from .base import BaseDataset
import numpy as np
import os
import torch
import pandas as pd
import copy
from utils.tools import get_sub_luna_point,get_sub_sun_point, dprint
from utils.timefeatures import time_features
from .normlizer import PreGauessNormlizer,PreUnitNormlizer,NoneNormlizer,PostGauessNormlizer,PostUnitNormlizer,TimewiseNormlizer
from tqdm.auto import tqdm
from utils.tools import get_center_around_indexes, get_center_around_indexes_3D
import json
from .utils import load_numpy_from_url

def get_init_file_list(dates):
    """
    ---> We will require the dates via np.datetime64 format
    ---> Old code use year list below
    # file_list = []
    # for year in years:
    #     if year == 1979: # 1979年数据只有8753个，缺少第一天前7小时数据，所以这里我们从第二天开始算起
    #         for hour in range(17, 8753, 1):
    #             file_list.append([year, hour])
    #     else:
    #         if year % 4 == 0:
    #             max_item = 8784
    #         else:
    #             max_item = 8760
    #         for hour in range(0, max_item, 1):
    #             file_list.append([year, hour])
    # return file_list
    """
    years          = dates.astype('datetime64[Y]').astype(int)   ## start from 1970, thus 1979 is 9
    start_of_years = years.astype('datetime64[Y]')
    hours          = (dates - start_of_years) / np.timedelta64(1, 'h')
    hours[years == 9] -= 7 # 1979 data lake 7 hours
    hours = np.maximum(hours, 0)  # ensure no negative hour values
    file_list      = np.vstack((years+1970, hours)).T.astype(int)
    return file_list

def get_default_timestamp_pool(config):
    time_unit = config.get('time_unit', 1 )
    return {'train':np.arange(np.datetime64("1979-01-02"), np.datetime64("2017-01-01"), np.timedelta64(time_unit, "h")),
            'valid':np.arange(np.datetime64("2017-01-01"), np.datetime64("2018-01-01"), np.timedelta64(time_unit, "h")),
            'test' :np.arange(np.datetime64("2018-01-01"), np.datetime64("2019-01-01"), np.timedelta64(time_unit, "h")),
            'ftest': np.arange(np.datetime64("2018-01-01"), np.datetime64("2019-01-01"), np.timedelta64(time_unit, "h")),
            }

def read_channel_list_file(file_path):
    with open(file_path, 'r') as f:
        channel_list = json.load(f)
    return channel_list

def get_timestamp_date(split, config):
    timestamps  = config.get('timestamps_list', None)
    defaultPool = get_default_timestamp_pool(config)
    time_unit   = config.get('time_unit', 1 )
    if timestamps is None:
        timestamps = defaultPool[split]
        dprint(f"you don't assign a timestamp list for {split}, then we use the default timestamp config  per {time_unit} hour")
    else:
        dprint(f"noice you assign your own year timestamps list from {timestamps.min} to {timestamps.max()}" )
    return timestamps  # [[year,idx],[year,idx],.....]

def get_vnames_from_config(config):
    channel_name_list = config.get('channel_name_list')
    if isinstance(channel_name_list, str):
        channel_name_list = read_channel_list_file(channel_name_list)
    elif isinstance(channel_name_list, list):
        channel_name_list = channel_name_list
    else:
        raise NotImplementedError(f"channel_name_list {channel_name_list} must be filepath or list of channel name")
    return channel_name_list

def preload_full_dataset(split, config):
    '''
    This will load the whole dataset into memory, it will be very slow and consume a lot of memory
    '''

    root           = config.get('root')
    ### first check whether the train.npy/test.npy/valid.npy is exist
    fast_offline_data_name = os.path.join(root, f"{split}.npy")
    if os.path.exists(fast_offline_data_name):
        dprint(f"detect fast offline data in {fast_offline_data_name}, use this")
        dataset_tensor     = torch.Tensor(np.load(fast_offline_data_name))
        record_load_tensor = torch.ones(len(dataset_tensor))
        return dataset_tensor,record_load_tensor
    else:
        init_file_list = get_init_file_list(get_timestamp_date(split, config))
        dprint(f"dont find offline data in {fast_offline_data_name}")
        dprint("then seek split file store in format {root}/{year}/{year}-{hour:04d}.npy")
        loaded_tensor  = None
        for i, (year, hour) in tqdm(enumerate(init_file_list)):
            url = f"{root}/{year}/{year}-{hour:04d}.npy"
            odata = load_numpy_from_url(None,url)
            if loaded_tensor is None:
                assert len(init_file_list)*np.prod(odata.shape) < 2**32, "The dataset is too large, we can't load it into memory"
                loaded_tensor = torch.empty(len(init_file_list), *odata.shape)
            loaded_tensor[i] = torch.from_numpy(odata)
        dprint("concat!")
        loaded_tensor  = torch.stack(loaded_tensor)
        loaded_flag    = torch.ones(len(loaded_tensor))
        dprint("finish preload the whole dataset into memory")
        return loaded_tensor, loaded_flag

def get_the_dataset_flag(config):
    channel_picked_name = os.path.split(config.get('channel_name_list'))[-1].split('.')[0]
    normlizer_name      = config.get('normlized_flag','N')
    return channel_picked_name+normlizer_name

class WeatherBenchBase(BaseDataset):
    time_unit  = 1

    single_vnames = [ "2m_temperature", "10m_u_component_of_wind", "10m_v_component_of_wind", "total_cloud_cover",
                  "total_precipitation", "toa_incident_solar_radiation"]
    level_vnames= []
    for physics_name in ["geopotential", "temperature",
                            "specific_humidity","relative_humidity",
                            "u_component_of_wind","v_component_of_wind",
                            "vorticity","potential_vorticity"]:
        for pressure_level in [50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 850, 925, 1000]:
            level_vnames.append(f"{pressure_level}hPa_{physics_name}")

    weatherbench_property_name_list = single_vnames + level_vnames
    ## the 110 channel for weatherbench dataset
    cons_vnames= ["lsm", "slt", "orography"]

    def get_channel_choice(self, property_names):
        if isinstance(property_names, (tuple,list)):
            return [self.get_channel_choice(property_name) for property_name in property_names]
        elif isinstance(property_names, str):
            return self.weatherbench_property_name_list.index(property_names)
        else:
            raise NotImplementedError(f"property_names {property_names} must be str or list of str")

class FakeWeatherBench(WeatherBenchBase):
    def __init__(self, split="train", shared_dataset_tensor_buffer=(None, None), with_idx=False, config={}, debug=False):

        self.split = split
        self.with_idx = with_idx
        self.timestamp_date = get_timestamp_date(split, config)
        self.single_data_path_list = get_init_file_list(self.timestamp_date)
        self.time_step = config.time_step
        self.time_intervel = config.time_intervel
        self.resolution_h, self.resolution_w = self.img_size = config.img_size

        self.normlized_flag = config.get('normlized_flag', 'N')
        self.use_time_feature = config.get('use_time_feature', False)
        self.time_reverse_flag = config.get('time_reverse_flag', 'only_forward')
        self.add_LunaSolarDirectly = config.get('add_LunaSolarDirectly', False)
        self.offline_data_is_already_normed = config.get('offline_data_is_already_normed', False)

        self.constant_channel_pick = config.get('constant_channel_pick', None)
        self.make_data_physical_reasonable_mode = config.get('make_data_physical_reasonable_mode', None)

        self.patch_range = patch_range = config.get('patch_range', None)
        self.cross_sample = config.get('cross_sample', True) and ((self.split == 'train') or debug) and (self.patch_range is not None)

        #self.vnames= [self.weatherbench_property_name_list[i] for i in self.channel_choice]
        self.dataset_flag = get_the_dataset_flag(config)

        self.vnames = get_vnames_from_config(config)
        self.channel_choice = np.array(self.get_channel_choice(self.vnames))

    def __len__(self):
        return len(self.single_data_path_list) - self.time_step*self.time_intervel + 1

    def __getitem__(self, idx):
        raise NotImplementedError("You must implement it")

class WeatherBench(WeatherBenchBase):
    '''
    In-memory dataset, if we activate the in memory procedure, please create shared tensor
    via Tensor.share_memory_() before call this module and pass the create tensor as args of this module
    the self.dataset_tensor should be same shape as the otensor, for example (10000, 110, 32, 64)
    the self.record_load_tensor is used to record which row is record.
    This implement will automatively offline otensor after load it once.

    no matter we use in-memory technology or not, we always initialized two tensor,
    one is the whole tensor dataset stored in (B, P, W, H) tensor
    one is the loadQ tensor that judge whether the real data is loaded.
    '''
    default_root = None
    reduce_Field_coef = np.array([1])
    clim_tensor = [0]
    img_shape = (32, 64)
    record_load_tensor = None
    dataset_tensor = None

    def __init__(self, split="train", shared_dataset_tensor_buffer=(None, None),with_idx=False, config={}):
        self.root = config.get('root', self.default_root)
        assert self.root is not None
        dprint(f"use dataset in {self.root} for {split}")
        self.split = split
        self.with_idx=with_idx
        self.timestamp_date=get_timestamp_date(split, config)
        self.single_data_path_list = get_init_file_list(self.timestamp_date)
        self.time_step     = config.time_step
        self.time_intervel = config.time_intervel
        self.resolution_h, self.resolution_w = self.img_size = config.img_size

        self.normlized_flag = config.normlized_flag
        self.use_time_feature = config.use_time_feature
        self.time_reverse_flag = config.time_reverse_flag
        self.add_LunaSolarDirectly = config.add_LunaSolarDirectly
        self.offline_data_is_already_normed = config.offline_data_is_already_normed
        self.constant_channel_pick = config.constant_channel_pick
        self.make_data_physical_reasonable_mode = config.make_data_physical_reasonable_mode

        self.data_processing_is_all_done = False

        #self.vnames= [self.weatherbench_property_name_list[i] for i in self.channel_choice]
        self.dataset_flag = get_the_dataset_flag(config)

        self.vnames = get_vnames_from_config(config)
        self.channel_choice = np.array(self.get_channel_choice(self.vnames))

        self.mean_std = self.load_original_meanstds()  # (2, P, W, H)
        self.normlizer = self.get_normlizer(
            self.mean_std[:, self.channel_choice][..., None, None], self.normlized_flag)

        #self.channel_choice, self.normlizer = self.get_config_from_preset_pool(self.dataset_flag)
        self.mean, self.std = self.normlizer.mean, self.normlizer.std
        self.unit_list      = self.std

        self.constants = torch.from_numpy(self.load_numpy_from_url(
            os.path.join(self.root, "constants.npy")))

        assert ((shared_dataset_tensor_buffer[0] is None and shared_dataset_tensor_buffer[1] is None) or
                (shared_dataset_tensor_buffer[0] is not None and shared_dataset_tensor_buffer[1] is not None))

        if shared_dataset_tensor_buffer[0] is None:
            self.dataset_tensor, self.record_load_tensor = self.create_offline_dataset_templete(
                split, create_buffer=False, config=config)
        else:
            self.dataset_tensor, self.record_load_tensor = shared_dataset_tensor_buffer

        if self.dataset_tensor is not None:
            assert len(self.dataset_tensor) == len(self.timestamp_date), f"{len(self.dataset_tensor)} == {len(self.timestamp_date)}"
            assert self.dataset_tensor.shape[-2] == self.resolution_h
            assert self.dataset_tensor.shape[-1] == self.resolution_w
        if self.single_data_path_list is not None:
            assert len(self.single_data_path_list) == len(self.timestamp_date)

        if self.time_reverse_flag: ## should add this before change vnames
            self.volicity_idx = [i for i, name in enumerate(self.vnames) if (
                ('v_component' in name) or ('u_component' in name))]

        # if self.constant_channel_pick: ### no need change vnames anymore, since we add constant at another key
        #     self.vnames = self.vnames + [self.cons_vnames[i] for i in self.constant_channel_pick]

        if self.use_time_feature:
            self.timestamp_date = time_features(pd.to_datetime(self.timestamp_date)).transpose(1, 0)

        if self.add_LunaSolarDirectly:
            self.LaLotude, self.LaLotudeVector = self.get_mesh_lon_lat(self.resolution_h, self.resolution_w)  # (32, 64 )

    def load_original_meanstds(self):
        mean_std_path = os.path.join(self.root, "mean_std.npy")
        if os.path.exists(mean_std_path):
            return self.load_numpy_from_url(mean_std_path)  # (2, P, W, H)
        else:
            dprint("""
                Warning!!!! The mean std file is not founed, we will use the mean=0 and std=1
                  """)
            return 0, 1

    @staticmethod
    def unique_name(config):
        return f"WeatherBench_{config.get('resolution_h')}x{config.get('resolution_w')}_{get_the_dataset_flag(config)}"

    @staticmethod
    def _add_constant_mean_std(mean_std, channel_num):
        mean, std = mean_std  # (70,1,1)
        mean = np.concatenate([mean, np.zeros(channel_num, *mean.shape[1:])])
        std = np.concatenate([std,  np.ones(channel_num, *std.shape[1:])])
        return np.stack([mean, std])  # (2, 71, 1, 1)

    @staticmethod
    def get_normlizer(mean_std, normlized_flag):

        if normlized_flag[-1] == 'N':
            return PreGauessNormlizer(mean_std)
        elif normlized_flag[-1] == 'U':
            return PreUnitNormlizer(mean_std)
        elif normlized_flag[-1] == 'O':
            return PostGauessNormlizer(mean_std)
        else:
            raise NotImplementedError(
                f"normlized_flag {normlized_flag} is not implemented")

    def get_config_from_preset_pool(self, dataset_flag):
        if dataset_flag == '2D110N':
            return list(range(0, 110)), PreGauessNormlizer(self.mean_std.reshape(2, 110, 1, 1))
        elif dataset_flag == '2D110U':
            return list(range(0, 110)),   PreUnitNormlizer(self.mean_std.reshape(2, 110, 1, 1))
        elif dataset_flag == '2D104N':
            return list(range(6, 110)), PreGauessNormlizer(self.mean_std[:, 6:].reshape(2, 104, 1, 1))
        elif dataset_flag == '2D104U':
            return list(range(6, 110)),   PreUnitNormlizer(self.mean_std[:, 6:].reshape(2, 104, 1, 1))
        elif dataset_flag == '3D104N':
            return self.threeD_channel, PreGauessNormlizer(self.mean_std[:, 6:].reshape(2, 8, 13, 1, 1).mean(2))
        elif dataset_flag == '3D104U':
            return self.threeD_channel,   PreUnitNormlizer(self.mean_std[:, 6:].reshape(2, 8, 13, 1, 1).mean(2))
        elif dataset_flag == '2D110O':
            return list(range(0, 110)),  PostGauessNormlizer(self.mean_std.reshape(2, 110, 1, 1))
        elif dataset_flag == '2D104O':
            return list(range(6, 110)),  PostGauessNormlizer(self.mean_std[:, 6:].reshape(2, 104, 1, 1))
        elif dataset_flag == '3D104O':
            return self.threeD_channel, PostGauessNormlizer(self.mean_std[:, 6:].reshape(2, 8, 13, 1, 1).mean(2))
        else:
            raise NotImplementedError(
                f"dataset_flag {dataset_flag} is not implemented")

    def __len__(self):
        return len(self.single_data_path_list) - self.time_step*self.time_intervel + 1

    @staticmethod
    def create_offline_dataset_templete(split='test', create_buffer=False, fully_loaded=False, config={}):
        fast_offline_data_name = os.path.join(config.get('root'), f"{split}.npy")
        if os.path.exists(fast_offline_data_name):
            create_buffer = fully_loaded = True
        if not create_buffer:return None, None
        if fully_loaded:
            return preload_full_dataset(split, config)
        else:
            ### will dynamic load during the iteration
            init_file_list = get_init_file_list(get_timestamp_date(split, config))
            vnames = get_vnames_from_config(config)
            batches = len(init_file_list)
            h  = config.get('resolution_h')
            w  = config.get('resolution_w')
            ### this create the whole buffer
            assert batches*len(vnames)*h*w < 2**32, "The dataset is too large, we can't load it into memory"
            return (torch.empty(batches, len(vnames), h , w),
                    torch.zeros(batches))

    def load_otensor(self, idx):
        '''
        This will load the whole tensor of timestamp into memory. For very large snapshot, it may be inefficient.
        '''
        if (self.record_load_tensor is None) or (not self.record_load_tensor[idx]):
            year, hour = self.single_data_path_list[idx]
            url = f"{self.root}/{year}/{year}-{hour:04d}.npy"
            odata = torch.from_numpy(self.load_numpy_from_url(url))
            if self.record_load_tensor is not None:
                '''
                <--- load data into shared buffer when dataset iteration
                '''
                self.record_load_tensor[idx] = 1
                self.dataset_tensor[idx] = odata

        if self.record_load_tensor is not None:
            # .clone() <--- be careful that do not inplace modify the source data
            odata =  self.dataset_tensor[idx]

        return odata

    def generate_runtime_data(self, idx, reversed_part=False):
        if self.data_processing_is_all_done:
            return self.dataset_tensor[idx]
        odata = self.load_otensor(idx)
        assert odata.shape[-2] == self.resolution_h
        assert odata.shape[-1] == self.resolution_w
        if isinstance(odata, torch.Tensor) and isinstance(self.channel_choice, np.ndarray):
            self.channel_choice = torch.LongTensor(self.channel_choice)
        data = odata[self.channel_choice]  # (110, W,H )->(P,W,H)
        data = self.make_data_physical_reasonable(data, odata, idx=idx)
        if not self.offline_data_is_already_normed:
            data = self.normlizer.do_pre_normlize(data)
        #assert len(data) == len(self.vnames) - len(self.constant_channel_pick)
        if isinstance(data, np.ndarray):
            data = torch.from_numpy(data)
        data = data.float()
        if reversed_part:  # the reversed_part should be
            data = data.clone() if isinstance(data, torch.Tensor) else data.copy()
            data[reversed_part] = -data[reversed_part]
        return data

    def get_item(self, idx, reversed_part=False):
        '''
        Notice for 3D case, we return (5,14,32,64) data tensor
        '''
        # if self.use_offline_data:
        #     data = self.dataset_tensor[idx]
        # else:
        data = self.generate_runtime_data(idx, reversed_part=reversed_part)

        output = {'field': data}

        if self.use_time_feature:
            output['timestamp'] = self.timestamp_date[idx]
        if self.add_LunaSolarDirectly:
            output['sun_mask'] = self.get_sum_mask(idx)
            output['luna_mask'] = self.get_moon_mask(idx)
        if self.constant_channel_pick and len(self.constant_channel_pick) > 0:
            output['constant'] = self.constants[self.constant_channel_pick]
        return output

    def get_sum_mask(self, idx):
        timenow = self.timestamp_date[idx]
        sun_lon, sun_lat = get_sub_sun_point(timenow.item())
        sun_vector = np.stack([np.cos(sun_lat/180*np.pi)*np.cos(sun_lon/180*np.pi),
                               np.cos(sun_lat/180*np.pi) *
                               np.sin(sun_lon/180*np.pi),
                               np.sin(sun_lat/180*np.pi)])

        sun_mask = (self.LaLotudeVector@sun_vector).reshape(1,
                                                            self.resolution_h, self.resolution_w)
        return sun_mask

    def get_moon_mask(self, idx):
        timenow = self.timestamp_date[idx]
        moon_lon, moon_lat = get_sub_luna_point(timenow.item())
        moon_vector = np.stack([np.cos(moon_lat/180*np.pi)*np.cos(moon_lon/180*np.pi),
                                np.cos(moon_lat/180*np.pi) *
                                np.sin(moon_lon/180*np.pi),
                                np.sin(moon_lat/180*np.pi)])
        moon_mask = (self.LaLotudeVector@moon_vector).reshape(1,
                                                              self.resolution_h, self.resolution_w)
        return moon_mask

    def make_data_physical_reasonable(self, data, odata, idx=None):
        '''
        make the data physical reasonable, for example, the total precipitation should be positive
        '''
        if self.make_data_physical_reasonable_mode is None:
            return data
        raise NotImplementedError

    @staticmethod
    def get_mesh_lon_lat(tH=32, tW=64):
        ### this is a general version
        resolution = tH
        assert tW == 2*tH
        theta_offset = (180/resolution/2)
        latitude = (np.linspace(0, 180, resolution+1) +
                    theta_offset)[:resolution]
        longitude = np.linspace(0, 360, 2*resolution+1)[:(2*resolution)]
        x, y = np.meshgrid(latitude, longitude)
        LaLotude = np.stack([y, x])/180*np.pi
        LaLotudeVector = np.stack([np.cos(LaLotude[1])*np.cos(LaLotude[0]), np.cos(
            LaLotude[1])*np.sin(LaLotude[0]), np.sin(LaLotude[1])], 2)
        return LaLotude, LaLotudeVector
        #### Below is 721x1440 origin data version
        # sH = 720
        # sW = 1440
        # steph = sH /float(tH)
        # stepw = sW /float(tW)
        # x     = np.arange(0, sW, stepw).astype('int')
        # y     = np.linspace(0, sH-1, tH, dtype='int')
        # x, y  = np.meshgrid(x, y)
        # latitude   = np.linspace(90,-90,721)
        # longitude  = np.linspace(0,360,1440)
        # LaLotude = np.stack([latitude[y],longitude[x]])/180*np.pi
        # LaLotudeVector = np.stack([np.cos(LaLotude[1])*np.cos(LaLotude[0]),np.cos(LaLotude[1])*np.sin(LaLotude[0]),np.sin(LaLotude[1])],2)
        # return LaLotude,LaLotudeVector

class WeatherBenchPhysical(WeatherBench):
    default_root='datasets/weatherbench'
    use_offline_data = False

    def make_data_physical_reasonable(self, data, odata, idx=None):
        '''
        make the data physical reasonable, for example, the total precipitation should be positive
        '''
        if self.make_data_physical_reasonable_mode is None:
            return data
        eg = torch if isinstance(data, torch.Tensor) else np
        if self.make_data_physical_reasonable_mode == 'use_constant_patch_lack_of_groud_geopotential':
            assert '2D70' in self.dataset_flag
            data = data.clone() # <-- avoid inplace modify
            data[14*4-1]    = eg.ones_like(data[14*4-1])*50 # modifiy the groud Geopotential, we use 5m height
            total_precipitaiton = odata[4]
            data[14*5-1]    = total_precipitaiton
            return data
        if self.make_data_physical_reasonable_mode == 'build_resonal_3D_tensor':
            assert '3D70' in self.dataset_flag
            raise NotImplementedError("TODO: notice the input tensor now is a 3D tensor, we need to modify the code to make it work")
            data[14*4-1] = eg.ones_like(data[14*4-1])*50 # modifiy the groud Geopotential, we use 5m height
            total_precipitaiton = odata[4]
            newdata = data[14*5-1].clone() if isinstance(data,torch.Tensor) else data[14*5-1].copy()
            newdata[total_precipitaiton>0] = 100
            newdata[data[14*5-1]>100]=data[14*5-1][data[14*5-1]>100]
            data[14*5-1] = newdata
            shape= data.shape
            data = data.reshape(5,14,*shape[-2:])
            return data

class WeatherBenchMultibranchRandom(WeatherBench):
    """
    This is a multibranch random fetcher, it will randomly select a time_intervel from multibranch_select.
    We will use outside random fetcher to call this module.
    For evaluation, we will use all time_intervel in multibranch_select via call set_time_intervel
    """
    def __init__(self, split="train", shared_dataset_tensor_buffer=(None,None),config = {}):
        super().__init__(split=split, shared_dataset_tensor_buffer=(None, None), config=config)
        use_offline_data = config.get('use_offline_data', 0)
        time_intervel    = config.get('time_intervel', 1)
        assert time_intervel == 1
        assert use_offline_data == 0
        self.multibranch_select= [int(t) for t in config['multibranch_select']]

    def set_time_intervel(self, time_intervel):
        self.time_intervel = time_intervel

    def __getitem__(self, idx):
        reversed_part = self.do_time_reverse(idx)
        time_intervel = self.time_intervel
        if self.split=='train':
            time_intervel = np.random.choice(self.multibranch_select)
            dprint(f"""
        ========> We use random time_intervel:{time_intervel}. However, this line should not appear here if your are in Train ,pde. You need call a random fetcher outside
                  """ )
        time_step_list = [idx+i*time_intervel for i in range(self.time_step)]
        if reversed_part:time_step_list = time_step_list[::-1]
        batch = [self.get_item(i, reversed_part) for i in time_step_list]
        return batch if not self.with_idx else (idx, batch)

class WeatherBench7066(WeatherBenchPhysical):
    """
    config = {
        'root':"/mnt/data/ai4earth/zhangtianning/datasets/WeatherBench/weatherbench_6hour/",
        'channel_name_list':"configs/datasets/WeatherBench/2D70.channel_list.json",
        'time_reverse_flag':'only_forward',
        'offline_data_is_already_normed':True,
        'time_unit':6
    }

    """

class WeatherBenchPatch(WeatherBench):
    """
    config = {
        'root':"/mnt/data/ai4earth/zhangtianning/datasets/WeatherBench/weatherbench_6hour/",
        'channel_name_list':"configs/datasets/WeatherBench/2D70.channel_list.json",
        'time_reverse_flag':'only_forward',
        'offline_data_is_already_normed':True,
        'time_unit':6
    }
    """
    def __init__(self, split="train", shared_dataset_tensor_buffer=(None,None),config = {}, debug=False):
        super().__init__(split=split, shared_dataset_tensor_buffer=(None, None), config=config)
        self.patch_range  = patch_range = config.data_patch_range
        self.cross_sample = ((self.split == 'train') or debug) and (self.patch_range is not None)
        self.patch_chunk  = config.patch_chunk

        if len(self.channel_choice.shape) == 2:
            patch_range = patch_range if isinstance(patch_range, (list, tuple)) else (patch_range, patch_range, patch_range)
            self.center_index, self.around_index = get_center_around_indexes_3D(patch_range, self.img_shape)
        else:
            patch_range = patch_range if isinstance(patch_range, (list, tuple)) else (patch_range, patch_range)
            self.center_index, self.around_index = get_center_around_indexes(patch_range, self.img_shape)
        self.patch_range = patch_range
        dprint(f"notice we will use around_index{self.around_index.shape} to patch data")
        #### due to the data efficiency, reshape the odata at beginning and do data preprocess
        assert self.dataset_tensor is not None
        assert (self.record_load_tensor == 1).all()
        self.dataset_tensor =  self.dataset_tensor[:,self.channel_choice] #(B, 70, 32, 64) or (B, 5, 14, 32, 64)
        self.dataset_tensor = torch.Tensor(self.dataset_tensor)
        if not self.offline_data_is_already_normed:
            self.dataset_tensor = self.normlizer.do_pre_normlize(self.dataset_tensor)
        self.dataset_tensor = self.dataset_tensor.float()
        self.data_processing_is_all_done = True
        self.img_shape  = self.dataset_tensor.shape[2:]
        self.length = len(self)

    def get_cross_sample_patch(self, idx):
        '''
        Notice for 3D case, we return (5,14,32,64) data tensor
        '''
        # if self.use_offline_data:
        #     data = self.dataset_tensor[idx]
        # else:
        if len(self.img_shape)==2:
            center_h = np.random.randint(self.img_shape[-2] - (self.patch_range[-2]//2)*2,size=(self.patch_chunk,))
            center_w = np.random.randint(self.img_shape[-1],size=(self.patch_chunk,))
            location = self.around_index[center_h, center_w] #(B,2,5,5)
            patch_idx_h = location[:,0]#(B,5,5)
            patch_idx_w = location[:,1]#(B,5,5)
            batch_idx = np.random.randint(self.length,size=(self.patch_chunk,)).reshape(self.patch_chunk,1,1) #(B,1,1)
            data = []
            for i in range(self.time_step):
                dt = i*self.time_intervel
                output = {'field':self.dataset_tensor[batch_idx+dt,:,patch_idx_h,patch_idx_w].permute(0,3,1,2)}
                if self.use_time_feature:
                    output['timestamp'] = [self.timestamp_date[idx+dt] for idx in batch_idx]
                if self.add_LunaSolarDirectly:
                    output['sun_mask']  = np.stack([self.get_sum_mask(idx)[0][a,b] for idx,a,b in zip(batch_idx,patch_idx_h,patch_idx_w)])
                    output['luna_mask'] = np.concatenate([self.get_moon_mask(idx)[0][a,b] for idx,a,b in zip(batch_idx,patch_idx_h,patch_idx_w)])
                assert not (self.constant_channel_pick and len(self.constant_channel_pick) > 0), 'do not use constant'
                output['location']  = location
                data.append(output)
            #data = [[self.data[batch_idx+i,:,patch_idx_h,patch_idx_w].permute(0,3,1,2)] for i in range(self.time_step)]
        elif len(self.img_shape)==3:
            center_z    = np.random.randint(self.img_shape[-3] - (self.patch_range[-3]//2)*2,size=(self.patch_chunk,))
            center_h    = np.random.randint(self.img_shape[-2] - (self.patch_range[-2]//2)*2,size=(self.patch_chunk,))
            center_w    = np.random.randint(self.img_shape[-1],size=(self.patch_chunk,))
            location    = self.around_index[center_z, center_h, center_w] #(B,2,5,5,5)
            patch_idx_z = location[:,0]#(B,5,5,5)
            patch_idx_h = location[:,1]#(B,5,5,5)
            patch_idx_w = location[:,2]#(B,5,5,5)
            #location = self.center_index[:,center_z, center_h, center_w].transpose(1,0)#(B,3)
            batch_idx = np.random.randint(self.length,size=(self.patch_chunk,)).reshape(self.patch_chunk,1,1,1) #(B,1,1,1)
            data = []
            for i in range(self.time_step):
                output = {'field': self.dataset_tensor[batch_idx+dt, :,patch_idx_z, patch_idx_h, patch_idx_w].permute(0, 4, 1, 2, 3)}
                if self.use_time_feature:
                    output['timestamp'] = [self.timestamp_date[idx+dt] for idx in batch_idx]
                if self.add_LunaSolarDirectly:
                    output['sun_mask']  = np.stack([self.get_sum_mask(idx)[0][a,b] for idx,a,b in zip(batch_idx[...,0],patch_idx_h,patch_idx_w)])
                    output['luna_mask'] = np.concatenate([self.get_moon_mask(idx)[0][a,b] for idx,a,b in zip(batch_idx[...,0],patch_idx_h,patch_idx_w)])
                assert not (self.constant_channel_pick and len(self.constant_channel_pick) > 0), 'do not use constant'
                output['location']  = location
                data.append(output)
            #data = [[self.data[batch_idx+i,:,patch_idx_z,patch_idx_h,patch_idx_w].permute(0,4,1,2,3)] for i in range(self.time_step)]
        else:
            raise NotImplementedError
        return data

    def get_normal_image(self, idx):
        time_step_list = [idx+i*self.time_intervel for i in range(self.time_step)]
        batch = [self.get_item(i, False) for i in time_step_list]
        dict_list =[]
        for i,data in enumerate(batch):
            _dict = {'field':data} if isinstance(data, (torch.Tensor, np.ndarray)) else data
            assert isinstance(_dict, dict)
            if self.with_idx:_dict['idx'] = idx
            dict_list.append(_dict)
        return dict_list

    def __getitem__(self, idx):
        if self.cross_sample:
            return self.get_cross_sample_patch(idx)
        else:
            return self.get_normal_image(idx)

    @staticmethod
    def collate_fn(batch):
        if isinstance(batch[0], dict):
            return {key: WeatherBenchPatch.collate_fn([d[key] for d in batch]) for key in batch[0]}
        elif isinstance(batch[0], list):
            return [WeatherBenchPatch.collate_fn(samples) for samples in zip(*batch)]
        elif isinstance(batch[0], torch.Tensor):
            return torch.cat(batch, dim=0)
        else:
            return batch

if __name__ == "__main__":
    pass
    # import sys
    # import time
    # from tqdm import tqdm
    # from multiprocessing import Pool
    # from petrel_client.client import Client
    # import petrel_client
    # dataset = WeatherBench64x128(split='test',root='weatherbench:s3://weatherbench/weatherbench64x128/npy',dataset_flag='2D68N')
    # for i in tqdm(range(0, 1000)):
    #     a = dataset[i][0]

    # def load_data_range(dataset, start, end):
    #     for i in tqdm(range(start, end)):
    #         _ = dataset[i]

    # def parallel_load_data(dataset):
    #     total_len = len(dataset)
    #     processor = 10
    #     range_list = np.linspace(0, total_len, processor+1)
    #     range_list = [int(a) for a in range_list]
    #     dprint(range_list)
    #     res = []
    #     p = Pool(processor)
    #     for i in range(processor):
    #         res.append(p.apply_async(load_data_range, args=(
    #             dataset, range_list[i], range_list[i+1])))
    #         dprint(str(i) + ' processor started !')
    #     p.close()
    #     p.join()

    # def check_range(dataset, start, end):
    #     fail_list = []
    #     for i in tqdm(range(start, end)):
    #         if i > len(dataset):
    #             continue
    #         now_time = time.time()
    #         try:
    #             _ = dataset[i]
    #         except:
    #             dprint(f"fail at {i}")
    #         fail_list.append(i)
    #         cost = time.time() - now_time
    #         #dprint(cost)
    #     return fail_list
    # dataset = ERA5CephDataset(split='train')
    # dprint(ERA5CephDataset.__name__)
    # dprint(dataset[0])

    # dprint(len(dataset[0]))
    # dprint(len(dataset[1]))

    raise
    dataset = ERA5CephDataset(split='valid')
    dprint(len(dataset))

    for idx in range(len(dataset)):
        dprint(dataset.get_url(idx))
    exit()
    processor = 10
    unit = len(dataset)//processor
    res = []
    p = Pool(processor)
    for i in range(processor):
        res.append(p.apply_async(
            check_range, args=(dataset, i*unit, (i+1)*unit)))
        #dprint(str(i) + ' processor started !')

    p.close()
    p.join()


import streamlit as st
from sklearn.preprocessing import MinMaxScaler
import pickle
import numpy as np

st.title("Employee Promotion Project")

st.info("Select values for each of the following features")

dept = st.selectbox("Department",('Sales & Marketing','Operations','Technology','Procurement','Analytics',
                                  'Finance','HR','Legal','R&D'))
region = st.selectbox("Region",('1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18',
                                '19','20','21','22','23','24','25','26','27','28','29','30','31','32','33','34'))
edu = st.selectbox("Education",("Bachelors","Masters & above","Below Secondary"))
gender = st.selectbox("Gender",("Male","Female"))
ch = st.selectbox("Recruitment channel",("referred","sourcing","other"))
no_of_train = st.selectbox("Number of training",('1','2','3','4','5','6','7','8','9','10'))
age = st.number_input("Enter age",min_value=20,max_value=60)
previous_rating = st.selectbox("Previous year rating",('1','2','3','4','5'))
service = st.number_input("Enter length of service",min_value=1,max_value=37)
award_won = st.selectbox("Award won in previous year",('Yes','No'))
train_score = st.number_input("Enter average training score",min_value=39,max_value=99)

X = np.zeros((1, 55))
if edu == 'Below Secondary':
    X[0][0] = 1
elif edu == 'Bachelors':
    X[0][0] = 2
elif edu == 'Masters & above':
    X[0][0] = 3

X[0][1] = float(no_of_train)
X[0][2] = age
X[0][3] = float(previous_rating)
X[0][4] = service

if award_won == 'Yes':
    X[0][5] = 1

X[0][6] = train_score

if dept == 'Analytics':
    X[0][7] = 1
elif dept == 'Finance':
    X[0][8] = 1
elif dept == 'HR':
    X[0][9] = 1
elif dept == 'Legal':
    X[0][10] = 1
elif dept == 'Operations':
    X[0][11] = 1
elif dept == 'Procurement':
    X[0][12] = 1
elif dept == 'R&D':
    X[0][13] = 1
elif dept == 'Sales & Marketing':
    X[0][14] = 1
elif dept == 'Technology':
    X[0][15] = 1

region = int(region)
if region == 1:
    X[0][16] = 1
elif region in range(10,20,1):
    X[0][region+7] = 1
elif region == 2:
    X[0][27] = 1
elif region in range(20,30,1):
    X[0][region+8] = 1
elif region == 3:
    X[0][38] = 1
elif region in range(30,35,1):
    X[0][region+9] = 1
elif region in range(4,10,1):
    X[0][region+40] = 1

if gender == 'Female':
    X[0][50] = 1
elif gender == 'Male':
    X[0][51] = 1

if ch == 'other':
    X[0][52] = 1
elif ch == 'referred':
    X[0][53] = 1
elif ch == 'sourcing':
    X[0][54] = 1

with open('P151_scaler' , 'rb') as sc:
    scaler = pickle.load(sc)

X_scaled = scaler.transform(X)

with open('P151_model' , 'rb') as file:
    model = pickle.load(file)

y = model.predict(X_scaled)

if y == 1:
    st.success("Employee may be promoted.")
else:
    st.warning("Employee may not be promoted.")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Apr 19 15:56:56 2020

@author: frederik
"""

import cv2
import numpy as np

img = cv2.imread("references/spatula.jpg", cv2.IMREAD_GRAYSCALE)

sift = cv2.xfeatures2d.SIFT_create()

kp = sift.detect(img, None)

img = cv2.drawKeypoints(img, kp, None)

cv2.imshow("image", img)
# Quit program is 'q' is pressed
if cv2.waitKey(0) & 0xFF == ord('q'):
    cv2.destroyAllWindows()


from datetime import datetime, timedelta
import os
from airflow import DAG
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.python_operator import PythonOperator
from helpers import SqlQueries
from crawlers.angel_co import main as angel_co_scraper
from crawlers.angel_co_parser import main as angel_co_crawler

default_args = {
    'owner': 'gabriel',
    'start_date': datetime(2019, 10, 19),
    'depends_on_past': False,
    'retries': 1,
    'retry_delay': timedelta(seconds=300)
}

dag = DAG('angel_co_jobs_dag',
          default_args=default_args,
          description='Load the jobs dataset and insert into PostgreSQL',
          schedule_interval='@daily',
          catchup=False
        )

# 1st step - download all htmls

run_selenium_scraper = PythonOperator(
    dag=dag,
    task_id='run_selenium_scraper',
    provide_context=False,
    python_callable=angel_co_scraper
)

# 2nd step - Parse all HTMLs into simple python data structures and insert into the jobs, companies and tags tables
# Mostly another python operator
run_crawler_on_scraped_data = PythonOperator(
    dag=dag,
    task_id='run_crawler_on_scraped_data',
    provide_context=False,
    python_callable=angel_co_crawler
)

# 3rd step - Delete all the HTML page files
# Yet Another python operator

run_selenium_scraper >> run_crawler_on_scraped_data


from Tkinter import*

my_app = Tk(className = 'Bangun Geometri')

D = Label(my_app, text ='Bangun Geometri',font='Helvetica 20 bold')
D.grid(row=0, column=0)

D1 = Label(my_app, text ='Balok adalah bangun ruang yang memiliki tiga pasang sisi yang ukurannya sama dan saling berhadapan serta memiliki bentuk persegi panjang.',font='Helvetica 10')
D1.grid(columnspan=4,row=1, column=0)

L1 = Label (my_app, text = 'Panjang : ')
L1.grid(row=2, column=0)
str1 = StringVar()
E1 = Entry(my_app, textvariable=str1)
E1.grid(columnspan=3 ,row=2, column=1)

L2 = Label (my_app, text = 'lebar : ')
L2.grid(row=3, column=0)
str2 = StringVar()
E2 = Entry(my_app, textvariable=str2)
E2.grid(columnspan=3 ,row=3, column=1)

L3 = Label (my_app, text = 'Tinggi : ')
L3.grid(row=4, column=0)
str3 = StringVar()
E3 = Entry(my_app, textvariable=str3)
E3.grid(columnspan=3 ,row=4, column=1)

def luas():
    p=float(str1.get())
    l=float(str2.get())
    t=float(str3.get())
    L=2*((p*l)+(p*t)+(l*t))
    LP.config(text=L)

B = Button(my_app, text='Hitung Luas', command= luas)
B.grid(row=5,column=0)
L=Label(my_app, text='luas :',font='Helvetica 10 bold')
L.grid(row=5,column=2)
LP=Label(my_app, text='0')
LP.grid(row=5,column=3)

my_app.mainloop()


#!/usr/bin/env python
# -*- coding: utf-8 -

from PIL import Image
import imagehash
import urllib
from io import BytesIO
import numpy as np
import time
from pathlib import Path

import logging
logging.getLogger("requests").setLevel(logging.WARNING)
log = logging.getLogger()

class Matcher:
    """
    Tries to match cards by image. #greatidea #notwaistingtimeatall
    """
    # self.list_cr
    # self.list_mid

    def __init__(self, list_cr, list_mid):
        """
        Runs matching on received lists.
        """
        self.list_cr = list_cr
        self.list_mid = list_mid
        self.info_status = []

    def get_matches(self):
        start = time.time()
        results = self.match(self.list_cr, self.list_mid)
        match_took = time.time() - start

        log.debug("Matching took {}.".format(self.readable_time(match_took)))

        if len(self.info_status):
            out_info = "(" + ",".join(self.info_status) + ")"
        else:
            out_info = None

        return results, out_info

    def readable_time(self, seconds):
        m, s = divmod(seconds, 60)
        h, m = divmod(m, 60)
        return "%d:%02d:%02d" % (h, m, s)

    def matrix(self, list_cr, list_mid):
        """
        Creates a nxn matrix with distances of all images to each other.
        """
        imgs_cr = [self.get_img("cr", id_cr) for id_cr in list_cr]
        imgs_api = [self.get_img("api", mid) for mid in list_mid]

        if (None not in imgs_cr) and (None not in imgs_api):
            mat = np.zeros([len(imgs_cr), len(imgs_api)])
            for i in range(len(imgs_cr)):
                for j in range(len(imgs_api)):
                    mat[i][j] = self.dist(imgs_cr[i], imgs_api[j])
        else:
            self.info_status.append("wrong url(s)")
            mat = None

        return mat

    def manual_match(self, mat):
        # Manual matching of images that are too similar.
        matches = {
            'BSLS_125': '214676',
            'BSLS_124': '214669',
            'BSLF_34': '244319',
            'BSLF_36': '244322',
            'BSLF_61': '157877',
            # ZEN lands
            'ZEN_255': '201959', 'ZEN_257': '195192', 'ZEN_259': '195177', 'ZEN_261': '201961',  # Forests
            'ZEN_263': '195187', 'ZEN_265': '195165', 'ZEN_267': '195161', 'ZEN_269': '201965',  # Islands
            'ZEN_271': '195178', 'ZEN_273': '195171', 'ZEN_275': '195181', 'ZEN_277': '195184',  # Mountains
            'ZEN_281': '201973', 'ZEN_283': '195173', 'ZEN_285': '195196', 'ZEN_287': '201971',  # Plains
            'ZEN_291': '195172', 'ZEN_293': '195194', 'ZEN_295': '201976', 'ZEN_197': '201975',  # Swamps

        }
        for i in range(len(self.list_cr)):
            if self.list_cr[i] in matches:
                for j in range(len(self.list_mid)):
                    if int(self.list_mid[j]) == int(matches[self.list_cr[i]]):
                        log.debug("Setting mat[{}][{}] to -1.".format(i, j))
                        mat[i][j] = -1  # -1 is lower than the lowest value of hash distance (0)
                    else:
                        log.debug("list_mid[j]={}, matches[list_cr[i]]={}".format(
                            self.list_mid[j], matches[self.list_cr[i]]))
        return mat

    def match(self, list_cr, list_mid):
        """
        Generates dictionary of cr_ids matching mids.
        """
        mat = self.matrix(list_cr, list_mid)
        if mat is not None:
            mat = self.manual_match(mat)
            minima_locations = self.get_minima(mat)
            if minima_locations is not None:
                out = {}
                for i in range(len(minima_locations)):
                    if minima_locations[i] != -1:
                        out[list_cr[i]] = list_mid[minima_locations[i]]

                return out
            else:
                return None
        else:
            return None

    def get_min_info(self, m, a):
        if a == 1:
            m = m.T
        min_loc = np.argmin(m.T, axis=0)
        min_val = np.amin(m.T, axis=0)
        occ = np.zeros_like(min_loc)
        duplicates = [None for i in range(len(occ))]
        for i in range(len(min_loc)):
            dupl = np.where(m[i] == min_val[i])[0]
            occ[i] = len(dupl) - 1
            if len(dupl) > 1:
                duplicates[i] = dupl

        return min_loc, min_val, occ, duplicates

    def get_minima(self, m):
        acceptable_minimum = 16
        xmin_loc, xmin_val, xocc, xdupl = self.get_min_info(m, 0)
        ymin_loc, ymin_val, yocc, ydupl = self.get_min_info(m, 1)
        # print(m)

        xout = np.negative(np.ones_like(xmin_loc))
        yout = np.negative(np.ones_like(ymin_loc))

        max_dif = 0

        # scroll by x and see perfect matches
        x_inserted = 0
        for i in range(len(xocc)):
            if xmin_val[i] <= acceptable_minimum:
                if xocc[i] <= 0:  # if no duplicates
                    if ymin_loc[xmin_loc[i]] == i:
                        xout[i] = xmin_loc[i]
                        yout[xmin_loc[i]] = ymin_loc[xmin_loc[i]]
                        x_inserted += 1
                    else:
                        log.debug('x minimum found, but not minimum for y, {} -> {} (i={}, xocc={}).'.format(
                            self.list_cr[xmin_loc[i]], self.list_mid[i], i, xocc[i]))
            elif xmin_val[i] > max_dif:
                max_dif = xmin_val[i]

        log.debug('x - duplicates ({} inserted)'.format(x_inserted))
        for k in range(len(xdupl)):
            if xdupl[k] is not None:
                for candidate in xdupl[k]:
                    log.debug(
                        "Ambiguous candidate: {} -> {}".format(self.list_cr[k], self.list_mid[candidate]))

        # scroll by y and for unmatched see matches within duplicates from x direction
        y_inserted = 0
        for j in range(len(yocc)):
            if yout[j] == -1:
                if ymin_val[j] <= acceptable_minimum:
                    if yocc[j] == 0:  # if no duplicates
                        if xdupl[ymin_loc[j]] is None:
                            log.debug('y minimum found, but x has no minima at all, {} -> {} (j={}).'.format(
                                self.list_mid[j], self.list_cr[ymin_loc[j]], j))
                        elif j in xdupl[ymin_loc[j]]:
                            yout[j] = ymin_loc[j]
                            xout[ymin_loc[j]] = xmin_loc[ymin_loc[j]]
                            y_inserted += 1
                        else:
                            log.debug('y minimum found, but not among minima for x, {} -> {} (j={}).'.format(
                                self.list_mid[j], self.list_cr[ymin_loc[j]], j))
                elif ymin_val[j] > max_dif:
                    max_dif = ymin_val[j]

        log.debug('y - duplicates ({} inserted)'.format(y_inserted))
        for k in range(len(ydupl)):
            if ydupl[k] is not None:
                for candidate in ydupl[k]:
                    log.debug(
                        "Ambiguous candidate: {} -> {}".format(self.list_mid[k], self.list_cr[candidate]))

        if max_dif > acceptable_minimum:
            self.info_status.append("max dif {} >= {}".format(max_dif, acceptable_minimum))

        return xout

    def get_minima_2(self, mat):
        minima = []
        minima_locations = []

        dim = len(mat[0])
        occurences = [0] * dim
        unambiguous = True
        for i in range(dim):
            minimum_location = np.argmin(mat.T[i])
            minimum = mat.T[i][minimum_location]
            all_minima = np.where(mat.T[i] == minimum)
            if len(all_minima[0]) != 1:
                unambiguous = False
                occurences[i] = all_minima[0]

            minima.append(minimum)
            minima_locations.append(minimum_location)

        acceptable_minimum = 11

        if unambiguous:
            status = 1
        else:
            status = 0
            self.info_status.append("ambiguous")
            log.debug("Some matches are ambiguous.")
            for i in range(len(occurences)):
                try:
                    for candidate in occurences[i]:
                        # TODO Not sure which index is which - might be switched CR/API
                        log.debug(
                            "Ambiguous candidate: {} -> {}".format(self.list_cr[i], self.list_mid[candidate]))
                except:
                    pass

        max_difference = np.max(minima)
        if max_difference < acceptable_minimum:
            log.debug("Maximum difference in images was {}.".format(max_difference))
        else:
            status = 0
            self.info_status.append("max dif {} >= {}".format(max_difference, acceptable_minimum))
            log.debug("WARNING! Maximum difference {} >= {}. Trouble?".format(
                max_difference, acceptable_minimum))

        if status:
            return minima_locations
        else:
            return None

    def dist(self, image_1, image_2):
        """
        Calculates distance of two images.
        """
        hash_1 = imagehash.average_hash(image_1)

        hash_2 = imagehash.average_hash(image_2)

        return hash_1 - hash_2

    def get_img(self, type_id, card_id):
        if type_id == "cr":
            url = self.url_cr(card_id)
            name = str(card_id) + ".jpg"
        else:
            url = self.url_api(card_id)
            name = str(card_id) + ".png"

        directory = Path('app/static/img/cards/')
        directory.mkdir(exist_ok=True)
        path = directory / name

        if path.exists():
            img = Image.open(path)
            return img
        else:
            try:
                urllib.request.urlretrieve(url, str(path))
                img = Image.open(path)
                return img
            except urllib.error.URLError as e:
                log.debug("URLError, trouble...")
                return None

    def url_cr(self, id_cr):
        """
        Generates cr_url from id_cr.
        """
        return 'http://cernyrytir.cz/images/kusovkymagic/' + id_cr.replace('_', '/') + '.jpg'

    def url_api(self, mid):
        """
        Generates api_url from mid.
        """
        return 'https://image.deckbrew.com/mtg/multiverseid/' + str(mid) + '.jpg'


"""
Pure subroutines of the MODL algorithm.

Author : Quentin Ferré <quentin.q.ferre@gmail.com>
"""

import time

from collections import defaultdict

import numpy as np
import pandas as pd

from sklearn.decomposition import SparseCoder
from sklearn.decomposition import MiniBatchDictionaryLearning

from pygtftk.stats.intersect.modl import tree

from pygtftk.utils import message

def learn_dictionary_and_encode(data, n_atoms = 20, alpha = 0.5,
        n_iter = 200, random_seed = 42, n_jobs = 1,
        fit_algorithm = 'cd',
        transform_algorithm = 'lasso_cd'):
    r"""
    Will learn a dictionary for the data (row-wise) and encode it, with the specified parameters.
    Returns the dictionary, components as a dataframe, and encoded data.

    By default, allows 20 words with alpha of 0.5.

    More info at : https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.MiniBatchDictionaryLearning.html

    >>> from pygtftk.stats.intersect.modl.dict_learning import test_data_for_modl
    >>> from pygtftk.stats.intersect.modl.subroutines import learn_dictionary_and_encode
    >>> import numpy as np
    >>> np.random.seed(42)
    >>> flags_matrix = test_data_for_modl(nflags = 1000, number_of_sets = 6)
    >>> import time
    >>> start_time = time.time()
    >>> U_df, V_df, error = learn_dictionary_and_encode(flags_matrix, n_atoms = 20, alpha = 0.5)
    >>> stop_time = time.time()

    """

    # TODO: Many operations here, such as recreating an object of a pandas
    # dataframe, might not be necessary ?

    data = np.array(data) # Force cast as array

    # Cannot ask for more rows than there are unique lines
    n_atoms = min(n_atoms, len(np.unique(data, axis = 0)))

    dico = MiniBatchDictionaryLearning(n_components=n_atoms, n_iter=n_iter,
        alpha = alpha,
        fit_algorithm = fit_algorithm,
        transform_algorithm = transform_algorithm, transform_alpha = alpha,
        positive_dict = True, positive_code = True,
        random_state = random_seed,
        n_jobs = n_jobs
        )

    # NOTE fit_algorithm is used during the learning and transform_algorithm
    # transforms the data once the estimator has been fitted.
    # We are using coordinate descent (CD) as LARS has troubles with correlated
    # features. Also because we want to be able to enforce positivity in the
    # dictionary and the code for interpretability.

    dico.fit(data)  # Fit the data

    # Get components (the dictionary).
    # NOTE: Use a try-except for future proofing, as sklearn (v 0.24) seems to
    # deprecate 'components_' across the board and replaced it with 'dictionary'
    try: V = dico.components_
    except: V = dico.dictionary
    V_df = pd.DataFrame(V)

    # If the alpha is inadapted, the learned dictionary may have failed to converge
    # and contain NaNs. If that happens, return it now and bypass the next steps
    # NOTE LARS is less vulnerable to it, use it as a fallback before calling it quits.
    if np.isnan(V).any():
        message("Learned dictionary by Coordinate descent contained NaNs. Alpha may have been indadapted. Defaulting to LARS.", type = 'DEBUG')

        dico = MiniBatchDictionaryLearning(n_components=n_atoms, n_iter=n_iter,
            alpha = alpha, transform_alpha = alpha, random_state = random_seed, n_jobs = n_jobs,
            fit_algorithm = 'lars',  transform_algorithm = 'lasso_lars')

        dico.fit(data)
        try: V = dico.components_
        except: V = dico.dictionary
        V_df = pd.DataFrame(V)

        # Only abort if it still does not work
        if np.isnan(V).any():
            message("Fallback still contains NaNs. Aborting.", type = 'DEBUG')
            return None, V_df, None # Return "U", V, "error"

    # Re-encode the data with this dictionary
    encoded_data = dico.transform(data)
    ed_df = pd.DataFrame(encoded_data)

    # Compute associated normalized L2 loss for reference
    reconstructed_features = np.matmul(encoded_data,V)
    error = np.sum((reconstructed_features-data)**2) / np.sum(data**2)

    return ed_df, V_df, error # Return U, V, error

# ---------------------------------------------------------------------------- #
#                        Step 1 of the MODL algorithm                          #
# ---------------------------------------------------------------------------- #

def generate_candidate_words(X, n_words, nb_threads = 1,
    discretization_threshold=0, alphas=None):
    """
    Given a data matrix, will generate a library of candidate words by trying many
    different Dictionary Learnings (different alphas only for now)
    """

    EPSILON = 1E-10

    ## Preliminary steps
    X = np.array(X) # Force cast to array for safety
    nb_features = X.shape[1] # word size

    # NOTE : n_atoms cannot be higher than the number of unique combis or there might be an Exception thrown.
    # It seems to vary and be due to alpha. To be safe, prevent it.
    nb_unique_combis = len(np.unique(X, axis = 0))
    if n_words > nb_unique_combis : n_words = nb_unique_combis

    # A too low alpha can result in learning giberrish words, thay may still be used as there is no penalty for it.
    # To counter this, we begin with an alpha of 1/nb_features
    # Rk: if alphas have been manually specified, use them instead
    if alphas is None: alpha = 1/nb_features
    else: alpha = alphas[0]

    # Remember all candidate words found at each step, and use a dictionary so we
    # can remember their total usage for later filtering
    all_candidate_words = defaultdict(lambda: 0)

    stop = False

    misbehaving = 0

    iternb = 0
    while not stop:

        # Encode using DL
        #start_time = time.time()
        message("> Alpha is "+str(alpha), type = 'DEBUG')

        # NOTE For some reason, the algorithm will never learn the word (1,1,1,...,1,1) even if it the only row in the data.
        # To prevent that, add a useless padding for the last column.
        pad = np.zeros((X.shape[0],1))
        Xpadded = np.append(X, pad, axis=1)

        # Perform the learning
        U, V, error = learn_dictionary_and_encode(Xpadded, n_atoms = n_words, alpha = alpha,
                                               n_jobs = nb_threads)

        #stop_time = time.time() # Careful not to name it 'stop' and overwrite the stop flag :)
        #print('One DL step took ', str(stop_time-start_time), 'seconds.')

        iternb += 1
        # Update alpha so DL will be more sparse in the next step
        # and look for higher-order combinations
        # We add iternb/nb_features to not take too long and not unduly favor high alphas (and longer words)

        # Rk: if alphas have been manually specified, use them instead
        if alphas is None: alpha += iternb/nb_features
        else:
            try: alpha = alphas[iternb]
            except: stop = True # Stop if we reached the end of the list

        # TODO: Also stop regardless after 2*k iterations ?

        # We stop once alpha is too high and preventing any word from being used
        # in the encoding U (which contained NaNs -- handled before -- or has a sum of 0)
        # Rq : can also happen with too low alpha or convergence errors, so stop only if it happens more than 2 times
        if (U is None): misbehaving += 1
        if (misbehaving >=2): stop = True

        if U is not None:

            if np.sum(U.values) == 0:
                stop = True
                continue # Do not do the following steps of word extraction !! The stop signal has been sent, and if U is None there is nothing to extract

            # -- Extract interesting words -- #
            # Here extract interesting words from dico and add to all_found_words

            # Binarize them (ie transform 0.9 - 0.9 - 0.001 into 1-1-0)
            # The sum of V_df**2 for each word is always 1.0 so our cutoff to say if a feature is used
            # is V**2 > 1/n_features**2 - epsilon for rounding errors.
            # We cannot simply binarize, since 0.001 would be binarized to 1 yet it is likely not interesting.
            # WARNING : no longer true if I have intra set overlap (ie 1-1-2 for example) so when looking for the combis I might need to ignore intra set overlap

            # Each learned word is remembered along with its usage in the reconstruction
            # This way when proceeding to step 2 we keep only the most relevant words from step 1 to save time.
            # Gibberish overcomplete words that are found when n_atoms is too high and were not used will have usage of 0
            for i in U.columns.tolist():
                this_word = (V**2).iloc[i,:].values

                # Remove padding
                this_word = this_word[:-1]

                # Above 1/k**2 ?
                first_thres = this_word - (1/nb_features**2) > EPSILON

                # Above discretization_threshold * max ?
                second_thres = this_word - (discretization_threshold * max(this_word)) > EPSILON

                # Above both ?
                this_word_binarized = np.logical_and(first_thres,second_thres).astype(int)

                this_word_binarized = tuple(this_word_binarized.tolist())

                this_word_usage = np.sum(U.values[:,i])

                # Now remember it
                all_candidate_words[this_word_binarized] += this_word_usage

                # Debug print
                message("Word = "+str(this_word_binarized)+" ; Usage = "+str(this_word_usage),
                    type = 'DEBUG')

    # Return the dictionary giving all candidate words and their total usage
    return all_candidate_words

# ---------------------------------------------------------------------------- #
#                        Step 2 of the MODL algorithm                          #
# ---------------------------------------------------------------------------- #

def normalize_and_jitter_matrix_rows(X, normalize = True, jitter = True):
    r"""
    Apply normalization so that sum of suquare of each row is 1,
    Add a slight epsilon so two rows may not have exact same dot product with a given vector
    Sort in lexicographic order

    >>> import numpy as np
    >>> import numpy.testing as npt
    >>> from pygtftk.stats.intersect.modl.subroutines import normalize_and_jitter_matrix_rows
    >>> D = np.array([[1,1,0],[0,1,0],[0,1,1]])
    >>> Dc = normalize_and_jitter_matrix_rows(D)
    >>> Dc_theory = [[0, 1, 0], [7.0736E-5, 7.07107E-1, 7.07107E-1], [7.07107E-1, 7.07107E-1, 9.99859E-05]]
    >>> npt.assert_almost_equal(Dc, Dc_theory, decimal = 5)
    """

    X = np.array(X) # Enforce NumPy array
    X = X[np.lexsort(np.rot90(X))] # Sort the dictionary in lexicographic order BEFORE applying all of this

    ## Applying corrections to the words : normalize and jitter
    new_X = []
    i = 0
    for row in X:

        # To prevent atoms in D from having the exact same dot product with any word
        # that LARS will try to rebuild, add a very small jitter to the words
        # -> sqrt(i) times 1E-4 to each element of the i-th row of D
        neo = np.array([x + np.sqrt(i)*1E-4 for x in row])
        i += 1

        # Normalize the words by the sum of their square. Helps counter a tendency to select longer words, which can increase sensitivity to noise.
        if normalize:
            squared_sum = np.sum(neo**2)
            if squared_sum > 0: # Avoid division by zero
                neo = np.sqrt(neo**2/squared_sum)

        new_X += [neo]

    # Now convert to array
    X = np.array(new_X)

    return X

def build_best_dict_from_library(data, library, queried_words_nb,
                error_function = None,
                nb_threads = 1, normalize_words = False,
                transform_alpha = None):
    r"""
    Given a data matrix and a library, will select the best n = queried_words_nb words
    with a greedy algorithm from the library to rebuild the data.

    `data` is a matrix with one transaction per row and one element per column, as usual

    The greeedy algorithm works because the problem in this particular instance
    is submodular. In broad strokes, it is assumed that the gain in term of reconstruction
    with the Sparse Coder of adding the word X to a set S is no higher than adding X to a
    set T that contains S. Also, more words cannot make the sparse coder do a worse job so
    the function is monotonous.

    Instead of the reconstruction error, you may pass a different callable of the form
    error_function(data, rebuilt_data, encoded, dictionary) that returns an error value so there
    can be a supervision, but the submodularity might no longer hold.

    >>> import numpy as np
    >>> from pygtftk.stats.intersect.modl import tree
    >>> from pygtftk.stats.intersect.modl import subroutines as modl_subroutines
    >>> X = np.array([[1,1,0,0,0,0],[1,1,1,1,0,0],[0,0,0,0,1,1]]*10)
    >>> candidates = [(1,1,0,0,0,0),(1,1,1,1,0,0),(0,0,0,0,1,1), (1,1,0,0,1,1), (1,0,1,0,1,0),(1,1,0,1,1,0),(1,0,1,1,0,0),(0,1,0,0,1,1)]
    >>> L = tree.Library()
    >>> L.build_nodes_for_words(candidates)
    >>> L.assign_nodes()
    >>> selection = modl_subroutines.build_best_dict_from_library(X, L, queried_words_nb = 3)
    >>> assert set(selection) == set([(1,1,0,0,0,0),(1,1,1,1,0,0),(0,0,0,0,1,1)])

    """

    ## Create objects

    # Initial dictionary
    nb_features = data.shape[1]

    dictionary = [tuple([0] * nb_features)] # Must initialize with a word, might as well be full zeros.

    # Use nonzero alpha for the coder as well, only if not manually specified.
    if transform_alpha is None:

        # To encourage use of fewer words, we by default use a rather high alpha (but capped at 0.5)
        transform_alpha = min(1/np.sqrt(nb_features), 0.5)

    ## Coder object, to be updated iteratively
    coder = SparseCoder(dictionary=np.array(dictionary).astype('float64'), # Rq : must make it float64 for some reason
        transform_algorithm='lasso_lars', #transform_algorithm='lasso_cd',
        transform_alpha = transform_alpha,
        positive_code = True, # Very important to ensure we also force positivity here !
        n_jobs = nb_threads)

    # We use lasso_lars, as lasso_cd often had mysterious convergence problems.
    # Lasso_lars can stil enforce positivity, and with precomputed words now the poor handling of degenerate
    # vectors (lars dropping them) is a strength

    # --- Filtering the candidates
    # Iteratively add candidates that improve the reconstruction
    stop = False
    while stop is False:

        ## Generate all_candidate_words for this iteration
        # For now, simply all words of the library that are not already used
        all_candidate_words = tree.get_all_candidates_except(library, dictionary)

        # Test all words for this iteration
        candidates_with_errors = dict()

        message("----------------------", type = 'DEBUG')
        message("Current dictionary : "+str(dictionary), type = 'DEBUG')
        message("> Candidate\tError", type = 'DEBUG')

        for candidate in all_candidate_words:

            # Create a new dictionary for this test
            dict_being_tested = dictionary + [candidate]

            Dt = np.array(dict_being_tested)

            # Normalization helps counter a tendency to selcet longer words, and jitter fixes a LARS bug with atoms that have the same dot product
            # Also lexicographic sort.
            Dt_corrected = normalize_and_jitter_matrix_rows(Dt, normalize_words)
            Dt = Dt_corrected

            # Update dictionary to the one being currently tested
            coder.dictionary = Dt.astype('float64')

            # Just to be safe
            assert np.allclose(coder.dictionary, Dt.astype('float64'))

            try:

                encoded = coder.transform(data)

                rebuilt_data = np.matmul(encoded, Dt)

                # We use Manhattan error so compromises are discouraged : with L2,
                # when rebuilding [(1,0,0)*many,(1,0,1),(0,0,1)*many], (1,0,1) would be picked first
                # Add an alpha that is nonzero so that using longer words is still
                # an improvement even when other words cover it (and as a tiebreaker)
                # but keep it low (1/k) so you don't re-encourage compromise
                def manhattan_dist_with_sparsity(X_true, X_rebuilt, encoded, dictionary):

                    dictionary = None # This particular function ignores the dictionary

                    error_mat = np.abs(X_rebuilt-X_true)
                    error = np.sum(error_mat)

                    alpha = transform_alpha # Fetch the alpha used when calling the main function

                    regul = np.sum(encoded) * alpha

                    final_error = error + regul
                    return final_error

                # Check for error function, default is manhattan dist
                if error_function is None:
                    error_function = manhattan_dist_with_sparsity

                # To compute the error, pass respectively : true data, rebuilt data, encoded data, and current dictionary
                # TODO: currently, the dictionary passed is BEFORE normalization and all that jazz
                error = error_function(data, rebuilt_data, encoded, dict_being_tested)

            # On rare occasion, convergence errors can result in a ValueError
            except ValueError:
                error = np.inf

            # Remember error
            candidates_with_errors[tuple(candidate)] = error

            message(str(candidate)+'\t'+str(error), type = 'DEBUG')

        # Add best candidate to final dict and remove from candidates.
        try:
            # In case of a tie, the first word is selected.
            best_candidate = min(candidates_with_errors, key=candidates_with_errors.get)
            dictionary += [best_candidate]
        # It should never happen, but if there are no more candidates add an empty word.
        except:
            dictionary += [tuple([0] * nb_features)]

        """
        # NOTE for improvement
        # To save time, we may only pass as candidates the "children" (see tree module)
        # of nodes already present under the assumption that if X is better than Y
        # the best parent of X should be better than the best parent of Y ?
        # Then we would need to wait for an optimum, with this draft code

        # If we have more words than the user wants, remove the worst one
        if (len(dictionary) > queried_words_nb):
            # Remove worst word with respect to error (try all, opposite of above)

        # When the word added and the word removed are the same (meaning
        # the word it wants to add does not improve on the dict) stop and
        # return
        if best_candidate == last_word_removed :
            best_words = final_dict
            return best_words
        """

        ## Stop condition : as soon as we have enough words
        # Remark : since the dictionary always contain the (0,0,0,...) word as a supplement, we
        # must stop when it contains the required number of words... plus one !
        if (len(dictionary) >= queried_words_nb + 1):
            stop = True

    message("Final dictionary :"+str(dictionary), type = 'DEBUG')

    # Conclusion of the function : remove the (0,0,0,...) word and make words unique, just in case
    best_words = list(set(dictionary))
    best_words.remove(tuple([0] * nb_features))

    return best_words


import os, shutil, ctypes
from tkinter.tix import DirList

# class Commit():

#     def __init__(self):                 #,szulo,szerzo,datum,commit_leiras,valtozott,commits=1
#         # self.commits = commits
#         # self.szulo = szulo
#         # self.szerzo = szerzo
#         pass

#     def commit_datails(self):
#         adatok_listaja = ["Szulo: ","Szerzo: ","Datum: ","Commit leiras: ","Valtozott: ",]

#         with open(f"{2}.commit.details.txt", "w", encoding="utf8") as f:
#             for adat in adatok_listaja:
#                 f.write(f"{adat} \n")

# Commit().Commit_datails()

path_here = os.getcwd()

# for e in os.listdir():
#     if e.name == ".dusza":
#         os.remove(".dusza")
os.mkdir(".dusza")
os.system("attrib +h myFile.txt")

source = f"{path_here}\\probaMappa"
destination = f"{path_here}\\probaMappa\\.dusza\\1.commit"

shutil.copytree(source, destination)


# theano_mnist_knn.py
import pandas as pd
import numpy as np
from theano import function
import theano.tensor as T
import time

# Read in data
training = pd.read_csv('MNIST_training.csv')
# training = pd.read_csv('../datasets/fashion-mnist_train.csv')
training_labels = training.iloc[:, 0]
training_data = training.drop('label', axis=1).as_matrix()
test = pd.read_csv('MNIST_test.csv')
# test = pd.read_csv('../datasets/fashion-mnist_test.csv')
test_labels = test.iloc[:, 0]
test_data = test.drop('label', axis=1).as_matrix()

X, Y = T.dvectors('X', 'Y')

distance = T.sqrt(T.sum(T.sqr(X - Y))) # equivalent to code below
# diff = X - Y
# diff_squared = T.sqr(diff)
# summed = T.sum(diff_squared)
# distance = T.sqrt(summed)
tdist = function([X, Y], distance)

def get_most_common(array):
	digits = np.zeros(10, dtype=int)
	for num in array:
		digits[num[1]] +=1
	return digits.argmax()

def evaluate_accuracy(knn_predictions):
	truth = np.array(test_labels) == np.array(knn_predictions)
	return sum(truth)

def dist(training, test):
	return np.sqrt(np.sum((training - test)**2))

start_time = time.time()
predictions = []
for test_case in test_data:
    distances = []
    for index, training_case in enumerate(training_data):
        distances.append((tdist(training_case, test_case), training_labels[index]))
    distances.sort(key=lambda tup: tup[0])
    predictions.append(get_most_common(distances[0:9]))
end_time = time.time()
elapsed_time = end_time - start_time

print("K = 9, Training Set Size: {}, Test Set Size: {}".format(len(training_labels), len(test_labels)))
print("Accuracy:")
print(evaluate_accuracy(predictions))
print("--")
print(len(test_labels))
print("Theano Runtime: {}".format(elapsed_time))

n_start_time = time.time()
n_predictions = []
for test_case in test_data:
    distances = []
    for index, training_case in enumerate(training_data):
        distances.append((dist(training_case, test_case), training_labels[index]))
    distances.sort(key=lambda tup: tup[0])
    n_predictions.append(get_most_common(distances[0:9]))
n_end_time = time.time()
n_elapsed_time = n_end_time - n_start_time

# print("Accuracy:")
# print(evaluate_accuracy(n_predictions))
# print("--")
# print(len(test_labels))
print("Numpy Runtime: {}".format(n_elapsed_time))

# a = [(0,2),(0,2),(0,1),(0,1),(0,1),(0,2)]
# print(get_most_common(a[0:2]))

# print(tdist(training_data[0], test_data[0]))
# print(tdist(training_data[0], test_data[40]))


import cv2
import subprocess as sp

rtspUrl = 'rtsp://11.73.12.184:554/test1' #这里改成本地ip，端口号不变，文件夹自定义
rtmpUrl = 'rtmp://8.136.105.93:10035/app/test'
# 视频来源 地址需要替换自己的可识别文件地址
sourceUrl='rtsp://admin:hk123456@11.73.12.20'
camera = cv2.VideoCapture(sourceUrl) # 从文件读取视频

# 视频属性
size = (int(camera.get(cv2.CAP_PROP_FRAME_WIDTH)), int(camera.get(cv2.CAP_PROP_FRAME_HEIGHT)))
sizeStr = str(size[0]) + 'x' + str(size[1])
fps = camera.get(cv2.CAP_PROP_FPS)  # 30p/self
fps = int(fps)
hz = int(1000.0 / fps)
print('size:'+ sizeStr + ' fps:' + str(fps) + ' hz:' + str(hz))

# 直播管道输出
# ffmpeg推送rtmp 重点 ： 通过管道 共享数据的方式
command = [
    'ffmpeg',
    # 're',#
    '-y', # 无需询问即可覆盖输出文件
    '-an',
    '-hwaccel', 'bmcodec',
    '-hwaccel_device', '0',
    '-f', 'rawvideo', # 强制输入或输出文件格式
    '-vcodec','rawvideo', # 设置视频编解码器。这是-codec:v的别名
    '-pix_fmt', 'bgr24', # 设置像素格式
    '-s', sizeStr, # 设置图像大小
    '-r', str(fps), # 设置帧率
    '-i', '-', # 输入
    '-is_dma_buffer','0',
    '-c:v', 'h264_bm',
    #'-pix_fmt', 'yuv420p',
    # '-preset', 'ultrafast',
    '-f', 'flv',# 强制输入或输出文件格式
    rtmpUrl]

# # ffmpeg推送rtsp 重点 ： 通过管道 共享数据的方式
# command = [
#     'ffmpeg',
#     # 're',#
#     '-y', # 无需询问即可覆盖输出文件
#     '-an',
#     '-hwaccel', 'bmcodec',
#     '-hwaccel_device', '0',
#     '-f', 'rawvideo', # 强制输入或输出文件格式
#     '-vcodec','rawvideo', # 设置视频编解码器。这是-codec:v的别名
#     '-pix_fmt', 'bgr24', # 设置像素格式
#     '-s', sizeStr, # 设置图像大小
#     '-r', str(fps), # 设置帧率
#     '-i', '-', # 输入
#     '-is_dma_buffer','0',
#     '-c:v', 'h264_bm',
#     #'-pix_fmt', 'yuv420p',
#     # '-preset', 'ultrafast',
#     '-f', 'rtsp',# 强制输入或输出文件格式
#     rtspUrl]

#管道特性配置
pipe = sp.Popen(command, stdin=sp.PIPE) #,shell=False
while (camera.isOpened()):
    ret, frame = camera.read() # 逐帧采集视频流
    if not ret:
        break
    # frame = cv2.cvtColor(frame,cv2.COLOR_BGR2YUV_I420)
    ############################图片输出
    # 结果帧处理 存入文件 / 推流 / ffmpeg 再处理
    pipe.stdin.write(frame.tobytes())  # 存入管道用于直播

camera.release()


import csv

def write_into_csv(info_list):
    with open('student_info.csv','a',newline='') as csv_file:
        writer = csv.writer(csv_file)

        if csv_file.tell() == 0:
            writer.writerow(["Name","Age","Contact Number","E-mail ID"])

        writer.writerow(info_list)
if __name__=='__main__':
    terms = True
    student_num = 1

    while(terms):
        student_info =input("enter sudent information for student # {} in the following format(name Age Contact_Number E-mail ID):".format(student_num))

        #split
        student_info_list=student_info.split(' ')

        print("\nThe entered information is :\nName: {}\nAge: {}\nContact_Number: {}\nE-mail ID : {}".format(student_info_list[0],student_info_list[1],student_info_list[2],student_info_list[3]))

        input_check = input("is the entered information is correct(yes/no):")

        if input_check=="yes":
            write_into_csv(student_info_list)
            terms_check =input("enter (yes/no) if  you want to enter information for another student:")
            if terms_check == "yes":
                terms = True
                student_num = student_num + 1
            elif terms_check == "no":
                terms = False
        elif input_check == "no":
            print("\nplease re-enter the values!")


from Language import Language
import re
import os
import main as m
import msvcrt as ms

def process_set(language):

    pattern = r'\s*(\w+)\s*=\s*{([^}]+)}'
    all_sets = re.findall(pattern, language)

    for _set in all_sets:
        name = _set[0]  # Nombre del conjunto
        values = re.findall(r'[^,\s]+', _set[1])  # Lista de valores del conjunto
        Language.add_language(name, values)

def show_all_language():
    print(f"Los Alfabetos registrados hasta el momento son: ")
    all_languages = Language.get_all_languages()
    for name in all_languages:
        if all_languages[name].elements:
            print(f"{name}: {all_languages[name].elements}")

def ask_name_language():
    language_names = input("\nIngrese los nombres de los lenguajes a operar de la forma L1 L2 L3 o L1, L2, L3: ")
    return re.findall(r'[^,\s]+', language_names)

def language_menu():

    os.system('cls')

    language_input = input("Digite los conjuntos de lenguanjess de la forma L1 = {cocina, pollo, Ingredientes} L2 = {programador, internet, ordenador, python}: ")
    language_input = "L1 = {fear, key, high} L2 = {hook, foot, kigs, 2}  L3 = {alex, daniel, deivis} D = {hello, bye, hi}" # esto es mientras se prueba el codigo
    process_set(language_input)

    #para este se hace igual que en el menu de alfabeto
    while True:

        os.system('cls')

        print("\n Menú de Operaciones con Lenguajes ")
        print("1. Unión")
        print("2. Intersección")
        print("3. Diferencia")
        print("4. Concatenación")
        print("5. Potencia")
        print("6. Inversión")
        print("7. Cardinalidad")
        print("8. Volver al menú principal")
        choice = input("Selecciona una operación: ")

        os.system('cls')

        if choice =="8":
            m.main()

        show_all_language()

        languages_names = ask_name_language()

        all_languages = Language.get_all_languages()

        result = None

        for name in languages_names:
            if name in all_languages:
                if result is None and choice in ("1", "2", "3", "4"):
                    result = Language.get_language(name)
                elif choice == "1":
                    result = Language(result._union(all_languages[name]))
                elif choice == "2":
                    result = Language(result._intersection(all_languages[name]))
                elif choice == "3":
                    result = Language(result._difference(all_languages[name]))
                elif choice == "4":
                    result = Language(result.concatenation(all_languages[name]))
                elif choice == "5":
                    exponent = int(input("ingrese el numero de veces que desee conectar el lenguaje con si mismo(exponente): "))
                    language = Language.get_language(name)
                    result = language.power(exponent)
                    print(f"La potencia de {name} elevado a {exponent} es {result}")
                elif choice == "6":
                    result = all_languages[name].inverse()
                    print(f"El resultado de invertir {name} es {result}")
                elif choice == "7":
                    print(f"La cardinalidad de {name} es {all_languages[name].cardinality()}")
                    result = all_languages[name].cardinality()
                else:
                    print("opcion no valida")
                    pass
            else:
                print(f'el conjunto {name} no se encuentra por ende no puede ser procesado')

        if choice in ("1", "2", "3", "4"):
            if result is None:
                print("ningun nombre coincide con los elementos que hay")
            else:
                print(f'El resultado es {result.elements}')

        ms.getch()
        os.system('cls')


import logging
import sys
import unittest

logger = logging.getLogger()
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(
    logging.Formatter(
        '[%(levelname)s] %(asctime)s %(filename)s:%(lineno)d %(message)s'))
logger.addHandler(handler)
logger.setLevel(logging.INFO)

if __name__ == '__main__':

  # Run the tests
  if ((len(sys.argv) < 2) or (sys.argv[1] == '?') or (sys.argv[1] == 'help')):

    print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n")

    print ("----- HELP :")
    print ("----- python test_RED.py <Type of test> <number of test>")
    print ("       or")
    print ("----- python test_RED.py <Type of test>")
    print ("")
    print ("----- Example: python test_RED.py REG 000 / python test_RED.py REG ")
    print ("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
    print ("")

  else :
    if (len(sys.argv) == 2):
      filename = 'RED_FT_S_' + sys.argv[1] + '_*' + '.py'
    else:
      filename = 'RED_FT_S_' + sys.argv[1] + '_' + str(sys.argv[2]) + '.py'
    #end_if

    print (filename)
    print (filename)
    tests = unittest.TestLoader().discover('testcases', filename)
    unittest.TextTestRunner(verbosity=0).run(tests)

  #end_if


import gettext
import os

from otopi import constants as otopicons
from otopi import context
from otopi import plugin
from otopi import util

from ovirt_hosted_engine_setup import constants as ohostedcons

def _(m):
    return gettext.dgettext(message=m, domain='ovirt-hosted-engine-setup')

@util.export
class Plugin(plugin.PluginBase):
    """Shell detection plugin."""

    def __init__(self, context):
        super(Plugin, self).__init__(context=context)

    @plugin.event(
        stage=plugin.Stages.STAGE_SETUP,
        priority=plugin.Stages.PRIORITY_HIGH,
        condition=lambda self: (
            not self.environment[otopicons.BaseEnv.ABORTED] and
            not self.environment[ohostedcons.CoreEnv.UPGRADING_APPLIANCE] and
            not self.environment[ohostedcons.CoreEnv.ROLLBACK_UPGRADE]
        ),
    )
    def _setup(self):
        self.environment.setdefault(
            ohostedcons.CoreEnv.SKIP_TTY_CHECK,
            False
        )
        self.environment.setdefault(
            ohostedcons.CoreEnv.SCREEN_PROCEED,
            None
        )
        ssh_connected = not os.getenv('SSH_CLIENT') is None
        skip_tty_check = self.environment[
            ohostedcons.CoreEnv.SKIP_TTY_CHECK
        ]
        if not skip_tty_check:
            if ssh_connected and os.getenv('TERM') is None:
                self.logger.error(
                    _(
                        'It has been detected that this program is executed '
                        'through an SSH connection without pseudo-tty '
                        'allocation.\n'
                        'Please run again ssh adding -t option\n'
                    )
                )
                raise context.Abort('Aborted due to missing requirement')
        screen_used = os.getenv('TERM') == 'screen'
        if ssh_connected and not screen_used:
            interactive = self.environment[
                ohostedcons.CoreEnv.SCREEN_PROCEED
            ] is None
            if interactive:
                self.environment[
                    ohostedcons.CoreEnv.SCREEN_PROCEED
                ] = self.dialog.queryString(
                    name=ohostedcons.Confirms.SCREEN_PROCEED,
                    note=_(
                        'It has been detected that this program is executed '
                        'through an SSH connection without using screen.\n'
                        'Continuing with the installation may lead to broken '
                        'installation if the network connection fails.\n'
                        'It is highly recommended to abort the installation '
                        'and run it inside a screen session using command '
                        '"screen".\n'
                        'Do you want to continue anyway? '
                        '(@VALUES@)[@DEFAULT@]: '
                    ),
                    prompt=True,
                    validValues=(_('Yes'), _('No')),
                    caseSensitive=False,
                    default=_('No')
                ) == _('Yes').lower()
                if not self.environment[ohostedcons.CoreEnv.SCREEN_PROCEED]:
                    raise context.Abort('Aborted by user')

# vim: expandtab tabstop=4 shiftwidth=4


import numpy as np
import pandas as pd
pd.set_option('display.float_format', lambda x: '%.3f' % x)
import matplotlib.pyplot as plt
import seaborn as sns
from subprocess import check_output
print(check_output(['ls', '../input']).decode('utf8'))
import glob, re
dfs = {re.search('/([^/\.]*)\.csv', fn).group(1):
      pd.read_csv(fn) for fn in glob.glob('../input/*.csv')}
for k, v in dfs.items(): locals()[k] = v
opp = order_products__prior
opp.tail()
opt = order_products__train
opt.tail()
opa = pd.concat([opp, opt], axis = 0) # 'order products all'
opa.tail()
# sanity check
print(len(opp))
print(len(opt))
print(len(opa))
print(opa.isnull().sum())
order_total = opa.groupby('order_id')['add_to_cart_order'].max().reset_index()
order_total = order_total.add_to_cart_order.value_counts()[:20] #nlargest(20)

sns.set_style('whitegrid')
f, ax = plt.subplots(figsize=(15,12))
plt.xticks(rotation = 'vertical')
sns.barplot(order_total.index, order_total.values)

plt.xlabel('Number of Orders', fontsize = 14)
plt.ylabel('Number of Products in Order', fontsize = 14)
#plt.xlabel('Number of Products in Order', fontsize=14)
plt.show()
pop = opa.groupby('product_id')['add_to_cart_order'].aggregate({'total_ordered': 'count'}).reset_index()
pop = pop.merge(products[['product_id', 'product_name']], how = 'left', on = ['product_id'])
pop = pop.sort_values(by = 'total_ordered', ascending = False)[:10]
pop
pop = pop.groupby(['product_name']).sum()['total_ordered'].sort_values(ascending = False)

f, ax = plt.subplots(figsize=(12,10))
sns.set_style('darkgrid')
sns.barplot(pop.index, pop.values)

plt.xticks(rotation = 'vertical')
plt.ylabel('Number of Orders', fontsize = 14)
plt.xlabel('Most Popular Products', fontsize = 14)
plt.show()
reorder_ratio = opa.groupby('reordered')['product_id'].agg({'total_products': 'count'}).reset_index()
reorder_ratio['ratio'] = reorder_ratio['total_products'].apply(lambda x: x / reorder_ratio['total_products'].sum())
reorder_ratio
reorder_ratio = reorder_ratio.groupby(['reordered']).sum()['total_products'].sort_values(ascending = False)

f, ax = plt.subplots(figsize = (5,8))
sns.set_style('whitegrid')
sns.barplot(reorder_ratio.index, reorder_ratio.values, palette = 'RdBu')

plt.xlabel('New Item/Reordered', fontsize = 14)
plt.ylabel('Total Number of Orders', fontsize = 14)
plt.ticklabel_format(style = 'plain', axis = 'y')
plt.show()

product_rr = opa.groupby('product_id')['reordered'].agg({'reorder_total': sum, 'order_total': 'count'}).reset_index()
product_rr['reorder_probability'] = product_rr['reorder_total'] / product_rr['order_total']
product_rr = product_rr.merge(products[['product_name', 'product_id']], how = 'left', on = 'product_id')
product_rr = product_rr[product_rr.order_total > 75].sort_values(['reorder_probability'], ascending = False)[:10]
product_rr
product_rr = product_rr.sort_values('reorder_probability', ascending = False)

plt.subplots(figsize = (12, 10))
sns.set_style('darkgrid')
sns.barplot(product_rr.product_name, product_rr.reorder_probability)
plt.ylim([.85, .95])
plt.xticks(rotation = 'vertical')
plt.xlabel('Most Reordered Products')
plt.ylabel('Reorder Probability')
plt.show()
time_of_day = orders.groupby('order_hour_of_day')['order_id'].agg('count').reset_index()

f, ax = plt.subplots(figsize = (15, 10))
sns.barplot(time_of_day['order_hour_of_day'], time_of_day['order_id'])
plt.xlabel('Orders by Hour', fontsize = 14)
plt.ylabel('Total Orders', fontsize = 14)
plt.show()
grouped = orders.groupby('order_dow')['order_id'].agg('count').reset_index()

f, ax = plt.subplots(figsize = (15,10))
sns.set_style('whitegrid')
current_palette = sns.color_palette('colorblind')
sns.set_palette(current_palette)
sns.barplot(x = 'order_dow', y = 'order_id', data = grouped)
plt.xlabel('Day of Week')
plt.ylabel('Total Orders')
plt.show()
grouped = orders.groupby('days_since_prior_order')['order_id'].agg('count').reset_index()

from matplotlib.ticker import FormatStrFormatter
f, ax = plt.subplots(figsize = (15,19))
sns.barplot(x = 'days_since_prior_order', y = 'order_id', data = grouped)
ax.xaxis.set_major_formatter(FormatStrFormatter('%.0f'))
plt.xlabel('Days to Reorder')
plt.ylabel('Total Orders')
plt.show()
grouped = orders.groupby('user_id')['order_id'].agg('count')
grouped = grouped.value_counts()

f, ax = plt.subplots(figsize=(15, 12))
sns.barplot(grouped.index, grouped.values)
plt.xlabel('Number of Orders')
plt.ylabel('Number of Customers')
plt.show()
items = products.merge(departments, how='left', on='department_id')
items = items.merge(aisles, how='left', on='aisle_id')
items.head()
grouped = items.groupby('department')['department_id'].agg({'total_products': 'count'}).reset_index()
grouped['percent_of_inv'] = grouped['total_products'].apply(lambda x: x / grouped['total_products'].sum())
grouped = grouped.sort_values(by='total_products', ascending=False)
grouped.head()
f, ax = plt.subplots(figsize=(12,10))
sns.barplot(x='department', y='total_products', data=grouped)
plt.xticks(rotation='vertical')
plt.xlabel('Department', fontsize=14)
plt.ylabel('Number of Products', fontsize=14)
plt.show
g1 = items.groupby(["department", "aisle"], as_index=False).size().reset_index(name="count")
g2 = g1.loc[g1['department'] == 'personal care']
g2 = g2.sort_values(by='count', ascending=False)[:5]
current_palette = sns.color_palette('colorblind')
sns.set_palette(current_palette)
sns.barplot(x='aisle', y='count', data=g2)
plt.xticks(rotation='vertical')
plt.title('Personal Care Department')
plt.show()
g2 = g1.loc[g1['department'] == 'snacks']
g2 = g2.sort_values(by='count', ascending=False)[:5]
current_palette = sns.color_palette('colorblind')
sns.set_palette(current_palette)
sns.barplot(x='aisle', y='count', data=g2)
plt.xticks(rotation='vertical')
plt.title('Snacks Department')
plt.show()
grouped = items.groupby('aisle')['product_id'].agg({'total_products': 'count'}).reset_index()
grouped['ratio'] = grouped['total_products'].apply(lambda x: x /grouped['total_products'].sum())
grouped = grouped.sort_values(by='total_products', ascending=False)[:10]
grouped
f, ax = plt.subplots(figsize=(12,10))
sns.barplot(x='aisle', y='total_products', data=grouped)
plt.xticks(rotation='vertical')
plt.show()
orders.head()
orders.eval_set.value_counts()
order_products = orders[['user_id', 'order_id']].merge(opa[['order_id', 'product_id']],
                                                 how='inner', on='order_id')
order_products = order_products.merge(items, how='inner', on='product_id')

grouped = order_products.groupby('department')['order_id'].agg({'total_orders': 'count'}).reset_index()
grouped['ratio'] = grouped['total_orders'].apply(lambda x: x / grouped['total_orders'].sum())
grouped = grouped.sort_values(by='total_orders', ascending=False)
grouped
f, ax = plt.subplots(figsize=(12,10))
sns.barplot(x='department', y='total_orders', data=grouped)
ax.yaxis.set_major_formatter(FormatStrFormatter('%.0f'))
plt.xticks(rotation='vertical')
plt.xlabel('Departments', fontsize=14)
plt.ylabel('Number of Orders', fontsize=14)
plt.show()
g1 = order_products.groupby(['department', 'aisle'], as_index=False).size().reset_index(name="count")
g2 = g1.loc[g1['department'] == 'produce']
g2 = g2.sort_values(by='count', ascending=False)[:5]
g2
current_palette = sns.color_palette('colorblind')
sns.set_palette(current_palette)
sns.barplot(x='aisle', y='count', data=g2)
plt.xticks(rotation='vertical')
plt.title('Top Produce Aisles')
plt.show()
grouped = order_products.groupby("aisle")["order_id"].aggregate({'Total_orders': 'count'}).reset_index()
grouped['Ratio'] = grouped["Total_orders"].apply(lambda x: x /grouped['Total_orders'].sum())
grouped.sort_values(by='Total_orders', ascending=False, inplace=True )
grouped.head(10)
grouped  = grouped.groupby(['aisle']).sum()['Total_orders'].sort_values(ascending=False)[:15]

f, ax = plt.subplots(figsize=(12, 15))
plt.xticks(rotation='vertical')
sns.barplot(grouped.index, grouped.values)
plt.ylabel('Number of Orders', fontsize=13)
plt.xlabel('Aisles', fontsize=13)
plt.show()