import zmq
import json
import os
import sys
import glob
import random
import string
import hashlib

COMMANDS = ['first', 'connect']

# UTILS------------------------------------

def random_string():
    chars = string.ascii_lowercase
    return ''.join((random.choice(chars)) for x in range(30))

def hash_string(cadena):
    h = hashlib.sha1()
    h.update(cadena.encode('utf-8'))
    hexa = h.hexdigest()
    return int(hexa, 16)

def upload(folder_name, re, byte_content):
    file_hash = re['file_hash']
    path = folder_name+'/files/{}.txt'.format(file_hash)
    message = add_file(path, byte_content)
    return message

def add_file(path, byte_content):
    file = open(path, 'wb+')
    file.write(byte_content)
    file.close()
    return [{'message': 'ok'}]

def download(folder_name, file_name):
    # path = folder_name+'/files/{}'.format(file_name)
    path = folder_name + "//files//" + file_name

    try:
        file = open(path, 'rb')
        content = file.read()
        file.close()
        return [{'message': 'ok-download'}, content]
    except:
        return [{
                'message': 'message',
                're': 'El archivo no existe'
            }, b'']

class Folder_Controller:

    def __init__(self, folder_name):
        self.name = str(folder_name)
        self.__create_folder()

    def __create_folder(self):
        os.makedirs(self.name+'/files')
        os.makedirs(self.name+'/data')
        self.__init_intervals_file()

    def __init_intervals_file(self):
        with open(self.name+'/data/interval.json', 'w+') as file:
            data = {}
            json.dump(data, file)

class Server:
    def __init__(self):
        self.command = ''
        self.port_server = ''
        self.port_reply = '8001'
        self.port_request = '8001'
        self.id = 0
        self.id_reply = 0
        self.id_request = 0
        self.interval = {}
        self.folder_controller = None

    def __init_folder_controller(self):
        self.folder_controller = Folder_Controller(self.id)

    def verify_command(self, command):
        if(command in COMMANDS):
            self.command = command
            return True
        else:
            return False

    def create_first(self, port_server, id_server):
        self.port_server = str(port_server)
        self.id = id_server
        self.__init_folder_controller()
        return True

    def assign_connection(self, port_server, port_reply, port_request, id_server):
        self.port_server = port_server
        self.port_reply = port_reply
        self.port_request = port_request
        self.id = id_server
        self.__init_folder_controller()
        return True

    def check_id(self):
        control_check_id = True
        port_temp = int(self.port_reply)

        while control_check_id:

            s.connect('tcp://localhost:{}'.format(port_temp))
            data = json.dumps(
                {'message': 'check-id', 'id_server': self.id}).encode('utf-8')

            s.send_multipart([data, b''])

            reply = s.recv_multipart()
            re = json.loads(reply[0])

            if(re['ok'] == 'accepted'):
                control_check_id = False

                self.port_request = str(re['port'])
                self.port_reply = str(port_temp)

                s.disconnect('tcp://localhost:{}'.format(port_temp))

            elif(re['ok'] == 'rejected'):

                if(int(re['port']) == 8001):
                    control_check_id = False

                    self.port_request = str(port_temp)
                    self.port_reply = str(re['port'])

                s.disconnect('tcp://localhost:{}'.format(port_temp))
                port_temp = int(re['port'])

    def create_interval(self):

        self.interval['intervals'] = []

        if(self.command == 'first'):
            max_interval = pow(2, 160) - 1
            print("ID Server:", max_interval)
            self.id_reply = max_interval
            self.interval['intervals'].append(
                [int(self.id_request), max_interval])

        if(self.command == 'connect'):

            self.interval['intervals'] = []
            self.interval['intervals'].append(
                [int(self.id_request), int(self.id) - 1])
            self.id_reply = int(self.id) - 1

        path = self.folder_controller.name + '/data/interval.json'
        with open(path, 'w+') as file:
            data = {}
            data['intervals'] = server.interval['intervals']
            data['port_origin'] = server.port_server
            data['port_destiny'] = server.port_reply
            json.dump(data, file)

    def create_connection_reply(self, s,):
        s.connect('tcp://localhost:{}'.format(self.port_reply))
        print(self.port_reply)
        data = json.dumps({'message': 'create-connection-reply',
                          'id_server': self.id, 'port': self.port_server}).encode('utf-8')

        s.send_multipart([data, b''])
        reply = s.recv_multipart()
        re = json.loads(reply[0])
        files_bytes_list = reply[1:]

        s.disconnect('tcp://localhost:{}'.format(self.port_reply))

        for file_name, content in zip(re['non_range_files'], files_bytes_list):
            path = self.folder_controller.name+'/files/{}'.format(file_name)
            add_file(path, content)

    def create_connection_request(self, s,):
        s.connect('tcp://localhost:{}'.format(self.port_request))
        data = json.dumps({'message': 'create-connection-request',
                          'port': self.port_server}).encode('utf-8')
        s.send_multipart([data, b''])
        reply = s.recv_multipart()

        re = json.loads(reply[0])
        server.id_request = re['id_server']
        print(re['ok'])
        server.create_interval()

        s.disconnect('tcp://localhost:{}'.format(self.port_request))

    def check_intervals(self, hash):
        dec = int(hash, 16)
        value = (pow(2, 160))
        value = (int((dec % value)))

        if(value >= self.interval['intervals'][0][0] and value <= self.interval['intervals'][0][1]):
            return True
        return False

    def delete_file_with_name(self, file_name):
        path = self.folder_controller.name+'/files/'+file_name
        if os.path.exists(path):
            os.remove(path)
            return True
        else:
            print("The file does not exist")
            return False

    def get_file_with_name(self, file_name):
        path = self.folder_controller.name+'/files/'+file_name
        file = open(path, 'rb')
        return file

    def get_files_list(self):
        path = self.folder_controller.name+'/files/*.txt'
        return glob.glob(path)

    def get_non_in_range_files(self):
        files_list = self.get_files_list()
        non_range_files_list = []  # Lista de archivos que no pertenecen al intervalo
        for file_path in files_list:
            file_name = os.path.split(file_path)[1]
            file_hash = file_name.split('.')[0]
            if not server.check_intervals(file_hash):
                non_range_files_list.append(file_name)
        return non_range_files_list

count = 0
context = zmq.Context()
command = sys.argv[1]

if(command == 'first'):
    s = context.socket(zmq.REP)
elif(command == 'connect'):
    s = context.socket(zmq.REQ)

control = True
server = Server()

if(server.verify_command(command=command)):

    if(command == 'first'):
        port_server = 8001
        server.create_first(port_server=8001, id_server=0)

        server.create_interval()

    if(command == 'connect'):
        port_server = input("Ingrese el puerto: ")
        number_interval = hash_string(random_string())

        server.id = number_interval
        print("ID Server:", number_interval)
        server.check_id()
        server.assign_connection(port_server=port_server, port_reply=server.port_reply,
                                 port_request=server.port_request, id_server=number_interval)

        server.create_connection_request(s=s)
        server.create_connection_reply(s=s)

else:
    control = False

s = context.socket(zmq.REP)
s.bind('tcp://*:{}'.format(port_server))

while control:

    print('\nEsperando solicitud {}\n\n'.format(count))
    count = count + 1

    m = s.recv_multipart()
    request = json.loads(m[0])

    if(request['message'] == 'check-id'):
        if(int(server.id) > int(request['id_server'])):
            print('Peticion aceptada')
            data_send = [json.dumps(
                {'ok': 'accepted', 'port': server.port_request}).encode('utf-8'), b'']
        else:
            print('Peticion rechazada')
            data_send = [json.dumps(
                {'ok': 'rejected', 'port': server.port_reply}).encode('utf-8'), b'']
        s.send_multipart(data_send)

    if(request['message'] == 'create-connection-reply'):
        server.port_request = request['port']
        server.id_request = request['id_server']

        server.create_interval()

        non_in_range_files = server.get_non_in_range_files()

        bytes_array = []

        for file_name in non_in_range_files:

            file = server.get_file_with_name(file_name)
            bytes_array.append(file.read())
            file.close()

        data_send = [json.dumps(
                {
                    'ok': 'El intervalo fue actualizado',
                    'non_range_files': non_in_range_files
                }).encode('utf-8'),*bytes_array]

        print(len(data_send))

        s.send_multipart(data_send)

        for file_name in non_in_range_files:
            server.delete_file_with_name(file_name)

    if(request['message'] == 'create-connection-request'):
        server.port_reply = request['port']

        data_send = [json.dumps(
            {'ok': 'ID fue enviado', 'id_server': server.id}).encode('utf-8'), b'']
        print('El id fue enviado al {}'.format(server.port_reply))
        server.create_interval()

        s.send_multipart(data_send)

    if(request['message'] == 'check_to_upload'):

        message_client = request['hash']
        control_upload = server.check_intervals(message_client)

        if(control_upload):
            data_send = [json.dumps(
                {'status': 'belongs', }).encode('utf-8'), b'']
            print('El archivo pertenece')
            s.send_multipart(data_send)
        else:
            data_send = [json.dumps(
                {
                    'status': 'continue_search',
                    'port': server.port_reply
                }).encode('utf-8'), b'']
            print('continuar la busqueda')
            s.send_multipart(data_send)

    if request['message'] == 'upload':

        message = upload(folder_name=server.folder_controller.name,re=request, byte_content=m[1])
        id = server.id
        data_send = [json.dumps(message[0]).encode('utf-8'), b'']
        s.send_multipart(data_send)
        s.recv_string()
        s.send_string(str(id))

    if request['message'] == 'download':
        file_name = request['file_name']
        message = download(folder_name=server.folder_controller.name,file_name=request['file_name'])
        byte_content = message[1]
        resp = message[0]
        s.send_multipart([json.dumps(resp).encode('utf-8'), byte_content])


# 0  1  2  3  4  5  6
# 0! 1! 2! 3! 4! 5! 6!
# 1  1  2  6  24 120...

# N = 5

# n = 0
# while n <= N:
#     n += 1
#     an = n * an

def fact_gen(N):
    an = 1
    yield an
    for n in range(1, N+1):
        an = n * an
        yield an

def fact_gen_to_inf():
    an = 1
    n = 0
    yield an
    while True:
        n += 1
        an = n * an
        yield an

if __name__ == '__main__':

    for f in fact_gen_to_inf():
        if f > 1000_000:
            break
        print(f)


import logbook
import pandas as pd
import zipline as zl
from datetime import datetime, timedelta
import pathlib
import azul
import numpy as np
from typing import List, Tuple

log = logbook.Logger('BasePriceManager')

class BasePriceManager(object):

    def __init__(self, calendar_name='NYSE'):
        self._calendar = zl.get_calendar(name=calendar_name)
        self._cols = ['open', 'high', 'low', 'close', 'volume', 'dividend', 'split']

        # The number of days the price manager will keep trying to pull data for a symbol that is not returning data.
        self.MISSING_DATE_THRESHOLD = 5

    def get_price_data(self, symbols: List[str], output_dir: str, start_date: datetime, end_date: datetime) -> None:
        minute_dir_path = pathlib.Path(output_dir, 'minute')
        daily_dir_path = pathlib.Path(output_dir, 'daily')

        for ticker in symbols:
            self._download_and_process_data(
                ticker, start_date, end_date, minute_dir_path, daily_dir_path)

    def _download_and_process_data(
            self,
            ticker: str,
            start_date: datetime,
            end_date: datetime,
            minute_dir_path: pathlib.Path,
            daily_dir_path: pathlib.Path
    ) -> None:
        df = self._minute_dataframe_for_dates(ticker, start_date, end_date)

        if df.empty:
            return

        df = self._check_sessions(df, ticker, frequency='minute')
        minute_dir_path.mkdir(parents=True, exist_ok=True)
        filename = pathlib.Path(minute_dir_path, ticker + '.csv')
        df.to_csv(filename)

        daily_df = self._resample_minute_data_to_daily_data(df)
        daily_df = self._check_sessions(daily_df, ticker, frequency='daily')
        daily_dir_path.mkdir(parents=True, exist_ok=True)
        filename = pathlib.Path(daily_dir_path, ticker + '.csv')
        daily_df.to_csv(filename)
        log.notice('Retrieved: {}'.format(ticker))

    def _resample_minute_data_to_daily_data(self, df):
        ohlc_dict = {
            'open': 'first',
            'high': 'max',
            'low': 'min',
            'close': 'last',
            'volume': 'sum',
            'dividend': 'last',
            'split': 'last'
        }

        if df.empty:
            daily_df = df
        else:
            daily_df = df.resample('D', closed='left', label='left').agg(ohlc_dict).dropna(how='any')

        # Resample mixes the columns so lets re-arrange them
        daily_df = daily_df[self._cols]
        return daily_df

    def _validated_start_and_end_dates(
            self,
            start_date: datetime,
            end_date: datetime
    ) -> Tuple[datetime, datetime]:
        """
        Creates valid start and end dates. Defaults to a start date of 30 calendar days ago and end date of today.

        Args:
            start_date (datetime): The start date.
            end_date (datetime): The end date.

        Returns:
            start_date (datetime): The validated start date.
            end_date (datetime): The validated end date.

        """
        today = datetime.today()
        if start_date is None:
            start_date = today - timedelta(days=30)

        if end_date is None:
            end_date = today

        if start_date > end_date:
            temp_date = start_date
            start_date = end_date
            end_date = temp_date

        return start_date, end_date

    def _minute_dataframe_for_dates(
            self,
            ticker: str,
            start_date: datetime,
            end_date: datetime
    ) -> pd.DataFrame:
        """
        Returns a DataFrame containing the all the minute bars for stock between the start and end dates.

        Args:
            ticker (str): Ticker symbol for the stock.
            start_date (datetime): Date to start pulling data.
            end_date (datetime): Date to stop pulling data.

        Returns:
            combined_df (DataFrame): Contains the all the minute bars for a stock between the start and end dates.

        """

        start_date, end_date = self._validated_start_and_end_dates(start_date, end_date)

        combined_df = pd.DataFrame(columns=self._cols)
        combined_df.index.name = 'date'

        #
        # # Get the date the symbol was listed on the exchange.
        # list_date = self._list_date(ticker)
        #
        # if list_date is not None:
        #     # If the we are asking for data from before the stock was listed, then set the start date to the day
        #     # the stock was listed.
        #     if list_date > start_date:
        #         log.info('The symbol {} was not listed until: {}. Adjusting start time.', ticker, list_date)
        #         start_date = list_date

        # Build a list of the trading days from the dates passed in.
        session_dates = self._calendar.sessions_in_range(start_date, end_date)

        if session_dates.empty:
            log.info('The symbol {} did not trade between {} and {} ', ticker, start_date, end_date)
            return combined_df

        # Iterate over the trading dates backwards. This means we don't need to know exactly
        # when the stock started trading. Note: this won't pull data for stocks that have been delisted.
        # TODO: Add code to capture data for delisted stocks.
        num_missing_dates = 0
        for timestamp in reversed(session_dates):
            df = self._minute_dataframe_for_date(ticker, timestamp)
            if df.empty:
                # Start counting the number of consecutive trading dates we are missing data.
                num_missing_dates += 1
                log.info('No minute data for {} on {}'.format(ticker, timestamp.date()))
            else:
                # reset missing date counter
                num_missing_dates = 0
                log.info('Retrieved minute data for {} on {}'.format(ticker, timestamp.date()))
                combined_df = pd.concat([combined_df, df])

            if num_missing_dates >= self.MISSING_DATE_THRESHOLD:
                log.info('No minute data for {} for {} days. Quitting.'.format(ticker, self.MISSING_DATE_THRESHOLD))
                break

        # Sort the dataframe oldest first, newest last.
        combined_df.sort_index(inplace=True)
        return combined_df

    # def _list_date(self, ticker: str) -> datetime:
    #     return None

    def _minute_dataframe_for_date(self, ticker: str, start_timestamp: pd.Timestamp) -> pd.DataFrame:
        raise NotImplementedError

    def _fixna(self, df, symbol):
        cols = ['close', 'high', 'low', 'open']
        df[cols] = df[cols].replace({0: np.nan})
        df[cols] = df[cols].replace({-1.0: np.nan})
        if df.isnull().sum().sum() > 0:
            # fixna_list.append(symbol)
            df['open'] = df['open'].bfill().ffill()
            df['close'] = df['close'].bfill().ffill()
            df.loc[df['low'].isnull(), 'low'] = df['open']
            df.loc[df['high'].isnull(), 'high'] = df['open']
            df.loc[df['close'].isnull(), 'close'] = df['open']
        return df

    def _check_sessions(self, df, ticker, frequency='daily'):

        # Remove any data that are outside of the trading sessions for the calendar.

        if df.empty:
            return df

        asset_first_day = df.index[0]
        asset_last_day = df.index[-1]
        sessions = self._calendar.sessions_in_range(asset_first_day, asset_last_day)
        asset_sessions = sessions[sessions.slice_indexer(asset_first_day, asset_last_day)]

        if frequency == 'minute':
            minutes_passed = len(df)
            asset_first_day = self._calendar.minute_to_session_label(asset_first_day, direction='next')
            asset_last_day = self._calendar.minute_to_session_label(asset_last_day, direction='previous')
            minutes_in_session = self._calendar.minutes_for_sessions_in_range(asset_first_day, asset_last_day)
            df = df[df.index.isin(minutes_in_session)]
            if (minutes_passed) > len(minutes_in_session):
                # print('Removed ' + str((minutes_passed) - len(minutes_in_session)) + ' minutes')
                pass
            elif minutes_passed < len(minutes_in_session):
                num_missing_sessions = len(minutes_in_session) - minutes_passed
                log.info('Missing sessions for {}'.format(ticker))
        elif frequency == 'daily' and len(df) != len(asset_sessions):
            missing_sessions = asset_sessions.difference(
                pd.to_datetime(np.array(df.index), unit='s', utc=True, )).tolist()
            extra_sessions = pd.to_datetime(np.array(df.index), unit='s', utc=True, ).difference(
                asset_sessions).tolist()
            if len(missing_sessions) > 0:
                # missing_sessions_list.append(symbol)
                # print('Adding ' + str(len(missing_sessions)) + ' sessions for ' + str(ticker))
                pass
            if len(extra_sessions) > 0:
                # extra_sessions_list.append(symbol)
                # print('Removing ' + str(len(extra_sessions)) + ' sessions for ' + str(symbol))
                pass
            for missing_session in missing_sessions:
                prev_date = self._calendar.previous_session_label(missing_session)
                row_to_copy = df[(df.index == prev_date)]
                row_to_copy_val = row_to_copy.values
                # from IPython import embed; embed()
                df.loc[missing_session] = row_to_copy_val[0]
                df.loc[missing_session].volume = 0
                # row = row_to_copy
                # table.append(row)

            for extra_session in extra_sessions:
                # delete stuff
                df.drop(extra_session)

        if frequency == 'minute':
            log.info('Downloaded and processed {} minute bars for {}', len(df), ticker)
        else:
            log.info('Downsampled {} daily bars for {}', len(df), ticker)

        return df


import scipy.io
import os
import cv2

count = 0
index = 0
picNum = 0
RandCor = []
FaceCor = []
imagePath = 'G:/baiduyun/faceresource/wider/WIDER_train/images/'
data = scipy.io.loadmat("G:/baiduyun/faceresource/wider/wider_face_split/wider_face_train.mat")
FileName = data['file_list']
FileValue = data['face_bbx_list']
FileOcc = data['occlusion_label_list']
num = 0

def file_name(temp):
    # print(temp)
    file_dir = 'G:/baiduyun/faceresource/wider/WIDER_train/images/'
    if (temp >= 10):
        for root, dirs, files in os.walk(file_dir):
            # print(root)  # 当前目录路径
            for f in dirs:
                if (f[1] != '-'):
                    if (temp == int(f[:2])):
                        #print("test1:", f[:2])
                        file_dir = file_dir + f + "/";

    else:
        # print("小于10")
        for root, dirs, files in os.walk(file_dir):
            # print(root)  # 当前目录路径
            for f in dirs:
                # print("printftest:",f[0])
                if (temp == int(f[0]) and f[1] == '-'):
                    # print("test2:", f[0])
                    file_dir = file_dir + f + "/";

    if(file_dir == 'G:/baiduyun/faceresource/wider/WIDER_train/images/'):
        file_dir = 'error'
    return file_dir

def FindAddress(step):
    if (step < 2):
        temp = step
    elif (step >= 2 and step <= 11):
        temp = 10 + step - 2
    # print("test3:",step)
    elif (step == 12):
        temp = 2
    elif (step > 12 and step <= 22):
        temp = 20 + step - 13
    elif (step == 23):
        temp = 3
    elif (step > 23 and step <= 33):
        temp = 30 + step - 24
    elif (step == 34):
        temp = 4
    elif (step > 34 and step <= 44):
        temp = 40 + step - 35
    elif (step == 45):
        temp = 5
    elif (step > 45 and step <= 55):
        temp = 50 + step - 46
    elif (step == 56):
        temp = 6
    elif (step > 56 and step <= 66):
        temp = 60 + step - 57
    string = file_name(temp)
    #print(step)
    return string

for step in range(61):
    print(step)
    #log = open("log.txt", 'w')
    # i遍历FileName[step][0],j遍历FileValue[step][0]，p遍历FileOcc[step][0]
    for (i, j, p) in zip(FileName[step][0], FileValue[step][0], FileOcc[step][0]):
        # print("test4", step)
        #print("THE k is", step)
        newstring = FindAddress(step)
        #print("findAddok")
        if (newstring == 'error'):
            print("string error continue")
            continue
        #print("readok")
        newstring = newstring + i[0][0] + '.jpg'
        num = num + 1
        print(newstring,num)
        if (os.path.exists(newstring) == False):
            print("path error")
            continue
        img = cv2.imread(newstring)
        print("readok")
        #FaceCor = []
        for k in range(j[0].shape[0]):  # 人头数
            if int(j[0][k][2]) <= 50 and int(j[0][k][3]) <= 50 or p[0][k] != 0:
                continue
            crop = img[int(j[0][k][1])+30:int(j[0][k][1]) + int(j[0][k][3]),
                   int(j[0][k][0])+20:int(j[0][k][0]) + int(j[0][k][2])]
            s=input('输入')
            print(s)
            count += 1
            print("write")
            filename = "F:/Traindata/eyes/" + str(count) + ".jpg"
            cv2.imwrite(filename, crop)
        print(step)
        print("writeok")


# This method is horribly inefficient because the content of sentences is incremented over the scans and we tokenize it again and again!
# Would probably be smarter to do this in the reader directly

# The tokenization code so far only works if the model is initialized as
# model = spacy.load('en_core_web_sm')
# Would probably have to refactor a lot if I want to change the tokenization

def tokenize_all(scan_events, model):
    tokenized_events = []
    print("Tokenizing")
    for event in scan_events:
        print(event.subject_id, event.block, event.timestamp)
        # Tokenize
        sentences = tokenize_sentences(event.sentences, model)
        current_sentence = tokenize_text(event.current_sentence, model)
        stimulus = tokenize_text(event.stimulus, model)

        # Put tokenized version back (this could be done in one step, but it is more readable like this)
        event.sentences = sentences
        event.current_sentence = current_sentence
        event.stimulus = stimulus
        tokenized_events.append(event)

    return tokenized_events

def tokenize_sentences(sentences, model):
    tokenized_sents = []
    for sentence in sentences:
        tok_sent = model(sentence)
        tokenized_sents.append([tok.text for tok in tok_sent])
    return tokenized_sents

def tokenize_text(text, model):
    return [tok.text for tok in model(text)]


# Exercises 1,2,3

# birthday = {
#     "Vasya" : "1994/04/26",
#     "Dima" : "1998/05/01",
#     "Vanya" : "2003/04/22",
#     "Ura" : "2000/11/23",
#     "Max" : "1997/01/12"
#     }
# print("You can look up the birthdays of the people in the list!")
# print(birthday.keys())
# new_name = input("Type your name:\n")
# new_date = input("Type your date(YYYY/MM/DD):\n")
# birthday.update({new_name:new_date})
# name = input("Choose name: \n")
# if name in birthday.keys():
#     print(f"{name} birtday is {birthday[name]}")
# else:
#     print(f"Sorry, we don’t have the birthday information for {name}")

# Exercise 4

# items = {
#     "banana": 4,
#     "apple": 2,
#     "orange": 1.5,
#     "pear": 3
# }

# for item, price in items.items():
#     print(f"The price of {item} is ${price}")

items = {
    "banana": {"price": 4 , "stock":10},
    "apple": {"price": 2, "stock":5},
    "orange": {"price": 1.5 , "stock":24},
    "pear": {"price": 3 , "stock":1}
}

total_cost = 0
for item in items.values():
    total_cost += item["price"] * item["stock"]
print(f"The total cost to buy everything in stock is ${total_cost}")


import argparse
import torch
import os
import DistGNN
import time

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--root',
                        default='dataset/',
                        help='Path of the dataset.')
    parser.add_argument(
        "--dataset",
        default="ogbn-papers100M",
        choices=["ogbn-products", "ogbn-papers100M", "ogbn-papers400M"])
    parser.add_argument("--save-path", default=None, type=str)
    parser.add_argument("--num-parts", default=None, type=int)
    parser.add_argument("--parts", default=None, type=str)
    args = parser.parse_args()
    print(args)

    start = time.time()
    graph, _ = DistGNN.dataloading.load_dataset(args.root,
                                                args.dataset,
                                                with_feature=False)
    end = time.time()
    print("Loading graph takes {:.3f} s".format(end - start))

    num_nodes = graph["indptr"].numel() - 1
    start = time.time()
    parts_mask_list = []
    for i in range(args.num_parts):
        parts_mask = torch.zeros((num_nodes, ), dtype=bool)
        parts_mask_list.append(parts_mask)
    parts_file = open(args.parts, "r")
    for nid, line in enumerate(parts_file):
        part_id = int(line)
        parts_mask_list[part_id][nid] = True
    end = time.time()
    print("Reading parts file and generating part masks takes {:.3f} s".format(
        end - start))

    train_nids_list = []
    for i, part_mask in enumerate(parts_mask_list):
        local_train_nids = graph["train_idx"][part_mask[graph["train_idx"]]]
        train_nids_list.append(local_train_nids)
        print("Part {:4d}, nodes num = {:12d}, train nids num = {:12d}".format(
            i,
            torch.nonzero(part_mask).numel(), local_train_nids.numel()))

    if args.save_path != None:
        save_path = args.save_path
    else:
        save_path = args.root
    mask_save_fn = os.path.join(
        save_path, args.dataset + str(args.num_parts) + "parts_mask.pkl")
    train_save_fn = os.path.join(
        save_path, args.dataset + str(args.num_parts) + "parts_train_nids.pkl")
    torch.save(parts_mask_list, mask_save_fn)
    torch.save(train_nids_list, train_save_fn)
    print("Result saved to {} and {}".format(mask_save_fn, train_save_fn))


import numpy as np

def reg_loss_(y, y_hat, theta, lambda_):
    """Computes the regularized loss of a linear regression model from two non-empty numpy.array, without any for loop.
    Args:
    y: has to be an numpy.ndarray, a vector of shape m * 1.
    y_hat: has to be an numpy.ndarray, a vector of shape m * 1.
    theta: has to be a numpy.ndarray, a vector of shape n * 1.
    lambda_: has to be a float.
    Returns:
    The regularized loss as a float.
    None if y, y_hat, or theta are empty numpy.ndarray.
    None if y and y_hat do not share the same shapes.
    Raises:
    This function should not raise any Exception.
    """
    try:
        theta = np.reshape(theta, (len(theta), ))
        l2 = float(theta[1:].T.dot(theta[1:]))
        m, n = y.shape
        diff = y_hat - y
        return float((1 / (2 * m)) * (diff.T.dot(diff) + lambda_ * l2))
    except Exception as inst:
        print(inst)
        return None


import pprint

import glob
from mkdocs.structure.files import File
import logging
import os
from datetime import datetime
log = logging.getLogger(__name__)

def add_hidden_files(files, config):
  #print("------ from on_pre_build")
  #pp = pprint.PrettyPrinter(indent=4)
  hidden_files_globs = [
    ".ladoc/api_helpers/**",
    ".ladoc/assets/**"
  ]
  #pp.pprint(config)
  for hidden_files_glob in hidden_files_globs:
    absolute_hidden_files_glob = os.path.join(config['docs_dir'], hidden_files_glob)
    hidden_files = glob.glob(absolute_hidden_files_glob, recursive=True)
    #pp.pprint(hidden_files)
    for hidden_filename in hidden_files:
      #pp.pprint(os.path.exists(hidden_filename))
      #pp.pprint(hidden_filename)
      if not os.path.exists(hidden_filename):
        continue
      if os.path.isdir(hidden_filename):
        continue
      file = File(hidden_filename.replace(config['docs_dir']+"/", ''), config['docs_dir'], config['site_dir'], config['use_directory_urls'])
      #pp.pprint(file)
      files.append(file)
  #pp.pprint(files)
  return files

def set_build_time_env_var(*args, **kwargs):
  #print("------ from on_post_build")
  os.environ['LADOC_BUILD_DATE'] = datetime.now().strftime("%Y-%m-%d")
  #pp.pprint(os.environ['LADOC_BUILD_TIME'])


# _*_ coding:utf-8 _*_
# Lee 2022-05-26 10:21 self_lr
# Note:

import torch
from torch.optim import lr_scheduler, Adam
import matplotlib.pyplot as plt
import torch.nn as nn
import math

class net(nn.Module):
    def __init__(self):
        super(net, self).__init__()
        self.fc = nn.Linear(1, 10)

    def forward(self, x):
        return self.fc(x)

model = net()
lr0 = 6.73e-6
optimizer = Adam(model.parameters(), lr=lr0)

lr_list = [lr0]

lf = lambda x: (((1 + math.cos(x * math.pi / 100)) / 2) ** 1.0) * 0.8 + 0.2 if x < 100 else 0.1  # cosine

def nre_lr(x):
    if x < 45:
        return (((1 + math.cos(x * math.pi / 15)) / 2) ** 1.0) * 0.38 + 0.62
    elif 45 <= x < 50:
        return 0.62
    elif 50 <= x < 95:
        return (((1 + math.cos((x - 50) * math.pi / 15)) / 2) ** 1.0) * 1.99 + 2.60
    else:
        return 2.6

# scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=self_lr)
scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=nre_lr)

for epoch in range(100):
    optimizer.step()
    scheduler.step()
    lr = scheduler.get_lr()[0]
    lr_list.append(lr)
t = range(len(lr_list))
plt.plot(t, lr_list, color='r')
plt.show()


from tkinter import *
from quiz_brain import QuizBrain

FONT_BODY = ("Ariel", 20, "italic")
THEME_COLOR = "#375362"
WHITE = "#FFFFFF"
BLACK = "#000000"
GREEN = "#00FF00"
RED = "#FF0000"

class QuizInterface():
    def __init__(self, quiz_brain: QuizBrain):
        self.quiz_brain = quiz_brain
        self.window = Tk()
        self.window.title("Quizzle")
        self.window.config(padx=20, pady=20, bg=THEME_COLOR)
        self.score_label = self._create_score_text()
        self.quiz_canvas = self._create_quiz_card()
        self.true_button = self._create_true_button()
        self.false_button = self._create_false_button()

        self.get_next_question()
        self.window.mainloop()

    def _create_score_text(self):
        score_text = Label(text=f"Score:{self.quiz_brain.score}", bg=THEME_COLOR, fg=WHITE, highlightthickness=0, anchor="center")
        score_text.grid(row=0, column=1)

        return score_text

    def _create_quiz_card(self):
        canvas = Canvas(width=300, height=250, bg=WHITE, highlightthickness=0)
        self.quiz_text = canvas.create_text(
            150,
            125,
            text="Give your quiz text here",
            fill=BLACK,
            font=FONT_BODY,
            width=280
        )
        canvas.grid(row=1, column=0, columnspan=2, pady=50)

        return canvas

    def _create_true_button(self):
        self.true_image = PhotoImage(file="./images/true.png")
        button = Button(
            image=self.true_image,
            highlightbackground=THEME_COLOR,
            highlightthickness=0,
            command=self.handle_true
        )
        button.grid(row=2, column=0)

        return button

    def _create_false_button(self):
        self.false_image = PhotoImage(file="./images/false.png")
        image_button = Button(
            image=self.false_image,
            highlightbackground=THEME_COLOR,
            highlightthickness=0,
            command=self.handle_false
        )
        image_button.grid(row=2, column=1)

        return image_button

    def get_next_question(self):
        self.quiz_canvas.config(bg=WHITE)
        if self.quiz_brain.still_has_questions():
            self.score_label.config(text=f"Score: {self.quiz_brain.score}")
            question = self.quiz_brain.next_question()
            self.quiz_canvas.itemconfig(self.quiz_text, text=question)
        else:
            self.quiz_canvas.itemconfig(self.quiz_text, text="The End.")
            self.true_button.config(state="disabled")
            self.false_button.config(state="disabled")

    def handle_true(self):
        self.give_feedback(self.quiz_brain.check_answer(user_answer="True"))

    def handle_false(self):
        self.give_feedback(self.quiz_brain.check_answer(user_answer="False"))

    def give_feedback(self, is_right: bool):
        if is_right:
            self.quiz_canvas.config(bg=GREEN)
        else:
            self.quiz_canvas.config(bg=RED)

        self.window.after(1000, self.get_next_question)


plane_info = [243 , 345]

def skrech_number(w):
    flag = True
    for r in w:
        if r == user_guess:
            return flag == True
        else:
            flag == False
    return flag

def resalt(q):
    if q ==True:
        print("Ваш рейс найден")
    elif q ==False:
        print("Ваш рейс не найден")
    else:
        print("Ошибка ")

def user_guess():
    return int(input("Введите номер вашего рейса(только цыфры):"))

def main():
    user_guess()
    skrech_of_plane = skrech_number(plane_info)
    resalt(skrech_of_plane)
main()


from typing import List, Optional

from fastapi import APIRouter

from app.apis import topic_views
from app.apis.util_routes import search
from app.utils.database import es_client, mongo_epigraphdb_web
from app.utils.logging import logger

router = APIRouter()

ES_EXCLUDE_PREFIX = ["embeddings-", ".apm-", ".kibana_"]

@router.get("/utils/cache/list", response_model=List[str])
def get_utils_cache_list() -> List[str]:
    """Returns the currently available cache collections."""
    colls = mongo_epigraphdb_web.list_collection_names()
    res = sorted(colls)
    return res

@router.get("/utils/cache/drop", response_model=bool)
def get_utils_cache_drop(
    collection_name: Optional[str] = None, all: bool = False
) -> bool:
    """Drop a mongodb collection, or every collection."""
    all_coll_names = mongo_epigraphdb_web.list_collection_names()
    if collection_name is not None and not all:
        if collection_name in all_coll_names:
            mongo_epigraphdb_web[collection_name].drop()
        new_all_coll_names = mongo_epigraphdb_web.list_collection_names()
        res = collection_name not in new_all_coll_names
        return res
    else:
        for collection_name in all_coll_names:
            mongo_epigraphdb_web[collection_name].drop()
        return True

@router.get("/utils/es/list", response_model=List[str])
def list_es_indices() -> List[str]:
    """Returns the currently available ES indices."""
    es_indices: List[str] = list(es_client.indices.get_alias("*").keys())
    return es_indices

@router.get("/utils/es/drop", response_model=bool)
def get_utils_es_drop(
    index_name: Optional[str] = None,
    all: bool = False,
    with_exclude: bool = False,
) -> bool:
    """Purge elasticsearch indices.

    - When specified with an `index_name`, drop it if it is not in exclude it
    - When `all`, drop all indices except those in exclude list
    - When `all` and `with_exclude`, drop all indices including those
      in exclude it
    """
    es_indices = list_es_indices()
    if index_name is not None:
        if index_name not in es_indices or not _es_index_not_exclude(
            index_name
        ):
            return False
        else:
            es_client.indices.delete(index=index_name, ignore=[400, 404])
            res = index_name not in list_es_indices()
            return res
    elif all:
        if not with_exclude:
            es_indices_subset = [
                _ for _ in es_indices if _es_index_not_exclude(_)
            ]
            for index in es_indices_subset:
                es_client.indices.delete(index=index, ignore=[400, 404])
            es_indices_subset = [
                _ for _ in list_es_indices() if _es_index_not_exclude(_)
            ]
            return len(es_indices_subset) == 0
        else:
            for index in es_indices:
                es_client.indices.delete(index=index, ignore=[400, 404])
            return len(list_es_indices()) == 0
    else:
        return False

@router.get("/utils/es/index", response_model=bool)
def get_utils_es_index(overwrite: bool = False) -> bool:
    logger.info("Index search indices.")
    assert search.get_index_all(overwrite=overwrite)
    logger.info("Index topic autocomplete indices.")
    index_funcs = [
        topic_views.mr.get_mr_ac_index,
        topic_views.obs_cor.get_obs_cor_ac_index,
        topic_views.genetic_cor.get_genetic_cor_ac_index,
        topic_views.confounder.get_confounder_ac_index,
        topic_views.drugs_risk_factors.get_drugs_risk_factors_ac_index,
        topic_views.pathway.get_pathway_ac_index,
        topic_views.prs.get_prs_ac_index,
        topic_views.xqtl.get_xqtl_ac_index,
        topic_views.literature_trait.get_literature_trait_ac_index,
        topic_views.ontology_trait_disease.get_ontology_trait_disease_ac_index,
    ]
    for func in index_funcs:
        logger.info(f"Indexing {func}")
        # NOTE: should refactor logics of overwrite
        # since some of the ac indices share the same
        assert func(overwrite=False)
    return True

def _es_index_not_exclude(index: str) -> bool:
    res = sum([not index.startswith(_) for _ in ES_EXCLUDE_PREFIX]) == len(
        ES_EXCLUDE_PREFIX
    )
    return res


from django.urls import path

from . import views

app_name = "auctions"

urlpatterns = [
    path("", views.index, name="index"),
    path("login", views.login_view, name="login"),
    path("logout", views.logout_view, name="logout"),
    path("register", views.register, name="register"),
    path("add_listing", views.add_listing, name="add_listing"),
    path("listing/<int:item_id>/<int:user_id>/<str:message>", views.render_item, name="render_item"),
    path("listing/<int:item_id>", views.render_item_not_auth, name="render_item_not_auth"),
    path("listing/<int:item_id>/add_watchlist/<str:user_id>", views.add_watchlist, name="add_watchlist"),
    path("listing/<int:item_id>/remove_watchlist/<str:user_id>", views.remove_watchlist, name="remove_watchlist"),
    path("listing/<int:item_id>/place_bid/<str:user_id>", views.place_bid, name="place_bid"),
    path("listing/<int:item_id>/close_auction/<str:user_id>", views.close_auction, name="close_auction"),
    path("listing/<int:item_id>/add_comment/<str:user_id>)", views.add_comment, name="add_comment"),
    path("watchlist/<str:user_id>", views.render_watchlist, name="render_watchlist"),
    path("categories", views.show_categories, name="show_categories"),
    path("categories/sort_by_category/<str:category>", views.sort_by_category, name="sort_by_category")
]


from web3 import Web3
import os.path
import json
import sys
import glob
import sys
import json
import rlp
from subprocess import Popen, PIPE, STDOUT
from values import MyGlobals
from blockchain import *

def compile_contract(filename):

    print('\033[1m[ ] Compiling Solidity contract from the file %s ... \033[0m' % filename, end='')

    source_file = filename
    if (not os.path.isfile(source_file) ):
        print('\n\033[91m[-] Contract file %s does NOT exist\033[0m' % source_file )
        return

    with open(filename, 'r') as myfile:
        code=myfile.read()

    p=Popen(['solc','--bin','--abi','-o','out',source_file,'--overwrite'], stdout=PIPE, stdin=PIPE, stderr=STDOUT)
    solo = ''
    while p.poll() is None:
        l = p.stdout.readline()
        solo += bytes.decode(l)
    if 'Error' in solo:
        print(solo)
        print('\n\033[91m[-] Cannot compile the contract \033[0m')
        exit()
    p.wait()

    print('\033[92m Done \033[0m')

def get_function_hashes(contract):

    data = json.load(open('out/'+contract+'.abi'))
    fhashes = {}
    for d in data:
        if 'type' in d and d['type'] == 'function':
            hs = ''
            if 'name' in d: hs = d['name']
            fr = True
            hs += '('
            if 'inputs' in d:
                for i in d['inputs']:
                    if not fr: hs +=','
                    hs += i['type']
                    fr = False
            hs += ')'
            hash_op = Web3.keccak(text=hs).hex()

            fhashes[hash_op[2:10]] = hs

    return fhashes

def deploy_contract(filename, etherbase, rawcode = False):

    if rawcode:
        with open(filename, 'r') as myfile: byt=myfile.read()
        filename_write_address = os.path.basename(filename)+'.address'

    else:
        filename_abi = os.path.basename(filename)+'.abi'
        filename_bin = os.path.basename(filename)+'.bin'
        filename_write_address = os.path.basename(filename)+'.address'
        with open('./out/'+filename_abi, 'r') as myfile: abi=myfile.read()
        with open('./out/'+filename_bin, 'r') as myfile: byt=myfile.read()
        if( len(abi) == 0 or len(byt) == 0 ):
            print('\033[91m[-] Some of the files is missing or empty: |%s|=%d  |%s|=%d' % (filename_abi, len(abi), filename_bin, len(byt) ) )
            print('The contracts have NOT been deployed\033[0m')
            return

        abi = json.loads(abi)

    print('\033[1m[ ] Deploying contract \033[0m', end='')
    MyGlobals.web3.geth.personal.unlock_account(etherbase, '1', 15000)

    try:
        tx = {'from':etherbase, 'data': ('0x' if byt[0:2]!='0x' else '') +byt , 'gas':6000000}
        transaction_creation_hash = MyGlobals.web3.eth.sendTransaction( tx ).hex()
        receipt = MyGlobals.web3.eth.wait_for_transaction_receipt(transaction_creation_hash,timeout=120,poll_latency=0.1)
        contract_address = receipt['contractAddress']
    except Exception as e:
        print ("Exception: "+str(e))
        return None

    print('\033[92m confirmed at address: %s \033[0m' % contract_address)
    fullcode = MyGlobals.web3.eth.getCode(contract_address)
    print('\033[1m[ ] Contract code length on the blockchain : \033[0m \033[92m  %d  : %s... \033[0m' % (len(fullcode), fullcode.hex()[:30] ) )
    with open('./out/'+filename_write_address, 'w') as f:
        f.write(contract_address)
        f.close()
    print('\033[1m[ ] Contract address saved in file: \033[0m\033[92m%s \033[0m' % ('./out/'+filename_write_address))

    return contract_address

def rlp_encode(input):
    if isinstance(input,str):
        if len(input) == 1 and ord(input) < 0x80: return input
        else: return encode_length(len(input), 0x80) + input
    elif isinstance(input,list):
        output = ''
        for item in input: output += rlp_encode(item)
        return encode_length(len(output), 0xc0) + output

def encode_length(L,offset):
    if L < 56:
         return chr(L + offset)
    elif L < 256**8:
         BL = to_binary(L)
         return chr(len(BL) + offset + 55) + BL
    else:
         raise Exception("input too long")

def to_binary(x):
    if x == 0:
        return ''
    else:
        return to_binary(int(x / 256)) + chr(x % 256)

def normalize_address(x, allow_blank=False):
    if allow_blank and x == '':
        return ''
    if len(x) in (42, 50) and x[:2] == '0x':
        x = x[2:]
    if len(x) in (40, 48):
        x = bytes.fromhex(x)
    if len(x) == 24:
        assert len(x) == 24 and Web3.keccak(x[:20])[:4] == x[-4:]
        x = x[:20]
    if len(x) != 20:
        raise Exception("Invalid address format: %r" % x)
    return x

def predict_contract_address(accountAddress):

    nonce = int(MyGlobals.web3.eth.getTransactionCount(accountAddress))
    adr = Web3.keccak(rlp.encode([normalize_address(accountAddress), nonce]))[-20:]
    return Web3.toChecksumAddress(adr.hex())


import boto3
import base64
import os
import json
from botocore.exceptions import ClientError

ddb = boto3.resource('dynamodb')

def getAddressList(tableName):
  try:
    table = ddb.Table(tableName)
    response = table.scan()
    return response['Items']
  except ClientError as err:
    print(err.response['Error'])
    raise

def lookupRecord(addressList, stationId):

  for rec in addressList:
    if stationId == rec['StationId']:
      return rec['Address']

def lambda_handler(event, context):
    output = []
    dynamoDBTableName = os.environ['dynamodb_table_name']
    addressList = getAddressList(dynamoDBTableName)

    for record in event['records']:
        kinesisRecord = json.loads(base64.b64decode(record['data']))
        stationAddRecord = lookupRecord(addressList,kinesisRecord['stationId'])
        kinesisRecord['stationAddress'] = stationAddRecord

        output_record = {
            'recordId': record['recordId'],
            'result': 'Ok',
            'data': base64.b64encode((json.dumps(kinesisRecord) +"\n").encode("utf-8"))
        }
        output.append(output_record)

    print('Successfully processed {} records.'.format(len(event['records'])))

    return {'records': output}


import os,sys
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

import skimage
import numpy as np
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import edge_detection

from skimage import io
from skimage import feature
from scipy.ndimage.morphology import binary_fill_holes
from skimage.measure import label
from skimage.measure import regionprops
from skimage.exposure import equalize_adapthist
from skimage.morphology import remove_small_objects
from scipy.signal import find_peaks
from irtemp import centikelvin_to_celsius
from edge_detection import inflection_point

##########################################################################################################################################################################
##########################################################################################################################################################################
                                                #######Sample detection using peaks#########
##########################################################################################################################################################################
##########################################################################################################################################################################

# Image equalization
def image_eq(n_frames,frames):
    '''Function to obtained an equalized image using all the frames
       in the video.
       Args:
       n_frames(int): The number of frames in the video.
       frames(List): List of arrays of frames in the video.
       Returns:
       img_eq: Equalized image
    '''
    for II in range(n_frames):
        frame = frames[II]
        img_eq = (frame - np.amin(frame))/(np.amax(frame)-np.amin(frame))
        if II == 0:
            img_ave = img_eq
        else:
            img_ave = img_ave + img_eq
    img_average = img_ave/n_frames
    img_eq = (img_ave - np.amin(img_ave))/(np.amax(img_ave)-np.amin(img_ave))
    return img_eq

# Function to obtain sum of pixels over all the rows and columns
def pixel_sum(frame):
    ''' Funtion to determine sum of pixels over all the rows and columns
        to obtain plots with peaks at the sample position in the array.
        Args:
        frame(array):Equalized image
        Returns:
        column_sum: Sum of pixels over all the columns
        row_sum: Sum of pixels over all the rows
        Also returns plots of column sum and row sum.
    '''
    rows = frame.shape[0]
    columns = frame.shape[1]
    column_sum = []
    for i in range(0,columns):
        column_sum.append(sum(frame[:,i]))
    row_sum = []
    for j in range(0,rows):
        row_sum.append(sum(frame[j,:]))
    column_sum = [x * -1 for x in column_sum]
    row_sum = [x * -1 for x in row_sum]
    plt.plot(range(len(column_sum)),column_sum)
    plt.xlabel('Column index')
    plt.ylabel('Sum of pixel values over columns')
    plt.title('Sum of pixel values over columns against column index')
    plt.show()
    plt.plot(range(len(row_sum)),row_sum)
    plt.xlabel('Row index')
    plt.ylabel('Sum of pixel values over rows')
    plt.title('Sum of pixel values over rows against row index')
    plt.show()
    return column_sum,row_sum

# To determine the peak values in the row and column sum and thus sample
# location.
def peak_values(column_sum,row_sum,n_columns,n_rows,image):
    ''' Function to find peaks from the column_sum and row_sum arrays
        and return a dataframe with sample locations.
        Args:
        column_sum: Sum of pixel values over all the columns in the
        image array.
        row_sum: Sum of pixel values over all the rows in the
        image array.
        n_columns: Number of columns of samples in the image
        n_rows: Number of rows of samples in the image.
        image: Image to be processed
        Returns:
        sample_location: A dataframe containing sample and plate locations and a plot with locations
        superimposed on the image to be processed.

    '''
    column_peaks = find_peaks(column_sum,distance=10)
    column_peaks = column_peaks[0]
    row_peaks = find_peaks(row_sum,distance=10)
    row_peaks = row_peaks[0]
    row = []
    column = []
    plate_location = []
    i = 0
    j = 0
    for i in range(0,n_columns):
        for j in range(0,n_rows):
            row.append(row_peaks[j])
            column.append(column_peaks[i])
            if j == 0:
                plate_location.append(int((row[j]-0)/2))
            else:
                plate_location.append(int((row[j] + row[j-1])/2))
            j = j + 1
        i = i + 1

    sample_location = pd.DataFrame(list(zip(row, column, plate_location)),columns =['Row', 'Column','plate_location'])
    plt.imshow(image)
    plt.scatter(sample_location['Column'],sample_location['Row'],s=4)
    plt.scatter(sample_location['Column'],sample_location['plate_location'],s=4)
    plt.title('Sample and plate location at which the temperature profile is monitored')
    plt.show()
    return sample_location

# To determine the samle and plate temperature using peak locations.
def pixel_intensity(sample_location, frames, x_name, y_name, plate_name):
    ''' Function to find pixel intensity at all sample locations
        and plate locations in each frame.
        Args:
        sample_location(dataframe): A dataframe containing sample and plate locations.
        frames(list or array): An array of arrays containing all the frames of a video.
        x_name(string): Name of the column in sample_location containing the row values of the samples.
        y_name(string): Name of the column in sample_location containing the column values of the samples.
        plate_name(string): Name of the column in sample_location containing the column values of the
        plate location.
    '''
    temp = []
    plate_temp = []
    x = sample_location[x_name]
    y = sample_location[y_name]
    p = sample_location[plate_name]
    for i in range(len(sample_location)):
        temp_well = []
        plate_well_temp = []
        for frame in frames:
            temp_well.append(centikelvin_to_celsius(frame[x[i]][y[i]]))
            plate_well_temp.append(centikelvin_to_celsius(frame[p[i]][y[i]]))
        temp.append(temp_well)
        plate_temp.append(plate_well_temp)
    return temp,plate_temp

##### Wrapping Function ######
def pixel_temp(frames,n_frames,n_columns,n_rows):
    ''' Function to determine the temperature of the samples and plate locations by analysing
    pixel values and finding peaks.
    Args:
    frames: The frames of a video to be analysed.
    n_frames: Number of frames in the video
    n_columns: Number of columns of samples in the image
    n_rows: Number of rows of samples in the image.
    Returns:
    m_df(Dataframe): A dataframe containing row and column coordinates of each sample
    and its respective inflection point obtained.
    '''
    flip_frames = edge_detection.flip_frame(frames)
    #Function to obtained an equalized image using all the frames
    #in the video.
    img_eq = image_eq(n_frames,flip_frames)
    #Funtion to determine sum of pixels over all the rows and columns
    #to obtain plots with peaks at the sample position in the array.
    column_sum,row_sum = pixel_sum(img_eq)
    # Function to find peaks from the column_sum and row_sum arrays
    # and return a dataframe with sample locations.
    sample_location = peak_values(column_sum,row_sum,n_columns,n_rows,img_eq)
    # Function to find pixel intensity at all sample locations
    # and plate locations in each frame.
    temp,plate_temp = pixel_intensity(sample_location, frames, x_name = 'Row', y_name = 'Column', plate_name = 'plate_location')
    # Function to obtain the inflection point(melting point) from the temperature profile.
    inf_temp, s_peaks, p_peaks = inflection_point(temp,plate_temp)
    # Dataframe with sample location (row and column coordinates) and respective inflection point.
    m_df = pd.DataFrame({'Row':sample_location.Row,'Column':sample_location.Column,'Melting point':inf_temp})
    return m_df


#!/usr/bin/python
# -*- coding: UTF-8 -*-

'''Logical operations on sets - symmetrical difference'''

import unittest

def roznica_zbiorow(zbior_a, zbior_b):
    return zbior_a - zbior_b

def rozbieznosc_zbiorow(zbior_a, zbior_b, verbose=1):
    mam_rozbieznosc = zbior_a != zbior_b
    if mam_rozbieznosc:
        if verbose:
            if 1:
                tmp_format = 'roznica_zbiorow(zbior_a, zbior_b)'
                print('Eval: %s %s' % (tmp_format, eval(tmp_format)))
            if 1:
                tmp_format = 'roznica_zbiorow(zbior_b, zbior_a)'
                print('Eval: %s %s' % (tmp_format, eval(tmp_format)))
    return mam_rozbieznosc

class TestOperacjiNaZbiorach(unittest.TestCase):
    def test_operacji_na_zbiorach(self):
        '''
        TestOperacjiNaZbiorach:
        '''

        self.assertEqual(roznica_zbiorow(frozenset('abc'), frozenset('ab')), frozenset('c'))
        self.assertEqual(roznica_zbiorow(frozenset('ab'), frozenset('abc')), frozenset(''))
        self.assertEqual(rozbieznosc_zbiorow(frozenset('ab'), frozenset('bc'), verbose=0), 1)
        self.assertEqual(rozbieznosc_zbiorow(frozenset('abc'), frozenset('cba')), 0)


# Databricks notebook source
# MAGIC %md
# MAGIC ##### Welcome  to Azurelib Academy By Deepak Goyal
# MAGIC ##### You can master the Databricks here: www.adeus.azurelib.com
# MAGIC ##### Author of this blog: Arud Seka Berne S in

# COMMAND ----------

# MAGIC %md
# MAGIC #### Create Pyspark Rdd in Azure Databricks

# COMMAND ----------

# MAGIC %md
# MAGIC ###### Gentle reminder:
# MAGIC In Databricks,
# MAGIC   - sparkSession made available as spark
# MAGIC   - sparkContext made available as sc
# MAGIC
# MAGIC In case, you want to create it manually, use the below code.

# COMMAND ----------

from pyspark.sql.session import SparkSession

spark = SparkSession.builder \
    .master("local[*]") \
    .appName("azurelib.com") \
    .getOrCreate()

sc = spark.sparkContext

# COMMAND ----------

# MAGIC %md
# MAGIC ##### 1) Creating RDD using paralelize()

# COMMAND ----------

rdd1 = sc.parallelize([1,2,3,4,5])
print(type(rdd1))
rdd1.collect()

# COMMAND ----------

# MAGIC %md
# MAGIC ##### 2) Creating RDD by reading files

# COMMAND ----------

# replace the file_path with the source file location which you have downloaded.

rdd2 = sc.textFile(file_path)
print(type(rdd2))
rdd2.collect()

# COMMAND ----------

# MAGIC %md
# MAGIC ##### 3) Creating empty RDD

# COMMAND ----------

# Method 1:
rdd3 = sc.emptyRDD()
rdd3.collect()

# Method 2:
rdd4 = sc.parallelize([])
rdd4.collect()

94. Create Pyspark Rdd/Create Pyspark Rdd - Notebook.py


#!/usr/bin/env python3
from ebook.markov import MarkovGenerator, twitter_tokenize
import random
import re

# get rid of annoying trailing ands.
andRe = re.compile(r'\s(and|the).$', re.I)

if __name__ == "__main__":
  with open('data/cleanCorpus.txt', mode='r') as fin:
    with open('data/markov-gamedev.txt', mode='a') as fout:
      tweetCorpus = fin.read()
      mc = MarkovGenerator(tweetCorpus, 90, tokenize_fun=twitter_tokenize)
      lengths = []
      for x in range(50):
        mc.length = min(25 + round(abs(random.gauss(0, 75))), 130)
        print(mc.length)
        lengths.append(mc.length)
        tweet = mc.generate_words()

        # exclude @ replies!
        if tweet.find('@') != -1:
          continue

        tweet = andRe.sub('.', tweet)

        fout.write(tweet)
        fout.write('\n')

      print('Average')
      print(sum(lengths) / len(lengths))


'''
    El Juego del Ahorcado
'''
# * LIBRERIAS
import os
import random

# * ACSII ART
# constante
IMAGENES = ['''

    +---+
    |   |
        |
        |
        |
        |
        ==========''','''

    +---+
    |   |
    O   |
        |
        |
        |
        ==========''','''

    +---+
    |   |
    O   |
    |   |
        |
        |
        ==========''','''

    +---+
    |   |
    O   |
   /|   |
        |
        |
        ==========''','''

    +---+
    |   |
    O   |
   /|\  |
        |
        |
        ==========''','''

    +---+
    |   |
    O   |
   /|\  |
    |   |
        |
        ==========''','''

    +---+
    |   |
    O   |
   /|\  |
    |   |
   /    |
        ==========''','''

    +---+
    |   |
    O   |
   /|\  |
    |   |
   / \  |
        ==========''','''

''']

# * JUEGO
# palabras para el juego
PALABRAS = [
    'sillas',
    'ordenador',
    'cursos',
    'desarrollador',
    'papelera'
]

# funciones
# elegir una palabra aleatoria
def palabraAleatoria():
    lista = random.randint(0, len(PALABRAS) - 1)
    return PALABRAS[lista]

# imprimir el escenario para poder jugar
def tablero(esconderPalabra, intentos):
    print(f'{IMAGENES[intentos]}\n')
    print(esconderPalabra)
    print('----*----*----*----*----*----*----*----*')

# arrancar
def run():
    palabra = palabraAleatoria()
    esconderPalabra = ['-'] * len(palabra)
    intentos = 0

    # bucle infinito
    while True:
        os.system('clear')
        tablero(esconderPalabra, intentos)
        print(f'\n\tINTENTOS: {intentos}')
        print(palabra)
        pedirLetras = input('Adivina una letra: ')

# * LOGICA del Juego
        indexarLetra = []
        for i in range(len(palabra)):
            if palabra[i] == pedirLetras:
                indexarLetra.append(i)

        if len(indexarLetra) == 0:
            intentos += 1

            # * Perder
            if intentos == 7:
                tablero(esconderPalabra, intentos)
                print('')
                print(f'¡Perdiste! La palabra correcta es {palabra}')
                break

        # * Acertar
        else:
            for i in indexarLetra:
                esconderPalabra[i] = pedirLetras

        # Vaciar al final
            indexarLetra = []

        # Ganar
        try:
            esconderPalabra.index('-')
        except ValueError:
            print('')
            print(f'¡Felicidades! La palabra ganadora es: {palabra}')
            # break

# * INICIO
if __name__ == "__main__":
    run()


import numpy as np
from scipy.optimize import minimize
from scipy.io import loadmat
from numpy.linalg import det, inv
from math import sqrt, pi
import scipy.io
import matplotlib.pyplot as plt
import pickle
import sys

def getUniqueY(y):
    uniqueY = np.unique(y) #returns vector of unique results

    return uniqueY

def findSigma(input, mewVector):
    numRows = input.shape[0] #read the var name its pretty self explanatory
    sigma = np.zeros((input.shape[1],input.shape[1])) #make sigma be Column number x Column number

    for i in range(0,numRows): # for every row in the input
       value1 = input[i] - mewVector
       value1 = value1.reshape(value1.size,1) #getting first value and reshapeing to (2,1)

       value2 = input[i] - mewVector
       value2 = value2.reshape(value2.size,1)#getting second value and reshaping to (2,1)

       sigma += np.matmul(value1, np.transpose(value2))#matmulling the values

    sigma = np.divide(sigma,numRows)  #some good ol' fashioned matrix divison?
    return sigma

def  Gaussian(mean,sigma,X):
    #Inputs
    #mean is vector of column means
    #sigma is a matrix
    #x is data
    D = X.shape[0]
    value1 = X-mean
    value1 = value1.reshape(value1.size,1)

    first = 1/(pow((2*pi),(D/2))*pow(float(det(sigma)),(1/2)))
    ePart = -1*np.matmul(np.matmul(np.transpose(value1),inv(sigma)),value1)/2

    return first*np.exp(ePart)

def findClasses(X,y):#splits up primary vector based on result
   #
   #Returns a dirctionary of str(y) as the key value is a matrix of the correct rows from X
    #first find number of unique result types in Y
    uniqueY = getUniqueY(y) #returns vector of unique results

    #Now, get all indices of each result type
    i = 0
    resultIndices = {}
    #print(uniqueY)
    for i in uniqueY:
        resultIndices[str(int(i))] = []
    i = 0

    while(i < np.shape(y)[0]):#iterate through entire result vector
        #store i-value in matching point in resultIndicies
        value = int(y[i])
        index = i

        resultIndices[str(value)]+=[index]
        #print(type(value))
        i=i+1
    for i in uniqueY:
       resultIndices[str(int(i))] = np.take(X,resultIndices[str(int(i))],axis=0)

    return resultIndices,uniqueY#returning the split X values, and unique Y values

def findMew(input):#finds the mean of each column in a matrix you pass to it
    numCol = np.shape(input)[1] #number of colums
    i = 0 #counter variable
    Mew = [] #initializing random Mew vector
    input= np.transpose(input)
    while(i < numCol): #for each column
        Mew = np.append(Mew,np.mean(input[i]) )#the output of Mew[i] is the mean of column i
        i = i+1 #increment i
    #print("Mew Vector = ", Mew)
    return Mew

def ldaLearn(X,y):
    mewVector = findMew(X) #get the mew vector for X
    sigma = findSigma(X, mewVector) #find the sigma matrix for X
    splitClasses,uniqueY = findClasses(X, y) #use findClasses to split up matrix
    numColumns = X.shape[1]

    means = np.ones((uniqueY.shape[0],numColumns))
    covmat = sigma

    for i in uniqueY:
       means[int(i-1)] = findMew(splitClasses[str(int(i))])

    # Inputs
    # X - a N x d matrix with each row corresponding to a training example
    # y - a N x 1 column vector indicating the labels for each training example
    #
    # Outputs
    # means - A k x d matrix containing learnt means for each of the k classes
    # covmat - A single d x d learnt covariance matrix

    return means,covmat

def qdaLearn(X,y):
    numColumns = X.shape[1]
    splitClasses,uniqueY = findClasses(X, y) #use findClasses to split up matrix
    means = np.ones((uniqueY.shape[0],numColumns))

    for i in uniqueY:
       means[int(i-1)] = findMew(splitClasses[str(int(i))])

    covmats =[]
    for i in uniqueY:
        covmats.append(findSigma(splitClasses[str(int(i))],means[int(i-1)]))

    # Inputs
    # X - a N x d matrix with each row corresponding to a training example
    # y - a N x 1 column vector indicating the labels for each training example
    #
    # Outputs
    # means - A k x d matrix containing learnt means for each of the k classes
    # covmats - A list of k d x d learnt covariance matrices for each of the k classes

    # IMPLEMENT THIS METHOD
    return means,covmats

def ldaTest(means,covmat,Xtest,ytest):

    uniqueY = getUniqueY(ytest) #fetch uniquey
    ypred = []

    for j in range(0,Xtest.shape[0]):
        minimum = sys.maxsize#minimum value

        for i in uniqueY:
            val2 = (Xtest[j] - means[int(i-1)])
            val1 = np.transpose(val2)
            val3 = np.matmul(val1,inv(covmat))
            val4 = np.matmul(val3,val2)

            if(val4 < minimum):
                minimum = val4
                predClass = int(i)

        ypred.append(predClass)
    numCorrect =0
    #print(ypred)
    ypred = np.array(ypred)
    ypred = ypred.reshape(len(ypred),1)

    for i in range(0,len(ypred)):
        if(ypred[i][0]==int(ytest[i][0])):
                numCorrect+=1
    acc = numCorrect/ytest.shape[0] *100
    # Inputs
    # means, covmat - parameters of the LDA model
    # Xtest - a N x d matrix with each row corresponding to a test example
    # ytest - a N x 1 column vector indicating the labels for each test example
    # Outputs
    # acc - A scalar accuracy value
    # ypred - N x 1 column vector indicating the predicted labels

    # IMPLEMENT THIS METHOD
    return acc,ypred

def qdaTest(means,covmats,Xtest,ytest):
    acc = None
    ypred = []

    counts = np.unique(ytest, return_counts=True)[1] #number of times each unique item appears

    X,uniqueY = findClasses(Xtest,ytest)
    numY = uniqueY.shape[0]

    for j in range(0,Xtest.shape[0]):
        maximum = -sys.maxsize-1
        for i in uniqueY:
            val1 = Gaussian(means[int(i-1)],covmats[int(i-1)],Xtest[j])
            if(val1 > maximum):
                maximum = val1
                predClass = int(i)
        ypred.append(predClass)
    numCorrect =0
    ypred = np.array(ypred)
    ypred = ypred.reshape(len(ypred),1)

    for i in range(0,len(ypred)):
        if(ypred[i][0]==int(ytest[i][0])):
                numCorrect+=1
    acc = numCorrect/ytest.shape[0] *100
    # Inputs
    # means, covmats - parameters of the QDA model
    # Xtest - a N x d matrix with each row corresponding to a test example
    # ytest - a N x 1 column vector indicating the labels for each test example
    # Outputs
    # acc - A scalar accuracy value
    # ypred - N x 1 column vector indicating the predicted labels

    # IMPLEMENT THIS METHOD
    return acc,ypred
#PART 1 ENDS HERE-------------------------------------------------------
def learnOLERegression(X,y):
    # Inputs:
    # X = N x d
    # y = N x 1
    # Output:
    # w = d x 1

    val1 = np.matmul(np.transpose(X),X)
    val1 = np.matmul(inv(val1),np.transpose(X))
    w = np.matmul(val1,y)

    return w

def learnRidgeRegression(X,y,lambd):
    # Inputs:
    # X = N x d
    # y = N x 1
    # lambd = ridge parameter (scalar)
    # Output:
    # w = d x 1

    # Ridge Estimate of W PartCSlides1 Page 20
    val1 = np.matmul(np.transpose(X),X)
    val1 = val1 + (lambd*np.identity(X.shape[1]))
    val1 = np.matmul(inv(val1),np.transpose(X))
    w = np.matmul(val1,y)

	# slides partc1, #20
    return w

def testOLERegression(w,Xtest,ytest):
    # Inputs:
    # w = d x 1
    # Xtest = N x d
    # ytest = X x 1
    # Output:
    # mse

    mse = 0
    numY = ytest.shape[0]

    for i in range(0,numY):
        wtx = np.dot(np.transpose(w),Xtest[i])

        val2 = (ytest[i][0] - wtx)[0]

        mse+=pow(val2,2)
    mse= mse/numY
    return mse

def regressionObjVal(w, X, y, lambd):

    # compute squared error (scalar) and gradient of squared error with respect
    # to w (vector) for the given data X and y and the regularization parameter
    # lambda

    # IMPLEMENT THIS METHOD

    squarer = lambda wj: wj ** 2
    square_func = np.vectorize(squarer)
    L2 = lambd * (np.sum(square_func(w))) / 2
    error = 0
    for i in range(0,X.shape[0]):
        error += np.square((y[i] - np.matmul(np.transpose(w), X[i]))) + L2

    error /= 2

    error_grad = np.zeros(w.shape)

    for j in range(0,len(w)):
        run_sum = 0
        for i in range(0,X.shape[0]):
            val = np.matmul(np.transpose(w),X[i])
            val = val - y[i]
            val = val *X[i,j]
            run_sum+= (lambd*w[j])
        error_grad[j]+=run_sum

    return error, error_grad

def mapNonLinear(x,p):
    # Inputs:
    # x - a single column vector (N x 1)
    # p - integer (>= 0)
    # Outputs:
    # Xp - (N x (p+1))

    N = x.shape[0]
    Xp = np.power(x, 0)
    Xp=Xp.reshape(N,1)

    for i in range(1,p+1):
        newVal = np.power(x,i)
        newVal = newVal.reshape(newVal.shape[0],1)
        Xp = np.hstack((Xp,newVal))

    return Xp

# Main script
# Problem 1
#This was added so it would run on my computer
"""
inp = open('sample.pickle', 'rb')
str_inp = inp.read().decode()
modified_file = str_inp.replace('\r\n', '\n')
inp.close()

out = open('sample.pickle', 'wb')
out.write(modified_file.encode())
out.close()

"""

# load the sample data

if sys.version_info.major == 2:
    X,y,Xtest,ytest = pickle.load(open('sample.pickle','rb'))
else:
    X,y,Xtest,ytest = pickle.load(open('sample.pickle','rb'),encoding = 'latin1')

# LDA
means,covmat = ldaLearn(X,y)
ldaacc,ldares = ldaTest(means,covmat,Xtest,ytest)
print('LDA Accuracy = '+str(ldaacc))
# QDA
means,covmats = qdaLearn(X,y)
qdaacc,qdares = qdaTest(means,covmats,Xtest,ytest)
print('QDA Accuracy = '+str(qdaacc))

# plotting boundaries
x1 = np.linspace(-5,20,100)
x2 = np.linspace(-5,20,100)
xx1,xx2 = np.meshgrid(x1,x2)
xx = np.zeros((x1.shape[0]*x2.shape[0],2))
xx[:,0] = xx1.ravel()
xx[:,1] = xx2.ravel()

fig = plt.figure(figsize=[12,6])
plt.subplot(1, 2, 1)

zacc,zldares = ldaTest(means,covmat,xx,np.zeros((xx.shape[0],1)))

plt.contourf(x1,x2,zldares.reshape((x1.shape[0],x2.shape[0])),alpha=0.3)
plt.scatter(Xtest[:,0],Xtest[:,1],c=ytest.flatten())
#plt.scatter(Xtest[:,0],Xtest[:,1],c=ytest)
plt.title('LDA')

plt.subplot(1, 2, 2)

zacc,zqdares = qdaTest(means,covmats,xx,np.zeros((xx.shape[0],1)))
plt.contourf(x1,x2,zqdares.reshape((x1.shape[0],x2.shape[0])),alpha=0.3)
#plt.scatter(Xtest[:,0],Xtest[:,1],c=ytest)
plt.scatter(Xtest[:,0],Xtest[:,1],c=ytest.flatten())
plt.title('QDA')

plt.show()

#Part 1 Ends Here
"""

#This part was added so it would run on my computer
inp = open('diabetes.pickle', 'rb')
str_inp = inp.read().decode()
modified_file = str_inp.replace('\r\n', '\n')
inp.close()

out = open('diabetes.pickle', 'wb')
out.write(modified_file.encode())
out.close()
"""
# Problem 2
if sys.version_info.major == 2:
    X,y,Xtest,ytest = pickle.load(open('diabetes.pickle','rb'))
else:
    X,y,Xtest,ytest = pickle.load(open('diabetes.pickle','rb'),encoding = 'latin1')

# add intercept

X_i = np.concatenate((np.ones((X.shape[0],1)), X), axis=1)
Xtest_i = np.concatenate((np.ones((Xtest.shape[0],1)), Xtest), axis=1)

w = learnOLERegression(X,y)
mle = testOLERegression(w,Xtest,ytest)
mle_TD = testOLERegression(w,X,y)
w_i = learnOLERegression(X_i,y)
mle_i = testOLERegression(w_i,Xtest_i,ytest)
mle_i_TD = testOLERegression(w_i,X_i,y)
print("TRAIN DATA")

print('MSE without intercept '+str(mle_TD))
print('MSE with intercept '+str(mle_i_TD))
print("TEST DATA")
print('MSE without intercept '+str(mle))
print('MSE with intercept '+str(mle_i))

# Problem 3
k = 101
lambdas = np.linspace(0, 1, num=k)
i = 0
mses3_train = np.zeros((k,1))
mses3 = np.zeros((k,1))
mintest = sys.maxsize
mintrain = sys.maxsize
opLamTest = 0
opLamTrain = 0
for lambd in lambdas:
    w_l = learnRidgeRegression(X_i,y,lambd)
    mses3_train[i] = testOLERegression(w_l,X_i,y)
    mses3[i] = testOLERegression(w_l,Xtest_i,ytest)
    #mins vvv
    if(mses3_train[i] < mintrain):
        mintrain = mses3_train[i]
        opLamTrain = lambd
    if(mses3[i] < mintest):
        mintest = mses3[i]
        opLamTest = lambd
    i = i + 1
fig = plt.figure(figsize=[12,6])
plt.subplot(1, 2, 1)
plt.plot(lambdas,mses3_train)
plt.title('MSE for Train Data')
plt.subplot(1, 2, 2)
plt.plot(lambdas,mses3)
plt.title('MSE for Test Data')

plt.show()
print("OpTrain = " , opLamTrain)
print("OpTest = " , opLamTest)

# Problem 4
k = 101
lambdas = np.linspace(0, 1, num=k)
i = 0
mses4_train = np.zeros((k,1))
mses4 = np.zeros((k,1))
opts = {'maxiter' : 20}    # Preferred value.
w_init = np.ones((X_i.shape[1],1))
for lambd in lambdas:
    args = (X_i, y, lambd)
    w_l = minimize(regressionObjVal, w_init, jac=True, args=args,method='CG', options=opts)
    w_l = np.transpose(np.array(w_l.x))
    w_l = np.reshape(w_l,[len(w_l),1])
    mses4_train[i] = testOLERegression(w_l,X_i,y)
    mses4[i] = testOLERegression(w_l,Xtest_i,ytest)
    i = i + 1

fig = plt.figure(figsize=[12,6])
plt.subplot(1, 2, 1)
plt.plot(lambdas,mses4_train)
plt.plot(lambdas,mses3_train)
plt.title('MSE for Train Data')
plt.legend(['Using scipy.minimize','Direct minimization'])

plt.subplot(1, 2, 2)
plt.plot(lambdas,mses4)
plt.plot(lambdas,mses3)
plt.title('MSE for Test Data')
plt.legend(['Using scipy.minimize','Direct minimization'])
plt.show()

# Problem 5
pmax = 7
lambda_opt = opLamTest # REPLACE THIS WITH lambda_opt estimated from Problem 3
mses5_train = np.zeros((pmax,2))
mses5 = np.zeros((pmax,2))
for p in range(pmax):
    Xd = mapNonLinear(X[:,2],p)
    Xdtest = mapNonLinear(Xtest[:,2],p)
    w_d1 = learnRidgeRegression(Xd,y,0)
    mses5_train[p,0] = testOLERegression(w_d1,Xd,y)
    mses5[p,0] = testOLERegression(w_d1,Xdtest,ytest)
    w_d2 = learnRidgeRegression(Xd,y,lambda_opt)
    mses5_train[p,1] = testOLERegression(w_d2,Xd,y)
    mses5[p,1] = testOLERegression(w_d2,Xdtest,ytest)

fig = plt.figure(figsize=[12,6])
plt.subplot(1, 2, 1)
plt.plot(range(pmax),mses5_train)
plt.title('MSE for Train Data')
plt.legend(('No Regularization','Regularization'))
plt.subplot(1, 2, 2)
plt.plot(range(pmax),mses5)
plt.title('MSE for Test Data')
plt.legend(('No Regularization','Regularization'))
plt.show()


from cell_type import CellType

ascii_repr = {
    CellType.UNKNOWN: '?',
    CellType.BLOCK: 'X',
    CellType.EMPTY: '.',
    CellType.PATH: '*',
    CellType.BOX: 'O',
    CellType.TARGET: '+',
    CellType.SATISFIED: '!',
    CellType.FIGURE: '|',
    CellType.TARGET_N_FIGURE: '|'
}

emoji_chick_repr = {
    CellType.UNKNOWN: '?',
    CellType.BLOCK: 'Ъфе',
    CellType.EMPTY: 'ЪЪФ',
    CellType.PATH: 'РГљ',
    CellType.BOX: 'ЪЦџ',
    CellType.TARGET: 'Ъф╣',
    CellType.SATISFIED: 'Ъф║',
    CellType.FIGURE: 'ЪљЦ',
    CellType.TARGET_N_FIGURE: 'ЪљЦ'
}


import pytest
import requests
from pytest_bdd import scenarios, when, then

# public api to get random single or two-part jokes of a random category
JOKES_API = "https://v2.jokeapi.dev/joke/Any"

# path to feature file
scenarios("../features/multiple_jokes.feature", example_converters=dict(number_of_jokes=int,
                                                                        response_number_of_jokes=int))

# 'when' behavior test
# calls jokes api with specified number of jokes
@pytest.fixture
@when('the Jokes API is queried with "<number_of_jokes>"')
def joke_response(number_of_jokes):
    params = {'format': 'json', 'amount': number_of_jokes}
    response = requests.get(JOKES_API, params=params)
    return response

# 'then' behavior test
# tests the data responded from the jokes api call, ensuring the correct number of jokes was returned
@pytest.fixture
@then('the response contains "<response_number_of_jokes>"')
def test_joke_response_amount(joke_response, response_number_of_jokes):
    # actual
    json_response = joke_response.json()
    # check actual vs expected and assert
    assert len(json_response["jokes"]) == response_number_of_jokes

# 'then' behavior test
# tests that the response status code from the jokes api was success (200)
@pytest.fixture
@then("the response status code is 200")
def test_joke_response_success(joke_response):
    # check actual vs expected and assert
    assert joke_response.status_code == 200


from __future__ import print_function
from sys import stderr
from consts import CMAconsts
from AssimCclasses import pyConfigContext
from AssimCtypes import CONFIGNAME_INSTANCE, CONFIGNAME_DEVNAME
from discoverylistener import DiscoveryListener
from graphnodes import NICNode, IPaddrNode, NetworkSegment, Subnet
from systemnode import SystemNode
from cmaconfig import ConfigFile
from linkdiscovery import discovery_indicates_link_is_up

@SystemNode.add_json_processor  # Register ourselves to process discovery packets
class ArpDiscoveryListener(DiscoveryListener):
    """
    Class for processing ARP cache discovery entries.
    The data section contains (IPaddress, MACaddress) pairs as a hash table (JSON object)
    Nanoprobes currently send their entire cache each time anything shows up

    For interest, here are some default ARP cache timeouts as of this writing:
        Linux         300 seconds
        Solaris       300 seconds
        Windows       600 seconds
        AIX          1200 seconds
        FreeBSD      1200 seconds
        NetBSD       1200 seconds
        OpenBSD      1200 seconds
        VMWare       1200 seconds
        Cisco       14400 seconds

    For large subnets, it would be much more efficient to send only changes --
    additions and deletions -- but that's not what we currently do :-D

    @TODO: Change ARP updates to give deltas

    If we changed the nanoprobes to send delta updates, then you wouldn't particularly
    want a 20 minute timeout - you'd want something more like a few minutes
    so that they came a few at a time to the CMA.  This would keep the CMA from
    being bottlenecked by massive updates if you have a subnet with a lot of IP
    addresses on it.  It also means that we would not get 1024 entries just because
    one came online...

    Of course, a lot of what causes this code to be really slow is the fact that we
    hit the database with a transaction for each IP and each MAC that we find in the
    message.
    """

    prio = DiscoveryListener.PRI_OPTION  # This is an optional feature
    # We are interested in two kinds of packets:
    # netconfig:    Packets from network configuration discovery
    #               When we hear these we send requests to listen to ARPs
    #               This is what eventually causes ARP discovery packets to be sent
    # ARP:          Packets resulting from ARP discovery - triggered by
    #               the requests we send above...
    wanted_packets = ("ARP", "netconfig")

    def processpkt(self, drone, src_addr, json_obj, discoverychanged):
        """
        Trigger ARP discovery or add ARP data to the database.

        :param drone: SystemNode: who discovered this?
        :param src_addr: pyNetAddr: address this came from
        :param json_obj: dict: discovery JSON as object
        :param discoverychanged: bool: TRUE if this discovery has changed
        :return: None
        """
        if not discoverychanged:
            return
        if json_obj["discovertype"] == "ARP":
            self.processpkt_arp(drone, src_addr, json_obj)
        elif json_obj["discovertype"] == "netconfig":
            self.processpkt_netconfig(drone, src_addr, json_obj)
        else:
            self.log.warning("Unexpected ArpDiscovery packet type [%s]" % json_obj["discovertype"])
            print(
                "OOPS! unexpected ArpDiscovery packet type [%s]" % json_obj["discovertype"],
                file=stderr,
            )

    def processpkt_netconfig(self, drone, _unused_srcaddr, jsonobj):
        """
        We want to trigger ARP discovery when we hear a 'netconfig' packet

        Build up the parameters for the discovery
        action, then send it to drone.request_discovery(...)
        To build up the parameters, you use ConfigFile.agent_params()
        which will pull values from the system configuration.

        :param drone: SystemNode: who discovered this?
        :param _unused_srcaddr: pyNetAddr: address this came from
        :param jsonobj: dict: discovery JSON as object
        :return: None
        """
        init_params = ConfigFile.agent_params(self.config, "discovery", "#ARP", drone.designation)

        data = jsonobj["data"]  # the data portion of the JSON message
        discovery_args = []
        for devname in data.keys():
            # print ("*** devname:", devname, file=stderr)
            devinfo = data[devname]
            if discovery_indicates_link_is_up(devinfo):
                params = pyConfigContext(init_params)
                params[CONFIGNAME_INSTANCE] = "_ARP_" + devname
                params[CONFIGNAME_DEVNAME] = devname
                # print('#ARP parameters:', params, file=stderr)
                discovery_args.append(params)
        if discovery_args:
            drone.request_discovery(discovery_args)

    def processpkt_arp(self, drone, _unused_srcaddr, jsonobj):
        """
        Process ARP (cache) packets when they come in

        We want to update the database when we hear a 'ARP' discovery packet
        These discovery entries are the result of listening to ARP packets
        in the nanoprobes.  Some may already be in our database, and some may not be.

        As we process the packets we create any IPaddrNode and NICNode objects
        that correspond to the things we've discovered.

        Since IP addresses can move around, we potentially need to clean up relationships to
        NICNodes - so that any given IP address is only associated with a single
        MAC address.

        We also create a NetworkSegment object for this network segment, and associate it with
        all the MAC addresses we deal with. We also try and update Subnets for any IP addresses
        we find on this subnet. If this network segment only has one Subnet associated with it,
        and this NIC has an IP address associated with it, then we get them all set up correctly.

        As noted in the class docs, the data we get is organized by IP address.
        This means that a single MAC address (NIC) may appear multiple times in the
        discovery data.

        As it stands now, not sure what to do about removed (MAC,IP) pairs...
        @FIXME: Nanoprobe code needs to time out MAC/IP entries, and we need to deal with
        having the IP address removed. MAC addresses in some sense never go away...
        We also need to deal with the fact that a given MAC address might show up on different
        subnets at different times - WiFi, VPN, etc

        :param drone: SystemNode: system node (Drone) that heard this packet
        :param _unused_srcaddr: IPaddrNode: unused - address that this packet came from
        :param jsonobj: dict-like: the JSON from the discovery - rendered into a dict-like thing
        :return:
        """

        data = jsonobj["data"]
        device_name = jsonobj["device"]
        device = drone.find_nic(device_name)
        if device is None:
            print(f"********************Cannot find NIC '{device_name}' for drone {drone}")
            return
        mac_ip_table = {}
        # Group the IP addresses by MAC address - inverting the map
        for ip, mac in data.items():
            macstr = str(mac)
            if macstr not in mac_ip_table:
                mac_ip_table[macstr] = []
            mac_ip_table[macstr].append(ip)

        net_segment = self.find_net_segment(drone, device, mac_ip_table)
        device.net_segment = net_segment.name
        self.fix_net_segment(
            domain=drone.domain, device=device, net_segment=net_segment, mac_ip_table=mac_ip_table
        )

    def find_net_segment(self, _drone, device, mac_ip_table):
        """
        Figure out which network segment this device belongs to or create one if its new...

        :param _drone: Drone:  this NIC segment is attached to
        :param device: NICNode: device these ARPs were heard on
        :param mac_ip_table: Table of IPs associated with NICs
        :return: NetworkSegment
        """
        if device.net_segment is not None:
            return self.store.load_or_create(
                NetworkSegment, domain=device.domain, name=str(device.net_segment)
            )
        mac_ip_pairs = []
        for mac, ip_list in mac_ip_table.items():
            for ip in ip_list:
                mac_ip_pairs.append((mac, ip))
        segment = NetworkSegment.guess_net_segment(self.store, device.domain, mac_ip_pairs)
        if self.debug:
            self.log.debug("Guess_net_segment() returned %s" % segment)
        if segment is None:
            result = self.store.load_or_create(NetworkSegment, domain=device.domain)
        else:
            result = self.store.load(NetworkSegment, domain=device.domain, name=segment.name)
        return result

    # pylint - too many local variables
    # pylint: disable=R0914
    def fix_net_segment(self, domain, device, net_segment, mac_ip_table):
        """
        Fix up the network segment for the MAC/IP pairs on this network segment
        :param domain: str: Domain for created IP/MAC addresses
        :param device: NICNode: device that heard the ARP packets
        :param net_segment: NetworkSegment: our network segment
        :param mac_ip_table: MAC/IP map - organized by MAC address
        :return: None
        """
        if self.debug:
            self.log.debug(
                "FIX_NET_SEGMENT: %s %s %s %s" % (domain, device, net_segment, mac_ip_table)
            )
        mac_ip_query = """
        MATCH(nic:Class_NICNode)-[:ipowner]->(ip:Class_IPaddrNode)
        WHERE ip.ipaddr in $ipaddrs AND nic.macaddr in $macaddrs
        AND (nic.net_segment = $net_segment OR nic.net_segment IS NULL)
        RETURN nic, ip
        """
        # Our algorithm is as follows:
        # - Find all the IP,MAC pairs that are either in our network segment, or could be part of
        #   our network segment, and update the network segment of those that aren't yet in any
        #   network segment.
        # - along the way, collect the set of subnets that the IP addresses belong to, and the
        #   IP addresses that don't belong to any subnet yet
        # - All those pairs that don't yet exist, create them, and link them together and
        #   make them part of our network segment
        # - Walk through the IP addresses we've collected, and make them part of one of these
        #   subnets if we can.
        #
        #   Something I haven't completely thought through yet...
        #   What happens when a MAC address gets a new IP address?
        #
        mac_ip_pairs = set()
        mac_set = set()
        ip_set = set()
        for mac, ip_list in mac_ip_table.items():
            if self.debug:
                self.log.debug("LOOKNG AT (%s,%s)" % (mac, str(ip_list)))
            mac_set.add(str(mac))
            for ip in ip_list:
                mac_ip_pairs.add((mac, ip))
                ip_set.add(str(ip))
        parameters = {
            "ipaddrs": sorted(ip_set),
            "macaddrs": sorted(mac_set),
            "net_segment": str(net_segment),
        }
        self.log.debug("MAC_IP_QUERY: %s %s" % (mac_ip_query, parameters))
        missing_pairs = mac_ip_pairs
        found_pairs = set()
        found_subnets = set()
        subnet_names = set()

        if self.debug:
            self.log.debug("MAC_IP_QUERY: %s::%s" % (mac_ip_query, str(parameters)))
        for mac, ip in self.store.load_cypher_query(mac_ip_query, parameters):
            if self.debug:
                self.log.debug("FIX_NET_SEGMENT: LOOKING AT (%s, %s)" % (mac, ip))
            if (mac.macaddr, ip.ipaddr) not in missing_pairs:
                # Could be a mismatched pair or a new/old IP for this same MAC
                continue
            missing_pairs.remove((mac.macaddr, ip.ipaddr))
            self.log.debug("FIX_NET_SEGMENT: ADDING FOUND PAIRS (%s, %s)" % (mac, ip))
            found_pairs.add((mac, ip))
            mac.net_segment = net_segment
            if ip.subnet is not None and ip.subnet not in subnet_names:
                found_subnets.add(Subnet.find_subnet_by_name(self.store, ip.subnet))
                subnet_names.add(ip.subnet)
        Subnet.normalize_subnet_set(found_subnets)  # This removes duplicate/overlapping subnets
        # Now we have a list of all the missing (MAC, IP) pairs - create them
        self.create_missing_mac_ip_pairs(
            domain, device.scope, net_segment, missing_pairs, found_subnets
        )
        self.fix_ip_subnets(found_subnets, [pair[1] for pair in found_pairs])

    def create_missing_mac_ip_pairs(self, domain, scope, net_segment, missing_pairs, found_subnets):
        """
        Create all the missing (mac, IP) pairs
        :param domain: str: domain for IP and MAC addresses
        :param scope: str: scope for created MAC addresses
        :param net_segment: str: the network segment to associate them with
        :param missing_pairs: set((str,str)) - set of (mac, IP) addresses in canonical form
        :param found_subnets: set(Subnet) - list of subnets found on this network segment
        :return: None
        """
        if self.debug:
            self.log.debug(
                "CREATE_MISSING_MAC_IP_PAIRS: %s / %s / %s" % (domain, scope, net_segment)
            )
            self.log.debug("CREATE_MISSING_MAC_IP_PAIRS: PAIRS: %s" % missing_pairs)
        for mac, ip in missing_pairs:
            self.log.debug("CREATE_MISSING_MAC_IP_PAIRS: PROCESSING (%s, %s)" % (mac, ip))
            # find_this_macaddr is a more "generous" find operation than load_or_create()...
            nic = NICNode.find_this_macaddr(
                self.store, domain=domain, macaddr=mac, net_segment=net_segment
            )
            if self.debug:
                self.log.debug("CREATE_MISSING_MAC_IP_PAIRS: FOUND NICs: %s" % nic)
            if nic is None:
                nic = self.store.load_or_create(
                    NICNode, domain=domain, macaddr=mac, scope=scope, net_segment=str(net_segment)
                )
            subnet = Subnet.find_matching_subnet(ip, found_subnets)
            if self.debug:
                self.log.debug("CREATE_MISSING_MAC_IP_PAIRS: FOUND SUBNET: %s" % subnet)
            other_ip = None
            for other_ip in self.store.load_related(nic, CMAconsts.REL_ipowner):
                if other_ip.ipaddr == ip:
                    other_ip.subnet = subnet
                    break  # Strange... It already exists...
            if other_ip is None or other_ip.ipaddr != ip:
                ip_node = self.store.load_or_create(
                    IPaddrNode, ipaddr=ip, domain=domain, subnet=subnet
                )
                if self.debug:
                    self.log.debug("CREATE_MISSING_MAC_IP_PAIRS: IP_NODE: %s" % ip_node)
                self.store.relate(nic, CMAconsts.REL_ipowner, ip_node)

    @staticmethod
    def fix_ip_subnets(found_subnets, ip_list):
        """
        Fix up the subnets of everything we can on this Network Segment.
        "We can" means we _know_ which subnet a particular IP address goes with.

        :param found_subnets: Iterable(Subnet): Candidate subnets
        :param ip_list: Iterable(IPaddrNode): IP addresses to fix up with their subnets
        :return:
        """
        for ip in ip_list:
            if hasattr(ip, "subnet") and ip.subnet is not None:
                subnet = Subnet.find_matching_subnet(ip, found_subnets)
                if subnet is not None:
                    ip.subnet = subnet.name


import sys
from OpenGL.GLUT import *
from OpenGL.GL import *
from OpenGL.GLU import *
import numpy as np
import pandas as pd
import math as m
import time
import os

win_w, win_h = 1024, 768

def reshape(w, h):
    global win_w, win_h

    win_w, win_h = w, h
    glViewport(0, 0, w, h)
    glMatrixMode(GL_PROJECTION)
    glLoadIdentity()
    gluPerspective(45, win_w/win_h, 0.01, 50)

wireframe, pause = False, True
def keyboard(key, x, y):
    global wireframe, pause

    key = key.decode("utf-8")
    if key == ' ':
        pause = not pause
        glutIdleFunc(None if pause else idle)
    elif key == 'w':
        wireframe = not wireframe
        glPolygonMode(GL_FRONT_AND_BACK, GL_LINE if wireframe else GL_FILL)
    elif key == 'q':
        os.exit(0)
    glutPostRedisplay()

tick, frame_cnt = 0, 0
def idle():
    global tick, frame_cnt

    tick += 1
    frame_cnt += 1
    glutPostRedisplay()

def display():
    global start_time, frame_cnt
    if frame_cnt == 20:
        print("%.2f fps" % (frame_cnt/(time.time()-start_time)), tick, end='\r')
        start_time = time.time()
        frame_cnt = 0
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
    glMatrixMode(GL_MODELVIEW)
    glLoadIdentity()

    #ตำแหน่งตา
    eye_pos = np.array((0,3,3))
    gluLookAt(*eye_pos, *centroid, 0, 1, 0)
    #ตำแหน่งแหล่งกำเนิดแสง
    light_pos = np.array((3,4,20))
    #ความเข้มแสง ค่าสีของแสง
    Il = np.array((1.0,1.0,1.0))

    Ka = np.array((0.1, 0.1, 0.1), dtype=np.float32)#ambient สีเทาอ่อน
    Kd = np.array((1, 0, 0), dtype=np.float32)#diffuse สีแดง
    Ks = np.array((1, 1, 0), dtype=np.float32) #specular สะท้อนสีเหลือง
    shininess = 50
    N = normals
    L = light_pos - positions
    L = L * 1/np.linalg.norm(L,axis=1).reshape(-1,1)

    NdotL = np.sum(N * L, axis=1).reshape(-1,1)

    V = eye_pos - positions
    V = V * 1/np.linalg.norm(V,axis=1).reshape(-1,1)
    H = L+V
    H = H * 1/np.linalg.norm(H,axis=1).reshape(-1,1)
    NdotH = np.sum(N * H, axis=1).reshape(-1,1)

    ambient = Ka*Il
    diffuse_reflection = Kd*NdotL*Il
    specular_reflection = Ks*pow(NdotH,shininess)*Il

    colors[:, :] = ambient + diffuse_reflection + specular_reflection

    glVertexPointer(3, GL_FLOAT, 0, positions)
    glColorPointer(3, GL_FLOAT, 0, colors)
    glDrawArrays(GL_TRIANGLES, 0, n_vertices)
    glutSwapBuffers()

def gl_init_models():
    global start_time
    global n_vertices, positions, colors, normals, uvs, centroid, bbox

    glClearColor(0, 0, 0, 0)
    glEnable(GL_DEPTH_TEST)
    glShadeModel(GL_SMOOTH)
    glEnableClientState(GL_VERTEX_ARRAY)
    glEnableClientState(GL_COLOR_ARRAY)

    df = pd.read_csv("../models/bunny_uv.tri", delim_whitespace=True,
                     comment='#', header=None, dtype=np.float32)
    centroid = df.values[:, 0:3].mean(axis=0)
    bbox = df.values[:, 0:3].max(axis=0) - df.values[:, 0:3].min(axis=0)

    n_vertices = len(df.values)
    positions = df.values[:, 0:3]
    colors = df.values[:, 3:6]
    normals = df.values[:, 6:9]
    uvs = df.values[:, 9:11]
    start_time = time.time() - 0.0001

def main():
    glutInit(sys.argv)
    glutInitDisplayMode(GLUT_RGB | GLUT_DOUBLE | GLUT_DEPTH)
    glutInitWindowSize(win_w, win_h)
    glutCreateWindow("Illumination with Client-side Verterx Arrays")
    glutDisplayFunc(display)
    glutReshapeFunc(reshape)
    glutKeyboardFunc(keyboard)
    glutIdleFunc(idle)
    gl_init_models()
    glutMainLoop()

if __name__ == "__main__":
    main()