from dataclasses import dataclass

from aiogram.fsm.state import State, StatesGroup
from aiogram.types import InlineKeyboardButton as Button, KeyboardButton as KButton, Message
from aiogram.utils.keyboard import InlineKeyboardBuilder as Board, ReplyKeyboardBuilder as KBoard

start_board = Board([
    [Button(text='Погода ⛅', callback_data='weather forecast')],
    [Button(text='Настройки ⚙', callback_data='settings')]
]).as_markup()
back_btn = lambda data='', text='': Button(text='🔙 Назад' if not text else text, callback_data=f'back_{data}')

settings_board = Board([
    [Button(text='🔔 Настроить уведомления', callback_data='notify_settings')],
    [Button(text='🗺 Обновить местоположение', callback_data='send_location')],
    [Button(text='🗑 Удалить все данные', callback_data='delete_data')],
    [back_btn()]
]).as_markup()
location_board = KBoard([
    [KButton(text='🗺 Отправить своё местоположение', request_location=True)],
    [KButton(text='🔙 Назад')]
]).as_markup(one_time_keyboard=True, resize_keyboard=True)

time_board = lambda h=None, m=None: lambda data='', text='': (
    Board().row(Button(text='Часы: ↘' if h is None else f'Часы: {h:02} ↘', callback_data='show_h'))
    .row(Button(text='Минуты: ↘' if m is None else f'Минуты: {m:02} ↘', callback_data='show_m'))
    .row(*([back_btn(data, text)]
         + ([] if h is None or m is None else [Button(text='Создать 🔔', callback_data=f'create_notify {h}:{m}')])))
    .as_markup()
)
hour_board = lambda h=None, m=None: lambda data='', text='': (
    Board().row(Button(text='Часы: ↖' if h is None else f'Часы: {h:02} ↖', callback_data='hide_h'))
    .row(*[Button(text=f'✅ {n:02}' if h == n else f'{n:02}', callback_data=f'set h {n}') for n in range(24)], width=6)
    .row(Button(text='Минуты: ↘' if m is None else f'Минуты: {m:02} ↘', callback_data='show_m'))
    .row(*([back_btn(data, text)]
           + ([] if h is None or m is None else [Button(text='Создать 🔔', callback_data=f'create_notify {h}:{m}')])))
    .as_markup()
)
minute_board = lambda h=None, m=None: lambda data='', text='': (
    Board().row(Button(text='Часы: ↘' if h is None else f'Часы: {h:02} ↘', callback_data='show_h'))
    .row(Button(text='Минуты: ↖' if m is None else f'Минуты: {m:02} ↖', callback_data='hide_m'))
    .row(*[Button(text=f'✅ {n:02}' if m == n else f'{n:02}',
                  callback_data=f'set m {n}') for n in range(0, 60, 5)], width=6)
    .row(*([back_btn(data, text)]
           + ([] if h is None or m is None else [Button(text='Создать 🔔', callback_data=f'create_notify {h}:{m}')])))
    .as_markup()
)

START = ('{}{}\n\nС помощью ветров знаний и сил солнца, неба и дождя я предсказываю прогноз погоды на каждый день! '
         'Нажми на кнопку «Погода», чтобы узнать погоду прямо сейчас и прогноз погоды на сегодня, на завтра или даже '
         'на целых 5 дней вперёд. ⛅\n\nТы можешь настроить меня в «Настройках» и указать своё местоположение или время'
         ', когда ты хочешь получать уведомления. 🔔')
SETTINGS = ('Здесь ты можешь обновить своё местоположение или настроить ежедневные уведомления о погоде. ⚙\n\nЯ за '
            'конфиденциальность, поэтому по желанию ты можешь удалить все данные о себе и заставить меня забыть тебя, '
            'нажав на кнопку «Удалить все данные». 😉')
DATA_DELETED = 'Ой, кто-то сверкнул у меня перед лицом Нейрализатором из «Людей в чёрном»... Я всё забыл про тебя...'

CURR_LOCATION = 'Я помню, что ты из {}. 😉'
LOCATION = ('Нажми на кнопку ниже, чтобы отправить своё текущее местоположение, и дальше я всё сделаю за тебя! 🔍\n\n'
            'Или можешь нажать на 📎 и поделиться местоположением так, если хочешь указать другое.\n\nНу и, наконец, '
            'можешь просто отправить мне название города.')
LOCATION_SET = 'Я запомнил, где ты живёшь ;) Приятно познакомиться с жителем {}! 🤝'
LOCATION_ERROR = ('Я старался, но не смог найти твой город. 😩 Может быть, где-то опечатка? Попробуй ещё раз или отправь'
                  ' мне своё местоположение через кнопку ниже или с помощью 📎.')
NO_LOCATION_FORECAST = 'Чтобы предсказать прогноз погоды, мне нужно знать, где ты находишься ;)'

CURR_NOTIFY = ('Список поставленных уведомлений ниже на кнопках.\n\nНажми на ненужное, если хочешь удалить. 🙅🏻♂',
               'Уведомления не установлены.\n\nНажми на кнопку ниже, чтобы установить уведомления. 👇🏻')
NO_LOCATION_NOTIFY = ('Я не смогу отправить тебе уведомление с погодой, потому что не знаю твоё местоположение. 🤷🏻♂\n'
                      'Покажи мне по кнопке ниже, где ты находишься ;)')
NEW_NOTIFY = 'Выбери часы и минуты для уведомления и нажми «Создать» или введи их в формате HH:MM. 🤓'
NOTIFY_ERROR = ('Что-то мне не разобрать, что ты написал. 🤨 Может быть, у тебя где-то ошибочка? Попробуй ещё раз или'
                'воспользуйся клавиатурой ниже ;)')
NOTIFY_EXISTS = 'Уведомление с таким временем уже есть. 🤷🏻♂\nВыбери другое время ;)'
NOTIFY_SUCCESS = 'Уведомление успешно создано!  🥳'

FORECAST = ('{icon} {adverb} в {city} {verb}{desc}.\n🌡 На улице {temp}°C ({feels_verb} как {feels_like}°C).\n🫠 '
            'Давление: {pressure} мм рт.ст.\n💦 Влажность: {humidity}%.\n🍃 {wind_side} ветер скоростью {wind_speed} '
            'м/c.\n☁ На небе облачность в {clouds}%.')
SUN_DESC = '🌅 Восход сегодня {verb_sr} в {sunrise}.\n🌇 Закат {verb_ss} в {sunset}.'

SOON = 'В разработке — ждите очень скоро! 🔜'

@dataclass
class CallbackData:
    data: str
    message: Message

class Dialog(StatesGroup):
    get_geo = State()
    get_notify_time = State()


import pandas as pd
import numpy as np
import pickle

from tqdm import tqdm

import vtk
from vtk.util.numpy_support import vtk_to_numpy

from glob import glob

def load_vtu(str_file):
    r = vtk.vtkXMLUnstructuredGridReader()

    r.SetFileName(str_file)
    r.Update()

    points = r.GetOutput().GetPoints().GetData()
    np_points = vtk_to_numpy(points)

    phi = r.GetOutput().GetPointData().GetArray(0)
    np_phi = vtk_to_numpy(phi)

    return {'points':np_points, 'phi' : np_phi}

file_list = glob("./*_motor_unit_* idx_muscle_fiber_*_1/t_*.vtu")

df = pd.DataFrame({'file_name' : file_list})

df['muscle_fiber_id'] = df.file_name.map(lambda x: int( x.split(' ')[-1].split('_')[3] ))
df['electrodes']      = df.file_name.map(lambda x: x.split('/')[-1].split('_')[0] == 't')
df['sources']         = df.file_name.map(lambda x: x.split('/')[-1].split('_')[0] == 'sources')
df['time']            = df.file_name.map(lambda x: int(x.split('/')[-1].split('_')[-1].split('.')[0]))

muscle_ids = df.muscle_fiber_id.unique()

muscle_fiber_phi = {}

def import_muscle_data(muscle_id : '[int]', df : 'pd.DataFrame' ) -> 'dict':
    sel = (df.muscle_fiber_id == muscle_id)

    df_local = df[sel & df.electrodes]

    muscle_fiber_phi = {}
    muscle_fiber_phi[muscle_id] = {}

    for t in df_local.time:
        file_selected = df_local.file_name[df_local.time == t]
        assert len(file_selected) == 1 , 'more than one file selected'

        try:
            data_dict = load_vtu( file_selected.item() )
        except :
            import pdb; pdb.set_trace()

        phi    = data_dict['phi']

        muscle_fiber_phi[muscle_id][t] = phi

    return muscle_fiber_phi

def import_electrode_positions(df : 'pd.DataFrame') -> 'nd.array' :
        data_dict = load_vtu( df.file_name[0])

        return data_dict['points']

def sort_time(d):
    time_idx = np.sort([t for t in d])
    return np.array( [ d[t] for t in time_idx ])

iterator = tqdm(map(lambda x : import_muscle_data(x, df), muscle_ids), total = len(muscle_ids))

muscle_fiber_phi = { k: sort_time(v) for d in iterator for k , v in d.items()  }

electtrode_df = pd.DataFrame({'timepoint' : [], 'phi': [], 'electrode_id': [], 'muscle_id' : []})

electrode_positions = import_electrode_positions(df)

for e_id in tqdm( range(len(muscle_fiber_phi[0][0,:])) ):
    for m_id in muscle_fiber_phi:

        df_length = len(muscle_fiber_phi[m_id][:,e_id])

        electtrode_df_m = pd.DataFrame({
            'timepoint':    [t for t in range(len(muscle_fiber_phi[m_id][:,e_id]))]
            , 'phi':          muscle_fiber_phi[m_id][:,e_id]
            , 'muscle_id':    [m_id] * df_length
            , 'electrode_id': [e_id] * df_length
            , 'p_electrode_x' : [electrode_positions[e_id,0]] * df_length
            , 'p_electrode_y' : [electrode_positions[e_id,1]] * df_length
            , 'p_electrode_z' : [electrode_positions[e_id,2]] * df_length
            })

        electtrode_df = pd.concat([electtrode_df, electtrode_df_m], ignore_index = True )

pd.to_pickle(electtrode_df, "./pandas_full_data.pkl")


from torch import Tensor
import kornia

# Use only if max_length >= 3000 or .mean often, otherwise use RunningMeanStack
# This implementation is faster than RunningMeanStack for large max_length
class RunningMeanStackFast(list):

    def __init__(self, max_length):
        super().__init__()
        self.max_length = max_length
        self.prev_sum = 0
        self.prev_mean = None
        self.added = []
        self.removed = []
        self.correction = 0

    def append(self, x):
        super().append(x)
        self.added.append(x)
        if len(self) > self.max_length:
            self.removed.append(self.tail)
            self.pop(0)
        else:
            self.prev_sum = sum(self)

    @property
    def mean(self):
        self.correction += 1
        if len(self) == 0:
            return 0

        if self.correction % 50 == 0:
            self.correction = 0
            self.prev_mean = sum(self) / len(self)
            return self.prev_mean

        if len(self.added) != 0 and len(self.removed) != 0:
            self.prev_sum += sum(self.added) - sum(self.removed)
            self.prev_mean = self.prev_sum / self.max_length
        else:
            self.prev_mean = self.prev_sum / len(self)

        self.added = []
        self.removed = []
        return self.prev_mean

    @property
    def head(self):
        return self[-1]

    @property
    def tail(self):
        return self[0]

# running mean for max_length samples using FIFO stack
class RunningMeanStack(list):

    def __init__(self, max_length):
        super().__init__()
        self.max_length = max_length

    def append(self, x):
        super().append(x)
        if len(self) > self.max_length:
            self.pop(0)

    @property
    def mean(self):
        if len(self) == 0:
            return 0

        return sum(self) / len(self)

    @property
    def head(self):
        return self[-1]

    @property
    def tail(self):
        return self[0]

# return image to normal rgb appearance and value range
def normalize_image(img):
    img = img.cpu().detach()

    # l_mean: float = 50, l_std: float = 29.59, ab_mean: float = 0, ab_std: float = 74.04
    img[0, 0] = img[0, 0] * 29.59 + 50
    img[0, 1] = img[0, 1] * 74.04
    img[0, 2] = img[0, 2] * 74.04

    # clip values
    img[0, 0] = img[0, 0].clamp(0, 100)
    img[0, 1] = img[0, 1].clamp(-128, 127)
    img[0, 2] = img[0, 2].clamp(-128, 127)

    img = kornia.color.lab_to_rgb(img)
    img = (img.squeeze(0).permute(1, 2, 0).numpy() * 255).astype('uint8')
    return img


from operator import itemgetter
import networkx as nx
import wikipedia
import matplotlib.pyplot as plt

def graphBuilder(pageTitle):
    SEED = pageTitle.title()
    STOPS = ("International Standard Serial Number",
            "International Standard Book Number",
            "National Diet Library",
            "International Standard Name Identifier",
            "International Standard Book Number (Identifier)",
            "Pubmed Identifier",
            "Pubmed Central",
            "Digital Object Identifier",
            "Arxiv",
            "Proc Natl Acad Sci Usa",
            "Bibcode",
            "Library Of Congress Control Number",
            "Jstor",
            "Doi (Identifier)",
            "Isbn (Identifier)",
            "Pmid (Identifier)",
            "Arxiv (Identifier)",
            "Bibcode (Identifier)")

    todo_lst = [(0, SEED)]  # The SEED is in layer 0
    todo_set = set(SEED)  # The SEED itself
    done_set = set()  # Nothing is done yet

    g = nx.DiGraph()
    layer, page = todo_lst[0]

    while layer < 2:
        # Remove the name page of the current page from the todo_lst,
        # and add it to the set of processed pages.
        # If the script encounters this page again, it will skip over it.
        del todo_lst[0]
        done_set.add(page)

        # Show progress
        print(layer, page)

        # Attempt to download the selected page.
        try:
            wiki = wikipedia.page(page)  # Specify language as 'en' for English
        except wikipedia.exceptions.DisambiguationError as e:
            print(f"Ambiguous page: {page}. Skipping.")
            layer, page = todo_lst[0]
            continue
        except:
            print("Could not load", page)
            layer, page = todo_lst[0]
            continue

        for link in wiki.links:
            link = link.title()
            if link not in STOPS and not link.startswith("List Of"):
                if link not in todo_set and link not in done_set:
                    todo_lst.append((layer + 1, link))
                    todo_set.add(link)
                g.add_edge(page, link)
        layer, page = todo_lst[0]

    print("{} nodes, {} edges".format(len(g), nx.number_of_edges(g)))
    return g

graphAlbion = graphBuilder("Albion Online")
graphOutlast = graphBuilder("Outlast 2")
graphAge = graphBuilder("Alan Wake 2")
graphCities = graphBuilder("Cities: Skylines II")

composeAlbionOutlast = nx.compose(graphAlbion, graphOutlast)
composeAgeCities = nx.compose(graphAge, graphCities)

composeAll = nx.compose(composeAlbionOutlast, composeAgeCities)

print("{} nodes, {} edges".format(len(composeAll), nx.number_of_edges(composeAll)))

# make a copy of raw graph
original = composeAll.copy()

# remove self loops
composeAll.remove_edges_from(nx.selfloop_edges(composeAll))

# identify duplicates like that: 'network' and 'networks'
duplicates = [(node, node + "s")
              for node in composeAll if node + "s" in composeAll]

for dup in duplicates:
  # *dup is a technique named 'unpacking'
  composeAll = nx.contracted_nodes(composeAll, *dup, self_loops=False)

print(duplicates)

duplicates = [(x, y) for x, y in
              [(node, node.replace("-", " ")) for node in composeAll]
                if x != y and y in composeAll]
print(duplicates)

for dup in duplicates:
  composeAll = nx.contracted_nodes(composeAll, *dup, self_loops=False)

# nx.contracted creates a new node/edge attribute called contraction
# the value of the attribute is a dictionary, but GraphML
# does not support dictionary attributes
nx.set_node_attributes(composeAll, 0,"contraction")
nx.set_edge_attributes(composeAll, 0,"contraction")

print("{} nodes, {} edges".format(len(composeAll), nx.number_of_edges(composeAll)))

plt.style.use("default")
# degree sequence
degree_sequence = sorted([d for n, d in composeAll.degree()], reverse=True)

fig, ax = plt.subplots(1,2,figsize=(8,4))

# all_data has information about degree_sequence and the width of each bin
ax[0].hist(degree_sequence)
ax[1].hist(degree_sequence,bins=[1,2,3,4,5,6,7,8,9,10])

ax[0].set_title("Degree Histogram")
ax[0].set_ylabel("Count")
ax[0].set_xlabel("Degree")
ax[0].set_ylim(0,15000)

ax[1].set_title("Degree Histogram - Zoom")
ax[1].set_ylabel("Count")
ax[1].set_xlabel("Degree")
ax[1].set_xlim(0,10)
ax[1].set_ylim(0,15000)

plt.tight_layout()
plt.show()

# filter nodes with degree greater than or equal to 2
core = [node for node, deg in dict(composeAll.degree()).items() if deg >= 2]

# select a subgraph with 'core' nodes
gsub = nx.subgraph(composeAll, core)

print("{} nodes, {} edges".format(len(gsub), nx.number_of_edges(gsub)))

nx.write_graphml(gsub, "cna.graphml")

print("Nodes removed: {:.2f}%".format(100*(1 - 3420/13775)))
print("Edges removed: {:.2f}%".format(100*(1 - 14771/24856)))
print("Edges per nodes: {:.2f}".format(13775/3420))

top_indegree = sorted(dict(gsub.in_degree()).items(),
                      reverse=True, key=itemgetter(1))[:100]
print("\n".join(map(lambda t: "{} {}".format(*reversed(t)), top_indegree)))

Atividade 03 - Unidade 02/Requisito_01/main.py


# -*- coding: utf-8 -*-
"""Public section, including homepage and signup."""
from flask import Blueprint, flash, redirect, render_template, request, url_for,current_app,send_from_directory
from flask_login import login_required, login_user, logout_user
import qrcode

import pyqrcode
from io import StringIO,BytesIO

from flask import make_response
import image,os
# try:
#     from PIL import Image
# except Exception as e:
#     print(str(e))
#     print('=====')
#     # import Image
#     from PIL.Image import core as image

from school.extensions import login_manager
from school.public.forms import LoginForm
from school.user.forms import RegisterForm
from school.user.models import User
from school.utils import flash_errors,templated

blueprint = Blueprint('public', __name__)

@login_manager.user_loader
def load_user(user_id):
    """Load user by ID."""
    return User.get_by_id(int(user_id))

@blueprint.route('/', methods=['GET', 'POST'])
@login_required
def home():
    """Home page."""
    return render_template('public/home.html')

@blueprint.route('/user_info')
@templated()
def user_info():
    return dict()

#简介
@blueprint.route('/introduction')
@templated()
def introduction():
    return dict()

#获取二维码
@blueprint.route('/get_rq/<string:rq_str>')
def get_rq(rq_str='0'):
    qr = qrcode.QRCode(
        version=1,
        error_correction=qrcode.constants.ERROR_CORRECT_L,
        box_size=10,
        border=4
    )
    qr.add_data(f'{rq_str}')
    qr.make(fit=True)
    img = qr.make_image()
    img_io = BytesIO()
    img.save(img_io, 'PNG')
    img_io.seek(0)
    response = make_response(img_io.read())
    response.headers['Content-Type'] = 'image/png'
    response.headers['Content-Description'] = 'attachment; filename=%s.png' % rq_str
    return response

#获取学生头像
@blueprint.route('/get_student_img')
@blueprint.route('/get_student_img/<path:student_img>')
def get_student_img(student_img='0'):
    path = os.getcwd()+'/'+current_app.config['STUDENTS_IMG']
    return send_from_directory(path, student_img)

@blueprint.route('/logout/')
@login_required
def logout():
    """Logout."""
    logout_user()
    flash('You are logged out.', 'info')
    return redirect(url_for('public.home'))

#flask-sse test
@blueprint.route('/hello')
def publish_hello():
    sse.publish({"user": "alice", "status": "Life is good!"}, type='greeting')
    # sse.publish({"message": "anaf!"}, type='greeting')
    return "Message sent!"
"""
<script>
    var source = new EventSource("{{ url_for('sse.stream') }}");
    source.addEventListener('greeting', function(event) {
        var data = JSON.parse(event.data);
        document.getElementById('inohtml').innerHTML +='<br/>hello';
        // alert(data);
    }, false);
    source.addEventListener('error', function(event) {
        // alert("Failed to connect to event stream. Is Redis running?");
    }, false);
  </script>
"""


__author__ = 'denislavrov'

import sqlite3

DATABASE = 'test.db'

def createtable(filename=''):  # function called to insert a table into a new database
    conn = sqlite3.connect(filename)  # opening/connecting to a database just like a file
    conn.execute('''CREATE TABLE SAVE
       (ID INTEGER PRIMARY KEY AUTOINCREMENT    NOT NULL,
       SAVENAME TEXT    NOT NULL,
       INV_TNAME TEXT     NOT NULL,
       LOC_TNAME TEXT  NOT NULL
       );
       ''')
    conn.close()

def createdb(filename=''):  # this function actually creates a physical file, for database to be stored in
    if filename == '':  # since some functions may be used just to reuse code they have keys to avoid user input
        while (len(filename) == 0) or (filename[len(filename) - 3:len(filename)] != '.db'):
            # check that input is not emtpy and ends with .db
            filename = input("Please enter a filename that ends with '.db': ")
    try:  # check if database exists by trying to open it for reading
        open(filename, mode='r')  # if file exists the following code is not run and jump to 'except:' is made.
        while 1:  # if database does exist ask user if he would like to switch to it
            useri = input("This database already exists. Would you like to switch to it? [Y,N]: ").lower()
            if useri == 'y':
                switchdb(filename=filename)  # invoke function to switch
                break
            elif useri == 'n':  # if user chooses not to switch send him back recursively,by calling this function again
                createdb()
                break
    except IOError:  # if file does not exist, then create it, create the table in it and switch to it.
        open(filename, mode='a').close()  # a quick way to create an empty file in one line.
        createtable(filename=filename)
        switchdb(filename=filename)

def switchdb(filename=''):  # a function to switch databases
    while 1:
        if filename == '':  # check for a preset key as usual if not set ask user for input
            filename = input("Please input a valid existing database name: ")
        try:  # try to open database for reading just to check that the file is there
            file = open(filename, mode='r')
        except IOError:
            print("Error switching the database.")
        else:  # no exception raised everything is fine, set that database to current database and reset current user
            global DATABASE
            DATABASE = filename
            file.close()
            break

def insertuser(username):  # little function to insert a new user into the database
    conn = sqlite3.connect(DATABASE)
    inv_tname = 'inv_' + username
    loc_tname = 'loc_' + username
    conn.execute("INSERT INTO SAVE (SAVENAME, INV_TNAME, LOC_TNAME) \
    VALUES ('%s','%s','%s')" % (username, inv_tname, loc_tname))
    conn.commit()
    conn.execute('''CREATE TABLE %s
       (ITEM_ID INTEGER  NOT NULL
       );
       ''' % inv_tname)

    conn.execute('''CREATE TABLE %s
       (LOC_ID INTEGER  NOT NULL
       );
       ''' % loc_tname)

    conn.close()

def save(inventory, visitedLocations):
    savename = input('Please input a savename: ')
    if savename != '' and not userexists(savename):
        insertuser(savename)
        conn = sqlite3.connect(DATABASE)
        for inv in inventory:
            conn.execute("INSERT INTO %s (ITEM_ID) \
    VALUES (%d)" % (get_inv_tname(savename), inv))
            conn.commit()

        for loc in visitedLocations:
            conn.execute("INSERT INTO %s (LOC_ID) \
    VALUES (%d)" % (get_loc_tname(savename), loc))
            conn.commit()
        conn.close()
    else:
        print('That save name is either invalid or already exists')

def checkdb(db, create=False):  # helper function to check if database exists
    # could be expanded to check data inside database
    try:
        open(db, mode='r').close()
        return True
    except IOError:
        print("Database %s does not exist, please create one." % db)
        if create:
            createdb()
        else:
            return False

def readusers():  # read the users in from the database
    conn = sqlite3.connect(DATABASE)
    rows = conn.execute("SELECT SAVENAME FROM SAVE")
    lrows = list([x[0] for x in rows])
    conn.close()
    return lrows

def readinv(savename):  # read the users in from the database
    conn = sqlite3.connect(DATABASE)
    rows = conn.execute("SELECT ITEM_ID FROM %s" % ('inv_' + savename))
    lrows = list([x[0] for x in rows])
    conn.close()
    return lrows

def readloc(savename):  # read the users in from the database
    conn = sqlite3.connect(DATABASE)
    rows = conn.execute("SELECT LOC_ID FROM %s" % ('loc_' + savename))
    lrows = list([x[0] for x in rows])
    conn.close()
    return lrows

def rmuser(username=''):  # function to remove a user
    global cUser
    if username == '':
        username = input("Please enter username to remove: ")
    conn = sqlite3.connect(DATABASE)
    conn.execute("DELETE FROM SAVE WHERE SAVENAME='%s'" % username)
    conn.commit()  # must commit when making changes
    conn.close()
    if username == cUser:  # if current user is the user being deleted set current user to no one
        cUser = ''
    return username

def getid(username):  # helper function to getid of user by username
    conn = sqlite3.connect(DATABASE)
    uid = list(conn.execute("SELECT ID FROM SAVE WHERE SAVENAME='%s'" % username))[0][0]
    conn.close()
    return uid

def get_inv_tname(username):  # helper function to getid of user by username
    conn = sqlite3.connect(DATABASE)
    uid = list(conn.execute("SELECT INV_TNAME FROM SAVE WHERE SAVENAME='%s'" % username))[0][0]
    conn.close()
    return uid

def get_loc_tname(username):  # helper function to getid of user by username
    conn = sqlite3.connect(DATABASE)
    uid = list(conn.execute("SELECT LOC_TNAME FROM SAVE WHERE SAVENAME='%s'" % username))[0][0]
    conn.close()
    return uid

def userexists(username):  # helper function to check if the user exists
    conn = sqlite3.connect(DATABASE)
    rows = list(conn.execute("SELECT SAVENAME FROM SAVE WHERE SAVENAME='%s'" % username))
    conn.close()
    return rows.__contains__((username,))


import imp
from django.shortcuts import render, redirect
from .models import Destination
from django.contrib.auth.models import User, auth
from django.contrib import messages
# Create your views here.

def game(request):

    gamehome = Destination()
    gamehome.img = 'gamelogo.png'
    return render(request, "game.html", {'gamehome': gamehome})

def register(request):

        if request.method == "POST":

            username = request.POST['username']
            email = request.POST['email']

            if User.objects.filter(username=username):
                messages.info(request,('Username Exists'))
                return redirect('register')
            elif User.objects.filter(email=email):
                messages.info(request, "Email Exists")
                return redirect('register')
            else:
                user = User.objects.create_user(username=username, email=email)
                user.save()
                print("User Created")
            return redirect('/')

        else:
            return render(request, "register.html")

def home(request):
    return redirect('/')


from src.config.configuration import queues_configuration
from src.utils.managers import ClientManager
import argparse
import collections
from src.utils.workers import Worker
from src.logging.dsys_logger_client import get_logger
from multiprocessing import Process
import time
import os
import signal

log = get_logger(__name__)

class ServerSetup(object):

    __slots__ = ('address', 'port', 'passkey')

    def __init__(self, address, port, passkey):
        self.address = address
        self.port = port
        self.passkey = passkey

    def __repr__(self):
        return "< ServerSetup address={} port={} passkey={}>".format(self.address, self.port, self.passkey)

class Watchdog(Process):
    def __init__(self, current_workers, jobs_queue, results_queue, rate, initial_workers, max_workers, sample_time,
                 poison_pill, logger):
        super(Watchdog, self).__init__()
        self.current_workers = current_workers
        self.jobs_queue = jobs_queue
        self.results_queue = results_queue
        self.rate = rate
        self.initial_workers = initial_workers
        self.max_workers = max_workers
        self.sample_time = sample_time
        self.poison_pill = poison_pill
        self.logger = logger

    def serve_forever(self):
        self.run()

    def run(self):
        while True:
            self.current_workers = collections.deque([w for w in self.current_workers if w.is_alive()])
            delta = analyze(self.jobs_queue, self.rate, self.current_workers,
                            self.initial_workers, self.max_workers, self.sample_time)
            if delta > 0 and len(self.current_workers) < self.max_workers:

                self.logger.info("====> Creating new worker: delta={}, threshold={}".format(delta, self.rate))
                worker = Worker(self.jobs_queue, self.results_queue, wait=False, poison_pill=self.poison_pill)
                worker.deamon = True
                worker.start()
                self.current_workers.append(worker)
                self.logger.info('{} - delta={}'.format(str(self), delta))

            elif delta < 0 and len(self.current_workers) > self.initial_workers:

                self.logger.info("====> Killing worker: delta={}, threshold={}".format(delta, self.rate))

                stop_worker(jobs_queue=self.jobs_queue,
                            poison_pill=self.poison_pill,
                            current_workers=self.current_workers,
                            workers_to_kill=1)

                self.current_workers = collections.deque([w for w in self.current_workers if w.is_alive()])
                self.logger.info('{} - delta={}'.format(str(self), delta))

            elif delta == self.rate and len(self.current_workers) > self.initial_workers:

                self.logger.info("====> Killing worker: delta={}, threshold={}".format(delta, self.rate))

                stop_worker(jobs_queue=self.jobs_queue,
                            poison_pill=self.poison_pill,
                            current_workers=self.current_workers,
                            workers_to_kill=len(self.current_workers) - self.initial_workers)

                self.current_workers = collections.deque([w for w in self.current_workers if w.is_alive()])
                self.logger.info('{} - delta={}'.format(str(self), delta))

            else:
                self.logger.info('{} - delta={}'.format(str(self), delta))

            time.sleep(1)

    def __repr__(self):
        return "< WatchDog - Running: {} Max: {}>".format(len(self.current_workers), self.max_workers)

def init_job_client(queues_conf):
    conf = ServerSetup(address=queues_conf['jobs_queue']['address'],
                       port=queues_conf['jobs_queue']['port'],
                       passkey=queues_conf['jobs_queue']['auth'])

    return ClientManager(address=(conf.address, conf.port), authkey=conf.passkey, queues=['tasks_queue'])

def init_result_client(queues_conf):
    conf = ServerSetup(address=queues_conf['results_queue']['address'],
                       port=queues_conf['results_queue']['port'],
                       passkey=queues_conf['results_queue']['auth'])

    return ClientManager(address=(conf.address, conf.port), authkey=conf.passkey, queues=['results_queue'])

def get_queues(job_client, results_client):
    return job_client.get_tasks_queue(), results_client.get_results()

def workerpool(jobs, results, initial_workers=2, max_workers=8, poison_pill='-STOP-', consume_rate=10., sample_time=1):

    current_workers = collections.deque()

    def inner(_current_workers):
        for _ in range(initial_workers):
            worker = Worker(jobs, results, wait=False, poison_pill=poison_pill)
            worker.deamon = True
            worker.start()
            _current_workers.append(worker)

        log.info("Started default workers")

        return Watchdog(current_workers=_current_workers,
                        jobs_queue=jobs,
                        results_queue=results,
                        rate=consume_rate,
                        initial_workers=initial_workers,
                        max_workers=max_workers,
                        sample_time=sample_time,
                        poison_pill=poison_pill,
                        logger=get_logger())

    return inner(current_workers)

def analyze(jobs, consume_rate_setpoint, current_workers, initial_workers, max_workers, sample_time=1. / 15.):

    size_t0 = jobs.qsize()
    time.sleep(sample_time)
    size_t1 = jobs.qsize()

    rate = size_t0 - size_t1 / len(current_workers)

    if len(current_workers) < initial_workers:
        return 1

    if len(current_workers) > max_workers:
        return -1

    return rate - consume_rate_setpoint

def stop_worker(jobs_queue, poison_pill, current_workers, workers_to_kill=1):

    if poison_pill is not None:

        _ = [jobs_queue.put(poison_pill, 1) for _ in range(workers_to_kill)]

    else:

        workers = [w.stats for w in current_workers]
        workers = [w for w in workers if w['cpu_percent'] == 0.]
        workers = sorted(workers, key=lambda x: x['memory_percent'])
        _ = [os.kill(workers[i * -1]['pid'], signal.SIGTERM) for i in range(workers_to_kill)]

def start_workers(queues_conf, initial_workers=2, max_workers=8, consume_rate=10., sample_time=1, poison_pill='-STOP-'):
    jobs, results = get_queues(init_job_client(queues_conf), init_result_client(queues_conf))
    pool = workerpool(jobs, results, initial_workers, max_workers, poison_pill,
                      consume_rate=consume_rate, sample_time=sample_time)
    pool.serve_forever()

def parse_commandline_args():
    parser = argparse.ArgumentParser('[dsys] Queues service')

    parser.add_argument(
        '--initial_workers',
        default=1,
        type=int,
        help='min workers',
        dest='initial_workers',
    )
    parser.add_argument(
        '--max_workers',
        default=6,
        type=int,
        help='max workers',
        dest='max_workers',
    )
    parser.add_argument(
        '--rate',
        default=10.,
        type=float,
        help='consuming rate threshold',
        dest='consume_rate',
    )
    parser.add_argument(
        '--sample_time',
        default=1.,
        type=float,
        help='check sample time',
        dest='sample_time',
    )
    parser.add_argument(
        '--poison_pill',
        default='-STOP-',
        type=str,
        help='message for stop',
        dest='poison_pill',
    )
    parsed_args = parser.parse_args()
    return parsed_args

def main():
    args = parse_commandline_args()
    start_workers(queues_configuration, args.initial_workers, args.max_workers,
                  args.consume_rate, args.sample_time, args.poison_pill)

if __name__ == '__main__':
    main()


#!/usr/bin/python
#
# Install SDK dependencies
#
#
# The <dependecies.json file> is a JSON file in its simplest form looking like this:
# [
#   {
#     "Url": "http://os4depot.net/share/library/misc/superduperlib.lha",
#     "SdkInstall":
#        [
#            { "From": "SuperDuper/Dev/include/amigaos4/inline4/superduper.h",    "To": "include/include_h/inline4/superduper.h"},
#            { "From": "SuperDuper/Dev/include/amigaos4/interfaces/superduper.h", "To": "include/include_h/interfaces/superduper.h"},
#        ]
#   }
# ]
# The JSON file can be expanded with SdkInstall instructions as below.

VERSION = "$ver 1.6 (2021-05-09)"
AUTHOR = "Joakim Nordstrom"

import sys
import json
import tempfile
#import urllib
#import shutil
#import urllib.request
#import requests
import argparse
import os.path
import re
from subprocess import call

# SDK location -- always end with slash
SDK_LOCATION = os.getenv('DEPGET_SDK_LOCATION', "/SDK/");
GLOBAL_DEPENDENCIES_FILE = os.getenv('DEPGET_GLOBAL_DEPENDENCIES_FILE', "/etc/amiga-dependencies.json");
GLOBAL_DEPENDENCIES_URL = "https://raw.githubusercontent.com/jaokim/amiga-dev-env/master/support-files/conf/dependencies.json"

parser = argparse.ArgumentParser(
     description='Install SDK dependencies',
     epilog='''\
dep-get uses a JSON file which contains install instructions for archives with SDK-material in them.
 [
   {
     "Url": "http://os4depot.net/share/library/misc/superduperlib.lha",
     "ArchiveType": "lha"
     "SdkInstall":
        [
            { "From": "SuperDuper/Dev/include/amigaos4/inline4/superduper.h",    "To": "include/include_h/inline4/superduper.h"},
            { "From": "SuperDuper/Dev/include/amigaos4/interfaces/superduper.h", "To": "include/include_h/interfaces/superduper.h"},
        ]
   }
 ]
Examples:
  dep-get --install libxml2 "pthreads 2005"       Install libxml2, libicu, and Pthreads
  dep-get --search icu                            Search for dependency "icu"
  dep-get -f myown-dependecies.json               Install dependecies listed in the JSON file
''')
parser.add_argument('-i', '--install', nargs='*', help="list of dependencies to install" )
parser.add_argument('-g', '--generate-dep-file', action="store_true",  help="for each installed package, update the local dependency file (see -f option)")
parser.add_argument('-f','--file', default="./dependencies.json", help="JSON file with dependecies to install")
parser.add_argument('-s', '--search', help="search for a dependency")
parser.add_argument('--update', help='download a new global dependencies file', action="store_true")
parser.add_argument('--test', help='only test what would happen', action="store_true")
parser.add_argument('--verbose', help='be verbose', action="store_true")
parser.add_argument('--version', help='show version', action="store_true")
parser.add_argument('--silent', help='be silent', action="store_true")

args = parser.parse_args()

if args.version:
    print("Version: {}".format(VERSION))
    print("Author:  {}".format(AUTHOR))
    sys.exit(0)

if args.verbose:
    COPY_COMMAND = "cp --verbose -rp {install_from} {install_to}";
else:
    COPY_COMMAND = "cp -rp {install_from} {install_to}";
UNPACK_CMD = {}
UNPACK_CMD['lha'] = "lha xw={tempdir} {archive}"
UNPACK_CMD['tar'] = "tar -C {tempdir} -xvf {archive}"
CHMOD_CMD = "chmod -R a+r {tempdir}"
DL_CMD = "wget {url} -O{destinationfile}"

#+tempdir+"/"+instruction['From']+" "+SDK_LOCATION+instruction['To'];

#######################################################################
# Download file
#######################################################################
def download_file(url, destinationfile):
    #with urllib.request.urlopen(url) as response:
    #    shutil.copyfileobj(response, destinationfile)
    download_cmd = DL_CMD.format(url=url, destinationfile=destinationfile)
    call(download_cmd, shell=True)
    #r = requests.get(url, allow_redirects=True)
    #open(destinationfile, 'wb').write(r.content)
    #urllib.request.FancyURLOpener().retrieve(url, filename=destinationfile)
    #urllib.urlretrieve(GLOBAL_DEPENDENCIES_URL, GLOBAL_DEPENDENCIES_FILE)
    #r = urllib.urlopen(url)
    #open(destinationfile, 'wb').write(r.read())

#######################################################################
# Update dependencies files
#######################################################################
if args.update:
    if not args.silent :
        print("Updating dependencies file from: "+GLOBAL_DEPENDENCIES_URL)

    if args.test:
        print("Downloading from {}, storing to: {}".format(GLOBAL_DEPENDENCIES_URL, GLOBAL_DEPENDENCIES_FILE))
    else:
        download_file(GLOBAL_DEPENDENCIES_URL, GLOBAL_DEPENDENCIES_FILE)

    sys.exit(0)

# Open default dependencies files. If the "local" dependency file doesnt contain
# an SdkInstall instructions, the global will be used.
if os.path.isfile(GLOBAL_DEPENDENCIES_FILE):
    with open(GLOBAL_DEPENDENCIES_FILE, 'r') as f:
        global_deps_dict = json.load(f)
else:
    if not args.silent:
        print("Global dependencies file couldn't be opened: {}", GLOBAL_DEPENDENCIES_FILE)
    global_deps_dict = {}

nothingFound = True
found = False;

#######################################################################
# Search
#######################################################################
if args.search:
    for global_dep in global_deps_dict:
        name = ""
        url = ""
        if "Url" in global_dep:
            url = global_dep["Url"]
        else:
            continue

        if "Name" in global_dep:
            name = global_dep["Name"]
        else:
            idx = url.rfind("/")
            if idx != -1:
                name = url[idx+1:len(url)]

        if url.lower() == args.search.lower():
            found = True;
        elif re.match(".*"+args.search.lower()+".*$", url.lower()):
            found = True;
        elif re.match(args.search.lower(), name.lower()):
            found = True;
        elif args.verbose :
            print("Doesnt match: "+name + " " + global_dep["Url"])

        if found :
            print(name.ljust(32)+" "+url)
            nothingFound = False
        found = False;
    if nothingFound:
        if not args.silent:
            print("No archives found matching \""+args.search+"\".")
        sys.exit(5)
    sys.exit(0)

#######################################################################
# Open JSON file listing and describing download URL and installation procedure.
# "Url" contains the URL to download
# "SdkInstall" is a list of From/To objects, where From is the path in the archive, and To is
# the target path in the SDK.
# [
#   {
#     "Url": "http://os4depot.net/share/library/misc/superduperlib.lha",
#     "SdkInstall":
#        [
#            { "From": "SuperDuper/Dev/include/amigaos4/inline4/superduper.h",    "To": "include/include_h/inline4/superduper.h"},
#            { "From": "SuperDuper/Dev/include/amigaos4/interfaces/superduper.h", "To": "include/include_h/interfaces/superduper.h"},
#        ]
#   }
# ]

if args.file:
    dependency_file = args.file

#######################################################################
# Read from already existing dep file, and then append stuff to it,
# adn then in the end, write to file.
#######################################################################
local_dependency_file_content = []
if args.generate_dep_file:
    if os.path.isfile(dependency_file) and os.path.getsize(dependency_file) > 0:
        with open(dependency_file, 'r') as f:
            local_dependency_file_content = json.load(f)
            if args.verbose:
                print("Read {} dependencies from {}", len(local_dependency_file_content), dependency_file)
    elif args.verbose:
        print("No existing, or empty dependency file found {}, will create new", dependency_file)

#######################################################################
# Install
#######################################################################
if args.install is not None:
    failure = False
    deps_to_install = []
    # Install from dependency file
    if len(args.install) == 0:
        dependencies = []
        if os.path.isfile(dependency_file) and os.path.getsize(dependency_file) > 0:
            if args.verbose:
                print("Loading dependencies from file {}".format(dependency_file))
            with open(dependency_file, 'r') as f:
                local_dep_file = json.load(f)
                if args.verbose:
                    print("Found {} dependencies to possibly install".format(len(local_dep_file)))
                for local_dep in local_dep_file:
                    # If we find the URL and isntall instructions in the local file,
                    # we can download and install, otherwise, we have to find it in global
                    if "Url" in local_dep and "SdkInstall" in local_dep:
                        deps_to_install.append(local_dep)
                    else:
                        for global_dep in global_deps_dict:
                            if ( ("Url" in global_dep and "Url" in local_dep and global_dep['Url'] == local_dep['Url']) or
                                    ("Name" in global_dep and "Name" in local_dep and global_dep['Name'] == local_dep['Name']) ):
                                if "SdkInstall" in global_dep:
                                    deps_to_install.append(global_dep)
                                else:
                                    print("No SDK installation instruction found for: {}, {}".format(dependency, url))
                                    failure = True
                            else:
                                continue
        elif args.verbose:
            failure = True
            print("Is not a file: {}".format(dependency_file))
    # Install from command line list
    else:
        dependencies = args.install

        if args.verbose:
            print("Loading dependencies from command-line {}".format(dependencies))
        for dependency in dependencies:
            deps_candidates = []
            # Find dependency to install
            for global_dep in global_deps_dict:
                name = ""
                url = ""
                if "Url" in global_dep:
                    url = global_dep["Url"]
                else:
                    continue

                archive_name = url.rsplit('/', 1)[-1]

                if "Name" in global_dep:
                    name = global_dep["Name"]
                if dependency == name or dependency.lower() == url or dependency.lower() == archive_name:
                    if "SdkInstall" in global_dep:
                        deps_candidates.append(global_dep)
                    else:
                        print("No SDK installation instruction found for: {}, {}".format(dependency, url))
                        failure = True

            # Make sure we have only matched one dependency
            if len(deps_candidates) == 0:
                failure = True
                if not args.silent :
                    print("Found no dependency for \"{}\"".format(dependency))
            elif len(deps_candidates) > 1:
                failure = True
                print("Found {} matching dependencies for \"{}\":".format(len(deps_candidates), dependency))
                for too_many_deps in deps_candidates :
                    print("   {}".format(too_many_deps['Url']))
                print("Please install using entire URL.")
            else:
                deps_to_install.append(deps_candidates[0])

    if failure:
        print("One or more errors occured, aborting")
        sys.exit(10)
    else:
        for dep_to_install in deps_to_install:
            url = dep_to_install['Url']
            archive_name = url.rsplit('/', 1)[-1]
            archive_type = dep_to_install['ArchiveType'] if 'ArchiveType' in dep_to_install else archive_name[-3:]

            if "SdkInstall" in dep_to_install:
                sdk_install = dep_to_install['SdkInstall']
            else :
                # This should be impossible, check is done above ^
                print("No SDK install instruction for {}".format(url))
                sys.exit(10)

            if not archive_type in UNPACK_CMD:
                print("No handler for archive type {} (filename: {}, url: {})".format(archive_type, archive_name, url))
                sys.exit(10)

            # Download archive and extract
            if args.test:
                tempdir = "tmp"
            else:
                tempdir = tempfile.mkdtemp()

            unpack_cmd = UNPACK_CMD[archive_type].format(tempdir=tempdir, archive=tempdir+"/"+archive_name)
            chmod_cmd = CHMOD_CMD.format(tempdir=tempdir)

            if args.test:
                print("Should download {} to {}/{}".format(url,tempdir,archive_name))
                print("call: "+unpack_cmd)
                print("call: "+chmod_cmd)
            else:
                if args.verbose:
                    print("Downloading {} to {}/{}".format(url,tempdir,archive_name))

                download_file(url, tempdir+"/"+archive_name)
                # r = requests.get(url, allow_redirects=True)
                # open(tempdir+"/"+archive_name, 'wb').write(r.content)

                # urllib.urlretrieve(url, tempdir+"/"+archive_name)

                if args.verbose:
                    print("call: "+unpack_cmd)
                call(unpack_cmd, shell=True)
                if args.verbose:
                    print("call: "+chmod_cmd)
                call(chmod_cmd, shell=True)

            for instruction in sdk_install:
                copy_cmd = COPY_COMMAND.format(install_from=tempdir+"/"+instruction['From'], install_to=SDK_LOCATION+instruction['To'])
                if args.test:
                    print("call: "+copy_cmd)
                else:
                    if args.verbose:
                        print("call: "+copy_cmd)
                    call(copy_cmd, shell=True)

            if not args.silent:
                print("Installed "+archive_name+" successfully.")

            if not dep_to_install in local_dependency_file_content:
                local_dependency_file_content.append({'Url':dep_to_install['Url']})
            elif args.verbose and args.generate_dep_file:
                print("Dependecy {} already in file, not adding".format(dep_to_install['Url']))
    if not args.silent:
        print("Installed {} dependencies".format(len(deps_to_install)))
if args.generate_dep_file:
    if args.verbose:
        print("Adding dependencies to {}".format(dependency_file))
    with open(dependency_file, 'w') as outfile:
        json.dump(local_dependency_file_content, outfile)

sys.exit(0)


from __future__ import print_function
import grpc
import msg_pb2
import msg_pb2_grpc

def run():
    # 客户端很好理解,网络连接得到一个channel,拿channel去实例化一个stub,通过stub调用RPC函数
    channel = grpc.insecure_channel('localhost:50051')
    # 使用grpc.insecure_channel('localhost:50051')进行连接服务端,接着在这个channel上创建stub
    stub = msg_pb2_grpc.MsgServiceStub(channel)
    # 在msg_pb2_grpc里可以找到MsgServiceStub这个类相关信息。这个stub可以调用远程的GetMsg函数

    a1 = float(input());# 输入运算数1，并转为float型
    op1 = input();# 输入运算符号类型
    b1 = float(input());# 输入运算数2，并转为float型

    response = stub.GetMsg(msg_pb2.MsgRequest(a=a1,op=op1,b=b1))# response为服务器端发来的内容
    # MsgRequest中的内容即msg.proto中定义的数据。在回应里可以得到msg.proto中定义的msg

    msg = "服务器端运算结果为：{}\n".format(response.result)#打印从服务器端接收到的数据，即运算结果
    print(msg)

if __name__ == '__main__':
    run()


# -*- coding:utf-8 -*-
import numpy as np
import pandas as pd
import gc
import lightgbm as lgb
import xgboost as xgb
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold
import matplotlib.pyplot as plt
import seaborn as sns
from hyperopt import fmin,tpe,hp,partial,space_eval
from sklearn.model_selection import KFold, StratifiedKFold
from hyperopt import STATUS_OK
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
pd.set_option('display.max_columns', None)
from sklearn.metrics import average_precision_score

train_df = pd.read_csv('creditcard_train.csv')

#特征工程1:时间特征处理
timedelta = pd.to_timedelta(train_df['Time'], unit='s')
train_df['Minute'] = (timedelta.dt.components.minutes).astype(int)
train_df['Hour'] = (timedelta.dt.components.hours).astype(int)

feats = [f for f in train_df.columns if f not in ['Class','Index']]
folds = StratifiedKFold(n_splits=5,shuffle=True,random_state=0)

def average_precision_score_vali(preds, data_vali):
    labels = data_vali.get_label()
    score_vali = average_precision_score(labels, preds)
    return 'average_precision_score', score_vali, True

def kfold_lightgbm(params):

    oof_preds = np.zeros(train_df.shape[0])

    for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train_df['Class'])):

        dtrain = lgb.Dataset(data=train_df[feats].iloc[train_idx],
                             label=train_df['Class'].iloc[train_idx],
                             free_raw_data=False, silent=True)
        dvalid = lgb.Dataset(data=train_df[feats].iloc[valid_idx],
                             label=train_df['Class'].iloc[valid_idx],
                             free_raw_data=False, silent=True)

        clf = lgb.train(
            params=params,
            train_set=dtrain,
            num_boost_round=1000,
            valid_sets=[dtrain, dvalid],
            early_stopping_rounds=20,
            feval=average_precision_score_vali,
            verbose_eval=False
        )

        oof_preds[valid_idx] = clf.predict(dvalid.data)

        del clf, dtrain, dvalid
        gc.collect()

    return average_precision_score(train_df['Class'], oof_preds)

#objective function
def lgb_objective(params,n_folds=5):

    loss = -kfold_lightgbm(params)

    return {'loss':-loss,'params':params,'status':STATUS_OK}

# Define the search space
space = {
    'objective':'regression',
    'boosting_type': 'gbdt',
    'subsample':0.8,
    'colsample_bytree':hp.uniform('colsample_bytree',0.8,0.9),
    'max_depth':7,
    'learning_rate':0.01,
    "lambda_l1":hp.uniform('lambda_l1',0.0,0.2),
    'seed':0,
}

#define algorithm
tpe_algorithm = tpe.suggest

best = fmin(fn = lgb_objective,space = space,algo=tpe_algorithm,max_evals=50)

print(best)
result=space_eval(space, best)
print(space_eval(space, best))


import re
from collections import Counter

import pytest

import gen_factor_sat.circuit.tseitin.encoding as te
from gen_factor_sat.factoring_sat import FactoringSat
from gen_factor_sat.formula.cnf import CNF
from gen_factor_sat.formula.symbol import variable

comment_line = re.compile('c (?P<comment>.*)')
problem_line = re.compile('p cnf (?P<variables>\\d*) (?P<clauses>\\d*)')
clause_line = re.compile('(-?[1-9][0-9]* )*0')

@pytest.mark.parametrize('clause', [
    te.empty_clause(),
    te.unit_clause(variable(42)),
    te.clause(list(map(variable, [1, -2, -4, 17])))
])
def test_clauses_to_dimacs_conversion(clause):
    dimacs_clause = CNF.clause_to_dimacs(clause)
    assert clause_line.match(dimacs_clause), 'Clauses should match the DIMACS format'

    numbers = list(map(int, dimacs_clause.split(' ')))
    assert numbers[-1] == 0, 'Clauses (including empty clauses) should have a trailing zero'

    literals = numbers[:-1]
    assert literals == list(clause), 'The conversion should write the same literals in the same order'

@pytest.mark.parametrize('num_vars, clauses', [
    (0, {}),
    (1, {te.clause(list(map(variable, [-1, 3])))}),
    (17, {te.clause(list(map(variable, [-1, -12, -4]))), te.empty_clause()})
])
def test_cnf_to_dimacs_conversion(num_vars, clauses):
    cnf = CNF(num_vars, clauses)
    dimacs = cnf.to_dimacs()

    lines = dimacs.splitlines(keepends=False)
    assert len(lines) == 1 + len(clauses), "All clauses should be written"

    assert problem_line.match(lines[0]), "The first line should contain the instance parameters"
    match = problem_line.match(lines[0])

    num_variables = int(match.group('variables'))
    num_clauses = int(match.group('clauses'))

    assert num_variables == num_vars, "The conversion should use the provided number of variables"
    assert num_clauses == len(clauses), "The conversion should write the number of clauses"

@pytest.mark.parametrize('comments', [[], ['Comment 1'], ['Comment 1', ' ', '', 'Comment 2']])
def test_comments_should_be_prepended(comments):
    num_vars = 13
    clauses = {te.clause(list(map(variable, [-1, 3]))),
               te.clause(list(map(variable, [-10, -5, 14])))}

    cnf = CNF(num_vars, clauses)
    dimacs = cnf.to_dimacs(comments=comments)

    lines = dimacs.splitlines(keepends=False)
    assert len(lines) == len(comments) + 1 + len(clauses)

    for index in range(len(comments)):
        assert comment_line.match(lines[index])
        match = comment_line.match(lines[index])
        parsed_comment = match.group('comment')
        assert parsed_comment == comments[index]

@pytest.fixture(scope='module', params=[2, 17, 2 ** 15 + 17896, 2 ** 23 + 1247561])
def factoring_instance(request):
    factor_sat = FactoringSat.factorize_number(request.param)
    return factor_sat

def test_dimacs_format(factoring_instance):
    dimacs = factoring_instance.to_dimacs()

    parsed_problem_line = False
    lines = dimacs.splitlines(keepends=False)
    for line in lines:
        if comment_line.match(line):
            assert not parsed_problem_line, 'Comments are only permitted before the problem line'
        elif problem_line.match(line):
            assert not parsed_problem_line, 'The DIMACS format should have exactly one problem line'
            parsed_problem_line = True
        elif clause_line.match(line):
            assert parsed_problem_line, 'Clauses are only permitted after the problem line'
        else:
            assert not line, "Only comment lines, problem lines, or clauses are allowed"

    assert parsed_problem_line, 'The DIMACS format should have exactly one problem line'

def test_encoded_cnf(factoring_instance):
    dimacs = factoring_instance.to_dimacs()

    clauses = []
    lines = dimacs.splitlines(keepends=False)
    for line in lines:
        if problem_line.match(line):
            match = problem_line.match(line)

            num_variables = int(match.group('variables'))
            num_clauses = int(match.group('clauses'))

            assert num_variables == factoring_instance.cnf.number_of_variables
            assert num_clauses == len(factoring_instance.cnf.clauses)

        elif clause_line.match(line):
            clause = list(map(int, line.split(' ')[:-1]))

            occurrences = Counter(clause)
            assert all(occurrences[-literal] == 0 for literal in clause)
            assert all(occurrences[literal] == 1 for literal in clause)

            clauses.append(frozenset(clause))

    assert len(clauses) == len(factoring_instance.cnf.clauses)
    assert set(clauses) == factoring_instance.cnf.clauses


import os
import json
import argparse
from collections import OrderedDict
import torch

from src.utils import bool_flag, initialize_exp
from src.models import build_model
from src.trainer import Trainer
from src.evaluation import Evaluator

VALIDATION_METRIC_SUP = 'precision_at_1-csls_knn_10'
VALIDATION_METRIC_UNSUP = 'mean_cosine-csls_knn_10-S2T-10000'

# main
parser = argparse.ArgumentParser(description='Supervised training')
parser.add_argument("--seed", type=int, default=-1, help="Initialization seed")
parser.add_argument("--verbose", type=int, default=2, help="Verbose level (2:debug, 1:info, 0:warning)")
parser.add_argument("--exp_path", type=str, default="", help="Where to store experiment logs and models")
parser.add_argument("--exp_name", type=str, default="debug", help="Experiment name")
parser.add_argument("--exp_id", type=str, default="", help="Experiment ID")
parser.add_argument("--cuda", type=bool_flag, default=True, help="Run on GPU")
parser.add_argument("--export", type=str, default="txt", help="Export embeddings after training (txt / pth)")

# data
parser.add_argument("--src_lang", type=str, default='en', help="Source language")
parser.add_argument("--tgt_lang", type=str, default='es', help="Target language")
parser.add_argument("--emb_dim", type=int, default=300, help="Embedding dimension")
parser.add_argument("--max_vocab", type=int, default=200000, help="Maximum vocabulary size (-1 to disable)")
# training refinement
parser.add_argument("--n_refinement", type=int, default=5, help="Number of refinement iterations (0 to disable the refinement procedure)")
# dictionary creation parameters (for refinement)
parser.add_argument("--dico_train", type=str, default="default", help="Path to training dictionary (default: use identical character strings)")
parser.add_argument("--dico_eval", type=str, default="default", help="Path to evaluation dictionary")
parser.add_argument("--dico_method", type=str, default='csls_knn_10', help="Method used for dictionary generation (nn/invsm_beta_30/csls_knn_10)")
parser.add_argument("--dico_build", type=str, default='S2T&T2S', help="S2T,T2S,S2T|T2S,S2T&T2S")
parser.add_argument("--dico_threshold", type=float, default=0, help="Threshold confidence for dictionary generation")
parser.add_argument("--dico_max_rank", type=int, default=10000, help="Maximum dictionary words rank (0 to disable)")
parser.add_argument("--dico_min_size", type=int, default=0, help="Minimum generated dictionary size (0 to disable)")
parser.add_argument("--dico_max_size", type=int, default=0, help="Maximum generated dictionary size (0 to disable)")
# reload pre-trained embeddings
parser.add_argument("--src_emb", type=str, default='', help="Reload source embeddings")
parser.add_argument("--tgt_emb", type=str, default='', help="Reload target embeddings")
parser.add_argument("--normalize_embeddings", type=str, default="", help="Normalize embeddings before training")

# parse parameters
params = parser.parse_args()

# check parameters
assert not params.cuda or torch.cuda.is_available()
assert params.dico_train in ["identical_char", "default"] or os.path.isfile(params.dico_train)
assert params.dico_build in ["S2T", "T2S", "S2T|T2S", "S2T&T2S"]
assert params.dico_max_size == 0 or params.dico_max_size < params.dico_max_rank
assert params.dico_max_size == 0 or params.dico_max_size > params.dico_min_size
assert os.path.isfile(params.src_emb)
assert os.path.isfile(params.tgt_emb)
assert params.dico_eval == 'default' or os.path.isfile(params.dico_eval)
assert params.export in ["", "txt", "pth"]

# build logger / model / trainer / evaluator
logger = initialize_exp(params)
src_emb, tgt_emb, mapping, _ = build_model(params, False)
trainer = Trainer(src_emb, tgt_emb, mapping, None, params)
evaluator = Evaluator(trainer)

# load a training dictionary. if a dictionary path is not provided, use a default
# one ("default") or create one based on identical character strings ("identical_char")
trainer.load_training_dico(params.dico_train)

# define the validation metric
VALIDATION_METRIC = VALIDATION_METRIC_UNSUP if params.dico_train == 'identical_char' else VALIDATION_METRIC_SUP
logger.info("Validation metric: %s" % VALIDATION_METRIC)

"""
Learning loop for Procrustes Iterative Learning
"""
for n_iter in range(params.n_refinement + 1):

    logger.info('Starting iteration %i...' % n_iter)

    # build a dictionary from aligned embeddings (unless
    # it is the first iteration and we use the init one)
    if n_iter > 0 or not hasattr(trainer, 'dico'):
        trainer.build_dictionary()

    # apply the Procrustes solution
    trainer.procrustes()

    # embeddings evaluation
    to_log = OrderedDict({'n_iter': n_iter})
    evaluator.all_eval(to_log)

    # JSON log / save best model / end of epoch
    logger.info("__log__:%s" % json.dumps(to_log))
    trainer.save_best(to_log, VALIDATION_METRIC)
    logger.info('End of iteration %i.\n\n' % n_iter)

# export embeddings
if params.export:
    trainer.reload_best()
    trainer.export()


from triplet_loss import batch_hard_triplet_loss, batch_all_triplet_loss
from torch.optim.lr_scheduler import MultiStepLR
import torch.nn as nn
import torch
import numpy as np
import pdb
import sys

def finetune_on_support(model, Dataloader, orig_target_dict, optimizer, epochs=6, logger=None, margin=0.1, Lambda=10):
    MSE = nn.MSELoss()
    model.train()

    for epoch in range(epochs):
        print("\n \n"+("-" * 20))
        print("Epoch : ", epoch)
        print("-" * 20)
        for batch_id, data in enumerate(Dataloader):
            imgs = data[0].cuda()
            labels = data[1].cuda()
            idxs = data[2]
            gt_label = data[3]

            optimizer.zero_grad()

            target_old_task = torch.Tensor(orig_target_dict[idxs,:]).cuda()
            orig_feats, new_feats = model(imgs)

            L_old = MSE(orig_feats,target_old_task)
            L_new, frac_valid_trip = batch_all_triplet_loss(labels, new_feats, margin=margin, squared=False)
            L_total = Lambda*L_old + L_new
            # L_total = L_new
            sys.stdout.write("\rLoss Total : %f L_trip : %f L_old : %f, Frac Valid Trips: %f"%(L_total.item(),L_new.item(), L_old.item(),frac_valid_trip.item()))
            sys.stdout.flush()

            if logger is not None:
                logger.add_scalar("loss/iter", L_total.item(), epoch*len(Dataloader)+batch_id)

            L_total.backward()
            optimizer.step()
    model.eval()
    return L_old, L_new


import tensorflow as tf

from .optimizers import gather_gradient, aggregate_grad_and_apply
from .logger.training_logging import valid_log, train_log
from .loss.loss import get_losses
import time
import wandb

@tf.function
def run_train_step(model, images, t_bbox, t_class, optimizers, config):

    if config.target_batch is not None:
        gradient_aggregate = int(config.target_batch // config.batch_size)
    else:
        gradient_aggregate = 1

    with tf.GradientTape() as tape:
        m_outputs = model(images, training=True)
        total_loss, log = get_losses(m_outputs, t_bbox, t_class, config)
        total_loss = total_loss / gradient_aggregate

    # Compute gradient for each part of the network
    gradient_steps = gather_gradient(model, optimizers, total_loss, tape, config, log)

    return m_outputs, total_loss, log, gradient_steps

@tf.function
def run_val_step(model, images, t_bbox, t_class, config):
    m_outputs = model(images, training=False)
    total_loss, log = get_losses(m_outputs, t_bbox, t_class, config)
    return m_outputs, total_loss, log

def fit(model, train_dt, optimizers, config, epoch_nb, class_names):
    """ Train the model for one epoch
    """
    # Aggregate the gradient for bigger batch and better convergence
    gradient_aggregate = None
    if config.target_batch is not None:
        gradient_aggregate = int(config.target_batch // config.batch_size)
    t = None
    for epoch_step , (images, t_bbox, t_class) in enumerate(train_dt):

        # Run the prediction and retrieve the gradient step for each part of the network
        m_outputs, total_loss, log, gradient_steps = run_train_step(model, images, t_bbox, t_class, optimizers, config)

        # Load the predictions
        if config.log:
            train_log(images, t_bbox, t_class, m_outputs, config, config.global_step,  class_names, prefix="train/")

        # Aggregate and apply the gradient
        for name in gradient_steps:
            aggregate_grad_and_apply(name, optimizers, gradient_steps[name]["gradients"], epoch_step, config)

        # Log every 100 steps
        if epoch_step % 100 == 0:
            t = t if t is not None else time.time()
            elapsed = time.time() - t
            print(f"Epoch: [{epoch_nb}], \t Step: [{epoch_step}], \t ce: [{log['label_cost']:.2f}] \t giou : [{log['giou_loss']:.2f}] \t l1 : [{log['l1_loss']:.2f}] \t time : [{elapsed:.2f}]")
            if config.log:
                wandb.log({f"train/{k}":log[k] for k in log}, step=config.global_step)
            t = time.time()

        config.global_step += 1

def eval(model, valid_dt, config, class_name, evaluation_step=200):
    """ Evaluate the model on the validation set
    """
    t = None
    for val_step, (images, t_bbox, t_class) in enumerate(valid_dt):
        # Run prediction
        m_outputs, total_loss, log = run_val_step(model, images, t_bbox, t_class, config)
        # Log the predictions
        if config.log:
            valid_log(images, t_bbox, t_class, m_outputs, config, val_step, config.global_step,  class_name, evaluation_step=evaluation_step, prefix="train/")
        # Log the metrics
        if config.log and val_step == 0:
            wandb.log({f"val/{k}":log[k] for k in log}, step=config.global_step)
        # Log the progress
        if val_step % 10 == 0:
            t = t if t is not None else time.time()
            elapsed = time.time() - t
            print(f"Validation step: [{val_step}], \t ce: [{log['label_cost']:.2f}] \t giou : [{log['giou_loss']:.2f}] \t l1 : [{log['l1_loss']:.2f}] \t time : [{elapsed:.2f}]")
        if val_step+1 >= evaluation_step:
            break


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 22 19:04:10 2017

@author: bernier2
"""
import argparse
import os
import multiprocessing
import numpy as np
import timeit
import sys
import yaml

from hexrd import config
from hexrd import constants as cnst
from hexrd import instrument
from hexrd.xrd import transforms_CAPI as xfcapi

from hedmutil import process_max_tth, fit_grain_FF_init, fit_grain_FF_reduced

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Fit grain parameters for and indexed ff-HEDM dataset')
    parser.add_argument(
        'cfg', metavar='cfg_filename',
        type=str, help='a YAML ff-HEDM config filename')
    parser.add_argument(
        'opts', metavar='options_filename',
        type=str, help='a YAML filename specifying parameters for indexing')

    args = vars(parser.parse_args(sys.argv[1:]))

    print("loaded options file %s" % args['opts'])

    cfg_filename = args['cfg']
    opts_filename = args['opts']

    cfg_list = config.open(cfg_filename)
    opts = yaml.load(open(opts_filename, 'r'), Loader=yaml.SafeLoader)

    # grab additional options from YAML file
    fitg_opts = opts['options']['fit_grains']
    block_number = fitg_opts['block_number']
    clobber_grains = fitg_opts['clobber_grains']

    # process block id options
    if block_number is not None:
        if isinstance(block_number, int):
            block_ids = [block_number, ]
        elif isinstance(block_number, list):
            block_ids = block_number
    else:
        block_ids = range(len(cfg_list))

    for block_id in block_ids:
        # =====================================================================
        # INITIALIZATION
        # =====================================================================
        print("fitting block %d" % block_id)

        cfg = cfg_list[block_id]

        analysis_id = '%s_%s' % (
            cfg.analysis_name.strip().replace(' ', '-'),
            cfg.material.active.strip().replace(' ', '-'),
            )

        grains_filename = os.path.join(
            cfg.analysis_dir, 'grains.out'
        )

        # load plane data
        plane_data = cfg.material.plane_data
        max_tth = process_max_tth(cfg)
        if max_tth is not None:
            plane_data.exclusions = None
            plane_data.tThMax = max_tth

        # load instrument
        instr = cfg.instrument.hedm

        # make output directories
        if not os.path.exists(cfg.analysis_dir):
            os.mkdir(cfg.analysis_dir)
            for det_key in instr.detectors:
                os.mkdir(os.path.join(cfg.analysis_dir, det_key))
        else:
            # make sure panel dirs exist under analysis dir
            for det_key in instr.detectors:
                if not os.path.exists(os.path.join(cfg.analysis_dir, det_key)):
                    os.mkdir(os.path.join(cfg.analysis_dir, det_key))

        # load imageseries
        imsd = cfg.image_series

        # grab eta ranges and ome_period
        eta_ranges = np.radians(cfg.find_orientations.eta.range)
        ome_period = np.radians(cfg.find_orientations.omega.period)

        ncpus = cfg.multiprocessing

        threshold = cfg.fit_grains.threshold

        # =====================================================================
        # FITTING
        # =====================================================================

        # make sure grains.out is there...
        if not os.path.exists(grains_filename) or clobber_grains:
            try:
                qbar = np.loadtxt(
                    'accepted_orientations_' + analysis_id + '.dat',
                    ndmin=2).T

                gw = instrument.GrainDataWriter(grains_filename)
                grain_params_list = []
                for i_g, q in enumerate(qbar.T):
                    phi = 2*np.arccos(q[0])
                    n = xfcapi.unitRowVector(q[1:])
                    grain_params = np.hstack(
                        [phi*n, cnst.zeros_3, cnst.identity_6x1]
                    )
                    gw.dump_grain(int(i_g), 1., 0., grain_params)
                    grain_params_list.append(grain_params)
                gw.close()
            except(IOError):
                if os.path.exists(cfg.fit_grains.estimate):
                    grains_filename = cfg.fit_grains.estimate
                else:
                    raise(RuntimeError, "neither estimate nor %s exist!"
                          % 'accepted_orientations_' + analysis_id + '.dat')
            pass

        grains_table = np.loadtxt(grains_filename, ndmin=2)
        spots_filename = "spots_%05d.out"
        params = dict(
                grains_table=grains_table,
                plane_data=plane_data,
                instrument=instr,
                imgser_dict=imsd,
                tth_tol=cfg.fit_grains.tolerance.tth,
                eta_tol=cfg.fit_grains.tolerance.eta,
                ome_tol=cfg.fit_grains.tolerance.omega,
                npdiv=cfg.fit_grains.npdiv,
                refit=cfg.fit_grains.refit,
                threshold=threshold,
                eta_ranges=eta_ranges,
                ome_period=ome_period,
                analysis_dirname=cfg.analysis_dir,
                spots_filename=spots_filename)

        # =====================================================================
        # EXECUTE MP FIT
        # =====================================================================

        # DO FIT!
        if len(grains_table) == 1 or ncpus == 1:
            print("INFO:\tstarting serial fit")
            start = timeit.default_timer()
            fit_grain_FF_init(params)
            fit_results = map(
                fit_grain_FF_reduced,
                np.array(grains_table[:, 0], dtype=int)
            )
            elapsed = timeit.default_timer() - start
        else:
            print("INFO:\tstarting fit on %d processes"
                  % min(ncpus, len(grains_table)))
            start = timeit.default_timer()
            pool = multiprocessing.Pool(min(ncpus, len(grains_table)),
                                        fit_grain_FF_init,
                                        (params, ))
            fit_results = pool.map(
                fit_grain_FF_reduced,
                np.array(grains_table[:, 0], dtype=int)
            )
            pool.close()
            elapsed = timeit.default_timer() - start
        print("INFO: fitting took %f seconds" % elapsed)

        # =====================================================================
        # WRITE OUTPUT
        # =====================================================================

        gw = instrument.GrainDataWriter(
            os.path.join(cfg.analysis_dir, 'grains.out')
        )
        for fit_result in fit_results:
            gw.dump_grain(*fit_result)
            pass
        gw.close()


from sklearn import cluster
from sklearn.cluster import KMeans

from functions import *
from KNN import *
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os

owd = os.getcwd()
os.chdir('brainnetworks/CSVdata/')

selection = 'math' #default creativity

data = []
order = []
for file in os.listdir():
    data.append(normalize(pd.read_csv(file, header=None).values).flatten())
    order.append(file[:9])

# GET LABELS
sex = getLabel(0, order)
math = getLabel(1, order)
creativity = getLabel(2, order)

nanMask = ~(sex.isna() | math.isna() | creativity.isna())
sex = sex[nanMask]
math = math[nanMask]
creativity = creativity[nanMask]
data = [data[i] for i in range(0,len(data)) if nanMask[i]]

# Used to assist in score calculation function to prevent wrong function/errors
label_set = sex if selection == 'sex' else math if selection == 'math' else creativity

all_scores = []
for seed in range(0, 10):
    X_train, X_test, y_train, y_test = train_test_split(data, label_set, test_size=0.2, random_state=seed)

    scaler = StandardScaler()
    X_train = scaler.fit_transform(X_train)
    X_test = scaler.transform(X_test)

    #####################################################################
    # PCA

    n_components = 20

    # Manual PCA
    X_train_mean = X_train - np.mean(X_train)
    covar = np.cov(X_train_mean, rowvar=False)
    eigVal, eigVec = np.linalg.eig(covar) # maybe use eigh instead rather than eig, because a covariance matrix is supposed to be symmetric
    q = [(eigVal[i], eigVec[:,i]) for i in range(0, len(eigVal))]
    q.sort(key = lambda x: x[0], reverse=True)
    matrix = q[0][1]
    for i in range (1, n_components):
        matrix = np.append(matrix, q[i][1])
    matrix = matrix.reshape(n_components,-1)
    X_train_pca = np.matmul(X_train, matrix.T)
    X_test_pca = np.matmul(X_test, matrix.T)

    explained_variance_ratio = [float(eigVal[i] / sum(eigVal)) for i in range(0, n_components)]
    #for i in range(n_components):
    #    print('Percentage of variance explained by PC {}: {}'.format(i+1, explained_variance_ratio[i]))
    #print('Total variance explained by 20 PCs: {}'.format(np.sum(explained_variance_ratio)))

    pca_fig, pca_ax = plt.subplots(1)
    pca_ax.plot(range(1, 21), [sum(explained_variance_ratio[0:x+1]) for x in range(0, len(explained_variance_ratio))])
    pca_ax.set_xlabel('Num principle components')
    pca_ax.set_ylabel('Fraction of Total Variance Explained')

    # This isn't currently working for creativity, needs to be fixed or just deleted.
    if selection == 'math' or selection == 'sex':
        #KNN via SKLEARN
        knn_classifier = KNeighborsClassifier()
        knn_classifier.fit(X_train_pca,y_train)
        testResult = knn_classifier.predict(X_test_pca)
        #print("KNN Prediction",testResult)
        #print("Given Test Data",y_test.tolist())

    # Manual KNN
    distance_types = [1,2]
    k_values = range(9,36,2)

    for dstType in distance_types:
        for k in k_values:
            pred_labels = []
            for i in range(0,len(y_test)):
                pred_labels.append(knn(X_test_pca[i], y_train, X_train_pca, k, dstType))
                #print('Document ' + str(i) + ' groundtruth ' + str(y_test[i]) + ' predicted as ' + str(pred_labels[-1]))

            # Accuracy is calculated as the average of 1 - (|ytrue-ypred| / ytrue) for all test points
            # used for creativity and math
            scores = []
            if selection == 'sex':
                # due to the nature of binary selection, the round still guarantees pred_labels[i]
                # is the most common of the nearest k labels
                scores = [(1 if y_test[i] == round(pred_labels[i]) else 0) for i in range(0, len(y_test))]
            else:
                scores = [(1 - (abs(y_test[i] - pred_labels[i]) / y_test[i])) for i in range(0, len(y_test))]
            accuracy_score = sum(scores) / len(scores)
            print("Seed:", seed, "Distance type:", dstType, "K value:", k, ", Accuracy Score:", accuracy_score)
            all_scores.append((seed, dstType, k, accuracy_score))

scores_df = pd.DataFrame(all_scores, columns=['Seed', 'Distance Type', 'K', 'Accuracy'])
scores_df.sort_values(['Distance Type', 'K', 'Accuracy'], ascending=False).groupby(['Distance Type', 'K']).mean()

os.chdir(owd)


import sys
import re
from argparse import ArgumentParser

from bs4 import BeautifulSoup

def arg_parser():
    p = ArgumentParser("Carteles Moviles generator")
    p.add_argument("--step-seconds", "-s", default="1", help="Seconds between steps (default=%(default)s)")
    p.add_argument("--overflow", "-o", default="hidden", help="Generated webpage CSS overflow (default=%(default)s)")
    p.add_argument("--disable-scrolling", "-d", action='store_true', help="Disable auto scrolling in generated web page")
    p.add_argument("--rotated", '-r', action='store_true', help="Show rotated (default no)")
    p.add_argument("files", nargs='+', help="Source .htm files to be used")

    return p

def make_soup(path):
    with open(path, "rb") as f:
        buf = f.read()

    raw_text = buf.decode("cp1252")
    return BeautifulSoup(
        raw_text,
        features="html.parser"
    )

START=r"(^|;)\s*"
END=r"\s*($|;)"
REMOVAL_REGEXES = [re.compile(START + r + END) for r in (
    r"background:\s*white",
    r"color:\s*black",
    r"background:\s*#\d+",
    r"color:\s*#\d+",
    r"[^;]*font-family[^;]*",
    r"[^;]*theme-font[^;]*",
)]

def remove_unwanted_style(bs):
    for e in bs.body.find_all(recursive=True):
        if hasattr(e, "get"):
            style = e.get("style")
            if not style is None:
                style = style.replace("\n", " ").replace("\r", " ")
                for regex in REMOVAL_REGEXES:
                    style = regex.sub(";", style)
                e["style"] = style

def main(argv):
    args = arg_parser().parse_args(argv[1:])

    print("""\

<meta charset="UTF-8">

""")

    for fname in (args.files):

        bs = make_soup(fname)
        remove_unwanted_style(bs)

        word_section = bs.body.find("div", {"class":"WordSection1"})

        for p in word_section.find_all("p", {"class": "MsoNormal"}, recursive=False):
            for enter_thing in p.find_all("span", {"style":"mso-spacerun:yes"}):
                del enter_thing["style"]
                enter_thing.string = " "
                enter_thing.append(bs.new_tag("br"))
            for c in p.children:
                align = p.get("align") or 'left'
                print(f'
')
                print(c)
                print("
")

    print("""\

""")

if __name__ == "__main__":
    main(sys.argv)


import requests
import os
path = "/Users/Johnny/Downloads/abc.jpg"
#url = "https://www.nationalgeographic.com/animals/2019/11/pharmaceuticals-pollution-dung-beetles-health/#/scarabs_07.jpg"
url = "http://img0.dili360.com/ga/M01/49/B7/wKgBzFqo7emAcq6PAATLZFhctQc857.tub.jpg@!rw17"
try:
    r = requests.get(url)
    print(r.status_code)
    print(r.headers)
    # with open(path, 'wb') as f:
    #     f.write(r.content)
except:
    print("Get data error")


import pygame
import key_input as IN
import copy
import convert
import math
import sys

pygame.init()
width = 400
height = 400
s = pygame.display.set_mode((width,height))
p_clock = pygame.time.Clock()

rule = 0
base = 3

while True:

    IN.last_keys_down = copy.deepcopy(IN.keys_down)
    for event in pygame.event.get():
        if event.type == pygame.QUIT: sys.exit()
        elif event.type == pygame.KEYDOWN: IN.keys_down[str(event.key)] = True
        elif event.type == pygame.KEYUP: IN.keys_down[str(event.key)] = False
        elif event.type == pygame.MOUSEBUTTONUP: IN.keys_down[str(event.button)] = False
        elif event.type == pygame.MOUSEBUTTONDOWN: IN.keys_down[str(event.button)] = True
        elif event.type == pygame.MOUSEMOTION: IN.mouse_position = event.pos

    pygame.draw.rect(s,(20,20,20),(0,0,width,height))

    if IN.down(pygame.K_UP) and rule < base**base-1: rule += 1
    elif IN.down(pygame.K_DOWN) and rule > 0: rule -= 1

    if IN.down(pygame.K_UP) and IN.down(pygame.K_s) and rule+int(math.sqrt(base**base)) < base**base: rule += int(math.sqrt(base**base))
    elif IN.down(pygame.K_DOWN) and IN.down(pygame.K_s) and rule-int(math.sqrt(base**base)) > 0: rule -= int(math.sqrt(base**base))

    if IN.pressed(pygame.K_RIGHT) and base < 10:
        base += 1
        rule = 0
    elif IN.pressed(pygame.K_LEFT) and base > 1:
        base -= 1
        rule = 0

    graphs = [[] for k in range(base)]
    r = convert.n2s(rule % (base**base),base,base)
    for x in range(base):
        for compose in range(base):
            graphs[compose].append(x)
            x = int(r[x])

    if IN.pressed(pygame.K_SPACE):
        print( graphs )

    if IN.pressed(pygame.K_m):
        rule = int((base**base)/2)

    scale = 350/(base+2)
    for g in range(len(graphs)):
        for e in range(len(graphs[g])):
            m = 255 / (len(graphs[g])+3)
            color = (255,0,0) if g == 0 else (0,m*(g+3),0)
            if e > 0:
                pygame.draw.line(s,color,(e*scale+width/10,-graphs[g][e]*scale+height-50),((e-1)*scale+width/10,-graphs[g][e-1]*scale+height-50))
            pygame.draw.rect(s,color,(e*scale + width/10,-graphs[g][e]*scale + height-50 ,1,1))
            if graphs[g][e] == e and g > 0:
                pygame.draw.ellipse(s,(255,0,255),(e*scale+width/10-scale/2,-graphs[g][e]*scale+height-50-scale/2,scale,scale),1)

    basicfont = pygame.font.SysFont(None,24)
    text = basicfont.render('Rule: '+str(rule),True, (90,90,90))
    s.blit(text, (50,height-20,100,20))

    basicfont = pygame.font.SysFont(None,24)
    text = basicfont.render('Base: '+str(base),True, (90,90,90))
    s.blit(text, (width-100,height-20,100,20))

    pygame.display.flip()
    p_clock.tick(60)


# importing libraries

import pandas as pd

# importing the data
df = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/wdbc.data', header=None)

# there are 30 fratures and one column for malignant or benign tumor  assign the 30 features to x and the labels to y

X = df.loc[:,2:].values
y = df.loc[:,1].values

# Tranform the B and M labels of the data into numeric data labels 0 and 1

from sklearn.preprocessing import LabelEncoder

le = LabelEncoder()
y = le.fit_transform(y)

# now y is tranformed into numeric labels , malignant = 1 and benign = 0

# Splliting the test and the training data

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.20, stratify=y, random_state=1)

from sklearn.metrics import confusion_matrix

from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline

#importing the gris search funtion
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC

# preparing the pipeline for grid search, including the scaling and the support vector machine
pipe_svc = make_pipeline(StandardScaler(), SVC(random_state=1))

# training the data using support vector machines
pipe_svc.fit(X_train, y_train)

# Predicting the labels for the test data
y_pred = pipe_svc.predict(X_test)

# preparing the condfusion matrix
confmat = confusion_matrix(y_true = y_test, y_pred = y_pred)
print(confmat)

import matplotlib.pyplot as plt

# specifying the size of the figure
fig , ax = plt.subplots(figsize=(2.5,2.5))

# the data, the color
ax.matshow(confmat, cmap=plt.cm.Blues, alpha=0.3)

# defining the text position for labelling
for i in range(confmat.shape[0]):
    for j in range(confmat.shape[1]):
        ax.text(x=j,y=i,s=confmat[i,j],va='center',ha='center')

plt.xlabel('Predicted Label')
plt.ylabel('True Label')
plt.show()

# other scoring metrics are precision and recall

from sklearn.metrics import precision_score
from sklearn.metrics import recall_score, f1_score

print('Precision: %.3f' % precision_score(y_true=y_test, y_pred= y_pred))
print('Recall: %.3f' % recall_score(y_true=y_test, y_pred=y_pred))
print('F1 Score:%.3f' % f1_score(y_true=y_test, y_pred=y_pred))

#importing the gris search funtion
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC

# preparing the pipeline for grid search, including the scaling and the support vector machine
pipe_svc = make_pipeline(StandardScaler(), SVC(random_state=1))

param_range = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]

param_grid = [{'svc__C':param_range, 'svc__kernel':['linear']},
    {'svc__C':param_range, 'svc__gamma':param_range, 'svc__kernel':['rbf']}]

# We can change the scoring method in the grid search by the following method

from sklearn.metrics import make_scorer, f1_score
scorer = make_scorer(f1_score, pos_label=0)

# Specifying the parameters,the estimator and the scoring method
gs = GridSearchCV(estimator=pipe_svc, param_grid=param_grid, scoring=scorer, cv=10)

gs = gs.fit(X_train, y_train)

print(gs.best_estimator_)
print(gs.best_score_)
print(gs.best_params_)

# Plotting the Receiver Operating Characteristic

""" ROC is used to select the model for classification based on their performance
    with repect to the True positive rate and the false  positive rate
    The diagonal in a ROC can be interpreted as random guessing, the models below the diagonal are far worse than random guessing
    A perfect classifier would fall into the top left ocrner of the graph with TPR = 1 and FPR = 0 """

from sklearn.metrics import roc_curve, auc
from scipy import interp

import numpy as np
from sklearn.model_selection import StratifiedKFold

pipe_lr = make_pipeline(StandardScaler(), PCA(n_components=2), LogisticRegression(penalty='l2', random_state=1,C=100.0,solver='lbfgs'))
X_train2 = X_train[:,[4, 14]]

"""  StratifiedKFold is used to divide the data(X_train and y_train) into 3 segments. This then follows the KFold method.
    Then the divided segments are stored in a list"""

cv = list(StratifiedKFold(n_splits=3, random_state=1).split(X_train, y_train))

# Intialising a figure
fig = plt.figure(figsize=(7, 5))

mean_tpr = 0.0
mean_fpr = np.linspace(0,1,100)
all_tpr = []

""" For each iteration in the KFold method here,
    the data is trained using the pipe_lr pipeline, that uses Logistic Regression to train the model.
    Then the test data is used to test and to find the probabilty (how truly/falsiliy it is classified).
    The roc_curve function is used to find the fpr, tpr and the threshold.

        The mean is calculated and then tha variable are intilised to 0 for the next iteration"""

for i , (train,test) in enumerate(cv):
    probas = pipe_lr.fit(X_train2[train],y_train[train]).predict_proba(X_train2[test])

    fpr , tpr, thresholds = roc_curve(y_train[test], probas[:,1], pos_label=1)
    mean_tpr += interp(mean_tpr,fpr, tpr)
    mean_tpr = 0.0
    roc_auc = auc(fpr,tpr)
    plt.plot(fpr, tpr, label = 'ROC fold %d (area = %0.2f)' %(i+1, roc_auc))

# Plotting the line for the random guessing
plt.plot([0,1], [0,1], linestyle='--', color=(0.6,0.6,0.6), label='random_guessing')

mean_tpr = mean_fpr/len(mean_fpr)
mean_auc = auc(mean_tpr, mean_fpr)

plt.plot(mean_fpr, mean_tpr, 'k--', label='mean ROC (area = %0.2f)' % mean_auc, lw =2)
plt.plot([0,0,1], [0,1,1], linestyle=':', color='black', label = 'perfect peformance')
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('false positive')
plt.ylabel('true positive rate')
plt.legend(loc='lower right')
plt.show()

PARAMETER EVALUATION AND HYPER_PARAMETER_TUNING/performance_evaluation_metrics.py


import sys
from collections import deque

def bfs(x = 1):
    visited[x] = True
    q = deque()
    q.append(x)

    while q:
        now = q.popleft()
        tree.append(now)
        for i in graph[now]:
            if not visited[i]:
                visited[i] = True
                q.append(i)

n = int(sys.stdin.readline())
graph = [[] for _ in range(n+1)]
visited = [False] * (n+1)
tree = []
answer = [0] * (n+1)

for i in range(n-1):
    x, y = map(int, sys.stdin.readline().rsplit())
    if x == 1:
        graph[x].append(y)
    elif y == 1:
        graph[y].append(x)
    else:
        graph[x].append(y)
        graph[y].append(x)

bfs()
tree.reverse()

for node in tree:
    if node == 1:
        break
    if graph[node] == []:
        answer[node] = [1]
    else:
        answer[node] = graph[node]
        graph[graph[node][0]].remove(node)

for node in answer[2:]:
    print(*node)

백준/트리/트리의 부모 찾기.py

트리의 부모 찾기.py


import openai
import os
from dotenv import load_dotenv
load_dotenv()
OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')

class OpenAIClient:

    # If need to accept user chat, implement this
    def sanitize_text(self, text: str) -> str:
        raise NotImplementedError
        # if '{' in text:
        #     text = text.replace('{', '{{')
        # if '}' in text:
        #     text = text.replace('}', '}}')
        # return text

    async def send_message(
        self,
        system_prompt: str,
        ai_prompt: str,
        user_prompt: str,
        model_name: str,
        frequency_penalty: int = 0,
        temperature: float = 0.9,
    ):
        """
        Send message to LLM.
        Returns the LLM's reply.
        """
        openai.api_key = OPENAI_API_KEY
        openai_prompt = []

        # Add initial prompt
        openai_prompt.append({"role": "system", "content": system_prompt})
        openai_prompt.append({"role": "assistant", "content": ai_prompt})
        openai_prompt.append({"role": "user", "content": user_prompt})

        completion = await openai.ChatCompletion.acreate(
            model=model_name,
            frequency_penalty=frequency_penalty,
            temperature=temperature,
            messages=openai_prompt,
        )

        reply = completion.choices[0].message.content

        return {"reply": reply}


import re
from itertools import chain

def ints(line):
    return tuple(map(int, line.split(',')))

with open('input.txt') as f:
    par1, par2, par3 = f.read().split('\n\n')
    constraints = {}
    for line in par1.splitlines():
        field, rhs = line.split(': ')
        min1, max1, min2, max2 = map(int, re.findall(r'\d+', rhs))
        constraints[field] = (range(min1, max1 + 1), range(min2, max2 + 1))
    mine = ints(par2.splitlines()[1])
    nearby = [ints(line) for line in par3.splitlines()[1:]]

valid_nearby = []
error_rate = 0

for values in nearby:
    error = False
    for value in values:
        if all(value not in range_
               for ranges in constraints.values()
               for range_ in ranges):
            error_rate += value
            error = True
    if not error:
        valid_nearby.append(values)

print(error_rate) # Part 1

rows    = chain((mine,), valid_nearby)
columns = list(zip(*rows))

choices = [{field
            for field, ranges in constraints.items()
            if all(any(value in range_ for range_ in ranges)
                   for value in column)}
           for column in columns]

seen              = set()
translated_fields = []

answer = 1
for index, fields in sorted(enumerate(choices), key=lambda tpl: len(tpl[1])):
    (field,) = fields - seen # only works for this particular input
    seen.add(field)
    if field.startswith('departure'):
        answer *= mine[index]
print(answer) # Part 2


import random
import threading
import time
from loguru import logger
import sys

from lib.rdt_listener.rdt_listener import RDTListener, STOP_AND_WAIT

config = {
    "handlers": [
        {"sink": sys.stdout, "level": "DEBUG"},
    ],
}
logger.configure(**config)

def new_handle_client(stream):
    logger.critical("Sending data to client")
    stream.send(b"Hello from server, this is a test, please ignore")
    time.sleep(random.random() * 5)
    stream.close()

def main():
    listener = RDTListener(STOP_AND_WAIT)

    listener.bind(("127.0.0.1", 1234))
    listener.listen(5)
    threads = []
    for i in range(1):
        stream = listener.accept()
        logger.critical("Stream accepted")
        thread = threading.Thread(target=new_handle_client, args=(stream,))
        thread.start()
        threads.append(thread)

    for thread in threads:
        thread.join()
    listener.close()

if __name__ == "__main__":
    main()


import time
import multiprocessing

# ~~ 코드를 실행했는데, 이 코드를 수행하는데 걸린 시간??

def add_func(start, end, li):
    for i in range(start, end + 1):
        li.put(i)

# main thread

# target -> 쓰레드가 실행할 함수
li = multiprocessing.Queue()
th = multiprocessing.Process(target = add_func, args=(0, 100000000 // 2, li))
th2 = multiprocessing.Process(target= add_func, args=(100000000 // 2, 100000000, li))

s = time.time()

th.start() # 해당 쓰레드가 일 하러 감.
th2.start()
th.join() # 이 매서드를 실행한 쓰레드에 th ( thread ) 를 join
th2.join()

#print(result) # print 를 실행한 장소 ( main thread )
f = time.time()

li.put('end')

sum = 0

while True:
    t = li.get()
    if t == 'end':
        break
    else:
        sum = sum + t

print(f"실행 소유시간 : {f-s}")


#Windows based
# import pyttsx3

# def speak(text):

#     engine = pyttsx3.init("sapi5")
#     voices =engine.getProperty('voices')
#     engine.setProperty('voices',voices[1].id)
#     engine.setProperty('rate',170)
#     # print(f"\n{text}")
#     engine.say(text)
#     engine.runAndWait()

import pai_window
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from time import sleep

chrome_options = Options()
chrome_options.add_argument('--log-level=3')
chrome_options.headless = True
Path = "DataBase\\chromedriver.exe"
driver = webdriver.Chrome(Path,options=chrome_options)
driver.maximize_window()

website = r"https://ttsmp3.com/text-to-speech/British%20English/"
driver.get(website)
ButtonSelection = Select(driver.find_element(by=By.XPATH,value='/html/body/div[4]/div[2]/form/select'))
ButtonSelection.select_by_visible_text('Indian English / Aditi')#British English / Brian

def speak(text):
    pai_window.ChatBot.addAppMsg(text)

    try:
        lengthoftext = len(str(text))
        if lengthoftext == 0:
            pass
        else:
            print("")
            print(f"PAI : {text}.")
            print("")
            Data = str(text)
            xpathofsec = '/html/body/div[4]/div[2]/form/textarea'
            driver.find_element(By.XPATH, value=xpathofsec).send_keys(Data)
            driver.find_element(By.XPATH, value='//*[@id="vorlesenbutton"]')\
                .click()
            driver.find_element(By.XPATH, value="/html/body/div[4]/div[2]/form/textarea")\
                .clear()
            if lengthoftext>=30:
                sleep(4)

            elif lengthoftext>=40:
                sleep(6)

            elif lengthoftext>=55:
                sleep(8)

            elif lengthoftext>=70:
                sleep(10)

            elif lengthoftext>=100:
                sleep(13)

            elif lengthoftext>=120:
                sleep(14)

            else:
                sleep(2)

    except Exception as e:
        print(f"An error occurred while processing the text, try after some time")
        print("\n\ntrying windows voice voice...")

        import pyttsx3

        engine = pyttsx3.init("sapi5")
        voices =engine.getProperty('voices')
        engine.setProperty('voices',voices[1].id)
        engine.setProperty('rate',170)
        # print(f"\n{text}")
        engine.say(text)
        engine.runAndWait()