import datetime
from typing import Union
from ..models import LavalinkPlayer
import disnake
from ..converters import fix_characters, time_format
import itertools

def load(player: LavalinkPlayer) -> dict:

    data = {
        "content": None,
        "embeds": []
    }

    embed = disnake.Embed(color=player.bot.get_color(player.guild.me))

    if not player.static:

        duration = "`🔴 Livestream`" if player.current.is_stream else \
            time_format(player.current.duration)

        embed.description = f"> ▶ **┃**[`{fix_characters(player.current.title, 45)}`]({player.current.uri})\n" \
                            f"> ℹ **┃**`{duration}`┃`{fix_characters(player.current.author, 18)}`┃<@{player.current.requester}>"

        if player.current.playlist_name:
            embed.description += f"\n> 🎼 **┃ Playlist:** [`{player.current.playlist_name}`]({player.current.playlist_url})"

        data["embeds"].append(embed)

        if player.current_hint:
            data["embeds"].append(disnake.Embed(color=player.bot.get_color(player.guild.me)).set_footer(text=f"💡 Dica: {player.current_hint}"))

        if player.controller_mode:
            player.hint_rate = 9
            player.controller_mode = False

    else:

        embed_queue = None
        position_txt = ""
        vc_txt = ""

        if not player.paused:
            embed.set_author(
                name="Tocando Agora:",
                icon_url="https://cdn.discordapp.com/attachments/480195401543188483/895862881105616947/music_equalizer.gif"
            )

            if not player.current.is_stream:
                position_txt = f"\n> ⏲ **⠂Termina:** " f"<t:{int((disnake.utils.utcnow() + datetime.timedelta(milliseconds=player.current.duration - player.position)).timestamp())}:R>"

        else:
            embed.set_author(
                name="Em Pausa:",
                icon_url="https://cdn.discordapp.com/attachments/480195401543188483/896013933197013002/pause.png"
            )

        if player.current_hint:
            embed.set_footer(text=f"💡 Dica: {player.current_hint}")
        else:
            embed.set_footer(
                text=str(player),
                icon_url="https://cdn.discordapp.com/attachments/480195401543188483/907119505971486810/speaker-loud-speaker.gif"
            )

        queue_text_size = 31
        queue_img = "https://cdn.discordapp.com/attachments/554468640942981147/937918500784197632/rainbow_bar.gif"
        playlist_text_size = 13

        duration = "> 🔴 **⠂Duração:** `Livestream`" if player.current.is_stream else \
            f"> ⏰ **⠂Duração:** `{time_format(player.current.duration)}`"

        txt = f"[`{player.current.single_title}`]({player.current.uri})\n\n" \
              f"{duration}\n" \
              f"> 💠 **⠂Por:** {player.current.authors_md}\n" \
              f"> ✋ **⠂Pedido por:** <@{player.current.requester}>\n" \
              f"> 🔊 **⠂Volume:** `{player.volume}%`"

        if player.current.track_loops:
            txt += f"\n> 🔂 **⠂Repetições restante:** `{player.current.track_loops}`"

        if player.loop:
            if player.loop == 'current':
                e = '🔂'; m = 'Música atual'
            else:
                e = '🔁'; m = 'Fila'
            txt += f"\n> {e} **⠂Modo de repetição:** `{m}`"

        if player.nightcore:
            txt += f"\n> 🇳 **⠂Efeito nightcore:** `ativado`"

        if player.current.album_name:
            txt += f"\n> 💽 **⠂Álbum:** [`{fix_characters(player.current.album_name, limit=playlist_text_size)}`]({player.current.album_url})"

        if player.current.playlist_name:
            txt += f"\n> 📑 **⠂Playlist:** [`{fix_characters(player.current.playlist_name, limit=playlist_text_size)}`]({player.current.playlist_url})"

        if player.keep_connected:
            txt += "\n> ♾ **⠂Modo 24/7:** `Ativado`"

        elif player.restrict_mode:
            txt += f"\n> 🔒 **⠂Modo restrito:** `Ativado`"

        txt += f"{vc_txt}{position_txt}\n"

        if player.command_log:
            txt += f"```ansi\nÚltima Interação```**┕ {player.command_log_emoji} ⠂**{player.command_log}\n"

        if len(player.queue):

            queue_txt = "\n".join(
                f"`{(n + 1):02}) [{time_format(t.duration) if not t.is_stream else '🔴 Livestream'}]` [`{fix_characters(t.title, queue_text_size)}`]({t.uri})"
                for n, t in (enumerate(itertools.islice(player.queue, 15)))
            )

            embed_queue = disnake.Embed(title=f"Músicas na fila: {len(player.queue)}", color=player.bot.get_color(player.guild.me),
                                        description=f"\n{queue_txt}")

            if not player.loop and not player.keep_connected:

                queue_duration = 0

                for t in player.queue:
                    if not t.is_stream:
                        queue_duration += t.duration

                embed_queue.description += f"\n`[⌛ As músicas acabam` <t:{int((disnake.utils.utcnow() + datetime.timedelta(milliseconds=(queue_duration + player.current.duration) - player.position)).timestamp())}:R> `⌛]`"

            embed_queue.set_image(url=queue_img)

        embed.description = txt
        embed.set_image(url="https://cdn.discordapp.com/attachments/554468640942981147/937918500784197632/rainbow_bar.gif")
        embed.set_thumbnail(url=player.current.thumb)

        if player.auto_update:
            player.auto_update = 0

        if not player.controller_mode:
            player.controller_mode = True

        data["embeds"] = [embed_queue, embed] if embed_queue else [embed]

    return data


# -*- coding: utf-8 -*-
"""
Created on Sun Nov  3 10:37:23 2019

@author: vyas
Assignment 2 of RL
| X | 1 | 2 | 3 |
| 4 | 5 | 6 | 7 |
| 8 | 9 | 10| 11|
|12 |13 | 14| X |

"""
import numpy as np
import matplotlib.pyplot as plt

def IsTerminating(current_state):
    if (current_state == 0):
        return "yes"
    elif (current_state == 15):
        return "yes"
    else:
        return "no"

def  GetReward(previous_state, current_state):
    if (current_state == 0):
        return 0
    elif (current_state == 15):
        return 0
    else:
        return -1

class GridPolicyValue:
    def __init__ (self, name, discount):
        self.name = name
        self.mvalues_sum = np.zeros(16)
        self.mvalues_count = np.zeros(16)
        self.mvalues = np.zeros(16)
        self.fvalues_sum = np.zeros(16)
        self.fvalues_count = np.zeros(16)
        self.fvalues = np.zeros(16)
        self.discount = discount
        self.episode_samples = 0
        return

    def UpdateValueFunction(self, first_or_multi, episode):
        self.episode_samples += 1
        #self.value_function.append()
        e_values = episode.GetEpisode()
        e_values = np.array(e_values)
        e_values = e_values[:: -1] # Reverse time (T-1, T-2...)
        #print (e_values)
        self.found_states = np.zeros(16) # Identify
        #print("length: " , len(e_values))
        G = 0
        for i in  np.arange(len(e_values)):
            G += float(self.discount* float(e_values[i][2]))
            current_state = int(e_values[i][0])
            if i != len(e_values):
                sequence = e_values[i+1:]  # Get from t .. 0
                sequence = sequence[:,0]
                sequence = sequence.astype(int)
                #print("current:", current_state,  "seq: ",sequence)
                if current_state in sequence:
                    if (first_or_multi == "first"):
                        #print("print.. first visit ignoring", current_state, self.fvalues_sum[current_state])

                        self.mvalues_count[current_state] += 1
                        self.mvalues_sum [current_state] += G
                        self.mvalues[current_state] = self.mvalues_sum[current_state]/self.mvalues_count[current_state]
                        #print("print.. first visit ignoring", current_state, self.mvalues_sum[current_state])
                    continue
                else:
                    #print("updating: ", current_state)
                    self.mvalues_count[current_state] += 1
                    self.mvalues_sum [current_state] += G
                    self.mvalues[current_state] = self.mvalues_sum[current_state]/self.mvalues_count[current_state]

                    self.fvalues_count[current_state] += 1
                    self.fvalues_sum [current_state] += G
                    self.fvalues[current_state] = self.fvalues_sum[current_state]/self.fvalues_count[current_state]

        #    current_state = e_value[0]
        #self.PrintValues()
        return

    def PrintValues(self):
        print("Multi-visit")
        print(" |  ", round(self.mvalues[0], 2), " | ", round(self.mvalues[1], 2),
              " | ", round(self.mvalues[2], 2), " | ", round(self.mvalues[3], 2))
        print(" | ", round(self.mvalues[4], 2), " | ", round(self.mvalues[5], 2),
              " | ", round(self.mvalues[6], 2), " | ", round(self.mvalues[7], 2))
        print(" | ", round(self.mvalues[8], 2), " | ", round(self.mvalues[9], 2),
              " | ", round(self.mvalues[10], 2), " | ", round(self.mvalues[11], 2))
        print(" | ", round(self.mvalues[12], 2), " | ", round(self.mvalues[13], 2),
              " | ", round(self.mvalues[14], 2), " | ", round(self.mvalues[15], 2))
        print("First visit")
        print(" | ", round(self.fvalues[0], 2), " | ", round(self.fvalues[1], 2),
              " | ", round(self.fvalues[2], 2), " | ", round(self.fvalues[3], 2))
        print(" | ", round(self.fvalues[4], 2), " | ", round(self.fvalues[5], 2),
              " | ", round(self.fvalues[6], 2), " | ", round(self.fvalues[7], 2))
        print(" | ", round(self.fvalues[8], 2), " | ", round(self.fvalues[9], 2),
              " | ", round(self.fvalues[10], 2), " | ", round(self.fvalues[11], 2))
        print(" | ", round(self.fvalues[12], 2), " | ", round(self.fvalues[13], 2),
              " |  ", round(self.fvalues[14], 2), " | ", round(self.fvalues[15], 2))
        return

    def GetMultiVisitValues(self):
        return self.mvalues

    def GetFirstVisitValues(self):
        return self.fvalues

ActionState = [ [0, 0, 0, 0, 0], # CurrentStqte = 0
                [0, 2, 1, 5, 1], # CurrentState = 1
                [1, 3, 2, 6, 2],
                [2, 3, 3, 7, 3],

                [4, 5, 0, 8, 4], #S = 4
                [4, 6, 1, 9, 5],
                [5, 7, 2, 10, 6],
                [6, 7, 3, 11, 7],

                [8, 9, 4, 12, 8],
                [8, 10, 5, 13, 9],
                [9, 11, 6, 14, 10],
                [10, 11, 7, 15, 11],

                [12, 13, 8, 12, 12],
                [12, 14, 9, 13, 13],
                [13, 15, 10, 14, 14],
                [15, 15, 15, 15, 15]
                ]

valid_actions = ['L', 'R', 'U', 'D', "Stay"]

equi_probable_policy = [
                        [0, 0, 0, 0, 1],
                        [1/3, 1/3, 0, 1/3, 0],
                        [1/3, 1/3, 0, 1/3, 0],
                        [1/2, 0, 1/2, 0, 0],

                        [0, 1/3, 1/3, 1/3, 0],
                        [1/4, 1/4, 1/4, 1/4, 0],
                        [1/4, 1/4, 1/4, 1/4, 0],
                        [1/3, 0, 1/3, 1/3, 0],

                        [0, 1/3, 1/3, 1/3, 0],
                        [1/4, 1/4, 1/4, 1/4, 0],
                        [1/4, 1/4, 1/4, 1/4, 0],
                        [1/3, 0, 1/3, 1/3, 0],

                        [0, 1/2, 1/2, 0, 0],
                        [1/3, 1/3, 1/3, 0, 0],
                        [1/3, 1/3, 1/3, 0, 0],
                        [0, 0, 0, 0, 1],
                        ]
assignment_policy = [
                        [0, 0, 0, 0, 1],
                        [0.7, 0.1, 0, 0.1, 0.1],
                        [0.7, 0.1, 0, 0.1, 0.1],
                        [0.45, 0, 0.45, 0, 0.1],

                        [0, 0.1, 0.7, 0.1, 0.1],
                        [0.4, 0.1, 0.1, 0.1, 0],
                        [0.1, 0.4, 0.1, 0.4, 0],
                        [0.1, 0, 0.1, 0.7, 0.1],

                        [0, 0.1, 0.7, 0.1, 0.1],
                        [0.4, 0.1, 0.1, 0.4, 0],
                        [0.1, 0.4, 0.1, 0.4, 0],
                        [0.1, 0, 0.1, 0.7, 0.1],

                        [0, 0.45, 0.45, 0, 0.1],
                        [0.1, 0.7, 0.1, 0, 0.1],
                        [0.1, 0.7, 0.1, 0, 0.1],
                        [0, 0, 0, 0, 1],
                        ]

def GetNextRandomAction(policy, current_state):
    policy_row = policy[current_state]

    r = np.random.rand(1)
    csum = np.cumsum(policy_row)
    new_state_index = 0
    #print ("GetNextRandomAction: " , current_state, " policy_row: " , policy_row, "\ncsum: " , csum, "Random: " , r)

    for i in np.arange(len(csum)):
        if (r > csum[i]):
            new_state_index = i+1
        else:
            #print("GetNextRandomAction:", valid_actions[new_state_index])
            return valid_actions[new_state_index]

    #print("GetNextRandomAction:", 4) # last state - stay
    return valid_actions[4]

def GetNextState(current_state, action):
    #print("GetNextState: ", current_state, "Action: ", action)
    return ActionState[current_state][valid_actions.index(action)]

class Episode:
    def __init__ (self, name):
        self.name = name
        self.episode_result = []
        self.elements_in_episode = 0

    def updateEpisode(self,state, Action, reward):
        episode = [state, Action, reward]
        self.episode_result.append(episode)
        self.elements_in_episode +=1
        return

    def Print(self):
        for i in np.arange(len(self.episode_result)):
            print("S= ", self.episode_result[i][0], ", A=", self.episode_result[i][1],
                  "R= ", self.episode_result[i][2])
        return

    def GetEpisode(self):
        return self.episode_result

class Grid:
    def __init__ (self, name, policy):
        self.name = name
        self.policy = policy
        self.episode_num = 0
        self.episodes = []
        str_val = self.name + "episode_num" + str(self.episode_num)
        self.current_episode = Episode(str_val)
        self.SetRandomStartState()
        self.grid_pv = GridPolicyValue(name, 0.95)
        self.mvalues = []
        self.fvalues = []
        for i in np.arange(16):
            m = []
            f = []
            self.mvalues.append(m)
            self.fvalues.append(f)

    def SetRandomStartState(self):
        self.start = np.ceil(np.random.rand(1)*14)
        self.current_state = int(self.start[0])
        return

    def SetStartState(self, state):
        self.start = state
        self.current_state = state
        return

    def RunNewEpisode(self, is_random_start, start_state):
        #Stash old episode
        self.episode_num += 1
        self.episodes.append(self.current_episode)
        str_val = self.name + "episode_num" + str(self.episode_num)
        self.current_episode = Episode(str_val)

        if (is_random_start == "random"):
            self.SetRandomStartState()
        else:
            self.SetStartState(start_state)

        while (IsTerminating(self.current_state) == "no"):
            #print("RunNewEpisode: ", self.current_state)
            action = GetNextRandomAction(self.policy, self.current_state)
            cstate = self.current_state
            self.current_state = GetNextState(self.current_state, action)
            reward = GetReward(cstate, self.current_state)

            self.current_episode.updateEpisode(cstate, action, reward)

        self.grid_pv.UpdateValueFunction("first", self.current_episode)
        gm = self.grid_pv.GetMultiVisitValues()
        gf = self.grid_pv.GetFirstVisitValues()
        for i in  np.arange(16):
            self.mvalues[i].append(gm[i])
            self.fvalues[i].append(gf[i])

        return

    def PrintCurrentEpisode(self):
        self.current_episode.Print()
        return

    def RunForNEpisodes(self, total, is_random_start, start_state):
        i = 0
        while (i < total):
            self.RunNewEpisode(is_random_start, start_state)
            i += 1
        return
    def PrintValues(self):
        print("Values for: ", self.name)
        self.grid_pv.PrintValues()

    def Assignment2plots(self):
        fig = plt.figure(1)
        fig.set_figheight(15)
        fig.set_figwidth(15)
        ax2_str = self.name + " first-visit simulation run" + "for " + str(len(self.mvalues[0])) + " samples"
        ax1_str = self.name + " every-visit simulation run" + "for " + str(len(self.mvalues[0])) + " samples"

        ax1 = fig.add_subplot(2,1,1)
        ax2 = fig.add_subplot(2,1,2)
        ax1.set_xlabel("Episode number")
        ax2.set_xlabel("Episode number")
        ax1.set_ylabel("Value function")
        ax2.set_ylabel("value function")
        ax2.set_title(ax2_str)
        ax1.set_title(ax1_str)
        for i in np.arange(14):
            #print(self.mvalues[i])
            label_str = "state: " + str(i+1)
            ax1.plot(self.mvalues[i+1], label= label_str)
            ax2.plot(self.fvalues[i+1], label= label_str)
        ax1.legend()
        ax2.legend()
        file_name = self.name + "-" + str(len(self.mvalues[0]))
        plt.savefig(file_name)
        plt.show()

def Test():
    g = Grid("Assignment Policy",assignment_policy)

    #g = Grid("test",equi_probable_policy)
    #g.RunNewEpisode("fixed", 12)
    #g.PrintCurrentEpisode()
    for j in np.arange(200):
        for i in np.arange(14):
            g.RunForNEpisodes(1, "fixed", i+1)
    g.PrintValues()

    #return
    g2 = Grid("equiprobable Policy",equi_probable_policy)
    #g2.RunNewEpisode("fixed", 12)
    #g2.PrintCurrentEpisode()
    for j in np.arange(5):
        for i in np.arange(14):
            g2.RunForNEpisodes(1, "fixed", i+1)
    g2.PrintValues()

    g.Assignment2plots()
    #g2.Assignment2plots()

if __name__ == "__main__":
    # execute only if run as a script
    Test()


import csv
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
from multiprocessing import Pool # 引用进程池
import time
import timeit

# -------小工具函数---------
# 检查缺失数据
def check_missing_data(df):
  # check for any missing data in the df (display in descending order)
  return df.isnull().sum().sort_values(ascending=False)

# 删除列中的字符串
def remove_col_str(df):
  # remove a portion of string in a dataframe column - col_1
  df['col_1'].replace('\n', '', regex=True, inplace=True)

  # remove all the characters after &# (including &#) for column - col_1
  df['col_1'].replace(' &#.*', '', regex=True, inplace=True)

# 删除列中的空格
def remove_col_white_space(df):
  # remove white space at the beginning of string
  df[col] = df[col].str.lstrip()
# ---------------

# 删掉字母和unknown,保存到data_test2_modified
def deal_with_char_and_unknown(filepath):
  print(filepath)
  df=pd.read_csv(filepath, header=None, sep=',', names=['i','e','c','p','l','PV'])

  df.replace('unknown', 0, inplace=True) #原地替换为0
  df.replace('i|e|c|p|l', '', regex=True, inplace=True)
  df = pd.DataFrame(df, dtype=np.float) #全部转为float类型
  save_path = os.path.join('../data_test2_modified',os.path.basename(filepath))
  df.to_csv(save_path, header=False, index=False)

# 处理某个时刻数据时，根据已存在的属性，对某个维度的属性去重
def deal_with_a_dim(df, dim):
  # 从对应维度的文件中读取已经存在的属性值
  path = '../dims'
  abspath = os.path.join(os.path.abspath(path), dim+'.csv')
  try:
    attributes = pd.read_csv(abspath, header=None, sep=',')
    existed_attributes_set = set(attributes.iloc[0:,0].values)
  except:
    attributes = pd.DataFrame()
    existed_attributes_set = set()

  # 当前时刻的文件中的属性值
  curr_attributes_set = set(df[dim].value_counts().index)

  # 两个集合合并
  combine_set = existed_attributes_set.union(curr_attributes_set)
  series = pd.Series(list(combine_set))

  # 把新的属性集合写回对应的维度属性文件
  series.to_csv(abspath, header=False, index=False)

# 统计各个维度属性个数
def count_attributes_of_dim(file):
  path = '../data_test2_modified'
  abspath = os.path.join(os.path.abspath(path), file)
  dims = ['i', 'e', 'c', 'p', 'l']
  df=pd.read_csv(abspath, header=None, sep=',', names=['i','e','c','p','l','PV'])
  df = pd.DataFrame(df, dtype=np.float) #全部转为float类型
  for dim in dims:
    # print(list(df[dim].value_counts().index))
    deal_with_a_dim(df, dim)

# 检测某个文件是否包含异常属性值
def detect_abnormal_attributes(file):
  path = '../data_test2_modified'
  abspath = os.path.join(os.path.abspath(path), file)
  dims = ['i', 'e', 'c', 'p', 'l']
  df=pd.read_csv(abspath, header=None, sep=',', names=['i','e','c','p','l','PV'])
  try:
    df = pd.DataFrame(df, dtype=np.float) #全部转为float类型
    dim_dict = {'i':147, 'e':13, 'c':9, 'p':35, 'l':5}
    for dim in dim_dict.keys():
      if dim_dict[dim] != len(list(df[dim].value_counts().index)):
        print(dim, len(list(df[dim].value_counts().index)), file)
  except:
    print(abspath, 'can not to float')

# 清洗data_test2目录下的数据，找出各维度的合理属性记录到文件(用0替换unknown)，并且把处理完的新数据放到data_test2_modified目录下
def clean_and_count_attributes():
  path = '../data_test2'
  files= os.listdir(path)
  for file in files:
    if not os.path.isdir(file):
      file_path = os.path.join(os.path.abspath(path), file)

      # replace char and 'unknown'
      print(file_path)
      df=pd.read_csv(file_path, header=None, sep=',', names=['i','e','c','p','l','PV'])
      df.replace('unknown', 0, inplace=True) #原地替换unknown为0
      df.replace('i|e|c|p|l', '', regex=True, inplace=True)
      df = pd.DataFrame(df, dtype=np.float) #全部转为float类型

      # count attributes of each dim
      attributes_of_all_dims = []
      dims = ['i', 'e', 'c', 'p', 'l']
      for dim in dims:
        print(list(df[dim].value_counts().index))
        attributes_of_all_dims.append(list(df[dim].value_counts().index))
        deal_with_a_dim(df, dim)

      # save new data
      save_path = os.path.join('../data_test2_modified', os.path.basename(file_path))
      df.to_csv(save_path, header=False, index=False)
  return attributes_of_all_dims

# 用于多进程版本的目标函数，清洗data_test2目录下的数据，找出各维度的合理属性记录到文件(用0替换unknown)，并且把处理完的新数据放到data_test2_modified目录下
def multi_clean_and_count_attributes(file):
  path = '../data_test2'

  file_path = os.path.join(os.path.abspath(path), file)

  # 把维度字符和unknown替换掉
  print(file_path)
  df=pd.read_csv(file_path, header=None, sep=',', names=['i','e','c','p','l','PV'])
  df.replace('unknown', 0, inplace=True) #原地替换unknown为0
  df.replace('i|e|c|p|l', '', regex=True, inplace=True)
  df = pd.DataFrame(df, dtype=np.float) #全部转为float类型

  # 计算每个维度的属性
  attributes_of_all_dims = []
  dims = ['i', 'e', 'c', 'p', 'l']
  for dim in dims:
    # print(list(df[dim].value_counts().index))
    attributes_of_all_dims.append(list(df[dim].value_counts().index))
    deal_with_a_dim(df, dim)

  # save new data
  save_path = os.path.join('../data_test2_modified', os.path.basename(file_path))
  df.to_csv(save_path, header=False, index=False)
  # return attributes_of_all_dims

# 从目录获取所有文件名(带后缀)，返回一个一维list
def get_file_list(path):
  file_list = os.listdir(path)
  # 分离文件名和后缀
  # for i in range(len(file_list)):
  #   file_list[i] = os.path.splitext(file_list[i])[0]
  # print(file_list)
  return file_list

if __name__=='__main__':

  # 检查每个处理后的文件的属性数据是否错误
  # start = timeit.default_timer()
  # path = '../data_test2_modified'
  # files= os.listdir(path)
  # for file in files:
  #   if not os.path.isdir(file):
  #     detect_abnormal_attributes(file)
  # end = timeit.default_timer()
  # print(end-start)

  # 单进程统计属性
  # clean_and_count_attributes()

  # 多进程统计属性
  start2 = timeit.default_timer()
  files = get_file_list("../data_test2")
  pool = Pool(12)
  results = pool.map(multi_clean_and_count_attributes, files)
  end2 = timeit.default_timer()
  print(end2-start2)


# conding = utf-8
# /usr/bin/env python
"""
author:帅气逼人的我
date:2019/11/26 9:00
"""
import logging
import os.path
import time

class Logger(object):
    def __init__(self, logger):
        """1"""
        # 创造一个logger
        self.logger = logging.getLogger(logger)
        self.logger.setLevel(logging.DEBUG)

        # 创造一个handler,用于写入日志文件
        rq = time.strftime('%Y%m%d%H%M', time.localtime(time.time()))  # 格式化时间
        log_path = os.path.dirname(os.path.abspath(__file__))
        log_name = log_path + '/' + rq + '.log'
        fh = logging.FileHandler(log_name)
        fh.setLevel(logging.INFO)

        # 再创建一个handler,用于输出控制台
        ch = logging.StreamHandler()
        ch.setLevel(logging.INFO)

        # 定义handler的输出格式
        formatter = logging.Formatter(
            '%(asctime)s -  %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        ch.setFormatter(formatter)

        # 给logger添加handler
        self.logger.addHandler(fh)
        self.logger.addHandler(ch)

    def getlog(self):
        return self.logger


# Copyright 2013 - Stratus, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import six

from highlanderclient.api.v1 import client as client_v1

def client(highlander_url=None, username=None, api_key=None,
           project_name=None, auth_url=None, project_id=None,
           endpoint_type='publicURL', service_type='maccleod',
           auth_token=None, user_id=None, cacert=None):

        if highlander_url and not isinstance(highlander_url, six.string_types):
            raise RuntimeError('Highlander url should be a string.')

        if not highlander_url:
            highlander_url = "http://localhost:8989/v1"

        return client_v1.Client(
            highlander_url=highlander_url,
            username=username,
            api_key=api_key,
            project_name=project_name,
            auth_url=auth_url,
            project_id=project_id,
            endpoint_type=endpoint_type,
            service_type=service_type,
            auth_token=auth_token,
            user_id=user_id,
            cacert=cacert
        )

def determine_client_version(highlander_url):
    if highlander_url.find("v1") != -1:
        return 1

    raise RuntimeError("Can not determine highlander API version")


# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import os
from dataclasses import dataclass
from pathlib import PurePath
from typing import Optional, Sequence

from pants.engine.engine_aware import EngineAwareParameter
from pants.util.dirutil import fast_relpath, longest_dir_prefix
from pants.util.strutil import strip_prefix

# Currently unused, but reserved for possible future needs.
BANNED_CHARS_IN_TARGET_NAME = frozenset(r"@!?/\:=")

class InvalidSpecPath(ValueError):
    """Indicate an invalid spec path for `Address`."""

class InvalidTargetName(ValueError):
    """Indicate an invalid target name for `Address`."""

@dataclass(frozen=True)
class AddressInput:
    """A string that has been parsed and normalized using the Address syntax.

    An AddressInput must be resolved into an Address using the engine (which involves inspecting
    disk to determine the types of its components).
    """

    path_component: str
    target_component: Optional[str] = None

    def __post_init__(self):
        if self.target_component is not None or self.path_component == "":
            if not self.target_component:
                raise InvalidTargetName(
                    f"Address spec {self.path_component}:{self.target_component} has no name part."
                )

        # A root is okay.
        if self.path_component == "":
            return
        components = self.path_component.split(os.sep)
        if any(component in (".", "..", "") for component in components):
            raise InvalidSpecPath(
                f"Address spec has un-normalized path part '{self.path_component}'"
            )
        if os.path.isabs(self.path_component):
            raise InvalidSpecPath(
                f"Address spec has absolute path {self.path_component}; expected a path relative "
                "to the build root."
            )

    @classmethod
    def parse(
        cls,
        spec: str,
        relative_to: Optional[str] = None,
        subproject_roots: Optional[Sequence[str]] = None,
    ) -> AddressInput:
        """Parse a string into an AddressInput.

        :param spec: Target address spec.
        :param relative_to: path to use for sibling specs, ie: ':another_in_same_build_family',
          interprets the missing spec_path part as `relative_to`.
        :param subproject_roots: Paths that correspond with embedded build roots under
          the current build root.

        For example:

            some_target(
                name='mytarget',
                dependencies=['path/to/buildfile:targetname'],
            )

        Where `path/to/buildfile:targetname` is the dependent target address spec.

        In there is no target name component, it defaults the default target in the resulting
        Address's spec_path.

        Optionally, specs can be prefixed with '//' to denote an absolute spec path. This is
        normally not significant except when a spec referring to a root level target is needed
        from deeper in the tree. For example, in `path/to/buildfile/BUILD`:

            some_target(
                name='mytarget',
                dependencies=[':targetname'],
            )

        The `targetname` spec refers to a target defined in `path/to/buildfile/BUILD*`. If instead
        you want to reference `targetname` in a root level BUILD file, use the absolute form.
        For example:

            some_target(
                name='mytarget',
                dependencies=['//:targetname'],
            )

        The spec may be a file, such as `a/b/c.txt`. It may include a relative address spec at the
        end, such as `a/b/c.txt:original` or `a/b/c.txt:../original`, to disambiguate which target
        the file comes from; otherwise, it will be assumed to come from the default target in the
        directory, i.e. a target which leaves off `name`.
        """
        subproject = (
            longest_dir_prefix(relative_to, subproject_roots)
            if relative_to and subproject_roots
            else None
        )

        def prefix_subproject(spec_path: str) -> str:
            if not subproject:
                return spec_path
            if spec_path:
                return os.path.join(subproject, spec_path)
            return os.path.normpath(subproject)

        spec_parts = spec.rsplit(":", 1)
        path_component = spec_parts[0]
        target_component = None if len(spec_parts) == 1 else spec_parts[1]

        normalized_relative_to = None
        if relative_to:
            normalized_relative_to = (
                fast_relpath(relative_to, subproject) if subproject else relative_to
            )
        if path_component.startswith("./") and normalized_relative_to:
            path_component = os.path.join(normalized_relative_to, path_component[2:])
        if not path_component and normalized_relative_to:
            path_component = normalized_relative_to

        path_component = prefix_subproject(strip_prefix(path_component, "//"))

        return cls(path_component, target_component)

    def file_to_address(self) -> Address:
        """Converts to an Address by assuming that the path_component is a file on disk."""
        if self.target_component is None:
            # Use the default target in the same directory as the file.
            spec_path, relative_file_path = os.path.split(self.path_component)
            # We validate that this is not a top-level file. We couldn't do this earlier in the
            # AddressSpec constructor because we weren't sure if the path_spec referred to a file
            # vs. a directory.
            if not spec_path:
                raise InvalidTargetName(
                    "Top-level file specs must include which target they come from, such as "
                    f"`{self.path_component}:original_target`, but {self.path_component} did not "
                    f"have an address."
                )
            return Address(spec_path=spec_path, relative_file_path=relative_file_path)

        # The target component may be "above" (but not below) the file in the filesystem.
        # Determine how many levels above the file it is, and validate that the path is relative.
        parent_count = self.target_component.count(os.path.sep)
        if parent_count == 0:
            spec_path, relative_file_path = os.path.split(self.path_component)
            return Address(
                spec_path=spec_path,
                relative_file_path=relative_file_path,
                target_name=self.target_component,
            )

        expected_prefix = f"..{os.path.sep}" * parent_count
        if self.target_component[: self.target_component.rfind(os.path.sep) + 1] != expected_prefix:
            raise InvalidTargetName(
                "A target may only be defined in a directory containing a file that it owns in "
                f"the filesystem: `{self.target_component}` is not at-or-above the file "
                f"`{self.path_component}`."
            )

        # Split the path_component into a spec_path and relative_file_path at the appropriate
        # position.
        path_components = self.path_component.split(os.path.sep)
        if len(path_components) <= parent_count:
            raise InvalidTargetName(
                "Targets are addressed relative to the files that they own: "
                f"`{self.target_component}` is too far above the file `{self.path_component}` to "
                "be valid."
            )
        offset = -1 * (parent_count + 1)
        spec_path = os.path.join(*path_components[:offset]) if path_components[:offset] else ""
        relative_file_path = os.path.join(*path_components[offset:])
        target_name = os.path.basename(self.target_component)
        return Address(spec_path, relative_file_path=relative_file_path, target_name=target_name)

    def dir_to_address(self) -> Address:
        """Converts to an Address by assuming that the path_component is a directory on disk."""
        return Address(spec_path=self.path_component, target_name=self.target_component)

class Address(EngineAwareParameter):
    """A target address.

    An address is a unique name for a `pants.engine.target.Target`, and optionally a particular file
    that it owns.

    While not their only use, a noteworthy use of addresses is specifying
    target dependencies. For example:

        some_target(
            name='mytarget',
            dependencies=['path/to/buildfile:targetname'],
        )

    Where `path/to/buildfile:targetname` is the dependent target address.
    """

    def __init__(
        self,
        spec_path: str,
        *,
        relative_file_path: Optional[str] = None,
        target_name: Optional[str] = None,
    ) -> None:
        """
        :param spec_path: The path from the build root to the directory containing the BUILD file
          for the target.
        :param relative_file_path: The relative path from the spec_path to an addressed file,
          if any. Because files must always be located below targets that apply metadata to
          them, this will always be relative.
        :param target_name: The name of the target applying metadata to the file, defined in a
          BUILD file in the spec_path directory, or None if this path refers to the default
          target in that directory.
        """
        self.spec_path = spec_path
        self._relative_file_path = relative_file_path

        # If the target_name is the same as the default name would be, we normalize to None.
        self._target_name: Optional[str]
        if target_name and target_name != os.path.basename(self.spec_path):
            banned_chars = BANNED_CHARS_IN_TARGET_NAME & set(target_name)
            if banned_chars:
                raise InvalidTargetName(
                    f"The target name {target_name} (defined in directory {self.spec_path}) "
                    f"contains banned characters (`{'`,`'.join(banned_chars)}`). Please replace "
                    "these characters with another separator character like `_` or `-`."
                )
            self._target_name = target_name
        else:
            self._target_name = None

        self._hash = hash((self.spec_path, self._relative_file_path, self._target_name))
        if PurePath(spec_path).name.startswith("BUILD"):
            raise InvalidSpecPath(
                f"The address {self.spec} has {PurePath(spec_path).name} as the last part of its "
                f"path, but BUILD is a reserved name. Please make sure that you did not name any "
                f"directories BUILD."
            )

    @property
    def is_file_target(self) -> bool:
        return self._relative_file_path is not None

    @property
    def is_default_target(self) -> bool:
        """True if this is address refers to the "default" target in the spec_path.

        The default target has a target name equal to the directory name.
        """
        return self._target_name is None

    @property
    def filename(self) -> str:
        if self._relative_file_path is None:
            raise ValueError("Only a file Address (`self.is_file_target`) has a filename.")
        return os.path.join(self.spec_path, self._relative_file_path)

    @property
    def target_name(self) -> str:
        if self._target_name is None:
            return os.path.basename(self.spec_path)
        return self._target_name

    @property
    def spec(self) -> str:
        """The canonical string representation of the Address.

        Prepends '//' if the target is at the root, to disambiguate root-level targets
        from "relative" spec notation.

        :API: public
        """
        prefix = "//" if not self.spec_path else ""
        file_portion = f"{prefix}{self.spec_path}"
        if self._relative_file_path is not None:
            file_portion = os.path.join(file_portion, self._relative_file_path)

        # Relativize the target name to the dirname of the file.
        parent_prefix = (
            "../" * self._relative_file_path.count(os.path.sep) if self._relative_file_path else ""
        )
        if self._target_name is None and not parent_prefix:
            return file_portion
        target_name = self._target_name or os.path.basename(self.spec_path)
        return f"{file_portion}:{parent_prefix}{target_name}"

    @property
    def path_safe_spec(self) -> str:
        """
        :API: public
        """
        if self._relative_file_path:
            parent_count = self._relative_file_path.count(os.path.sep)
            parent_prefix = "@" * parent_count if parent_count else "."
            file_portion = f".{self._relative_file_path.replace(os.path.sep, '.')}"
        else:
            parent_prefix = "."
            file_portion = ""
        if parent_prefix == ".":
            target_portion = f"{parent_prefix}{self._target_name}" if self._target_name else ""
        else:
            target_name = self._target_name or os.path.basename(self.spec_path)
            target_portion = f"{parent_prefix}{target_name}"
        return f"{self.spec_path.replace(os.path.sep, '.')}{file_portion}{target_portion}"

    def maybe_convert_to_build_target(self) -> Address:
        """If this address is for a file target, convert it back into its BUILD target.

        Otherwise, return itself unmodified.
        """
        if not self.is_file_target:
            return self
        return self.__class__(self.spec_path, relative_file_path=None, target_name=self.target_name)

    def __eq__(self, other):
        if not isinstance(other, Address):
            return False
        return (
            self.spec_path == other.spec_path
            and self._relative_file_path == other._relative_file_path
            and self._target_name == other._target_name
        )

    def __hash__(self):
        return self._hash

    def __repr__(self) -> str:
        return f"Address({self.spec})"

    def __str__(self) -> str:
        return self.spec

    def __lt__(self, other):
        return (self.spec_path, (self._relative_file_path or ""), (self._target_name or "")) < (
            other.spec_path,
            (other._relative_file_path or ""),
            (other._target_name or ""),
        )

    def debug_hint(self) -> str:
        return self.spec

@dataclass(frozen=True)
class BuildFileAddress:
    """Represents the address of a type materialized from a BUILD file.

    TODO: This type should likely be removed in favor of storing this information on Target.
    """

    address: Address
    # The relative path of the BUILD file this Address came from.
    rel_path: str


from flask import jsonify, request
from flask_restful import Api, Resource, reqparse, abort
from src.services.CourseService import *

from src import api

@api.route('/courses', methods=['GET'])
def get_all_courses():
    args = request.args

    return jsonify(get_all(args))

@api.route('/course/<id>', methods=['GET', 'PATCH', 'DELETE'])
def one_course(id):
    course_id = id
    if request.method == 'DELETE':
        return jsonify(delete_one(course_id))
    if request.method == 'PATCH':
        return jsonify(update_one(course_id,request.json))
    if request.method == 'GET':
        return jsonify(get_one(course_id))

@api.route('/course', methods=['POST'])
def create_course():
    return jsonify(create_one_(request.json))


import os
import glob
import gzip
import matplotlib
matplotlib.use('TkAgg')
from matplotlib import pylab as plt
# 读取.nii文件
import nibabel as nib
from nibabel import nifti1
from nibabel.viewers import OrthoSlicer3D
import numpy as np

# 查看当前文件路径
path = '/data/data'
file_paths = glob.glob(os.path.join(path, '*.gz'))
print(file_paths)

# 解压数据文件，数据文件格式为.nii.gz
for file in file_paths:
    print(file)
    file = file.split('/')[-1]
    print(file)
    new_file_name = file.replace(".gz", "")
    print(new_file_name)
    # 开始解压
    g_file = gzip.GzipFile(file)
    print(g_file)
    #读取解压后的文件，并写入去掉后缀名的同名文件（即得到解压后的文件）
    new_file_name = os.path.join(path, new_file_name)
    open(new_file_name, "wb+").write(g_file.read())
    g_file.close()

# 读取数据文件
file1 = 'tr_im.nii'
file2 = 'tr_mask.nii'
file3 = 'val_im.nii/val_im.nii'
files = [file1, file2]
train_data_dir = 'data/data/train/data'
valid_data_dir = 'data/data/valid/data'

# 提取训练集和验证集的数据
for file in files:
    img = nib.load(file)
    width,height,queue=img.dataobj.shape
    j = 0
    for i in range(0, queue, 1):
        img_arr = img.dataobj[:,:,i]
        if i<70:
            # cur_file = '/%s.npy'%(i+1)
            filename = train_data_dir + cur_file
            np.save('train/data/%s.npy'%(i+1), img_arr)
        else:
            filename = valid_data_dir + cur_file
            np.save('valid/data/%s.npy'%(j+1), img_arr)
            j = j+1
        print(img_arr)
    print (img)

# 提取训练集和验证集的label
img_label = nib.load(file3)
width,height,queue=img_label.dataobj.shape
### 编写label
def convert(array):
    a = np.copy(array)
    a[a!=0] = 3
    a[a==0] = 1
    a[a==3] = 0

    b = np.copy(array)
    b[b!=1] = 3
    b[b==3] = 0

    c = np.copy(array)
    c[c!=2] = 3
    c[c==2] = 1
    c[c==3] = 0

    new_array = np.concatenate((a,b,c),axis=0).reshape(3,512,512)
    new_array.shape
    # (3, 512, 512)
    return new_array

j = 0
for i in range(0, queue, 1):
    img_arr = img_label.dataobj[:,:,i]
    new_arr = convert(img_arr)
    if i<70:
        cur_file = '/%s.npy'%(i+1)
#         filename = train_data_dir + cur_file
        np.save('train/label/%s.npy'%(i+1), new_arr)
    else:
#         filename = valid_data_dir + cur_file
        np.save('valid/label/%s.npy'%(j+1), new_arr)
        j = j+1
    print(img_arr)


#  F28PL 2015/2016
#  Labs H- (http://www.macs.hw.ac.uk/~gabbay/F28PL/labs.html)
#  Christian Gregg
#  Lab Helpers:
#    Helen Rankin - hr80@hw.ac.uk
#    Pierre - pl96@hw.ac.uk
#    Andrea - att31@hw.ac.uk

###################################################################
# Question H (Python lists, polymorphism, recursion, and reduce). #
###################################################################

#   Make sure to test all your programs below, including on [].

#   Using iteration (a for-next or a while loop or similar), write a program mult1 that inputs a list of integers or reals, and returns their product (the result of multiplying all the numbers together). mult1 should return 1 for the empty list, because 1 is the multiplicative unit. Thus
#       mult1([]) calculates 1, and
#       mult1([1,2,3,10]) calculates 60.
def mult1(list):
    sum = 1
    for x in list:
        sum *= x
    return sum
#   Using recursion, write a program mult2 that inputs a list of integers and
#   returns their product (warning: this question is very slightly harder than
#   it seems; make sure to test your answer).
def mult2(list):
    if list:
        return list[0]*(mult2(list[1:]))
    else:
        return 1
#   Using range or otherwise, write a one-line piece of code that breaks mult2
#   (i.e. make it crash when it “should” return a value). Why did this happen?
mult2(range(5)) # range creates an iterator, not a list, iterators do not have
                # the random access property ie you cannot perform i[j] on
                # an iterator i to access its jth element, because that 'element'
                # has not yet been computed, iterators do not actually have 'elements'.

#   Using Python lambda and reduce (see specifically the example on line 2 of
#   this link (https://docs.python.org/3/library/functools.html?highlight=reduce#functools.reduce))
#   , write a program mult3 that inputs a list of integers and returns their product.

import functools
def mult3(l):
    if l == []:
        return 0
    return functools.reduce(lambda x, y: x*y, l)
#  What do mult1, mult2, and mult3 return if applied to [1.0,2,3.0,10]?
#  What would corresponding programs do if they were written in ML and applied to such an input?
#  What does this tell us about ML vs Python typing?

#       All three return 60.0.
#       ML would return a type error when trying to create a list of ints and reals to input.
#       Python typing is less strict, and python allows lists containing different types

#   Order mult1, mult2, and mult3 from slowest to fastest and explain your order.
# TODO
#   An issue with the programs above is that they only work for numbers, however,
#   there is no reason in principle they should not also work with strings or
#   lists (by concatenation). Note that in Python you can check whether a
#   variable x has type str (for example) with if type(x) is str:
#   print("x is a string!"), and similarly for lists if type(x) is
#   list: print("x is a list!").
#   Write a program multpoly that outputs the product of a list of numbers,
#   and the concatenation (i.e. chaining) of a list of strings or lists. So
#       multpoly([1,2,3,10]) calculates 60, and
#       multpoly(["1","2","3","10"]) calculates "12310", and
#       multpoly([[1],[2,3],[10]]) calculates [1,2,3,10].
#   We do not care what multpoly calculates on the empty list. We do care about
#   a clear and elegant implementation.
def multpoly(l):
    if type(l[0]) is int:
        product = 1
        for x in l:
            product *= x
        return product
    elif type(l[0]) is str:
        concat = ""
        for x in l:
            concat += x
        return concat
    elif type(l[0]) is list:
        concat =[]
        for x in l:
            concat += x
        return concat
    else:
        return l

#   Using if type(x) is list or otherwise, write a recursive program flatten
#   that inputs an arbitrarily nested list structure and outputs a “flattened”
#   list consisting of the list of all the data in the list, but with any nested
#   list structure removed. So for instance,
#       flatten([]) returns [], and
#       flatten([["hi"],5]) returns ["hi",5], and
#       flatten([[[],[["hello"],[" "]],[1],[[["world"]],[]]],"!"]) returns ["hello"," ",1,"world!"].
def flatten(arg):
    if arg == []:
        return arg
    elif type(arg[0]) == list:
        return flatten(arg[0]) + flatten(arg[1:])
    else:
        return [arg[0]] + flatten(arg[1:])

############################
# Question I (More Python) #
############################

#   Write a program factors that inputs a number n, which you may assume is
#   greater than 2, and outputs the list of its prime factors (prime numbers
#   are 2, 3, 5, 7, 11, 13, 17, and so on). You may assume basic arithmetic
#   operations, including modulus n % k and divisibility. However, if you find a
#   library call “prime_factors_of”, you can’t use that, sorry.
def factors(n):
    current_n = n
    factor = 0
    for div in range(2, current_n+1):
        if current_n % div == 0:
            factor = div
            current_n //= div
            break
    if n == 1:
        return []
    elif factor == n:
        return [n]
    else:
        return [factor]+factors(current_n)

#   Write a program largest that inputs a nonempty list of numbers and outputs
#   its greatest element. You do not need to worry about error-handling,
#   e.g. if the input is the empty list.
def largest(arg):
    largest = arg[0]
    for i in arg:
        if i > largest:
            largest = i
    return largest

#   Write a program largest_factor that inputs a number n and outputs its
#   largest prime factor.
def largest_factor(n):
    return largest(factors(n))

#   Write a program firstbigfib that inputs a number n and outputs the index of
#   the first Fibonacci number to contain n digits.
def fib():
    a, b = 0, 1
    while True:
        yield a
        a, b = b, a + b

def firstbigfib(n):
    index = 0
    fibgen = fib()
    current = next(fibgen)
    while n > len(str(abs(current))):
        current = next(fibgen)
        index += 1
    return index

#   Write a program firstbigf that inputs a number n and a function f mapping
#   numbers to numbers, and outputs the least strictly positive number i
#   (so i is in [1,2,3,...]) such that f(i) contains n digits. So for example
#   if f=lambda x:10**x then firstbigf f 10 should calculate 9 (because 10**9,
#   which is equal to 1000000000, has ten digits).
def firstbigf(f,n):
    i = 1
    current = f(1)
    while n > len(str(abs(current))):
        i += 1
        current = f(i)
    return i
#   Suggest, in very general terms, how we might optimise firstbigf to run on
#   an m-core architecture where m>1, with a roughly m-fold speedup.

#   A Pythagorean triple is a 3-tuple of strictly positive numbers (x,y,z)
#   such that z*z=x*x+y*y. Using Python generators (page 97 of these slides or
#   the generator that yields items example here), write a Python generator
#   function triples() to generate all Pythagorean triples. So:
#       x=triples() and then next(x) should return a first triple, and then
#       next(x) should return a second triple, and then
#       next(x) should return a third one, and so on.
# TODO: Need to find an algorithm that created triples in order
def triples():
    n, m = 1, 2
    while True:
        a = (m*m) - (n*n)
        b = 2 * n * m
        c = (m*m) + (n*n)
        yield (a, b, c)
        m = m+1

#   Test your program further by running it on list(itertools.islice(triples(), 1, 300))
#   (using the Internet or otherwise, make sure you understand what this does;
#   you may need to import itertools).

#####################################################
# Question J (Python lists, sets, and dictionaries) #
#####################################################

# State what the following Python programs calculate, and explain why:
#   1. { 0:"zero", 1:"one", 2:"two" }[1]
#       Calculates the value that is associsted with the key 1
#   2. range(1000)[0]
#       Calculates the first (0th) integer calculated by range()
#   3. for x in range(1,1000): print(list(enumerate(range(x)))[0][0])
#       the program prints out the print out the first element of the first element
#       in the enumerated list of the range 0 to x-1, 999 times. (just prints 0 999 times).
#
#       >>> list(enumerate(range(10)))
#           [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8), (9, 9)]
#       >>> list(enumerate(range(10)))[0]
#           (0, 0)
#       >>> list(enumerate(range(10)))[0][0]
#
#   4. "hello world"[:-2]
#       returns the string with all the characters bar the last 2, ie each characters
#       is indexed, it returns all the characters from index 0 to index -3 (index -2 is not inclusive)

# Explain the difference in behaviour between the following two code snippets:
#   x=[""] # Snippet 1
#   for i in x: x.append("")
#   x=[""] # Snippet 2
#   for i in x[:]: x.append("")
#
#       Snippet 1 will append "" to the list x infinetly, as it is iterating over the list x
#       which is being updated to add a new object to it on every iteration, ie the for loop has no end.
#
#       Snippet 2 on the other hand will only append "" to the list once, as at the beginning there is
#       only one object in list x, this is because the for loop is iterating over a copy of x that is
#       created at the the beginning of the iteration (it iterates over that value returned by x[:])
#       Snippet 2 will double the number of objects in the list.

# Explain the behaviour of the following code snippet:
#   x=[[]] ; x[0].extend(x)
#   while True:
#       print(x)
#       x = x[0]
#
#       By extending x[0] by x, x[0] is extended by itself, this means x[0] contains a reference to
#       itself. (it is recursive, x[0] is equal to x[0] which is equal to x[0] and so on... )
#       Because x[0] contains a refernce to itself, assigning x to x[0] is eqivalent to assigning x to itself.
#       This can be seen by adding a call to id(x) underneath print(x), which shows that
#       after the first iteration, the element being printed is the same (held at the same point in memory).

# The following Python program returns an error.
#   {[0]:0}[[0]]
# Explain what the error is and why it arises.
#       The error is caused due to the fact that the keys in a dictionary must be imutable,
#       lists are mutable objects.
# Suggest how to fix it.
#       Use tuples, which are immutable instead of lists. ie {(0):0}[(0)]

# Using list comprehensions and lambda, write a program that if given a number n, calculates the first n even numbers (starting from 0).
[x for x in list(range(n*2)) if x % 2 == 0]

# Using list comprehensions and lambda, write a program that if given a list of numbers l, will return the sublist of those numbers in l that are divisible by 3.
[x for x in l if x % 3 == 0 ]
# The zip function inputs two lists (which you may assume have the same length) and outputs the list of pairs of the lists ‘zipped’ together. For instance, zip([1,2,3],["one","two","three"]) returns [(1,"one"),(2,"two"),(3,"three")]. Implement zip yourself, from first principles, using list comprehension (so not using a for-next loop and the append method; too easy).
def my_zip(list1,list2):
    return [(x,y) for x in list1 for y in list2 if list1.index(x) == list2.index(y)]

# The following Python programs calculates the first ten squares and the first ten pentagonal numbers:
#   squares = [x*x for x in range(10)]
#   pentagonals = [x*(3*x-1)//2 for x in range(10)]
#   (For this question, 0 is a square and a pentagonal number.)
# Based on those examples, and using list comprehensions if possible, write a Python program that calculates a list of numbers that are both square and pentagonal (up to some computational bound, i.e. the program is allowed to stop after a while). Test your program by calculating the first four such numbers.

# TODO make this better ffs
squares = [x*x for x in range(100000)]
pentagonals = [x*(3*x-1)//2 for x in range(100000)]
square_pent = [x for x in squares if (x in pentagonals)]

# Nonnegative integers can be implemented in Python by nest(0)=[] and nest(n+1)=[nest(n)]. Write programs nest and unnest which translate between nonnegative integers and the nesting implemenation.
def nest(n):
    if n == 0:
        return []
    else:
        return [nest(n-1)]

def unnest(n):
    if n == []:
        return 0
    else:
        return 1 + unnest(n[0])
# We do not care what happens at incorrect input. Implement addition and multiplication as direct functions on this implementation (i.e. not by untranslating, adding the numbers, and retranslating).
def add(x, y):
    if y == []:                 # if y is 0 return x (nothing added)
        return x
    else:
        return add([x], y[0])  # call add with x+1 and y-1

def multiply_help(n1,n2,n3):
    if n1 == [] or n2 == []:
        return []
    elif n2 == [[]]:
        return n1
    else:
        return multiply_help(add(n1,n3), n2[0],n3)

def multiply(n1,n2):
    return multiply_help(n1,n2,n1)
#(The student who finds this kind of thing amusing, might be interested in the surreal numbers.)


import datetime

DATE_FORMAT = '%Y-%m-%d'

def date_range(start, end=None, days=None, day_step=1, include_end=True):
    assert end or days, 'one of "end" and "days" should be given'
    assert not (end and days), 'at most one of "end" and "days" could be given'

    start = datetime.datetime.strptime(start, DATE_FORMAT).date()
    if end:  # and not days
        end = datetime.datetime.strptime(end, DATE_FORMAT).date()
        days = (end - start).days
        if include_end:
            days += 1

    return [start + datetime.timedelta(days=x) for x in range(0, days, day_step)]

def test():
    print(date_range('2018-07-02', days=5, day_step=1))

    print(date_range('2018-07-02', end='2018-08-24', day_step=1))

    print(date_range('2018-07-02', end='2018-08-23', day_step=3))
    print(date_range('2018-07-02', end='2018-08-24', day_step=3))
    print(date_range('2018-07-02', end='2018-08-25', day_step=3))

    # bad
    # print(range_date('2018-07-02', end='2018-08-24', days=4, day_step=1))
    # print(range_date('2018-07-02', day_step=1))


# -*- coding: utf-8 -*-
from django.db import models

PATTERN_TYPE = (('A', u'Дуга'),
                ('At', u'Тентовая Дуга'),
                ('L', u'Петля'),
                ('Lp', u'Павлиний глаз'),
                ('Ws', u'Завиток спиралькой'),
                ('W', u'Завиток кружками'),
                ('2L', u'Двойная петля'),
                ('C', u'Сложный узор'))

ORIENT_TYPE = (('U', u'Ульнарная'), # Завиток: Прав рука - против, Левая - по; Петля: хвостик от Бол пал
               ('R', u'Радиальная')) # Завиток: Прав рука - по, Левая - против;  Петля: хвостик к Бол пал

class FingerPrints(models.Model):
    pattern = models.CharField(max_length=2, choices=PATTERN_TYPE, verbose_name=u'Тип узора')
    orient = models.CharField(max_length=1, choices=ORIENT_TYPE, null=True, default=None, verbose_name=u'Ориентация узора')
    rc_u = models.IntegerField(default=0, blank=True, verbose_name = u'Ульнарный ГС')
    rc_r = models.IntegerField(default=0, blank=True, verbose_name = u'Радиальный ГС')

    class Meta:
        verbose_name = u'палец'
        verbose_name_plural = u'пальцы'

    def __unicode__(self):
        return "%s %s (%s, %s)" % (self.pattern, self.orient, self.rc_u, self.rc_r)

    @property
    def is_complicated(self):
        return self.pattern == 'Ws' or self.pattern == 'W' or self.pattern == '2L' or self.pattern == 'C'

class Human(models.Model):
    name = models.CharField(max_length=256, unique=True, verbose_name=u'имя')
    desc = models.TextField(blank=True, verbose_name=u'описание')
    # далее - вектора и вероятность от 0 до 10 присутствия этого вектора
    ur = models.NullBooleanField(default=None, verbose_name='У')
    an = models.NullBooleanField(default=None, verbose_name='А')
    ko = models.NullBooleanField(default=None, verbose_name='К')
    my = models.NullBooleanField(default=None, verbose_name='М')
    zr = models.NullBooleanField(default=None, verbose_name='Зр')
    zv = models.NullBooleanField(default=None, verbose_name='Зв')
    o  = models.NullBooleanField(default=None, verbose_name='Ор')
    ob = models.NullBooleanField(default=None, verbose_name='Об')

    # далее - руки от 1(Бол) до 5(Миз) пальца
    r1 = models.ForeignKey(FingerPrints, verbose_name=u'П Бол', related_name = 'r1')
    r2 = models.ForeignKey(FingerPrints, verbose_name=u'П Ук', related_name = 'r2')
    r3 = models.ForeignKey(FingerPrints, verbose_name=u'П Ср', related_name = 'r3')
    r4 = models.ForeignKey(FingerPrints, verbose_name=u'П Без', related_name = 'r4')
    r5 = models.ForeignKey(FingerPrints, verbose_name=u'П Миз', related_name = 'r5')
    l1 = models.ForeignKey(FingerPrints, verbose_name=u'Л Бол', related_name = 'l1')
    l2 = models.ForeignKey(FingerPrints, verbose_name=u'Л Ук', related_name = 'l2')
    l3 = models.ForeignKey(FingerPrints, verbose_name=u'Л Ср', related_name = 'l3')
    l4 = models.ForeignKey(FingerPrints, verbose_name=u'Л Без', related_name = 'l4')
    l5 = models.ForeignKey(FingerPrints, verbose_name=u'Л Миз', related_name = 'l5')

    class Meta:
        verbose_name = u'человек'
        verbose_name_plural = u'люди'

    def __unicode__(self):
        return self.name

    @property
    def sum_rc(self):
        s = 0
        s += self.r1.rc_u + self.r1.rc_r
        s += self.r2.rc_u + self.r2.rc_r
        s += self.r3.rc_u + self.r3.rc_r
        s += self.r4.rc_u + self.r4.rc_r
        s += self.r5.rc_u + self.r5.rc_r

        s += self.l1.rc_u + self.l1.rc_r
        s += self.l2.rc_u + self.l2.rc_r
        s += self.l3.rc_u + self.l3.rc_r
        s += self.l4.rc_u + self.l4.rc_r
        s += self.l5.rc_u + self.l5.rc_r
        return s

    @property
    def complicated_count(self):
        s = 0
        if self.r1.is_complicated: s += 1
        if self.r2.is_complicated: s += 1
        if self.r3.is_complicated: s += 1
        if self.r4.is_complicated: s += 1
        if self.r5.is_complicated: s += 1
        if self.l1.is_complicated: s += 1
        if self.l2.is_complicated: s += 1
        if self.l3.is_complicated: s += 1
        if self.l4.is_complicated: s += 1
        if self.l5.is_complicated: s += 1

        return s


import warnings
warnings.filterwarnings("ignore")
import nltkModules
import pandas as pd
import numpy as np
import seaborn as sns
from matplotlib import pyplot as plt
plt.rcParams['figure.figsize']=(15,5)
pd.set_option('display.max_columns', 100)

import nltk
from scipy import stats
import re
from  scipy.stats import ttest_ind
from wordcloud import WordCloud

from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score,roc_curve,fbeta_score,make_scorer
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from xgboost import XGBClassifier
from cf_matrix import make_confusion_matrix

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import KFold, GridSearchCV, cross_val_score,train_test_split,RandomizedSearchCV
from sklearn.pipeline import Pipeline

from sklearn.base import BaseEstimator,TransformerMixin
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.preprocessing import FunctionTransformer
from sklearn.pipeline import FeatureUnion

import emoji
from nltk.corpus import wordnet
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
import string
import unicodedata
from nltk.stem import WordNetLemmatizer
import textstat

class Thresholder:
    """This function computes Precision, Recall, Accuracy,
        F1 Score and Fbeta Score for various values for threshold
        between 0 and 1.

        Parameters
        ----------
        gs : estimator
        X : Predictor Variables
        y : Target Variable
        beta : value of beta to use for calculating Fbeta Score
        plotChart: Boolean (Default=False)

        Returns
        -------
        Dataframe with computed values for all the mectric for
        100 values of threshold.
    """

    def __init__(self,gs,X,y,plotOptimalThreshold=False,plotROCCurve=False):
        self.gs=gs

        self.y=y
        self.plotOptimalThreshold=plotOptimalThreshold
        self.plotROCCurve=plotROCCurve
        self.X=X

    def fit(self,beta=1):

        self.beta=beta
        self.metric_df=self.findOptimalThreshold(self.gs,self.X,self.y,self.beta)

        if self.plotOptimalThreshold==True:
            self.plotOptimalThreshold_(self.metric_df)

        if self.plotROCCurve==True:
            self.plotROCCurve_(self.gs,self.X,self.y)

        return self

    def findOptimalThreshold(self,gs,X,y,beta):

        predict_proba=self.gs.predict_proba(X)[:,1]

        thresholdRange=list(set(predict_proba))
#         thresholdRange=np.arange(0.1,1,0.1)

        precision_list=[]
        f1Score_list=[]
        fbetaScore_list=[]
        accuracy_list=[]
        recall_list=[]
        threshold_list=[]
        for t in thresholdRange:
            predict_newThreshold=[]
            for prob in predict_proba:
                if prob<t:
                    predict_newThreshold.append(0)
                else:
                    predict_newThreshold.append(1)
            cm=confusion_matrix(y.astype(int),predict_newThreshold)
            tp = cm[0][0]
            fp = cm[0][1]
            fn = cm[1][0]
            tn = cm[1][1]
            precision=tp/(tp+fp)
            recall=tp/(tp+fn)
            F1=2*(precision*recall)/(precision+recall)
            FBeta=((1 + beta**2) * precision * recall) / (beta**2 * precision + recall)
            accuracy=(tp + tn) /(tp + tn + fp + fn)

            precision_list.append(precision)
            recall_list.append(recall)
            f1Score_list.append(F1)
            fbetaScore_list.append(FBeta)
            accuracy_list.append(accuracy)
            threshold_list.append(t)

        metric_df=pd.DataFrame(data={'Threshold':threshold_list,
                         'Precision':precision_list,
                         'Recall':recall_list,
                         'F1':f1Score_list,
                         'FBeta':fbetaScore_list,
                         'Accuracy':accuracy_list})
        metric_df['Beta']=self.beta

        metric_df.reset_index(drop=True,inplace=True)
#         metric_df.fillna(0,inplace=True)
        return metric_df

    def plotOptimalThreshold_(self,metric_df):
            xmark=metric_df[metric_df['FBeta']==max(metric_df['FBeta'])]['Threshold'].iloc[0]
            ymark=metric_df[metric_df['FBeta']==max(metric_df['FBeta'])]['FBeta'].iloc[0]
            beta=metric_df[metric_df['FBeta']==max(metric_df['FBeta'])]['Beta'].iloc[0]

            text= "FBeta={:.4f}".format(ymark)
            sns.lineplot(x='Threshold',y='FBeta',data=metric_df,label='Fbeta Score')
            sns.lineplot(x='Threshold',y='Precision',data=metric_df,label='Precision')
            sns.lineplot(x='Threshold',y='Recall',data=metric_df,label='Recall')
            plt.axvline(x=xmark,ymin=0,ymax=1,color='red')
            plt.annotate(s=text,xy=(xmark,ymark))
            plt.ylabel('Metrics')
            title='Threshold vs Metrics'
            plt.title(title);

    def plotROCCurve_(self,gs,X,y):
            fpr, tpr, thresholds=roc_curve(y,gs.predict_proba(X)[:,1])
            self.false_positive_rate=fpr
            self.true_positive_rate=tpr
            self.thresholds=thresholds

            plt.plot(fpr, tpr)
            title='ROC curve for '+ str(gs.best_estimator_.named_steps['clf']).split('(')[0]
            plt.title(title)
            plt.xlabel("False Positive Rate")
            plt.ylabel("True Positive Rate")
            plt.plot([0, 1], [0, 1], 'k--')
            plt.show()

class TransformInput(BaseEstimator, TransformerMixin):

    def __init__(self):
        pass

    def fit(self, X, y=None):
        """
        Learn how to transform data based on input data, X.
        """
        return self

    def transform(self, X):
        """
        Transform X into a new dataset, Xprime and return it.
        """
        X=pd.DataFrame(X)
        def countCaps(comment):
            count=0
            for c in comment:
                if c.isupper():
                    count+=1
            return round(count*100/len(comment),2)
        X['%OfUpperCaseLetters']=X['Comment'].apply(countCaps)

        pattern='https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,}'
        X['NoOfURL']=X['Comment'].apply(lambda x: len(re.findall(pattern,x)))

        X['NoOfWords']=X['Comment'].apply(lambda x: (len(word_tokenize(x))))

        X['AvgSentenceLength']=X['Comment'].apply(lambda x: textstat.avg_sentence_length(x))

        X['TextStandard']=X['Comment'].apply(lambda x: textstat.text_standard(x,float_output=True))

        self.X=X
        return self.X

class GridSearcher:
    """
    This function performs grid search using given text
    vectorizer and estimator for predictor and target variables.

    Parameters
    ----------
    X: Predictor Variables
    y: target Variable
    n_splits: Number of folds. Must be at least 2.
    vectorizer: text vectorizer
    clf: estimator
    param_grid: The parameter grid to explore, as a dictionary
    mapping estimator parameters to sequences of allowed values.

    Returns
    -------
    None
    """
    def __init__(self,n_splits,vectorizer,clf,param_grid):
        self.n_splits=n_splits
        self.vectorizer=vectorizer
        self.clf=clf
        self.param_grid=param_grid

    def fit(self,X,y):
        return self.findBestEstimator(X,y,self.n_splits,self.vectorizer,self.clf,self.param_grid)

    def findBestEstimator(self,X,y,n_splits,vectorizer,clf,param_grid):

        cv = KFold(n_splits=n_splits, random_state=42, shuffle=True)

        get_text_data = FunctionTransformer(lambda x: x['Comment'], validate=False)
        get_numeric_data = FunctionTransformer(lambda x: x[['%OfUpperCaseLetters',
                                                            'NoOfURL',
                                                            'NoOfWords',
                                                            'AvgSentenceLength',
                                                            'TextStandard']], validate=False)
#        Pipepline with FeatureUnion. Feature Union joins the text features and meta features generated during Feature engineering
        pipe = Pipeline([('transformInput',TransformInput()),
                        ('features', FeatureUnion([
                                                   ('meta_features', Pipeline([
                                                                               ('selector', get_numeric_data)
                                                                               ])),
                                                   ('text_features', Pipeline([
                                                                               ('selector', get_text_data),
                                                                               ('vectorizer', vectorizer)
                                                                               ]))
                                                  ])),
                        ('clf', clf)
        ])

        gs = RandomizedSearchCV(pipe,param_grid,n_jobs=-1, cv=cv, verbose=0,return_train_score=True,scoring='roc_auc')
        gs.fit(X,y);

        self.pipe=pipe
        self.cv=cv
        self.best_params=gs.best_params_
        self.best_score=gs.best_score_
        self.fittedWinnerModel=gs
        self.X=X

        return self

def lemmatize(cleanTextList):
        wordnet_map = {"N":wordnet.NOUN, "V":wordnet.VERB, "J":wordnet.ADJ, "R":wordnet.ADV}
        pos_tagged_text = nltk.pos_tag(cleanTextList)
        lemmatizer=WordNetLemmatizer()

        return " ".join([lemmatizer.lemmatize(word, wordnet_map.get(pos[0], wordnet.NOUN)) for word, pos in pos_tagged_text])

def TextPreprocessor(comment):
    """
     This function takes list of text and performs a
        list of processing steps and return text ready
        to be fed to machine learning model.

        The list of cleaning steps:
        1. Remove URLs
        2. Convert emojis to text
        3. Convert numbers to '9's
        4. Remove non-english text
        5. Remove english stop words
        6. Remove punctuations
        7. Lemmatize text

        Parameters
        ----------
        CleanWordList : List

        Returns
        -------
        List of words
    """
    pattern1='(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?'
    comment=re.sub(pattern1,' ',comment)
    comment=re.sub('<[^<]+?>', '', comment)
    comment=comment.replace(u'\ufeff', '')
    comment=emoji.demojize(comment, delimiters=("", ""))
    comment=comment.lower()
    comment=re.sub(r'\d','9',comment)
    comment = re.sub("([^\x00-\x7F])+"," ",comment)
    wordList=word_tokenize(comment)
    wordList=[word for word in wordList if word not in stopwords.words('english')]
    wordList=[word for word in wordList if word not in string.punctuation]
    cleanedTextList=[unicodedata.normalize('NFKD',word).encode('ascii', 'ignore').decode('utf-8', 'ignore') \
                             for word in wordList]
    lemmatizedText=lemmatize(cleanedTextList)
    return lemmatizedText


import csv
import requests
from pymongo import MongoClient
import click

# globle vars
default_url = "https://docs.google.com/spreadsheets/d/1MDTs7jD1D8fSlYfp3lBzCqk8U9F27QxF1PMN40nu454/export?format=csv"

# helper method
def get_attribute(name, sample, returnType='none', nullValue='null'):

	try:
		val = sample[name]
	except KeyError:
		return nullValue

	if returnType == 'str':
		try:
			return str(val)
		except (ValueError, TypeError) as e:
			return nullValue
	elif returnType == 'float':
		try:
			return float(val)
		except (ValueError, TypeError) as e:
			return nullValue
	elif returnType == 'int':
		try:
			return int(val)
		except (ValueError, TypeError) as e:
			return nullValue
	else:
		return val

# Generate ICDO <-> NCIT mapping
# key : ICDO_morph + ICDO_topo
# values : [NCIT.CODE, NCIT.TERM]

@click.command()
@click.option('-u', '--url', default=default_url, help='Get the mapping from an url')
@click.option('-f', '--file', help='Get the mapping from a file')
@click.option('-l', '--log', type=click.File('w'), default='log.txt', help='Generate a log file')
@click.option('-d', '--database', default='arraymap', help='Use a specific database')
@click.option('-c', '--collection', default='samples', help='Use a specific collection')
def cli(url, file, log, database, collection):

	"""
	This script convert the icdo morphology and icdo topology codes of a sample to
	generate new ncit code and ncit term.
	The mapping is based on a provided file with expected structure and format.
	"""

	######################
	# init the db handler
	######################
	client = MongoClient()
	if database not in client.database_names():
		click.echo(database + ' does not exist')
		sys.exit()
	if collection not in client[database].collection_names():
		click.echo(collection + ' does not exist')
		sys.exit()
	db = client['arraymap']['samples']

	# init variables
	ctr_h = 0 # hit counter
	mappings = {}
	no_samples = db.count()

	# read in the mapping from either an url or a file
	if file :
		with open(file) as f:
			reader = csv.reader(f)
			for line in reader:
				key = line[2].strip() + line[4].strip()
				mappings[key] = [line[5], line[6]]
	else:
		csvf = requests.get(url)
		reader = csv.reader(csvf.content.decode('utf-8').splitlines(), delimiter=',')
		for line in reader:
			key = line[2].strip() + line[4].strip()
			mappings[key] = [line[5], line[6]]

	# pull mappings into a dictionary
	# key: ICDO_morphology + ICDO_topology
	# values: [NCIT.CODE, NCIT.TERM]

	# draw the progress bar
	with click.progressbar(db.find(), label='Processing', fill_char=click.style('>',fg='green'), length=no_samples) as bar:

		# Go through the database
		for sample in bar:

			# get icdo values
			icd_m = get_attribute('ICDMORPHOLOGYCODE', sample)
			icd_t = get_attribute('ICDTOPOGRAPHYCODE', sample)

			# if the sample has both icdo attributes
			if (icd_m != 'null') and (icd_t != 'null'):

				# build the key
				key = icd_m.strip() + icd_t.strip()

				# if the ICDOs match a key in the mapping
				if key in mappings:
					# assign ncit values
					ncit_code = mappings[key][0].strip()
					ncit_term = mappings[key][1].strip()

					# if the NCIT.CODE is in the neoplasm core set
					term_id = 'NCIT:' + ncit_code
					qy = client['progenetix']['classifications'].find_one({'term_id':term_id})
					if qy != None:

						# wrtie the NCIT values to database
						db.update(	{'UUID': get_attribute('UUID', sample)},
									{'$set': {	'NCIT:CODE': ncit_code,
												'NCIT:TERM': ncit_term
											 }
									}
								 )

						# update counter
						ctr_h += 1
						click.echo('HIT ' + str(ctr_h) + ':' + '\t', nl=False, file=log)
					else:
						click.echo('MISS CORE SET:' + term_id + '\t', nl=False, file=log)
						# assign basic tissue type

				else:
					click.echo('MISS MAPPING:' + '\t', nl=False, file=log)
					# assign basic tissue type

			else:
				click.echo('MISS ICD:' + '\t', nl=False, file=log)
			click.echo(icd_m + ' ' + icd_t + ' ' + get_attribute('UUID', sample), file=log)

	# close databse connection
	client.close()

# main
if __name__ == '__main__':
    cli()


from django.shortcuts import render, redirect
from django.views.generic import View
from django.utils.decorators import method_decorator
from django.contrib.auth.decorators import login_required
from .forms import RegistrationForm
from .models import Perfil

class PerfilView(View):
	@method_decorator(login_required)
	def get(self,request):
		template_name='accounts/perfil.html'
		context={
		}
		return render(request,template_name,context)

class Registro(View):
	def get(self,request):
		template_name='accounts/registro.html'
		form = RegistrationForm()
		context = {
			'form':form,
		}
		return render(request,template_name,context)

	def post(self,request):
		template_name = "accounts/registro.html"
		form = RegistrationForm(request.POST)
		if form.is_valid():
			new_user = form.save(commit=False)
			new_user.set_password(form.cleaned_data['password'])
			# perfil = Profile(instance=new_user)
			new_user.save()
			perfil_nuevo=Perfil()
			perfil_nuevo.user=new_user
			perfil_nuevo.save()
			# perfil = Profile.objects.create(user=new_user)
			return redirect('profile')
		else:
			template_name = "accounts/registro.html"
			context = {
			'form':form
			}
			return render(request,template_name,context)


#!/usr/bin/python3
""" Handle RESTful API request for states"""

from models.user import User
from api.v1.views import app_views
from flask import jsonify, abort, request, make_response
from models import storage

__CLASS = User

@app_views.route('/users',
                 methods=['GET'],
                 strict_slashes=False)
def all_users():
    """ GET ALL USERS """
    objs = storage.all(User).values()
    list_obj = []
    for obj in objs:
        list_obj.append(obj.to_dict())

    return jsonify(list_obj)

@app_views.route('/users/<user_id>',
                 methods=['GET'],
                 strict_slashes=False)
def get_users(user_id):
    """ Retrieves a specific State """
    instance = storage.get(User, user_id)
    if not instance:
        abort(404)

    return jsonify(instance.to_dict())

@app_views.route('/users/<user_id>',
                 methods=['DELETE'],
                 strict_slashes=False)
def delete_user(user_id):
    obj = storage.get(User, user_id)
    if obj:
        storage.delete(obj)
        storage.save()
        return make_response(jsonify({}), 200)
    else:
        abort(404)

@app_views.route('/users',
                 methods=['POST'],
                 strict_slashes=False)
def create_users():
    """Creates a amenity """

    if not request.get_json():
        abort(400, description="Not a JSON")
    else:
        data = request.get_json()

    if not 'name' in data:
        abort(400, description="Missing name")
    if not 'email' in data:
        abort(400, description="Missing email")
    if not 'password' in data:
        abort(400, description="Missing password")

    new_instance = User(**data)
    new_instance.save()

    return make_response(jsonify(new_instance.to_dict()), 201)

@app_views.route('/users/<user_id>',
                 methods=['PUT'],
                 strict_slashes=False)
def update_user(user_id):
    """update a State: POST /api/v1/states"""
    if not request.get_json():
        abort(400, description="Not a JSON")

    obj = storage.get(User, user_id)

    if not obj:
        abort(404)

    data = request.get_json()

    ignore = ['id', 'created_at', 'updated_at']

    data = request.get_json()
    for key, value in data.items():
        if key not in ignore:
            setattr(obj, key, value)
    storage.save()

    return make_response(jsonify(obj.to_dict()), 200)


from . import telegram_bot
from .utils import Access, Navigation, Helpers
from telebot.types import Message
from resources import strings, keyboards
from core import files, users

@telegram_bot.message_handler(content_types=['text'], func=Access.search)
def search_index(message: Message):
    user_id = message.from_user.id
    Navigation.to_search(user_id)

def search_handler(message: Message):
    user_id = message.from_user.id

    def error():
        error_message = strings.get_string('search.type_query')
        telegram_bot.send_message(user_id, error_message)
        telegram_bot.register_next_step_handler_by_chat_id(user_id, search_handler)

    def not_found():
        not_found_message = strings.get_string('search.not_found')
        telegram_bot.send_message(user_id, not_found_message)
        telegram_bot.register_next_step_handler_by_chat_id(user_id, search_handler)

    if not message.text:
        error()
        return
    if strings.get_string('back') in message.text:
        Navigation.to_main_menu(user_id)
        return
    found_files = files.search_files(message.text)
    if not found_files:
        not_found()
        return
    files_keyboard = keyboards.from_files_list_to_keyboard(found_files)
    select_message = strings.get_string('catalog.files.select')
    telegram_bot.send_message(user_id, select_message, reply_markup=files_keyboard)
    telegram_bot.register_next_step_handler_by_chat_id(user_id, file_handler)

def file_handler(message: Message):
    user_id = message.from_user.id
    user = users.get_user_by_telegram_id(user_id)

    def error():
        telegram_bot.send_message(user_id, strings.get_string('catalog.files.select'))
        telegram_bot.register_next_step_handler_by_chat_id(user_id, file_handler)

    if not message.text:
        error()
        return
    if strings.get_string('back') in message.text:
        Navigation.to_search(user_id)
    else:
        file = files.get_file_by_name(message.text)
        if not file:
            error()
            return
        Helpers.send_file(user_id, file, user)
        telegram_bot.register_next_step_handler_by_chat_id(user_id, file_handler)


from typing import List

class Solution:
    def rob(self, nums: List[int]) -> int:
        n: int = len(nums)
        if n == 0:
            return 0
        if n <= 3:
            return max(nums)

        res = nums[:]
        res[2] = nums[0] + nums[2]

        for i in range(3, n-1):
            value = max(res[i-2], res[i-3]) + nums[i]
            res[i] = value
        r1 = max(res[-2], res[-3])

        res = nums[:]
        res[-3] = nums[-1] + nums[-3]
        for i in range(n-4, 0, -1):
            value = max(res[i+2], res[i+3])
            res[i] = value
        r2 = max(res[1], res[2])
        return max(r1, r2)


import numpy
import scipy.linalg
import scipy.optimize
from joblib import Parallel, delayed
from operator import itemgetter

from moe.optimal_learning.python.geometry_utils import ClosedInterval
from moe.optimal_learning.python.python_version.domain import TensorProductDomain as pythonTensorProductDomain

__author__ = 'matthiaspoloczek'

'''
Optimization of the Hyperparameters of the MISO statistical model:
signal variance and length scales of the SE kernel,
and observational noise parameter.
'''

def covariance(hyper_without_noise, lengths_sq_0, point_one, point_two):
    '''Calculate the covariance between two points (not accounting for noise) under the MISO model'''
    # sigma = numpy.array([[ hyper_without_noise[-4], hyper_without_noise[-3] ],
    #                      [ hyper_without_noise[-2], hyper_without_noise[-1] ]])
    # phi_s1 = numpy.array([ 1, point_one[0] ])
    # phi_s2 = numpy.array([ 1, point_two[0] ])
    # first = numpy.dot(phi_s1, sigma)
    # ker_s = numpy.dot(first, phi_s2)
    ker_s = hyper_without_noise[-4] + hyper_without_noise[-3] * point_two[0] + \
            hyper_without_noise[-2] * point_two[0] + hyper_without_noise[-1] * point_two[0] * point_two[0]
    # print(point_two[0])
    # -------------------- Matern52 -------------------- #
    norm_x_sq = numpy.divide( numpy.power(point_two[1:] - point_one[1:], 2.0), lengths_sq_0[:4] ).sum()
    matern_norm_x = numpy.sqrt(5) * numpy.sqrt(norm_x_sq)
    norm_x_sum = 1.0 + matern_norm_x + 5.0/3.0*norm_x_sq
    result = hyper_without_noise[0] * ker_s * norm_x_sum * numpy.exp(-matern_norm_x)
    # -------------------- exponential -------------------- #
    # exp_value = numpy.exp(-0.5 * numpy.divide(numpy.power(point_two[1:] - point_one[1:], 2.0), lengths_sq_0[1:]).sum())
    # result = hyperparameters_without_noise[0] * exp_value * poly_s
    # -------------------- all exponential -------------------- #
    # exp_value = numpy.exp(-0.5 * numpy.divide(numpy.power(point_two - point_one, 2.0), lengths_sq_0).sum())
    # result = hyperparameters_without_noise[0] * exp_value
    return result

def compute_covariance_matrix(hyperparameters_without_noise, noise_hyperparameters, points_sampled):
    '''
    Compute the covariance matrix of the points that have been sampled so far
    Args:
        dim: 1 + the dimension of the search space that the points are from
        hyperparameters_without_noise: the signal variance followed by the length scales
        noise_hyperparameters: the hyperparameters of the observational noise
        points_sampled: the points that have been sampled so far
    Returns: the covariance matrix
    '''
    lengths_sq_0 = numpy.power(hyperparameters_without_noise[1:], 2.0)
    cov_mat = numpy.zeros((points_sampled.shape[0], points_sampled.shape[0]), order='F')
    for j, point_two in enumerate(points_sampled):
        for i, point_one in enumerate(points_sampled[j:, ...], start=j):
            cov_mat[i, j] = covariance(hyperparameters_without_noise, lengths_sq_0, point_one, point_two)
            if i != j:
                cov_mat[j, i] = cov_mat[i, j]

        # add noise on the main diagonal
        cov_mat[j, j] += numpy.power(noise_hyperparameters[0], 2.0)
        # We have to square the noise hyperparameter. This is how it is done in MOE:
        # noise_variance: the ``\sigma_n^2`` (noise variance) associated w/observation, points_sampled_value.
        # In build_covariance_matrix() in moe/optimal_learning/python/python_version/python_utils.py:
        # noise_variance: i-th entry is amt of noise variance to add to i-th diagonal entry; i.e., noise measuring i-th point
    return cov_mat

def compute_hyper_prior(problem_search_domain, points_sampled, points_sampled_value):
        # print(problem_search_domain[1:])
        prior_mean = []
        # print points_sampled
        prior_mean = numpy.concatenate(( [max(0.01, numpy.var(points_sampled_value))],
                                         [ (d[1]-d[0]) for d in problem_search_domain[1:] ] ))
        (lb,ub) = problem_search_domain[0]
        for _ in range(4):
            prior_mean = numpy.concatenate(( prior_mean, [ub-lb] ))
        # print(prior_mean)
        prior_mean = numpy.array(prior_mean)
        prior_sig = numpy.diag(numpy.power(prior_mean/2., 2.0)) # variance
        hyper_prior = NormalPrior(prior_mean, prior_sig)

        return hyper_prior

class NormalPrior(object):
    ''' class for a multivariate normal prior '''
    def __init__(self, mu, sig):
        self._mu = mu
        self._sig_inv = numpy.linalg.inv(sig)

    def compute_log_likelihood(self, x):
        x_mu = (x - self._mu).reshape((-1, 1))
        return -0.5 * numpy.dot(x_mu.T, numpy.dot(self._sig_inv, x_mu))

    def compute_grad_log_likelihood(self, x):
        x_mu = (x - self._mu).reshape((-1, 1))
        return -0.5 * numpy.dot(self._sig_inv + self._sig_inv.T, x_mu).flatten()

def compute_log_likelihood(K_chol, K_inv_y, points_sampled_value):
    log_marginal_term1 = -0.5 * numpy.inner(points_sampled_value, K_inv_y)
    # print(log_marginal_term1)

    log_marginal_term2 = -numpy.log(K_chol[0].diagonal()).sum()

    log_marginal_term3 = -0.5 * numpy.float64(points_sampled_value.size) * numpy.log(2.0 * numpy.pi)
    return log_marginal_term1 + log_marginal_term2 + log_marginal_term3

def compute_log_likelihood_K(K, points_sampled_value):
    yk = numpy.inner(points_sampled_value, scipy.linalg.inv(K))
    log_marginal_term1 = -0.5 * numpy.inner(yk, points_sampled_value.T)
    # print(log_marginal_term1)

    log_marginal_term2 = -0.5 * numpy.log(K.diagonal()).sum()

    log_marginal_term3 = -0.5 * numpy.float64(points_sampled_value.size) * numpy.log(2.0 * numpy.pi)
    return log_marginal_term1 + log_marginal_term2 + log_marginal_term3

def hyper_opt(points_sampled, points_sampled_value, init_hyper, hyper_bounds, approx_grad, hyper_prior=None):
    '''
    Hyperparameter optimization
    Args:
        dim: 1 + the dimension of the search space that the points are from
        points_sampled:
        points_sampled_value:
        init_hyper: starting point of hyperparameters
        hyper_bounds: list of (lower_bound, upper_bound)
        approx_grad: bool
        hyper_prior:
    Returns: (optimial hyper, optimal marginal loglikelihood, function output)
    '''
    # hyper_bounds = [(0.0,numpy.inf) for hyperparameter in init_hyper]

    def obj_func(x):
        '''
        The negative marginal loglikelihood for hyperparameters x
        x: the hyperparameters, including noise hyperparameters appended to the hyperparameters of the kernels
        Returns: The negated value of the marginal loglikelihood at hyperparameters x
        '''
        # split x into hyperparameters and noise_hyperparameters
        # there are dim signal variances and length scales
        hyperparameters_without_noise = x[:-1]
        noise_hyperparameters = x[[-1]]

        # # compute the parts of the marginal loglikelihood
        covariance_matrix = compute_covariance_matrix(hyperparameters_without_noise, noise_hyperparameters, points_sampled)
        eigen = scipy.linalg.eigvalsh(covariance_matrix)
        if numpy.any(eigen<=0):
            print('not positive definite')
            if hyper_prior is not None:
                # This BFGS minimizes but we wish to maximize, thus negate the log marginal likelihood + log prior
                return -1.0 * ( compute_log_likelihood_K(covariance_matrix, points_sampled_value)
                               + hyper_prior.compute_log_likelihood(hyperparameters_without_noise) )
            else:
                # This BFGS minimizes but we wish to maximize, thus negate the log marginal likelihood
                return -1.0 * compute_log_likelihood_K(covariance_matrix, points_sampled_value)
        else:
            print('positive definite, take K_chol')
            K_chol = scipy.linalg.cho_factor(covariance_matrix, lower=True, overwrite_a=True)
            K_inv_y = scipy.linalg.cho_solve(K_chol, points_sampled_value)
            if hyper_prior is not None:
                # This BFGS minimizes but we wish to maximize, thus negate the log marginal likelihood + log prior
                return -1.0 * ( compute_log_likelihood(K_chol, K_inv_y, points_sampled_value)
                               + hyper_prior.compute_log_likelihood(hyperparameters_without_noise) )
            else:
                # This BFGS minimizes but we wish to maximize, thus negate the log marginal likelihood
                return -1.0 * compute_log_likelihood(K_chol, K_inv_y, points_sampled_value)

    best_hyper = scipy.optimize.fmin_l_bfgs_b(func=obj_func, x0=init_hyper, args=(), approx_grad=True,
                                              bounds=hyper_bounds, m=10, factr=10.0, pgtol=0.01,
                                              epsilon=1e-08, iprint=-1, maxfun=15000, maxiter=100, disp=0, callback=None)
    return best_hyper

def generate_hyperbounds(problem_search_domain, upper_bound_noise_variances, upper_bound_signal_variances=1000.):
    '''
    Bounds on each hyperparameter to speed up BFGS. Zero is a trivial lower bound. For the upper bound keep in mind that
    the parameters will be squared, hence in particular the bounds for variances should cover all reasonable values.
    Args:
        num_IS: number of IS, including truth as IS 0
        problem_search_domain: the search domain of the problem, usually problem.obj_func_min._search_domain
        upper_bound_signal_variances: an upper bound on the signal variance and the sample/noise variance
    Returns: bounds as an array of pairs (lower_bound, upper_bound)
    '''
    general_lower_bound = 1e-1 # set a general lower bound for hyperparameters -- allowing 0. seems to cause singularities
    noise_var_lower_bound = 1e-2 # SPEARMINT uses a tophat prior with lower bound 0.01 on the squared hyperparameters for length scales

    # arbitrary large value for signal variance
    hyper_bounds = [(general_lower_bound, upper_bound_signal_variances)]
    # trivial upper bounds on length scales. Remember that they will be squared.
    for (lb,ub) in problem_search_domain[1:]:
        hyper_bounds = numpy.append( hyper_bounds, [(general_lower_bound, (ub-lb))] )
    # bounds on hyper of s.
    (lb,ub) = problem_search_domain[0]
    for _ in range(4):
        hyper_bounds = numpy.append( hyper_bounds, [(general_lower_bound, (ub-lb))] )

    # add bounds on noise/sample variances
    hyper_bounds = numpy.append(hyper_bounds, numpy.array([ [noise_var_lower_bound, upper_bound_noise_variances] ]))
    # print(hyper_bounds)

    return hyper_bounds.reshape(-1,2) # reshape so that the array becomes an array of pairs (lower_bound, upper_bound)

def optimize_hyperparameters(problem_search_domain, points_sampled, points_sampled_value,
                             upper_bound_noise_variances = 10., consider_small_variances = True,
                             hyper_prior = None, num_restarts = 32, num_jobs = 16):
    '''
    Fit hyperparameters from data using MLE or MAP (described in Poloczek, Wang, and Frazier 2016)
    :param problem_search_domain: The search domain of the benchmark, as provided by the benchmark
    :param points_sampled: An array that gives the points sampled so far. Each points has the form [IS dim0 dim1 ... dimn]
    :param points_sampled_value: An array that gives the values observed at the points in same ordering
    :param upper_bound_noise_variances: An upper bound on the search interval for the noise variance parameters (before squaring)
    :param consider_small_variances: If true, half of the BFGS starting points have entries for the noise parameters set to a small value
    :param hyper_prior: use prior for MAP estimate if supplied, and do MLE otherwise
    :param num_restarts: number of starting points for BFGS to find MLE/MAP
    :param num_jobs: number of parallelized BFGS instances
    :return: An array with the best found values for the hyperparameters
    '''
    approx_grad = True
    upper_bound_signal_variances = numpy.maximum(10., numpy.var(points_sampled_value)) # pick huge upper bounds
    hyper_bounds = generate_hyperbounds(problem_search_domain, upper_bound_noise_variances, upper_bound_signal_variances)
    hyperparam_search_domain = pythonTensorProductDomain([ClosedInterval(bd[0], bd[1]) for bd in hyper_bounds])
    hyper_multistart_pts = hyperparam_search_domain.generate_uniform_random_points_in_domain(num_restarts)

    for i in xrange(num_restarts):
        init_hyper = hyper_multistart_pts[i]

        # if optimization is enabled, make sure that small variances are checked despite multi-modality
        # this optimization seems softer than using a MAP estimate
        if consider_small_variances and (i % 2 == 0):
            init_hyper[-1] = 0.1 # use a small value as starting point for noise parameters in BFGS

        hyper_multistart_pts[i] = init_hyper

    # num_jobs
    parallel_results = Parallel(n_jobs=1)(delayed(hyper_opt)(points_sampled, points_sampled_value,
                                init_hyper, hyper_bounds, approx_grad, hyper_prior) for init_hyper in hyper_multistart_pts)
    # print min(parallel_results,key=itemgetter(1))
    best_hyper = min(parallel_results,key=itemgetter(1))[0] # recall that we negated the log marginal likelihood when passing it to BFGS
    return best_hyper

def create_array_points_sampled_noise_variance(points_sampled, noise_hyperparameters):
    points_sampled_noise_variance = []
    for point in points_sampled:
        points_sampled_noise_variance.append(noise_hyperparameters[0])
    return numpy.array(points_sampled_noise_variance)


import matplotlib.pyplot as plt
import time as t

times=[]
mistakes=0

print("This program will help you type faster. You will have to type the word 'programming' as fast as you can for five times")

input("Press enter to ccontinue.")

while len(times)<5:
    start=t.time()
    word=input("Type the given word: ")
    end=t.time()
    time_elapsed=end-start

    times.append(time_elapsed)

    if (word.lower()!= "programming"):
        mistakes += 1

print("You made",str(mistakes),"mistake(s)")
print("Okay!Now let's see your evolution chart.")

t.sleep(5)

x=[1,2,3,4,5]
y=times

plt.plot(x,y)
legends=["1","2","3","4","5"]
plt.xticks(x,legends)
plt.ylabel("Time in seconds")
plt.xlabel("Attempts")
plt.title("Typing Evolution")

plt.show()