def get_google_flights_data(origin,dest,dates,currency,non_stop=True,save_csv=True,headless=True):

    import os
    from selenium import webdriver
    from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
    from selenium.webdriver.chrome.options import Options
    from bs4 import BeautifulSoup
    import time

    import pandas as pd
    import re
    from datetime import datetime

    #Set up driver
    caps = DesiredCapabilities().CHROME
    caps["pageLoadStrategy"] = "eager"

    chrome_options = Options()
    chrome_options.add_experimental_option("prefs", {
            "download.default_directory": os.getcwd()+"\\Downloads",
            "download.prompt_for_download": False,
            "download.directory_upgrade": True,
            "safebrowsing_for_trusted_sources_enabled": False,
            "safebrowsing.enabled": False})

    if headless == True:
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')

    driver = webdriver.Chrome(executable_path=os.getcwd()+"/chromedriver.exe",desired_capabilities=caps,chrome_options=chrome_options)

    #Get code corresponding to city when all city airports are selected
    multiple_airports = {'London':'/m/04jpl', 'Paris':'/m/05qtj', 'New York':'/m/02_286', 'Rome':'/m/06c62'}
    if len(origin)>3:
        origin = multiple_airports[origin]
    if len(dest)>3:
        dest = multiple_airports[dest]

    if non_stop:
        params = 'e:1;s:0;sd:1;t:f;tt:o' #url end if non-stop is selected
    else:
        params = ';e:1;sd:1;t:f;tt:o' #url end if non-stop isn't selected

    #Create dataframes to store data for the date and all dates combined
    data = pd.DataFrame()

    for i, date in enumerate(dates):
        #Load website
        url = 'https://www.google.com/flights?hl=en' + '#flt=' + origin + '.' + dest + '.' + date + ';c:' + currency + params
        driver.get(url)
        time.sleep(5)

        soup=BeautifulSoup(driver.page_source, 'lxml')

        #Extract airline names
        airlines = []
        for airline in soup.find_all('span', attrs={'class': 'gws-flights__ellipsize'}):
            airlines.append(airline.getText())
        airlines = airlines[::2]

        #Extract Dep and Arr airports
        airports = []
        for airport_pair in soup.find_all('div', attrs={'class': 'gws-flights-results__airports flt-caption'}):
            airports.append(airport_pair.getText().strip())

        #Extract times
        times = []
        for time_ in soup.find_all('div', attrs={'class': 'gws-flights-results__times flt-subhead1'}):
            times.append(time_.getText().strip())

        #Extract prices (cheapest and other)
        prices = []
        for price in soup.find_all('div', attrs={'class': 'flt-subhead1 gws-flights-results__price gws-flights-results__cheapest-price'}):
            price = price.getText()
            price = price[re.search(r"\d", price).start():].strip()
            prices.append(price)

        for price in soup.find_all('div', attrs={'class': 'flt-subhead1 gws-flights-results__price'}) :
            price = price.getText()
            price = price[re.search(r"\d", price).start():].strip()
            prices.append(price)
        prices = prices[::2]

        data_date = pd.DataFrame()

        if airlines: #check if there are any flights on the selected date

            #Store extracted values
            data_date['Airline'] = airlines
            data_date['Airport Pair'] = airports
            data_date['Time'] = times
            data_date['Price'] = prices

            #Separate dep and arr airports
            data_date['Origin'] = data_date['Airport Pair'].str.split('–').str[0]
            data_date['Destination'] = data_date['Airport Pair'].str.split('–').str[1]
            data_date = data_date.drop(['Airport Pair'], axis=1)

            #Separate dep and arr times
            data_date['Dep Time (local)'] = data_date['Time'].str.split('–').str[0]
            data_date['Arr Time (local)'] = data_date['Time'].str.split('–').str[1]
            data_date['Arr Time (local)'] = data_date['Arr Time (local)'].str.strip()
            data_date = data_date.drop(['Time'], axis=1)

            #Add other info
            data_date['Flight Date'] = date
            data_date['Flight Date'] = pd.to_datetime(data_date['Flight Date'])
            data_date['Weekday'] = data_date['Flight Date'].dt.weekday_name
            data_date['Currency'] = currency
            data_date['Fare Extraction Date'] = datetime.today().strftime('%Y-%m-%d')
            data_date['Fare Extraction Date'] = pd.to_datetime(data_date['Fare Extraction Date'])

            #Reorder columns
            data_date = data_date[['Origin','Destination','Airline','Flight Date','Weekday','Dep Time (local)', 'Arr Time (local)', 'Price', 'Currency', 'Fare Extraction Date']]

            #Append date's results to data
            data = data.append(data_date, ignore_index=True)

        #Print progress indicator
        print(str(i+1) + '/' + str(len(dates)) + ' days scraped')

    #Save data
    if save_csv:
        origin = list(multiple_airports.keys())[list(multiple_airports.values()).index(origin)]
        filename = origin + '-' + dest + '_' + 'fares.csv'
        if os.path.exists(os.getcwd()+'/Downloads/' + filename):
            data_full = pd.read_csv(os.getcwd()+'/Downloads/' + filename)
            data_full['Flight Date'] = pd.to_datetime(data_full['Flight Date'])
            data_full['Fare Extraction Date'] = pd.to_datetime(data_full['Fare Extraction Date'])
            data_full = data_full.append(data, ignore_index=True)
            data_full.to_csv(os.getcwd() + '/Downloads/' + filename, index=False)
        else:
            data.to_csv(os.getcwd() + '/Downloads/' + filename, index=False)

    return data


from django import forms
#モデルクラスを呼出
from .models import People

#フォームクラス作成
class Votes_Form(forms.ModelForm):

    class Meta():

        model = People

        fields = ('MyName','GoodPerson','comment')

        labels = {'MyName':"名前",
                    'GoodPerson':"素敵さん",
                    'comment':"コメント",
        }


import gym
import gym_simplifiedtetris
import time

env = gym.make('simplifiedtetris-binary-v0')
obs = env.reset()

# Run 10 games of Tetris, selecting actions uniformly at random.
num_episodes = 0
while num_episodes < 100:
    env.render()
    action = env.action_space.sample()
    obs, rwd, done, info = env.step(action)
    print('Intermediary sleep')
    time.sleep(2)
    if done:
        print(f"Episode {num_episodes + 1} has terminated.")
        num_episodes += 1
        time.sleep(8)
        print('Long sleep')
        obs = env.reset()

env.close()


from aws_quota.exceptions import InstanceWithIdentifierNotFound
import typing

import boto3
import botocore.exceptions
import cachetools
from .quota_check import QuotaCheck, InstanceQuotaCheck, QuotaScope

def check_if_vpc_exists(session: boto3.Session, vpc_id: str) -> bool:
    client = session.client('ec2')
    try:
        client.describe_vpcs(VpcIds=[vpc_id])
    except botocore.exceptions.ClientError as e:
        return False
    return True

@cachetools.cached(cache=cachetools.TTLCache(1, 60))
def get_all_vpcs(session: boto3.Session) -> typing.List[dict]:
    return session.client('ec2').describe_vpcs()['Vpcs']

def get_vpc_by_id(session: boto3.Session, vpc_id: str) -> dict:
    try:
        return next(filter(lambda vpc: vpc_id == vpc['VpcId'], get_all_vpcs(session)))
    except StopIteration:
        raise KeyError

@cachetools.cached(cache=cachetools.TTLCache(1, 60))
def get_all_sgs(session: boto3.Session) -> typing.List[dict]:
    return session.client('ec2').describe_security_groups()['SecurityGroups']

def get_sg_by_id(session: boto3.Session, sg_id: str) -> dict:
    try:
        return next(filter(lambda sg: sg_id == sg['GroupId'], get_all_sgs(session)))
    except StopIteration:
        raise KeyError

@cachetools.cached(cache=cachetools.TTLCache(1, 60))
def get_all_rts(session: boto3.Session) -> typing.List[dict]:
    return session.client('ec2').describe_route_tables()['RouteTables']

def get_rt_by_id(session: boto3.Session, rt_id: str) -> dict:
    try:
        return next(filter(lambda rt: rt_id == rt['RouteTableId'], get_all_rts(session)))
    except StopIteration:
        raise KeyError

@cachetools.cached(cache=cachetools.TTLCache(1, 60))
def get_all_network_acls(session: boto3.Session) -> typing.List[dict]:
    return session.client('ec2').describe_network_acls()['NetworkAcls']

class VpcCountCheck(QuotaCheck):
    key = "vpc_count"
    description = "VPCs per Region"
    scope = QuotaScope.REGION
    service_code = 'vpc'
    quota_code = 'L-F678F1CE'

    @property
    def current(self):
        return len(get_all_vpcs(self.boto_session))

class InternetGatewayCountCheck(QuotaCheck):
    key = "ig_count"
    description = "VPC internet gateways per Region"
    scope = QuotaScope.REGION
    service_code = 'vpc'
    quota_code = 'L-A4707A72'

    @property
    def current(self):
        return len(self.boto_session.client('ec2').describe_internet_gateways()['InternetGateways'])

class NetworkInterfaceCountCheck(QuotaCheck):
    key = "ni_count"
    description = "VPC network interfaces per Region"
    scope = QuotaScope.REGION
    service_code = 'vpc'
    quota_code = 'L-DF5E4CA3'

    @property
    def current(self):
        return len(self.boto_session.client('ec2').describe_network_interfaces()['NetworkInterfaces'])

class SecurityGroupCountCheck(QuotaCheck):
    key = "sg_count"
    description = "VPC security groups per Region"
    scope = QuotaScope.REGION
    service_code = 'vpc'
    quota_code = 'L-E79EC296'

    @property
    def current(self):
        return len(self.boto_session.client('ec2').describe_security_groups()['SecurityGroups'])

class RulesPerSecurityGroupCheck(InstanceQuotaCheck):
    key = "vpc_rules_per_sg"
    description = "Rules per VPC security group"
    service_code = 'vpc'
    quota_code = 'L-0EA8095F'
    instance_id = 'Security Group ID'

    @staticmethod
    def get_all_identifiers(session: boto3.Session) -> typing.List[str]:
        return [sg['GroupId'] for sg in get_all_sgs(session)]

    @property
    def current(self):
        try:
            sg = get_sg_by_id(self.boto_session, self.instance_id)
            return len(sg['IpPermissions']) + len(sg['IpPermissionsEgress'])
        except KeyError:
            raise InstanceWithIdentifierNotFound(self)

class RouteTablesPerVpcCheck(InstanceQuotaCheck):
    key = "vpc_route_tables_per_vpc"
    description = "Route Tables per VPC"
    service_code = 'vpc'
    quota_code = 'L-589F43AA'
    instance_id = 'VPC ID'

    @staticmethod
    def get_all_identifiers(session: boto3.Session) -> typing.List[str]:
        return [vpc['VpcId'] for vpc in get_all_vpcs(session)]

    @property
    def current(self):
        if check_if_vpc_exists(self.boto_session, self.instance_id):
            return len(self.boto_session.client('ec2').describe_route_tables(Filters=[
                {
                    'Name': 'vpc-id',
                    'Values': [self.instance_id]
                }])['RouteTables'])
        else:
            raise InstanceWithIdentifierNotFound(self)

class RoutesPerRouteTableCheck(InstanceQuotaCheck):
    key = "vpc_routes_per_route_table"
    description = "Routes per Route Table"
    service_code = 'vpc'
    quota_code = 'L-93826ACB'
    instance_id = 'Route Table ID'

    @staticmethod
    def get_all_identifiers(session: boto3.Session) -> typing.List[str]:
        return [rt['RouteTableId'] for rt in get_all_rts(session)]

    @property
    def current(self):
        try:
            rt = get_rt_by_id(self.boto_session, self.instance_id)
            return len(rt['Routes'])
        except KeyError:
            raise InstanceWithIdentifierNotFound(self)

class SubnetsPerVpcCheck(InstanceQuotaCheck):
    key = "vpc_subnets_per_vpc"
    description = "Subnets per VPC"
    service_code = 'vpc'
    quota_code = 'L-407747CB'
    instance_id = 'VPC ID'

    @staticmethod
    def get_all_identifiers(session: boto3.Session) -> typing.List[str]:
        return [vpc['VpcId'] for vpc in get_all_vpcs(session)]

    @property
    def current(self):
        if check_if_vpc_exists(self.boto_session, self.instance_id):
            return len(self.boto_session.client('ec2').describe_subnets(Filters=[
                {
                    'Name': 'vpc-id',
                    'Values': [self.instance_id]
                }])['Subnets'])
        else:
            raise InstanceWithIdentifierNotFound(self)

class AclsPerVpcCheck(InstanceQuotaCheck):
    key = "vpc_acls_per_vpc"
    description = "Network ACLs per VPC"
    service_code = 'vpc'
    quota_code = 'L-B4A6D682'
    instance_id = 'VPC ID'

    @staticmethod
    def get_all_identifiers(session: boto3.Session) -> typing.List[str]:
        return [vpc['VpcId'] for vpc in get_all_vpcs(session)]

    @property
    def current(self) -> int:
        if check_if_vpc_exists(self.boto_session, self.instance_id):
            return len(self.boto_session.client('ec2').describe_network_acls(Filters=[
                {
                    'Name': 'vpc-id',
                    'Values': [self.instance_id]
                }])['NetworkAcls'])
        else:
            raise InstanceWithIdentifierNotFound(self)

class RulesPerAclCheck(InstanceQuotaCheck):
    key = "vpc_rules_per_acl"
    description = "Rules per Network ACL"
    service_code = 'vpc'
    quota_code = 'L-2AEEBF1A'
    instance_id = 'Network ACL ID'

    @staticmethod
    def get_all_identifiers(session: boto3.Session) -> typing.List[str]:
        return [acl['NetworkAclId'] for acl in get_all_network_acls(session)]

    @property
    def current(self) -> int:
        acls = get_all_network_acls(self.boto_session)
        if self.instance_id in [acl['NetworkAclId'] for acl in acls]:
            return len(next(filter(lambda acl: self.instance_id == acl['NetworkAclId'], acls))['Entries'])
        else:
            raise InstanceWithIdentifierNotFound(self)

class Ipv4CidrBlocksPerVpcCheck(InstanceQuotaCheck):
    key = "vpc_ipv4_cidr_blocks_per_vpc"
    description = "IPv4 CIDR blocks per VPC"
    service_code = 'vpc'
    quota_code = 'L-83CA0A9D'
    instance_id = 'VPC ID'

    @staticmethod
    def get_all_identifiers(session: boto3.Session) -> typing.List[str]:
        return [vpc['VpcId'] for vpc in get_all_vpcs(session)]

    @property
    def current(self) -> int:
        try:
            vpc = get_vpc_by_id(self.boto_session, self.instance_id)
            return len(list(filter(lambda cbas: cbas['CidrBlockState']['State'] == 'associated', vpc['CidrBlockAssociationSet'])))
        except KeyError:
            raise InstanceWithIdentifierNotFound(self)

class Ipv6CidrBlocksPerVpcCheck(InstanceQuotaCheck):
    key = "vpc_ipv6_cidr_blocks_per_vpc"
    description = "IPv6 CIDR blocks per VPC"
    service_code = 'vpc'
    quota_code = 'L-085A6257'
    instance_id = 'VPC ID'

    @staticmethod
    def get_all_identifiers(session: boto3.Session) -> typing.List[str]:
        return [vpc['VpcId'] for vpc in get_all_vpcs(session)]

    @property
    def current(self) -> int:
        try:
            vpc = get_vpc_by_id(self.boto_session, self.instance_id)
            if 'Ipv6CidrBlockAssociationSet' not in vpc:
                return 0

            return len(list(filter(lambda cbas: cbas['Ipv6CidrBlockState']['State'] == 'associated', vpc['Ipv6CidrBlockAssociationSet'])))
        except KeyError:
            raise InstanceWithIdentifierNotFound(self)


#!pip install PyDictionary
import inflect
p = inflect.engine()
from PyDictionary import PyDictionary
import re
import numpy
import nltk

#nltk.download('stopwords')
from nltk.corpus import stopwords
import string
import requests
import ast
from nltk.tokenize import word_tokenize
#nltk.download('punkt')
import sys
import os

# Disable
def blockPrint():
    sys.stdout = open(os.devnull, 'w')

# Restore
def enablePrint():
    sys.stdout = sys.__stdout__

dictionary = PyDictionary()
def findWords(stri):
    result = []
    data = re.split(', |_|-|!| ', stri)
    for words in data:
        blockPrint()
        synonym = dictionary.synonym(words)
        antonym = dictionary.antonym(words)
        enablePrint()
        if not antonym and not synonym:
            temp = []
        elif not synonym and antonym:
            temp = antonym
        elif not antonym and synonym:
            temp = synonym
        else:
            temp = numpy.concatenate([synonym, antonym])
        result = numpy.concatenate([result, temp])

    return result

def wikipedia_solution(wikipedia_clues):
    WIKIPEDIA_API = "https://en.wikipedia.org/w/api.php?action=query&utf8=&format=json&list=search&srlimit=50&srsearch="
    stop = stopwords.words('english') + list(string.punctuation)
    #clue_mapping = dict()
    #print("wikipedia fetching word for " + wikipedia_clues)
    if '#' in wikipedia_clues:
        wikipedia_clues = wikipedia_clues.replace("#", "")

    for sentence in wikipedia_clues:
        req = requests.get(WIKIPEDIA_API + sentence)
        wiki_json = ast.literal_eval(req.text)
        #print(wiki_json["query"])
        solutions = list(set([word for word in [[word for word in word_tokenize(info["title"].lower()) if word not in stop] for info in wiki_json["query"]["search"]] for word in word]))
    return solutions

def wikipedia_sentence_solution(wikipedia_clues):
    WIKIPEDIA_API = "https://en.wikipedia.org/w/api.php?action=query&utf8=&format=json&list=search&srlimit=50&srsearch="
    stop = stopwords.words('english') + list(string.punctuation)
    #clue_mapping = dict()
    #print(">>> STARTING WIKI FETCH sentence.....")
    if '#' in wikipedia_clues:
        wikipedia_clues = wikipedia_clues.replace("#", "")
    req = requests.get(WIKIPEDIA_API + wikipedia_clues)
    wiki_json = ast.literal_eval(req.text)
    solutions = list(set([word for word in [[word for word in word_tokenize(info["title"].lower()) if word not in stop] for info in wiki_json["query"]["search"]] for word in word]))
    return solutions

def possible_answers(clues, length):
    myarr = wikipedia_solution(clues)
    myarr = numpy.array(myarr)
    myarr = numpy.concatenate([myarr, findWords(clues)])
    myarr = numpy.concatenate([myarr, wikipedia_sentence_solution(clues)])
    finalResult = []
    returnResult = []
    for i in myarr:
    #if len(i) <= length:
       if i not in finalResult:
        finalResult.append(i.upper())
        if p.plural(i.upper()):
            finalResult.append(p.plural(i.upper()))

        if p.singular_noun(i.upper()):
            finalResult.append(p.singular_noun(i.upper()))

    for j in finalResult:
        #sorting based on length
        if len(j) == length:
            if j not in returnResult:
                returnResult.append(j)

    return returnResult


#!/usr/bin/env python3

import glob
import requests
from Bio.Blast import NCBIWWW

def build_url(database: str, sequence: str) -> str:
    """
    URL address builder.
    :param database: database to be queried
    :param sequence: query sequence
    :return: URL address
    """
    with open(sequence, 'r') as f:
        seq = f.readline()
    return 'http://www.boldsystems.org/index.php/Ids_xml?db=' + database + '&sequence=' + seq

def query_bold(database: str, sequence: str) -> str:
    """
    Function queries a database and returns text results.
    :param database: database to be queried
    :param sequence: query sequence
    :return: query text results
    """
    url = build_url(database, sequence)
    result = requests.get(url)
    return result.text

def query_ncbi(database: str, sequence: str) -> str:
    """
    Function queries a database and returns text results.
    :param database: database to be queried
    :param sequence: query sequence
    :return: query text results
    """
    with open(sequence, 'r') as f:
        seq = f.read()
    result_handle = NCBIWWW.qblast("blastn", database, seq, hitlist_size=65)
    return result_handle.read()

def store_query(database: str, sequence: str, file: str) -> None:
    """
    Function stores the results of database query in desired file.
    :param database: database to be queried
    :param sequence: query sequence
    :param file: file where the results will be stored
    """
    with open(file, 'w+') as f:
        if database in ['COX1', 'COX1_SPECIES', 'COX1_SPECIES_PUBLIC', 'COX1_640bp']:
            f.write(query_bold(database, sequence))
        else:
            f.write(query_ncbi(database, sequence))

def query_all(database: str, sequence: str, out: str) -> None:
    """
    Function queries all the files that match given pattern.
    :param database: database to be queried
    :param sequence: query sequence
    :param out: file where the results will be stored
    """
    files = glob.glob(sequence)
    for file in files:
        store_query(database, file, get_out_file(sequence, out, database))

def get_out_file(file: str, specify: str, database: str) -> str:
    """
    A proper name of a file for storing results is created
        - either specified by the user or in format original_name_database_results.
    :param file: Name of a file with a sequence.
    :param specify: User specified file name for storing results.
    :param database: database to be queried
    :return: Name of a file for storing results.
    """
    if specify is not None:
        return specify
    else:
        return file + '_' + database + '_result'

def list_ncbi() -> str:
    """
    :return: Returns list of NCBI databases possible to search.
    """
    return """List of NCBI databases possible to be searched:
    nt: All GenBank + EMBL + DDBJ + PDB sequences. Non-redundant, records with identical sequences collapsed into a single entry.
    rRNA/ITS databases: A collection of four databases: a 16S Microbial rRNA sequences from NCBI’s Targeted Loci \
Projects, an 18S and a 26S RNA rRNA dataabses for fungi, plus an ITS database for fungi.
    refseq_rna: Curated (NM_, NR_) plus predicted (XM_, XR_) sequences from NCBI Reference Sequence Project."""

def list_bold() -> str:
    """
    :return: Returns list of BOLD databases possible to search.
    """
    return """List of BOLD databases possible to be searched:
    COX1 - Every COI barcode record on BOLD with a minimum sequence length of 500bp.
    COX1_SPECIES - Every COI barcode record with a species level identification and a minimum sequence length of 500bp.
    COX1_SPECIES_PUBLIC - All published COI records from BOLD and GenBank with a minimum sequence length of 500bp.
    COX1_L640bp - Subset of the Species library with a minimum sequence length of 640bp and containing both public and \
private records."""


class Solution:
    def canReach(self, arr: list[int], start: int) -> bool:
        # sol1 294ms 85% 20.8MB 73%
        from collections import deque
        queue = deque([start])
        while queue:
            node = queue.popleft()
            if arr[node] == 0:
                return True
            if arr[node] > 0:
                for v in [node+arr[node], node-arr[node]]:
                    if 0 <= v < len(arr):
                        queue.append(v)
            arr[node] = -1
        return False
print(Solution().canReach([3,0,2,1,2], 2))


# -*- coding: utf-8 -*-
import glob
import os
import numpy as np
from PIL import Image

from sklearn.manifold import TSNE
from sklearn.decomposition import PCA

from bokeh.plotting import figure, curdoc
from bokeh.models import ColumnDataSource
from bokeh.layouts import layout

# Dependencies
# Make sure to install the additional dependencies noted in the requirements.txt using the following command:
# pip install -r requirements.txt

# You might want to implement a helper function for the update function below or you can do all the calculations in the
# update callback function.

# Only do this once you've followed the rest of the instructions below and you actually reach the part where you have to
# configure the callback of the lasso select tool. The ColumnDataSource containing the data from the dimensionality
# reduction has an on_change callback routine that is triggered when certain parts of it are selected with the lasso
# tool. More specifically, a ColumnDataSource has a property named selected, which has an on_change routine that can be
# set to react to its "indices" attribute and will call a user defined callback function. Connect the on_change routine
# to the "indices" attribute and an update function you implement here. (This is similar to the last exercise but this
# time we use the on_change function of the "selected" attribute of the ColumnDataSource instead of the on_change
# function of the select widget).
# In simpler terms, each time you select a subset of image glyphs with the lasso tool, you want to adjust the source
# of the channel histogram plot based on this selection.
# Useful information:
# https://docs.bokeh.org/en/latest/docs/reference/models/sources.html
# https://docs.bokeh.org/en/latest/docs/reference/models/tools.html
# https://docs.bokeh.org/en/latest/docs/reference/models/selections.html#bokeh.models.selections.Selection

# Fetch the number of images using glob or some other path analyzer
N = len(glob.glob("static/*.jpg"))

# Find the root directory of your app to generate the image URL for the bokeh server
ROOT = os.path.split(os.path.abspath("."))[1] + "/"

# Number of bins per color for the 3D color histograms
N_BINS_COLOR = 16
# Number of bins per channel for the channel histograms
N_BINS_CHANNEL = 50

# Define an array containing the 3D color histograms. We have one histogram per image each having N_BINS_COLOR^3 bins.
# i.e. an N * N_BINS_COLOR^3 array
color_histograms=np.zeros((N, N_BINS_COLOR**3))

# Define an array containing the channel histograms, there is one per image each having 3 channel and N_BINS_CHANNEL
# bins i.e an N x 3 x N_BINS_CHANNEL array
channel_histograms=np.zeros((N, 3, N_BINS_CHANNEL))

# initialize an empty list for the image file paths
image_paths=[]

# Compute the color and channel histograms
for idx, f in enumerate(glob.glob("static/*.jpg")):
    # open image using PILs Image package
    image = Image.open(f)

    # Convert the image into a numpy array and reshape it such that we have an array with the dimensions (N_Pixel, 3)
    image_array=np.asarray(image);
    image_array=np.reshape(image_array, (image_array.shape[0]*image_array.shape[1],3))

    # Compute a multi dimensional histogram for the pixels, which returns a cube
    # reference: https://numpy.org/doc/stable/reference/generated/numpy.histogramdd.html
    multi_histogram, edges=np.histogramdd(image_array, bins=(N_BINS_COLOR, N_BINS_COLOR, N_BINS_COLOR))

    # However, later used methods do not accept multi dimensional arrays, so reshape it to only have columns and rows
    # (N_Images, N_BINS^3) and add it to the color_histograms array you defined earlier
    # reference: https://numpy.org/doc/stable/reference/generated/numpy.reshape.html
#     color_histograms.append(multi_histogram)
    multi_histogram=np.reshape(multi_histogram, (1,N_BINS_COLOR**3))
    color_histograms[idx]=multi_histogram

    # Append the image url to the list for the server
    url = ROOT + f

    # Compute a "normal" histogram for each color channel (rgb)
    # reference: https://numpy.org/doc/stable/reference/generated/numpy.histogram.html
    red_hist, red_edges=np.histogram(image_array[:,0], bins=N_BINS_CHANNEL)
    green_hist, green_edges=np.histogram(image_array[:,1], bins=N_BINS_CHANNEL)
    blue_hist, blue_edges=np.histogram(image_array[:,2], bins=N_BINS_CHANNEL)

    # and add them to the channel_histograms
    channel_histograms[idx, 0, :]=red_hist
    channel_histograms[idx, 1, :]=green_hist
    channel_histograms[idx, 2, :]=blue_hist

    image_paths.append(url)

# Calculate the indicated dimensionality reductions
# references:
# https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html
# https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html
color_tsne = TSNE(n_components=2, random_state=7).fit_transform(color_histograms)
color_pca = PCA(n_components=2, random_state=7).fit_transform(color_histograms)

# Construct a data source containing the dimensional reduction result for both the t-SNE and the PCA and the image paths
color_source = ColumnDataSource(dict(
        tsne_x = color_tsne[:, 0],
        tsne_y = color_tsne[:, 1],
        pca_x = color_pca[:,0],
        pca_y = color_pca[:,1],
        paths = image_paths))

# Create a first figure for the t-SNE data. Add the lasso_select, wheel_zoom, pan and reset tools to it.

tsne_plot = figure(plot_width=500, plot_height=400, tools=['lasso_select', 'wheel_zoom', 'pan', 'reset'])
tsne_plot.title.text = 'tSNE'
tsne_plot.image_url(url="paths", x="tsne_x", y="tsne_y", h=1, w=1.5 ,source=color_source, anchor="center")
tsne_plot.circle(x="tsne_x", y="tsne_y", source=color_source, fill_alpha=0, line_alpha=0)
tsne_plot.xaxis.axis_label = 'x'
tsne_plot.yaxis.axis_label = 'y'

# And use bokehs image_url to plot the images as glyphs
# reference: https://docs.bokeh.org/en/latest/docs/reference/models/glyphs/image_url.html

# Since the lasso tool isn't working with the image_url glyph you have to add a second renderer (for example a circle
# glyph) and set it to be completely transparent. If you use the same source for this renderer and the image_url,
# the selection will also be reflected in the image_url source and the circle plot will be completely invisible.

# Create a second plot for the PCA result. As before, you need a second glyph renderer for the lasso tool.
# Add the same tools as in figure 1
pca_plot = figure(plot_width=500, plot_height=400, tools=['lasso_select', 'wheel_zoom', 'pan', 'reset'])
pca_plot.title.text = 'PCA'
pca_plot.image_url(url="paths", x="pca_x", y="pca_y", h=8000, w=10000 ,source=color_source, anchor="center")
pca_plot.circle(x="pca_x", y="pca_y", source=color_source, fill_alpha=0, line_alpha=0)
pca_plot.xaxis.axis_label = 'x'
pca_plot.yaxis.axis_label = 'y'

# Construct a datasource containing the channel histogram data. Default value should be the selection of all images.
# Think about how you aggregate the histogram data of all images to construct this data source
aggregation=channel_histograms.sum(axis=0)
red=aggregation[0]
green=aggregation[1]
blue=aggregation[2]

# Construct a datasource containing the channel histogram data. Default value should be the selection of all images.
# Think about how you aggregate the histogram data of all images to construct this data source
channel_source=ColumnDataSource(dict(
                                bins=[i for i in range(0,50)],
                                red=red,
                                green=green,
                                blue=blue))

# Construct a histogram plot with three lines.
# First define a figure and then make three line plots on it, one for each color channel.
# Add the wheel_zoom, pan and reset tools to it.
channel_hist_plot= figure(plot_width=500, plot_height=400, tools=['wheel_zoom', 'pan', 'reset'])
channel_hist_plot.title.text = 'Channel Histogram'
channel_hist_plot.line(x="bins", y="red", source=channel_source, line_color="red")
channel_hist_plot.line(x="bins", y="green", source=channel_source, line_color="green")
channel_hist_plot.line(x="bins", y="blue", source=channel_source, line_color="blue")
channel_hist_plot.xaxis.axis_label = 'bin'
channel_hist_plot.yaxis.axis_label = 'Frequency'

# Connect the on_change routine of the selected attribute of the dimensionality reduction ColumnDataSource with a
# callback/update function to recompute the channel histogram. Also read the topmost comment for more information.
def update(attr, old, new):
    selected_rows = new

    # we don't want to do anything if nothing is selected
    if len(selected_rows) == 0:
        aggregation=channel_histograms.sum(axis=0)
    else:
    	aggregation=channel_histograms[selected_rows].sum(axis=0)

    red=aggregation[0]
    green=aggregation[1]
    blue=aggregation[2]

    channel_source.data = dict(
        bins=[i for i in range(0,50)],
        red=red,
        green=green,
        blue=blue)

# Construct a layout and use curdoc() to add it to your document.

color_source.selected.on_change("indices", update)
lt=layout([[tsne_plot, pca_plot, channel_hist_plot]])

curdoc().add_root(lt)

# You can use the command below in the folder of your python file to start a bokeh directory app.
# Be aware that your python file must be named main.py and that your images have to be in a subfolder name "static"

# bokeh serve --show .
# python -m bokeh serve --show .

# dev option:
# bokeh serve --dev --show .
# python -m bokeh serve --dev --show .


from tqdm import tqdm
import cv2 as cv
import numpy as np
import os
import json
from ImageMasker.generator.BaseGenerator import BaseGenerator, MaskedImage

class PosGenerator(BaseGenerator):
    def __init__(self, df, image_queue, save_dir, mask_factor_list, target_size, plot_out):
        super().__init__(df, image_queue, save_dir, mask_factor_list, target_size, plot_out)

        # initalize max n rows/cols vars to track largest seen images for cropping
        self.max_n_rows = 0
        self.max_n_cols = 0

        # start generator loop
        self.generator_loop()

    def generator_loop(self): # mask_factor_list, target_size, save_out: bool, plot_out: bool
        # get output dataframe
        self.out_df = self.df.copy()

        # iterate over dataframe
        for i, data in tqdm(self.df.iterrows(), total=self.n_images):
            # load img
            img = cv.imread(data.png_path)
            print(f'\nimg {i} loaded')

            # get img view type
            view_axis_idx = self.get_view_axis_idx(data)

            # load roi
            roi_list = extract_roi(data.ROI_coords)

            # check img laterality
            laterality = self.check_side(img)

            # if laterality is 'right' flip img and roi coords
            if laterality == 'R':
                img = np.fliplr(img)
                roi_list = switch_coord_side(roi_list, img.shape)

            print('laterality checked')

            # record roi and tissue distribution
            self.record_roi_tissue_dist(img, view_axis_idx, roi_list, i)
            print('tissue dist recorded')

            # initalize masked_img_list
            masked_img_list = []

            # initalize masked_img_paths string
            masked_img_paths = ''

            # iterate over mask factors
            for mask_factor in self.mask_factor_list:
                # generate masked images
                masked_img = self.mask_image(img, i, roi_list, mask_factor, 'pos')

                # append masked images to list
                masked_img_list.append(masked_img)

                if self.save_dir is not None:
                    self.save_image(masked_img, data.png_filename)
                    masked_img_paths += (masked_img.save_path + '$')

            print('images generated')

            self.out_df.at[i, 'masked_factors'] = str(self.mask_factor_list)
            self.out_df.at[i, 'masked_png_paths'] = masked_img_paths

            print('dataframe updated')

            if self.plot_out:
                self.plot_images(img, masked_img_list)

        # send stop signal to the queue if it exists
        self.send_stop_signal()

        # crop rows/cols arrays to max recorded img size
        self.crop_out_arrays()

        # if saving, save pos image attributes
        if self.save_dir is not None:
            self.save_attrs()

    def record_roi_tissue_dist(self, img, view_axis_idx, roi_list, i):
        """
        Move to PosGenerator.py
        """
        img_rows, img_cols, _ = img.shape

        # if img shape is bigger than max recorded, record it
        # WOULD THE MAX() FUNCTION BE FASTER HERE <-------------------------?
        if img_rows > self.max_n_rows:
            self.max_n_rows = img_rows
        if img_cols > self.max_n_cols:
            self.max_n_cols = img_cols

        # record tissue tissue shape to cols/rows_array --------------------------
        # get a low percentile value of the image to deal with diff normalization
        bg_threshold = np.percentile(img, 2)

        # count the number of non-bg pixels in each of the cols/rows
        px_count_cols = (img[:, :, 0] > bg_threshold).sum(axis=0)
        px_count_rows = (img[:, :, 0] > bg_threshold).sum(axis=1)

        # record current tissue distribution to main arrays
        self.cols_array[view_axis_idx, i, :img_cols] = px_count_cols
        self.rows_array[view_axis_idx, i, :img_rows] = px_count_rows

        # record current roi_list to roi_dict
        if view_axis_idx == 0:
            self.cc_roi_dict[i] = roi_list
        else:
            self.mlo_roi_dict[i] = roi_list

    def crop_out_arrays(self):
        """
        crop cols/rows arrays to the max recorded size
        """
        self.crop_cols_array = self.cols_array[:, :, :self.max_n_cols]
        self.crop_rows_array = self.rows_array[:, :, :self.max_n_rows]

    def save_attrs(self):
        """
        save cols/rows arrays and roi dict to file
        """
        # save crop cols/rows arrays as .npy files
        np.save(
            os.path.join(self.save_dir, 'cols_arr.npy'),
            self.crop_cols_array
        )
        np.save(
            os.path.join(self.save_dir, 'rows_arr.npy'),
            self.crop_rows_array
        )

        # save roi dicts to .json files
        with open(os.path.join(self.save_dir, 'cc_roi_dict.json'), 'w') as json_dict:
            json.dump(self.cc_roi_dict, json_dict)
        with open(os.path.join(self.save_dir, 'mlo_roi_dict.json'), 'w') as json_dict:
            json.dump(self.mlo_roi_dict, json_dict)

def switch_coord_side(roi_list, full_shape):
    """
    static function to switch the side of an roi
    if the tissue is right-aligned
    """
    roi_list_out = []

    for roi in roi_list:
        minX = roi[1]
        maxX = roi[3]
        roi[1] = full_shape[1] - maxX
        roi[3] = full_shape[1] - minX
        roi_list_out.append(roi)

    return roi_list_out

def extract_roi(roi: str):
    """
    static function to convert ROI string from
    dataframe to nested list
    """
    roi = roi.translate({ord(c): None for c in "][)(,"})
    roi = list(map(int, roi.split()))
    roi_list = []
    for i in range(len(roi) // 4):
        roi_list.append(roi[4*i:4*i+4])
    return roi_list


from __future__ import print_function
from itertools import izip

import codecs
import argparse
import tempfile
import subprocess
import sys
import os

JOSHUA = os.environ.get('JOSHUA')

EXAMPLE = r"""
Example invocation:

$JOSHUA/scripts/support/run_thrax.py \
  /path/to/thrax.config \
  /path/to/corpus.SOURCE \
  /path/to/corpus.TARGET \
  /path/to/alignment \
  [-o grammar.gz] \
  [-T /tmp] \
  [-v]
"""
parser = argparse.ArgumentParser(description='Run thrax')
parser.add_argument('-o', dest='output_file', default='grammar.gz', help='Location of output grammar')
parser.add_argument('-f', dest='force', default=False, action='store_true', help='Force overwrite')
parser.add_argument('-T', dest='tmp_dir', default='/tmp', help='Temporary directory')
parser.add_argument('-v', dest='verbose', default=False, action='store_true', help='Be verbose')
parser.add_argument('-d', '--debug', dest='debug', default=False, action='store_true', help='Don\'t cleanup')
parser.add_argument('thrax_config', help='Path to thrax config file')
parser.add_argument('corpora', nargs='+', help='Either (a) the Thrax input file or (b) the source, target, and aligned corpus files')
args = parser.parse_args()

HADOOP   = os.environ['HADOOP']
THRAX_JAR = os.path.join(os.environ['JOSHUA'], 'thrax', 'bin', 'thrax.jar')

THRAXDIR = 'pipeline-%s' % ( os.getcwd().replace('/','_') )

def run(cmd):
    if args.verbose:
        print(cmd)
    subprocess.call(cmd, shell=True)

def utf8open(file, flags='r'):
    return codecs.open(file, flags, 'utf-8')

def paste(source, target, align, out_file):
    out = utf8open(out_file, 'w')
    for s, t, a in izip(utf8open(source), utf8open(target), utf8open(align)):
        out.write(' ||| '.join([s.strip(), t.strip(), a.strip()]) + '\n')
    out.close()

if os.path.exists(args.output_file) and not args.force:
    sys.stderr.write('Fatal: output path "%s" already exists\n' % (args.output_file))
    sys.stderr.write('  (use -f to force overwrite)\n')
    sys.exit(1)

if len(args.corpora) not in [1,3]:
    sys.stderr.write('Fatal: corpora argument must be either')
    sys.stderr.write('  (a) a single consolidated Thrax input file, or')
    sys.stderr.write('  (b) three parallel files: source, target, and alignments')
    sys.exit(2)

# Cleanup
run('%s/bin/hadoop fs -rm -r %s' % (HADOOP, THRAXDIR))
run('%s/bin/hadoop fs -mkdir %s' % (HADOOP, THRAXDIR))

# Create thrax input file
if len(args.corpora) == 3:
    # Paste together the source, target, and alignment files
    thrax_file = 'thrax.input_file'
    paste(args.corpora[0], args.corpora[1], args.corpora[2], thrax_file)
    rm_thrax_file = True
else:
    # Assume the thrax input file is already created
    thrax_file = args.corpora[0]
    rm_thrax_file = False

run('%s/bin/hadoop fs -put %s %s/input-file' % (HADOOP, thrax_file, THRAXDIR))

# Copy the template
conf_file = tempfile.NamedTemporaryFile(prefix='/tmp/thrax.conf', delete=False)
for line in open(args.thrax_config):
    if not line.startswith('input-file'):
        conf_file.write(line)
conf_file.write('input-file %s/input-file\n' % (THRAXDIR))
conf_file.close()
conf_file_name = conf_file.name

# Run Hadoop
run('%s/bin/hadoop jar %s -D mapred.child.java.opts="-Xmx%s" -D hadoop.tmp.dir=%s %s %s' % (HADOOP, THRAX_JAR, '4g', args.tmp_dir, conf_file_name, THRAXDIR))
run('rm -f %s' % (args.output_file))
run('%s/bin/hadoop fs -getmerge %s/final/ %s' % (HADOOP, THRAXDIR, args.output_file))

# Cleanup
if not args.debug:
    os.remove(conf_file_name)
    if rm_thrax_file:
      os.remove(thrax_file)
    run('%s/bin/hadoop fs -rm -r %s' % (HADOOP, THRAXDIR))


# implementation of Extreme Learning Machine
import numpy as np
import copy
from numpy import linalg
from scipy.special import expit
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.ensemble import AdaBoostClassifier

class BaseELM(BaseEstimator, ClassifierMixin):
    upper_bound = 1.
    lower_bound = -1.
    def __init__(self, n_hidden, dropout_prob=None):
        self.n_hidden = n_hidden
        self.dropout_prob = dropout_prob

    def fit(self, X, y, sample_weight=None):
        # check label has form of 2-dim array
        X, y, = copy.deepcopy(X), copy.deepcopy(y)
        self.sample_weight = None
        if y.shape.__len__() != 2:
            self.classes_ = np.unique(y)
            self.n_classes_ = self.classes_.__len__()
            y = self.one2array(y, self.n_classes_)
        else:
            self.classes_ = np.arange(y.shape[1])
            self.n_classes_ = self.classes_.__len__()
        self.W = np.random.uniform(self.lower_bound, self.upper_bound, size=(X.shape[1], self.n_hidden))
        if self.dropout_prob is not None:
            self.W = self.dropout(self.W, prob=self.dropout_prob)
            # X = self.dropout(X, prob=self.dropout_prob)
        self.b = np.random.uniform(self.lower_bound, self.upper_bound, size=self.n_hidden)
        H = expit(np.dot(X, self.W) + self.b)
        # H = self.dropout(H, prob=0.1)
        if sample_weight is not None:
            self.sample_weight = sample_weight / sample_weight.sum()
            extend_sample_weight = np.diag(self.sample_weight)
            inv_ = linalg.pinv(np.dot(
                np.dot(H.transpose(), extend_sample_weight), H))
            self.B = np.dot(np.dot(np.dot(inv_, H.transpose()), extend_sample_weight), y)
        else:
            self.B = np.dot(linalg.pinv(H), y)
        return self

    def one2array(self, y, n_dim):
        y_expected = np.zeros((y.shape[0], n_dim))
        for i in range(y.shape[0]):
            y_expected[i][y[i]] = 1
        return y_expected

    def predict(self, X, prob=False):
        X = copy.deepcopy(X)
        H = expit(np.dot(X, self.W) + self.b)
        output = np.dot(H, self.B)
        if prob:
            return output
        return output.argmax(axis=1)

    def get_params(self, deep=True):
        params = {'n_hidden': self.n_hidden, 'dropout_prob': self.dropout_prob}
        return params

    def set_params(self, **parameters):
        for parameter, value in parameters.items():
            setattr(self, parameter, value)
        return self

    def predict_proba(self, X):
        return self.predict(X, prob=True)

    def dropout(self, x, prob=0.2):
        """
        prob:
        :param x:
        :param prob:
        :return:
        """
        if prob < 0. or prob >= 1:
            raise Exception('Dropout level must be in interval [0, 1]')
        # discon_prob = 1. - prob
        sample = np.random.binomial(n=1, p=prob, size=x.shape)
        x *= sample
        # x /= retain_prob
        return x

    def softmax(self, x):
        """
        Compute the softmax function for each row of the input x.

        Arguments:
        x -- A N dimensional vector or M x N dimensional numpy matrix.

        Return:
        x -- You are allowed to modify x in-place
        """
        orig_shape = x.shape

        if len(x.shape) > 1:
            # Matrix
            exp_minmax = lambda x: np.exp(x - np.max(x))
            denom = lambda x: 1.0 / np.sum(x)
            x = np.apply_along_axis(exp_minmax, 1, x)
            denominator = np.apply_along_axis(denom, 1, x)

            if len(denominator.shape) == 1:
                denominator = denominator.reshape((denominator.shape[0], 1))

            x = x * denominator
        else:
            # Vector
            x_max = np.max(x)
            x = x - x_max
            numerator = np.exp(x)
            denominator = 1.0 / np.sum(numerator)
            x = numerator.dot(denominator)

        assert x.shape == orig_shape
        return x

#
'''
----------
ELM test
----------
'''
# import sklearn.datasets as dt
# from sklearn.preprocessing import normalize
# from sklearn.cross_validation import train_test_split
# from sklearn.metrics import accuracy_score
# from sklearn import preprocessing
# iris = dt.load_iris()
# X, y = iris.get('data'), iris.get('target')  # start with 0
# X = normalize(X)
# X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.6)
#
# lb = preprocessing.LabelBinarizer()
# Y_train = lb.fit_transform(y_train)
# # Y_test = lb.fit_transform(y_test)
#
# elm = BaseELM(200, dropout_prob=0.9)
# elm.fit(X_train, Y_train, sample_weight=None)
# labels_tr = elm.predict(X_train)
# labels_ts = elm.predict(X_test)
#
# print 'Train err:', 1-accuracy_score(y_train, labels_tr)
# print 'Test err:', 1-accuracy_score(y_test, labels_ts)
# print 'predicted labels:', labels
# print 'actual labels:', y_test-labels

#
# elm_ab = AdaBoostClassifier(ELM(10), algorithm="SAMME", n_estimators=100)
# elm_ab.fit(X_train, y_train)
# y_pre_elm_ab = elm_ab.predict(X_test)
# print 'AdBoost ELM:', accuracy_score(y_test, y_pre_elm_ab)

# '''
# --------
# compare with HPELM
# --------
# '''
# from hpelm import ELM as HPELM
# model = HPELM(4, 3)
# model.add_neurons(10, "sigm")
# model.train(X_train, Y_train)
# Y_pre = model.predict(X_test)
# y = Y_pre.argmax(axis=1)
# print 'HPELM acc:', accuracy_score(y_test, y)


import os
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from io import BytesIO
from urllib.request import urlopen
from zipfile import ZipFile

import pytorch_lightning as pl

from data import IMDBDataModule
from model import TextClassifier

os.environ["TOKENIZERS_PARALLELISM"] = "true"

if __name__ == '__main__':
    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('--datadir', default=os.getcwd(), type=str, help="IMDB data directory")
    parser.add_argument('--imdb', default="https://pl-flash-data.s3.amazonaws.com/imdb.zip", type=str, help="IMDB data source")
    parser = pl.Trainer.add_argparse_args(parser)
    parser = IMDBDataModule.add_argparse_args(parser)
    parser = TextClassifier.add_argparse_args(parser)
    args = parser.parse_args()

    pl.seed_everything(args.seed)

    if not(os.path.isdir(args.datadir+"/imdb")):
        os.makedirs(args.datadir, exist_ok=True)
        with urlopen(args.imdb) as zipresp:
            with ZipFile(BytesIO(zipresp.read())) as zfile:
                zfile.extractall(args.datadir)

    dm = IMDBDataModule.from_argparse_args(args)
    dm.setup('fit',datadir=args.datadir)
    model = TextClassifier(
        model_name_or_path=dm.model_name_or_path,
        label2id=dm.label2id,
        learning_rate=args.learning_rate,
        adam_epsilon=args.adam_epsilon,
        weight_decay=args.weight_decay,
        warmup_steps=args.warmup_steps,
        predictions_file=args.predictions_file
    )
    model.tokenizer = dm.tokenizer
    model.total_steps = (
            (len(dm.ds['train']) // (args.batch_size * max(1, (args.gpus or 0))))
            // args.accumulate_grad_batches
            * float(args.max_epochs)
    )
    trainer = pl.Trainer.from_argparse_args(args)
    trainer.fit(model, dm)
    trainer.test(datamodule=dm)
    model.save_pretrained("/lightning_logs/outputs")


#!/usr/bin/env python3

import os
import re
import sys
import csv
import logging
import pkg_resources
from typing import Union
from pathlib import Path
from argparse import ArgumentParser
from collections import defaultdict
from importlib.resources import files

__version__ = pkg_resources.require("isostate")[0].version

LOG = logging.getLogger('isostate')

PACKAGE_DATA_PATH = files('isostate').joinpath('data')
DEFAULT_CSV_PATH = PACKAGE_DATA_PATH.joinpath('isostate.csv')

SOURCE_BASENAME_RE = r"^1\.([a-z-]+)\.([a-z]{2})\.csv$"

DEFAULT_LANG = "en"
DEFAULT_NAME = "short"

class LookupNotFoundException(Exception):
    pass

class NameNotFoundException(Exception):
    pass

def get_source_dict() -> dict[tuple[str, str]: Path]:
    path = PACKAGE_DATA_PATH
    source_list = []
    for source_path in sorted(path.glob("*.csv")):
        match = re.compile(SOURCE_BASENAME_RE).match(source_path.name)
        if not match:
            continue
        source_list.append((match.groups(), source_path))
    return dict(sorted(source_list))

def codes(name=DEFAULT_NAME, lang=DEFAULT_LANG, source=None):
    code_dict = {}
    reader = Iso.source_iter(name, lang, source)
    for row in reader:
        if not len(row):
            continue
        iso2, _features, lang, name = row
        code_dict[iso2] = name
    return code_dict

class Iso():
    def __init__(
            self,
            source="text",
            cache: Union[None, str, Path] = None,
            batch: bool = False,
    ):
        self._source = None
        self._cache_path = None
        self._lookup = {}
        self._batch = batch

        if cache:
            self._cache_path = cache

        self.set_source(source)

    def set_source(self, source="text"):
        data = load_data(self._cache_path)

        assert source == "text"

        self._source = SourceText(data, sub=False)

    @staticmethod
    def key(name=DEFAULT_NAME, lang=DEFAULT_LANG):
        return "%s.%s" % (name, lang)

    @classmethod
    def source_iter(
            cls,
            name: str = DEFAULT_NAME,
            lang: str = DEFAULT_LANG,
            source: Union[None, str, Path] = None
    ):
        if source is None:
            try:
                path = get_source_dict()[(name, lang)]
            except IndexError:
                LOG.error(
                    "Error: no source file available for name `{%s}` and lang `{%s}`.",
                    name, lang
                )
                sys.exit(1)
        else:
            path = Path(source)

        try:
            csv_data = path.open("r", encoding="utf-8")
        except Exception as e:
            LOG.error(
                "Could not open file '%s' for name '%s' and language '%s'.",
                path, name, lang)
            raise e

        reader = csv.reader(csv_data, delimiter=';')
        return reader

    def add_lookup(self, name=DEFAULT_NAME, lang=DEFAULT_LANG, source=None):
        key = self.key(name, lang)
        reader = self.source_iter(name, lang, source)
        for row in reader:
            if not len(row):
                continue
            row = [v.strip() for v in row]
            iso2, _features, lang, name = row
            if not key in self._lookup:
                self._lookup[key] = {}
            self._lookup[key][iso2] = name

    def name(self, iso2, name=DEFAULT_NAME, lang=DEFAULT_LANG):
        key = self.key(name, lang)

        if not key in self._lookup:
            raise LookupNotFoundException("No lookup found with key '%s'." % key)

        if not iso2 in self._lookup[key]:
            raise NameNotFoundException(
                "No name found for code '%s' in lookup '%s'." % (iso2, key))

        return self._lookup[key][iso2]

    def iso2(
            self,
            text,
            batch: Union[None, bool] = None,
            accept_sub: bool = False,
    ):
        search_text = clean_text(text)

        assert self._source

        iso2, sub = self._source.match(search_text)

        if batch is None:
            batch = self._batch

        if not batch and not iso2:
            iso2, sub = self._source.search(search_text, insert=self.insert)

        if iso2 and sub and not accept_sub:
            iso2 = None

        if iso2 == "  ":
            iso2 = None

        return iso2

    def insert(self, iso2, sub, lang, name):
        if self._cache_path:
            with open(self._cache_path, "a", encoding="utf-8") as fp:
                fp.write("%2s; %s; %2s; %s\n" % (
                    iso2 or "  ",
                    sub and ">" or "",
                    lang,
                    name,
                ))
        data = load_data(self._cache_path)
        self._source.reload(data)

class SourceText():
    def __init__(self, data, lang=DEFAULT_LANG, sub=False, size=(3, 5, 7)):
        """
        size: ngram size
        """

        self.lang = lang
        self.sub = sub
        self.size = size

        self.reload(data)

    def reload(self, data):
        self.data = data
        self.data = [row for row in self.data if row["lang"] == self.lang]

        self.by_name = {}
        for row in self.data:
            self.by_name[row["name"]] = {
                "iso2":row["iso2"],
                "sub":row["sub"],
            }

        def ngram_dict():
            return {
                "name": defaultdict(float),
                "total": 0.0
            }

        names = defaultdict(list)
        for row in self.data:
            iso2 = row["iso2"]
            name = row["name"]
            names[iso2].append(name)

        self.ngrams = defaultdict(ngram_dict)
        for iso2, name_list in list(names.items()):
            weight = 1.0 / len(name_list)
            for name in name_list:
                for ngram in text_to_ngrams(name, self.size):
                    self.ngrams[ngram]["name"][name] += weight
                    self.ngrams[ngram]["total"] += weight

    def match(self, text):
        iso2 = None
        sub = None

        row = self.by_name.get(text, None)
        if row:
            iso2, sub = row["iso2"], row["sub"]

        return iso2, sub

    def search_all(self, text):
        candidates = defaultdict(float)

        for ngram in text_to_ngrams(text, self.size):
            matches = self.ngrams.get(ngram, None)
            if not matches:
                continue
            total = matches["total"]
            for name, value in list(matches["name"].items()):
                candidates[name] += float(value) / total

        def score():
            return {
                "value": 0.0,
                }

        high_scores = defaultdict(score)

        for name, value in list(candidates.items()):
            row = self.by_name.get(name, None)
            key = row["iso2"]
            if row["sub"]:
                key += ">"
            if value > high_scores[key]["value"]:
                high_scores[key] = {
                    "iso2": row["iso2"],
                    "value": value,
                    "name": name,
                    "sub": row["sub"],
                }

        high_scores = sorted(high_scores.values(),
                             key=lambda x: x["value"], reverse=True)

        return high_scores

    def search(self, text, insert=None):
        iso2 = None
        sub = None

        result = pick_one(text, self.search_all)
        if result:
            iso2, sub = result

        if insert:
            insert(iso2, sub, self.lang, text)

        return iso2, sub

def pick_one(text, search_func, limit=10):
    search_text = text
    while True:
        high_scores = search_func(search_text)[:limit]
        max_score = None

        sys.stderr.write("\n             Find:  %s\n\n" % repr(text))

        for i, row in enumerate(high_scores):
            if max_score is None:
                max_score = row["value"]
            show_name = row["name"]
            if row["sub"]:
                show_name += " (subregion)"
            sys.stderr.write(" %4s:  %10s  %s\n" % (
                i + 1,
                "+" * int(10 * row["value"] / max_score),
                show_name
            ))

        sys.stderr.write("\n")
        sys.stderr.write("    0:  None of the above\n")
        sys.stderr.write("   #>:  Confirm text as sub-region of option #\n")
        sys.stderr.write(" Text:  Alternative search string\n")
        sys.stderr.write("\n")

        sys.stderr.write("> ")
        choice = input().strip()
        sys.stderr.write("\n")

        if not choice:
            sys.stderr.write("Please select an option.\n\n")
            continue

        if re.match("^[0-9]+$", choice):
            choice = int(choice) - 1
            sub = False
        elif re.match("^[0-9]+>$", choice):
            choice = int(choice[:-1]) - 1
            sub = True
        else:
            search_text = choice
            continue

        if choice == -1:
            return None

        if not 0 <= choice < len(high_scores):
            sys.stderr.write("Choice out of range.\n\n")
            continue

        row = high_scores[choice]

        return row["iso2"], sub or row["sub"]

def clean_text(text):
    text = text.strip()
    text = text.lower()
    text = text.replace("\u2013", "-")
    text = text.replace("/", " ")
    text = text.replace(" - ", " ")
    text = re.compile(r"[^\w '-]+", re.U).sub("", text)
    text = re.compile(r"[\s]+", re.U).sub(" ", text)
    return text

def load_data(cache_path=None):
    data = {}

    def process_line(line):
        line = line[:-1]
        iso2, features, lang, name = line.split("; ")

        sub = ">" in features
        name = clean_text(name)
        assert name

        data[(lang, name)] = {
            "iso2": iso2,
            "sub": sub,
            "lang": lang,
            "name": name,
        }

    path = DEFAULT_CSV_PATH
    n = 0

    with path.open("r", encoding="utf-8") as csv_fp:
        for line in csv_fp.readlines():
            process_line(line)

    LOG.debug(f"Loaded {len(data) - n:d} entries from `{path}`")
    n = len(data)

    if cache_path:
        if os.path.exists(cache_path):
            with open(cache_path, "r", encoding="utf-8") as csv_fp:
                for line in csv_fp.readlines():
                    process_line(line)
        LOG.debug(f"Loaded {len(data) - n:d} entries from `{cache_path}`")

    return list(data.values())

def text_to_ngrams(text, size=3):
    ngrams = []

    if hasattr(size, "__iter__"):
        for value in size:
            ngrams += text_to_ngrams(text, value)
        return ngrams

    text = clean_text(text)

    for word in text.split():
        length = len(word)
        space = " " * (size - 1)
        word = space + word + space
        for i in range(length + size - 1):
            ngrams.append(word[i: i + size])
    return ngrams

def main():
    LOG.addHandler(logging.StreamHandler())

    parser = ArgumentParser("Search for ISO country codes.")
    parser.add_argument(
        "--verbose", "-v",
        action="count", default=0,
        help="Print verbose information for debugging.")
    parser.add_argument(
        "--quiet", "-q",
        action="count", default=0,
        help="Suppress warnings.")

    parser.add_argument(
        "--batch", "-b",
        action="store_true",
        help="Non-interactive mode. Search is disabled; only exact matches will be printed.")

    parser.add_argument(
        "--cache", "-c",
        action="store",
        help="Path to cache file.")

    parser.add_argument(
        "--list-sources", "-l",
        action="store_true",
        help="List sources.")

    parser.add_argument(
        "search", metavar="SEARCH",
        nargs="*",
        action="store",
        help="Search string`.")

    args = parser.parse_args()

    level = (logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG)[
        max(0, min(3, 1 + args.verbose - args.quiet))]
    LOG.setLevel(level)

    if args.list_sources:
        for (name, lang) in get_source_dict():
            print(f"{name}.{lang}")
        return

    if not args.search:
        LOG.warning("No search terms given. Nothing to do.")
        sys.exit(0)

    iso = Iso(cache=args.cache, batch=args.batch)

    iso.add_lookup("short", "en")
    for needle in args.search:
        iso2 = iso.iso2(needle)
        short_en = iso.name(iso2, "short", "en")
        print("%s\t%s" % (iso2, short_en))


import openpyxl
import json
from tqdm import tqdm

def convert_to_json(path):
    # 엑셀파일 열기
    wb = openpyxl.load_workbook(path)

    # 현재 Active Sheet 얻기
    ws = wb.active

    # 국영수 점수를 읽기
    high_school_data = []
    for r in tqdm(ws.rows):
        if r[0].row  < 6 or r[1].value not in ['대학교', '각종대학(대학)', '교육대학', '전공대학', '전문대학(4년제)']:
            continue

        high_school_data.append({
            "school_type":r[1].value,
            "region":r[2].value,
            "sub_region":r[3].value,
            "name":r[4].value,
            "eng_name":r[5].value,
            "foundation":r[8].value,
            "homePage":r[13].value
        })
    return high_school_data
#
# if __name__ == '__main__':
#     //analyze()


import json
import re

from search import get_best_k_completions
from init_data_structure import write_to_files

size_of_string = 7
size_of_res = 5

def print_result(json_lines, auto_list):
    for i in auto_list:
        line = json_lines[str(i.file_id)][str(i.offset)]
        print(
            f'\t\033[35m\033[1m{line[:len(line) - 1]}\033[0m    ,\033[37m( file:\033[94m\033[4m{i.source_text}\033[0m\033[37m ) ,(arg:{i.offset})\033[0m')
def run(json_data, json_lines):
    print("Hi user!")
    while 1:
        input_user = input("\nPlease enter the string you want us to search for you or enter exit\n")
        if input_user == "exit":
            print("GoodBye:)")
            break
        print_result(json_lines,
                     get_best_k_completions(json_data, json_lines,
                                            re.sub(r"[^a-z0-9]+", ' ', input_user.lower()[:size_of_string])) [:size_of_res])
        while 1:
            input_user += input(input_user)
            if input_user[-1] == '#':
                break
            print_result(json_lines,
                         get_best_k_completions(json_data, json_lines,
                                                re.sub(r"[^a-z0-9]+", ' ', input_user.lower()[:size_of_string])) [:size_of_res])

def read_from_file():
    file_data = open("data2.json")
    json_data = json.load(file_data)
    file_data.close()
    file_data = open("line2.json")
    json_lines = json.load(file_data)
    file_data.close()
    return json_data, json_lines

if __name__ == '__main__':
    # write_to_files()
    file = read_from_file()
    run(file[0], file[1])


from django.urls import path, re_path, include
from .views import get_pos, search_key_view, select_points, index, search, analyze, count_city, other, charCity, charXueli, charWorkJy, charWorkIndustry, charWorkSize, get_data_list, get_word_view

# 防止不同app中的URL的name产生冲突   reverse(appName:hello)
app_name = 'position'

urlpatterns = [
    path('', index, name='index'),
    path('search/', search, name='search'),
    path('get_pos/', get_pos, name='get_pos'),
    path('search_key/', search_key_view, name='search_key'),
    path('select_points/', select_points, name='select_points'),
    path('analyze/', analyze, name='analyze'),
    path('count_city/', count_city, name='count_city'),
    path('other/', other, name='other'),
    path('charCity/', charCity, name='charCity'),
    path('charXueli/', charXueli, name='charXueli'),
    path('charWorkJy/', charWorkJy, name='charWorkJy'),
    path('charWorkIndustry/', charWorkIndustry, name='charWorkIndustry'),
    path('charWorkSize/', charWorkSize, name='charWorkSize'),
    path('get_data_list/', get_data_list, name='get_data_list'),
    path('get_word_cloud/', get_word_view, name='get_word_cloud'),
]


from django.core.exceptions import PermissionDenied
from django.http import HttpResponseForbidden
from django.http import HttpResponse
from django.contrib import messages
from app_accounts.models import ConnectionRequestsUrlsAndPermissions
from django.db.models import Q

def request_validate_url_access(self, request):
    request_url = request.path[1:]

    if len(request_url) == 0:
        request_url = '/'
    else:
        if request_url[-1] != '/':
            request_url += '/'

    if "admin/" not in request_url and not request.user.is_superuser:
        has_access = False
        text = "403 Forbidden"

        url_filter = (Q(url=request_url) &
                      Q(url_include_nested_urls=False))
        obj_url_permission = ConnectionRequestsUrlsAndPermissions.objects.filter(url_filter).first()

        if not obj_url_permission:
            request_url_parts = request_url.split('/')
            request_url_parts = list(filter(("").__ne__, request_url_parts))
            if len(request_url_parts) > 1:
                tmp_list = []
                j = 0
                for idx, i in enumerate(request_url_parts):
                    if j == 0:
                        v = request_url_parts[idx-j]
                        tmp_list.append(v+"/")
                    else:
                        v = request_url_parts[idx-j]
                        for f in range((idx+1)-j, idx+1):
                            v += "/" + request_url_parts[f]
                        tmp_list.append(v+"/")
                    j += 1
                request_url_parts = tmp_list
                request_url_parts_filter = "Q(" + ' | '.join([f"Q(url=request_url_parts[{i}])" for i in range(0, len(request_url_parts))]) + ")"
                url_filter = (eval(request_url_parts_filter) &
                              Q(url_include_nested_urls=True))
            else:
                url_filter = (Q(url=request_url) &
                              Q(url_include_nested_urls=True))

            obj_url_permission = ConnectionRequestsUrlsAndPermissions.objects.filter(url_filter).first()

        if obj_url_permission:
            url_has_access_anonymous = obj_url_permission.users_include_anonymous
            url_has_access_users = list(obj_url_permission.access_via_users.values_list('id', flat=True))
            url_has_access_roles = list(obj_url_permission.access_via_roles.values_list('name', flat=True))

            if request.user.is_anonymous:
                if url_has_access_anonymous:
                    has_access = True
            elif hasattr(request.user, 'id'):
                # No access restrictions by Users or Roles
                if len(url_has_access_users) == 0 and len(url_has_access_roles) == 0:
                    has_access = True
                # Users + Roles
                elif len(url_has_access_users) > 0 and len(url_has_access_roles) > 0:
                    if request.user.id in url_has_access_users:
                        if len(url_has_access_roles) > 0:
                            user_roles = list(request.user.groups.values_list('name', flat=True))
                            if len(user_roles) > 0:
                                for i in user_roles:
                                    if i in url_has_access_roles:
                                        has_access = True
                                        break
                        else:
                            has_access = True
                # Only Roles
                elif len(url_has_access_roles) > 0:
                    user_roles = list(request.user.groups.values_list('name', flat=True))
                    if len(user_roles) > 0:
                        for i in user_roles:
                            if i in url_has_access_roles:
                                has_access = True
                                break
                # Only Users
                else:
                    if len(url_has_access_users) > 0:
                        if request.user.id in url_has_access_users:
                            has_access = True

            # print(request.user.is_anonymous, url_has_access_anonymous)
            # print(url_has_access_users)
            # print(url_has_access_roles)
            # print(user_roles)

        if not has_access:
            user_id = "0"
            if hasattr(request.user, 'id'):
                if not request.user.is_anonymous:
                    user_id = str(request.user.id)
            claim_id = f"0-{user_id}"
            message_text = f'
[ Claim id: {claim_id} ]

$$$message_text$$$
'
            message_template = f'
$$$message_text$$$
'
            message_template = message_template.replace("$$$message_text$$$",
                                                        message_text.replace("$$$message_text$$$", text))
            response = HttpResponse(message_template)
            response.status_code = 403
            return response


from HAR_Feature_Selection import *
import umap.umap_ as umap
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.utils import shuffle
from sklearn.ensemble import RandomForestClassifier, StackingClassifier, GradientBoostingClassifier
from sklearn.linear_model import RidgeClassifier
from sklearn.svm import SVC
from sklearn.metrics import f1_score, confusion_matrix, roc_auc_score, roc_curve
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
import plotly.figure_factory as ff
import plotly.graph_objects as go
import plotly.io as pio
pio.renderers.default='browser'
import pandas as pd
import warnings
warnings.filterwarnings("ignore")

### Functions used ..

def ConfusionMatrix_plot(cm, labels, title):

    """
    ConfusionMatrix_plot function plots the Confusion Matrix with labels.

    :param cm: Confusion Matrix.
    :param labels: Labels for the Dataset.
    :param tittle: tittle of the plot.

    Returns
    -----------------------
    2D figure of Confusion Matrix.
    """

    cm_text = [[str(y) for y in x] for x in cm]
    # set up figure
    fig = ff.create_annotated_heatmap(cm, x=labels, y=labels, annotation_text=cm_text, colorscale='Viridis')

    # add title
    fig.update_layout(title_text=f"{title}")

    # add custom xaxis title
    fig.add_annotation(dict(font=dict(color="black", size=14),
                            x=0.5,
                            y=-0.15,
                            showarrow=False,
                            text="Predicted value",
                            xref="paper",
                            yref="paper"))

    # add custom yaxis title
    fig.add_annotation(dict(font=dict(color="black", size=14),
                            x=-0.35,
                            y=0.5,
                            showarrow=False,
                            text="Real value",
                            textangle=-90,
                            xref="paper",
                            yref="paper"))

    # adjust margins to make room for yaxis title
    fig.update_layout(margin=dict(t=50, l=200))

    # add colorbar
    fig['data'][0]['showscale'] = True
    fig.show()

def AUROC_curve(model, y, y_scores, title):

    """
    plots the AUROC (Area Under Reciever Operating Characteristics) Curve which is used
    for showing the performance of classification model at all classification thresholds.

    :param model: model used for classification.
    :param y: Actual labels.
    :param y_scores: Predicted Labels.
    :param tittle: tittle of the plot.

    Returns
    ---------------------
    AUROC curve figure.
    """

    # One hot encode the labels in order to plot them
    y_onehot = pd.get_dummies(y, columns=model.classes_)

    # Create an empty figure, and iteratively add new lines
    # every time we compute a new class
    fig = go.Figure()
    fig.add_shape(
        type='line', line=dict(dash='dash'),
        x0=0, x1=1, y0=0, y1=1
    )

    for i in range(y_scores.shape[1]):
        y_true = y_onehot.iloc[:, i]
        y_score = y_scores[:, i]

        fpr, tpr, _ = roc_curve(y_true, y_score)
        auc_score = roc_auc_score(y_true, y_score)

        name = f"{y_onehot.columns[i]} (AUC={auc_score:.2f})"
        fig.add_trace(go.Scatter(x=fpr, y=tpr, name=name, mode='lines'))

    fig.update_layout(
        title=title,
        xaxis_title='False Positive Rate',
        yaxis_title='True Positive Rate',
        yaxis=dict(scaleanchor="x", scaleratio=1),
        xaxis=dict(constrain='domain'),
        width=700, height=500
    )
    fig.show()

def f_score(x_train, x_test, y_train, y_test, classifiers):

    """
    f_score computes the f1-Score of the all models.

    :param x_train: training features.
    :param x_test: testing features.
    :param y_train: training labels.
    :param y_test: testing labels.
    :param classifier: list of classification algorithm

    Return
    """

    for clf in classifiers:

        clf.fit(x_train,y_train)
        y_pred = clf.predict(x_test)
        f = f1_score(y_true=y_test,y_pred=y_pred,average="macro")

        print(f"Score: {round(f,3)*100}% \t Classifier: {clf.__class__.__name__}")

# Splitting the Dataset into Train-Test

train = pd.read_csv("Human Activity Recognition with Smartphone/train.csv")
test = pd.read_csv("Human Activity Recognition with Smartphone/test.csv")
train = shuffle(train)
test = shuffle(test)
x = train.drop(['Activity'],axis=1)
x = StandardScaler().fit_transform(x)
y = train['Activity']
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=40)

xTest = test.drop(['Activity'],axis=1)
xTest = StandardScaler().fit_transform(xTest)
yTest = test['Activity']
labels = ['LAYING', 'SITTING', 'STANDING', 'WALKING', 'WALKING_DOWNSTAIRS', 'WALKING_UPSTAIRS']

# Different Classifiers to find the best model.
classifiers = [
    KNeighborsClassifier(5),
    SVC(kernel="rbf"),
    DecisionTreeClassifier(),
    RandomForestClassifier(),
    GaussianNB(),
    RidgeClassifier(),
    LogisticRegression(max_iter=200)
]
f_score(x_train, x_test, y_train, y_test, classifiers)

estimators = [
        ('RFC' ,RandomForestClassifier(n_estimators=500, random_state = 42)),
        ('KNC', KNeighborsClassifier(5)),
        ('DTC', DecisionTreeClassifier()),
        ('SVC', SVC(kernel="rbf")),
        ('RC',  RidgeClassifier()),
]

clf = StackingClassifier(
    estimators=estimators,
    final_estimator=GradientBoostingClassifier()
)

# Experimentation

### Before Feature selection #########

clf.fit(x_train,y_train)

y_pred = clf.predict(x_test)
f1 = f1_score(y_true=y_test,y_pred=y_pred,average="macro")
print("Accuracy for Train data before feature selection")
print(f"Score: {round(f1, 3) * 100}%")
y_scores = clf.predict_proba(x_test)
AUROC_curve(clf, y_test, y_scores, title = 'AUROC Curve for Training Data')

### Accuracy on original Test Data ###
y_pred = clf.predict(xTest)
f2 = f1_score(y_true=yTest,y_pred=y_pred,average="macro")
print("Accuracy for Original Test data before feature selection")
print(f"Score: {round(f2, 3) * 100}%")
y_scores = clf.predict_proba(xTest)
AUROC_curve(clf, yTest, y_scores, title = 'AUROC Curve for Original Test Data')

### After Feature selection #########

features = featureSelection()

x1, xTest1 = train.filter(items=features), test.filter(items=features)
x1 = StandardScaler().fit_transform(x1)
xTest1 = StandardScaler().fit_transform(xTest1)
x_train, x_test, y_train, y_test = train_test_split(x1, y, test_size=0.2, random_state=40)

clf.fit(x_train,y_train)

y_pred = clf.predict(x_test)
f1 = f1_score(y_true=y_test,y_pred=y_pred,average="macro")
cm = confusion_matrix(y_true=y_test,y_pred=y_pred, labels=['LAYING', 'SITTING', 'STANDING', 'WALKING', 'WALKING_DOWNSTAIRS', 'WALKING_UPSTAIRS'])
print("Accuracy for Train data after feature selection")
print(f"Score: {round(f1, 3) * 100}%")
ConfusionMatrix_plot(cm, labels, title = "Confusion Matrix (Training)")    # Confusion matrix for training data
y_scores = clf.predict_proba(x_test)
AUROC_curve(clf, y_test, y_scores, title = 'AUROC Curve for Training Data (After Feature Selection)')

y_pred = clf.predict(xTest1)
f2 = f1_score(y_true=yTest,y_pred=y_pred,average="macro")
cm = confusion_matrix(y_true=yTest,y_pred=y_pred, labels=['LAYING', 'SITTING', 'STANDING', 'WALKING', 'WALKING_DOWNSTAIRS', 'WALKING_UPSTAIRS'])
print("Accuracy for Original Test data after feature selection")
print(f"Score: {round(f2, 3) * 100}%")
ConfusionMatrix_plot(cm, labels, title = "Confusion Matrix (Testing)")    # Confusion matrix for original test data
# AUC on original test data
y_scores = clf.predict_proba(xTest1)
AUROC_curve(clf, yTest, y_scores, title = 'AUROC Curve for Original Test Data (After Feature Selection)')

### Rather than removing Features we can use the reduced features using Advanced Dimensionality Reduction Techniques

trans = umap.UMAP(n_neighbors=5, n_components=5 ,random_state=42).fit(x_train)
clf.fit(trans.embedding_,y_train)
test_embedding = trans.transform(x_test)
y_pred = clf.predict(test_embedding)
f1 = f1_score(y_true=y_test,y_pred=y_pred,average="macro")
print(f"Score: {round(f1, 3) * 100}%")
y_scores = clf.predict_proba(test_embedding)
AUROC_curve(clf, y_test, y_scores, title = 'AUROC Curve for Transformed UMAP Training Data')

# UMAP on original test data

test_embedding = trans.transform(xTest1)
y_pred = clf.predict(test_embedding)
f1 = f1_score(y_true=yTest,y_pred=y_pred,average="macro")
print(f"Score: {round(f1, 3) * 100}%")
y_scores = clf.predict_proba(test_embedding)
AUROC_curve(clf, yTest, y_scores, title = 'AUROC Curve for Transformed UMAP Test Data (Original Test Dataset)')


from setuptools import setup
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

with open(path.join(here, 'README.md')) as f:
    long_description = f.read()

setup(
    name='jupyter-snippets',

    version='0.6',

    python_requires='>3.7',

    description='Useful jupyter startup/initialisation snippets',
    long_description=long_description,

    url='https://github.com/hottwaj/jupyter-snippets',

    author='Jonathan Clarke',
    author_email='jonathan.a.clarke@gmail.com',

    license='Copyright 2020',

    classifiers=[
    ],

    keywords='',

    packages=["jupyter_snippets"],

    install_requires=[
        "matplotlib",
        "seaborn",
        "ipython",
        "notebook",
        'beautifulsoup4'
    ],
)


from django.contrib.auth import get_user_model
from django.db import models

from CTRS_course_project.projection.models import Projection

UserModel = get_user_model()

class Reservation(models.Model):
    date = models.DateField(
        null=False,
        blank=False,
    )
    projection = models.ForeignKey(
        Projection,
        on_delete=models.CASCADE,
        null=False,
        blank=False,
    )
    user = models.ForeignKey(
        UserModel,
        on_delete=models.CASCADE,
        null=True,
        blank=True,
    )
    number_of_tickets = models.PositiveIntegerField(
        null=True,
        blank=True
    )
    type_of_tickets = models.TextField(
        null=True,
        blank=True,
    )
    total_price = models.DecimalField(
        max_digits=8,
        decimal_places=2,
        null=True,
        blank=True,
    )
    reserved_seats = models.TextField(
        null=True,
        blank=True,
    )
    is_finished = models.BooleanField(
        default=False,
        null=False,
        blank=False
    )

    def __str__(self):
        return f'{self.date} - {self.projection.movie} - {self.projection.hall} - {self.user}'


#parts of speech classification using logistic regression w/ softmax in keras and pytorch
#each word is a separate input, its tag is a separate output
# https://www.clips.uantwerpen.be/conll2000/chunking/

import numpy as np
import matplotlib.pyplot as plt
import string

#loading in data..
train = []
for l in open('train.txt'):
    train.append(l)
train = [t for t in train if len(t) > 2]

test = []
for l in open('test.txt'):
    test.append(l)
test = [t for t in test if len(t) > 2]

def tokenize(s):
    s = s.lower()
    # s = s.translate(str.maketrans('', '', string.punctuation)) #dont want to remove punctuations in this case
    words = s.split()
    return words

Xtrain = []
Ytrain = []
for l in train:
    words = tokenize(l)
    Xtrain.append(words[0])
    Ytrain.append(words[1])

Xtest = []
Ytest = []
for l in test:
    words = tokenize(l)
    Xtest.append(words[0])
    Ytest.append(words[1])

#word2idx
word2idx = {'UNK':0} #add an unknown key for test words not encountered during training
total_vocab = 1

for w in Xtrain:
    if w not in word2idx.keys():
        word2idx[w] = total_vocab
        total_vocab += 1

#target2idx - convert each target label to an idx
K = len(set(Ytrain)) #44 different labels
target2idx = {'UNK':0} #add an extra unknown label for test labels not encountered during training
total_labels = 1

for w in Ytrain:
    if w not in target2idx.keys():
        target2idx[w] = total_labels
        total_labels += 1

#convert train and test to their idx form
# Xtrain2 = np.zeros((len(Xtrain), total_vocab)) #<- cant do this, file would be way too large, just store as an array of idx
Xtrain2 = np.zeros((len(Xtrain)))
for n in range(len(Xtrain)):
        Xtrain2[n] = word2idx[Xtrain[n]]
Xtrain2 = Xtrain2.astype(int)

Ytrain2 = np.zeros((len(Ytrain)))
for n in range(len(Ytrain)):
        Ytrain2[n] = target2idx[Ytrain[n]]
Ytrain2 = Ytrain2.astype(int)

Xtest2 = np.zeros((len(Xtest)))
for n in range(len(Xtest)):
    word = Xtest[n]
    if word not in word2idx.keys():
        word = 'UNK'
    Xtest2[n] = word2idx[word]
Xtest2 = Xtest2.astype(int)

Ytest2 = np.zeros((len(Ytest)))
for n in range(len(Ytest)):
    label = Ytest[n]
    if label not in target2idx.keys():
        label = 'UNK'
    Ytest2[n] = target2idx[label]
Ytest2 = Ytest2.astype(int)

#
#logistic regresion w/ softmax in pytorch
import torch
from torch.autograd import Variable
from torch import optim
import torch.nn as nn

#Use GPU if possible, if not, then default to CPU
device = "cuda" if torch.cuda.is_available() else "cpu"
device = torch.device(device)

Xtrain2 = np.reshape(Xtrain2, (len(Xtrain2), 1, 1))
Xtest2 = np.reshape(Xtest2, (len(Xtest2), 1, 1))

Xtrain = torch.from_numpy(Xtrain2).long()
Ytrain = torch.from_numpy(Ytrain2).long()
Xtest = torch.from_numpy(Xtest2).long()
Ytest = torch.from_numpy(Ytest2).long()

class LogisticRegression(nn.Module):
    def __init__(self):
        super(LogisticRegression, self).__init__()

        self.emb = nn.Embedding(total_vocab, total_labels) #N x 1 x total_labels

    def forward(self, x):
        out = self.emb(x)
        out = out.reshape(-1,total_labels) #N x 1 x total_labels -> N x total_labels
        return out

model = LogisticRegression()
model.to(device)

#binary cross entropy loss
loss = torch.nn.CrossEntropyLoss(size_average=True)
loss.to(device)

optimizer = optim.Adam(model.parameters())

#train, and get train cost
def train(model, loss, optimizer, inputs, targets):
    model.train()

    inputs = Variable(inputs, requires_grad = False)
    inputs.to(device)
    targets = Variable(targets, requires_grad = False)
    targets.to(device)

    optimizer.zero_grad()

    out = model.forward(inputs)
    out = loss.forward(out, targets)

    out.backward()
    optimizer.step()
    return out.item()

#get a loss without training
def get_loss(model, loss, inputs, targets):
    model.eval()

    inputs = Variable(inputs, requires_grad = False)
    inputs.to(device)
    targets = Variable(targets, requires_grad = False)
    targets.to(device)

    out = model.forward(inputs)
    out = loss.forward(out, targets)
    return out.item()

#get a prediction
def predict(model, inputs):
    model.eval()

    inputs = Variable(inputs, requires_grad = False)

    pred = model.forward(inputs)
    pred = pred.data.cpu().numpy().argmax(axis = 1)
    return pred

#get a prediction
def get_accuracy(model, inputs, targets):
    p = predict(model, inputs)
    return np.round(np.mean(p == targets.cpu().numpy()), 2)

#params
B = 200
N = Xtrain.shape[0]
n_batches = int(N//B)
n_iter = 40

#store costs and acc
train_costs = []
test_costs = []
train_accs = []
test_accs = []

#train
for i in range(n_iter):
    train_cost = 0.

    for j in range(n_batches):
        Xbatch = Xtrain[j*B:(j+1)*B].to(device)
        Ybatch = Ytrain[j*B:(j+1)*B].to(device)

        train_cost += train(model, loss, optimizer, Xbatch, Ybatch)

    test_cost = get_loss(model, loss, Xtest.to(device), Ytest.to(device))

    train_acc = get_accuracy(model, Xtrain.to(device), Ytrain.to(device))
    test_acc = get_accuracy(model, Xtest.to(device), Ytest.to(device))

    print('iter: ', i,
          'train cost: ', train_cost / n_batches,
          'test cost: ', test_cost,
          'train accuracy: ', train_acc,
          'test accuracy: ', test_acc)

    train_costs.append(train_cost / n_batches)
    test_costs.append(test_cost)

    train_accs.append(train_acc)
    test_accs.append(test_acc)

pred = predict(model, Xtest.to(device))
from sklearn.metrics import f1_score

f1_score(Ytest2, pred, average=None).mean()


import glob
import os
import logging
import random

from modularirc import BaseModule

class Module(BaseModule):
    def cmd_quote(self, **kwargs):
        """!quote: to get a random quote"""
        quote = self.random_quote()
        if quote:
            return [quote]

    def random_quote(self):
        """Read a quote from a text file"""
        try:
            quote_dir = self.get_config('path')
        except:
            return 'Error: path not set'
        try:
            files = glob.glob(os.path.join(quote_dir, '*.txt'))
            quotes = []
            for filename in files:
                with open(filename, 'rt', encoding='utf-8') as f:
                    quotes.extend(f.readlines())
            return random.choice(quotes)
        except IOError as e:
            logging.exception('Quote IOError')
            return 'Error: quote file not found: {}'.format(e)
        except:
            logging.exception('Quote Exception')
            return 'Error: no quote file defined'


# Definition for a binary tree node.
class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None
import collections

class Codec:

    def serialize(self, root):
        """Encodes a tree to a single string.

        :type root: TreeNode
        :rtype: str
        """
        if root is None:
            return ""
        res = []
        queue = collections.deque()
        queue.append(root)
        while queue:
            node = queue.popleft()
            if node is not None:
                res.append(str(node.val))
                queue.append(node.left)
                queue.append(node.right)
            else:
                res.append("#")
        return ",".join(res)

    def deserialize(self, data):
        """Decodes your encoded data to tree.

        :type data: str
        :rtype: TreeNode
        """
        if data == "":
            return None

        nodes = data.split(",")

        i = 0
        queue = collections.deque()
        root = TreeNode(int(nodes[i]))
        i += 1
        queue.append(root)
        while queue:
            node = queue.popleft()
            if nodes[i] != "#":
                node.left = TreeNode(int(nodes[i]))
                queue.append(node.left)
            i += 1
            if nodes[i] != "#":
                node.right = TreeNode(int(nodes[i]))
                queue.append(node.right)
            i += 1

        return root

# Your Codec object will be instantiated and called as such:
codec = Codec()
codec.deserialize(codec.serialize(None))


'''
This module is responsible for preprocessing images:
grayscale, normalizing, smoothing etc
'''
from skimage import color
from skimage.exposure import rescale_intensity
import numpy as np
from scipy.ndimage.filters import gaussian_filter
def convert2grayscale(img):
    '''
    Takes an image (numpy.ndarray) and converts it to grayscale
    INPUT:
    img - numpy.ndarray
    OUTPUT
    img_gray - image in grayscale
    '''
    assert isinstance(img, np.ndarray)
    img_gray = color.rgb2gray(img)
    assert img_gray.shape == (img.shape[0], img.shape[1])
    return img_gray

def normalize_image(img, mean_flag=True, rescale_flag=True):
    '''
    Takes an image and
    - normalizes by the mean
    - rescales intensity to the range (0,255)
    INPUT:
    img - image of typ np.ndarray
    OUTPUT:
    img_norm - normalized image
    '''
    assert isinstance(img, np.ndarray)
    # normalize by the mean
    if mean_flag:
        img_norm = img - np.mean(img)
    else:
        img_norm = img
    # rescale intensity to (0,255)
    if rescale_flag:
        # avoid division by 0
        assert np.min(img_norm) != np.max(img_norm)
        img_norm = rescale_intensity(img_norm, out_range=(0, 255))
        assert np.max(img_norm) <= 255
        assert np.min(img_norm) >= 0
    assert img_norm.shape == img.shape
    assert isinstance(img_norm, np.ndarray)
    return img_norm

def smooth_image(img, sigma=2):
    '''
    Smooths image using a gaussian filter
    INPUT:
    img - image to be smoothed type
    sigma - standard deviation for Gaussian kernel
    OUTPUT
    img_smooth - smoothed image
    '''
    assert isinstance(img, np.ndarray)
    img_smooth = gaussian_filter(img, sigma)
    assert img.shape == img_smooth.shape
    assert isinstance(img_smooth, np.ndarray)
    return img_smooth


import unittest
from common import pro_path
from ddt import ddt,data
import logging
from common import my_logger
from common.do_excel import DoExcel
from common.http_request import HttpRequest

#获取测试数据
test_data = DoExcel(pro_path.test_data_path,'Sheet2').do_excel()
COOKIES = {} #全局变量

@ddt
class TestHttpRequest(unittest.TestCase):
    def setUp(self):
       self.t = DoExcel(pro_path.test_data_path, 'Sheet2') #创建一个实例

    @data(*test_data)
    def test_api(self,item):
        global COOKIES#声明全局变量，必要的时候去更新从cookie的值
        logging.info('正在执行第{0}条用例：{1}'.format(item['CaseId'],item['Title']))
        logging.info('发起请求的地址是：{0}'.format(item['URL']))
        logging.info('发起请求的参数是：{0}'.format(item['Param']))
        res = HttpRequest().http_request(item["URL"], eval(item["Param"]), item["Method"],cookies=COOKIES)
       #登录请求之后会产生一个cookie
        if res.cookies!={}:#如果cookie不为空  就对全局变量进行修改
            COOKIES = res.cookies
       # 断言
        try:
            self.assertEqual(item['ExpectedResult'],res.json()['Message'])
            TestResult = 'PASS'#存储测试用例的执行结果  通过是PASS  失败是fail
        except Exception as e:
            logging.error('出错了，错误是：{0}'.format(e))
            TestResult = 'FAIL'
            raise e
        finally:
            self.t.write_back(item['CaseId']+1,7,str(res.json()))#因为Excel里面只能支持传递字符串和整数，所以这里要强制转换一下
            self.t.write_back(item['CaseId'] + 1, 8, TestResult)


import jieba
import numpy
import re
import os
import json
from collections import defaultdict

spam_file_num=7775
normal_file_num=7063

#todo 原始邮件数据的预处理
'''
获取中文的停用词表
【停用词】（https://zh.wikipedia.org/wiki/%E5%81%9C%E7%94%A8%E8%AF%8D）
'''

def get_stopwords():
    return [
        i.strip() for i in open('./data/中文停用词表.txt', encoding='gbk')
    ]

'''
    读取原始文件，进行简单的处理和分词，返回分词列表

'''

def get_raw_str_list(path):
    stop_list = get_stopwords()
    with open(path, encoding='gbk') as f:
        raw_str = f.read()
    pattern = '[^\u4E00-\u9FA5]'  #中文的unicode编码的范围
    regex = re.compile(pattern)
    ret = regex.findall(raw_str)
    handled_str = re.sub(pattern, '', raw_str)
    str_list = [
        word for word in jieba.cut(handled_str) if word not in stop_list
    ]
    return str_list

#分词以及统计词频，得到词汇表
'''
返回一个字典，记录了分词后的词汇表
path:k可以使dir或者file路径，默认为存放文本的dir
is_file_path:表示输入路径是不是文件

'''

def get_voca(path, is_file_path=False):
    if is_file_path:
        return read_voca_from_file(path)

    voca = defaultdict(int)
    # 获取垃圾邮件列表
    file_list = [file for file in os.listdir(path)]
    #获得词汇表
    for file in file_list:  #测试用
        raw_str_list = get_raw_str_list(path + str(file))
        for raw_str in raw_str_list:
            voca[raw_str] = voca[raw_str] + 1

    return voca

'''
    将得到的词汇表保存到json文件中，以便下次读取
    voca:词汇表字典
    path:文件的保存路径
    sort_by_value:是否按value值排序

'''

def save_voca2json(voca, path, sort_by_value=False,indent_=4):
    if sort_by_value:
        sorted_by_value(voca)

    with open(path, 'w+') as f:
        f.write(json.dumps(voca, ensure_ascii=False, indent=indent_))

'''
从json文件中读取voca
'''

def read_voca_from_file(path):
    voca = None
    with open(path) as f:
        voca = json.load(f)
    return voca

'''
将字典基于value排序
'''

def sorted_by_value(_dict):
    _dict = dict(sorted(spam_voca.items(), key=lambda x: x[1], reverse=True))

#计算 P(Spam|word)

'''
    计算邮件和邮件分类最相关词语及其 P(spam|word)
    words_size:最终使用词语的数量，用于最后的预测

'''

def get_top_words_prob(path,spam_voca,normal_voca,words_size=30):
    critical_words=[]
    for word in get_raw_str_list(path):
        if word in spam_voca.keys() and word in normal_voca.keys():
            # 如果word在两边都出现
            p_w_s=spam_voca[word]/spam_file_num
            p_w_n=normal_voca[word]/normal_file_num
            p_s_w=p_w_s/(p_w_n+p_w_s)

        elif word in spam_voca.keys() and word not in normal_voca.keys():
            # 如果word只在spam出现
            p_w_s=spam_voca[word]/spam_file_num
            p_w_n=0.01
            p_s_w=p_w_s/(p_w_n+p_w_s)

        elif word not in spam_voca.keys() and word in normal_voca.keys():
            # 如果word只在normal出现
            p_w_s=0.01
            p_w_n=normal_voca[word]/normal_file_num
            p_s_w=p_w_s/(p_w_n+p_w_s)
        else:
            #如果都不出现
            p_s_w=0.4
        #以上设定的数值均是基于前人的研究得到的数据
        critical_words.append([word,p_s_w])

    return dict(sorted(critical_words[:words_size],key=lambda x:x[1],reverse=True))

'''
    计算贝叶斯概率
    words_prob：包含词汇以及其P(spam|word)概率的一个字典
    spam_voca:垃圾邮件词汇表
    normal_voca:正常邮件词汇表
'''

def caculate_bayes(words_prob,spam_voca,normal_voca):
    p_s_w=1
    p_s_nw=1
    for word,prob in words_prob.items():
        p_s_w*=prob
        p_s_nw*=(1-prob)

    return p_s_w/(p_s_w+p_s_nw)

def predict(bayes,threshold=0.9):
    return bayes>=threshold

'''
    返回文件名和label
'''

def get_files_labels(dir_path,is_spam=True):
    raw_files_list=os.listdir(dir_path)
    files_list= [dir_path+file for file in raw_files_list]
    labels=[is_spam for i in range(len(files_list))]
    return files_list,labels

'''
    预测测试集结果并打印
    file_list：测试集文件路径集合
    y:测试集结果标签
    word_size：预测的指标
'''

def predict_result(file_list,y,spam_voca,normal_voca,word_size=30):
    ret=[]
    right=0
    for file in file_list:
#         raw_strs=get_raw_str_list(str(file))
        words_prob=get_top_words_prob(file,spam_voca,normal_voca,words_size=word_size)
        bayes=caculate_bayes(words_prob,spam_voca,normal_voca)
        ret.append(predict(bayes))
    for i in range(len(ret)):
        if ret[i]==y[i]:
            right+=1

    print(right/len(y))


import os
import json
import logging

with open(os.path.join(os.getcwd(), "data", "statistics_parameters.json"), "r") as st_f:
    stat_prmtrs = json.load(st_f)

logging.basicConfig(
    # filename=os.path.join(PROJECT_ROOT_DIR, "data", "expert_bot_update.logs"),
    # filemode='a',
    level=logging.INFO,
    format='%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S', )

logger = logging.getLogger()
logger.setLevel(logging.INFO)