import re, emoji

class Twikenizer():

    def __init__(self):

        '''
        Symbols regex
          - Punctuation marks: ,.;:?!
          - Parenthesis: ()[]{}
          - Quotes: "'
          - Others: -

        Emojies regex
          - All unicode emojies

        Special characters regex
          - %$&€#~º+^<>|\£§¬/
        '''

        # Initializing regular expressions
        self.symbols_regex = r'["\'.!?,;:\()\[\]\{\}_-]'
        self.emojies_regex = emoji.get_emoji_regexp()
        self.special_chars_regex = r'^[%$&€#@~º+\^<>|\\£§¬\/]+$'

        self.other_inv_chars_regex = r'[%$&€#@~º+\^<>|\\£§¬\/]'
        self.invalid_hashtags_mentions_chars = r'["\'.!?,;:\()\[\]\{\}%$&€#@~º+\^<>|\\£§¬\/-]'

    def tokenize(self, tweet):

        '''
        Transforms a lists of raw tokens (whitespace separated) into a
        list of processed tokens using predefined metrics
        '''

        # Splitting first by whitespace
        raw_tokens = tweet.split()

        # Initializing list of final tokens
        tokens = []

        for raw_token in raw_tokens:

            # Initializing auxiliar variables
            subtokens, substring = [], ''

            '''
            First, we handle hashtags and mentions
            '''

            # Getting first character of the raw token
            first_char = raw_token[0]
            len_raw_token = len(raw_token)

            # Looking for valid hashtags
            if first_char == '#' and len_raw_token > 1:
                # Parsing raw token
                subtokens, valid_hashtag = self.parse_hashtags_mentions(raw_token)
                # Hashtag is fully valid - no more parsing needed
                if valid_hashtag == 1:
                    tokens += subtokens
                    continue
                # Left hand side of the hashtag is valid - right hands side needs parsing
                elif valid_hashtag == 0:
                    tokens.append(subtokens[0])
                    raw_token = subtokens[1]

            # Looking for valid mentions
            elif first_char == '@' and len_raw_token > 1:
                # Parsing raw token
                subtokens, valid_mention = self.parse_hashtags_mentions(raw_token, False)
                # Mention is fully valid - no more parsing needed
                if valid_mention == 1:
                    # Hashtag or mention handled correctly, nothing else to parse
                    tokens += subtokens
                    continue
                # Left hand side of the mention is valid - right hands side needs parsing
                elif valid_mention == 0:
                    # Hashtag or mention handled correctly, parse right hand side
                    tokens.append(subtokens[0])
                    raw_token = subtokens[1]

            '''
            - Valid hashtags and mentions have been handled.
            - Invalid hashtags and mentions and respective invalid right hand sides are parsed as any other raw token
                e.g. #hashtag       - Fully parsed at this point
                     #hashtag.lol   - Left hand side parsed (#hashtag), right hand side is parsed as any other token (.lol)
                     #$hashtag      - Invalid hashtag, parsed as a whole raw token (#$hashtag)

            - Now parsing the rest of the raw tokens
            '''

            # Initializing lists characters found per raw token
            symbols = re.findall(self.symbols_regex, raw_token)
            specials = re.findall(self.other_inv_chars_regex, raw_token)
            emojies = re.findall(self.emojies_regex, raw_token)
            special_chars = re.findall(self.special_chars_regex, raw_token)

            if (symbols and specials) or (symbols and specials and emojies) or (emojies and specials) or (emojies and symbols):
                subtokens = self.parser(raw_token, r'[a-zA-Z0-9]', True)
            # Parsing strings containing only symbols and alphanumerics
            elif symbols:
                subtokens = self.parser(raw_token, self.symbols_regex)
            # Parsing strings containing only emojies and alphanumerics
            elif emojies:
                subtokens = self.parser(raw_token, self.emojies_regex)
            # Parsing strings containing only special characters
            elif special_chars:
                subtokens += raw_token
            # Parsing alphanumeric tokens
            else:
                #print('5subtokens', subtokens)
                subtokens.append(raw_token)
                #print('5subtokens', subtokens)

            # Merging tokens
            tokens += subtokens

        return tokens

    def parser(self, raw_token, regex, general=False):

        '''
        - General parser -
        Parses any raw token that isn't a valid hashtag or mention
        '''

        # Initializing auxiliar variables
        subtokens, substring, position = [], '', 0

        # Processing each character individually
        for char in raw_token:

            # 'General' includes all possible combinations of emojies, special characters
            # and symbols (plus alphanumeric) characters
            if general:
                # Found alphanumeric character
                if re.match(regex, char):
                    substring += char
                    # Appending substring if this is the last position of the raw token
                    if position == len(raw_token) - 1 and substring:
                        subtokens.append(substring)
                # Found an emoji, special char or symbol
                else:
                    # Appending constructed substring before handling character
                    if substring:
                        subtokens.append(substring)
                    # If symbol, append substring to the tokens list
                    substring = '' # Reset substring after appending
                    subtokens.append(char) # Appending to tokens symbol char indivually

            # Raw token containg only alphanumeric characters plus symbols, emojies or special characters
            # Or only special characters
            else:
                # Found alphanumeric character
                if not re.match(regex, char):
                    substring += char
                    # Appending substring if this is the last position of the raw token
                    if position == len(raw_token) - 1 and substring:
                        subtokens.append(substring)
                # Found an emoji, special char or symbol
                else:
                    # Appending constructed substring before handling character
                    if substring:
                        subtokens.append(substring)
                    # If symbol, append substring to the tokens list
                    substring = '' # Reset substring after appending
                    subtokens.append(char) # Appending to tokens symbol char indivually
            # Updating raw token index
            position += 1

        return subtokens

    def parse_hashtags_mentions(self, raw_token, hashtag=True):

        '''
        - Hashtags and mentions parser -

        Returns (un)parsed token plus:
         -1: invalid           e.g. #-hashtag (no parsing done)
          0: partially valid   e.g. #hash.tag
          1: fully valid       e.g. #hashtag
        '''

        # Initializing validity of raw token (as invalid)
        valid_hashtag_mention = -1

        # Initializing auxiliar variables
        subtokens, substring, index = [], '', 0

        # Processing each character excluding the first one
        for char in raw_token[1:]:

            # Checking for invalid characters
            if not re.match(self.invalid_hashtags_mentions_chars, char):
                substring += char # Building substring hashtag until invalid token is found
                index += 1

            # Character is invalid
            else:
                # Invalid hashtag or mention (contains invalid character right after '#' or '@')
                if index == 0:
                    return [], valid_hashtag_mention
                break

        # Updating truth value of valid_hashtag_mention
        # Raw token as a whole is a valid hashtag or mention
        if substring and not raw_token[index+1:len(raw_token)]:
            valid_hashtag_mention = 1
        # Raw token contains a valid hashtag or mention but has an invalid right hand side
        else:
            valid_hashtag_mention = 0

        # Tokenizes valid hashtag / mention
        subtokens.append('#' + substring) if hashtag else subtokens.append('@' + substring)

        # Tokenizes rest of the (invalid) hashtag so it can be handled as any other split
        if raw_token[index+1:len(raw_token)]:
            subtokens.append(raw_token[index+1:len(raw_token)])

        return subtokens, valid_hashtag_mention

    # Exemplifies how Twikenizer handles different situations
    def examplify(self):

        tweet = 'Tw33t a_!aa&!a?b #%lol # @dude_really #hash_tag $hit (g@y) (retard#d) @dude. \U0001F600\U0001F600 !\U0001F600abc %\U0001F600lol #hateit #hate.it $%&/ f*ck-'

        print('Generated tweet\n---------------')
        print(tweet + '\n')

        tokens = self.tokenize(tweet)

        print('Generated tokens\n----------------')
        print(tokens)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Set process name and cpu priority,return socket.socket or None."""

import logging as log
import socket
import sys

def set_single_instance(name, port=8888):
    """Set process name and cpu priority,return socket.socket or None."""
    try:  # Single instance app ~crossplatform, uses udp socket.
        log.info("Creating Abstract UDP Socket Lock for Single Instance.")
        __lock = socket.socket(
            socket.AF_UNIX if sys.platform.startswith("linux")
            else socket.AF_INET, socket.SOCK_STREAM)
        __lock.bind(
            "\0_{name}__lock".format(name=str(name).lower().strip())
            if sys.platform.startswith("linux") else ("127.0.0.1", port))
    except (socket.error, OSError, Exception) as e:
        __lock = None
        log.critical("Another instance of App is already running!, Exiting!.")
        log.exception(e)
        sys.exit()
        exit()
    else:
        log.info("Socket Lock for Single Instance: {}.".format(__lock))
    finally:
        return __lock


from typing import List
import random
import numpy as np
from deap import base, creator, tools, algorithms, cma
import os, sys

p = os.path.abspath('.')
sys.path.insert(1, p)

from controllers.deap_controller import player_controller_demo
from assignment2.environment import New_Environment as Environment

def create_environment(name: str, enemies: List[int]):
    environment = Environment(
        experiment_name=name,
        multiplemode="yes",  # yes or no
        enemies=enemies,  # array with 1 to 8 items, values from 1 to 8
        level=2,  # integer
        playermode="ai",  # ai or human
        player_controller=player_controller_demo(10),
        enemymode="static",  # ai or static
        speed="fastest",  # normal or fastest
        randomini="yes",  # yes or no
    )
    return environment

def evaluate(individual, env):
    f,p,e,t = env.play(pcont=individual)
    # print('Overall results:', f,p,e,t)
    return (f,)

# --- from DEAP documentation: https://deap.readthedocs.io/en/master/examples/es_fctmin.html
def generateES(icls, size, imin, imax):
    ind = icls(random.uniform(imin, imax) for _ in range(size))
    return ind

def best_individual(population):
    fitnesses = [ind.fitness.values for ind in population]
    return population[np.argmax(fitnesses)]

def run_deap(env, **config):
    n_pop = config.get('n_pop', 150)
    n_gens_ga = config.get('n_gens_ga', 30)
    n_gens_cma = config.get('n_gens_cma', 15)
    mutpb = config.get('mutpb', 0.25)
    indpb = config.get('indpb', 0.25)
    tournsize = config.get('tournsize', 2)
    cxpb = config.get('cxpb', 0.5)
    optimize = config.get('optimize', False)
    """
        n_pop: The population size
        n_gens_ga: The number of generations for ga
        n_gens_cma: The number of generations for cma
        mutpb: The probability of mutating an individual.
        indpb: Independent probability for each attribute to be mutated.
        tournsize: Number of individuals participating in a tournament
        cxpb: The probability of mating two individuals.
        mu: The mean for Gaussian operations
        sigma: The standard deviation for Gaussian operations.
        optimize: A boolean value for running the optimize algorithm
    """

    print('Setting up DEAP!\n')

    ind_size = 265 #number of nodes in network (250 weights (20*10*5) + 15 bias (10+5))

    # Creation of required types
    creator.create("FitnessMax", base.Fitness, weights=(1.0,))
    creator.create("Individual", np.ndarray, fitness=creator.FitnessMax)

    toolbox = base.Toolbox()

    # --- partially from DEAP documentation: https://deap.readthedocs.io/en/master/examples/es_fctmin.html
    MIN_VALUE = -5
    MAX_VALUE = 5

    toolbox.register("attr_float", random.random)
    toolbox.register("individual", generateES, creator.Individual, ind_size, MIN_VALUE, MAX_VALUE)

    toolbox.register("population", tools.initRepeat, list, toolbox.individual)
    toolbox.register('mate', tools.cxOnePoint)
    toolbox.register("mutate", tools.mutGaussian, mu=0, sigma=0.15, indpb=indpb)
    toolbox.register("select", tools.selTournament, tournsize=tournsize)
    toolbox.register("evaluate", evaluate, env=env)

    # --- Tools for saving the statistics
    stats = tools.Statistics(lambda ind: ind.fitness.values)
    stats.register("Mean", np.mean)
    stats.register("Max", np.max)
    stats.register("Min", np.min)

    pop = toolbox.population(n=n_pop)
    hof = tools.HallOfFame(1, similar=np.array_equal)

    print('Setup is complete, now starting GA!\n')

    pop_ga, log_ga = algorithms.eaSimple(pop, toolbox, cxpb=cxpb, mutpb=mutpb, ngen=n_gens_ga, halloffame=hof, stats=stats)

    print('GA is done, CMA-ES next!')

    stats_per_gen = []

    for gen in log_ga:
        stats_per_gen.append([gen['Mean'], gen['Max']])

    if n_gens_cma > 0:
        # implementation of CMA-ES
        centroid = best_individual(pop_ga)
        strategy= cma.Strategy(centroid, sigma=0.05)
        toolbox.register("generate", strategy.generate, creator.Individual)
        toolbox.register("update", strategy.update)
        pop_cma, log_cma = algorithms.eaGenerateUpdate(toolbox=toolbox, ngen=n_gens_cma, stats=stats, halloffame=hof)

        for gen in log_cma:
            stats_per_gen.append([gen['Mean'], gen['Max']])

    if optimize:
        return hof[0].fitness.values[0]

    return stats_per_gen, hof[0]

def main(**config):
    experiment_name = config.get('name', 'test')
    enemies = config.get('enemies', [1,4])
    env = create_environment(experiment_name, enemies)
    return run_deap(env, **config)

if __name__ == '__main__':
    # choose this for not using visuals and thus making experiments faster
    headless = True
    if headless:
        os.environ["SDL_VIDEODRIVER"] = "dummy"
    config = dict()
    main(**config)


from SimpleXMLRPCServer import SimpleXMLRPCServer
from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler

from parser import get_paras

import seq2seq_attention_decode_server
import os
import nltk
import re
import codecs

# Special tokens
PARAGRAPH_START = '<p>'
PARAGRAPH_END = '</p>'
SENTENCE_START = '<s>'
SENTENCE_END = '</s>'
DOCUMENT_START = '<d>'
DOCUMENT_END = '</d>'

UNKNOWN_TOKEN = '<UNK>'

# Restrict to a particular path.
class RequestHandler(SimpleXMLRPCRequestHandler):
    rpc_paths = ('/RPC2',)

def sentence_tokenizer():
	# Train a Punkt tokenizer if required
	return nltk.data.load('tokenizers/punkt/english.pickle')

def word_tokenizer():
	return nltk.tokenize.treebank.TreebankWordTokenizer()

def remove_digits(parse):
	return re.sub(r'\d', '#', parse)

sentence_segmenter = sentence_tokenizer()
word_tok = word_tokenizer()

def get_doc_from_tokens(tokens):
    docuemnt = [DOCUMENT_START] + tokens + [DOCUMENT_END]
    return " ".join(docuemnt)

def process_sentence(sentence):
	## Add more processing if required
    tokens = [SENTENCE_START]
    tokens.extend(word_tok.tokenize(sentence))
    tokens.append(SENTENCE_END)
    return tokens

def para_to_tokens(paragraph):
	return [token for sent in sentence_segmenter.tokenize(paragraph.strip())
        for token in process_sentence(sent)]

def getDocumentFromJson(document_json):
    docuemnt = [DOCUMENT_START]
    for key in document_json:
        section = document_json[key]
        section_name = key
        tokens = para_to_tokens(section)
        docuemnt.extend(tokens)
    docuemnt.append(DOCUMENT_END)
    return " ".join(docuemnt)

def pre_example_process(text):
    #print(text)
    text = text.replace("< ref >", "||REF||")
    return remove_digits(text).lower()

def article_to_tokens(article):
    return [token for sent in article.split('\n')
        for token in process_sentence(sent)]

class DecodeServer(object):

	def __init__(self, decoder_for_server, port):
		# Create server
		self.server = SimpleXMLRPCServer(("0.0.0.0", port),
		                            requestHandler=RequestHandler)
		self.server.register_introspection_functions()

		self.decoder = decoder_for_server

		self.server.register_function(self.summarize)

		self.server.register_function(self.update_session)

		self.server.register_function(self.summarize_with_string)

		print("Server running at port "+str(port))

		self.server.serve_forever()

	def update_session(self):
		self.decoder.update_session()
		return "done"

	def summarize_with_string(self, latex):
		inp_file = codecs.open("/home/bt1/13CS10060/snlp16/Test_Data/Input/temp.tex", "w", "utf-8")
		inp_file.write(latex)
		inp_file.close()

		return self.summarize("temp.tex")

	def summarize(self, filename):

		summary = ""
		## Preprocess article to generate input for the model

		# Parse Latex File - Convert to Text
		os.system('python2 ../../../Codes/pylatexenc/parse_latex_file.py '+filename)

		# Parse Text File - Convert to Dict (Section: Text)
		os.system('python2 ../../../Codes/pylatexenc/parse_text_file.py '+filename)

		# Parse Dict - Dict to File

		print("Here1")
		(para_list, actual_abstract) = get_paras(filename)

		print("JHere2")

		print(len(para_list), actual_abstract)
		print(para_list[0])

		article_list = []
		for para in para_list:
			article = pre_example_process(getDocumentFromJson({"abstract" :para}))
			if(len(article.split(" ")) < 4):
				continue
			article_list.append(article)

		print("Here3")

		if(len(article_list) < 1):
			return (actual_abstract, " ")

		#print(article)

		summary_list = []
		for article in article_list[0:1]:
			summary = self.decoder.DecodeOne(article)
			summary_list.append(summary)

		summary = " ".join(summary_list)

		print("ehre4")

		return (actual_abstract,summary)

## Test Case

# source = codecs.open("my.tex", "r","utf-8")
# print( summarize_with_string(source.read()))
# source.close()
#summarize('comm_magazine_FDMIMO_overview.tex')


import os
from PIL import Image
import numpy as np
import shutil

def mkdirs(paths):
    if isinstance(paths, list) and not isinstance(paths, str):
        for path in paths:
            mkdir(path)
    else:
        mkdir(paths)

# if directory does not already exist, create path
def mkdir(path):
    if not os.path.exists(path):
        os.makedirs(path)

# return image saved in path
def load_image(path):
    with Image.open(path).convert('L') as img:      # convert image to b&w
        img = np.array(img)
        img = np.expand_dims(img, axis=2)       # expand shape of array, new axis is placed in position 2
        return img

#
def load_label(txt):        # txt = dataroot, cap_scheme, label, label + '_train/test.txt'
    with open(txt, 'r') as fp_txt:
        labels = fp_txt.read()
        labels = labels.split('#')
        return labels

# return dictionary created from character set used in captcha scheme
def char2dict(char_set):
    dict = {}
    for i, char in enumerate(char_set):
        dict[i] = char
    return dict

# return character dictionary used in given captcha scheme
def create_dict(cap_scheme):
    char_set = []
    scheme = CaptchaSchemes()
    if cap_scheme == 'min24':
        char_set = scheme.min24
    return char2dict(char_set)

# vector -> text
# vector: indicate where character position in dictionary
def vec2text(vect, dict):
    text = ''
    for i, key in enumerate(vect):
        value = dict[key]
        text += value
    return text

# print result
def print_predict(opt):
    scheme = CaptchaSchemes()
    preds = opt.preds
    preds_right_num = 0     # initialize
    char_set = []       # initialize
    total_num = len(preds)
    print('total_num : ', total_num)
    if opt.cap_scheme == 'min24':       # if the scheme is 민원24
        char_set = scheme.min24     # return character set of min24
    dict = char2dict(char_set)      # and create dictionary based on set
    if opt.isTune:
        label = 'real'
    else:
        label = 'synthetic'    # create and sort labels
    if opt.phase == 'train':
        real_label_name = os.path.join(opt.dataroot, opt.cap_scheme, label, label + '_train.txt')
    else:       # if phase is validation or testing
        real_label_name = os.path.join(opt.dataroot, opt.cap_scheme, label, label + '_test.txt')
    real_labels = load_label(real_label_name)
    for i, pred in enumerate(preds):        # for all images
        pred = vec2text(pred, dict)     # recognized text value of iamge
        if pred.lower() == real_labels[i].lower():      # convert to lowercase and compare
            preds_right_num += 1
            print('Correct Prediction')
        print('No.{} \t Predict: {} \t Real: {}' .format(i, pred, real_labels[i]))
    print('Recognition Accuracy : ', preds_right_num/total_num)

# define character set of each captcha scheme
class CaptchaSchemes:
    def __init__(self):
        self.min24 = [
            '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'
        ]

#
if __name__ == '__main__':
    dict = create_dict('min24')     # create dictionary for min24 captcha scheme
    c = 'Z'

    def char2pos(c):
        k = -1
        for (key, value) in dict.items():
            if value == c:
                k = key
        return k

    key = char2pos(c)
    print(key)

def create_labels():
    img_path = '../data/min24/synthetic/min240_4246/'
    img_save_path = '../data/min24/synthetic/min244000-4246/'

    j = 0
    for i in range(4000, 4247, 1):
        shutil.copyfile(img_path + str(i) + '.jpg',
                        img_save_path + str(j) + '.jpg')
        j += 1


import pygame
import math
import random
class Cursor:

    def __init__(self, screen, width):
        self.start = pygame.mouse.get_pos()
        self.colour = [250, 250, 250]
        self.screen = screen
        self.width = width
        self.cursor = pygame.image.load('cursor.png')
        self.cursor = pygame.transform.scale(self.cursor, (20, 20))

    def update(self):

        curr_pos = [i-10 for i in list(pygame.mouse.get_pos())]

        self.screen.blit(self.cursor, curr_pos)

class Planet:

    def __init__(self, screen, id, GRAV):
        self.screen = screen
        self.last_pos = None
        self.pos = list(pygame.mouse.get_pos())
        self.GRAV = 1
        self.DENSITY = 1
        self.radius = 1
        self.growing = True
        self.id = id
        self.vel = [0,0] if self.id == 0 else  [random.uniform(0,0.10),random.uniform(0,0.10)]

    def grow(self):
        if pygame.mouse.get_pressed()[0] == 1 and self.growing:
            self.radius += 0.5
        else:
            self.growing = False
            print(self.radius)

    def get_volume(self):
        return (4/3) * math.pi * ((self.radius) ** 3)

    def get_mass(self):
        return self.get_volume() * self.DENSITY

    def set_vels(self, planets):
        if not self.growing:
            for planet in planets:
                if planet.id != self.id and not planet.growing:
                    dt = 0.00001
                    #r_hat = self.pos
                    # calculate x and y total distance
                    r = tuple(map(lambda i, j: i -j, planet.pos, self.pos))
                    distance = math.sqrt(r[0]**2 + r[1]**2)
                    # calculate gravitational force
                    G = self.GRAV * ((planet.get_mass() * self.get_mass()) / (distance ** 2))

                    # get rs
                    r_mag = math.sqrt(r[0]**2 + r[1]**2)
                    r_hat = tuple([i/r_mag for i in r])

                    r_hat_x = ((self.GRAV * (self.get_mass()/r_mag**2)) * r_hat[0]) * dt
                    r_hat_y = ((self.GRAV * (self.get_mass()/r_mag**2)) * r_hat[1]) * dt
                    planet.vel[0] -= r_hat_x
                    planet.vel[1] -= r_hat_y

    def update_pos(self):
        self.pos[0] += self.vel[0]
        self.pos[1] += self.vel[1]
        self.last_pos = self.pos

    def update(self, planets):

        if self.growing:
            self.grow()

        self.set_vels(planets)
        self.update_pos()
        #print(self.vel, self.pos)
        pygame.draw.circle(self.screen, (255,255,255), self.pos, self.radius)
        if self.id == 1:
            print(self.pos)


# -*- coding:utf-8 -*-
import argparse
import threading
import time
from Modules import ReadyWork
from Modules import HttpServer
from Modules import SecureHttpServer
from Modules import StartAP_airbase

def main():
    parser = argparse.ArgumentParser(description="一款简单的伪造AP钓鱼工具")
    parser.add_argument("--route_page",default='original',help="指定登录验证页面类型")
    parser.add_argument("--wifiname",default="Iphone",help="指定伪造WIFI名称")
    args = parser.parse_args()

    routepage_dict={'original':"Modules/access-point-pages/minimal", \
                'mianliao':'Modules/access-point-pages/mianliao'}

    route_page=routepage_dict[args.route_page]

    ReadyWork.Start(IP="127.0.0.1",PORT=8080,SSL_IP="127.0.0.1",SSL_PORT=443)

    ts=[]
    ts.append(threading.Thread(target=HttpServer.Start, args=('127.0.0.1',8080,route_page,)))
    ts.append(threading.Thread(target=SecureHttpServer.Start, args=("127.0.0.1",443,'127.0.0.1',8080,)))
    for t in ts:
        t.setDaemon(True)
        t.start()

    time.sleep(3) #等待上面线程完成准备工作，否则会出现显示混乱
    AP=StartAP_airbase.StartAP(a_iface='eth0',ap_iface="wlan0",wifiname=args.wifiname,channel=1)
    AP.run()

main()


from stats.models import StatLine, CustomStat, CustomMetric
from stats.interpreter import interpreter
from django.db import transaction

def get_custom_metric(username, label, metric_string):
    """
    [get_custom_metric] returns a CustomMetric object with fields
        equal to username and metric_string. If there is currently
        a metric with the same values, it will delete it and create
        a new one to return.
    [username]: string
    [label]: string
    [metric_string]: string
    Returns: CustomMetric object
    """
    metric_queryset = CustomMetric.objects.filter(
        username__exact = username
    ).filter(
        label__exact = label
    )

    if len(metric_queryset) > 0:
        metric = metric_queryset.get(username = username)
        metric.customstat_set.all().delete()
        metric.delete()

    # If there is no current metric with the given values
    metric = CustomMetric(username = username, label = label, string = metric_string)
    metric.save()

    return metric

@transaction.atomic
def assign_values(metric, objects):
    """
    [assign_values] takes in a metric object and a queryset of StatLine
        objects and creates a CustomStat for each statline in the queryset.
    [metric]: CustomMetric object
    [objects]: StatLine queryset
    Returns: None
    """
    delete_metric_stats(metric)
    metric_string = metric.string

    # statline_to_value is a function that optimizes for repeated
    # calculation of aggregate functions
    # Takes in a statline and returns a float or error
    statline_to_value = interpreter(metric_string, objects)

    stats = [
        CustomStat(
            player = statline,
            value = statline_to_value(statline),
            metric = metric
            ) for statline in objects
        ]

    # If there was an error, return it
    # otherwise, create the stats and return the empty string
    if len(stats) > 0 and type(stats[0].value) == type(""):
            return stats[0].value
    else:
        CustomStat.objects.bulk_create(stats)
        return ""

def delete_metric_stats(metric):
    """
    [delete_metric_stats] is used to delete all CustomStat objects associate
        with the given metric. This is to be called right before assigning
        new objects to the metric so that all old objects are cleared.
    [metric]: CustomMetric object
    Returns: None
    """
    for stat in metric.customstat_set.all():
        stat.delete()


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.

author: Sammi Chekroud
"""
import numpy   as np
import scipy   as sp
from scipy import interpolate
import copy
import sys
import os

class DataError(Exception):
    '''
    Raised when there is an error in the datafile and expectations do not align with what is present
    '''
    def _init_(self, msg):
        self.msg = msg

class ArgumentError(Exception):
    """
    Raised when there is an error in the arguments used in a function here
    """
    def _init_(self, msg):
        self.msg = msg

def Eucdist(x1, y1, x2, y2):
    """
    calculate euclidian distance between two points

    formula: sqrt( (x2-x1)^2 + (y2-y1)^2 )

    """
    distance = np.sqrt( (x2-x1)**2 + (y2-y1)**2)
    return distance

def parse_eye_data(eye_fname, block_rec, trial_rec, nblocks, ntrials = None):
    """

    eye_fname    -- full path to a file to be parsed. this is used as a template to create the parsed data in pickle form

    block_rec    -- boolean as to whether you stopped/started recording blockwise in your task

    trial_rec    -- boolean as to whethre you stopped/started recording trialwise in your task

    nblocks      -- the number of blocks of task run in your experiment. This must be defined in all cases.

    ntrials      -- the number of trials in your data if you recorded the eyes with trialwise stop/start
                    if you recorded blockwise, then ntrials is not needed.

                    if you recorded trialwise, then the number of trials per block
                    is calculated and used to separate the data into defined blocks
    """
    if not os.path.exists(eye_fname): #check that the path to the file exists
        raise Exception('the filename: %s does not exist. Please check!' %eye_fname)

    if 'block_rec' not in locals() and 'trial_rec' not in locals():
        raise ArgumentError('Please indicate whether the recording was stopped/started on each trial or each block')

    if 'block_rec' in locals():
        if block_rec == True:
            trial_rec = False
    if 'trial_rec' in locals():
        if trial_rec == True:
            block_rec = False

    if 'block_rec' in locals():
        if block_rec == True and 'nblocks' not in locals():
            raise ArgumentError('Please specify how many blocks you expect in your data recording')

    if 'trial_rec' in locals():
        if trial_rec == True and 'ntrials' not in locals():
            raise ArgumentError('Please specify how many trials you expect in your data recording')
        elif trial_rec == True and 'ntrials' in locals() and ntrials == None:
            raise ArgumentError('Please specify a number of trials to expect')
        elif trial_rec == True and 'ntrials' in locals() and ntrials != None and 'nblocks' not in locals():
            raise ArgumentError('Please specify how many blocks of data you recorded')

    if block_rec:
        d = _parse_eye_data_blockwise(eye_fname, nblocks)
    else:
        d = _parse_eye_data_trialwise(eye_fname, nblocks, ntrials)

    return d #return the parsed data

def _parse_eye_data_blockwise(eye_fname, nblocks):
    """
    this function is called by parse_eye_data and will operate on data where
    the recording was stopped/started for each block of task data
    """

    d = open(eye_fname, 'r')
    raw_d = d.readlines()
    d.close()

    split_d = []
    for i in range(len(raw_d)):
        tmp = raw_d[i].split()
        split_d.append(tmp)

    start_inds = [x for x in range(len(split_d)) if len(split_d[x]) == 6 and split_d[x][0] == 'START']
    if len(start_inds) != nblocks:
        raise DataError('%d blocks are found in the data, not %d as has been input in nblocks' %(len(start_inds),nblocks))

    #get points where recording stopped
    end_inds   = [x for x in range(len(split_d)) if len(split_d[x]) == 7 and split_d[x][0] == 'END']

    if len(start_inds) != len(end_inds):
        raise DataError('the number of times the recording was started and stopped does not align. check problems with acquisition')

    #assign some empty lists to get filled with information
    trackertime = []
    lx   = []; rx   = []
    ly   = []; ry   = []
    lp   = []; rp   = []

    Efix = []; Sfix = []
    Esac = []; Ssac = []
    Eblk = []; Sblk = []
    Msg  = []

    for i in range(len(start_inds)):
        start_line = start_inds[i]
        end_line   = end_inds[i]

        iblk = np.array(split_d[start_line:end_line]) #get only lines with info for this block

        iblk_event_inds, iblk_events     = [], []
        iblk_blink_inds, iblk_blink      = [], []
        iblk_fix_inds, iblk_fix          = [], []
        iblk_sac_inds, iblk_sac          = [], []
        iblk_input_inds                  = []
        for x,y in np.ndenumerate(iblk):
            if y[0] == 'MSG':
                iblk_event_inds.append(x[0]) # add line index where a message was sent to the eyetracker
                iblk_events.append(y)        # add the line itself to list
            elif y[0] in ['EFIX', 'SFIX']:
                iblk_fix_inds.append(x[0])   # add line index where fixation detected (SR research)
                iblk_fix.append(y)           # add fixation event structure to list
            elif y[0] in ['ESACC', 'SSACC']:
                iblk_sac_inds.append(x[0])   # add line index where saccade detected (SR research)
                iblk_sac.append(y)           # add saccade event structure to list
            elif y[0] in ['EBLINK', 'SBLINK']:
                iblk_blink_inds.append(x[0]) # add line index where blink detected (SR research)
                iblk_blink.append(y)         # add blink event structure to list
            elif y[0] == 'INPUT':
               iblk_input_inds.append(x[0])  # find where 'INPUT' is in data (sometimes appears, has no use...)

        #get all non-data line indices
        iblk_nondata    = sorted(iblk_blink_inds + iblk_sac_inds + iblk_fix_inds + iblk_event_inds + iblk_input_inds)

        iblk_data = np.delete(iblk, iblk_nondata) #remove these lines so all you have is the raw data

        iblk_data = iblk_data[6:] #remove first five lines as these are filler after the recording starts

        try:
            iblk_data = np.vstack(iblk_data)
        except ValueError:
            for item in range(len(iblk_data)):
                iblk_data[item] = iblk_data[item][:7] #make sure each item has only 7 strings. sometimes the eyetracker hasn't appended the random .... at the end
            iblk_data = np.vstack(iblk_data) #shape of this should be number of columns in the file (data)

        iblk_data = iblk_data[:,:7]            # only take first 7 columns as these contain data of interest. last one is redundant (if exists)

        #before you can convert to float, need to replace missing data where its '.' as nans (in pupil this is '0.0')
        eyecols = [1,2,4,5] #leftx, lefty, rightx, right y col indices
        for col in eyecols:
            missing_inds = np.where(iblk_data[:,col] == '.') #find where data is missing in the gaze position, as probably a blink (or its missing as lost the eye)
            for i in missing_inds:
                iblk_data[i,col] = np.NaN #replace missing data ('.') with NaN
                iblk_data[i,3]   = np.NaN #replace left pupil as NaN (as in a blink)
                iblk_data[i,6]   = np.NaN #replace right pupil as NaN (as in a blink)

        iblk_data = iblk_data.astype(np.float) # convert data from string to floats for computations

        #for binocular data, the shape is:
        # columns: time stamp, left x, left y, left pupil, right x, right y, right pupil
        iblk_trackertime = iblk_data[:,0]
        iblk_lx, iblk_ly, iblk_lp = iblk_data[:,1], iblk_data[:,2], iblk_data[:,3]
        iblk_rx, iblk_ry, iblk_rp = iblk_data[:,4], iblk_data[:,5], iblk_data[:,6]

        # split Efix/Sfix and Esacc/Ssacc into separate lists
        iblk_efix = [iblk_fix[x] for x in range(len(iblk_fix)) if
                     iblk_fix[x][0] == 'EFIX']
        iblk_sfix = [iblk_fix[x] for x in range(len(iblk_fix)) if
                     iblk_fix[x][0] == 'SFIX']

        iblk_ssac = [iblk_sac[x] for x in range(len(iblk_sac)) if
                     iblk_sac[x][0] == 'SSACC']
        iblk_esac = [iblk_sac[x] for x in range(len(iblk_sac)) if
                     iblk_sac[x][0] == 'ESACC']
        iblk_sblk = [iblk_blink[x] for x in range(len(iblk_blink)) if
                     iblk_blink[x][0] == 'SBLINK']
        iblk_eblk = [iblk_blink[x] for x in range(len(iblk_blink)) if
                     iblk_blink[x][0] == 'EBLINK']

        #append to the collection of all data now
        trackertime.append(iblk_trackertime)
        lx.append(iblk_lx)
        ly.append(iblk_ly)
        rx.append(iblk_rx)
        ry.append(iblk_ry)
        lp.append(iblk_lp)
        rp.append(iblk_rp)
        Efix.append(iblk_efix)
        Sfix.append(iblk_sfix)
        Ssac.append(iblk_ssac)
        Esac.append(iblk_esac)
        Sblk.append(iblk_sblk)
        Eblk.append(iblk_eblk)
        Msg.append(iblk_events)

    iblock = {
    'trackertime': [],
    'lx'  : [], 'ly'  : [], 'lp'  : [],
    'rx'  : [], 'ry'  : [], 'rp'  : [],
    'Efix': [], 'Sfix': [],
    'Esac': [], 'Ssac': [],
    'Eblk': [], 'Sblk': [],
    'Msg' : []
    }
    dummy = copy.deepcopy(iblock)
    blocked_data = np.repeat(dummy, nblocks)

    for i in np.arange(nblocks):
        iblock_data = copy.deepcopy(iblock)
        iblock_data['trackertime'] = trackertime[i]
        iblock_data['lx'] = lx[i]
        iblock_data['ly'] = ly[i]
        iblock_data['lp'] = lp[i]
        iblock_data['rx'] = rx[i]
        iblock_data['ry'] = ry[i]
        iblock_data['rp'] = rp[i]
        iblock_data['Efix'] = Efix[i]
        iblock_data['Sfix'] = Sfix[i]
        iblock_data['Esac'] = Esac[i]
        iblock_data['Ssac'] = Ssac[i]
        iblock_data['Eblk'] = Eblk[i]
        iblock_data['Sblk'] = Sblk[i]
        iblock_data['Msg'] = Msg[i]
        blocked_data[i] = iblock_data

    return blocked_data

def _parse_eye_data_trialwise(eye_fname, nblocks, ntrials):

    """
    this function is called by parse_eye_data and will operate on data where the recording was stopped/started for each trial of task data

    """

    d = open(eye_fname, 'r')
    raw_d = d.readlines()
    d.close()

    split_d = []
    for i in range(len(raw_d)):
        tmp = raw_d[i].split()
        split_d.append(tmp)

    #get all lines where 'START'i s seen, as this marks the start of the recording
    start_inds = [x for x in range(len(split_d)) if len(split_d[x]) == 6 and split_d[x][0] == 'START']
    if len(start_inds) != ntrials:
        raise DataError('%d trials are found in the data, not %d as has been input in ntrials' %(len(start_inds),ntrials))

    #get points where recording stopped
    end_inds   = [x for x in range(len(split_d)) if len(split_d[x]) == 7 and split_d[x][0] == 'END']

    if len(start_inds) != len(end_inds):
        raise DataError('the number of times the recording was started and stopped does not align. check problems with acquisition')

    #assign some empty lists to get filled with information
    trackertime = []
    lx   = []; rx   = []
    ly   = []; ry   = []
    lp   = []; rp   = []

    Efix = []; Sfix = []
    Esac = []; Ssac = []
    Eblk = []; Sblk = []
    Msg  = []

    for i in range(len(start_inds)):
        start_line = start_inds[i]
        end_line   = end_inds[i]

        itrl = np.array(split_d[start_line:end_line])

        itrl_event_inds, itrl_events     = [], []
        itrl_blink_inds, itrl_blink      = [], []
        itrl_fix_inds, itrl_fix          = [], []
        itrl_sac_inds, itrl_sac          = [], []
        itrl_input_inds                  = []
        for x,y in np.ndenumerate(itrl):
            if y[0] == 'MSG':
                itrl_event_inds.append(x[0]) # add line index where a message was sent to the eyetracker
                itrl_events.append(y)        # add the line itself to list
            elif y[0] in ['EFIX', 'SFIX']:
                itrl_fix_inds.append(x[0])   # add line index where fixation detected (SR research)
                itrl_fix.append(y)           # add fixation event structure to list
            elif y[0] in ['ESACC', 'SSACC']:
                itrl_sac_inds.append(x[0])   # add line index where saccade detected (SR research)
                itrl_sac.append(y)           # add saccade event structure to list
            elif y[0] in ['EBLINK', 'SBLINK']:
                itrl_blink_inds.append(x[0]) # add line index where blink detected (SR research)
                itrl_blink.append(y)         # add blink event structure to list
            elif y[0] == 'INPUT':
               itrl_input_inds.append(x[0])  # find where 'INPUT' is in data (sometimes appears, has no use...)

        #get all non-data line indices
        itrl_nondata    = sorted(itrl_blink_inds + itrl_sac_inds + itrl_fix_inds + itrl_event_inds + itrl_input_inds)

        itrl_data = np.delete(itrl, itrl_nondata) #remove these lines so all you have is the raw data

        itrl_data = itrl_data[6:] #remove first five lines as these are filler after the recording starts

        try:
            itrl_data = np.vstack(itrl_data) #if this fails its most likely because the end string of dots is missing for some reason
        except ValueError: #if fails, fix
            for item in range(len(itrl_data)):
                itrl_data[item] = itrl_data[item][:7] #make sure each item has only 7 strings. sometimes the eyetracker hasn't appended the random .... at the end

        itrl_data = itrl_data[:,:7]            # only take first 6 columns as these contain data of interest (redundant if above section trips up

        itrl_data = np.vstack(itrl_data) #shape of this should be number of columns in the file (data)

        #before you can convert to float, need to replace missing data where its '.' as nans (in pupil this is '0.0')
        eyecols = [1,2,4,5] #leftx, lefty, rightx, right y col indices
        for col in eyecols:
            missing_inds = np.where(itrl_data[:,col] == '.') #find where data is missing in the gaze position, as probably a blink (or its missing as lost the eye)
            for i in missing_inds:
                itrl_data[i,col] = np.NaN #replace missing data ('.') with NaN
                itrl_data[i,3]   = np.NaN #replace left pupil as NaN (as in a blink)
                itrl_data[i,6]   = np.NaN #replace right pupil as NaN (as in a blink)
        itrl_data = itrl_data.astype(np.float) #convert data from string to floats for computations

        #for binocular data, the shape is:
        # columns: time stamp, left x, left y, left pupil, right x, right y, right pupil
        itrl_trackertime = itrl_data[:,0]
        itrl_lx, itrl_ly, itrl_lp = itrl_data[:,1], itrl_data[:,2], itrl_data[:,3]
        itrl_rx, itrl_ry, itrl_rp = itrl_data[:,4], itrl_data[:,5], itrl_data[:,6]

        # split Efix/Sfix and Esacc/Ssacc into separate lists
        itrl_efix = [itrl_fix[x] for x in range(len(itrl_fix)) if
                     itrl_fix[x][0] == 'EFIX']
        itrl_sfix = [itrl_fix[x] for x in range(len(itrl_fix)) if
                     itrl_fix[x][0] == 'SFIX']

        itrl_ssac = [itrl_sac[x] for x in range(len(itrl_sac)) if
                     itrl_sac[x][0] == 'SSACC']
        itrl_esac = [itrl_sac[x] for x in range(len(itrl_sac)) if
                     itrl_sac[x][0] == 'ESACC']

        itrl_sblk = [itrl_blink[x] for x in range(len(itrl_blink)) if
                     itrl_blink[x][0] == 'SBLINK']
        itrl_eblk = [itrl_blink[x] for x in range(len(itrl_blink)) if
                     itrl_blink[x][0] == 'EBLINK']

        #append to the collection of all data now
        trackertime.append(itrl_trackertime)
        lx.append(itrl_lx)
        ly.append(itrl_ly)
        rx.append(itrl_rx)
        ry.append(itrl_ry)
        lp.append(itrl_lp)
        rp.append(itrl_rp)
        Efix.append(itrl_efix)
        Sfix.append(itrl_sfix)
        Ssac.append(itrl_ssac)
        Esac.append(itrl_esac)
        Sblk.append(itrl_sblk)
        Eblk.append(itrl_eblk)
        Msg.append(itrl_events)

    trialsperblock = ntrials/nblocks
    rep_inds  = np.repeat(np.arange(nblocks),trialsperblock)
    #plt.hist(rep_inds, bins = nblocks) # all bars should be the same height now if its work? @ trials per block (80)

    iblock = {
    'trackertime': [],
    'lx'  : [], 'ly'  : [], 'lp'  : [],
    'rx'  : [], 'ry'  : [], 'rp'  : [],
    'Efix': [], 'Sfix': [],
    'Esac': [], 'Ssac': [],
    'Eblk': [], 'Sblk': [],
    'Msg' : []
    }
    dummy = copy.deepcopy(iblock)
    blocked_data = np.repeat(dummy, nblocks)

    for i in np.arange(nblocks):
        inds = np.squeeze(np.where(rep_inds == i))
        iblock_data = copy.deepcopy(iblock)
        for ind in inds: #add trialwise info into the blocked structure, continuous data rather than sectioned
            iblock_data['trackertime'].append(trackertime[ind])
            iblock_data['lx'].append(lx[ind])
            iblock_data['ly'].append(ly[ind])
            iblock_data['lp'].append(lp[ind])
            iblock_data['rx'].append(rx[ind])
            iblock_data['ry'].append(ry[ind])
            iblock_data['rp'].append(rp[ind])
            iblock_data['Efix'].append(Efix[ind])
            iblock_data['Sfix'].append(Sfix[ind])
            iblock_data['Esac'].append(Esac[ind])
            iblock_data['Ssac'].append(Ssac[ind])
            iblock_data['Eblk'].append(Eblk[ind])
            iblock_data['Sblk'].append(Sblk[ind])
            iblock_data['Msg'].append(Msg[ind])
            blocked_data[i] = iblock_data
    for block in range(len(blocked_data)): #concatenate trialwise signals into whole block traces to make artefact removal easier
        blocked_data[block]['trackertime'] = np.hstack(blocked_data[block]['trackertime'])
        blocked_data[block]['lx']          = np.hstack(blocked_data[block]['lx']         )
        blocked_data[block]['ly']          = np.hstack(blocked_data[block]['ly']         )
        blocked_data[block]['lp']          = np.hstack(blocked_data[block]['lp']         )
        blocked_data[block]['rx']          = np.hstack(blocked_data[block]['rx']         )
        blocked_data[block]['ry']          = np.hstack(blocked_data[block]['ry']         )
        blocked_data[block]['rp']          = np.hstack(blocked_data[block]['rp']         )
    return blocked_data

def find_missing_periods(data, nblocks):
    '''
    this function will find all sections of missing data. this typically relates to blinks
    but it will also find small patches where an eye has been dropped from the data.
    this just finds all patches, no matter the size, that are missing and outputs meaningful info into your data structure

    data argument expects a list, where each item in the list is a dictionary containing the data for a continuous block.

    binocular recorded data is expected here, where in each block the following keys are in the data dict:
        lx, ly, rx, ry.
    these correspond to the left and right gaze values. this function will iterate over each one to identify missing periods
    in each signal, and then output this information into new keys in your data:
    Eblk_lx, Eblk_ly, Eblk_rx, Eblk_ry
    '''

    if not isinstance(data, np.ndarray):
        raise Exception('check the format of your data. an array of dictionaries for each block is expected')

    if nblocks != len(data): #len(data) should give you the number of blocks in the data file
        raise Exception('there are not as many blocks in the data as you think. check this!')

    for block in data: #iterate over each block of data
        if 'lx' not in block.keys():
            raise Exception('A signal relating to the left eye is missing. Make sure the left x is labelled \'lx\'')
        if 'ly' not in block.keys():
            raise Exception('A signal relating to the left eye is missing. Make sure the left y is labelled \'ly\'')
        if 'rx' not in block.keys():
            raise Exception('A signal relating to the right eye is missing. Make sure the right x is labelled \'rx\'')
        if 'ry' not in block.keys():
            raise Exception('A signal relating to the right eye is missing. Make sure the right y is labelled \'ry\'')

        s_lx = []; s_rx = [];
        s_ly = []; s_ry = [];
        e_lx = []; e_rx = [];
        e_ly = []; e_ry = [];

        #find missing data in each gaze trace (left x & y, right x & y) separately for precision
        mlx = np.array(np.isnan(block['lx']) == True, dtype = int) # array of 1's and 0's for if data is missing at that time point
        dlx = np.diff(mlx) #find change-points, +1 means goes from present to absent, -1 from absent to present
        s_lx = np.squeeze(np.where(dlx ==  1)) #find points where it starts to be missing
        e_lx = np.squeeze(np.where(dlx == -1))+1 #find points where it stops being missing

        mly = np.array(np.isnan(block['ly']) == True, dtype = int)
        dly = np.diff(mly)
        s_ly = np.squeeze(np.where(dly ==  1))
        e_ly = np.squeeze(np.where(dly == -1))+1

        mrx = np.array(np.isnan(block['rx']) == True, dtype = int)
        drx = np.diff(mrx)
        s_rx = np.squeeze(np.where(drx ==  1))
        e_rx = np.squeeze(np.where(drx == -1))+1

        mry = np.array(np.isnan(block['ry']) == True, dtype = int)
        dry = np.diff(mry)
        s_ry = np.squeeze(np.where(dry ==  1))
        e_ry = np.squeeze(np.where(dry == -1))+1

        Eblk_lx = []; Eblk_ly = []
        Eblk_rx = []; Eblk_ry = []
        #left x
        for i in range(s_lx.size):
            if s_lx.size == 1:
                start = s_lx.tolist()
                end   = e_lx.tolist()
            else:
                start = s_lx[i]
                if i < e_lx.size: #check within range of the number of missing periods in data
                    end = e_lx[i]
                elif i == e_lx.size:
                    end = e_lx[-1]
                else:
                    end = e_lx[-1]
            ttime_start = block['trackertime'][start]
            ttime_end   = block['trackertime'][end]
            dur = end-start
                #create a blink event structure:
                # blink_code, start (blocktime), end (blocktime), start_trackertime, end_trackertime, duration
            evnt = ['LX_BLK', start, end, ttime_start, ttime_end, dur]
            Eblk_lx.append(evnt)
        #left y
        for i in range(s_ly.size):
            if s_ly.size == 1:
                start = s_ly.tolist()
                end   = e_ly.tolist()
            else:
                start = s_ly[i]
                if i < e_ly.size: #check within range of the number of missing periods in data
                    end = e_ly[i]
                elif i == e_ly.size:
                    end = e_ly[-1]
                else:
                    end = e_ly[-1]
            ttime_start = block['trackertime'][start]
            ttime_end   = block['trackertime'][end]
            dur = end-start
            #create a blink event structure:
            # blink_code, start (blocktime), end (blocktime), start_trackertime, end_trackertime, duration
            evnt = ['LY_BLK', start, end, ttime_start, ttime_end, dur]
            Eblk_ly.append(evnt)
        #right x
        for i in range(s_rx.size):
            if s_rx.size == 1:
                start = s_rx.tolist()
                end   = e_rx.tolist()
            else:
                start = s_rx[i]
                if i < e_rx.size: #check within range of the number of missing periods in data
                    end = e_rx[i]
                elif i == e_rx.size:
                    end = e_rx[-1]
                else:
                    end = e_rx[-1]
            ttime_start = block['trackertime'][start]
            ttime_end   = block['trackertime'][end]
            dur = end-start
            #create a blink event structure:
            # blink_code, start (blocktime), end (blocktime), start_trackertime, end_trackertime, duration
            evnt = ['RX_BLK', start, end, ttime_start, ttime_end, dur]
            Eblk_rx.append(evnt)
        #right y
        for i in range(s_ry.size):
            if s_ry.size == 1:
                start = s_ry.tolist()
                end   = e_ry.tolist()

            else:
                start = s_ry[i]
                if i < e_ry.size: #check within range of the number of missing periods in data
                    end = e_ry[i]
                elif i == e_ry.size:
                    end = e_ry[-1]
                else:
                    end = e_ry[-1]
            ttime_start = block['trackertime'][start]
            ttime_end   = block['trackertime'][end]
            dur = end-start
            #create a blink event structure:
            # blink_code, start (blocktime), end (blocktime), start_trackertime, end_trackertime, duration
            evnt = ['RY_BLK', start, end, ttime_start, ttime_end, dur]
            Eblk_ry.append(evnt)
        #append these new structures to the dataset
        block['Eblk_lx'] = Eblk_lx
        block['Eblk_rx'] = Eblk_rx
        block['Eblk_ly'] = Eblk_ly
        block['Eblk_ry'] = Eblk_ry

    return data

def interpolateBlinks_Blocked(block, trace):

    """
    - This function will interpolate the blinks in eyelink data that is in blocks (longer continuous segments)
    - The data that it will handle can either be segmented trials (due to trialwise stop/start of recording in the task) stitched back together
    - or just continuous block data, depending on the structure you give it.

    - If you use the script AttSacc_ParseData.py to parse the data, and are running AttSacc_CleanBlockedData.py, then the block structure should be suitable for this function.

    block argument will expect a dictionary that has the following fields:
    - trackertime: the timeseries of timepoints defined by the eyelink time, rather than trial time
    - lx      - timeseries of the x value for the left eye
    - ly      - timeseries of the y value for the left eye
    - rx      - timeseries of the x value for the right eye
    - ry      - timeseries of the y value for the right eye

    - Eblk_lx - list of events characterising missing periods of data in lx
    - Eblk_rx - list of events characterising missing periods of data in rx
    - Eblk_ly - list of events characterising missing periods of data in ly
    - Eblk_ry - list of events characterising missing periods of data in ry
    ----------- these blink structures have the following format:
    ----------- ['Event_code', start_blocktime, end_blocktime, start_trackertime, end_trackertime, duration]
    ----------- block code e.g. 'LX_BLK'

    trace argument expects a string specifying what trace you want to clean (e.g. 'lx' for left eye, x signal)

    Missing periods of data will be removed in the following way:

    periods of missing data of under 10 samples will be linearly interpolated within a window of 10 samples either side of the start and end of the period
    """
    if not isinstance(block, dict):
        raise Exception('data supplied is not a dictionary')

    if not isinstance(trace, str):
        raise Exception('the trace indicator supplied is not a string. \n please input a string e.g. \'lx\'')

    if trace not in block.keys():
        raise Exception('the signal you want to clean does not exist in the data. check your trace labels in your data')

    signal = block[trace] #extract the signal that needs cleaning

    eventlabel = 'Eblk_%s'%trace

    if eventlabel not in block.keys():
        raise Exception('the missing period information for this signal is not in your data structure')

    blinks = np.array(block[eventlabel])       #get the desired blink structures
    if blinks.size != 0:
        blinks = blinks[:,1:]            #remove the first column as it's a string for the event code, not needed now
        blinks = blinks.astype(float).astype(int) # change the strings to integers. need to go via float or it fails.

        short_duration_inds  = np.where(np.in1d(blinks[:,4], range(21)))[0]    # find the blinks that are below 20 samples long
        medium_duration_inds = np.where(np.in1d(blinks[:,4], range(21,51)))[0] # find the blinks that are between 21 and 50 samples long
        long_duration_inds   = np.where(blinks[:,4] > 50)[0]                   # find blinks that are over 50 samples long

        short_blinks  = blinks[short_duration_inds,:]
        medium_blinks = blinks[medium_duration_inds,:]
        long_blinks   = blinks[long_duration_inds,:]

        #linear interpolate across these smaller periods before proceeding.
        for blink in short_blinks:
            start, end               = blink[0], blink[1] #get start and end periods of the missing data
            to_interp                = np.arange(start, end) #get all time points to be interpolated over

            #set up linear interpolation
            inttime                  = np.array([start,end])
            inttrace                 = np.array([signal[start], signal[end]])
            fx_lin                   = sp.interpolate.interp1d(inttime, inttrace, kind = 'linear')

            interptrace              = fx_lin(to_interp)
            signal[to_interp] = interptrace
        for blink in medium_blinks:
            start, end               = blink[0], blink[1] #get start and end periods of the missing data
            to_interp                = np.arange(start, end) #get all time points to be interpolated over

            #set up linear interpolation
            inttime                  = np.array([start,end])
            inttrace                 = np.array([signal[start], signal[end]])
            fx_lin                   = sp.interpolate.interp1d(inttime, inttrace, kind = 'linear')

            interptrace              = fx_lin(to_interp)
            signal[to_interp] = interptrace

        #now cubic spline interpolate across the larger missing periods (blinks)
        for blink in long_blinks:
            start, end            = blink[0], blink[1] #get start and end of these missing samples
            if end+40 >= signal.size: #this blink happens just before the end of the block, so need to adjust the window
                window            = [start-80, start - 40, end, signal.size-1] #reduce the window size but still cubic spline
            elif end+40 <= signal.size and end+80 >= signal.size:
                window            = [start-80, start-40, end+40, signal.size-1]
            else:
                window            = [start-80, start-40, end+40, end+80] #set the window for the interpolation
            inttime               = np.array(window)
            if end + 40 >= signal.size:
                inttrace          = np.array([np.nanmedian(signal[start-80:start-40])                 , np.nanmedian(signal[start-40:start-1]),
                                              np.nanmedian(signal[end:int(np.floor((signal.size-1-end)/2))]), np.nanmedian(signal[int(np.ceil((signal.size-1-end)/2)):signal.size-1]) ])
            elif end+40 <= signal.size and end + 80 >= signal.size:
                inttrace          = np.array([np.nanmedian(signal[start-80:start-40])                 , np.nanmedian(signal[start-40:start-1]),
                                              np.nanmedian(signal[end:end+40]), np.nanmedian(signal[end+40:signal.size-1]) ])
            else:
                inttrace          = np.array([np.nanmedian(signal[start-80:start-40]), np.nanmedian(signal[start-40:start-1]), # by giving the nanmedian between these points,
                                              np.nanmedian(signal[end+1:end+40])     , np.nanmedian(signal[end+40:end+80]) ])  # points, it accounts for variance of the signal
            fx_cub                = sp.interpolate.interp1d(inttime, inttrace, kind = 'cubic')

            if end+30 >= signal.size:
                to_interp         = np.arange(start-30, signal.size-1)
            else:
                to_interp         = np.arange(start-30, end+30) #interpolate just outside the start of the missing period, for cases of large changes due to blinks
            interptrace           = fx_cub(to_interp)
            signal[to_interp]     = interptrace
    #output the data into the block structure
    block[trace] = signal
    return block


import datetime
import functools
import json
import logging
import pathlib
import time
from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Type, Union

import numpy as np
from matplotlib import pyplot as plt
from msi_zarr_analysis import VERSION
from msi_zarr_analysis.ml.dataset import GroupCollection
from msi_zarr_analysis.ml.dataset.cytomine_ms_overlay import (
    collect_spectra_zarr,
    get_overlay_annotations,
    translate_annotation_mapping_overlay_to_template,
)
from msi_zarr_analysis.utils.autocrop import autocrop
from msi_zarr_analysis.utils.check import open_group_ro
from msi_zarr_analysis.utils.cytomine_utils import (
    get_lipid_dataframe,
    get_page_bin_indices,
)
from sklearn.base import BaseEstimator
from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier
from sklearn.model_selection import GridSearchCV, StratifiedGroupKFold, StratifiedKFold
from sklearn.tree import DecisionTreeClassifier

@functools.lru_cache(maxsize=1)
def datetime_str() -> str:
    "return a datetime representation, constant through the program"
    return datetime.datetime.now().strftime("%W-%w_%H-%M-%S")

def build_segmentation_mask(
    zarr_path: str,
    estimator: BaseEstimator,
) -> np.ndarray:

    z_group = open_group_ro(zarr_path)
    z_ints = z_group["/0"]
    z_lens = z_group["/labels/lengths/0"]

    # assume small dataset
    n_ints = z_ints[:, 0, ...]
    n_lens = z_lens[0, 0, ...]
    selection_mask = n_lens > 0

    # select valid spectra only
    dataset = n_ints[:, selection_mask].T
    prediction = estimator.predict(dataset)

    # build prediction 2D mask
    prediction_mask = np.zeros(shape=n_lens.shape, dtype=np.uint8)
    prediction_mask[selection_mask] = prediction

    # build colored mask
    #   no prediction: transparent
    #   red: class 1
    #   green: class 2
    mask = np.stack(
        [
            np.where(prediction_mask, 200, 50),  # R
            np.where(prediction_mask, 50, 200),  # G
            np.where(prediction_mask, 50, 30),  # B
            np.where(n_lens, 255, 0),  # A
        ],
        axis=-1,
    )

    return mask

class MLConfig(NamedTuple):
    choices: Dict[Type[BaseEstimator], Dict[str, Any]]
    cv_fold_outer: Optional[int] = None
    cv_fold_inner: Optional[int] = None

class DSConfig(NamedTuple):
    image_id_overlay: int  # Cytomine ID for the overlay image
    local_overlay_path: str  # local path of the (downloaded) overlay
    lipid_tm: str  # name of the lipid to base the template matching on

    project_id: int  # project id
    annotated_image_id: int  # image with the annotations

    zarr_path: str  # path to the non-binned zarr image

    classes: Dict[str, List[int]]

    save_image: Union[bool, str] = False

    transform_rot90: int = 0
    transform_flip_ud: bool = False
    transform_flip_lr: bool = False

    annotation_users_id: Tuple[int] = ()  # select these users only

    zarr_template_path: str = None  # use another group for the template matching

def run(
    base: str,
    config_path: str,
    ml_config: MLConfig,
    bin_csv_path: str,
    *ds_config: DSConfig,
    grouped_cv: bool,
):
    if not ds_config:
        raise ValueError("a list of dataset configuration is required")

    from cytomine import Cytomine

    with open(config_path) as config_file:
        config_data = json.loads(config_file.read())
        host_url = config_data["HOST_URL"]
        pub_key = config_data["PUB_KEY"]
        priv_key = config_data["PRIV_KEY"]

    lipid_df = get_lipid_dataframe(bin_csv_path)

    # build ds
    with Cytomine(host_url, pub_key, priv_key):

        dataset_to_be_merged: List[GroupCollection] = []

        for ds_config_itm in ds_config:

            # build all datasets and merge them if there are more than one
            page_idx, bin_idx, *_ = get_page_bin_indices(
                ds_config_itm.image_id_overlay, ds_config_itm.lipid_tm, bin_csv_path
            )

            annotation_dict = get_overlay_annotations(
                ds_config_itm.project_id,
                ds_config_itm.image_id_overlay,
                ds_config_itm.classes,
                select_users=ds_config_itm.annotation_users_id,
            )

            annotation_dict = translate_annotation_mapping_overlay_to_template(
                annotation_dict,
                ds_config_itm.zarr_template_path,
                bin_idx,
                ds_config_itm.local_overlay_path,
                page_idx,
                ds_config_itm.transform_rot90,
                ds_config_itm.transform_flip_ud,
                ds_config_itm.transform_flip_lr,
            )

            dataset_to_be_merged.append(
                collect_spectra_zarr(
                    annotation_dict,
                    lipid_df.Name,
                    ds_config_itm.zarr_path,
                )
            )

        if len(dataset_to_be_merged) == 1:
            ds = dataset_to_be_merged[0]
        else:
            ds = GroupCollection.merge_collections(*dataset_to_be_merged)

    estimator = model_selection_assessment(
        ds,
        ml_config,
        grouped_cv=grouped_cv,
    )

    dir = pathlib.Path("results") / datetime_str() / "class_masks"
    dir.mkdir(parents=True, exist_ok=True)
    path = str(dir / base)

    with Cytomine(host_url, pub_key, priv_key):

        for ds_config_itm in ds_config:
            mask = build_segmentation_mask(ds_config_itm.zarr_path, estimator)
            stem = pathlib.Path(ds_config_itm.zarr_path).stem

            # build all datasets and merge them if there are more than one
            page_idx, bin_idx, *_ = get_page_bin_indices(
                ds_config_itm.image_id_overlay, ds_config_itm.lipid_tm, bin_csv_path
            )

            template_group = open_group_ro(ds_config_itm.zarr_template_path)
            template_lipid = template_group["/0"][bin_idx, 0, ...]

            crop_idx = autocrop(template_lipid)
            mask = mask[crop_idx]
            template_lipid = template_lipid[crop_idx]

            fig, ax = plt.subplots(dpi=250)

            ax.imshow(template_lipid, interpolation="nearest")
            ax.imshow(mask, interpolation="nearest", alpha=0.4)

            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(False)

            fig.tight_layout()
            fig.savefig(path + "_" + stem + "_annotated.png")
            plt.close(fig)

            fig, ax = plt.subplots(dpi=250)

            ax.imshow(template_lipid, interpolation="nearest")

            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(False)

            fig.tight_layout()
            fig.savefig(path + "_" + stem + ".png")
            plt.close(fig)

def model_selection_assessment(
    collection: GroupCollection,
    ml_config: MLConfig,
    *,
    grouped_cv: bool,
):
    logger = logging.getLogger()

    collection.dataset.check_dataset(print_=True)
    class_names = collection.dataset.class_names()
    logger.info(f"terms: {class_names}")

    # StratifiedKFold ignores groups
    CVType = StratifiedGroupKFold if grouped_cv else StratifiedKFold
    # one for each loop
    outer = CVType(n_splits=ml_config.cv_fold_outer, shuffle=True, random_state=31)
    inner = CVType(n_splits=ml_config.cv_fold_inner, shuffle=True, random_state=32)

    def select_model_(ds_x: np.ndarray, ds_y: np.ndarray, groups: np.ndarray):
        "select one model that best fits a dataset"

        best_res = None
        best_model = None
        best_score = -1

        for model, arg_dict in ml_config.choices.items():

            folds = list(inner.split(ds_x, ds_y, groups))
            search = GridSearchCV(model(), arg_dict, cv=folds, n_jobs=-1)
            search.fit(ds_x, ds_y)

            logger.debug(
                "in-loop: %s, %s, %s", model, search.best_params_, search.best_score_
            )

            if search.best_score_ > best_score:
                best_res = search
                best_model = model
                best_score = search.best_score_

        if best_model is None or best_res is None:
            raise ValueError("empty choice dict!")

        return best_model, best_res

    def assessment_(ds_x: np.ndarray, ds_y: np.ndarray, groups: np.ndarray):
        "assess the quality of the selection procedure"

        scores = []

        for train_idx, test_idx in outer.split(ds_x, ds_y, groups):
            train_set = (ds_x[train_idx], ds_y[train_idx])
            test_set = (ds_x[test_idx], ds_y[test_idx])

            model, res = select_model_(*train_set, groups[train_idx])

            try:
                estimator = model(**res.best_params_, n_jobs=-1)
            except TypeError:
                estimator = model(**res.best_params_)
            estimator.fit(*train_set)

            scores.append(estimator.score(*test_set))

        return np.array(scores)

    logger.info("starting assessment procedure")
    start = time.time()
    scores = assessment_(collection.data, collection.target, collection.groups)
    logger.info("duration: %s", time.time() - start)
    logger.info("scores: %s (%s pm %s)", scores, scores.mean(), scores.std(ddof=1))

    logger.info("starting selection procedure")
    start = time.time()
    model, res = select_model_(collection.data, collection.target, collection.groups)
    logger.info("duration: %s", time.time() - start)
    logger.info("model: %s, parameters: %s", model, res.best_params_)

    logging.info("training model")
    try:
        estimator = model(**res.best_params_, n_jobs=-1)
    except TypeError:
        estimator = model(**res.best_params_)
    estimator.fit(collection.data, collection.target)

    avg_gini = estimator.feature_importances_
    if hasattr(estimator, "estimators_"):
        std_gini = np.std(
            [tree.feature_importances_ for tree in estimator.estimators_], axis=0
        )
    else:
        std_gini = np.full_like(avg_gini, np.nan)

    # 1: feature importance
    logging.info("feature importances:")
    for idx, (feature, importance, stddev) in enumerate(
        zip(collection.dataset.attribute_names(), avg_gini, std_gini)
    ):
        if len(feature) > 20:
            feature = feature[:17] + "..."
        logging.info("  %02d (%20s) : %.4f +/- %.4f", idx, feature, importance, stddev)

    return estimator

def main():
    def filter(record: logging.LogRecord) -> bool:
        if record.name == "root":
            return True

        if record.name == "cytomine.client":
            record.name = "cyt-client"
            return record.levelno != logging.DEBUG

        return False

    # setup logger
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    logger.addFilter(filter)

    formatter = logging.Formatter(
        "%(asctime)s.%(msecs)03d [%(name)s] [%(levelname)s] : %(message)s",
        datefmt="%j %H:%M:%S",
    )

    # main handler is stdout
    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(logging.NOTSET)
    stream_handler.setFormatter(formatter)
    stream_handler.addFilter(filter)
    logger.addHandler(stream_handler)

    logger.info("starting %s (VERSION: %s)", __file__, str(VERSION))

    data_sources = [
        {
            "name": "region_13",
            "args": {
                "image_id_overlay": 545025763,
                "local_overlay_path": "datasets/Adjusted_Cytomine_MSI_3103_Region013-Viridis-stacked.ome.tif",
                "lipid_tm": "LysoPPC",
                "project_id": 542576374,
                "annotated_image_id": 545025783,
                "transform_rot90": 1,
                "transform_flip_ud": True,
                "transform_flip_lr": False,
                "annotation_users_id": (),
                "zarr_template_path": "datasets/comulis13_binned.zarr",
            },
            "base": "datasets/comulis13",
        },
        {
            "name": "region_14",
            "args": {
                "image_id_overlay": 548365416,
                "local_overlay_path": "datasets/Region014-Viridis-stacked.ome.tif",
                "lipid_tm": "LysoPPC",
                "project_id": 542576374,
                "annotated_image_id": 548365416,
                "transform_rot90": 1,
                "transform_flip_ud": True,
                "transform_flip_lr": False,
                "annotation_users_id": (),
                "zarr_template_path": "datasets/comulis14_binned.zarr",
            },
            "base": "datasets/comulis14",
        },
        {
            "name": "region_15",
            "args": {
                "image_id_overlay": 548365463,
                "local_overlay_path": "datasets/Region015-Viridis-stacked.ome.tif",
                "lipid_tm": "LysoPPC",
                "project_id": 542576374,
                "annotated_image_id": 548365463,
                "transform_rot90": 1,
                "transform_flip_ud": True,
                "transform_flip_lr": False,
                "annotation_users_id": (),
                "zarr_template_path": "datasets/comulis15_binned.zarr",
            },
            "base": "datasets/comulis15",
        },
    ]

    normalizations = [
        "",
        "_norm_2305",
        "_norm_max",
        "_norm_tic",
        "_norm_vect",
    ]

    problem_classes = {
        "LS SC": {  # both merged
            "LS": [544926097, 544926081],
            "SC": [544926052, 544924846],
        },
        "LS_n LS_p": {  # irradiation on LS
            "LS_n": [544926097],
            "LS_p": [544926081],
        },
        "SC_n SC_p": {  # irradiation on SC
            "SC_n": [544926052],
            "SC_p": [544924846],
        },
        "LS_n SC_n": {  # LS vs SC when irradiated
            "LS_n": [544926097],
            "SC_n": [544926052],
        },
        "LS_p SC_p": {  # LS vs SC before irradiation
            "LS_p": [544926081],
            "SC_p": [544924846],
        },
    }

    ml_config = MLConfig(
        choices={
            ExtraTreesClassifier: {
                "n_estimators": [200, 500, 1000],
                "max_features": ["sqrt", None],
                "max_depth": [20, None],
            },
            RandomForestClassifier: {
                "n_estimators": [200, 500, 1000],
                "max_features": ["sqrt", None],
                "max_depth": [20, None],
            },
            DecisionTreeClassifier: {
                "max_depth": [1, 20, None],
            },
        },
        cv_fold_inner=5,
        cv_fold_outer=5,
    )

    for normalization in normalizations:

        for name, classes in problem_classes.items():

            ds_lst = []
            for source in data_sources:
                zarr_path = source["base"] + normalization + "_binned.zarr"
                ds_lst.append(
                    DSConfig(
                        **source["args"],
                        classes=classes,
                        save_image=False,
                        zarr_path=zarr_path,
                    )
                )

            for grouped_cv in [True, False]:

                base = (
                    name
                    + (normalization or "_no_norm")
                    + ("_gcv" if grouped_cv else "_cv")
                )
                file_handler = logging.FileHandler(
                    f"logs/classification_task/{base}.log", mode="a"
                )
                file_handler.setLevel(logging.INFO)
                file_handler.setFormatter(formatter)
                file_handler.addFilter(filter)
                logger.addHandler(file_handler)

                logger.info("norm: '%s', problem: '%s'", normalization, name)

                run(
                    base,
                    "config_cytomine.json",
                    ml_config,
                    "mz value + lipid name.csv",
                    *ds_lst,
                    grouped_cv=grouped_cv,
                )

                logger.info("done")

                logger.removeHandler(file_handler)

if __name__ == "__main__":
    main()


import os
import pathlib
from sklearn.model_selection import train_test_split

import pandas as pd
import numpy as np

import utils
from accuracy import Accuracy_Categorical
from model import Model
from layers import Dense
from optimizers import SGD, Optimizer_Adam
from regularizarions import RegL1
from trainCallback import EarlyStopping

def convertOneShot(x_input):
    l = []
    for x in x_input:
        b = np.zeros((x.size, 4))
        b[np.arange(6), x - 1] = 1
        b = np.reshape(b, 24)
        l.append(b)
    l = np.array(l)
    dropped = np.delete(l, [3, 7,10,11,15,22,23], 1)
    return dropped

# configuration ########################################################
DATASET_PATH = "classificazione/monks-3.csv"
# ######################################################################

dataset = pd.read_csv(DATASET_PATH)

X_train = dataset.drop(["output"], axis=1)
y_train = dataset.drop(["input1", "input2", "input3", "input4", "input5", "input6"], axis=1)

X_values = np.array(X_train.values.tolist())
y = np.array(y_train.values.tolist())

X = convertOneShot(X_values)

X_train, X_val, y_train, y_val = train_test_split(X, y,test_size=0.3,random_state=3)
X_val, X_test, y_val, y_test = utils.straified_split(X_val, y_val, y_idx=0, test_size=0.2)

model = Model()
model.add(Dense(8, activationFunName="tanh"))
model.add(Dense(1, activationFunName="sigmoid"))

accuracy = Accuracy_Categorical()
earlystopping = EarlyStopping(patience=500, monitor="val_data_loss")
# model.compile(lossname="bce", optimizer=SGD(learning_rate=0.002, decay=0.0, momentum=0.0), accuracy= accuracy)
model.compile(lossname="bce", optimizer=Optimizer_Adam(learning_rate=0.0005, decay=0.00001), accuracy=accuracy)

model.train(X_train, y_train, epochs=10000, batch_size=16, log_every=500, X_val=X_val, y_val=y_val, callbacks=[earlystopping])

utils.printLossHistory(model,0,1)
utils.printAccuracyHistory(model)

res_test = model.model_assessment(X_test, y_test)
print("--------------------------------------")
print("TEST dataset results")
print(res_test)


import csv
import requests
import json
from pprint import pprint
from decouple import config

movies = []
result = []
api_key = config('API_KEY')

# Discover
# 최신 영화 60개를 result에 담기

for i in range(3):
# for i in range(1):

    page = i+1
    base_url = 'https://api.themoviedb.org/3/discover/movie?'
    url = base_url + f'api_key={api_key}&language=ko-KR&sort_by=popularity.desc&include_adult=false&include_video=false&page={page}'

    response = requests.get(url)
    response_dict = response.json()

    # len(response_dict['results']) => 20개
    for i in range(len(response_dict['results'])):
        movieId = response_dict['results'][i]['id']
        movies.append(movieId)

for i in range(len(movies)):
# for i in range(1):
    movie_id = movies[i]

    # model : Movie
    movie_tmp = {}
    movie_tmp['model'] = 'movies.movie'
    movie_tmp['pk'] = movies[i]
    base_url_movies = 'https://api.themoviedb.org/3/movie/'

    ## detail
    url_detail = base_url_movies + f'{movie_id}?api_key={api_key}&language=ko-KR'

    response = requests.get(url_detail)
    response_dict = response.json()

    title = response_dict["title"]
    original_title = response_dict["original_title"]
    summary = response_dict["overview"]
    poster_url = response_dict["poster_path"]
    backdrop_url = response_dict["backdrop_path"]
    come_out = response_dict["release_date"]
    audience = response_dict["popularity"] * 1000

    genres = []
    tmps = response_dict["genres"]
    for tmp in tmps:
        genres.append(tmp["id"])

    ## credits
    url_credits = base_url_movies + f'{movie_id}/credits?api_key={api_key}'

    response = requests.get(url_credits)
    response_dict = response.json()

    tmps = response_dict["crew"]
    director = ''

    for tmp in tmps:
        if tmp["job"] == "Director":
            director = tmp["name"]
            break

    ## videos
    base_url_video = 'https://api.themoviedb.org/3/movie/'
    url_video = base_url_video + f'{movie_id}/videos?api_key={api_key}&language=ko-KR'

    response = requests.get(url_video)
    response_dict = response.json()

    if response_dict["results"]:
        teaser = response_dict["results"][0]["key"]
    else:
        teaser = ""

    # ## actor
    # url_actor = f'http://api.themoviedb.org/3/movie/{movie_id}/casts?api_key={api_key}'

    # response = requests.get(url_actor)
    # response_dict = response.json
    # tmp_casts = response_dict["cast"]
    # cnt = 0
    # casts = []
    # # pprint(response_dict['cast'])
    # for cast in tmp_casts:
    #     if cnt == 5:
    #         break
    #     name = cast["name"]
    #     cnt += 1

    movie_tmp['fields'] = {
        'title': title,
        'original_title': original_title,
        'summary': summary,
        'poster_url': poster_url,
        'backdrop_url': backdrop_url,
        'teaser': teaser,
        'come_out': come_out,
        'audience': audience,
        'director': director,
        'genres': genres,
    }
    result.append(movie_tmp)

with open('movies.json', 'w', encoding='utf-8') as f:
    json.dump(result, f, ensure_ascii=False, indent=4)