# Look for #IMPLEMENT tags in this file.
'''
All models need to return a CSP object, and a list of lists of Variable objects
representing the board. The returned list of lists is used to access the
solution.

For example, after these three lines of code

    csp, var_array = caged_csp_model(board)
    solver = BT(csp)
    solver.bt_search(prop_FC, var_ord)

var_array[0][0].get_assigned_value() should be the correct value in the top left
cell of the FunPuzz puzzle.

The grid-only models do not need to encode the cage constraints.

1. binary_ne_grid (worth 10/100 marks)
    - A model of a FunPuzz grid (without cage constraints) built using only
      binary not-equal constraints for both the row and column constraints.

2. nary_ad_grid (worth 10/100 marks)
    - A model of a FunPuzz grid (without cage constraints) built using only n-ary
      all-different constraints for both the row and column constraints.

3. caged_csp_model (worth 25/100 marks)
    - A model built using your choice of (1) binary binary not-equal, or (2)
      n-ary all-different constraints for the grid.
    - Together with FunPuzz cage constraints.

'''
from cspbase import *
import itertools

def binary_ne_grid(fpuzz_grid):
    ##IMPLEMENT
    constraints, constraint_values, constraint_names = [], [], []
    initial_variables = get_initial_variables(fpuzz_grid)
    size_of_board = initial_variables["size"]
    cell_values = initial_variables["cell_values"]
    variables = initial_variables["variables"]

    for cell in itertools.product(list(range(size_of_board)), list(range(size_of_board)), list(range(size_of_board))):
        col_c1 = cell_name(cell[1], cell[0]) + ", " + cell_name(cell[2], cell[0])
        col_c2 = cell_name(cell[2], cell[0]) + ", " + cell_name(cell[1], cell[0])
        if col_c1 not in constraint_names and col_c2 not in constraint_names and cell[2] != cell[1]:
            satisfying_col_constraints = []
            for v1 in cell_values:
                for v2 in cell_values:
                    if v1 != v2:
                        satisfying_col_constraints.append((v1, v2))
            c1 = Constraint(cell_name(cell[1], cell[0]) + ", " + cell_name(cell[2], cell[0]),
                            [variables[cell[1]][cell[0]], variables[cell[2]][cell[0]]])
            c1.add_satisfying_tuples(satisfying_col_constraints)
            constraints.append(c1)
            constraint_values.append(c1)
            constraint_names.append(c1.name)
            row_c1 = cell_name(cell[0], cell[1]) + ", " + cell_name(cell[0], cell[2])
            row_c2 = cell_name(cell[0], cell[2]) + ", " + cell_name(cell[0], cell[1])
            if row_c1 not in constraint_names and row_c2 not in constraint_names and cell[2] != cell[1]:
                satisfying_row_constraints = []
                for v1 in cell_values:
                    for v2 in cell_values:
                        if v1 != v2:
                            satisfying_row_constraints.append((v1, v2))

                added_constraints = Constraint(cell_name(cell[0], cell[1]) + ", " + cell_name(cell[0], cell[2]),
                                               [variables[cell[0]][cell[1]], variables[cell[0]][cell[2]]])
                added_constraints.add_satisfying_tuples(satisfying_row_constraints)
                constraint_values.append(added_constraints)
                constraint_names.append(added_constraints.name)

    csp = CSP('binary_ne', [variable for rows in variables for variable in rows])
    for constraint in constraints:
        csp.add_constraint(constraint)
    return (csp, variables)

def nary_ad_grid(fpuzz_grid):
    ##IMPLEMENT
    constraints, scope = [], []
    initial_variables = get_initial_variables(fpuzz_grid)
    cell_values = initial_variables["cell_values"]
    variables = initial_variables["variables"]

    for col in cell_values:
        for row in cell_values:
            scope.append(variables[row][col])
        cells1 = []
        for value_pair1 in itertools.permutations(cell_values):
            cells1.append(value_pair1)
        c1 = Constraint(hash(col), scope)
        c1.add_satisfying_tuples(cells1)
        constraints.append(c1)
        cells2 = []
        for value_pair2 in itertools.permutations(cell_values):
            cells2.append(value_pair2)
        c2 = Constraint(hash(col), variables[col])
        c2.add_satisfying_tuples(cells2)
        constraints.append(c2)

    csp = CSP('nary_ad', [variable for rows in variables for variable in rows])
    for constraint in constraints:
        csp.add_constraint(constraint)
    return (csp, variables)

def caged_csp_model(fpuzz_grid):
    ##IMPLEMENT
    constraints, constraint_values, constraint_names = [], [], []
    initial_variables = get_initial_variables(fpuzz_grid)
    size_of_board = initial_variables["size"]
    cell_values = initial_variables["cell_values"]
    cage_constraints = range(1, size_of_board)
    csp, variables = binary_ne_grid(fpuzz_grid)

    for cage in cage_constraints:
        row = list(fpuzz_grid[cage])
        operation, target, scope_values = row[-1], row[-2], row[:-2]
        scope, cells = [], []
        for scope_value in scope_values:
            value = variables[(scope_value // 10) - 1][(scope_value % 10) - 1]
            scope.append(value)
        constraint_name = "Operation: " + str(operation) + "Target:" + str(target)
        constraint = Constraint(constraint_name, scope)
        op = check_operation(operation)
        if op['addition']:
            for cell in itertools.product(tuple(cell_values), repeat=len(scope)):
                if sum(cell) == target:
                    cells.append(cell)
        elif op['subtraction']:
            for cell in itertools.product(tuple(cell_values), repeat=len(scope)):
                for i in range(len(scope)):
                    difference = cell[i] - sum(cell[:i] + cell[i + 1:])
                    if difference == target:
                        cells.append(cell)
        elif op['multiplication']:
            for cell in itertools.product(tuple(cell_values), repeat=len(scope)):
                for i in range(len(scope)):
                    product = float(cell[i])
                    for v1 in cell[:i] + cell[i + 1:]:
                        product *= v1
                    if product == target:
                        cells.append(cell)
        elif op['division']:
            for cell in itertools.product(tuple(cell_values), repeat=len(scope)):
                for i in range(len(scope)):
                    quotient = float(cell[i])
                    for v1 in cell[:i] + cell[i + 1:]:
                        quotient = quotient / v1
                    if quotient == target:
                        cells.append(cell)

        constraint.add_satisfying_tuples(cells)
        constraints.append(constraint)
        constraint_values.append(constraint)
        constraint_names.append(constraint.name)

    for constraint in constraints:
        csp.add_constraint(constraint)
    return (csp, variables)

def check_operation(operation):
    operation_dictionary = {'addition': False, 'subtraction': False, 'multiplication': False, 'division': False}
    if operation == 0:
        operation_dictionary['addition'] = True
    elif operation == 1:
        operation_dictionary['subtraction'] = True
    elif operation == 3:
        operation_dictionary['multiplication'] = True
    elif operation == 2:
        operation_dictionary['division'] = True
    return operation_dictionary

def cell_name(row, column):
    # Return cell name used for constraints
    return "Row: " + str(row) + " Col: " + str(column)

def get_initial_variables(fpuzz_grid):
    # Return size_of_board, cell_values, variables in that order
    size_of_board = fpuzz_grid[0][0]
    max_value = size_of_board + 1
    cell_values = list(range(1, max_value))
    variables = []
    for r in range(size_of_board):
        row = []
        for c in range(size_of_board):
            variable = Variable(cell_name(r, c), list(range(1, size_of_board + 1))[:])
            row.append(variable)
        variables.append(row)
    return {"size": size_of_board, "cell_values": cell_values, "variables": variables}


import requests

STEAMDB_SALE_URL = "https://steamdb.info/sales/?merged=true&cc=cn"

class SaleRequester:
    def __init__(self):
        self.fake_header = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Charset': 'UTF-8,*;q=0.5',
            'Accept-Encoding': 'gzip,deflate,sdch',
            'Accept-Language': 'en-US,en;q=0.8',
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:13.0) Gecko/20100101 Firefox/13.0'
        }

    def get_sale_page(self):
        try:
            content = requests.get(STEAMDB_SALE_URL,headers=self.fake_header).text
        except TimeoutError:
            pass
        return content

if __name__ == "__main__":
    s = SaleRequester()
    print(s.get_sale_page())


from sqlite3 import *
from typing import Union

class DB:
    def __init__(self):
        self.db = connect("app.db")
        self.cr = self.db.cursor()
        self.cr.execute("create table if not exists `users`(user_id INTEGER, username TEXT, chat_name TEXT, "
                        "chat_username TEXT, chat_id INTEGER)")

    def __del__(self):
        self.db.close()

    async def get_user(
            self, user_id=None, username=None
    ) -> list:

        self.cr.execute("select * from `users` where user_id = ?", (user_id,)) if user_id else self.cr.execute(
            "select * from `users` where username = ?", (username,))
        data = self.cr.fetchall()
        return data

    async def get_all(
            self
    ) -> list:
        self.cr.execute("select * from users")
        res = self.cr.fetchall()
        return res

    async def insert_user(
            self,
            user_id: Union[str, int],
            username: str,
            chat_id: Union[str, int],
            chat_name: str,
            chat_username: str
    ):
        self.cr.execute("select * from `users` where user_id = ? and chat_id = ?", (user_id, chat_id))
        results = self.cr.fetchall()
        if results:
            return
        self.cr.execute(
            "insert into `users`(user_id, username, chat_id, chat_name, chat_username) values(?, ?, ?, ?, ?)",
            (user_id, username, chat_id, chat_name, chat_username)
        )
        self.db.commit()


"""
This script is used for 'writing' songs in musical notation form, with recording of key downs and ups being used to define the time durations and delays of notes. Notes are shown line by line and a single key on your keyboard can be used to set the timing for each note in a song - of course, you'll need to know the song by heart.

All that is needed is the pynput module installed and a file containing your music notes. You will need to pass the music notes file as an argument. To start, call this script from python in the terminal - e.g.:

$ python3 musical_timing_recorder.py <path to your music notes file>

Once all the music notes have been 'played' by you, an out file gets written to in the same directory as your music notes file. This text file will contain 3 separate sections - the notes, the note durations, and the note gaps.
"""

from pynput import keyboard
import time
import os,sys

DEBUG = False
KILL = False

# Keyboard key handling
key_held = False

def key_down(key):
    global key_held,KILL
    key_held = True
    if key == keyboard.Key.esc: KILL = True

def key_up(key):
    global key_held
    key_held = False

kl = keyboard.Listener(on_press=key_down,on_release=key_up,suppress=True)
kl.start()

# Execute
if __name__ == "__main__":
    # Init
    if not DEBUG:
        args = sys.argv
        if os.path.isfile(args[1]): notes_file = args[1]
        else: raise FileNotFoundError("Error: missing positional argument - music notes file")
    else:
        notes_file = "<INSERT MUSIC NOTES FILE HERE>" # Ignore this
    with open(notes_file,"r") as f:
        x = f.readlines()
        lines = [i.strip() for i in x]

    # Clear screen and show greeter
    os.system("cls" if os.name == "nt" else "clear")
    print(f"Musical Timing Recorder ({os.path.basename(notes_file)})\nPress 'Esc' to quit\n\n")
    print("When you are ready, Maestro...\n")

    # Recording loop system
    line_count = len(lines)
    durations,gaps = [],[]
    t,time_d,time_g = 0,0,0
    recording_started = False
    for n,line in enumerate(lines):
        # Print out the current sheet line for the user
        print(f"\nLine: {n+1}\n \t{line}\n> \t",end="")
        if not recording_started:
            # Wait for user to press key for first time
            while not key_held: continue
            if KILL: exit()
            recording_started = True
        # Notes per line recording loop start
        notes = line.split()
        note_count = len(notes)
        for i,note in enumerate(notes):
            while not key_held: continue # Failsafe
            if key_held:
                print(note+" ",end="",flush=True)
                t = time.time()
                while key_held: time_d = (time.time() - t)
                if KILL: exit()
                durations.append(time_d)
            if not key_held:
                # Abruptly stop and don't record gap for last note
                if n+1 >= line_count and i+1 >= note_count:
                    gaps.append(0)
                    break
                t = time.time()
                while not key_held: time_g = (time.time() - t)
                if KILL: exit()
                gaps.append(time_g)
        durations.append("\n")
        gaps.append("\n")
        print("\n")
    # Finished recording - cleanup and write data to output file
    out_file = os.path.basename(notes_file).split(".")[0]+"_output.txt"
    melody = []
    for l in lines:
        clean = l.split()
        x = [f"\"{n}\"," for n in clean]
        x.append("\n")
        melody.extend(x)
    for i in range(len(gaps)):
        if gaps[i] == "\n":continue
        durations[i] = f"{durations[i]:.3F},"
        gaps[i] = f"{gaps[i]:.3F},"
    with open(f"{out_file}","w") as f:
        f.write("Melody:\n")
        f.writelines(melody)
        f.write("Durations:\n")
        f.writelines(durations)
        f.write("Gaps:\n")
        f.writelines(gaps)
    print(f"Finished - Data written to ./{out_file}")


from __future__ import print_function
import sys
import xml.etree.ElementTree as ET
import os

sys.path.extend(['.', '..', './pycparser/'])

from pycparser import c_parser, c_ast

filehandle = open('dummy3.c', 'r')
#filehandle = open('reverse_noinclude.c', 'r')
#filehandle = open('reverse.c', 'r')
text = ''.join(filehandle.readlines())
#print(text)

# create a pycparser
parser = c_parser.CParser()
ast = parser.parse(text, filename='<none>')

# generate the XML tree
ast.show()
codeAstXml = open('code_ast.xml','w')
ast.showXml(codeAstXml)
codeAstXml.close()
tree = ET.parse('code_ast.xml')
root = tree.getroot()

kernelsVars=[]
kernelsTyps=[]
kernelNames=['__ungenerated_kernel_function_region__0']
for kn in kernelNames:
    # go through all functions in the code (C/C++ code)
    # find the function which the kernel is called there
    # then find the type of all variables
    for func in root.findall(".//FuncDef"):
        kernelFound=0
        kernelVars=[]
        kernelTyps=[]
        print('we have found '+str(len(func.findall(".//FuncCall/ID")))+' function calls')
        for fcall in func.findall(".//FuncCall/ID"):
            if str(fcall.get('uid')).strip()==kn.strip():
                kernelFound=1
                #print(fcall.get('uid'))
        if kernelFound==1:
            print('<'+kn+'> is found in <'+func.find('Decl').get('uid')+'>')
            # go through all declerations and find the varibales
            funcBody=func.find('Compound')
            for var in funcBody.findall(".//Decl"):
                # single variable Decl
                kernelVars.append(var.get('uid'))
                kernelTyps.append(var.find('.//IdentifierType').get('uid')+((len(var.findall(".//PtrDecl")))*'*'))
#                print('< '+var.get('uid')+' > is defined as <'+var.find('.//IdentifierType').get('uid')+((len(var.findall(".//PtrDecl")))*'*')+'>')
            kernelsVars.append(kernelVars)
            kernelsTyps.append(kernelTyps)
            break

for i in range(0,len(kernelsVars)):
    var=kernelsVars[i]
    typ=kernelsTyps[i]
    print('=======> kernel #'+str(i)+':')
    for g in range(0,len(var)):
        print(var[g]+'->'+typ[g])

os.remove('code_ast.xml')


from apivk.function_vk import vkinder
from datetime import date
from database.script_bd import check_users_vk, check_search_results, save_users_vk, save_search_results
from botvk.function_botvk import write_msg, send_photo

# определение статуса отношений
def find_relation(search_user_id):
    res = vkinder.about_user(search_user_id)['response']
    for item in res:
        if 'relation' not in item:
            relation = None
        elif 'relation' in item:
            relation = item['relation']
        return relation

# определение пола
def find_sex(search_user_id):
    res = vkinder.about_user(search_user_id)['response']
    for item in res:
        if 'sex' not in item:
            any_sex = None
        elif 'sex' in item:
            sex = item['sex']
            if sex == 2:
                any_sex = 1
            elif sex == 1:
                any_sex = 2
            else:
                any_sex = 0
        return any_sex

# определение города
def find_city(search_user_id):
    res = vkinder.about_user(search_user_id)['response']
    for item in res:
        if 'city' not in item:
            city = None
        elif 'city' in item:
            city = item['city']['id']
        return city

# определение возраста
def find_age(search_user_id):
    res = vkinder.about_user(search_user_id)['response']
    for item in res:
        if 'bdate' not in item:
            age = None
        elif 'bdate' in item:
            bdate = item['bdate']
            if len(bdate) >= 8:
                day, mon, year = bdate.split('.')
                day = int(day)
                mon = int(mon)
                year = int(year)
                today = date.today()
                age = today.year - year - ((today.month, today.day) < (mon, day))
            else:
                age = None
        return age

# отправить ссылку на человека и топ-3 фото
def choose_photo(age_from, age_to, relation, sex, city, search_user_id, user_id):
    try:
        search = vkinder.users_search(age_from, age_to, relation, sex, city)
        people_id = search['response']['items']
        for people in people_id:
            try:
                id_people = int(people['id'])
                first_name = people['first_name']
                last_name = people['last_name']
                status = people['is_closed']
                city_ = people['city']['id']
            except KeyError:
                pass
            if status is False and city_ == city:
                if city_ == city:
                    if check_users_vk(id_people) is None:
                        save_users_vk(id_people, first_name, last_name)
                    if check_search_results(search_user_id, id_people) is None:
                        save_search_results(search_user_id, id_people)
                        write_msg(user_id, f'Зацени {first_name} {last_name} https://vk.com/id{id_people}')
                        for i in vkinder.photo_user(id_people):
                            owner_id = i[1][2]
                            photo_id = i[1][3]
                            photo = f'photo{owner_id}_{photo_id}'
                            send_photo(user_id, photo)
                        write_msg(user_id, 'Для продолжения поиска повторно введите команду "поиск"')
                        break
    except TypeError:
        write_msg(user_id, 'Не хватает данных для поиска')

# проверка полноты информации о пользователе
def check(age, city, sex, relation, user_id):
    count = 0
    if age is not None:
        count += 1
    if city is not None:
        count += 1
    if sex is not None:
        count += 1
    if relation is not None:
        count += 1
    if count == 4:
        write_msg(user_id, 'Отлично! Для начала поиска введите команду "поиск"')


import argparse
import glob
import logging
import os
import random
import timeit

import numpy as np
import torch
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
from torch.utils.data.distributed import DistributedSampler
from tqdm import tqdm, trange

from transformers import (
    WEIGHTS_NAME,
    AdamW,
    AlbertConfig,
    AlbertForQuestionAnswering,
    AlbertTokenizer,
    #BertConfig,
    #BertForQuestionAnswering,
    #BertForSequenceClassification,
    #BertTokenizer,
    RobertaConfig,
    RobertaForQuestionAnswering,
    RobertaTokenizer,
    get_linear_schedule_with_warmup,
    openqa_convert_examples_to_features,
)
from transformers.data.metrics.squad_metrics import (
    compute_predictions_log_probs,
    compute_predictions_logits,
    squad_evaluate,
)
from transformers.data.processors.openqa import OpenQAResult, OpenQAV1Processor, OpenQAV2Processor

try:
    from torch.utils.tensorboard import SummaryWriter
except ImportError:
    from tensorboardX import SummaryWriter

## added by Jong-Hoon Oh
import torchtext
import cnn_utils
import train_utils

NUM_PARALLEL_EXEC_UNITS = 4
os.environ['OMP_NUM_THREADS'] = str(NUM_PARALLEL_EXEC_UNITS)
os.environ["KMP_AFFINITY"] = "granularity=fine,verbose,compact,1,0"
os.environ['KMP_WARNINGS'] = 'off'

logger = logging.getLogger(__name__)

ALL_MODELS = sum(
    (tuple(conf.pretrained_config_archive_map.keys()) for conf in (AlbertConfig, RobertaConfig,)),
    (),
)

MODEL_CLASSES = {
    "albert": (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer),
    "roberta": (RobertaConfig, RobertaForQuestionAnswering, RobertaTokenizer),
}

## added by Jong-Hoon Oh
class TTDataset(torchtext.data.Dataset):
    '''Dummy Dataset for build_vocab'''
    def __init__(self, words, fields):
        data_fields = [('text', fields['text'])]
        ex = (words,)
        examples = [torchtext.data.Example.fromlist(ex, data_fields)]
        super(TTDataset, self).__init__(examples, data_fields)

## added by Jong-Hoon Oh
def load_cnn_model_and_vocab(args, cnn_file, words):
    assert args.emb_file and args.min_freq

    fields = cnn_utils.get_fields()
    train_utils.build_vocab(args, fields, TTDataset(words, fields), [])
    vocab = fields['text'].vocab
    model, pre_fields = train_utils.load_cnn_model(args, cnn_file, fields)

    return model, pre_fields['text'].vocab.stoi

def set_seed(args):
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if args.n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

def to_list(tensor):
    return tensor.detach().cpu().tolist()

# modified by Jong-Hoon Oh
# DATA PROCESSING PART
# - Converting input examples to cached examples
# - cnn_stoi: vocab.stoi for the cnn model
def load_and_cache_examples(args, filename, tokenizer, cnn_stoi, evaluate=False, output_examples=False):
    if args.local_rank not in [-1, 0] and not evaluate:
        # Make sure only the first process in distributed training process the dataset, and the others will use the cache
        torch.distributed.barrier()

    bert_token_str = "ot0"
    input_dir = args.feat_dir if args.feat_dir else "."
    fstem =  list(filter(None,filename.split("/"))).pop()
    fstem =  fstem.split(".")[0]
    fstem =  fstem
    cached_file = "cached_{}_{}_{}_{}_{}_{}".format(
            fstem,
            list(filter(None, args.model_name_or_path.split("/"))).pop(),
            args.cnn_stem,
            list(filter(None, args.cnn_model.split("_"))).pop(),
            bert_token_str,
            str(args.max_seq_length),
        )
    # split the input data into data, positive_data, feature, and example
    dset_dir = input_dir + '/dset'
    pdset_dir = input_dir + '/pdset'
    feat_dir = input_dir + '/feat'
    exset_dir = input_dir + '/exset'
    cached_dset_file = os.path.join(dset_dir,cached_file)
    cached_feat_file = os.path.join(feat_dir,cached_file)
    cached_pdset_file = os.path.join(pdset_dir,cached_file)
    cached_exset_file = os.path.join(exset_dir,cached_file)
    if evaluate:
       logger.info("Specified cached file %s for dev or predict files", cached_dset_file)
    else:
       logger.info("Specified cached file %s for train files", cached_dset_file)

    # Init features and dataset from cache if it exists
    if os.path.exists(cached_dset_file) and not args.overwrite_cache:
        logger.info("Feature files already exist: %s", cached_dset_file)
    else:
        logger.info("Creating features from dataset file at %s", input_dir) # input_dir="." by defaults

        # if no predict file for evaluation or no train file for training
        if not args.data_dir and ((evaluate and not args.predict_file) or (not evaluate and not args.train_file)):
            try:
                import tensorflow_datasets as tfds
            except ImportError:
                raise ImportError("If not data_dir is specified, tensorflow_datasets needs to be installed.")

            if args.version_2_with_negative:
                logger.warn("tensorflow_datasets does not handle version 2 of SQuAD.")

            tfds_examples = tfds.load("openqa")
            examples = OpenQAV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate)
        else:
            # The main part of data processing in our OpenQA experiments
            processor = OpenQAV1Processor()

            if evaluate:
                # initializer
                examples = processor.get_dev_examples(args.data_dir, filename=filename)
            else:
                # initializer
                examples = processor.get_train_examples(args.data_dir, filename=filename)

        features, dataset, possible_dataset = openqa_convert_examples_to_features(
            examples=examples,
            tokenizer=tokenizer,
            cnn_stoi=cnn_stoi,
            max_seq_length=args.max_seq_length,
            doc_stride=args.doc_stride,
            max_query_length=args.max_query_length,
            is_training=not evaluate,
            return_dataset="pt", # "pt" represents 'pytorch dataset'
            threads=args.threads,
        )

        if args.local_rank in [-1, 0]:
            logger.info("Saving features into cached file %s", cached_dset_file)
            if evaluate:
               logger.info("dataset:{}".format(len(dataset)))
               torch.save({"dataset": dataset}, cached_dset_file)
               logger.info("features")
               torch.save({"features": features}, cached_feat_file)
               logger.info("examples")
               torch.save({"examples": examples}, cached_exset_file)
            else:
               logger.info("dataset:{}".format(len(dataset)))
               torch.save({"dataset": dataset}, cached_dset_file)
               logger.info("possible_dataset:{}".format(len(possible_dataset)))
               torch.save({"possible_dataset": possible_dataset}, cached_pdset_file)

    if args.local_rank == 0 and not evaluate:
        # Make sure only the first process in distributed training process the dataset, and the others will use the cache
        torch.distributed.barrier()

def main():
    parser = argparse.ArgumentParser()

    # Required parameters
    parser.add_argument(
        "--model_type",
        default=None,
        type=str,
        required=True,
        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()),
    )
    parser.add_argument(
        "--model_name_or_path",
        default=None,
        type=str,
        required=True,
        help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS),
    )
    parser.add_argument(
        "--prep_vocab_file",
        default=None,
        type=str,
        help="preprocessed_vocab_file with the train/dev/predict file. see make_openqa_cnn_vocab.py",
    )
    parser.add_argument(
        "--emb_file",
        default=None,
        type=str,
        help="The embedding vector file used for cnn",
    )
    parser.add_argument(
        "--cnn_model",
        default=None,
        type=str,
        help="The cnn model file name",
    )
    parser.add_argument(
        "--cnn_stem",
        default="enwiki",
        type=str,
        help="stem for cnn models for caching (different vocab.stoi for each model)",
    )
    parser.add_argument(
        "--min_freq",
        default=5,
        type=int,
        help="min freq. for unknown words",
    )
    parser.add_argument(
        "--emb_dim",
        default=300,
        type=int,
        help="dim for representation of fastText",
    )

    # Other parameters
    parser.add_argument(
        "--data_dir",
        default=None,
        type=str,
        help="The input data dir. Should contain the .json files for the task."
        + "If no data dir or train/predict files are specified, will run with tensorflow_datasets.",
    )
    parser.add_argument(
        "--train_file",
        default=None,
        type=str,
        help="The input training file. If a data dir is specified, will look for the file there"
        + "If no data dir or train/predict files are specified, will run with tensorflow_datasets.",
    )
    parser.add_argument(
        "--predict_file",
        default=None,
        type=str,
        help="The input evaluation file. If a data dir is specified, will look for the file there"
        + "If no data dir or train/predict files are specified, will run with tensorflow_datasets.",
    )
    parser.add_argument(
        "--dev_file",
        default=None,
        type=str,
        help="The input development file. If a data dir is specified, will look for the file there"
        + "If no data dir or devel files are specified, will run with tensorflow_datasets.",
    )
    parser.add_argument(
        "--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name"
    )
    parser.add_argument(
        "--tokenizer_name",
        default="",
        type=str,
        help="Pretrained tokenizer name or path if not the same as model_name",
    )
    parser.add_argument(
        "--cache_dir",
        default="",
        type=str,
        help="Where do you want to store the pre-trained models downloaded from s3",
    )
    parser.add_argument(
        "--feat_dir",
        default="",
        type=str,
        help="Where do you want to store the processed data whose features were extracted from the input data",
    )

    parser.add_argument(
        "--max_seq_length",
        default=384,
        type=int,
        help="The maximum total input sequence length after WordPiece tokenization. Sequences "
        "longer than this will be truncated, and sequences shorter than this will be padded.",
    )
    parser.add_argument(
        "--doc_stride",
        default=128,
        type=int,
        help="When splitting up a long document into chunks, how much stride to take between chunks.",
    )
    parser.add_argument(
        "--max_query_length",
        default=64,
        type=int,
        help="The maximum number of tokens for the question. Questions longer than this will "
        "be truncated to this length.",
    )
    parser.add_argument(
        "--do_lower_case", action="store_true", help="Set this flag if you are using an uncased model."
    )

    parser.add_argument(
        "--max_answer_length",
        default=30,
        type=int,
        help="The maximum length of an answer that can be generated. This is needed because the start "
        "and end predictions are not conditioned on one another.",
    )
    parser.add_argument(
        "--verbose_logging",
        action="store_true",
        help="If true, all of the warnings related to data processing will be printed. "
        "A number of warnings are expected for a normal SQuAD evaluation.",
    )
    parser.add_argument(
        "--lang_id",
        default=0,
        type=int,
        help="language id of input for language-specific xlm models (see tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)",
    )

    parser.add_argument("--no_cuda", action="store_true", help="Whether not to use CUDA when available")
    parser.add_argument(
        "--overwrite_cache", action="store_true", help="Overwrite the cached training and evaluation sets"
    )
    parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")

    parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus")
    parser.add_argument(
        "--fp16",
        action="store_true",
        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
    )
    parser.add_argument(
        "--fp16_opt_level",
        type=str,
        default="O1",
        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
        "See details at https://nvidia.github.io/apex/amp.html",
    )
    parser.add_argument("--server_ip", type=str, default="", help="Can be used for distant debugging.")
    parser.add_argument("--server_port", type=str, default="", help="Can be used for distant debugging.")

    parser.add_argument("--threads", type=int, default=1, help="multiple threads for converting example to features")
    args = parser.parse_args()

    assert args.prep_vocab_file is not None
    assert args.cnn_model is not None
    assert args.cnn_stem is not None
    assert args.emb_dim is not None
    assert args.emb_file is not None

    if (not os.path.exists(args.prep_vocab_file)):
        raise ValueError(
            "prep_vocab_file ({}) does not exist. Check the --prep_vocab_file option.".format( args.prep_vocab_file) )

    if (not os.path.exists(args.cnn_model)):
        raise ValueError(
            "cnn_model ({}) does not exist. Check the --cnn_model option.".format( args.cnn_model) )

    if (not os.path.exists(args.emb_file)):
        raise ValueError(
            "emb_file ({}) does not exist. Check the --emb_file option.".format( args.emb_file) )

    if args.doc_stride >= args.max_seq_length - args.max_query_length:
        logger.warning(
            "WARNING - You've set a doc stride which may be superior to the document length in some "
            "examples. This could result in errors when building features from the examples. Please reduce the doc "
            "stride or increase the maximum length to ensure the features are correctly built."
        )

    # Setup CUDA, GPU & distributed training
    if args.local_rank == -1 or args.no_cuda:
        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
        args.n_gpu = torch.cuda.device_count()
    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
        torch.cuda.set_device(args.local_rank)
        device = torch.device("cuda", args.local_rank)
        torch.distributed.init_process_group(backend="nccl")
        args.n_gpu = 1
    args.device = device

    # Setup logging
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
    )
    logger.warning(
        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
        args.local_rank,
        device,
        args.n_gpu,
        bool(args.local_rank != -1),
        args.fp16,
    )

    # Set seed
    set_seed(args)

    # Load pretrained model and tokenizer
    if args.local_rank not in [-1, 0]:
        # Make sure only the first process in distributed training will download model & vocab
        # The barrier starts
        torch.distributed.barrier()

    # added by Jong-Hoon Oh
    #  - Load cnn model and pre-processed vocab.
    #  - prep_vocab_file: see vocab/
    prep_tokens = torch.load(args.prep_vocab_file)
    all_tokens = prep_tokens['tokens']
    cnn_model, cnn_stoi = load_cnn_model_and_vocab(args, args.cnn_model, all_tokens)
    cnn_dim = len(cnn_model.args.filter_widths) * cnn_model.args.filter_size
    args.cnn_dim = cnn_dim

    args.model_type = args.model_type.lower()
    # "albert": (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer),
    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
    config = config_class.from_pretrained(
        args.config_name if args.config_name else args.model_name_or_path,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
    config.num_of_TIERs = 3
    config.cnn_dim = args.cnn_dim
    config.emb_dim = args.emb_dim
    config.cnn_model = args.cnn_model
    config.cnn_stem = args.cnn_stem
    # tokenizer_class: AlbertTokenizer
    tokenizer = tokenizer_class.from_pretrained(
        args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
        do_lower_case=args.do_lower_case,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
    # model_class: AlbertForQuestionAnswering
    model = model_class.from_pretrained(
        args.model_name_or_path,
        from_tf=bool(".ckpt" in args.model_name_or_path), # ckpt: tensorflow file, pt: pytorch file
        config=config,
        cache_dir=args.cache_dir if args.cache_dir else None,
    )
    ###########

    if args.local_rank == 0:
        # Make sure only the first process in distributed training will download model & vocab
        # The barrier ends
        torch.distributed.barrier()

    model.to(args.device)
    cnn_model.to(args.device)

    logger.info("Training/evaluation parameters %s", args)

    # Before we do anything with models, we want to ensure that we get fp16 execution of torch.einsum if args.fp16 is set.
    # Otherwise it'll default to "promote" mode, and we'll get fp32 operations. Note that running `--fp16_opt_level="O2"` will
    # remove the need for this code, but it is still valid.
    if args.fp16:
        try:
            import apex

            apex.amp.register_half_function(torch, "einsum")
        except ImportError:
            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
    if args.train_file is not None:
       load_and_cache_examples(args, args.train_file, tokenizer, cnn_stoi, evaluate=False, output_examples=False)
    if args.predict_file is not None:
       load_and_cache_examples(args, args.predict_file, tokenizer, cnn_stoi, evaluate=True, output_examples=True)
    if args.dev_file is not None:
       load_and_cache_examples(args, args.dev_file, tokenizer, cnn_stoi, evaluate=True, output_examples=True)

if __name__ == "__main__":
    main()


from os.path import join
from typing import Optional

from dagger.dag_creator.airflow.operator_creator import OperatorCreator
from dagger.dag_creator.airflow.operators.redshift_sql_operator import (
    RedshiftSQLOperator,
)

class RedshiftLoadCreator(OperatorCreator):
    ref_name = "redshift_load"

    def __init__(self, task, dag):
        super().__init__(task, dag)

        self._input_path = join(self._task.inputs[0].rendered_name, "")
        self._input_s3_bucket = self._task.inputs[0].bucket
        self._input_s3_prefix = self._task.inputs[0].path

        self._output_schema = self._task.outputs[0].schema
        self._output_table = self._task.outputs[0].table
        self._output_schema_quoted = f'"{self._output_schema}"'
        self._output_table_quoted = f'"{self._output_table}"'

        self._tmp_table = (
            f"{self._task.tmp_table_prefix}_{self._output_table}"
            if self._task.tmp_table_prefix
            else None
        )
        self._tmp_table_quoted = f'"{self._tmp_table}"' if self._tmp_table else None

        self._copy_ddl_from = self._task.copy_ddl_from
        self._alter_columns = self._task.alter_columns

        self._sort_keys = self._task.sort_keys

    @staticmethod
    def _read_sql(directory, file_path):
        full_path = join(directory, file_path)

        with open(full_path, "r") as f:
            sql_string = f.read()

        return sql_string

    def _get_create_table_cmd(self) -> Optional[str]:
        if self._tmp_table and self._task.create_table_ddl:
            ddl = self._read_sql(
                self._task.pipeline.directory, self._task.create_table_ddl
            )
            return ddl.format(
                schema_name=self._output_schema_quoted,
                table_name=self._tmp_table_quoted,
            )
        if self._tmp_table and self._copy_ddl_from:
            return (
                f"CREATE TABLE {self._output_schema_quoted}.{self._tmp_table_quoted}"
                f"(LIKE {self._copy_ddl_from})"
            )
        elif self._tmp_table:
            return (
                f"CREATE TABLE {self._output_schema_quoted}.{self._tmp_table_quoted}"
                f"(LIKE {self._output_schema_quoted}.{self._output_table_quoted})"
            )
        elif self._task.create_table_ddl:
            ddl = self._read_sql(
                self._task.pipeline.directory, self._task.create_table_ddl
            )
            return ddl.format(
                schema_name=self._output_schema_quoted,
                table_name=self._output_table_quoted,
            )
        elif self._copy_ddl_from:
            return (
                f"CREATE TABLE IF NOT EXISTS {self._output_schema_quoted}.{self._output_table}"
                f"(LIKE {self._copy_ddl_from})"
            )

        return None

    def _get_sort_key_cmd(self) -> Optional[str]:
        sort_key_cmd = None
        if self._sort_keys:
            sort_key_cmd = (
                f"ALTER TABLE {self._output_schema_quoted}.{self._tmp_table_quoted} "
                f"ALTER COMPOUND SORTKEY({self._sort_keys})"
            )
        return sort_key_cmd

    def _get_delete_cmd(self) -> Optional[str]:
        if self._task.incremental:
            return (
                f"DELETE FROM {self._output_schema_quoted}.{self._output_table_quoted}"
                f"WHERE {self._task.delete_condition}"
            )

        if not self._task.incremental and self._tmp_table is None:
            return f"TRUNCATE TABLE {self._output_schema_quoted}.{self._output_table_quoted}"

        return None

    def _get_load_cmd(self) -> Optional[str]:
        table_name = self._tmp_table_quoted or self._output_table_quoted
        columns = "({})".format(self._task.columns) if self._task.columns else ""
        extra_parameters = "\n".join(
            [
                "{} {}".format(key, value)
                for key, value in self._task.extra_parameters.items()
            ]
        )

        return (
            f"copy {self._output_schema_quoted}.{table_name}{columns}\n"
            f"from '{self._input_path}'\n"
            f"iam_role '{self._task.iam_role}'\n"
            f"{extra_parameters}"
        )

    def _get_replace_table_cmd(self) -> Optional[str]:
        if self._tmp_table is None:
            return None

        return (
            f"BEGIN TRANSACTION;\n"
            f"DROP TABLE IF EXISTS {self._output_schema_quoted}.{self._output_table_quoted};\n"
            f"ALTER TABLE {self._output_schema_quoted}.{self._tmp_table_quoted} "
            f"RENAME TO {self._output_table_quoted};\n"
            f"END"
        )

    def _get_alter_columns_cmd(self) -> Optional[str]:
        if self._alter_columns is None:
            return None

        alter_column_commands = []
        alter_columns = self._alter_columns.split(",")
        for alter_column in alter_columns:
            [column_name, column_type] = alter_column.split(":")
            alter_column_commands.append(
                f"ALTER TABLE {self._output_schema_quoted}.{self._tmp_table_quoted} "
                f"ALTER COLUMN {column_name} TYPE {column_type}"
            )

        return ";\n".join(alter_column_commands)

    def _get_drop_tmp_table_cmd(self) -> Optional[str]:
        if self._tmp_table is None:
            return None

        return f"DROP TABLE IF EXISTS {self._output_schema_quoted}.{self._tmp_table_quoted}"

    def _get_cmd(self) -> str:
        raw_load_cmd = [
            self._get_drop_tmp_table_cmd(),
            self._get_create_table_cmd(),
            self._get_alter_columns_cmd(),
            self._get_sort_key_cmd(),
            self._get_delete_cmd(),
            self._get_load_cmd(),
            self._get_replace_table_cmd(),
        ]

        load_cmd = [cmd for cmd in raw_load_cmd if cmd]

        return ";\n".join(load_cmd)

    def _create_operator(self, **kwargs):
        load_cmd = self._get_cmd()

        redshift_op = RedshiftSQLOperator(
            dag=self._dag,
            task_id=self._task.name,
            sql=load_cmd,
            redshift_conn_id=self._task.postgres_conn_id,
            autocommit=True,
            **kwargs,
        )
        return redshift_op


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""--------------------------------------------------------------------
GENETIC ALGORITHMS EXPERIMENTS
Started on the 2018/01/03
theo.alves.da.costa@gmail.com
https://github.com/theolvs
------------------------------------------------------------------------
"""

from scipy import stats
import seaborn as sns
import os
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import sys
import time
from tqdm import tqdm
import itertools

#=============================================================================================================================
# DISTRIBUTIONS
#=============================================================================================================================

class Dist(object):
    def __init__(self,mu = None,std = None,label = None):
        self.mu = np.random.rand()*20 - 10 if mu is None else mu
        self.std = np.random.rand()*10 if std is None else std
        self.label = "" if not label else " - "+label
        self.func = lambda x : stats.norm.cdf(x,loc = self.mu,scale = self.std)

    def __repr__(self,markdown = False):
        return "Norm {1}mu={2}{0}, {0}std={3}{0}{4}".format("$" if markdown else "","$\\" if markdown else "",
                                                             round(self.mu,2),round(self.std,2),self.label)

    def plot(self,fill = True):
        x = np.linspace(-20, 20, 100)
        y = stats.norm.pdf(x,loc = self.mu,scale = self.std)
        plt.plot(x,y,label = self.__repr__(markdown = True))
        if fill:
            plt.fill_between(x, 0, y, alpha=0.4)

    def __add__(self,other):
        mu = np.mean([self.mu,other.mu])
        std = np.mean([self.std,other.std])
        return Dist(mu,std)

    def mutate(self,alpha = 1):
        self.mu = self.mu + 1/(1+np.log(1+alpha)) * np.random.randn()
        self.std = max(self.std + 1/(1+np.log(1+alpha)) * np.random.randn(),0.5)

    def fitness(self,x):
        return 1 - stats.kstest(x,self.func).statistic

class Population(object):
    def __init__(self,distributions = None,n = 100):
        if distributions is not None:
            self.distributions = distributions
        else:
            self.distributions = [Dist() for i in range(n)]

    def __getitem__(self,key):
        if type(key) == tuple or type(key) == list:
            d = []
            for i in key:
                d.append(self.distributions[i])
            return d
        else:
            return self.distributions[key]

    def __iter__(self):
        return iter(self.distributions)

    def __len__(self):
        return len(self.distributions)

    def plot(self,title = "Normal distributions",figsize = None):
        if figsize:
            plt.figure(figsize = figsize)
        plt.title(title)
        fill = len(self) < 5
        for d in self:
            d.plot(fill = fill)
        plt.legend()
        plt.xlabel("x")
        plt.show()

    def evaluate(self,x):
        fitnesses = [(i,dist.fitness(x)) for i,dist in enumerate(self)]
        indices,fitnesses = zip(*sorted(fitnesses,key = lambda x : x[1],reverse = True))
        return indices,fitnesses

    def selection(self,x,top = 0.1):
        indices,fitnesses = self.evaluate(x)
        n = int(top*len(fitnesses))
        return indices[:n]

    def crossover(self,indices):
        combinations = list(itertools.combinations(indices,2))
        np.random.shuffle(combinations)
        combinations = combinations[:len(self)]
        new_population = []
        for i,j in combinations:
            new_population.append(self[i]+self[j])
        self.distributions = new_population

    def mutate(self,generation = 1):
        for d in self:
            d.mutate(generation)

    def evolve(self,x,top = 0.25,n_generations = 20,last_selection = True):
        all_fitnesses = [self.evaluate(x)[1]]

        for generation in tqdm(range(n_generations)):

            indices = self.selection(x,top)
            self.crossover(indices)
            self.mutate(generation)

            indices,fitnesses = self.evaluate(x)
            all_fitnesses.append(fitnesses)

        self._plot_fitnesses(all_fitnesses)

        if last_selection:
            indices = self.selection(x,top)

        return Population(self[indices])

    def _plot_fitnesses(self,fitnesses):
        sups = []
        infs = []
        means = []
        for step in fitnesses:
            sups.append(np.max(step))
            infs.append(np.min(step))
            means.append(np.mean(step))

        plt.figure(figsize=(10,6))
        plt.plot(means)
        plt.fill_between(range(len(means)),sups,infs, alpha = 0.2)
        plt.xlabel('# Generation')
        plt.ylabel('Fitness')
        plt.legend()
        plt.show()

#=============================================================================================================================
# LOGREG
#=============================================================================================================================

import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F

class LogReg(torch.nn.Module):
    def __init__(self, n_feature,n_output = 1,alpha = 10e-1):
        self.alpha = alpha
        self.args = n_feature,n_output
        super(LogReg, self).__init__()
        self.out = torch.nn.Linear(n_feature,n_output,bias = False)   # output layer

    def forward(self, x):
        x = Variable(torch.FloatTensor(x))
        x = F.sigmoid(self.out(x))
        return x

    def __add__(self,other):
        new = LogReg(*self.args)
        new.out.weight.data = torch.FloatTensor(0.5 * (self.out.weight.data.numpy() + other.out.weight.data.numpy()))
        return new

    def mutate(self,generation):
        out = self.out.weight.data.numpy()
        noise_out = self.alpha * np.random.randn(*out.shape)
        self.out.weight.data = torch.FloatTensor(self.out.weight.data.numpy() + noise_out)

    def evaluate(self,x,y):
        pred = self.forward(x).data.numpy()
        loss_1 = np.sum(np.log(pred + 10e-9)*y.reshape(-1,1))
        loss_0 = np.sum(np.log(1-pred + 10e-9)*(1-y).reshape(-1,1))
        return loss_1 + loss_0

    def plot_coefs(self):
        plt.figure(figsize = (15,4))
        plt.title("Coefficients")
        plt.axhline(0,c = "black")
        plt.plot(self.out.weight.data.numpy()[0])
        plt.xlabel("# Pixel")
        plt.show()

class PopulationLogReg(object):
    def __init__(self,x,y,regs = None,n = 20,top = 0.25,**kwargs):

        self.x = x
        self.y = y
        self.kwargs = kwargs

        if regs is None:
            self.regs = [LogReg(**kwargs) for i in range(n)]
        else:
            self.regs = regs

    def __getitem__(self,key):
        if type(key) == tuple or type(key) == list:
            d = []
            for i in key:
                d.append(self.regs[i])
            return d
        else:
            return self.regs[key]

    def __iter__(self):
        return iter(self.regs)

    def __len__(self):
        return len(self.regs)

    def evaluate(self):
        fitnesses = [(i,element.evaluate(self.x,self.y)) for i,element in enumerate(self)]
        indices,fitnesses = zip(*sorted(fitnesses,key = lambda x : x[1],reverse = True))
        return indices,fitnesses

    def selection(self,top = 0.5):
        indices,fitnesses = self.evaluate()
        n = int(top*len(fitnesses))
        return indices[:n]

    def crossover(self,indices):
        combinations = list(itertools.combinations(indices,2))
        np.random.shuffle(combinations)
        combinations = combinations[:len(self)]
        new_population = []
        for i,j in combinations:
            new_population.append(self[i]+self[j])

        if len(new_population) < len(self):
            new_population.extend([LogReg(**self.kwargs) for i in range(len(self)-len(new_population))])
        self.regs = new_population

    def mutate(self,generation):
        for d in self:
            d.mutate(generation)

    def evolve(self,top = 0.25,n_generations = 20,last_selection = True):
        n_fittest = int(top*len(self))
        offsprings = len(list(itertools.combinations(range(n_fittest),2)))
        print("- Generations {}".format(len(self)))
        print("- Fittest : {}".format(n_fittest))
        print("- Offsprings : {}".format(offsprings))

        all_fitnesses = [self.evaluate()[1]]

        for generation in tqdm(range(n_generations)):

            indices = self.selection(top)
            self.crossover(indices)
            self.mutate(generation)

            indices,fitnesses = self.evaluate()
            all_fitnesses.append(fitnesses)

        self._plot_fitnesses(all_fitnesses)

        if last_selection:
            indices = self.selection(top)

        return PopulationLogReg(self.x,self.y,regs = self[indices])

    def _plot_fitnesses(self,fitnesses):

        from sklearn.linear_model import LogisticRegression
        lr = LogisticRegression()
        lr.fit(self.x,self.y)
        pred_bench = lr.predict_proba(self.x)
        loss_bench = np.sum(np.log(pred_bench + 10e-9)*self.y.reshape(-1,1)) + np.sum(np.log(1-pred_bench + 10e-9)*(1-self.y).reshape(-1,1))

        sups = []
        infs = []
        means = []
        for step in fitnesses:
            sups.append(np.max(step))
            infs.append(np.min(step))
            means.append(np.mean(step))

        plt.figure(figsize=(10,6))
        plt.plot(means)
        plt.fill_between(range(len(means)),sups,infs, alpha = 0.2)
        plt.axhline(loss_bench)
        plt.xlabel('# Generation')
        plt.ylabel('Fitness')
        plt.legend()
        plt.show()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        ('main', '0001_initial'),
    ]

    operations = [
        migrations.AlterField(
            model_name='image',
            name='uploaded_by',
            field=models.ForeignKey(to=settings.AUTH_USER_MODEL),
        ),
        migrations.AlterField(
            model_name='website',
            name='owner',
            field=models.ForeignKey(to=settings.AUTH_USER_MODEL),
        ),
        migrations.DeleteModel(
            name='SiteOwner',
        ),
    ]


#!/usr/bin/env python
#

"""
Name:       Jesus Hernandez                         Partner:    Zechariah Neak
Email:      jherna83@ucsc.edu                       Email:      zneak@ucsc.edu
ID:         1420330
Course:     CMPM146 Game AI
Professor:  Daniel G Shapiro

                                \\\\\\\ Program 4 ///////

Description:
    This is a bot that is designed to win at Planet Wars against 5 other bots using
    a behavior tree. The root acts as a Selector composite parent that checks through
    each Sequence composite child top to bottom, and performs the action for whatever
    Sequence child returns true. Each Sequence child only returns true if all its
    checks and actions come out as successful.

"""

"""
// There is already a basic strategy in place here. You can use it as a
// starting point, or you can throw it out entirely and replace it with your
// own.
"""
import logging, traceback, sys, os, inspect
logging.basicConfig(filename=__file__[:-3] +'.log', filemode='w', level=logging.DEBUG)
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.append(parentdir)

from behavior_tree_bot.behaviors import *
from behavior_tree_bot.checks import *
from behavior_tree_bot.bt_nodes import Selector, Sequence, Action, Check

from planet_wars import PlanetWars, finish_turn

# You have to improve this tree or create an entire new one that is capable
# of winning against all the 5 opponent bots
def setup_behavior_tree():

    # Top-down construction of behavior tree
    root = Selector(name='High Level Ordering of Strategies')

    # Define available actions to take.
    colonize = Action(take_defenseless_territory)
    invade = Action(attack_with_no_mercy)
    reinforce = Action(reinforce_with_vengeance)
    retaliate = Action(retaliate_with_fury)

    # *** Begin preliminary suprise invasion over the galaxy. ***
    imperial_ambition = Sequence(name='Expansion Strategy: Manifest Destiny')
    imperial_ambition.child_nodes = [colonize, invade]

    # *** Consolidate and retaliate if under attack by hostiles. ***
    imperial_shield = Sequence(name='Security Strategy: Cereberus')
    danger_check = Check(if_under_attack)
    imperial_shield.child_nodes = [danger_check, reinforce, retaliate]

    # *** If the advantage is ours, attack with full force. ***
    imperial_aggression = Sequence(name='Aggressive Strategy: Crush All Remaining Resistance')
    largest_fleet_check = Check(have_largest_fleet)
    imperial_aggression.child_nodes = [largest_fleet_check, invade]

    # Begin selecting strategies.
    root.child_nodes = [imperial_ambition, imperial_aggression, imperial_shield, invade.copy()]

    logging.info('\n' + root.tree_to_string())
    return root

# You don't need to change this function
def do_turn(state):
    behavior_tree.execute(planet_wars)

if __name__ == '__main__':
    logging.basicConfig(filename=__file__[:-3] + '.log', filemode='w', level=logging.DEBUG)

    behavior_tree = setup_behavior_tree()
    try:
        map_data = ''
        while True:
            current_line = input()
            if len(current_line) >= 2 and current_line.startswith("go"):
                planet_wars = PlanetWars(map_data)
                do_turn(planet_wars)
                finish_turn()
                map_data = ''
            else:
                map_data += current_line + '\n'

    except KeyboardInterrupt:
        print('ctrl-c, leaving ...')
    except Exception:
        traceback.print_exc(file=sys.stdout)
        logging.exception("Error in bot.")


"""
Module to handle a local InfoKinds with unique name.

NOTE: this is taken from python-common in nomad-lab-base.
It is copied here to remove the dependency from nomad-lab-base.
For more info on python-common visit:
https://gitlab.mpcdf.mpg.de/nomad-lab/python-common

The author of this code is: Dr. Fawzi Roberto Mohamed
E-mail: mohamed@fhi-berlin.mpg.de

"""
from past.builtins import cmp
from future import standard_library
standard_library.install_aliases()

from builtins import str
from builtins import range
from builtins import object
import logging
from ai4materials.external import compact_sha
import json
import os, re
from ai4materials.external.json_support import jsonCompactS, jsonCompactD, jsonIndentD
from io import open

class InfoKindEl(object):
    """Info kind (tipically from a file, without shas but with locally unique names)"""
    __slots__ = ["name","description","kindStr","units","superNames","dtypeStr", "repeats", "shape", "extra_args"]
    IGNORE_EXTRA_ARGS = 1
    ADD_EXTRA_ARGS = 2
    RAISE_IF_EXTRA_ARGS = 3

    def __init__(self, name, description, kindStr = "type_document_content", units = None, superNames = None,
            dtypeStr = None, shape = None, extraArgsHandling = ADD_EXTRA_ARGS, repeats = None, **extra_args):
        if superNames is None:
            superNames = []
        self.name = name
        self.description = description
        self.kindStr = kindStr
        self.superNames = superNames
        self.units = units
        self.dtypeStr = dtypeStr
        if dtypeStr in ["None", "null"]:
            self.dtypeStr = None
        self.shape = shape
        self.repeats = repeats
        if extraArgsHandling == self.ADD_EXTRA_ARGS:
            self.extra_args = extra_args
        elif extraArgsHandling == self.IGNORE_EXTRA_ARGS:
            self.extra_args = {}
        else:
            raise Exception("extra arguments to InfoKindEl:" + str(extra_args))

    def __eq__(o1, o2):
        try:
            if not (o1.name == o2.name and o1.description == o2.description and o1.kindStr == o2.kindStr and
                o1.units == o2.units and o1.shape == o2.shape):
                return False
            if o1.dtypeStr != o2.dtypeStr:
                return False
            if o1.repeats != o2.repeats:
                return False
            if o1.extra_args != o2.extra_args:
                return False
            if o1.superNames == o2.superNames:
                return True
            if len(o1.superNames) != len(o2.superNames):
                return False
            if o1.superNames[0] != o2.superNames[0]:
                return False
            a1 = o1.superNames[1:]
            a2 = o2.superNames[1:]
            a1.sort()
            a2.sort()
            for i in range(len(a1)):
                if a1[i] != a2[i]:
                    return False
            return True
        except:
            raise
            return False

    def __cmp__(k1, k2):
        c = cmp(k1.name, k2.name)
        if c != 0: return c
        c = cmp(k1.kindStr, k2.kindStr)
        if c != 0: return c
        c = cmp(k1.description, k2.description)
        if c != 0: return c
        if len(k1.superNames) > 0:
            if len(k2.superNames) > 0:
                c = cmp(k1.superNames[0], k2.superNames[0])
                if c != 0: return c
                s1 = k1.superNames[1:]
                s2 = k2.superNames[1:]
                c = cmp(s1, s2)
                if c != 0: return c
            else:
                return 1
        elif len(k2.superNames) > 0:
            return -1
        if c != 0: return c
        c = cmp(k1.units, k2.units)
        if c != 0: return c
        c = cmp(k1.dtypeStr, k2.dtypeStr)
        if c != 0: return c
        c = cmp(k1.repeats, k2.repeats)
        if c != 0: return c
        c = cmp(k1.shape, k2.shape)
        if c != 0: return c
        if k1.extra_args == k2.extra_args:
            return 0
        if k1.extra_args is None:
            return 1
        if k2.extra_args is None:
            return -1
        extraK1 = list(k1.extra_args.keys())
        extraK1.sort()
        extraK2 = list(k2.extra_args.keys())
        extraK2.sort()
        i = 0
        while (i < len(extraK1) and i < len(extraK2)):
            kk1 = extraK1[i]
            kk2 = extraK2[i]
            c = cmp(kk1, kk2)
            if c != 0: return c # use -c ?
            c = cmp(k1.extra_args[kk1], k2.extra_args[kk2])
            if c != 0: return c
        c = cmp(len(extraK1), len(extraK2))
        return c

    def __ne__(o1, o2):
        return not o1.__eq__(o2)

    def prepare(self, env):
        if len(self.superNames) > 1:
            a = self.superNames[1:]
            a.sort(lambda x, y: cmp(env.gidOf(x, precalculatedGid = True), env.gidOf(y, precalculatedGid = True)))
            self.superNames[1:] = a

    def evalGid(self, env):
        self.prepare(env)
        sha = env.newSha()
        self.serialize(env,sha.update, precalculatedGid = True, selfGid = False)
        return 'p' + sha.b64digest()[:28]

    def serialize(self, env, writeOut, subGids = True, addExtraArgs = True, precalculatedGid = False, selfGid = True):
        d = self.toDict(env, subGids = subGids, addExtraArgs = addExtraArgs, precalculatedGid = precalculatedGid, selfGid = selfGid)
        jsonCompactD(d, writeOut)

    def toDict(self, env = None, addExtraArgs = True, inlineExtraArgs = True , selfGid = False, subGids = False, precalculatedGid = False):
        res = {
            "description": self.description,
            "name": self.name,
            "superNames": self.superNames,
        }
        try:
            if self.kindStr != "type_document_content":
                if self.kindStr is None or self.kindStr == "":
                    res["kindStr"] = "MetaType"
                else:
                    res["kindStr"] = self.kindStr
            if env:
                if selfGid:
                    res["gid"] = env.gidOf(self.name, precalculatedGid = precalculatedGid)
                if subGids:
                    res["superGids"] = [ env.gidOf(sName, precalculatedGid = precalculatedGid) for sName in self.superNames ]
            elif subGids or selfGid:
                raise Exception("env required in toDict for subGids or selfGid")
            if self.units is not None:
                res["units"] = self.units
            if self.dtypeStr is not None:
                res["dtypeStr"] = self.dtypeStr
            if self.repeats is not None:
                res["repeats"] = self.repeats
            if self.shape is not None:
                res["shape"] = self.shape
            if addExtraArgs:
                if inlineExtraArgs:
                    res.update(self.extra_args)
                else:
                    res["extraArgs"] = self.extra_args
        except:
            logging.exception("error in InfoKindEl.toDict, partial dict is %s", res)
        return res

    def __unicode__(self):
        s = StringIO.StringIO()
        self.serialize(s)
        return s.string

class RelativeDependencySolver(object):
    def __init__(self):
        self.deps = {}

    def __call__(self, infoKindEnv, source, dep):
        if "relativePath" not in dep:
            raise Exception('Invalid dependency for relativeDependencySolver there must be a relativePath')
        basePath = source.get('path')
        if basePath:
            baseDir = os.path.dirname(os.path.abspath(basePath))
        else:
            baseDir = os.getcwd()
        dPath = os.path.realpath(os.path.join(baseDir, dep['relativePath']))
        if dPath in self.deps:
            return self.deps[dPath]
        depInfo = None
        depIKEnv = InfoKindEnv(path = dPath, dependencyLoader=infoKindEnv.dependencyLoader)
        self.deps[dPath] = depIKEnv
        with open(dPath, encoding="utf-8") as f:
            try:
                depInfo = json.load(f)
            except:
                logging.exception("Error while loading dependency %s" % f)
                raise
        if depInfo:
            depIKEnv.fromJsonList(depInfo, name = os.path.basename(dPath), source = { 'path': dPath }, dependencyLoad = False)
        return depIKEnv

class InfoKindEnv(object):
    """An environment keeping locally unique InfoKinds and their gids"""
    def __init__(self, infoKinds = None, name = None, description = None, newSha = compact_sha.sha512, gids = None,
            dependencyLoader = None, path = None, uri = None, deps = None):
        self.newSha = newSha
        self.clear()
        self.name = name
        self.description = description
        self.dependencyLoader = dependencyLoader
        if dependencyLoader is None:
            self.dependencyLoader = RelativeDependencySolver()
        self.path = path
        self.uri  = uri
        if not infoKinds is None:
            for ik in infoKinds:
                self.addInfoKindEl(ik)
        if not gids is None:
            self.gids = gids
        if deps:
            self.deps = deps

    def __str__(self):
        if self.path:
            return "InfoKindEnv loaded from {}".format(self.path)

    def clear(self):
        self.gids = {}
        self.infoKinds = {}
        self.deps = []

    def depNames(self):
        res = set()
        for dep in self.deps:
            for name in dep.infoKinds.keys():
                res.add(name)
        return res

    def noDepNames(self):
        return set(self.infoKinds.keys()).difference(self.depNames())

    def embedDeps(self):
        hidden = []
        duplicate = set()
        for dep in self.deps:
            for name, ikEl in dep.infoKinds.items():
                oldVal=self.infoKinds.get(name, None)
                if oldVal is None:
                    self.infoKinds[name] = ikEl
                elif ikEl != oldVal:
                    hidden.append(ikEl)
                else:
                    duplicate.add(name)
        return { "hidden": hidden, "duplicate": duplicate }

    def addInfoKindEl(self, infoKind):
        if infoKind.name in self.infoKinds and infoKind != self.infoKinds[infoKind.name]:
            raise Exception('InfoKindEnv has collision for name {0}: {1} vs {2}'
                    .format(infoKind.name, infoKind, self.infoKinds[infoKind.name]))
        self.infoKinds[infoKind.name] = infoKind

    def addDependenciesFrom(self, infoKindEnv):
        toAdd = set(self.infoKinds.keys())
        missing = set()
        while len(toAdd):
            ikName = toAdd.pop()
            ik = self.infoKinds.get(ikName,None)
            if ik is None:
                depInfoKindEl = infoKindEnv.infoKinds.get(ikName, None)
                if depInfoKindEl:
                    self.infoKinds[ikName] = depInfoKindEl
                    toAdd.add(ikName)
                else:
                    missing.add(ikName)
            else:
                for dep in ik.superNames:
                    if not dep in self.infoKinds:
                        toAdd.add(dep)
        return missing

    def gidOf(self, name, precalculatedGid=False):
        res = self.gids.get(name,None)
        if res is None:
            if precalculatedGid:
                raise Exception("non precalculated gid for %s" % name)
            res = self.calcGid(name)
        return res

    def calcGid(self, name):
        inProgress = []
        toDo = [name]
        hasPending = False
        for i in range(2):
            while len(toDo) > 0:
                if not hasPending and inProgress:
                    now = inProgress.pop()
                else:
                    now = toDo.pop()
                if now in self.gids and now in inProgress:
                    inProgress.remove(now)
                hasPending = False
                nowVal = self.infoKinds.get(now, None)
                if nowVal is None:
                    raise Exception("while calculating gid of %r found unknown key %r" % (name, now))
                for subName in nowVal.superNames:
                    if subName in self.gids:
                        continue
                    hasPending = True
                    if subName in toDo:
                        toDo.remove(subName)
                    if subName in inProgress:
                        raise Exception('found loop to %s evaluating %s, currently in progress: %s' % (subName, now, inProgress))
                    toDo.append(subName)
                if not hasPending:
                    self.gids[now] = nowVal.evalGid(self)
                    if now in inProgress:
                        inProgress.remove(now)
                else:
                    if now in inProgress:
                        raise Exception('found loop to %s, currently in progress: %s' % (now, inProgress))
                    inProgress.append(now)
            toDo = list(inProgress)
        return self.gids[name]

    def keyDependsOnKey(self, k1Name, k2Name):
        """partial ordering given by the dependencies
        1: k1Name depends on k2Name
        0: k1Name == k2Name
        -1: k2Name depends on k1Name
        None: no dependency"""
        if k1Name == k2Name: return 0
        k1 = self.infoKinds[k1Name]
        k2 = self.infoKinds[k2Name]
        if k1.superNames != k2.superNames:
            allSuperK1 = set()
            toDoK1 = list(k1.superNames)
            allSuperK2 = set()
            toDoK2 = list(k2.superNames)
            while (len(toDoK1) > 0 or len(toDoK2) > 0):
                if len(toDoK1) > 0:
                    el1Name = toDoK1.pop()
                    if k2Name == el1Name:
                        return 1
                    el1 = self.infoKinds[el1Name]
                    if el1.kindStr in self and not el1.kindStr in allSuperK1:
                        toDoK1.append(el1.kindStr)
                    for subEl in el1.superNames:
                        if not subEl in allSuperK1:
                            toDoK1.append(subEl)
                    allSuperK1.update(el1.superNames)
                if len(toDoK2) > 0:
                    el2Name = toDoK2.pop()
                    if k1Name == el2Name:
                        return -1
                    el2 = self.infoKinds[el2Name]
                    if el2.kindStr in self and not el2.kindStr in allSuperK2:
                        toDoK2.append(el2.kindStr)
                    for subEl in el2.superNames:
                        if not subEl in allSuperK2:
                            toDoK2.append(subEl)
                    allSuperK2.update(el2.superNames)
        return None

    def __contains__(self, name):
        "if an item with the given name is contained in this environment"
        return name in self.infoKinds

    def __len__(self):
        """returns the number of InfoKindEl stored in this environment"""
        return len(self.infoKinds)

    def __getitem__(self, name):
        """returns a dictionary representing the entry with the given name, or None if it does not exist"""
        ikEl = self.infoKinds.get(name, None)
        if ikEl:
            return ikEl.toDict(self)
        return None

    def infoKindEls(self):
        return list(self.infoKinds.values())

    def infoKindEl(self, name):
        """returns the InfoKindEl with the given name, or None if it does not exist"""
        return self.infoKinds.get(name, None)

    def calcGids(self):
        for k in self.infoKinds.keys():
            if not k in self.gids:
                self.gids[k]=self.calcGid(k)

    def serialize(self, writeOut, subGids = True, selfGid = True):
        infoKinds = self.sortedIKs()
        writeOut("""{
  "type": "nomad_meta_info_1_0",
  "description": """)
        if self.description:
            jsonIndentD(self.description, writeOut, extraIndent = 4)
        else:
            writeOut('""')
        writeOut(',\n')
        if not self.path:
            baseDir = os.getcwd()
        else:
            baseDir = os.path.normpath(os.path.dirname(os.path.abspath(self.path)))
        depKeys = set()
        if self.deps:
            writeOut('  "dependencies": [ ')
            depColon = False
            for d in self.deps:
                path = d.path
                uri = d.uri
                depKeys.update(d.infoKinds.keys())
                if path:
                    path = os.path.normpath(os.path.abspath(path))
                    if path.startswith(baseDir) or not uri:
                        if depColon:
                            writeOut(", ")
                        else:
                            depColon = True
                        jsonIndentD({"relativePath": os.path.relpath(path, baseDir)}, writeOut, extraIndent = 4)
                        continue
                if uri:
                    if depColon:
                        writeOut(", ")
                    else:
                        depColon = True
                    jsonIndentD({"uri": uri}, writeOut, extraIndent = 4)
                    continue
                raise Exception("Dependency on serializable %s" % d)
            writeOut('],\n')
        addColon = False
        writeOut('  "metaInfos": [ ')
        for ik in infoKinds:
            if ik.name in depKeys:
                continue
            if addColon:
                writeOut(", ")
            else:
                addColon = True
            jsonIndentD(ik.toDict(env = self, subGids = subGids, selfGid = selfGid), writeOut, extraIndent = 4, check_circular = True)
        writeOut("]\n}\n")

    def sortedIKs(self):
        infoKinds = list(self.infoKinds.values())
        infoKinds.sort(lambda x, y: cmp(x.name.lower()+x.name, y.name.lower()+y.name))
        return infoKinds # self.sortAndComplete(infoKinds, ignoreMissing = True)

    def toJsonList(self, withGids):
        infoKinds = list(self.infoKinds.keys())
        infoKinds.sort(lambda x, y: self.compareKeys(x.name, y.name))
        return [self.infoKinds[x].toDict(self,
                self if withGids else None) for x in infoKinds]

    def verifyGids(self, preserveAbsent=False):
        changes = {}
        oldGids = self.gids
        self.gids = {}
        self.calcGids()
        for k,v in oldGids.items():
            newVal = self.gids.get(k, None)
            if newVal is None:
                if preserveAbsent:
                    self.gids[k] = v
                else:
                    changes[k] = (v, None)
            elif v != newVal:
                changes[k] = (v, newVal)
        return changes

    def fromJsonList(self, jsonDict, name,  source, extraArgsHandling = InfoKindEl.ADD_EXTRA_ARGS, dependencyLoad=False):
        typeStr = jsonDict.get("type","nomad_meta_info_1_0")
        typeRe = re.compile(r"nomad_meta_info_(?P<major>[0-9]+)_(?P<minor>[0-9]+)$")
        self.name = name
        m = typeRe.match(typeStr)
        if not m:
            raise Exception("unexpected type '%s', expected nomad_meta_info_1_0" % typeStr)
        if int(m.group("major")) != 1:
            raise Exception("Unsupported major version %s, expeced 1")
        dependencies = jsonDict.get("dependencies",[])
        jsonList = jsonDict.get("metaInfos",[])
        self.description = jsonDict.get("description","")
        overwritten = []
        gidToCheck = {}
        deps = []
        for d in dependencies:
            if self.dependencyLoader is None:
                raise Exception("no dependencyLoader while loading local_in")
            dep = self.dependencyLoader(self, source, d)
            if dep:
                self.deps.append(dep)
        index = -1
        for ii in jsonList:
            index += 1
            val = dict(ii)
            if not "name" in ii:
                raise Exception("InfoKind at %d is without name: %s" % (index, ii) )
            oldVal=self.infoKinds.get(ii['name'],None)
            gid=None
            if "gid" in ii:
                gid = ii['gid']
                del val['gid']
            if "superGids" in ii:
                if not "superNames" in ii:
                    raise Exception("superGids without superNames in fromJsonList")
                superNames = ii["superNames"]
                superGids = ii["superGids"]
                if len(superNames) != len(superGids):
                    raise Exception("superGids incompatible with superNames in fromJsonList: %s vs %s" % (ii["superGids"], ii["superNames"]))
                toCheck = {}
                for i in range(len(superNames)):
                    assert not superNames[i] in toCheck.keys(), "duplicate superName %r in %r" % (superNames[i], ii["name"])
                    toCheck[superNames[i]] = superGids[i]
                gidToCheck[ii["name"]] = toCheck
                del val['superGids']
            val['extraArgsHandling'] = extraArgsHandling
            ikEl = InfoKindEl(**val)
            if not oldVal is None and ikEl != oldVal:
                overwritten.append((oldVal, ikEl))
            if gid:
                self.gids[ii['name']] = gid
            self.infoKinds[ikEl.name] = ikEl
        res = { "overwritten": overwritten }
        if not dependencyLoad:
            res.update(self.embedDeps())
            for childName, gids in gidToCheck.items():
                for name, gid in gids.items():
                    if self.gidOf(name) != gid:
                        raise Exception("incompatible superGid for superName %s of %s (%s vs %s)" % (name, ii["name"], gid, self.gidOf(name)))
        if res.get("overwritten", False) or res.get("duplicate", False) or res.get("hidden", False):
            res["hasWarnings"] = True
        else:
            res["hasWarnings"] = res.get("hasWarnings", False)
        return res

    def sortAndComplete(self, propsToSort, ignoreMissing = False):
        """builds a list of properties in propsToSort, so that all the dependecies of each
        property are present before them"""
        toDo = list(propsToSort)
        done = set()
        deps = []
        res = []
        while len(toDo)>0:
            pAtt = toDo.pop()
            nameAtt = pAtt.name
            if nameAtt in done:
                continue
            deps = [nameAtt]
            while len(deps)>0:
                nameAtt = deps[-1]
                pAtt = self.infoKinds.get(nameAtt, None)
                if pAtt is None:
                    if ignoreMissing:
                        deps.pop()
                        done.add(nameAtt)
                        continue
                    else:
                        raise Exception("missing dependent InfoKindEl {0} following chain {1}".format(nameAtt, pAtt))
                hasDepsToDo = False
                kindStr = pAtt.kindStr
                kindType = self.infoKindEl(kindStr)
                for superName in pAtt.superNames:
                    if not superName in done:
                        if superName in deps:
                            raise Exception("circular dependency {0}, {1}".format(deps,superName))
                        deps.append(superName)
                        hasDepsToDo = True
                if kindType and not kindStr in done:
                    if kindStr in deps:
                        raise Exception("circular dependency in kindStr {0}, {1}".format(deps,kindStr))
                    deps.append(kindStr)
                    hasDepsToDo = True
                if not hasDepsToDo:
                    deps.pop()
                    res.append(pAtt)
                    done.add(nameAtt)
        return res

    def metaInfoNameWithAllSuper(self, name):
        """returns the meta info names of name and all its dependencies"""
        toAdd = set([name])
        res = set([name])
        while toAdd:
            e = toAdd.pop()
            for superName in self.infoKinds[e].superNames:
                if not superName in res:
                    res.add(superName)
                    toAdd.add(superName)
        return res

    def firstAncestorsByType(self, name):
        """Returns the first acestors of each type separated in roots and children.
        (scala conversion, could be improved a bit)"""
        metaInfoNames = self.metaInfoNameWithAllSuper(name)
        metaInfoNames.remove(name)
        mInfo = list(metaInfoNames)
        edges = {}
        for i, metaName in enumerate(mInfo):
            metaInfo = self.infoKinds[metaName]
            edges[i] = [mInfo.index(x) for x in metaInfo.superNames]
        typeGroups = {}
        for mIdx, metaName in enumerate(mInfo):
            kindStr = self.infoKinds[metaName].kindStr
            tNow = typeGroups.get(kindStr, None)
            if tNow is None:
                typeGroups[kindStr] = [mIdx]
            else:
                tNow.append(mIdx)
        childsByType = {}
        toDo  = set(range(len(mInfo)))
        while (toDo):
            now = toDo.pop()
            kindNow = self.infoKinds[mInfo[now]].kindStr
            toDo2 = set(edges[now])
            known2 = set(edges[now])
            while (toDo2):
                now2 = toDo2.pop()
                if (self.infoKinds[mInfo[now2]].kindStr == kindNow):
                    childs = childsByType.get(kindNow, None)
                    if childs:
                        childs.add(now2)
                    else:
                        childsByType[kindNow] = set([now2])
                    if now2 in toDo:
                        toDo.remove(now2)
                for el in edges[now2]:
                    if not el in known2:
                        toDo2.add(el)
                        known2.add(el)
        res = {}
        for typeName, allChilds in typeGroups.items():
            childs = childsByType.get(typeName, set())
            allForKind = set(allChilds)
            rootNames = [mInfo[x] for x in (allForKind - childs)]
            childNames = [mInfo[x] for x in childs]
            res[typeName] = (rootNames, childNames)
        return res

def loadJsonFile(filePath, dependencyLoader = None, extraArgsHandling = InfoKindEl.ADD_EXTRA_ARGS, uri = None):
    env = InfoKindEnv(dependencyLoader = dependencyLoader, path = filePath, uri = uri)
    try:
        with open(filePath, encoding="utf-8") as f:
            o = json.load(f)
            warnings = env.fromJsonList(o, name = os.path.basename(filePath), source = {'path': filePath}, extraArgsHandling = extraArgsHandling)
    except:
        logging.exception("Error while loading file %s" % filePath)
        raise
    return env, warnings

def load_metainfo(filename, dependencyLoader=None, extraArgsHandling=InfoKindEl.ADD_EXTRA_ARGS, uri=None):
    """Loads a metainfo environment for a filename. The filename should not
    contain the full path, as the full path is resolved here and not by the
    caller.

    Args:
        filename: filename as a string.

    Returns:
        Tuple containing the metainfo environment, and any possible warnings
        that were encountered in the loading.
    """
    path = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../../nomad-meta-info/meta_info/nomad_meta_info/{}".format(filename)))
    return loadJsonFile(path, dependencyLoader, extraArgsHandling, uri)

def loadJsonStream(fileStream, name = None, dependencyLoader = None, extraArgsHandling = InfoKindEl.ADD_EXTRA_ARGS, filePath = None, uri = None):
    if filePath is None:
        try:
            filePath = fileStream.name
        except:
            filePath = None
    if name is None and not filePath is None:
        name = os.path.basename(filePath)
    env = InfoKindEnv(dependencyLoader = dependencyLoader, name = name, path = filePath, uri = uri)
    try:
        o = json.load(fileStream)
        warnings = env.fromJsonList(o, source = {'path': filePath}, extraArgsHandling = extraArgsHandling)
    except:
        logging.exception("Error while loading file %s" % filePath)
        raise
    return env, warnings


'''
Created on 9 Apr 2019

@author: qubix
'''

from typing import Tuple

import numpy as np
from sklearn.base import BaseEstimator

from modAL.utils.data import modALinput
from math import floor

from asreview.query_strategies.max_sampling import max_sampling
from asreview.query_strategies.random_sampling import random_sampling

def rand_max_sampling(classifier: BaseEstimator,
                      X: modALinput,
                      pool_idx=None,
                      n_instances: int = 1,
                      query_kwargs={},
                      **kwargs
                      ) -> Tuple[np.ndarray, modALinput]:
    """
    Combination of random and maximum sampling.
    By default samples the 95% of the instances with max sampling,
    and 5% of the samples with random sampling.

    Parameters
    ----------
    classifier: BaseEstimator
        The classifier for which the labels are to be queried.
    X: modALinput
        The whole input matrix.
    pool_idx: np.array
        Indices of samples that are in the pool.
    n_instances: int
        Total number of samples to be queried.
    extra_vars: dict
        dictionary to pass through settings (such as the max/rand ratio),
        as well as the indices that were obtained using max & random sampling.
    **kwargs:
        Keyword arguments to be passed on to random/max sampling.

    Returns
    -------
    np.ndarray, modALinput
        The indices of the instances from X chosen to be labelled;
        the instances from X chosen to be labelled.
    """

    n_samples = X.shape[0]
    if pool_idx is None:
        pool_idx = np.arange(n_samples)

    # Set the fraction of maximum sampling. Defaults to 95% max, 5% rand.
    rand_max_frac = query_kwargs.get('rand_max_frac', 0.05)
    max_frac = 1-rand_max_frac

    # Get the discrete number of instances for rand/max sampling.
    n_instance_max = floor(n_instances*max_frac)
    if np.random.random_sample() < n_instances*max_frac-n_instance_max:
        n_instance_max += 1
    n_instance_rand = n_instances-n_instance_max

    # Do max sampling.
    max_idx, _ = max_sampling(classifier, X, pool_idx=pool_idx,
                              n_instances=n_instance_max,
                              query_kwargs=query_kwargs,
                              **kwargs)

    # Remove indices found with max sampling from the pool.
    query_idx = np.delete(np.arange(n_samples), pool_idx, axis=0)
    query_idx = np.append(query_idx, max_idx)
    new_pool_idx = np.delete(np.arange(n_samples), query_idx, axis=0)

    # Random sampling.
    rand_idx, _ = random_sampling(classifier, X, pool_idx=new_pool_idx,
                                  n_instances=n_instance_rand,
                                  query_kwargs=query_kwargs,
                                  **kwargs)

    if "max" not in query_kwargs['src_query_idx']:
        query_kwargs["src_query_idx"]["max"] = np.array(max_idx, dtype=np.int)
    else:
        query_kwargs["src_query_idx"]["max"] = np.append(
            query_kwargs["src_query_idx"]["max"], max_idx)

    if "random" not in query_kwargs['src_query_idx']:
        query_kwargs["src_query_idx"]["random"] = np.array(
            rand_idx, dtype=np.int)
    else:
        query_kwargs["src_query_idx"]["random"] = np.append(
            query_kwargs["src_query_idx"]["random"], rand_idx)

    query_kwargs['rand_max_frac'] = rand_max_frac
    query_kwargs['last_bounds'] = [
        ("max", 0, n_instance_max),
        ("random", n_instance_max, n_instances),
    ]
    query_idx = np.append(max_idx, rand_idx)

    return query_idx, X[query_idx]


# Packages
import pandas as pd
import os
import json
from gensim.utils import simple_preprocess
from gensim.summarization.textcleaner import split_sentences
from functools import reduce
from fuzzywuzzy import fuzz

## Functions
## Returns marked html from iucn notes
def find_country(text, country):
	'''Function to id country names in iucn notes and insert mark tags for
	highlighting in html'''

	# # Split text into individual words
	txt_ls = text.split(" ")
	q_ls = country.split(" ")

	# given length of country
	q_len = len(q_ls)

	interest = [0]*len(txt_ls)

	# check each subset of n words for matches
	for i in range(len(txt_ls)-q_len+1):
		tmp_txt = (" ").join(txt_ls[i:i+q_len])

		if fuzz.token_set_ratio(tmp_txt, country)>=90:
			interest[i:i+q_len] = [1]*q_len

	# use index list to find words to highlight
	for w in range(len(txt_ls)):
		if interest[w] == 1:
			txt_ls[w] = ""+txt_ls[w]+""

	recomb_html = " ".join(txt_ls)
	# If consecutive words highlighted, rm end and start
	recomb_html = recomb_html.replace(" ", " ")

	# for t in range(len(word_ls)):
	# 	# Match word against country
	# 	pr = fuzz.token_set_ratio(word_ls[t], country)
	# 	# If match is good, add html marks...
	# 	if pr>90:
	# 		# print (word_ls[t])
	# 		word_ls[t] = ""+word_ls[t]+""

	# Split text into sentences within paragraphs
	# split_txt = [split_sentences(para) for para in text.split("\n") if len(split_sentences(para))>0]
	# # interest_idx = [[0] * len(inner) for inner in split_txt]

	# for p in range(len(split_txt)):
	# 	for s in range(len(split_txt[p])):
	# 		# for each sentence fuzzy match against country
	# 		pr = fuzz.token_set_ratio(split_txt[p][s], country)
	# 		# If match is good, indicate in interest list or add marks...
	# 		if pr>90:
	# 			# interest_idx[p][s] += 1
	# 			# Add "" to start and "" end of sentence?
	# 			split_txt[p][s] = ""+split_txt[p][s]+""

	# recomb_html = "\n".join([" ".join(inner) for inner in split_txt])
	# recomb_html = " ".join(word_ls)
	return(recomb_html)

# Extracts data from dictionary level given a list of indices to that level
def get_from_dict(dataDict, pathlist):
	"""Iterate nested dictionary"""
	return reduce(dict.get, pathlist, dataDict)

## Constants
notes_paths = {"taxonomic_notes" : ["taxonomy", "taxonomic_notes", "value"],
				"red_list_notes" : ["iucn_status", "red_list_notes", "value"],
				"range_notes" : ["habitat", "range_notes", "value"],
				"population_notes" : ["population", "population_notes", "value"],
				"use_trate_notes" : ["trade", "use_trade_notes", "value"],
				"conservation_notes" : ["conservation", "conservation_notes", "value"],
				"threats_notes" : ["threats", "threats_notes", "value"]}

## Main code
# Load cites df for relevant countries
cites_df = pd.read_csv("../Data/CitesParrots.csv")
cites_country_code = list(set(list(cites_df["Importer"])+(list(cites_df["Exporter"]))))

# Load country list data data
country_df = pd.read_csv("../Data/countries.csv")
# Subset to countries of interest
country_df = country_df.loc[country_df["Code"].isin(cites_country_code)]
# Create a simpler, single word country name
country_df["Basic"] = [country.split("(")[0].split(",")[0] for country in country_df["Name"]]

# List all json files
dat_dir = "../Data/parrot_data/"
f_ls = os.listdir(dat_dir)
# Calc no of rows need in output df
n_row = len(f_ls) * country_df.shape[0]

out_df = pd.DataFrame({"SpeciesID" : ["NA"]*n_row,
						"Country" : ["NA"]*n_row,
							"taxonomic_notes" : ["NA"]*n_row,
							"red_list_notes" : ["NA"]*n_row,
							"range_notes" : ["NA"]*n_row,
							"population_notes" : ["NA"]*n_row,
							"use_trate_notes" : ["NA"]*n_row,
							"conservation_notes" : ["NA"]*n_row,
							"threats_notes" : ["NA"]*n_row})

row_count = 0
for f in f_ls:
	# Load json
	with open(dat_dir+f) as json_file:
		parrot_dat = json.load(json_file)
	parrot = f.split(".")[0]

	# Is IUCN data there?
	if len(parrot_dat["iucn"])>0:
		iucn_dat = parrot_dat["iucn"]
		for country in country_df["Basic"]:
			for key in notes_paths.keys():
				# Obtain data
				tmp_dat = get_from_dict(iucn_dat, notes_paths[key])
				# If not "NA" or "value" add to notes dict
				if ((tmp_dat != "NA") & (tmp_dat != "value")):
					out_df.iloc[row_count][key] = find_country(tmp_dat, country)
			out_df.iloc[row_count]["SpeciesID"] = parrot
			out_df.iloc[row_count]["Country"] = country
			row_count +=1

# print(row_count)
out_df = out_df.loc[0:row_count-1]
# out_df.to_csv("../../Local_Code/Data/marked_text.csv")

idx = int((row_count-1)/3.0)

out_df1 = out_df.loc[0:idx]
out_df2 = out_df.loc[idx+1:2*idx]
out_df3 = out_df.loc[2*idx+1:row_count-1]

out_df1.to_csv("../Data/parrot_csv/marked_text1.csv")
out_df2.to_csv("../Data/parrot_csv/marked_text2.csv")
out_df3.to_csv("../Data/parrot_csv/marked_text3.csv")


'''
Dependencies: gettext, playsound
installing
$ pip install gTTS pyttsx3 playsound soundfile transformers datasets sentencepiece
$ pip install playsound (may need to use "$ pip install --upgrade wheel" if install fails)
'''
import gtts
from playsound import playsound

with open("sample.ini") as fileDescriptor:
     data = fileDescriptor.read()

tts = gtts.gTTS(data)
tts.save("audioReader.mp3")
playsound("audioReader.mp3")


from pyowm import OWM
from pyowm.utils import config
from pyowm.utils import timestamps
from config import owm_key

owm = OWM(owm_key)
mgr = owm.weather_manager()

# info on looking up cities.
#To make it more precise put the city's name, comma, 2-letter country code (ISO3166). You will get all proper cities in chosen country.
#The order is important - the first is city name then comma then country. Example - London, GB or New York, US.

city = 'Leesburg,US'
short_city = city.split(",", 1)[0]

# Creating an empty "database" or dictionary.
# I'm using this to train myself in dictionaries and how to read them.
database = {}

# Searching location for data on owm
location = mgr.weather_at_place(city)
w = location.weather

# Create key + value
database['wind'] = w.wind()
database['temp'] = w.temperature('fahrenheit')

# printing out the default looking dictionary
print(database)

# Print out the status of the dictionary
print(f"Wind Speed for {city} : {database['wind']['speed']}")

# Printing out the temp instead
print(f"Temp for {short_city} = {database['temp']['temp']}")
print(f"The high for today in {short_city} is, {database['temp']['temp_max']}")