import os
import re
import torch
import shutil

from enum import Enum
from collections import Counter
from safetensors.torch import load_file
from diffusers import StableDiffusionPipeline, AutoencoderKL

from .convert_vae_pt_to_diffusers import vae_pt_to_vae_diffuser
from .diffusion_error import ModelFormatError, ModelIdentificationError, ModelLoadError

class LoadModelType(Enum):
    SD_model_Diffusers      = 'Stable Diffusion Model (Diffusers Model)'
    SD_model_Safetensors    = 'Stable Diffusion Model (Safetensors Model)'
    SD_model_Ckpt           = 'Stable Diffusion Model (CKPT Model)'
    VAE_model_Diffusers     = 'Stable Diffusion Vae (Diffusers Model)'
    VAE_model_Pt            = 'Stable Diffusion Vae (PT Model)'
    LORA_model_Safetensors  = 'Stable Diffusion Lora (Safetensors Model)'
    HuggingFace             = 'Huggingface Repository Path'

class ModelType(Enum):
    SD_model = 'Stable Diffusion Model'
    VAE_model = 'Stable Diffusion Vae'
    LORA_model = 'Stable Diffusion Lora'

class BaseLoader():
    def __init__(self, dtype: torch.dtype = torch.float16, cache: str = './cache') -> None:
        self.dtype = dtype
        self.cache = cache
        self.basename = None
        self.basetype = None
        os.makedirs(cache, exist_ok=True)

    def determine_model_from_file(self, model_path: str) -> LoadModelType:
        def __logger(lmt: LoadModelType) -> LoadModelType:
            print(f'The model loaded was \"{lmt.value}\".')
            return lmt
        print('Determining model.')
        if os.path.isfile(model_path):
            _, ext = os.path.splitext(model_path)
            if ext == '.safetensors':
                dict = load_file(model_path)
                if any('diffusion_model' in key for key in dict.keys()):
                    return __logger(LoadModelType.SD_model_Safetensors)
                elif any('lora_unet' in key for key in dict.keys()):
                    return __logger(LoadModelType.LORA_model_Safetensors)
            elif ext == '.ckpt':
                return __logger(LoadModelType.SD_model_Ckpt)
            elif ext == '.pt':
                return __logger(LoadModelType.VAE_model_Pt)
        elif os.path.isdir(model_path):
            vlist = ['config.json', 'diffusion_pytorch_model.safetensors']
            flist = os.listdir(model_path)
            if Counter(flist) == Counter(vlist):
                return __logger(LoadModelType.VAE_model_Diffusers)
            else:
                return __logger(LoadModelType.SD_model_Diffusers)
        elif re.fullmatch(r'^([^/]*?)/([^/]*?)$', model_path):
            return __logger(LoadModelType.HuggingFace)
        raise ValueError('The specified argument is malformed.')

    def remove_model(self, model_path:str) -> (str, ModelType):
        model_type = self.determine_model_from_file(model_path)

        if model_type != LoadModelType.HuggingFace:
            cache_path = os.path.abspath(self.cache)
            target_path = os.path.abspath(model_path)
            if not os.path.commonprefix([target_path, cache_path]) == cache_path:
                raise ValueError('The specified path does not exist in the cache directory.')

        if model_type in [LoadModelType.SD_model_Diffusers, LoadModelType.VAE_model_Diffusers]:
            shutil.rmtree(model_path)
        elif model_type in [LoadModelType.SD_model_Safetensors,
                            LoadModelType.LORA_model_Safetensors,
                            LoadModelType.SD_model_Ckpt,
                            LoadModelType.VAE_model_Pt]:
            os.remove(model_path)
        elif model_type == LoadModelType.HuggingFace:
            names = model_path.split('/')
            shutil.rmtree(os.path.join(self.cache, f'models--{names[0]}--{names[1]}'))
        else:
            raise ModelIdentificationError('Could not identify type of model to delete.')
        return model_path

    def set_model(self, model_path: str) -> (str, ModelType):
        pass

    def get_path(self):
        return self.basename

    def get_type(self):
        return self.basetype

class DiffusionModelLoader(BaseLoader):
    def __model_cacher(self, model_path: str) -> str:
        path = os.path.join(self.cache, os.path.basename(model_path) + '_from_file')
        pipe = StableDiffusionPipeline.from_single_file(
            pretrained_model_link_or_path=model_path,
            torch_dtype=self.dtype,
            cache_dir=self.cache
        )
        pipe.save_pretrained(
            save_directory=path,
            safe_serialization=True
        )
        return path

    def __model_check_and_buffer(self, model_path: str) -> bool:
        try:
            pipe = StableDiffusionPipeline.from_pretrained(
                pretrained_model_name_or_path=model_path,
                torch_dtype=self.dtype,
                cache_dir=self.cache)
        except:
            return False
        return True

    def set_model(self, model_path: str) -> (str, ModelType):
        modelType = super().determine_model_from_file(model_path)
        if not modelType in [LoadModelType.SD_model_Ckpt,
                             LoadModelType.SD_model_Safetensors,
                             LoadModelType.SD_model_Diffusers,
                             LoadModelType.HuggingFace]:
            raise ModelFormatError('The specified model is not a Diffusion model.')

        if modelType in [LoadModelType.SD_model_Safetensors,
                         LoadModelType.SD_model_Ckpt]:
            print('Creating cache from file.')
            self.basename = self.__model_cacher(model_path)
        elif modelType == LoadModelType.SD_model_Diffusers:
            print('Creating cache from Diffusers model.')
            if not self.__model_check_and_buffer(model_path):
                raise ModelLoadError('The specified path could not be loaded as a Diffusion model.')
            path = os.path.join(self.cache, os.path.basename(model_path))
            shutil.copytree(model_path, path)
            self.basename = path
        elif modelType == LoadModelType.HuggingFace:
            print('Creating cache from HuggingFace.')
            if not self.__model_check_and_buffer(model_path):
                raise ModelLoadError('Could not successfully retrieve model from HuggingFace.')
            self.basename = model_path

        self.basetype = ModelType.SD_model
        return self.basename, self.basetype

class VaeModelLoader(BaseLoader):
    def __model_checker(self, model_path: str) -> bool:
        try:
            pipe = AutoencoderKL.from_pretrained(
                pretrained_model_name_or_path=model_path,
                torch_dtype=self.dtype,
                cache_dir=self.cache)
        except:
            return False
        return True

    def set_model(self, model_path: str) -> (str, ModelType):
        modelType = super().determine_model_from_file(model_path)
        if not modelType in [LoadModelType.VAE_model_Diffusers,
                             LoadModelType.VAE_model_Pt]:
            raise ModelFormatError('The specified model is not a Vae model.')

        if modelType == LoadModelType.VAE_model_Diffusers:
            print('Creating cache from Diffusers model.')
            if not self.__model_checker(model_path):
                raise ModelLoadError('The specified path could not be loaded as a Vae model.')
            path = os.path.join(self.cache, os.path.basename(model_path))
            shutil.copytree(model_path, path)
            self.basename = path
        elif modelType == LoadModelType.VAE_model_Pt:
            print('Creating cache from pt model.')
            path = os.path.join(self.cache, os.path.basename(model_path) + '_from_file')
            vae_pt_to_vae_diffuser(model_path, path)
            self.basename = path
        self.basetype = ModelType.VAE_model
        return self.basename, self.basetype

class LoraModelLoader(BaseLoader):
    def set_model(self, model_path: str) -> (str, ModelType):
        modelType = super().determine_model_from_file(model_path)
        if modelType != LoadModelType.LORA_model_Safetensors:
            raise ModelFormatError('The specified model is not a Lora model.')
        print('Copying file to cache.')
        path = os.path.join(self.cache, os.path.basename(model_path))
        shutil.copy(model_path, path)
        self.basename = path
        self.basetype = ModelType.LORA_model
        return self.basename, self.basetype

class AutoModelLoader(BaseLoader):
    def __model_type_convert(self, model_path: str):
        def __logger(mt: ModelType) -> ModelType:
            print(f'The model was classified as \"{mt.value}\".')
            return mt
        modelType = super().determine_model_from_file(model_path)
        print('Broadly classify model types.')
        if modelType in [LoadModelType.SD_model_Ckpt,
                             LoadModelType.SD_model_Safetensors,
                             LoadModelType.SD_model_Diffusers,
                             LoadModelType.HuggingFace]:
            return __logger(ModelType.SD_model)
        elif modelType in [LoadModelType.VAE_model_Diffusers,
                             LoadModelType.VAE_model_Pt]:
            return __logger(ModelType.VAE_model)
        elif modelType == LoadModelType.LORA_model_Safetensors:
            return __logger(ModelType.LORA_model)
        raise ModelFormatError('The specified model could not be determined.')

    def set_model(self, model_path: str) -> (str, ModelType):
        model_type = self.__model_type_convert(model_path)
        print('Loading by model')
        if model_type == ModelType.SD_model:
            self.basename, self.basetype = DiffusionModelLoader(self.dtype, self.cache).set_model(model_path)
        elif model_type == ModelType.VAE_model:
            self.basename, self.basetype = VaeModelLoader(self.dtype, self.cache).set_model(model_path)
        elif model_type == ModelType.LORA_model:
            self.basename, self.basetype = LoraModelLoader(self.dtype, self.cache).set_model(model_path)
        else:
            raise ModelLoadError('Could not load model.')
        print(f'Model loading is complete. It is saved in \"{self.basename}\". The model type is \"{self.basetype.value}\"')
        return self.basename, self.basetype


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jun  7 14:41:30 2019

@author: nk7g14
"""
import pandas as pd
import fluxquery.auxil as aux
import os
import requests

import logging
import re
from astropy.io import fits
import numpy as np
from pathlib import Path
import glob

logging.basicConfig(level=logging.DEBUG, format=' %(asctime)s -- %(message)s')
class RXTE:
    def __init__(self):
        super(RXTE, self).__init__()
        self.RXTE_OBS_LIST = aux.GetObservationList(self.SOURCE_NAME, 'xtemaster')

    def _RXTE_DownloadObservation(self, obsID):
        print('Downloading RXTE observation:', obsID)
        filepath = '/sources/{}/rxte/{}.tar'.format(self.SOURCE_NAME, obsID)
        #RXTE observation IDS have a structure like XXXXX-XX-XX-XX
        #and are saved in folders on the archive with the first bit XXXXX with
        #a P in front of it.
        first_bit = obsID.split('-')[0]
        filepath_folder = 'P' + first_bit #Extracted OBSid folder

        tar_exists = Path(filepath).is_file()
        folder_exists = Path(filepath_folder).is_file()

        if tar_exists or folder_exists:
            print('Folder or tar file already exists, not downloading.')
        else:
            url = 'http://heasarc.gsfc.nasa.gov/cgi-bin/W3Browse/xteTar.pl?obsid={}&prnb={}'.format(obsID, first_bit)
            myfile = requests.get(url, allow_redirects=True)
            cwd = os.getcwd()
            open(cwd + filepath, 'wb').write(myfile.content)

    def RXTE_DownloadAllObservations(self):
        for obsID in self.RXTE_OBS_LIST['OBSID']:
            self._RXTE_DownloadObservation(obsID)

    def RXTE_GetCountsVpXPcu(self, xfl_file):
        df = pd.DataFrame()
        time = np.array(xfl_file[1].data['Time'], dtype=float)
        count1 = np.array(xfl_file[1].data['VpX1LCntPcu0'], dtype=float)
        count2 = np.array(xfl_file[1].data['VpX1RCntPcu0'], dtype=float)
        count3 = np.array(xfl_file[1].data['VpX1LCntPcu1'], dtype=float)
        count4 = np.array(xfl_file[1].data['VpX1RCntPcu1'], dtype=float)
        count5 = np.array(xfl_file[1].data['VpX1LCntPcu2'], dtype=float)
        count6 = np.array(xfl_file[1].data['VpX1RCntPcu2'], dtype=float)
        count7 = np.array(xfl_file[1].data['VpX1LCntPcu3'], dtype=float)
        count8 = np.array(xfl_file[1].data['VpX1RCntPcu3'], dtype=float)
        count9 = np.array(xfl_file[1].data['VpX1LCntPcu4'], dtype=float)
        count10 = np.array(xfl_file[1].data['VpX1RCntPcu4'], dtype=float)

        df['Time'] = time
        df['VpX1LCntPcu0'] = count1
        df['VpX1RCntPcu0'] = count2
        df['VpX1LCntPcu1'] = count3
        df['VpX1RCntPcu1'] = count4
        df['VpX1LCntPcu2'] = count5
        df['VpX1RCntPcu2'] = count6
        df['VpX1LCntPcu3'] = count7
        df['VpX1RCntPcu3'] = count8
        df['VpX1LCntPcu4'] = count9
        df['VpX1RCntPcu4'] = count10
        return df

    def RXTE_GetCountsQ6VPcu(self, xfl_file):
        df = pd.DataFrame()
        time = np.array(xfl_file[1].data['Time'], dtype=float)
        count1 = np.array(xfl_file[1].data['Q6VxVpXeCntPcu0'], dtype=float)
        count2 = np.array(xfl_file[1].data['Q6VxVpXeCntPcu1'], dtype=float)
        count3 = np.array(xfl_file[1].data['Q6VxVpXeCntPcu2'], dtype=float)
        count4 = np.array(xfl_file[1].data['Q6VxVpXeCntPcu3'], dtype=float)
        count5 = np.array(xfl_file[1].data['Q6VxVpXeCntPcu4'], dtype=float)

        df['Time'] = time
        df['Q6VxVpXeCntPcu0'] = count1
        df['Q6VxVpXeCntPcu1'] = count2
        df['Q6VxVpXeCntPcu2'] = count3
        df['Q6VxVpXeCntPcu3'] = count4
        df['Q6VxVpXeCntPcu4'] = count5
        return df

    def RXTE_GetCountsXPcu(self, xfl_file):
        df = pd.DataFrame()
        time = np.array(xfl_file[1].data['Time'], dtype=float)
        count1 = np.array(xfl_file[1].data['X1LX2LCntPcu0'], dtype=float)
        count2 = np.array(xfl_file[1].data['X1RX2RCntPcu0'], dtype=float)
        count3 = np.array(xfl_file[1].data['X2LX2RCntPcu0'], dtype=float)
        count4 = np.array(xfl_file[1].data['X2LX3LCntPcu0'], dtype=float)
        count5 = np.array(xfl_file[1].data['X2RX3RCntPcu0'], dtype=float)
        count6 = np.array(xfl_file[1].data['X3LX3RCntPcu0'], dtype=float)

        df['Time'] = time
        df['X1LX2LCntPcu0'] = count1
        df['X1RX2RCntPcu0'] = count2
        df['X2LX2RCntPcu0'] = count3
        df['X2LX3LCntPcu0'] = count4
        df['X2RX3RCntPcu0'] = count5
        df['X3LX3RCntPcu0'] = count6
        return df

    def RXTE_GetcountsGood(self, xfl_file):
        df = pd.DataFrame()
        time = np.array(xfl_file[1].data['Time'])
        count_good_1 = np.array(xfl_file[1].data['evXEgood_PCU0'], dtype=float)
        count_good_2 = np.array(xfl_file[1].data['evXEgood_PCU1'], dtype=float)
        count_good_3 = np.array(xfl_file[1].data['evXEgood_PCU2'], dtype=float)
        count_good_4 = np.array(xfl_file[1].data['evXEgood_PCU3'], dtype=float)
        count_good_5 = np.array(xfl_file[1].data['evXEgood_PCU4'], dtype=float)

        df['Time'] = time
        df['evXEgood_PCU0'] = count_good_1
        df['evXEgood_PCU1'] = count_good_2
        df['evXEgood_PCU2'] = count_good_3
        df['evXEgood_PCU3'] = count_good_4
        df['evXEgood_PCU4'] = count_good_5
        return df

    def RXTE_GetCounts(self, xfl_file):
        #Tempoarily removed good counts as for some observations don't have em
        countsVpX = self.RXTE_GetCountsVpXPcu(xfl_file)
        counts6VP = self.RXTE_GetCountsQ6VPcu(xfl_file)
        countsXP = self.RXTE_GetCountsXPcu(xfl_file)
        #counts_good = self.RXTE_GetcountsGood(xfl_file)

        #mapping = [countsVpX, counts6VP, countsXP, counts_good]
        mapping = [countsVpX, counts6VP, countsXP]
        df = pd.concat(mapping, axis=1)
        df = df.loc[:, ~df.columns.duplicated()] #Drop duplicate time columns
        return df

    def RXTE_GetAllCounts(self):
        '''
        Runs through all stdprod folders and obtains the flux for each obsID
        returns a dictionary containing all the flux dataframes
        '''
        walk = os.walk('sources/{}/rxte'.format(self.SOURCE_NAME))

        obsIDregex = re.compile(r'\d{5}-\d{2}-\d{2}-\d{2}')
        df_dict = {}

        gz_files = glob.glob('sources/{}/rxte/*/*/stdprod/*.xfl.gz'.format(self.SOURCE_NAME))
        xfl_files = glob.glob('sources/{}/rxte/*/*/stdprod/*.xfl'.format(self.SOURCE_NAME))

        len(gz_files)
        len(xfl_files)

        for walk_iter in walk:
            if 'stdprod' in walk_iter[0]:
                result = obsIDregex.search(walk_iter[0])
                obsID = result.group()
                logging.debug('Getting counts rxte %s', obsID)
                aux.UnzipAllgzFiles(walk_iter[0])
                xfl_search = glob.glob(walk_iter[0] + '/*.xfl')[0]
                xfl_file = fits.open(xfl_search)
                count_df = self.RXTE_GetCounts(xfl_file)
                df_dict[obsID] = count_df
        return df_dict

    def RXTE_MergeDataframeDictionary(self, df_dict):
        df = pd.concat(df_dict.values(), ignore_index=True)
        df = df.sort_values(by=['Time'])
        df = df.reset_index(drop=True)
        return df

    def RXTE_GetLightcurve(self):
        df_dict = self.RXTE_GetAllCounts()
        df = self.RXTE_MergeDataframeDictionary(df_dict)
        self.LIGHTCURVE_RXTE = df
        return df

    def RXTE_Complete(self):
        self.RXTE_DownloadAllObservations()
        aux.UnzipalltarFiles('sources/{}/rxte'.format(self.SOURCE_NAME))
        aux.RemoveAlltarFiles('sources/{}/rxte'.format(self.SOURCE_NAME))
        lc = self.RXTE_GetLightcurve()
        lc.plot(x='Time')
        return lc


from app.models.posts_models import Posts
from flask import jsonify, request
from datetime import datetime
import os

ACCEPTED_KEYS = os.getenv("ACCEPTED_KEYS")

def get_posts_controller():
    get_posts = Posts.get_posts_models()

    get_posts_list = list(get_posts)

    for i in get_posts_list:
        del i["_id"]

    return jsonify(get_posts_list), 200

def get_posts_by_id_controller(id):
    try:
        get_post_by_id = Posts.get_posts_by_id_models(id)

        del get_post_by_id["_id"]

        return get_post_by_id, 200
    except:
        return {"msg": f"ID {id} not found"}, 404

def create_posts_controller():
    try:
        date = datetime.now()
        actual_date = date.strftime("%d/%m/%y %H:%M:%S")

        data = request.get_json()

        get_posts = Posts.get_posts_models()

        get_posts_list = list(get_posts)

        if(len(get_posts_list) == 0):
            new_id = 1
        else:
            previous_object = get_posts_list[-1]
            new_id = previous_object["id"] + 1

        new_data = Posts(new_id, actual_date, actual_date, **data)

        found = Posts.create_posts_models(new_data.__dict__)

        del found["_id"]

        return found, 201

    except:
        return {
            "error": "wrong fields",
            "required_fields": [
                {"title": "Title"},
                {"author": "Author"},
                {"tags": "Tags"},
                {"content": "Content"}
            ]
        }, 400

def delete_post_controller(id):
    try:
        deleted_post = Posts.delete_post_models(id)

        del deleted_post["_id"]

        return deleted_post, 200

    except:
        return {"error": f"ID {id} not found"}, 404

def update_post_controller(id):
    try:
        data = request.get_json()

        accepted = ACCEPTED_KEYS.split(", ")

        for key in data.keys():
            if(key not in accepted):
                return {
                    "error": f"field '{key}' doesn't exists and cannot be updated!"
                }, 400

        if(len(data) == 0):
            return {
                "error": "You must pass the data!"
            }, 400

        date = datetime.now()
        actual_date = date.strftime("%d/%m/%y %H:%M:%S")

        data.update({"updated_at": actual_date})

        updated_post = Posts.update_post_models(data, id)

        show_updated_post = Posts.get_posts_by_id_models(id)

        del show_updated_post["_id"]

        return show_updated_post, 200

    except:
        return {
            "error": f"ID {id} not found!"
        }, 404


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# pylint: disable=C0103,W0621
"""
Dataset preparing utils.
"""
__author__ = "GW [http://gw.tnode.com/] <gw.2016@tnode.com>"
__license__ = "GPLv3+"

import numpy as np
import random
from keras.engine.training import make_batches

def build_index(sequences, max_new=None, min_count=1, index=None, vocab_start=2, none_key=None, none_ids=0, oov_key="", oov_ids=1):
    """Build vocabulary index from dicts or lists of strings (reserved ids: 0 = none/padding, 1 = out-of-vocabulary)."""
    if index is None:
        index = {}

    def _traverse_cnt(obj, cnts):
        """Recursively traverse dicts and lists of strings."""
        if isinstance(obj, dict):
            for s in obj.itervalues():
                _traverse_cnt(s, cnts)
        elif isinstance(obj, list) or isinstance(obj, tuple):
            for s in obj:
                _traverse_cnt(s, cnts)
        else:
            try:
                cnts[obj] += 1
            except KeyError:
                cnts[obj] = 1

    # count string occurrences
    cnts = {}
    _traverse_cnt(sequences, cnts)

    # ignore strings with low occurrences
    for k, cnt in cnts.items():
        if cnt < min_count:
            del cnts[k]

    # rank strings by decreasing occurrences and use as index
    index_rev = sorted(cnts, key=cnts.get, reverse=True)
    if max_new is not None:
        index_rev = index_rev[:max_new]  # limit amount of added strings

    # mapping of strings to vocabulary ids
    index.update([ (k, i) for i, k in enumerate(index_rev, start=vocab_start) ])
    index_size = vocab_start + len(index)  # largest vocabulary id + 1

    # add none/padding and out-of-vocabulary ids
    index[none_key] = none_ids
    index[oov_key] = oov_ids
    return index, index_size

def map_sequence(sequence, index, oov_key=""):
    """Map sequence of strings to vocabulary ids."""

    ids = []
    for s in sequence:
        try:
            ids.append(index[s])
        except KeyError:  # missing in vocabulary
            ids.append(index[oov_key])
    return ids

def pad_sequence(sequence, max_len, value=0, max_rand=None):
    """Post-pad sequence of ids as numpy array."""

    # crop sequence if needed
    sequence = sequence[:max_len]

    # convert to numpy array with masked and random post-padding
    if isinstance(value, int):
        x = np.hstack([sequence, np.ones((max_len - len(sequence),), dtype=np.int) * value])
    elif isinstance(value, float):
        x = np.hstack([sequence, np.ones((max_len - len(sequence),), dtype=np.float32) * value])
    elif value == 'rand' and isinstance(max_rand, int):
        x = np.hstack([sequence, np.random.randint(1, max_rand, size=max_len - len(sequence),)])
    else:
        raise ValueError("Padding value '{}' not understood".format(value))
    return x

def batch_generator(dataset, indexes, indexes_size, arg1_len, arg2_len, conn_len, punc_len, batch_size, random_per_sample=1):
    """Batch generator where each sample represents a different discourse relation."""

    batch_size = int(batch_size / (1.0 + random_per_sample))

    rel_ids = list(dataset['rel_ids'])  # copy list
    while True:
        # shuffle relations on each epoch
        random.shuffle(rel_ids)
        for batch_start, batch_end in make_batches(len(rel_ids), batch_size):

            # prepare batch data
            _rel_id = []
            data_in = {}
            data_out = {}
            for rel_id in rel_ids[batch_start:batch_end]:
                # choose sample
                doc_id = dataset['rel_parts'][rel_id]['DocID']
                words_len = len(dataset['words'][doc_id])

                _rel_id.append(rel_id)

                def tokens_np(token_ids, max_len):
                    words_slice = [ dataset['words'][doc_id][i]  for i in token_ids ]

                    ids = map_sequence(words_slice, indexes['words2id'])
                    x_np = pad_sequence(ids, max_len, value=0)
                    return x_np

                # arg1/2/conn/punc/rand inputs
                arg1_np = tokens_np(dataset['rel_parts'][rel_id]['Arg1'], arg1_len)
                try:
                    data_in['arg1_ids'].append(arg1_np)
                except KeyError:
                    data_in['arg1_ids'] = [arg1_np]

                arg2_np = tokens_np(dataset['rel_parts'][rel_id]['Arg2'], arg2_len)
                try:
                    data_in['arg2_ids'].append(arg2_np)
                except KeyError:
                    data_in['arg2_ids'] = [arg2_np]

                conn_np = tokens_np(dataset['rel_parts'][rel_id]['Connective'], conn_len)
                try:
                    data_in['conn_ids'].append(conn_np)
                except KeyError:
                    data_in['conn_ids'] = [conn_np]

                punc_np = tokens_np(dataset['rel_parts'][rel_id]['Punctuation'], punc_len)
                try:
                    data_in['punc_ids'].append(punc_np)
                except KeyError:
                    data_in['punc_ids'] = [punc_np]

                # relation senses output
                def rsenses_np(cat, oov_key=""):
                    try:
                        i = indexes['rel_senses2id'][cat]
                    except KeyError:  # missing in vocabulary
                        i = indexes['rel_senses2id'][oov_key]
                    x_np = np.zeros((indexes_size['rel_senses2id'],), dtype=np.float32)
                    x_np[i] = 1
                    return x_np

                if dataset['rel_senses']:
                    rsenses = rsenses_np(dataset['rel_senses'][rel_id])
                else:
                    rsenses = rsenses_np("")
                try:
                    data_out['rsenses'].append(rsenses)
                except KeyError:
                    data_out['rsenses'] = [rsenses]

                # random noise for each sample
                for _ in range(random_per_sample):
                    j = np.random.randint(3)
                    if j == 0:
                        # random arg1
                        arg1_np = np.random.randint(1, 1 + np.max(arg1_np), size=arg1_np.shape)
                    elif j == 1:
                        # random arg2
                        arg2_np = np.random.randint(1, 1 + np.max(arg2_np), size=arg2_np.shape)
                    else:
                        # random arg1 and arg2
                        arg1_np = np.random.randint(1, 1 + np.max(arg1_np), size=arg1_np.shape)
                        arg2_np = np.random.randint(1, 1 + np.max(arg2_np), size=arg2_np.shape)
                    # random connective
                    if np.max(conn_np) > 0:
                        conn_np = np.random.randint(1, 1 + np.max(conn_np), size=conn_np.shape)
                    # random punctuation
                    if np.max(punc_np) > 0:
                        punc_np = np.random.randint(1, 1 + np.max(punc_np), size=punc_np.shape)
                    # mark as out-of-vocabulary in rsenses
                    rsenses = rsenses_np("")

                    _rel_id.append(rel_id)
                    data_in['arg1_ids'].append(arg1_np)
                    data_in['arg2_ids'].append(arg2_np)
                    data_in['conn_ids'].append(conn_np)
                    data_in['punc_ids'].append(punc_np)
                    data_out['rsenses'].append(rsenses)

            # convert to NumPy array
            for k, v in data_in.items():
                data_in[k] = np.asarray(v)
            for k, v in data_out.items():
                data_out[k] = np.asarray(v)

            # append meta data
            data_in['_rel_id'] = _rel_id

            # yield batch
            yield (data_in, data_out)

def decode_relation(y_np, cats2id, cats2id_size):
    """Decode categories from one-hot vector (sample, cats2id)."""

    # normalize by rows to [0,1] interval
    y_sum = np.sum(y_np)
    totals = y_np / y_sum
    totals[y_sum == 0.] = y_np[y_sum == 0.]  # prevent NaN

    # return most probable category
    cat = None
    max_total = -1.
    for t, j in cats2id.items():
        if totals[j] > max_total:
            max_total = totals[j]
            cat = t
    return cat, totals

def load_word2vec(words2id, words2id_size, words_dim, words2vec_bin=None, words2vec_txt=None):
    if not words2vec_bin and not words2vec_txt:
        return None  # no pre-trained word embeddings

    import numpy as np
    from gensim.models import word2vec
    if words2vec_bin:
        model = word2vec.Word2Vec.load_word2vec_format(words2vec_bin, binary=True)
    else:
        model = word2vec.Word2Vec.load_word2vec_format(words2vec_txt)
    init_weights = np.zeros((words2id_size, words_dim), dtype=np.float32)
    for k, i in words2id.iteritems():
        if not isinstance(k, str):
            continue
        try:
            init_weights[i] = model[k][:words_dim]
        except KeyError:  # missing in word2vec
            pass
    return [init_weights]


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# EXPLORE THE BEST characteristics of rt

import numpy as np
from pylab import *
import cPickle as pickle
from scipy.stats import sem

monkeys =   { 'g':12701,
              'm':34752,
              'p':27692,
              'r':11634,
              's':13348 }

data = {}
for s in monkeys.keys():
	with open("../data/data_pickle/"+s+".pickle", 'rb') as f:
		data[s] = pickle.load(f)

rt = {}
rtmean = {}
count = {}

for s in monkeys.keys():
	rt[s] = dict({'search':{},'repeat':{}})
	rtmean[s] = dict({'search':{},'repeat':{}})
	count[s] = dict({'search':np.zeros(60),'repeat':np.zeros(60)})
	problem = data[s][0,4]
	search = [data[s][0,8]]
	repeat = []
	phase = 0.0
	for t in xrange(1, len(data[s])):
		phase = data[s][t,2]-data[s][t-1,2] # Si phase == -1, on vient de changer de problem
		if data[s][t,4] == problem and (phase == 0.0 or phase == 1.0): # same problem
			if data[s][t,2] == 0.0: # search trial
				search.append(data[s][t,8]) # append rt of search trial
			elif data[s][t,2] == 1.0: # repeat trial
				repeat.append(data[s][t,8]) # append rt of repeat trial
		else:
			nb_incorrect = len(search) # number of search trial before correct
			nb_repeat = len(repeat)
			count[s]['search'][nb_incorrect] += 1
			count[s]['repeat'][nb_repeat] += 1
			if nb_incorrect in rt[s]['search'].keys(): # append search trial in corresponding error length
				rt[s]['search'][nb_incorrect].append(np.array(search))
				rt[s]['repeat'][nb_incorrect].append(np.array(repeat))
			else:
				rt[s]['search'][nb_incorrect] = [np.array(search)]
				rt[s]['repeat'][nb_incorrect] = [np.array(repeat)]

			problem = data[s][t,4]  # number of new problem
			search = [data[s][t,8]]
			repeat = []

	t = 'search'
	for l in rt[s][t].keys():
		rt[s][t][l] = np.array(rt[s][t][l])
		rtmean[s][t][l] = np.zeros((2,l))
		rtmean[s][t][l][0] = rt[s][t][l].mean(0)
		rtmean[s][t][l][1] = sem(rt[s][t][l])

	t = 'repeat'
	for l in rt[s][t].keys():
		meann = np.zeros((2,7))
		for i in xrange(7):
			tmp = []
			for j in xrange(len(rt[s][t][l])):
				if len(rt[s][t][l][j]) > i:
					tmp.append(rt[s][t][l][j][i])
			meann[0,i] = np.mean(tmp)
			meann[1,i] = sem(tmp)

		rtmean[s][t][l] = meann

# figure(figsize = (20,15))
figure()
colors = {'p':'blue','s':'cyan','r':'black','m':'red','g':'green'}

for i in xrange(1,7):
	subplot(2,3,i)
	for s in rtmean.keys():
	# for s in ['p','s','r','m']:
	# for s in ['p','m']:
		# errorbar(range(i), rtmean[s]['search'][i][0], rtmean[s]['search'][i][1], fmt='o', label = str(s), color = colors[s])
		# errorbar(range(i,i+7), rtmean[s]['repeat'][i][0], rtmean[s]['repeat'][i][1], fmt='*', color = colors[s])
		plot(range(i), rtmean[s]['search'][i][0], 'o', label = str(s), color = colors[s])
		plot(range(i,i+7), rtmean[s]['repeat'][i][0], '*', color = colors[s])
		plot(range(i+7), np.hstack((rtmean[s]['search'][i][0],rtmean[s]['repeat'][i][0])), '-', color = colors[s])
	grid()
	title(str(i)+" SEARCH | 8 REPEAT")
	legend()

figure()
ind = 1
for k in count.keys():
	subplot(2,3,ind)
	plot(count[k]['search'][0:10], 'o-', label = 'Search')
	plot(count[k]['repeat'][0:10], '*-', label = 'Repeat')
	ylabel("Count")
	xlabel("Lenght of search/repeat bloc")
	legend()
	ind+=1
	title(k)

show()


from datetime import datetime

from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.pagination import PageNumberPagination

from config.constants import *
from models import Board
from serializers import BoardListSerializer

@api_view(['GET', 'POST'])
def board(request):
    response = {}
    result = {}

    if request.method == "POST":
        member_id = request.data["memberId"]
        title = request.data["title"]
        content = request.data["content"]

        if member_id is None or title is None or content is None:
            response["STS"] = ERR_USER_PARAM
            response["MSG"] = MSG[ERR_USER_PARAM]

            return Response(response)

        board = Board(member_id=member_id, title=title, content=content)
        board.save()

        response["STS"] = SUCCESS
        response["MSG"] = MSG[SUCCESS]
        response["DAT"] = board.id

        return Response(response)

    if request.method == "GET":
        paginator = PageNumberPagination()

        board_list = Board.objects.order_by("-id").all()
        result_page = paginator.paginate_queryset(board_list, request)
        board_list_serializer = BoardListSerializer(result_page, many=True)

        result["board_list"] = board_list_serializer.data
        result["total_cnt"] = len(board_list)

        response["STS"] = SUCCESS
        response["MSG"] = MSG[SUCCESS]
        response["DAT"] = result

        return Response(response)

@api_view(['GET', 'DELETE', "PUT"])
def board_detail(request, id):
    response = {}
    result = {}

    if request.method == "GET":

        posting = Board.objects.get(id=id)
        posting_serializer = BoardListSerializer(posting)

        result["posting"] = posting_serializer.data

        response["STS"] = SUCCESS
        response["MSG"] = MSG[SUCCESS]
        response["DAT"] = result

        return Response(response)

    if request.method == "DELETE":
        result["id"] = id

        posting = Board.objects.filter(id=id)

        if len(posting) != 1:
            response["STS"] = ERR_DB_DELETE
            response["MSG"] = MSG[ERR_DB_DELETE]
            response["DAT"] = result
        else :
            posting[0].delete()
            response["STS"] = SUCCESS
            response["MSG"] = MSG[SUCCESS]
            response["DAT"] = result

        return Response(response)

    if request.method == "PUT":
        result["id"] = id

        member = request.data["memberId"]
        posting_id = request.data["postingId"]
        title = request.data["title"]
        content = request.data["content"]

        posting = Board.objects.filter(id=posting_id, member=member)

        if id != posting_id:
            response["STS"] = ERR_DB_UPDATE
            response["MSG"] = MSG[ERR_DB_UPDATE]
            response["DAT"] = result

            return Response(response)

        if len(posting) != 1:
            response["STS"] = ERR_DB_UPDATE
            response["MSG"] = MSG[ERR_DB_UPDATE]
            response["DAT"] = result

            return Response(response)
        else:
            posting = posting[0]

        posting.title = title
        posting.content = content
        posting.moddt = datetime.now()

        posting.save()

        response["STS"] = SUCCESS
        response["MSG"] = MSG[SUCCESS]
        response["DAT"] = result

        return Response(response)


import yfinance as yf
import pandas as pd

tickers = ['BBDC4.SA', 'BBAS3.SA', 'ITUB4.SA']

start_date = '2018-01-01'

end_date = '2023-04-01'

df = yf.download(tickers, start=start_date, end=end_date)['Adj Close']
jls_extract_var = 'Adj Close'
df_mensal = df = yf.download(tickers, start=start_date, end=end_date).resample("Y").last()

retorno_mensal = df_mensal.pct_change()

df = pd.DataFrame(df)
print(df_mensal)


import argparse

arr = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
limit = 10

# boxes = vibrational nodes
# num boxes = 3N-6

def sixBoxes(A, B, C, D, E, F, sum):
    count = 0  # Initialize result
    len_A = len(A)
    len_B = len(B)
    len_C = len(C)
    len_D = len(D)
    len_E = len(E)
    len_F = len(F)

    for i in range(0, len_A):
        for j in range(0, len_B):
            for k in range(0, len_C):
                for l in range(0, len_D):
                    for m in range(0, len_E):
                        for n in range(0, len_F):
                            if arr[i] + arr[j] + arr[k] + arr[l] + arr[m] + arr[n] <= sum:
                                # print(arr[i], arr[j], arr[k], arr[l], arr[m], arr[n])
                                count += 1
    return count

# finds ints that are equal to or less than the given int
# returns array
def findLessNums(x, arr):
    nums = []
    for item in arr:
    	if item <= x:
            nums.append(item)
    return nums

def main():

    parser = argparse.ArgumentParser()
    parser.add_argument("a",type=int)
    parser.add_argument("b",type=int)
    parser.add_argument("c",type=int)
    parser.add_argument("d",type=int)
    parser.add_argument("e",type=int)
    parser.add_argument("f",type=int)

    args = parser.parse_args()
    # load integers
    a = args.a
    b = args.b
    c = args.c
    d = args.d
    e = args.e
    f = args.f

    A = findLessNums(a, arr)
    B = findLessNums(b, arr)
    C = findLessNums(c, arr)
    D = findLessNums(d, arr)
    E = findLessNums(e, arr)
    F = findLessNums(f, arr)

    print("Number of Acceptable Combinations: ", sixBoxes(A, B, C, D, E, F, limit))

if __name__ == "__main__":
    main()


from django.urls import path

from . import views

app_name = 'navi'

urlpatterns = [
	path('',views.index,name='hi'),
	path('msgb/',views.msgb,name='hi'),
	path('msgb/save/',views.msgsave,name='save'),
	path('wttr/',views.wttr,name='save'),
	path('jcount/',views.viscount,name='jcount')

]


# -*- coding: utf-8 -*-
"""
Created on Wed Apr  3 21:58:54 2019

@author: kartik
"""

import speech_recognition as sr

r = sr.Recognizer()
mic_list = sr.Microphone.list_microphone_names()

with sr.Microphone(device_index = 1) as source:
    r.adjust_for_ambient_noise(source)
    print ("Say Something")
    i=1

    audio = r.listen(source,phrase_time_limit=2)

    try:
        text = r.recognize_google(audio)
        print( "you said: " + text )

    except sr.UnknownValueError:
        print("Google Speech Recognition could not understand audio")

    except sr.RequestError as e:
        print("Could not request results from Google Speech Recognition service; {0}".format(e))

print("copmpleted")


import unittest
from tree.buttons.Button_tempo import Button_tempo
from unittest.mock import Mock
from mock import patch
import threading
from time import sleep

class TestButtonTempo(unittest.TestCase):

    @patch('time.sleep', return_value=None)
    @patch('time.time', return_value=0)
    def test_press(self, time, mock_sleep):
        manager = Mock()
        scenar_on, scenar_off = Mock(), Mock()
        button = Button_tempo("test", manager, scenar_on, scenar_off, 12)
        button.press()
        self.assertEqual(manager.do_scenar_principal.call_count, 1)
        self.assertEqual(manager.do_scenar_principal.call_args[0][0], scenar_on)
        self.assertEqual(threading.active_count(), 2)
        self.assertTrue("test" in "".join(i.name for i in threading.enumerate()))
        self.assertTrue("tempo" in "".join(i.name for i in threading.enumerate()))
        time.return_value = 11
        button.press()
        self.assertEqual(threading.active_count(), 2)
        time.return_value = 13
        sleep(0.005)
        self.assertEqual(threading.active_count(), 2)
        time.return_value = 24
        sleep(0.005)
        self.assertEqual(manager.do_scenar_principal.call_count, 2)
        self.assertEqual(manager.do_scenar_principal.call_args[0][0], scenar_off)

    def test_state(self):
        manager = Mock()
        scenar_on, scenar_off = Mock(), Mock()
        button = Button_tempo("test", manager, scenar_on, scenar_off, 0)
        self.assertEqual(button.state(), manager.get_state())

    def test_eq(self):
        manager = Mock()
        scenar_on, scenar_off = Mock(), Mock()
        button = Button_tempo("test", manager, scenar_on, scenar_off, 0)
        button2 = Button_tempo("test", manager, scenar_on, scenar_off, 0)
        self.assertEqual(button, button2)
        self.assertNotEqual(button, 8)
        button2 = Button_tempo("test", manager, Mock(), scenar_off, 0)
        self.assertNotEqual(button, button2)
        button2 = Button_tempo("test", manager, scenar_on, Mock(), 0)
        self.assertNotEqual(button, button2)
        button2 = Button_tempo("test", manager, scenar_on, scenar_off, 12)
        self.assertNotEqual(button, button2)

    def test_str(self):
        manager = Mock()
        scenar_on, scenar_off = Mock(), Mock()
        button = Button_tempo("test", manager, scenar_on, scenar_off, 254)
        self.assertTrue(str(scenar_on.name) in str(button))
        self.assertTrue(str(scenar_off.name) in str(button))
        self.assertTrue("tempo" in str(button))
        self.assertTrue("254" in str(button))


from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from .forms import NoteCreationForm

@login_required
def notes(request):
    author = request.user
    user_notes = author.notes_set.all()
    context = {'notes': user_notes}
    return render(request, 'notes/notes.html', context)

@login_required
def note_detail(request, pk):
    author = request.user
    user_note = author.notes_set.get(id=pk)
    context = {'note': user_note}
    return render(request, 'notes/note.html', context)

@login_required
def add_note(request):
    author = request.user
    if request.method == 'POST':
        form = NoteCreationForm(request.POST)
        if form.is_valid():
            note = form.save(commit=False)
            note.author = author
            note.save()
            return redirect('notes')
    else:
        form = NoteCreationForm()
    context = {'form': form}
    return render(request, 'notes/note_form.html', context)

@login_required
def update_note(request, pk):
    author = request.user
    note = author.notes_set.get(id=pk)
    if request.method == 'POST':
        form = NoteCreationForm(request.POST, instance=note)
        if form.is_valid():
            form.save()
            return redirect('notes')
    else:
        form = NoteCreationForm(instance=note)
    context = {'form': form}
    return render(request, 'notes/note_form.html', context)

@login_required
def delete_note(request, pk):
    author = request.user
    note = author.notes_set.get(id=pk)
    if request.method == 'POST':
        note.delete()
        return redirect('notes')
    context = {'note': note}
    return render(request, 'notes/delete_note.html', context)


import logging
import time as pytime
from pathlib import Path
from typing import Any, Dict, List

import dolfin as df
import ufl
from dolfin import grad, inner

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
df.set_log_level(df.LogLevel.WARNING)

from multidiffusion.boundary import DirichletBoundary, indexed_boundary_conditions, process_dirichlet
from multidiffusion.fenicsstorage import FenicsStorage
from multidiffusion.interpolator import vectordata_interpolator
from multidiffusion.timekeeper import TimeKeeper
from multidiffusion.utils import assign_mixed_function

def print_progress(t, T, rank=0):
    if rank != 0:
        return
    progress = int(40 * t / T)
    print(
        f"[{'=' * progress}{' ' * (40 - progress)}] {round(progress * 100 / 40, 1)}%",
        end="\r",
        flush=True,
    )

def read_concentration_data(filepath, funcname="cdata") -> list[df.Function]:
    store = FenicsStorage(filepath, "r")
    tvec = store.read_timevector(funcname)
    c = store.read_function(funcname, idx=0)
    C = [df.Function(c.function_space()) for _ in range(tvec.size)]
    for idx in range(len(C)):
        store.read_checkpoint(C[idx], funcname, idx)
    return tvec, C

def read_function_element(filepath, funcname) -> df.FunctionSpace:
    store = FenicsStorage(filepath, "r")
    el = store.read_element(funcname)
    store.close()
    return el

def compartment_form(
    idx_j: int,
    u: ufl.Argument,
    v: ufl.Argument,
    c0: df.Function,
    D: Dict[str, float],
    phi: Dict[str, float],
    alpha: float,
    P: float,
    compartments: List[str],
    dt: float,
) -> df.Form:
    j = compartments[idx_j]
    sj = sum(
        [
            (alpha * P / phi[j]) * (u[idx_i] - u[idx_j])
            for idx_i, i in enumerate(compartments)
            if idx_i != idx_j
        ]
    )
    return (u[idx_j] - c0[idx_j] - dt * sj) * v[idx_j] + dt * inner(
        D[j] * grad(u[idx_j]), grad(v[idx_j])
    )

def multicomp_diffusion_form(
    V: df.FunctionSpace,
    coefficients: Dict[str, Any],
    c0: df.Function,
    compartments: List[str],
    dt: float,
) -> df.Form:
    dx = df.Measure("dx", domain=V.mesh())
    u = df.TrialFunction(V)
    v = df.TestFunction(V)
    D = coefficients["diffusion_coefficient"]
    phi = coefficients["porosity"]
    alpha = coefficients["alpha"]
    P = coefficients["permeability"]
    return (
        sum(
            [
                compartment_form(idx_j, u, v, c0, D, phi, alpha, P, compartments, dt)
                for idx_j, _ in enumerate(compartments)
            ]
        )
        * dx
    )

def get_default_coefficients() -> Dict[str, Any]:
    return {
        "porosity": {
            "ecs": 0.14,
            "pvs": 0.01,
        },
        "diffusion_coefficient": {"ecs": 3.4e-4, "pvs": 3.4e-3},
        "alpha": 5.0,
        "permeability": 8.3e-5,
    }

def multicomp_diffusion(compartments, coefficients, inputfile, outputfile=None):
    if outputfile is None:
        outputfile = Path(inputfile)
    logger.info(f"Reading data from {inputfile}")
    timevec, data = read_concentration_data(inputfile)
    interpolator = vectordata_interpolator(data, timevec)
    c0 = data[0].copy(deepcopy=True)

    domain = c0.function_space().mesh()
    el = read_function_element(inputfile, "cdata")
    element = df.MixedElement([el] * 2)
    V = df.FunctionSpace(domain, element)

    dt = 3600
    T = timevec[-1]
    time = TimeKeeper(dt=dt, endtime=T)

    # Define boundary conditions
    u_interp = {compartment: c0.copy(deepcopy=True) for compartment in compartments}
    phi = coefficients["porosity"]
    phi_tot = sum(phi.values())
    for val in u_interp.values():
        val.vector()[:] = (1.0 / phi_tot) * interpolator(0.0)

    boundary_data = {
        idx: [DirichletBoundary(u_interp[compartment], "everywhere")]
        for idx, compartment in enumerate(compartments)
    }
    boundaries = indexed_boundary_conditions(boundary_data)
    bcs = process_dirichlet(V, domain, boundaries)

    # Define variational problem
    u0 = assign_mixed_function(u_interp, V, compartments)
    F = multicomp_diffusion_form(V, coefficients, u0, compartments, dt)
    a = df.lhs(F)
    l = df.rhs(F)
    A = df.assemble(a)

    u = df.Function(V)
    u.assign(u0)
    storage = FenicsStorage(outputfile, "a")
    storage.write_function(u, "multidiffusion", overwrite=True)

    logger.info("Starting time loop...")
    tic = pytime.time()
    time.reset()
    for ti in time:
        print_progress(float(ti), time.endtime, rank=df.MPI.comm_world.rank)
        for j, uj in u_interp.items():
            uj.vector()[:] = ((1.0 / phi_tot)) * interpolator(float(ti))
        b = df.assemble(l)
        for bc in bcs:
            bc.apply(A, b)
        df.solve(A, u.vector(), b, "gmres", "hypre_amg")
        storage.write_checkpoint(u, "multidiffusion", float(ti))
        u0.assign(u)
    storage.close()
    logger.info("Time loop finished.")
    toc = pytime.time()

    df.MPI.comm_world.barrier()
    if df.MPI.comm_world.rank == 0:
        logger.info(f"Elapsed time in loop: {toc - tic:.2f} seconds.")
    return storage.filepath


import time
from fuzzywuzzy import process
from cachetools import TTLCache
from bs4 import BeautifulSoup

from rsi.conf import DEFAULT_RSI_URL
from .session import RSISession

DEFAULT_CACHE_TTL = 300

class OrgAPI(object):
    def __init__(self, symbol, session=None, admin_mode=False, url=DEFAULT_RSI_URL, endpoint='/orgs',
                 members_endpoint='/api/orgs/getOrgMembers', cache_ttl=DEFAULT_CACHE_TTL):
        self.symbol = symbol
        self.url = url.rstrip('/')
        self.endpoint = endpoint
        self.members_endpoint = members_endpoint
        self.admin_mode = admin_mode
        self.session = session or RSISession(url=url)

        self.org_url = "{}/{}/{}".format(self.url, self.endpoint.lstrip('/'), symbol)
        self.members_api = "{}/{}".format(self.url, self.members_endpoint.lstrip('/'))
        self._ttlcache = TTLCache(maxsize=1, ttl=cache_ttl)

        self._update_details()   # pull and cache the org details which will raise 404 if not found

    def clear_cache(self):
        """ Resets the cache """
        for key in self._ttlcache.keys():
            del self._ttlcache[key]

    def _cache(self, key, update_func, *args, **kwargs):
        if key not in self._ttlcache:
            self._ttlcache[key] = update_func(*args, **kwargs)
        return self._ttlcache[key]

    def _update_members(self, search):
        members = []

        params = {
            'symbol': self.symbol,
            'search': search,
            'page': 1
        }

        if self.admin_mode:
            params['admin_mode'] = 1

        # this just gets us going
        totalsize = 1
        members_scanned = 0

        while members_scanned < totalsize:
            r = self.session.post(self.members_api, data=params)

            if r.status_code == 200:
                r = r.json()
                if r is None:
                    continue

                if 'data' in r and r['data'] and 'totalrows' in r['data']:
                    totalsize = int(r['data']['totalrows'])

                if r['success'] == 1:
                    apisoup = BeautifulSoup(r['data']['html'], features='html.parser')
                    for member in apisoup.select('.member-item'):
                        members_scanned += 1
                        if member.select('.member-visibility-restriction'):
                            print('skipping hidden member')
                            continue

                        members.append({
                            'name': member.select_one('.name').text,
                            'handle': member.select_one('.nick').text,
                            'avatar': '{}{}'.format(self.url, member.select_one('img').attrs['src']),
                            'affiliate': member.select_one('.title').text == 'Affiliate',
                            'rank': member.select_one('.rank').text,
                            'roles': [_.text for _ in member.select('.rolelist .role')],
                            'url': '{}{}'.format(self.url, member.select_one('a.membercard').attrs['href']),

                            # defaults for things online admins will be able to get the real values of
                            'id': '',
                            'visibility': 'Membership: Visible',
                            'last_online': '',
                        })

                        if self.admin_mode:
                            members[-1].update({
                                'id': member.attrs.get('data-member-id', ''),
                                'last_online': member.select_one('.frontinfo .lastonline').text,
                                'visibility': member.select_one('.frontinfo .visibility').text,
                            })

                    params['page'] = params['page'] + 1
                else:
                    raise ValueError('Received error fetching Org members: {}'.format(r))
            else:
                raise Exception('Received error fetching Org members: {}'.format(r.status_code))
            time.sleep(0.5)
        return members

    def _update_details(self):
        r = self.session.get(self.org_url)
        data = {}
        r.raise_for_status()

        orgsoup = BeautifulSoup(r.text, features='html.parser')
        data['banner'] = '{}{}'.format(self.url, orgsoup.select_one('.banner img')['src'])
        data['logo'] = '{}{}'.format(self.url, orgsoup.select_one('.logo img')['src'])
        data['name'], data['symbol'] = orgsoup.select_one('.inner h1').text.split(' / ')
        data['model'] = orgsoup.select_one('.inner .tags .model').text
        data['commitment'] = orgsoup.select_one('.inner .tags .commitment').text
        data['primary_focus'] = orgsoup.select_one('.inner .focus .primary img')['alt']
        data['secondary_focus'] = orgsoup.select_one('.inner .focus .secondary img')['alt']
        data['join_us'] = orgsoup.select_one('.join-us .body').text.strip()
        return data

    def search(self, handle, score_cutoff=80, limit=None):
        """
        Return members that match the given handle using fuzzy matching.

        :param handle: Handle to match
        :param score_cutoff: minimum matching score to return
        :param limit: limit the number of matches found
        :return: List of matched results in the form of [(dict, int)] where dict is the ship data and in is the
                 matching confidence
        """
        choices = {i: _['handle'] for i, _ in enumerate(self.members)}
        return [(self.members[_[2]], _[1]) for _ in process.extractBests(handle, choices,
                                                                         score_cutoff=score_cutoff, limit=limit)]

    def search_one(self, handle):
        """
        Return the first member that matches the given handle using fuzzy matching, or None

        :param handle: Handle to match
        :return: The best matching member, or None
        """
        choices = self.search(handle, limit=1)
        if choices:
            return choices[0][0]
        return None

    @property
    def members(self):
        return self._cache('members', self._update_members, search='')

    @property
    def details(self):
        return self._cache('details', self._update_details)

    @property
    def banner(self):
        return self.details['banner']

    @property
    def logo(self):
        return self.details['logo']

    @property
    def name(self):
        return self.details['name']

    @property
    def model(self):
        return self.details['model']

    @property
    def commitment(self):
        return self.details['commitment']

    @property
    def primary_focus(self):
        return self.details['primary_focus']

    @property
    def secondary_focus(self):
        return self.details['secondary_focus']

    @property
    def spectrum_url(self):
        return '{}/spectrum/community/{}'.format(self.url, self.symbol)

    @property
    def join_us(self):
        return self.details['join_us']


"""The workflow execution:
1) Get all the EQUITY_* data of the week and collect it into one CSV file WEEKLY_*
2) Create a back of all the EQUITY_* files in a back_up location.
3) Remove the daily logs of the week and the EQUITY_* files.
4) Convert WEEKLY_* files in JSON.
5) Insert it inES index as docs. If it fails retain the WEEKLY_JSON_TMP"""

from datetime import datetime
import glob
import shutil
import os
import pandas as pd
from elasticsearch.exceptions import ConnectionTimeout, ConflictError
from LoggerApi.Logger import Logger
from Harvester import Utils

LOGGER = Logger(__file__, "WEEKLY_GATHERING")
DESTINATION_FOLDER = "/backup_stock_data/"
DAILY_DATA_FILES = [daily_data for daily_data in
                    glob.glob(".\\resources\\*.csv") if "EQUITY" in daily_data]
LOG_FILES = [logs for logs in
             glob.glob(os.path.dirname(os.path.dirname(__file__)) + "\\*LOG*") if "LOG" in logs]
FIRST_FILE_INDICATOR = True
FINAL_DATA = None

for file in DAILY_DATA_FILES:
    date = file.split("\\")[2].split(".")[0].split("_")[-1:][0]
    daily_df = pd.read_csv(file)
    daily_df['processingDate'] = date
    if FIRST_FILE_INDICATOR:
        FIRST_FILE_INDICATOR = False
        column_names = daily_df.columns
        FINAL_DATA = daily_df.copy()
        LOGGER.add("INFO", "First files parsed."
                           " Date:{}".format(date[:2] + "/" + date[2:4] + "/" + date[4:]))
        continue
    FINAL_DATA = pd.concat([FINAL_DATA, daily_df], axis=0)
    LOGGER.add("INFO",
               "Data added for Date:{} ".format(date[:2] + "/" + date[2:4] + "/" + date[4:]))

try:
    FINAL_DATA.to_csv(".\\resources\\WEEKLY_{}.csv".format(datetime.now().strftime('%d%m%Y')),
                      index=False)
    LOGGER.add("INFO", "COMPLETED")
    LOGGER.add("INFO", "Moving the daily data of the week to the back-up folder.")
    for file in DAILY_DATA_FILES:  # move the daily collected files to backup folder
        try:
            shutil.move(file, DESTINATION_FOLDER)
        except shutil.Error:
            LOGGER.add("ERROR", "The file already exists in back-up")
            os.remove(file)
            LOGGER.add("INFO", "The duplicates removed from cwd")
    for file in LOG_FILES:  # removing the logs for the week
        os.remove(file)

    Utils.weekly_report_to_json()  # Converting all the csv weekly report to JSON file
    LOGGER.add("INFO", "Converted to JSON successfully.")

    try:
        Utils.insert_to_es_index(LOGGER)  # Insert the file to ES
        LOGGER.add("INFO", "Insertion into ES was successful")
        for file in [weekly_json for weekly_json in glob.glob(".\\resources\\*JSON*")]:
            os.remove(file)
    except ConnectionTimeout:
        LOGGER.add("ERROR", "Connection Timed out while inserting")
    except ConflictError:
        LOGGER.add("ERROR", "Error while establishing connection. Please check ES host is up.")

    for file in [daily_data for daily_data in glob.glob(".\\resources\\*.csv") if
                 "WEEKLY" in daily_data]:  # Removing the temporary WEEKLY* csv files
        os.remove(file)
    LOGGER.add("INFO", "Weekly temp csv files removed.")
except NameError:
    LOGGER.add("ERROR", "There is not data present in the resource folder.")


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/8/18 19:23
# @Author  : hearecho
# @Site    :
# @File    : get_list.py
# @Software: PyCharm

import requests
from spider.yundownload.utils import headers,tools
import json
import time

def get_list():
    path = input("请输入路径:/*")
    encode_path = tools.encodeURIComponent(path)
    url = 'http://pan.baidu.com/api/list?dir={}' \
          '&num=100&order=time&desc=0&clienttype=0' \
          '&showempty=0&web=1&page=1'.format(encode_path)
    resp = requests.get(url,headers=headers.setheaders())
    print(resp.apparent_encoding)
    resp.encoding = resp.apparent_encoding
    resu_json = resp.json()
    print(json.dumps(resu_json,indent=2))

def get_item(fs_id):
    timestamp = int(time.time())
    fidlist = tools.encodeURIComponent("[{}]".format(fs_id))
    url = "http://pan.baidu.com/api/download?" \
          "sign=IVB6GC7Y665X6pNpiZz293vavj6qLsrdOFQtaJ8z4RNAUXReoQrjZA==" \
          "&timestamp={}" \
          "&fidlist={}".format(timestamp,fidlist)
    resp = requests.get(url,headers=headers.setheaders())
    resu_json = resp.json()
    print(json.dumps(resu_json, indent=2))
    print(url)

if __name__ == '__main__':
    get_list()


import sys
import logging
import datetime
from backend.block import operations
from scripts import witness_service

taskname = "witness schedule"
if __name__ == "__main__":

    try:
        if len(sys.argv) > 1:
            time_str = sys.argv[1]
            utc_date_time = datetime.datetime.strptime(time_str, "%Y-%m-%d")
        else:
            utc_date_time = datetime.datetime.now() - datetime.timedelta(days=1)
        witness_service.witness_services("make_witness_schedule", utc_date_time)
        logging.info('execute crontab [%s] success.', taskname)
    except Exception as e:
        logging.error('execute crontab [%s] error: %s', taskname, e)


import json
import operator
import io

f = open(r'G:\twitter\Twitter\user-tweet-similar.json')
df =json.loads(f.read())
#user_tweet = df['user_tweet']

g = open(r'G:\twitter\Twitter\graph\nodes.json')
dg = json.loads(g.read())
nodes = dg['nodes']
#print(nodes)
d = dict()

fun = operator.itemgetter('id')
l = list()
for u in df:
    #r=str(u).lstrip()
    for su in df[u]:
        #t=str(su).lstrip()
        l.append(dict(source= map(fun,nodes).index(u),target=map(fun,nodes).index(su)))

d['links']=l
with open('link-ut-sim.json','w')as jj:
    json.dump(d,jj)


from __future__ import print_function
import torch
import os
import numpy as np
import argparse
import yaml
from tqdm import tqdm
import cv2
import sys

sys.path.append(".")
sys.path.append("..")

from layers.functions.prior_box import PriorBox
from utils.nms.py_cpu_nms import py_cpu_nms
from models.retinaface import RetinaFace
from utils.box_utils import decode, decode_landm

def check_keys(model, pretrained_state_dict):
    ckpt_keys = set(pretrained_state_dict.keys())
    model_keys = set(model.state_dict().keys())
    used_pretrained_keys = model_keys & ckpt_keys
    unused_pretrained_keys = ckpt_keys - model_keys
    missing_keys = model_keys - ckpt_keys
    print('Missing keys:{}'.format(len(missing_keys)))
    print('Unused checkpoint keys:{}'.format(len(unused_pretrained_keys)))
    print('Used keys:{}'.format(len(used_pretrained_keys)))
    assert len(used_pretrained_keys) > 0, 'load NONE from pretrained checkpoint'
    return True

def remove_prefix(state_dict, prefix):
    ''' Old style model is stored with all names of parameters sharing common prefix 'module.' '''
    print('remove prefix \'{}\''.format(prefix))
    f = lambda x: x.split(prefix, 1)[-1] if x.startswith(prefix) else x
    return {f(key): value for key, value in state_dict.items()}

def load_model(model, pretrained_path, load_to_cpu):
    print('Loading pretrained model from {}'.format(pretrained_path))
    if load_to_cpu:
        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage)
    else:
        device = torch.cuda.current_device()
        pretrained_dict = torch.load(pretrained_path, map_location=lambda storage, loc: storage.cuda(device))
    if "state_dict" in pretrained_dict.keys():
        pretrained_dict = remove_prefix(pretrained_dict['state_dict'], 'module.')
    else:
        pretrained_dict = remove_prefix(pretrained_dict, 'module.')
    check_keys(model, pretrained_dict)
    model.load_state_dict(pretrained_dict, strict=False)
    return model

@torch.no_grad()
def test(_args):
    with open(_args.cfg, "r") as f:
        cfg = yaml.load(f.read(), Loader=yaml.FullLoader)

    net = RetinaFace(cfg=cfg)
    net = load_model(net, _args.ckpt, _args.device == "cpu")
    net.eval()

    confidence_threshold = _args.confidence_threshold
    nms_threshold = _args.nms_threshold
    vis_threshold = _args.vis_threshold
    device = _args.device
    origin_size = _args.origin_size
    target_size = _args.target_size
    max_size = _args.max_size

    print('Finished loading model!')

    device = torch.device(device)
    net = net.to(device)

    images_path = [x.path for x in os.scandir(_args.images_path) if x.name.endswith(("png", "jpg", "jpeg"))]

    for i, image_path in tqdm(enumerate(images_path), desc="testing..."):
        img_raw = cv2.imread(image_path, cv2.IMREAD_COLOR)
        img = np.float32(img_raw)
        im_shape = img.shape
        im_size_min = np.min(im_shape[0:2])
        im_size_max = np.max(im_shape[0:2])
        resize = float(target_size) / float(im_size_min)

        # prevent bigger axis from being more than max_size:
        if np.round(resize * im_size_max) > max_size:
            resize = float(max_size) / float(im_size_max)
        if origin_size:
            resize = 1

        if resize != 1:
            img = cv2.resize(img, None, None, fx=resize, fy=resize, interpolation=cv2.INTER_LINEAR)

        im_height, im_width, _ = img.shape
        scale = torch.Tensor([img.shape[1], img.shape[0], img.shape[1], img.shape[0]])
        img -= (104, 117, 123)
        img /= (57, 57, 58)
        img = img.transpose(2, 0, 1)
        img = torch.from_numpy(img).unsqueeze(0)
        img = img.to(device)
        scale = scale.to(device)

        if cfg['gender']:
            loc, conf, landms, gender = net(img)  # forward pass
        else:
            loc, conf, landms = net(img)  # forward pass

        priorbox = PriorBox(cfg, image_size=(im_height, im_width))
        priors = priorbox.forward()
        priors = priors.to(device)
        prior_data = priors.data
        boxes = decode(loc.data.squeeze(0), prior_data, cfg['variance'])
        boxes = boxes * scale / resize
        boxes = boxes.cpu().numpy()
        scores = conf.squeeze(0).data.cpu().numpy()[:, 1]
        if cfg['gender']:
            genders = gender.squeeze(0).data.cpu().numpy()

        landms = decode_landm(landms.data.squeeze(0), prior_data, cfg['variance'])
        scale1 = torch.Tensor([img.shape[3], img.shape[2], img.shape[3], img.shape[2],
                               img.shape[3], img.shape[2], img.shape[3], img.shape[2],
                               img.shape[3], img.shape[2]])
        scale1 = scale1.to(device)
        landms = landms * scale1 / resize
        landms = landms.cpu().numpy()

        # ignore low scores
        inds = np.where(scores > confidence_threshold)[0]
        boxes = boxes[inds]
        landms = landms[inds]
        scores = scores[inds]
        if cfg['gender']:
            genders = genders[inds]

        # keep top-K before NMS
        order = scores.argsort()[::-1]
        # order = scores.argsort()[::-1][:args.top_k]
        boxes = boxes[order]
        landms = landms[order]
        scores = scores[order]
        if cfg['gender']:
            genders = genders[order]

        # do NMS
        dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32, copy=False)
        keep = py_cpu_nms(dets, nms_threshold)
        # keep = nms(dets, args.nms_threshold,force_cpu=args.cpu)
        dets = dets[keep, :]
        landms = landms[keep]
        if cfg['gender']:
            genders = genders[keep]
        # keep top-K faster NMS
        # dets = dets[:args.keep_top_k, :]
        # landms = landms[:args.keep_top_k, :]
        if cfg['gender']:
            dets = np.concatenate((dets, landms, genders * 10000), axis=1).astype(np.float32)
        else:
            dets = np.concatenate((dets, landms), axis=1).astype(np.float32)

        for b in dets:
            if b[4] < vis_threshold:
                continue
            text = "{:.4f}".format(b[4])
            b = list(map(int, b))
            cv2.rectangle(img_raw, (b[0], b[1]), (b[2], b[3]), (0, 0, 255), 2)
            cx = b[0]
            cy = b[1] + 12
            cv2.putText(img_raw, text, (cx, cy),
                        cv2.FONT_HERSHEY_DUPLEX, 0.5, (255, 255, 255))
            if cfg['gender']:
                gender = (b[15] / 10000, b[16] / 10000)
                gender_str = "male:" if gender.index(max(gender)) else "female:"
                gender_str += str(max(gender))

                cv2.putText(img_raw, gender_str, (cx, cy - 15),
                            cv2.FONT_HERSHEY_DUPLEX, 0.5, (255, 255, 0))
            # landms
            cv2.circle(img_raw, (b[5], b[6]), 3, (0, 0, 255), 3)
            cv2.circle(img_raw, (b[7], b[8]), 3, (0, 255, 255), 3)
            cv2.circle(img_raw, (b[9], b[10]), 3, (255, 0, 255), 3)
            cv2.circle(img_raw, (b[11], b[12]), 3, (0, 255, 0), 3)
            cv2.circle(img_raw, (b[13], b[14]), 3, (255, 0, 0), 3)

        if not os.path.exists(_args.save):
            os.makedirs(_args.save)
        name = os.path.join(_args.save, image_path.split('/')[-1].split('.')[0] + "_{}.jpg".format(cfg["name"]))
        cv2.imwrite(name, img_raw)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="RetinaFace tester.")

    parser.add_argument(
        "--ckpt",
        type=str,
        help="path to the model checkpoint",
    )

    parser.add_argument(
        "--cfg",
        type=str,
        default="",
        help="path to the config yaml file",
    )

    parser.add_argument(
        "--device",
        type=str,
        default="cpu",
        help="path to the config yaml file",
    )

    parser.add_argument(
        "--confidence_threshold",
        type=float,
        default=0.02,
        help="confidence thresh",
    )

    parser.add_argument(
        "--nms_threshold",
        type=float,
        default=0.04,
        help="nms thresh",
    )

    parser.add_argument(
        "--vis_threshold",
        type=float,
        default=0.5,
        help="visual thresh",
    )

    parser.add_argument(
        "--target_size",
        type=int,
        default=512,
        help="target output size",
    )

    parser.add_argument(
        "--max_size",
        type=int,
        default=512,
        help="max output size",
    )

    parser.add_argument(
        '--origin_size',
        action="store_true",
        help='keep origin size.')

    parser.add_argument(
        "--images_path", type=str, help="images path to be test."
    )

    parser.add_argument(
        "--save",
        type=str,
        default="./result",
        help="path to save the result",
    )

    args = parser.parse_args()

    test(args)

    print("Test Finished!")


from typing import List

# Approach 1: Max Heap
import heapq
from collections import Counter
class Solution1(object):
    def topKFrequent(self, nums, k):
        heap = [(-val, key) for key, val in Counter(nums).items()]
        heapq.heapify(heap) # O(n)
        result = []

        for _ in range(k):
            result.append(heapq.heappop(heap)[1])
        return result

# Approach 2: Min-Heap
import heapq
from collections import Counter
class Solution:
    def topKFrequent(self, nums, k):
        heap = []
        for num, freq in Counter(nums).items():
            heapq.heappush(heap, (freq, num))

            if len(heap) > k:
                heapq.heappop(heap)

        return [item[1] for item in heap]

# Approach 3: Bucket Sort (Worst case this is the best performant in time, but uses O(n) space)
import collections
class Solution3:
    def topKFrequent(self, nums: List[int], k: int) -> List[int]:
        n, result = len(nums), []
        counts_bucket = [[] for _ in range(n)]

        for num, freq in collections.Counter(nums).items():
            counts_bucket[freq - 1].append(num)

        for i in range(n - 1, -1, -1):
            if not counts_bucket[i]: continue
            result.extend(counts_bucket[i])
            if len(result) >= k: break

        return result[:k]

# # Approach 4: Quick-Select
import random
import collections
class Solution:
    def topKFrequent(self, nums, k):
        def quick_select(a, l, r, K):
            if l == r: return
            mid = rand_partition(a, l, r)

            if K < mid:     quick_select(a, l, mid - 1, K)
            elif K > mid:   quick_select(a, mid + 1, r, K)
            else:           return

        def rand_partition(a, l, r):
            i = random.randint(l, r)
            a[r], a[i] = a[i], a[r]
            return partition(a, l, r)

        def partition(a, l, r):
            i = l - 1
            pivot = a[r][1]
            for j in range(l, r):
                if a[j][1] > pivot: continue
                i += 1
                a[i], a[j] = a[j], a[i]

            a[i + 1], a[r] = a[r], a[i + 1]
            return i + 1

        counts = list(collections.Counter(nums).items())
        n = len(counts)
        quick_select(counts, 0, n - 1, n - k)
        return [x[0] for x in counts[n - k:]]

# 347. Top K Frequent Elements
# https://leetcode.com/problems/top-k-frequent-elements/

# Key Insight:
# 1. Within top K order is not important, we just need the top K frequent elements

# Approach 1: Max heap
# Steps:
# 1. Create a frequency map
# 2. Add every tuple (frequency, item) to a max heap
# 3. Then extract the top k elements

# Time: nlog(n)(n + k) log(n) => nlog(n)
# Space: O(n)

# Approach 2: Min-heap
# 1. Create a frequency map.
# 2. Add k tuples (frequency, item) to min-heap.
# 3. Iterate from k + 1 th tuple to nth tuple.
#    Now that size is k, Do a heapreplace only if new tuple's count is > than smallest on the pqueue
# 4. Finally the heap will have k largest frequency numbers, return it.
# Note: Order doesn't matter for the answer

# Time: O(nlog(k))
# Space: O(k)

# Approach 3: Bucket Sort
# Steps:
# 1. Create a count_bucket array of size = len(nums), initialized with values []
# 2. Use Counter class to get frequencies of each number
# 3. Use 'count' as buckets to append every element with count c to index c - 1 in 'count'
# 4. Now we move from back of the count array, pick buckets that aren't empty and add them to the result
#    until the result is of size k.

# Time: O(n)
# Space: O(n)

# Approach 4: Quick Select
# Steps:
# 1. Get the counts using collections.counter
# 2. Do a quick select on the counts list using n - k as K,
#    Thus, converting kth smallest to kth largest
# 3. Now we'll have the elements after n - kth positing all having k largest frequencies
# 4. Return the actual elements from those items

# Time: O(n): Avg case due to randomization, O(n^2): Worst Case Time
# Space: O(1)

sol = Solution()
assert sol.topKFrequent([1, 1, 1, 2, 2, 3], 2) in ([1, 2], [2, 1])


import os
import warnings
from datetime import datetime
from typing import List

import pandas as pd
import psycopg2
import requests
from psycopg2.extras import execute_values

from core.config import quest

warnings.simplefilter("ignore", UserWarning)

class QuestBase:
    def __init__(self) -> None:
        self.dsn = quest.dsn
        self.host = quest.host
        self.rest_port = quest.rest_port

    def sql_exec(self, sql: str):
        with psycopg2.connect(self.dsn) as conn:
            with conn.cursor() as cursor:
                cursor.execute(sql)
                try:
                    return cursor.fetchall()
                except:
                    pass

class StockDay(QuestBase):
    def bulk_insert_csv(self, file_path: str) -> None:
        if os.path.exists(file_path):
            schema = """[{"name": "symbol",  "type": "symbol"},
                         {"name": "open",    "type": "double"},
                         {"name": "high",    "type": "double"},
                         {"name": "low",     "type": "double"},
                         {"name": "close",   "type": "double"},
                         {"name": "amount",  "type": "double"},
                         {"name": "volume",  "type": "double"},
                         {"name": "date",    "type": "TIMESTAMP", "pattern": "yyyy-MM-ddTHH:mm:ss"}]"""
            params = {'name': 'trade'}
            files = {
                'schema': schema,
                'data': open(file_path, 'rb'),
            }
            requests.post('http://{host}:{port}/imp'.format(
                host=self.host, port=self.rest_port),
                params=params, files=files)

    def current_date(self):
        sql = '''SELECT date FROM 'trade'
                 WHERE symbol = 'sh999999'
                 ORDER BY date DESC
                 LIMIT 1;'''
        date = self.sql_exec(sql)[0][0]
        return date

    def format_sql(self, item: dict) -> tuple:
        return (
            item['symbol'],
            item['open'],
            item['high'],
            item['low'],
            item['close'],
            item['amount'],
            item['volume'],
            item['date']
        )

    def query(self, symbol, start_date):
        sql = '''
            select symbol, open, high, low, close, volume, date
            from trade where symbol = '{symbol}' and
            date >= to_timestamp('{start_date}','yyyyMMdd');
            '''.format(symbol=symbol, start_date=start_date)

        with psycopg2.connect(self.dsn) as conn:
            data = pd.read_sql(sql, conn)
        if not data.empty:
            data['date'] = pd.to_datetime(data['date'])
            data = data.set_index('date')
            return data

    def create_table(self) -> None:
        sql = '''CREATE TABLE IF NOT EXISTS 'trade' (
                    symbol SYMBOL INDEX,
                    open DOUBLE,
                    high DOUBLE,
                    low DOUBLE,
                    close DOUBLE,
                    amount DOUBLE,
                    volume DOUBLE,
                    date TIMESTAMP);
                '''
        self.sql_exec(sql)

class AdjFactor(QuestBase):
    def bulk_insert(self, items: List) -> None:
        sql = '''INSERT INTO 'adj_factor' (symbol, factor, date, factor_type)
                 VALUES %s;'''
        with psycopg2.connect(self.dsn) as conn:
            with conn.cursor() as cursor:
                execute_values(cursor, sql, items)

    def create_table(self):
        sql = '''CREATE TABLE IF NOT EXISTS 'adj_factor' (
                factor_type SYMBOL,
                symbol SYMBOL INDEX,
                factor STRING,
                date TIMESTAMP);'''
        self.sql_exec(sql)

    def recreate_table(self):
        sql = '''
        DROP TABLE 'adj_factor';
        '''
        self.sql_exec(sql)
        self.create_table()

    def format_sql(self, symbol, dict):
        def gen_date(date_str):
            return datetime.strptime(date_str, '%Y-%m-%d').isoformat()
        l = []

        for qfq in dict['qfq']:
            l.append(tuple([symbol, qfq['f'], gen_date(qfq['d']), 'qfq']))
        for hfq in dict['hfq']:
            l.append(tuple([symbol, hfq['f'], gen_date(hfq['d']), 'hfq']))
        return l

    def query(self, symbol, factor_type):
        sql = '''
              SELECT * FROM 'adj_factor'
              WHERE symbol = '{symbol}'
              AND factor_type = '{factor_type}';
              '''.format(symbol=symbol, factor_type=factor_type)

        with psycopg2.connect(self.dsn) as conn:
            data = pd.read_sql(sql, conn)
        if not data.empty:
            data['date'] = pd.to_datetime(data['date'])
            data = data.set_index('date')
            return data

sd = StockDay()
af = AdjFactor()


import openai
import pandas as pd
import argparse
import json
from collections import Counter
from tqdm import tqdm
import backoff
from openai.error import APIError, Timeout, APIConnectionError
import jsonlines
import random

@backoff.on_exception(backoff.expo, openai.error.RateLimitError)
def completions_with_backoff(**kwargs):
    return openai.ChatCompletion.create(**kwargs)

KNOWLEDGE_INSTRUCTIONS = {"nq": "Please answer the following questions using the shortest possible response. For example, if the question asks 'What is the capital of France?'', you can simply reply with 'Paris'.",
                          "fever": "Determine whether the following statement is true or false.",
                          "wow": "You have been provided with a chat history between two agents, separated by new lines. Generate a response that is informative and engaging based on the latest message."}

PROMPT_DICT = {
    "multi": (
        "You'll be provided with an instruction, along with evidence and possibly some preceding sentences. "
        "When there are preceding sentences, your focus should be on the sentence that comes after them. "
        "Your job is to determine if the evidence is relevant to the initial instruction and the preceding context, and provides useful information to complete the task described in the instruction. "
        "If the evidence meets this requirement, respond with [Relevant]; otherwise, generate [Irrelevant].\n\n"
        "###\nInstruction: Given four answer options, A, B, C, and D, choose the best answer.\n\n"
        "Input: Earth rotating causes\n"
        "A: the cycling of AM and PM\nB: the creation of volcanic eruptions\nC: the cycling of the tides\nD: the creation of gravity\n\n"
        "Evidence: Rotation causes the day-night cycle which also creates a corresponding cycle of temperature and humidity creates a corresponding cycle of temperature and humidity. Sea level rises and falls twice a day as the earth rotates.\n\n"
        "Rating: [Relevant]\n"
        "Explanation: The evidence explicitly mentions that the rotation causes a day-night cycle, as described in the answer option A.\n\n"
        "###\nInstruction: age to run for us house of representatives\n\n"
        "Evidence: The Constitution sets three qualifications for service in the U.S. Senate: age (at least thirty years of age); U.S. citizenship (at least nine years); and residency in the state a senator represents at the time of election.\n\n"
        "Rating: [Irrelevant]\n"
        "Explanation: The evidence only discusses the ages to run for the US Senate, not for the House of Representatives.\n\n"
        "###\nInstruction: {instruction}\n\n"
        "Evidence: {evidence}\n\n"
        "Rating:"
    ),
    "multi_no_preceding": (
        "You'll be provided with an instruction, along with evidence and possibly some preceding sentences. "
        "When there are preceding sentences, your focus should be on the sentence that comes after them. "
        "Your job is to determine if the evidence is relevant to the initial instruction and the preceding context, and provides useful information to complete the task described in the instruction. "
        "If the evidence meets this requirement, respond with [Relevant]; otherwise, generate [Irrelevant].\n\n"
        "###\nInstruction: Given four answer options, A, B, C, and D, choose the best answer.\n\n"
        "Input: Earth rotating causes\n"
        "A: the cycling of AM and PM\nB: the creation of volcanic eruptions\nC: the cycling of the tides\nD: the creation of gravity\n\n"
        "Evidence: Rotation causes the day-night cycle which also creates a corresponding cycle of temperature and humidity creates a corresponding cycle of temperature and humidity. Sea level rises and falls twice a day as the earth rotates.\n\n"
        "Rating: [Relevant]\n"
        "Explanation: The evidence explicitly mentions that the rotation causes a day-night cycle, as described in the answer option A.\n\n"
        "###\nInstruction: Describe a leader or a politician whom you admire. \n\n"
        "Preceding sentences: Leaders and politicians have the power to shape the course of history and impact the lives of countless individuals. Among the myriad of notable figures, Nelson Mandela stands as an exemplary leader whose indomitable spirit, unwavering commitment to justice, and remarkable ability to unite a divided nation have made him an admired and revered personality on a global scale. "
        "Evidence: Barack Obama was one of the most influential people of the world and the man with a difference. He has served as the President of the United States of America. He was the 44th President of America. He was elected in the year 2009 to the office of the President. He was the first-ever African-American President of America.\n\n"
        "Rating: [Irrelevant]\n"
        "Explanation: While the evidence discuss Barack Obama, who is known as an influential political leader, the preceding sentences describe Nelson Mandela, so this evidence doesn't provide useful information to generate an helpful continuation.\n\n"
        "###\nInstruction: {instruction}\n\n"
        "Evidence: {evidence}\n\n"
        "Rating:"
    ),
}

def load_jsonlines(file):
    with jsonlines.open(file, 'r') as jsonl_f:
        lst = [obj for obj in jsonl_f]
    return lst

def postprocess(results):
    raw_output = results["choices"][0]["message"]["content"]
    print(raw_output)
    if "\nExplanation:" in raw_output:
        explanation = raw_output.split("\nExplanation:")[1]
        if explanation[0] == " ":
            explanation = explanation[1:]
        score_string = raw_output.split("\nExplanation:")[0]
        score = None
        for i in range(1, 6):
            if str(i) in score_string:
                score = int(i)
        if score is None:
            return "", explanation
        else:
            return score, explanation
    else:
        return "", ""

def process_input(example, multi_retrieval=False):
    if multi_retrieval is False:
        return PROMPT_DICT["context"].format_map(example)
    else:
        if "sent_idx" not in example or example["sent_idx"] == 0 or len(example["preceding_sentences"]) == 0:
            return PROMPT_DICT["multi_no_preceding"].format_map(example)
        else:
            return PROMPT_DICT["multi"].format_map(example)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--input_files', type=str, nargs='+')
    parser.add_argument('--output_file_name', type=str)
    parser.add_argument('--multi_retrieval', action="store_true")
    parser.add_argument('--api_key', type=str)
    parser.add_argument('--org_name', type=str)
    parser.add_argument('--model_name', type=str, default="gpt-3.5-turbo")
    parser.add_argument('--n', type=int, default=None)
    args = parser.parse_args()

    with open(args.api_key) as f:
        openai.api_key = f.read()[:-1]
    openai.organization = args.org_name

    examples = []
    for input_file in args.input_files:
        if input_file.endswith(".json"):
            examples += json.load(open(input_file))
        else:
            examples += load_jsonlines(input_file)

    result_list = []
    if args.n is not None and len(examples) > args.n:
        examples = random.sample(examples, k=args.n)

    task_types = Counter([item["dataset_name"]
                         for item in examples if "dataset_name" in item])

    print(Counter(task_types))

    for idx, example in tqdm(enumerate(examples)):
        if "output" not in example and "answers" in example:
            example["output"] = example["answers"][0] if type(
                example["answers"]) is list else example["answers"]
        if "target_output" not in example and "output" in example:
            example["target_output"] = example["output"]
        if "instruction" not in example and "question" in example:
            data_type = example["q_id"].split("_")[0]
            if data_type in KNOWLEDGE_INSTRUCTIONS:
                example["instruction"] = KNOWLEDGE_INSTRUCTIONS[data_type] + \
                    example["question"]
            else:
                example["instruction"] = example["question"]
        if "As a language model, I cannot" in example["output"]:
            continue
        input = process_input(example, multi_retrieval=args.multi_retrieval)
        try:
            results = completions_with_backoff(
                model=args.model_name,
                messages=[
                    {"role": "user",
                        "content": input},
                ],
                request_timeout=60,
                max_tokens=200,
            )
            score, explanation = postprocess(results)
            result_list.append({"input": example, "score": score, "explanation": score,
                               "raw_output": results["choices"][0]["message"]["content"]})
            if idx % 20 == 0:
                print("Input: {}".format(example["instruction"]))
                print("Output: {}".format(example["output"]))
                print("Evidence: {}".format(example["evidence"]))
                print("Score: {0} ({1})".format(score, explanation))

        except (APIError, Timeout, APIConnectionError):
            results = "ERROR: API error outputs"
        if idx % 20 == 0:
            print("saved output at {}".format(args.output_file_name + "_tmp"))
            with open(args.output_file_name + "_tmp", "w") as outfile:
                json.dump(result_list, outfile)

    with open(args.output_file_name, "w") as outfile:
        json.dump(result_list, outfile)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3

#
# Usage: see lc0_graph_activations.py --help
#

import chess
import chess.pgn
import chess.uci
import chess.svg
import re
import matplotlib.pyplot as plt
import matplotlib.axes
import pandas as pd
import numpy as np
import os
import math
import argparse
from collections import OrderedDict
import svgutils.transform as sg
from svgutils.compose import *
import sys
#import cairosvg

COLS = 32
ROWS = 8

DIR_VECTORS = {'N': (0, 1), 'NE': (1, 1), 'E': (1, 0), 'SE': (1, -1),
        'S':(0, -1), 'SW':(-1, -1), 'W': (-1, 0), 'NW': (-1, 1)}

KNIGHT_DIR_VECTORS = {'N': (1, 2), 'NE': (2, 1), 'E': (2, -1), 'SE': (1, -2),
        'S':(-1, -2), 'SW':(-2, -1), 'W': (-2, 1), 'NW': (-1, 2)}

LOCMAP = []

for direction in ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']:
    for steps in range(1, 8):
        x, y = DIR_VECTORS[direction]
        x *= steps
        y *= steps
        #print(len(LOCMAP), "normal", direction, x, y)
        LOCMAP.append((x, y))

for direction in ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']:
    x, y = KNIGHT_DIR_VECTORS[direction]
    #print(len(LOCMAP), "knight", direction, x, y)
    LOCMAP.append((x,y))

LOCMAP.extend([(-5,8),(-4,8),(-3,8), (-1,8),(0,8),(1,8), (3,8),(4,8),(5,8)])

def plot(data, layernum, board):
    if len(data) == 80:
        fig, ax = plt.subplots(16, 15, figsize=(15*2,16*2), subplot_kw={'title':"", 'xticks':[], 'yticks':[]}, frameon=False)
        for e, d in enumerate(data):
            if e >= len(LOCMAP): break
            # Data in logfile has side to move going down the board,
            # so flip vertically.
            d = np.flip(d, 0)
            x, y = LOCMAP[e]
            x += 7
            y = -y + 8
            #ax[y, x].imshow(d, norm=matplotlib.colors.SymLogNorm(linthresh=0.03, linscale=0.03, vmin=-50, vmax=50))
            ax[y, x].imshow(d, norm=matplotlib.colors.SymLogNorm(linthresh=0.03, linscale=0.03, vmin=-1, vmax=1))
            #ax[y, x].imshow(d, vmin=0, vmax=1)
            #ax[y, x].imshow(d, norm=matplotlib.colors.Normalize(vmin=0., vmax=1.))
            ax[y, x].get_xaxis().set_visible(False)
            ax[y, x].get_yaxis().set_visible(False)
        #plt.tight_layout(w_pad=0.01, h_pad=0.01)
        filename = "%s/activation_layer_%03d.png" % (args.savedir, layernum)
        plt.savefig(filename, transparent=True)
        plt.close("all")
        print("saved " + filename)
    else:
        fig, ax = plt.subplots(ROWS, COLS, figsize=(32,8), subplot_kw={'title':"", 'xticks':[], 'yticks':[]}, frameon=False)
        for e, d in enumerate(data):
            # Data in logfile has side to move going down the board,
            # so flip vertically.
            d = np.flip(d, 0)
            #ax[e//COLS, e%COLS].imshow(d, norm=matplotlib.colors.SymLogNorm(linthresh=0.03, linscale=0.03, vmin=-50, vmax=50))
            ax[e//COLS, e%COLS].imshow(d, norm=matplotlib.colors.SymLogNorm(linthresh=0.03, linscale=0.03, vmin=-1, vmax=1))
            #ax[e//COLS, e%COLS].imshow(d, vmin=0, vmax=1)
            ax[e//COLS, e%COLS].get_xaxis().set_visible(False)
            ax[e//COLS, e%COLS].get_yaxis().set_visible(False)
        plt.tight_layout(w_pad=0.01, h_pad=0.01)
        filename = "%s/activation_layer_%03d.png" % (args.savedir, layernum)
        plt.savefig(filename, transparent=True)
        plt.close("all")
        print("saved " + filename)

def parse():
    alldata = []
    layernum = 0
    board = chess.Board()
    if not os.path.exists(args.savedir):
        os.makedirs(args.savedir)
    for line in open(args.logfile).readlines():
        m = re.search(">> position startpos", line)
        if m:
            m = re.search(">> (position startpos moves .*)", line)
            if m:
                ucipos = m.group(1)
            else:
                ucipos = "position startpos"
            f = open("%s/activation_game.txt" % (args.savedir), "w")
            for m in ucipos.split()[3:]:
                board.push_uci(m)
            f.write(str(chess.pgn.Game.from_board(board)) + "\n")
            f.write(str(ucipos) + "\n")
            f.write(str(board.fen()) + "\n")
            f.write(str(board) + "\n")
            f.close()
        data = line.split()
        if len(data) == 0 or not data[0].startswith("channel:"): continue
        _,channel_num = data[0].split(":")
        channel_num = int(channel_num)
        if channel_num == 0 and len(alldata) > 0:
            plot(alldata, layernum, board)
            alldata = []
            layernum += 1
        data = [float(x) for x in data[1:]]
        alldata.append(np.array(data).reshape((8,8)))
    plot(alldata, layernum, board)

if __name__ == "__main__":
    usage_str = """
lc0_graph_activations.py lc0log_mypos.txt activations_mypos

This script parses a logfile generated using a special lc0 binary that dumps
activation data of the NN. It outputs a text file of the position, and images
of the activations for every layer of the NN.

To generate a logfile:
* `git clone -b nndebug http://github.com/killerducky/lc0`
* Build lc0
* Run lc0 with Backend=blas, and LogFile=lc0log_mypos.txt
* setposition you want to analyze
  * currently script assumes there is only 1 position!
  * fens won't work right now, only full setposition
* go nodes 1 (it's important the NN only analyzes one position)
* quit lc0
* Now you can run this script on lc0log_mypos.txt

Side to move is always going up in these images.

Output files generated:

activation_game.txt - pgn, uci, fen, and ascii board
activation_layer_000.png - Inputs:
    For 8 plies of hisory, most recent first:
        Our     P, N, B, R, Q, K
        Their   P, N, B, R, Q, K
        Repetition plane
    We can O-O-O
    We can O-O
    They can O-O-O
    They can O-O
    We are Black
    50 move rule counter
    Unused (used to be move count)
    Plane of 1s
activation_layer_001.png - Output of input convolution
activation_layer_002.png - Output of 1st resblock, first half
activation_layer_003.png - Output of 1st resblock, second half
...
activation_layer_040.png - Output of 20th resblock, second half
activation_layer_041.png - Output of 20th resblock, second half
activation_layer_042.png - Output of Policy FC layer (T50 convolutional Policy will have two convolves instead)
(Final Policy move selection not supported yet)
"""

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=usage_str)
    parser.add_argument("logfile", type=str, help="logfile to parse")
    parser.add_argument("savedir", type=str, help="directory to put files")

    args = parser.parse_args()
    print(args.logfile)
    parse()


# -*- coding: utf-8 -*-

import numpy as np
from matplotlib import pyplot as plt
from matplotlib import animation

class Parameter:
    def __init__(self, *args, **kwargs):

        if args and isinstance(args[0],Parameter):
            xb = args[0].xb
            xe = args[0].xe
            N = args[0].N
            dt = args[0].dt
            kappa = args[0].kappa

        elif kwargs :
            xb = kwargs['xb']
            xe = kwargs['xe']
            N = kwargs['N']
            dt = kwargs['dt']
            kappa = kwargs['kappa']

        self.xb = xb
        self.xe = xe
        self.N = N
        self.dx = (self.xe-self.xb)/float(self.N)
        self.dt = dt
        self.kappa = kappa
        self.cfl=1

class PDE(Parameter):
    """
    solves: u_t = \kappa \Delta u
    """
    def __init__(self, p ,t0 = 0.0):
        Parameter.__init__(self,p)
        self.kx = np.linspace(0,self.N/2,self.N/2+1)*2*np.pi/(self.xe-self.xb)
        self.kx2 = np.square(self.kx)

        self.x = self.xb + self.dx * np.arange(self.N)
        self.u = np.sin(self.x)
        self.uhat = np.fft.rfft(self.u)
        print(len(self.uhat))
        #self.u = np.piecewise(self.x, [np.abs(self.x-np.pi) < 0.01], [1])
        #self.u = np.sign(np.cos(self.x))
        self.t = t0

        self.scheme = self.heun

        # attributes used for dynamic plotting
        #  self.x_line = None
        #  self.u_line = None

    def time_step(self, Nsteps = 1):
        for i in range(Nsteps):
            self.scheme()
        self.u = np.fft.irfft(self.uhat)
        #print self.t, np.max(-np.fft.irfft(self.kx*1j*self.uhat))
        self.cfl = np.max(self.u)*self.dt/self.dx
        #print "cfl =",self.cfl
        self.t += self.dt * Nsteps

    def rhs(self, uhat):
        # aliasing
        uhat[int(2.*len(self.kx)/3.):len(self.kx)] = 0
        return  uhat - 0.5 * self.kx * 1j * np.fft.rfft(np.fft.irfft(uhat)**2)*self.dt

    def prop(self, delta):
        return np.exp(-self.kappa*self.dt*self.kx2*delta)

    def euler(self):
        self.uhat = self.rhs(self.uhat)*self.prop(1.)

    def heun(self):
        uone = self.rhs(self.uhat)*self.prop(1.)
        self.uhat = 0.5*(self.uhat*self.prop(1.) + self.rhs(uone))

    def shuOsher(self):
        uone =  self.rhs(self.uhat)*self.prop(1.)
        utwo = 3./4.*self.uhat*self.prop(0.5) + 1./4.*self.rhs(uone)*self.prop(-0.5)
        self.uhat = 1./3.*self.uhat*self.prop(1.) + 2./3.*self.rhs(utwo)*self.prop(0.5)

p = Parameter(xb=0,xe=2*np.pi,N=256,dt=0.01,kappa=0.02)

N_steps = 10
t_max = 100
frames = int(t_max / float(N_steps * p.dt))
frames = 500

pde = PDE(p)

def step():
    pde.time_step(N_steps)
    return np.array([pde.x, pde.u])

res = np.array([step() for _ in range(frames)])

######################################################################
# Set up plot
fig = plt.figure()
ax = plt.axes(xlim=(p.xb,p.xe), ylim=(-1.2, 1.2))
u_line, = ax.plot([], [],lw=2,marker='.',markerfacecolor='red', markersize=12)
time_text = ax.text(0.02, 0.95, '', transform=ax.transAxes)
cfl_text = ax.text(0.02, 0.90, '', transform=ax.transAxes)
#ax.legend(prop=dict(size=14))
ax.set_xlabel('$x$')
ax.set_ylabel('$u$')

######################################################################
# Animate plot
def init():
    u_line.set_data([], [])

    time_text.set_text('')
    cfl_text.set_text('')
    return (u_line, time_text,cfl_text)

def integrate(i):
    #  pde.time_step(N_steps)
    u_line.set_data(res[i,0], res[i,1])

    #  time_text.set_text('time = %.2f' % pde.t)
    #  cfl_text.set_text('cfl = %.2f' % pde.cfl)
    #  return (u_line, time_text,cfl_text)
    return u_line,

anim = animation.FuncAnimation(fig, integrate, init_func=init, frames=frames,
                               interval=100, blit=True,repeat=False)

plt.show()