from ArrayQueue import *
import ctypes  # provides low-level arrays

# q1

class MeanQueue:
    def __init__(self):
        self.data = ArrayQueue()
        self.total = 0

    def __len__(self):
        '''Return the number of elements in the queue'''
        return len(self.data)

    def is_empty(self):
        ''' Return True if queue is empty'''
        return self.data.is_empty()

    def enqueue(self, e):
        ''' Add element e to the front of the queue. If e is not an int or float, raise a TypeError '''
        if isinstance(e, (int, float)):
            self.data.enqueue(e)
            self.total += e
        else:
            raise TypeError("Invalid type")

    def dequeue(self):
        ''' Remove and return the first element from the queue. If the queue is empty, raise an exception'''
        if self.is_empty():
            raise Exception()
        else:
            dequeued = self.data.dequeue()
            self.total -= dequeued
            return dequeued

    def first(self):
        ''' Return a reference to the first element of the queue without removing it. If the queue is empty, raise an exception '''
        if self.is_empty():
            raise Exception()
        else:
            return self.data.first()

    def sum(self):
        ''' Returns the sum of all values in the queue'''
        return self.total

    def mean(self):
        ''' Return the mean (average) value in the queue'''
        return self.total / len(self.data)

# q2

def flatten_list_by_depth(lst):
    """
    : lst type: list
    : return type: list
    """
    q = ArrayQueue()
    new_lst = []
    ''' Remove and return the top element from the stack. If the
    stack is empty, raise an exception'''
    for i in lst:
        q.enqueue(i)
    while not q.is_empty():
        if isinstance(q.first(), list):
            total = len(q) - 1
            for i in q.dequeue():
                q.enqueue(i)
            for _ in range(total):
                q.enqueue(q.dequeue())
        else:
            new_lst.append(q.dequeue())
    return new_lst

# q3

def make_array(n):
    return (n * ctypes.py_object)()

class ArrayDeque:
    INITIAL_CAPACITY = 8

    def __init__(self):
        self.data_arr = make_array(ArrayQueue.INITIAL_CAPACITY)
        self.capacity = ArrayQueue.INITIAL_CAPACITY
        self.n = 0
        self.front_ind = None
        self.back_ind = None

    def __len__(self):
        return self.n

    def is_empty(self):
        return (len(self) == 0)

    def enqueue_last(self, elem):
        if (self.n == self.capacity):
            self.resize(2 * self.capacity)
        if (self.is_empty()):
            self.data_arr[2] = elem
            self.front_ind = 2
            self.back_ind = 2
            self.n += 1
        else:
            self.back_ind = (self.front_ind + self.n) % self.capacity
            self.data_arr[self.back_ind] = elem
            self.n += 1

    def enqueue_first(self, elem):
        if (self.n == self.capacity):
            self.resize(2 * self.capacity)
        if self.is_empty():
            self.data_arr[2] = elem
            self.front_ind = 2
            self.back_ind = 2
            self.n += 1
        else:
            self.front_ind = (self.back_ind - self.n) % self.capacity
            self.data_arr[self.front_ind] = elem
            self.n += 1

    def dequeue_first(self):
        if (self.is_empty()):
            raise Exception("Queue is empty")
        value = self.data_arr[self.front_ind]
        self.data_arr[self.front_ind] = None
        self.front_ind = (self.front_ind + 1) % self.capacity
        self.n -= 1
        if(self.is_empty()):
            self.front_ind = None
            self.back_ind = None
        if((self.n < self.capacity // 4) and
                (self.capacity > ArrayQueue.INITIAL_CAPACITY)):
            self.resize(self.capacity // 2)
        return value

    def dequeue_last(self):
        if self.is_empty():
            raise Exception("Queue is empty")
        value = self.data_arr[self.back_ind]
        self.data_arr[self.back_ind] = None
        self.back_ind = (self.back_ind - 1) % self.capacity
        self.n -= 1
        if self.is_empty():
            self.front_ind = None
            self.back_ind = None
        if((self.n < self.capacity // 4) and
                (self.capacity > ArrayQueue.INITIAL_CAPACITY)):
            self.resize(self.capacity // 2)
        return value

    def first(self):
        if self.is_empty():
            raise Exception("Queue is empty")
        return self.data_arr[self.front_ind]

    def resize(self, new_cap):
        new_data = make_array(new_cap)
        old_ind = self.front_ind
        offset = (new_cap - len(self.data_arr)) // 2
        for new_ind in range(self.n):
            new_data[new_ind + offset] = self.data_arr[old_ind]
            old_ind = (old_ind + 1) % self.capacity
        self.data_arr = new_data
        self.capacity = new_cap
        self.front_ind = 0

# q4

class QueueStack_fastpush:
    def __init__(self):
        self.data = ArrayQueue()
        self.first_in_front = False

    def __len__(self):
        return len(self.data)

    def is_empty(self):
        return len(self) == 0

    def push(self, e):
        ''' Add element e to the top of the stack '''
        self.first_in_front = False
        self.data.enqueue(e)

    def pop(self):
        ''' Remove and return the top element from the stack. If the stack is empty, raise an exception'''
        if not self.first_in_front:
            for _ in range(len(self.data) - 1):
                self.data.enqueue(self.data.dequeue())
        self.first_in_front = False
        return self.data.dequeue()

    def top(self):
        ''' Return a reference to the top element of the stack without removing it. If the stack is empty, raise an exception '''
        if not self.first_in_front:
            for _ in range(len(self.data) - 1):
                self.data.enqueue(self.data.dequeue())
        self.first_in_front = True
        return self.data.first()

class QueueStack_fastpoptop:
    def __init__(self):
        self.data = ArrayQueue()
        self.first_in_front = False

    def __len__(self):
        return len(self.data)

    def is_empty(self):
        return len(self) == 0

    def push(self, e):
        ''' Add element e to the top of the stack '''
        self.data.enqueue(e)
        for _ in range(len(self.data) - 1):
            self.data.enqueue(self.data.dequeue())

    def pop(self):
        ''' Remove and return the top element from the stack. If the stack is empty, raise an exception'''
        return self.data.dequeue()

    def top(self):
        ''' Return a reference to the top element of the stack without removing it. If the stack is empty, raise an exception '''
        return self.data.first()

if __name__ == "__main__":
    lst = [ [[[0]]], [1, 2], 3, [4, [5, 6, [7]], 8], 9]
    new_lst = flatten_list_by_depth(lst)
    print(new_lst)
    print()

    dq = ArrayDeque()
    dq.enqueue_first(0)
    dq.enqueue_first(1)
    dq.enqueue_last(2)
    dq.enqueue_last(3)
    print(dq.dequeue_first())
    print(dq.dequeue_first())
    dq.enqueue_first(11)
    dq.enqueue_first(12)
    dq.enqueue_first(13)
    dq.enqueue_first(14)
    print(dq.dequeue_first())
    print(dq.dequeue_first())
    print(dq.dequeue_first())
    print(dq.dequeue_first())
    print(dq.dequeue_last())
    dq.enqueue_last(15)
    dq.enqueue_last(16)
    print(dq.dequeue_last())
    print(dq.dequeue_last())

    print()

    qs = QueueStack_fastpush()
    qs.push(1)
    qs.push(2)
    qs.push(3)
    qs.push(4)
    qs.push(5)
    qs.push(6)
    qs.push(7)
    print(qs.top())
    print(qs.pop())
    qs.push(8)
    qs.push(9)
    print(qs.top())
    print(qs.pop())
    print(qs.pop())
    print(qs.pop())
    print(qs.pop())
    print(qs.pop())

    print()

    qs2 = QueueStack_fastpoptop()
    qs2.push(1)
    qs2.push(2)
    qs2.push(3)
    qs2.push(4)
    qs2.push(5)
    qs2.push(6)
    qs2.push(7)
    print(qs2.data.first())
    print(qs2.top())
    print(qs2.pop())
    print(qs2.pop())
    print(qs2.pop())
    print(qs2.pop())


#!/usr/bin/env python3

import argparse
from datetime import datetime
import sys
import pandas

def parseArgs(args):
    parser = argparse.ArgumentParser()
    parser.add_argument('file', type=str,
            help='File to process.')
    parser.add_argument('--event', type=str,
            help='File to process.')
    parser.add_argument('-b', '--begin', type=str,
            help='Start timestamp')
    parser.add_argument('-e', '--end', type=str,
            help='End timestamp')
    return parser.parse_args(args)

def main(args):
    progress_trace = pandas.read_csv(args.file,
                              converters={'TIMESTAMP': datetime.fromisoformat}
                                     )

    have_trace_id = 'TRACE ID' in progress_trace.keys()

    sync_from_events = progress_trace[
            (progress_trace['DATASTORE'] == 'running') &
            ((progress_trace['EVENT TYPE'] == 'start') |
             (progress_trace['EVENT TYPE'] == 'stop'))
    ]
    if args.event:
        sync_from_events = sync_from_events[
            sync_from_events['MESSAGE'] == args.event
        ]
    if args.begin:
        begin = datetime.fromisoformat(args.begin)
        sync_from_events = sync_from_events[
            sync_from_events['TIMESTAMP'] >= begin
        ]
    if args.end:
        end = datetime.fromisoformat(args.end)
        sync_from_events = sync_from_events[
            sync_from_events['TIMESTAMP'] <= end
        ]

    tids = []
    tids_msgs = []
    for name, fields in sync_from_events.iterrows():
        evt = fields['EVENT TYPE']
        tid = fields['TRANSACTION ID']
        msg = fields['MESSAGE']
        ann = fields['ANNOTATION']
        if evt == 'start':
            if tid not in tids:
                tids.append(tid)
                tids_msgs.append((tid, msg))
        elif evt == 'stop':
            if (tid, msg) in tids_msgs:
                tids_msgs.remove((tid, msg))
                tids.remove(tid)
                print(f"{name}  {msg:15} {tid} {ann}")

if __name__ == '__main__':
    main(parseArgs(sys.argv[1:]))


import os
import logging
import time
import pymongo
from random import choice
MONGO_DB = pymongo.MongoClient(host='172.16.75.38',port=27017)
STATIC_IP = MONGO_DB["IP"]["STATIC_IP"]
def logger(FILE_NAME):
    """
    日志配置
    :param FILE_NAME: 日志文件名(全路径 )
    :return:日志记录生成器
    """
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
        datefmt='%Y %H:%M:%S',
        filename=FILE_NAME,
        filemode='w'
    )
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('[%(asctime)s] %(filename)s[Line:%(lineno)d] [%(levelname)s] %(message)s')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)
    return logging

def get_log():
    real_path = os.path.dirname(os.path.realpath(__file__)).replace('\\', '/') + "/log/"
    file_name = "{}_爬虫程序_{}.log".format(real_path,
                                             time.strftime("%Y-%m-%d",
                                                           time.localtime()))
    log = logger(file_name)
    return log

def ABY():
    try:
        IP = STATIC_IP.find()

        ip_lt = choice([_ for _ in IP])
        ip = ip_lt["ip_parmas"]
        _id = ip_lt["_id"]
        proxies = {
            "http": "http://" + ip,
            "https": "https://" + ip,
        }
        return proxies,_id

    except Exception as e:
        print(e)
        return None


from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
import tensorflow.keras.backend as K
import os, random, math
import numpy as np
from agent.prioritized_memory import Memory
from agent.model import Build_model
from tensorflow.compat.v1.train import get_or_create_global_step
from tensorflow.keras.utils import Progbar

config = tf.compat.v1.ConfigProto()
config.intra_op_parallelism_threads = 44
config.inter_op_parallelism_threads = 44
tf.compat.v1.Session(config=config)

class Agent:
	def __init__(self, ticker, state_size, neurons, m_path, is_eval=False):
		self.state_size = state_size # normalized previous days
		self.action_size = 3
		self.neurons = neurons
		self.memory_size = 20000 #記憶長度
		self.memory = Memory(self.memory_size)
		self.epsilon = 0.3
		self.epsilon_min = 0.05
		self.epsilon_decay = 0.995
		self.gamma = 0.95
		self.batch_size = 128
		self.epoch_loss_avg = tf.keras.metrics.Mean()
		self.epochs = 1
		self.bar = Progbar(self.epochs)
		self.is_eval = is_eval
		self.checkpoint_path = m_path
		self.checkpoint_dir = os.path.dirname(self.checkpoint_path)
		self.check_index = self.checkpoint_path + '.index'   #checkpoint裡面的檔案多加了一個.index
		if is_eval==False:
			self.training = True
			self.model = self._model('  Model')
			self.target_model = self._model(' Target')
		else:
			self.training = False
			self.model = self._model('  Model')
		self.optimizer = tf.optimizers.Adam(learning_rate=0.0000625, epsilon=0.00015)
		self.loss_function = tf.keras.losses.Huber()

	def _model(self, model_name):
		ddqn = Build_model()
		model = ddqn.build_model(self.state_size, self.neurons, self.action_size, self.training)
		if os.path.exists(self.check_index):
			#如果已經有訓練過，就接著load權重
			print('-'*52+'{} Weights loaded!!'.format(model_name)+'-'*52)
			model.load_weights(self.checkpoint_path)
		else:
			print('-'*53+'Create new model!!'+'-'*53)
		return model

	# 把model的權重傳給target model
	def update_target_model(self):
		self.target_model.set_weights(self.model.get_weights())

	def act(self, state):
		if not self.is_eval and np.random.rand() <= self.epsilon:
			return random.randrange(self.action_size)
		options = self.model(state)
		options = options.numpy()
		action_out = np.argmax(options[0])  # array裡面最大值的位置號
		return action_out

	# Prioritized experience replay
	# save sample (error,<s,a,r,s'>) to the replay memory
	def append_sample(self, state, action, reward, next_state, done):
		if reward != 0:
			max_p = 100 + reward  # 有reward的記憶給大的priority
		else:
			max_p = 10  # 預設給10
		self.memory.add(max_p, (state, action, reward, next_state, done))  # set the max p for new p

	# loss function
	def _loss(self, model, x, y):
		y_ = self.model(x)
		return self.loss_function(y_true=y, y_pred=y_)

	# gradient
	def _grad(self, model, inputs, targets):
		with tf.GradientTape() as tape:
			loss_value = self._loss(self.model, inputs, targets)
		return loss_value, tape.gradient(loss_value, self.model.trainable_variables)

	def train_model(self):
		# pick samples from prioritized replay memory (with batch_size)
		mini_batch, idxs, is_weights = self.memory.sample(self.batch_size)
		state_inputs = np.zeros((self.batch_size,self.state_size[0],self.state_size[1]))
		next_states = np.zeros((self.batch_size,self.state_size[0],self.state_size[1]))
		action, reward, done = [], [], []

		for i in range(self.batch_size):
			state_inputs[i] = mini_batch[i][0][0]
			action.append(mini_batch[i][1])
			reward.append(mini_batch[i][2])
			next_states[i][:][:] = mini_batch[i][3][0]
			done.append(mini_batch[i][4])
		old_q = []
		# 主model動作
		q_result = self.model(state_inputs)
		q_result = q_result.numpy()
		# 下一個state
		q_next_result = self.model(next_states)
		q_next_result = q_next_result.numpy()
		next_action = np.argmax(q_next_result, axis=1)
		# target model動作
		q_t_next_result = self.target_model(next_states)
		q_t_next_result = q_t_next_result.numpy()
		# 更新Q值: Double DQN的概念
		for i in range(self.batch_size):
			old_q.append(q_result[i][action[i]])
			q_result[i][action[i]] = reward[i]
			if not done[i]:
				q_result[i][action[i]] +=  self.gamma * q_t_next_result[i][next_action[i]]
			# 計算error給PER
			error = abs(old_q[i] - q_result[i][action[i]])
			error *= is_weights[i]
			self.memory.update(idxs[i], error)

		# train model
		for i in range(self.epochs):
			loss_value, grads = self._grad(self.model, state_inputs, q_result)
			self.optimizer.apply_gradients(zip(grads, self.model.trainable_variables),
				get_or_create_global_step())
			self.epoch_loss_avg(loss_value)


from dotenv import load_dotenv, dotenv_values
import pathlib
from pathlib import Path

load_dotenv()
config = dotenv_values("secret.env")

tk = config['OPENAI_API_KEY']
TOKEN = config['VK_TOKEN']

GROUP_ID = 220253752
FLOOD_DELAY = 10

IMAGE_KW = '/image'

HI_MESSAGE = "Добро пожаловать, {}!\nЧтобы начать работу введите запрос, и искусственный интелект вам ответит"
COMPLETE_ERROR_MSG = 'Ошибка при получении данных с https://chat.openai.com\nПовторите попытку позже'
IMAGE_ERROR_MSG = 'Невозможно сгенерировать картинку по данному запросу'
GENERATING_MSG = "Ответ генерируется, подождите..."
FLOOD_WARN = "Не флуди"
END_DIALOG_BTN = 'Завершить предыдущий диалог'
DIALOG_ENDED = 'Диалог успешно завершён, можете начинать новый!'
IMAGE_SENT = 'Вот ваше изображение!'

audio_path = Path(pathlib.Path.cwd(), 'audio')
db_path = Path(pathlib.Path.cwd(), 'db', 'conv.sqlite')
images_path = Path(pathlib.Path.cwd(), 'images')


#!/usr/bin/env python3

import argparse
import calendar
import getpass
import happybase
import logging
import random
import sys
from collections import Counter

logging.basicConfig(level="DEBUG")

HOSTS = ["mipt-node0%d.atp-fivt.org" % i for i in range(1, 5)]

USAGE = """
Format of query:
  $ {0} --table_name (e.g. s201723Table) --period_from (e.g. 2014-09-01) --period_to (e.g. 2014-09-10)
""".format(sys.argv[0])

def connect(table_name):
    host = random.choice(HOSTS)
    conn = happybase.Connection(host)

    logging.debug("Connecting to HBase Thrift Server on %s", host)
    conn.open()

    logging.debug("Using table %s", table_name)
    return happybase.Table(table_name, conn)

def parse_date(date):
    year, month, day = date.split('-')
    return int(year), int(month), int(day)

def get_period(period_from, period_to):
    year_from, month_from, day_from = parse_date(period_from)
    year_to, month_to, day_to = parse_date(period_to)
    def get_row_name(year, month):
        return "%04d%02d" % (year, month)

    if year_from != year_to or month_from != month_to:
        start_days = ['col_day%02d' % i for i in range(day_from, 32)]
        end_days = ['col_day%02d' % i for i in range(1, day_to)]

        first_month_range = (get_row_name(year_from, month_from), get_row_name(year_from, month_from + 1), start_days)
        last_month_range = (get_row_name(year_to, month_to), get_row_name(year_to, month_to + 1), end_days)
        inner_range = (get_row_name(year_from, month_from + 1), get_row_name(year_to, month_to))
        return first_month_range, inner_range, last_month_range
    else:
        inner_range = (get_row_name(year_from, month_from),
            get_row_name(year_to, month_to + 1),
            ['col_day%02d' % i for i in range(day_from, day_to + 1)])
        return None, inner_range, None

def main():
    parser = argparse.ArgumentParser(usage=USAGE)
    parser.add_argument("--table_name", type=str, required=True)
    parser.add_argument("--period_from", type=str, required=True)
    parser.add_argument("--period_to", type=str, required=True)
    args = parser.parse_args()

    table = connect(args.table_name)

    first_month_range, inner_range, last_month_range = get_period(args.period_from, args.period_to)

    def convert_data(data):
        return Counter({col.decode("utf-8")[10:] : int.from_bytes(data[col], byteorder='big') for col in data})

    domain_hits = Counter()
    if first_month_range:
        for key, data in table.scan(row_start=first_month_range[0], row_stop=first_month_range[1], columns=first_month_range[2]):
            domain_hits += convert_data(data)
    for key, data in table.scan(row_start=inner_range[0], row_stop=inner_range[1]):
        domain_hits += convert_data(data)
    if last_month_range:
        for key, data in table.scan(row_start=last_month_range[0], row_stop=last_month_range[1], columns=last_month_range[2]):
            domain_hits += convert_data(data)

    counts_sum = sum(domain_hits.values())
    for pair in domain_hits.most_common():
        print( pair[0], pair[1] / counts_sum, sep='\t' )

if __name__ == "__main__":
    main()


#!/usr/bin/env python3

import os
import unittest
import logging
from lofar.sas.resourceassignment.resourceassignmentestimator.service import ResourceEstimatorHandler
from lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators import ObservationResourceEstimator
from unittest import mock

class TestObservationResourceEstimator(unittest.TestCase):
    def test_raises_ValueError_on_incomplete_parset(self):
        parset = {}

        estimator = ObservationResourceEstimator()

        with self.assertRaises(ValueError):
            estimator.verify_and_estimate(parset)

    def generate_complete_parset(self):
        parset = {'Observation.sampleClock': None,
                  'Observation.startTime': "2019-03-04 13:00:00",
                  'Observation.stopTime': "2019-03-04 14:00:00",
                  'Observation.antennaSet': "LBA_INNER",
                  'Observation.nrBeams': 1,
                  'Observation.Beam[0].subbandList': [40, 41],
                  'Observation.nrBitsPerSample': 16,
                  'Observation.VirtualInstrument.stationList': ["CS001"],
                  'Observation.ObservationControl.OnlineControl.Cobalt.Correlator.nrChannelsPerSubband': 64,
                  'Observation.ObservationControl.OnlineControl.Cobalt.Correlator.integrationTime': 1,
                  'Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.flysEye': None,
                  'Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor': None,
                  'Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor': None,
                  'Observation.DataProducts.Output_Correlated.enabled': True,
                  'Observation.DataProducts.Output_Correlated.identifications': None,
                  'Observation.DataProducts.Output_Correlated.storageClusterName': None,
                  'Observation.DataProducts.Output_CoherentStokes.enabled': False,
                  'Observation.DataProducts.Output_CoherentStokes.identifications': None,
                  'Observation.DataProducts.Output_CoherentStokes.storageClusterName': None,
                  'Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.which': None,
                  'Observation.DataProducts.Output_IncoherentStokes.enabled': False,
                  'Observation.DataProducts.Output_IncoherentStokes.identifications': None,
                  'Observation.DataProducts.Output_IncoherentStokes.storageClusterName': None,
                  'Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.which': None,
                  }

        return parset

    def test_does_not_raise_ValueError_on_complete_parset(self):
        parset = self.generate_complete_parset()

        estimator = ObservationResourceEstimator()

        try:
            estimator.verify_and_estimate(parset)
        except ValueError:
            self.fail("estimator.verify_and_estimate(parset) raised ValueError unexpectedly!")

    def test_reports_error_when_no_output_was_enabled(self):
        parset = self.generate_complete_parset()

        parset['Observation.DataProducts.Output_Correlated.enabled'] = False
        parset['Observation.DataProducts.Output_CoherentStokes.enabled'] = False
        parset['Observation.DataProducts.Output_IncoherentStokes.enabled'] = False

        estimator = ObservationResourceEstimator()

        estimate = estimator.verify_and_estimate(parset)

        self.assertEqual(
            ['Produced observation resource estimate list has no data product estimates!'],
            estimate['errors'])

    def test_raise_reports_error_when_no_subbands_are_given(self):
        parset = self.generate_complete_parset()

        parset['Observation.DataProducts.Output_Correlated.enabled'] = True
        parset['Observation.Beam[0].subbandList'] = []

        estimator = ObservationResourceEstimator()

        estimate = estimator.verify_and_estimate(parset)

        self.assertTrue("Correlated data output enabled, but empty subband list for sap 0" in
            estimate['errors'])

    def test_reports_error_when_stationlist_is_empty(self):
        parset = self.generate_complete_parset()

        parset['Observation.DataProducts.Output_Correlated.enabled'] = True
        parset['Observation.VirtualInstrument.stationList'] = []

        estimator = ObservationResourceEstimator()

        estimate = estimator.verify_and_estimate(parset)

        self.assertTrue("Observation.VirtualInstrument.stationList is empty" in
            estimate['errors'])

    def test_reports_error_when_nrbeams_less_than_one(self):
        parset = self.generate_complete_parset()

        parset['Observation.DataProducts.Output_Correlated.enabled'] = True
        parset['Observation.nrBeams'] = 0

        estimator = ObservationResourceEstimator()

        estimate = estimator.verify_and_estimate(parset)

        self.assertTrue("Correlated data output enabled, but nrBeams < 1" in
            estimate['errors'])

class TestResourceEstimatorHandler(unittest.TestCase):
    def setUp(self):
        self.unique_otdb_id = 0
        self.data_sets_dir = os.path.join(os.environ.get('srcdir', os.path.dirname(os.path.abspath(__file__))),
                                          "data_sets")

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.ObservationResourceEstimator')
    def test_ObservationResourceEstimator_gets_called_on_observation_parset(self, ObservationResourceEstimator_mock):
        specification_tree = self.generate_observation_spec()

        resource_estimator_handler = ResourceEstimatorHandler()

        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        ObservationResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_observation_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_beam_observation')
        task_type = 'observation'
        task_subtype = 'bfmeasurement'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.ReservationResourceEstimator')
    def test_ReservationResourceEstimator_gets_called_on_reservation_maintenance_parset(self, ReservationResourceEstimator_mock):
        specification_tree = self.generate_maintenance_reservation_spec()

        resource_estimator_handler = ResourceEstimatorHandler()

        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        ReservationResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_maintenance_reservation_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_maintenance_reservation')
        task_type = 'reservation'
        task_subtype = 'maintenance'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.ReservationResourceEstimator')
    def test_ReservationResourceEstimator_gets_called_on_reservation_project_parset(self, ReservationResourceEstimator_mock):
        specification_tree = self.generate_project_reservation_spec()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        ReservationResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_project_reservation_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_project_reservation')
        task_type = 'reservation'
        task_subtype = 'project'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.CalibrationPipelineResourceEstimator')
    def test_CalibrationPipelineResourceEstimator_gets_called_on_calibrator_pipeline_parset(self, CalibrationPipelineResourceEstimator_mock):
        specification_tree = self.generate_calibration_pipeline_spec()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        CalibrationPipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_calibration_pipeline_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_calibration_pipeline')
        task_type = 'pipeline'
        task_subtype = 'calibration pipeline'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        self.add_predecessor_to_specification_tree(os.path.join(self.data_sets_dir,
                                                                't_resource_estimator.in_calibration_pipeline_predecessor_558022'),
                                                   # predecessor also used for imaging pipeline test
                                                   'observation',
                                                   'bfmeasurement',
                                                   specification_tree['predecessors'])
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.CalibrationPipelineResourceEstimator')
    def test_ImagePipelineResourceEstimator_gets_called_on_averaging_pipeline_parset(self, CalibrationPipelineResourceEstimator_mock):
        specification_tree = self.generate_averaging_pipeline_spec()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        CalibrationPipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_averaging_pipeline_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_preprocessing_pipeline')
        task_type = 'pipeline'
        task_subtype = 'averaging pipeline'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        # Pipelines need a predecessor so give it one
        predecessor_data_set_filepath = os.path.join(self.data_sets_dir,
                                                     't_resource_estimator.in_interferometer_observation')
        predecessor_task_type = 'observation'
        predecessor_task_subtype = 'interferometer'
        self.add_predecessor_to_specification_tree(predecessor_data_set_filepath,
                                                   predecessor_task_type,
                                                   predecessor_task_subtype,
                                                   specification_tree['predecessors'])
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.ImagePipelineResourceEstimator')
    def test_ImagePipelineResourceEstimator_gets_called_on_imaging_pipeline_parset(self, ImagePipelineResourceEstimator_mock):
        data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_imaging_pipeline')
        task_type = 'pipeline'
        task_subtype = 'imaging pipeline'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)

        self.add_predecessor_to_specification_tree(os.path.join(self.data_sets_dir,
                                                   't_resource_estimator.in_calibration_pipeline_predecessor_558022'),  # predecessor also used for calibration pipeline test
                                                   'observation',
                                                   'bfmeasurements',
                                                   specification_tree['predecessors'])

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        ImagePipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.ImagePipelineResourceEstimator')
    def test_ImagePipelineResourceEstimator_gets_called_on_imaging_pipeline_msss_parset(self, ImagePipelineResourceEstimator_mock):
        specification_tree = self.generate_imageing_pipeline_mss()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        ImagePipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_imageing_pipeline_mss(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_imaging_pipeline')
        task_type = 'pipeline'
        task_subtype = 'imaging pipeline msss'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        self.add_predecessor_to_specification_tree(os.path.join(self.data_sets_dir,
                                                                't_resource_estimator.in_calibration_pipeline_predecessor_558022'),
                                                   # predecessor also used for calibration pipeline test
                                                   'observation',
                                                   'bfmeasurements',
                                                   specification_tree['predecessors'])
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.LongBaselinePipelineResourceEstimator')
    def test_LongBaselinePipelineResourceEstimator_get_called_on_long_baseline_pipeline_parset(self, LongBaselinePipelineResourceEstimator_mock):
        specification_tree = self.generate_long_baseline_pipeline_spec()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        LongBaselinePipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_long_baseline_pipeline_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_long_baseline_pipeline')
        golden_output_filepath = os.path.join(self.data_sets_dir,
                                              't_resource_estimator.out_long_baseline_pipeline')
        task_type = 'pipeline'
        task_subtype = 'long baseline pipeline'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        tree = specification_tree
        predecessor_tree = self.get_specification_tree(
            os.path.join(
                self.data_sets_dir,
                't_resource_estimator.in_long_baseline_pipeline_predecessor_556601'),
            'pipeline',
            'averaging pipeline')
        tree['predecessors'].append(predecessor_tree)
        tree = predecessor_tree
        predecessor_tree = self.get_specification_tree(
            os.path.join(
                self.data_sets_dir,
                't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429'),
            'pipeline',
            'calibration pipeline')
        tree['predecessors'].append(predecessor_tree)
        tree = predecessor_tree
        predecessor_tree_branch_a = self.get_specification_tree(
            os.path.join(
                self.data_sets_dir,
                't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_556373'),
            'pipeline',
            'calibration pipeline')
        tree['predecessors'].append(predecessor_tree_branch_a)
        predecessor_tree_branch_b = self.get_specification_tree(
            os.path.join(
                self.data_sets_dir,
                't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_556375'),
            'observation',
            'bfmeasurement')
        tree['predecessors'].append(predecessor_tree_branch_b)
        predecessor_tree = self.get_specification_tree(
            os.path.join(
                self.data_sets_dir,
                't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_xxxxxx_556371'),
            'observation',
            'bfmeasurement')
        predecessor_tree_branch_a['predecessors'].append(predecessor_tree)
        predecessor_tree_branch_b['predecessors'].append(predecessor_tree)
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.PulsarPipelineResourceEstimator')
    def test_PulsarPipelineResourceEstimator_get_called_on_a_pulsar_pipeline_parset(self, PulsarPipelineResourceEstimator_mock):
        specification_tree = self.generate_pulsar_pipeline_spec()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        PulsarPipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_pulsar_pipeline_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_pulsar_pipeline')
        task_type = 'pipeline'
        task_subtype = 'pulsar pipeline'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        # Pulsar pipelines need a beamformer observation as their predecessor
        predecessor_data_set_filepath = os.path.join(self.data_sets_dir,
                                                     't_resource_estimator.in_beam_observation')
        predecessor_task_type = 'observation'
        predecessor_task_subtype = 'bfmeasurement'
        self.add_predecessor_to_specification_tree(predecessor_data_set_filepath,
                                                   predecessor_task_type,
                                                   predecessor_task_subtype,
                                                   specification_tree['predecessors'])
        return specification_tree

    def test_add_otdb_id_to_output(self):
        specification_tree = self.generate_averaging_pipeline_spec()

        resource_estimator_handler = ResourceEstimatorHandler()

        estimate = resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        estimate_list = estimate['estimates']

        for est in estimate_list:
            output_files = est.get('output_files')
            for dptype in output_files:
                for dptype_dict in output_files[dptype]:
                    self.assertEqual(specification_tree['otdb_id'], dptype_dict['properties'][dptype + '_otdb_id'])

    def add_predecessor_to_specification_tree(self, data_set_filepath, task_type, task_subtype, predecessor_list):
        """ Adds a predecessor specification tree to an existing specification tree

        :param data_set_filepath: predecessor specification data set filepath
        :param task_type: predecessor task's type
        :param task_subtype: predecessor task's subtype
        :param predecessor_list: specification tree to with to add predecessor specification tree
        """
        predecessor_specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        predecessor_list.append(predecessor_specification_tree)

    def get_specification_tree(self, data_set_filepath, task_type, task_subtype):
        """ Create a specification tree from a specification data set text file

        :param data_set_filepath: specification data set's filepath
        :param task_type: the task's type
        :param task_subtype: the task's subtype
        :return: specification tree
        """
        with open(data_set_filepath) as spec_file:
            parset = eval(spec_file.read().strip())
            return {
                'otdb_id': self.get_unique_otdb_id(),
                'task_type': task_type,
                'task_subtype': task_subtype,
                'predecessors': [],
                'specification': parset
            }

    def get_unique_otdb_id(self):
        """ Generates a unique OTDB ID (for use in parsets with predecessors)

        :return: unique OTDB ID
        """
        self.unique_otdb_id += 1
        return self.unique_otdb_id

    def get_golden_imate(self, golden_output_filepath, estimator_function=None, *estimator_args):
        """ Obtain the golden estimation from file (and create one if DO_GENERATE_GOLDEN_OUTPUTS is True)

        :param golden_output_filepath: the path to the golden estimate output file
        :param estimator_function: the estimator function to be called with estimator_args as it argument(S)
        :param estimator_args: the estimator function's arguments
        :return: the golden estimate
        """
        # Generate the golden output prior to fetching it if user requests so
        if DO_GENERATE_GOLDEN_OUTPUTS:
            estimation = estimator_function(*estimator_args)
            error_messages = self.get_uut_errors(estimation)
            # Make sure that there no errors are returned by uut
            if len(error_messages) == 0:
                self.store_datastructure(estimation, golden_output_filepath)
            else:
                raise Exception("\nThe uut reported errors:\n" + '\n- '.join(error_messages))

        # Fetch the golden output
        f = open(golden_output_filepath, "r")
        golden_output = f.read()
        f.close()

        # Remove trailing newline, and trailing and heading double-quotes
        stringified = golden_output.strip()
        stringified = stringified.strip('\n')
        stringified = stringified.strip('"')
        return stringified

    def store_datastructure(self, estimation={}, output_file=""):
        """ Stores the estimation data structure such that it can be used as golden output to verify against.

        :param estimation: resource estimator data structure
        :param output_file: file name to store the estimation to
        """
        output_filepath = os.path.join(self.data_sets_dir, output_file)
        f = open(output_filepath, 'w+')
        pprint(repr(estimation).strip(), stream=f)
        f.close()

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
    unittest.main()


"""
This script is intended to test base implementations and should be deleted as soon as infrastructure is established
When run, this script will calculate a given amount of prime numbers, beginning at 2 for an input of 0
"""

import argparse
import copy
import logging.config

import yaml

with open('logging_conf.yaml') as conf_file:
    config = yaml.safe_load(conf_file.read())
    logging.config.dictConfig(config)

logger = logging.getLogger(__name__)

class PrimeCalculator:
    """
    Responsible for handling calls to prime calculation
    """
    def calculate_primes(self, n, i=2, primes=()):
        """
        recursively tests if each number is a prime
        n: the number of primes left to calculate
        i: the number we have reached
        primes: the primes we have already found
        """

        # base case: we have calculated the last prime
        if n == 0:
            return i - 1

        for prime in primes:
            if i % prime != 0:
                continue

            return self.calculate_primes(n, i + 1, primes)

        # no divisions have succeeded, we have found another prime
        primes = (*primes, i)
        logging.debug(f"Found a prime, current prime list is now \n{str(primes)}\n")
        return self.calculate_primes(n - 1, i + 1, primes)

def build_namespace_argument_dict(namespace):
    """
    Returns a dictionary version of a namespace for more natural parsing
    """
    # Namespace is mutable, copy it before making changes
    dict_args = vars(copy.copy(namespace))

    # Remove the function paramter if it exists, as its not an actual parameter
    dict_args.pop('func', None)
    return dict_args

def build_arg_parser():
    """
    Create an argparser configuration for command line arguments
    """
    parser = argparse.ArgumentParser(
        description="Module for getting the nth prime"
    )
    parser.add_argument('-n', '--number', type=int, help="The nth prime to retrieve")

    return build_namespace_argument_dict(parser.parse_known_args()[0])

if __name__ == "__main__":
    arguments = build_arg_parser()
    argument_str = '\n'.join([f'--- {key} = {arg}' for key, arg in arguments.items()])
    logger.info(f"Prime calculator running with the following arguments: \n{argument_str}")

    prime_calc = PrimeCalculator()
    nth_prime = prime_calc.calculate_primes(arguments['number'])
    with open('prime.txt', 'w') as prime_file:
        prime_file.write(str(nth_prime))

    logging.info(f"COMPLETED RUN, the {arguments['number']}th prime is {nth_prime}")


nValues = [1000, 10000, 100000, 1000000, 10000000]
pValues = [1, 2, 4, 8, 16, 20, 32, 40, 64, 128]

# create Dir if not exists
import os
if not os.path.exists("Graf1Tests"):
    os.makedirs("Graf1Tests")

for n in nValues:
    for p in pValues:
        f = open("Graf1Tests/testGraf1_" + str(n) + "_" + str(p) + ".sh", "w")
        f.write("#!/bin/bash" + "\n" + "#SBATCH --time=1:00" + "\n" + "#SBATCH --ntasks=" + str(p) + "\n" + "#SBATCH --partition=cpar" + "\n" + "module load gcc/7.2.0" + "\n" + "export CP_CLUSTERS=4" + "\n" + "export N_SIZE=" + str(n) + "\n" + "export PROCESSES=" + str(p) + "\n" + "make clean" + "\n" + "make k_means" + "\n" + "perf stat -r 1 -e L1-dcache-load-misses -M cpi make runseq" + "\n"
        + "echo \"---------------------------------------------------------------------\"" + "\n" + "echo \"--- TEST WITH || SIZE: " + str(n) + " || PROCESSES: " +str(p) + "----\"")


import pandas as pd
import math
import random
import warnings
import re
from copy import deepcopy
from typing import Optional
from mob_sets import FULL_MONSTERS, FullMob

STAT_TABLE = pd.read_csv('generic_stats.csv', index_col=0)
STAT_TABLE['pos index'] = range(len(STAT_TABLE))
STAT_TABLE['Prof Bonus low'] = STAT_TABLE['Prof Bonus']
STAT_TABLE['Prof Bonus high'] = STAT_TABLE['Prof Bonus']
STAT_TABLE['DC low'] = STAT_TABLE['DC']
STAT_TABLE['DC high'] = STAT_TABLE['DC']

CRS = tuple(['0', '1/8', '1/4', '1/2'] + [str(x) for x in range(1,21)])

def shift_cr(cr: str, amount: int):
    cr = str(cr)
    start_index = CRS.index(cr)
    end_index = start_index + amount
    if end_index < 0:
        return 0, -1
    if end_index >= len(CRS):
        return CRS[-1], 1
    return CRS[end_index], 0

def cr_offset(stat, value, current_cr):
    current_cr = str(current_cr)
    expected_index = STAT_TABLE.loc[current_cr, 'pos index']
    possible_cr = STAT_TABLE[(STAT_TABLE[f'{stat} low'] <= value) & (STAT_TABLE[f'{stat} high'] >= value)].copy()
    possible_cr['diff'] = possible_cr['pos index'] - expected_index
    possible_cr['abs diff'] = possible_cr['diff'].apply(abs)
    closest_cr = possible_cr.loc[possible_cr['abs diff'] == possible_cr['abs diff'].min()].iloc[0]

    cr_diff = closest_cr['diff']
    if closest_cr[f'{stat} high'] - closest_cr[f'{stat} low'] == 0:
        frac = 0
    else:
        frac = (value - closest_cr[f'{stat} low'])/(closest_cr[f'{stat} high'] - closest_cr[f'{stat} low'])

    return cr_diff, frac

def generate_value(stat, cr, frac=0):
    cr = str(cr)
    return round(STAT_TABLE.loc[cr, f'{stat} low'] + frac * (
            STAT_TABLE.loc[cr, f'{stat} high'] - STAT_TABLE.loc[cr, f'{stat} low']))

def generate_hp(cr, frac, size):
    target_hp = generate_value('HP', cr, frac)
    hit_dice = {
        'tiny': (4, 2.5), 'small': (6, 3.5), 'medium': (8, 4.5), 'large': (10, 5.5), 'huge': (12, 6.5),
        'gargantuan': (20, 10.5)}
    hit_die = hit_dice[size]
    return dice_from_hp(target_hp, hit_die[1])

def dice_from_hp(target_hp: int, hit_die: int):
    ave_value = hit_die/2 + 0.5
    n_dice = int(target_hp // ave_value)
    make_fixed = random.choice(range(0, max(1, n_dice-4)))
    n_dice -= make_fixed
    fixed = round((target_hp-make_fixed*ave_value) % ave_value+make_fixed*ave_value)
    return target_hp, f'{n_dice}d{hit_die}+{fixed}'

def scale_monster(monster: FullMob, new_cr: str, new_size: Optional[str] = None):
    new_cr = str(new_cr)
    new_monster = deepcopy(monster)
    old_cr = monster.cr
    if new_size is None:
        new_size = monster.size
    new_monster.stat_block['Source'] = 'DSO Homebrew'
    new_monster.stat_block['size'] = new_size
    new_monster.stat_block['Version'] = ""
    new_monster.stat_block['Type'] = f"{new_size} {new_monster.stat_block['category']}, {new_monster.stat_block['alignment']}"

    new_monster.set_cr(new_cr)
    new_name = input('What is the monster\'s name? ')
    new_monster.stat_block['Name'] = new_name
    new_monster.stat_block['Id'] = new_name.replace(' ', '_')
    old_to_hit = int(input('What is the monster\'s current to hit? '))
    old_damage = int(input('What is the monster\'s current damage per round? '))
    old_dc = input('What is the monster\'s current save DC for abilities? ')

    # Set HP
    cr_offset_amount, cr_frac = cr_offset('HP', monster.hp[0], old_cr)

    new_stat_cr, shift_flag = shift_cr(new_cr, cr_offset_amount)
    if shift_flag == -1:
        warnings.warn('The scaled monster should have HP CR of below 0. Its HP has been set to CR 0 which might make it '
                      'stronger than expected.')
    if shift_flag == 1:
        warnings.warn('The scaled monster should have HP CR of above 20. Its HP has been set to CR 20 which might make it '
                      'weaker than expected.')
    new_hp = generate_hp(new_stat_cr, cr_frac, new_size)
    new_monster.set_hp(*new_hp)
    print(f'Suggested HP: {new_hp[0]}({new_hp[1]})')

    # Set AC
    cr_offset_amount, cr_frac = cr_offset('AC', monster.ac, old_cr)

    new_stat_cr, shift_flag = shift_cr(new_cr, cr_offset_amount)
    if shift_flag == -1:
        warnings.warn(
            'The scaled monster should have AC CR of below 0. Its AC has been set to CR 0 which might make it '
            'stronger than expected.')
    if shift_flag == 1:
        warnings.warn(
            'The scaled monster should have AC CR of above 20. Its AC has been set to CR 20 which might make it '
            'weaker than expected.')
    new_ac = generate_value('AC', new_stat_cr, cr_frac)
    new_monster.set_ac(new_ac)
    print(f'Suggested AC: {new_ac}')

    # Print the suggested new to hit
    cr_offset_amount, cr_frac = cr_offset('Hit', old_to_hit, old_cr)

    new_stat_cr, shift_flag = shift_cr(new_cr, cr_offset_amount)
    if shift_flag == -1:
        warnings.warn(
            'The scaled monster should have To Hit CR of below 0. Its To Hit has been set to CR 0 which might make it '
            'stronger than expected.')
    if shift_flag == 1:
        warnings.warn(
            'The scaled monster should have To Hit CR of above 20. Its To Hit has been set to CR 20 which might make it '
            'weaker than expected.')
    print(f'Suggested To Hit: +{generate_value("Hit", new_stat_cr, cr_frac)}')

    # Print the suggested new damage
    cr_offset_amount, cr_frac = cr_offset('Damage', old_damage, old_cr)

    new_stat_cr, shift_flag = shift_cr(new_cr, cr_offset_amount)
    if shift_flag == -1:
        warnings.warn(
            'The scaled monster should have Damage CR of below 0. Its Damage has been set to CR 0 which might make it '
            'stronger than expected.')
    if shift_flag == 1:
        warnings.warn(
            'The scaled monster should have To Damage of above 20. Its Damage has been set to CR 20 which might make it '
            'weaker than expected.')
    print(f'Suggested Damage per turn: {generate_value("Damage", new_stat_cr, cr_frac)}')

    # Print the suggested new DC
    if old_dc != '':
        cr_offset_amount, cr_frac = cr_offset('DC', old_dc, old_cr)

        new_stat_cr, shift_flag = shift_cr(new_cr, cr_offset_amount)
        if shift_flag == -1:
            warnings.warn(
                'The scaled monster should have DC CR of below 0. Its DC has been set to CR 0 which might make it '
                'stronger than expected.')
        if shift_flag == 1:
            warnings.warn(
                'The scaled monster should have To DC of above 20. Its DC has been set to CR 20 which might make it '
                'weaker than expected.')
        print(f'Suggested DC: {generate_value("DC", new_stat_cr, cr_frac)}')
    return new_monster

x = FULL_MONSTERS['ankheg']

#t = scale_monster(x, '6', 'huge')
#h = t.to_homebrewery()
#print(h)


import mysql.connector
import discord
import time
import re
import os
from discord.ext import commands
from functions import wantcardsuser, cardnamechecker, cardnamecheckername

class want(commands.Cog):

    def __init__(self,client):
        self.client = client

    #commands
    @commands.command(brief='Will @ People who have the card and how many')
    async def want(self, ctx, *, card):
        await ctx.send(f'looking up the users who have {card}......')
        if "}" in str(f'{card}') or "{" in str(f'{card}'):
            await ctx.send('Hey now. You stop that!')
        elif "None" in str(await cardnamechecker(f'{card}')):
            await ctx.send('The card name is incorrect')
        elif "None" in str(await wantcardsuser(f'{card}')):
            await ctx.send('No one has the card')
        else:
            from database import db_info
            db = mysql.connector.connect(**db_info)
            mycursor = db.cursor()
            atcard = ""
            noatcard = ""
            guild = str(ctx.guild.id)
            cardoutputname = await cardnamecheckername(f'{card}')
            mycursor.execute(
                f'SELECT i.discordid AS usid, SUM(i.count) AS count, i.name AS magicname FROM inventory i JOIN discorduser d ON i.discordid = d.discordid LEFT JOIN discordserver s ON i.discordid = s.discordid Where i.name = "{cardoutputname}" AND serverid = {guild} AND active = 1 AND at = 1 GROUP BY i.discordid')
            rows = mycursor.fetchall()
            for (usid, count, magicname) in rows:
                atcard += ("<@!{}> has {} {}\n".format(usid, count, magicname))
            mycursor.execute(
                f'SELECT d.name AS uname, SUM(i.count) AS count, i.name AS magicname FROM inventory i JOIN discorduser d ON i.discordid = d.discordid LEFT JOIN discordserver s ON i.discordid = s.discordid Where i.name = "{cardoutputname}" AND serverid = {guild} AND active = 1 AND at = 0 GROUP BY i.discordid')
            rows = mycursor.fetchall()
            for (uname, count, magicname) in rows:
                noatcard += ("{} has {} {}\n".format(uname, count, magicname))
            await ctx.send(
                f'<@!{ctx.author.id}> here are the users who have the card {card}:\n\n{atcard + noatcard}')
        mycursor.close()
        db.close()

    @commands.command(brief='Wll look up a mass import of who has the cards. ')
    async def masswant(self, ctx, *, card):
        cardsplit = (re.split("\n+", card))
        timeofsend = int(time.time())
        nonumber = ""
        lessthan10 = ""
        greaterthan10 = ""
        numbercheck = list(range(10))
        cardwantedlist = []
        if "}" in str(f'{card}') or "{" in str(f'{card}'):
            await ctx.send('Please remove { or } from your list!')
        else:
            await ctx.send('looking up the users who have cards from the list......')
            for card in cardsplit:
                if card[0] not in str(numbercheck):
                    nonumber += ('i.name = "{}" OR '.format(card))
                    cardwantedlist.append('{}'.format(card))
                elif card[0] == " ":
                    nonumber += ('i.name = "{}" OR '.format(card))
                    cardwantedlist.append('{}'.format(card))
                elif int(card[0] + card[1]) < 10:
                    lessthan10 += ('i.name = "{}" OR '.format(card[2:]))
                    cardwantedlist.append('{}'.format(card[2:]))
                else:
                    greaterthan10 += ('i.name = "{}" OR '.format(card[3:]))
                    cardwantedlist.append('{}'.format(card[3:]))
                whereintoquery = (nonumber + greaterthan10 + lessthan10)[:-3]
                from database import db_info
                db = mysql.connector.connect(**db_info)
                mycursor = db.cursor()
                atcard = ""
                noatcard = ""
                currentuser = ""
                userlist = []
                cardlist = []
                guild = str(ctx.guild.id)
                mycursor.execute(
                    f'SELECT i.discordid AS usid, SUM(i.count) AS count, i.name AS magicname FROM inventory i JOIN discorduser d ON i.discordid = d.discordid LEFT JOIN discordserver s ON i.discordid = s.discordid WHERE ({whereintoquery}) AND serverid = {guild} AND active = 1 AND at = 1 GROUP BY i.discordid, i.name ORDER BY i.name')
                rows = mycursor.fetchall()
                for (usid, count, magicname) in rows:
                    if currentuser == "" or currentuser != usid:
                        currentuser = usid
                        userlist.append(currentuser)
                userlist = list(set(userlist))
                for user in userlist:
                    atcard += "<@!{}> ".format(user)
                mycursor.execute(
                    f'SELECT d.name AS uname, SUM(i.count) AS count, i.name AS magicname FROM inventory i JOIN discorduser d ON i.discordid = d.discordid LEFT JOIN discordserver s ON i.discordid = s.discordid WHERE ({whereintoquery}) AND serverid = {guild} AND active = 1 GROUP BY i.discordid, i.name ORDER BY i.name')
                rows = mycursor.fetchall()
                currentcard = ""
                for (uname, count, magicname) in rows:
                    if currentcard == "" or currentcard != magicname:
                        currentcard = magicname
                        cardlist.append(magicname)
                        noatcard += "\n{}: \n".format(currentcard)
                    noatcard += ("{} has {} {}\n".format(uname, count, magicname))
                cardlist = list(set(cardlist))
                cardlist_lower = [card.lower() for card in cardlist]
                cardwantedlist_lower = [card.lower() for card in cardwantedlist]
                cardsnonehas = list(set(cardwantedlist_lower).difference(cardlist_lower))
                cardsnonehasmessege = ""
                for card in cardsnonehas:
                    cardsnonehasmessege += ('\n{}:\nNo one has this card or the card name was missed spelled\n'.format(card))
                filename = f'{timeofsend}{ctx.author.id}.txt'
                savepath = 'F:/Documents/Python Projects/MagicBot/Massexports'
                filepath = os.path.join(savepath, filename)
                with open(filepath, "w") as f:
                    f.write(F'{noatcard}\n{60 * "-"}\n{cardsnonehasmessege}')
                file = discord.File(filepath)
                flushdir(savepath)
        await ctx.send(file=file,
                       content=f'<@!{ctx.author.id}> we have found your cards!\n\nHere are users for quick whisper: {atcard}\n\nHere is the users who have cards on your list:')
        mycursor.close()

##removes files that are a minute old to keel the massexports folder from filling up
def flushdir(dir):
    now = time.time()
    for f in os.listdir(dir):
        fullpath = os.path.join(dir, f)
        if os.stat(fullpath).st_mtime < (now - 60):
            if os.path.isfile(fullpath):
                os.remove(fullpath)
            elif os.path.isdir(fullpath):
                flushdir(fullpath)

async def setup(client):
    await client.add_cog(want(client))


from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
from locators import form_locators
import time

class Proficiency(object):

    def __init__(self, driver):
        self.driver = driver

    def get_proficiency_text(self):
        chack_box_gride = WebDriverWait(self.driver, 3).until(
            EC.presence_of_all_elements_located((By.CSS_SELECTOR, form_locators.chack_box_gride_selc))
        )
        return chack_box_gride

    def set_proficiency_check_lang(self, lang):
            reading = WebDriverWait(self.driver, 10).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, form_locators.reading_selector.format(lang)))
            )
            reading.click()

            WebDriverWait(self.driver, 10).until(
                EC.invisibility_of_element_located((By.CSS_SELECTOR, '[aria-label="Reading, response for {}"]'
                                                                     '[aria-checked="false"]'.format(lang)))
            )
            listing = WebDriverWait(self.driver, 10).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, form_locators.listing_selector.format(lang)))
            )
            listing.click()

    # def proficiency_check_french(self, fre):
    #         listing = WebDriverWait(self.driver, 10).until(
    #             EC.element_to_be_clickable((By.CSS_SELECTOR, form_locators.listing_selector.format(fre)))
    #         )
    #         listing.click()
    #
    # def proficiency_check_urdu(self, urdu):
    #         reading = WebDriverWait(self.driver, 10).until(
    #             EC.element_to_be_clickable((By.CSS_SELECTOR, form_locators.reading_selector.format(urdu)))
    #         )
    #         reading.click()
    #         writing = self.driver.find_element_by_css_selector(
    #             form_locators.writing_selector.format(urdu))
    #         writing.click()
    #         listing = WebDriverWait(self.driver, 10).until(
    #             EC.element_to_be_clickable((By.CSS_SELECTOR, form_locators.listing_selector.format(urdu)))
    #         )
    #         listing.click()


#!/usr/bin python3
# -*- encoding: utf-8 -*-
# @Author : cobra
# @File : preprocess.py
# @Time : 2020/9/20 23:06
import pandas as pd
import jieba
import keras
import json
from keras import backend as K

from sklearn.preprocessing import LabelEncoder
from model.text_cnn import simple_text_cnn
from sklearn.model_selection import train_test_split

from keras.preprocessing.text import Tokenizer

from keras.preprocessing.sequence import pad_sequences

def convert_json2excel(infile, outfile):
    """
    转换json格式为excel
    :param infile:
    :param outfile:
    :return:
    """
    f = open(infile, encoding='utf-8')
    df_out = pd.DataFrame()
    label_id_list = []
    label_list = []
    text_list = []
    keyword_list = []

    for line in f.readlines():
        line = line.strip()
        line = json.loads(line)
        label_id_list.append(line['label'])
        label_list.append(line['label_desc'])
        text_list.append(line['sentence'])
        keyword_list.append(line['keywords'])
    df_out['label_id'] = label_id_list
    df_out['label'] = label_list
    df_out['text'] = text_list
    df_out['keywords'] = keyword_list
    df_out.to_excel(outfile)

def cut_x(x):

    return list(jieba.cut(x)) # jieba.cut(x)返回值是一个generator，所以前面要加list

def excel_preprocess(infile):
    df = pd.read_excel(infile)
    function_cut = lambda x: list(jieba.cut(x))
    df['words'] = df['text'].apply(cut_x)
    return df

def train_test_split_dataset(text,label,test_size = 0.1):
    return train_test_split(text,label,test_size = 0.1)

def get_token_id(text_list):
    tokenizer = Tokenizer()
    tokenizer.fit_on_texts(text_list)
    vocab = tokenizer.word_index
    return tokenizer

def pad_sequence(text,max_len =50,padding='post'):
    text_pad = pad_sequences(text,maxlen=50,padding=padding)
    return text_pad

def self_loss(y_true, y_pred):
    return K.sum((y_pred + y_true), axis=-1)

if __name__ == '__main__':
    # convert_json2excel('../data/train.json', '../data/train.xlsx')

    df = excel_preprocess('../data/train_demo.xlsx')
    tokenizer= get_token_id(list(df['words']))
    x_train, x_test, y_train, y_test = train_test_split(list(df['words']), list(df['label']), test_size=0.1)
    x_train = tokenizer.texts_to_sequences(x_train)
    x_test = tokenizer.texts_to_sequences(x_test)
    x_train_pad = pad_sequence(x_train,max_len =50)
    x_test_pad = pad_sequence(x_test, max_len=50)
    encoder = LabelEncoder()
    y_train_id = encoder.fit_transform(y_train)
    model = simple_text_cnn((50,),len(tokenizer.word_index))
    model.summary()
    model.compile(loss=self_loss, optimizer='adam', metrics=['accuracy'])
    one_hot_labels = keras.utils.to_categorical(y_train_id, num_classes=15)  # 将标签转换为one-hot编码
    model.fit(x_train_pad, one_hot_labels, batch_size=128, epochs=10)


from create_bot import db
from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton, ReplyKeyboardMarkup, KeyboardButton

menu_markup = InlineKeyboardMarkup(
    inline_keyboard=[
        [InlineKeyboardButton("Я подтверждаю ▶", callback_data='i_approve')]
    ]
)

form_conditional = ReplyKeyboardMarkup(
    [
        ["online", "offline"]
    ], resize_keyboard=True
)

contact = ReplyKeyboardMarkup(
    resize_keyboard=True,
    keyboard=[
        [KeyboardButton("Номер телефона",request_contact=True)]
    ]
)

admin_markup = InlineKeyboardMarkup(
    inline_keyboard=[
        [InlineKeyboardButton("ОТПРАВИТЬ СООБЩЕНИЕ", callback_data='send_message')],
        [InlineKeyboardButton("ИЗМЕНИТЬ ТЕКСТ", callback_data="change_text")],
        [InlineKeyboardButton("ДОБАВИТЬ ВОПРОС", callback_data='add_question')],
        [InlineKeyboardButton('РЕДАКТИРОВАТЬ ВОПРОС', callback_data='edit_questions')]
    ]
)

optinos = InlineKeyboardMarkup(inline_keyboard=[
    [InlineKeyboardButton("РЕДАКТИРОВАТЬ ТЕКСТ", callback_data='edit_text')],
    [InlineKeyboardButton("ИЗМЕНИТЬ ТИП ОТВЕТЫ", callback_data='edit_type_answer')],
    [InlineKeyboardButton("УДАЛИТЬ ВОПРОС", callback_data='delete')],
    [InlineKeyboardButton("ВЕРНУТЬСЯ К АДМИНАМ ПАНЕЛИ", callback_data='exit')]
])

def list_texts ():
    markup = InlineKeyboardMarkup()
    for key, in db.sqlite('SELECT key FROM Texts'):
        markup.add(InlineKeyboardButton(key, callback_data=f'change_{key}'))
    markup.add(InlineKeyboardButton("НАЗАД", callback_data='admin'))
    return markup

def list_question_markup():
    markup = InlineKeyboardMarkup()
    for key,rowid in db.sqlite('SELECT question, rowid FROM Questions'):
        markup.add(InlineKeyboardButton(key, callback_data=f'change_{rowid}'))
    markup.add(InlineKeyboardButton("НАЗАД", callback_data='exit'))
    return markup


from django.shortcuts import render
from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.parsers import FormParser, MultiPartParser, JSONParser
from .models import LectureVideos
from .serializers import LectureVideoSerialzer

@api_view(['GET'])
def apiOverview(request):

    api_urls = {

        'Lecture-Videos' : 'videos-list/',
        'Upload-lecture-video': 'video-upload/',
        'Delete-lecture-video' : 'delete-lecture/'
    }

    return Response(api_urls)

@api_view(['GET'])
def getLectureVideos(request):

    videos = LectureVideos.objects.all()
    serializer = LectureVideoSerialzer(videos, many=True)

    return Response(serializer.data)

@api_view(['POST'])
def uploadLectureVideo(request):

    parser_classes = (JSONParser, FormParser, MultiPartParser)
    serializer = LectureVideoSerialzer(data=request.data)

    if serializer.is_valid():
        serializer.save()
        return Response(serializer.data)

    else:
        print('Errors: ' + str(serializer.data))

    return Response(serializer.data)


from typing import Dict, Set, List
from MetaVec.embedding import Embedding
import MetaVec.matrix_math as matrix_math
import argparse
import os
from collections import Counter
from tqdm import tqdm
import numpy as np
from MetaVec.utils import batch

def get_union_vocab(
    list_of_vocabularies: List[List[str]], min_freq: int = 1
) -> Set[str]:
    if min_freq > 1:
        c: Counter[str, int] = Counter()
        for list_vocab in list_of_vocabularies:
            c.update(list_vocab)

        return set([word for word, freq in c.items() if freq > min_freq])
    else:
        return set.union(*[set(list_vocab) for list_vocab in list_of_vocabularies])

def save_meta_embedding(
    embedding: Embedding,
    new_words: List[str],
    new_vectors: np.ndarray,
    save_path: str,
):

    with open(save_path, "w+", encoding="utf-8") as output_file, tqdm(
        total=len(embedding) + len(new_words), desc=f"Saving embedding to {save_path}"
    ) as pbar:
        print(f"{len(embedding)+len(new_words)} {embedding.dims}", file=output_file)

        for words in batch(embedding.words, n=100):
            print(
                "\n".join(
                    [
                        word
                        + " "
                        + " ".join(["%.6g" % x for x in embedding.word2vector(word)])
                        for word in words
                    ]
                ),
                file=output_file,
            )

            pbar.update(len(words))

        for words, vectors in zip(batch(new_words, n=100), batch(new_vectors, n=100)):
            print(
                "\n".join(
                    [
                        word + " " + " ".join(["%.6g" % x for x in vector])
                        for word, vector in zip(words, vectors)
                    ]
                ),
                file=output_file,
            )
            pbar.update(len(words))

    print(f"Embedding exported to : {save_path}")

def oov_generator(
    embeddings_paths: List[str],
    save_dir: str,
    min_k_value: int = 2,
    max_k_value: int = 50,
    metric: str = "cosine",
    batch_size: int = 2000,
    min_freq_words: int = 1,
    normalize_method: str = "none",
    fp: int = 32,
    filter_words: bool = False,
    debug: bool = False,
):

    # PREPARE DATA

    assert normalize_method in [
        "l2",
        "none",
        "scale",
    ], f"Normalize method {normalize_method} not implemented. Available methods: [l2,none,scale]"

    assert (
        max_k_value >= min_k_value > 0
    ), f"max_k_value >= min_k_value > 0. min_k_value:{min_k_value}. max_k_value:{max_k_value}"

    # - Load embeddings

    embeddings = [
        Embedding.from_file(
            path,
            fp=fp,
            filter_words=filter_words,
        )
        for path in embeddings_paths
    ]

    # - Create vocabulary  and normalize embeddings

    dims = set([emb.dims for emb in embeddings])
    assert (
        len(dims) == 1
    ), f"All the embeddings should have the same number of dimensions. Dimensions found: {dims}"

    dims = dims.pop()

    normalize = normalize_method == "l2"

    union_vocabulary: Set[str] = get_union_vocab(
        [embedding.words for embedding in embeddings], min_freq=min_freq_words
    )

    v_sum = "\n".join(
        [
            f"{os.path.basename(ep)} has {len(emb)} words"
            for ep, emb in zip(embeddings_paths, embeddings)
        ]
    )
    print(
        f"\n--> Vocabulary summary <--\n"
        f"{v_sum}\n"
        f"- Union of vocabularies has {len(union_vocabulary)} words\n\n"
    )

    del v_sum

    if normalize_method == "l2":
        print("Normalization method: L2")
        for emb in embeddings:
            emb.normalize_length()
            emb.mean_center()
            emb.normalize_length()

    elif normalize_method == "scale":

        print("Normalization method: Translate + Scale")
        for embedding in tqdm(embeddings, desc="Embedding Normalization"):
            embedding.mean_center()
            avg_scale: float = matrix_math.get_average_length(embedding.vectors)
            embedding.scale_vectors(scale_factor=1.0 / avg_scale)

    # RUN THE OOV ALGORITHM

    print(f"--> Running OOV generation algorithm <--")
    print(
        f"Save dir: {save_dir}\n"
        f"No. Embeddings: {len(embeddings)}\n"
        f"Metric: {metric}\n"
        f"FP{fp}\n"
        f"Batch size. {batch_size}\n"
        f"min_k_value: {min_k_value}\n"
        f"max_k_value: {max_k_value}\n"
        f"Debug mode: {debug}\n"
        f"GPU available: {matrix_math.cupy_available}\n"
        f"GPU Free memory: {matrix_math.get_gpu_memory()} Mb \n\n"
    )

    for embedding_oov_id in range(len(embeddings)):
        print(f"--> Running OOV generation for embedding {embedding_oov_id} <--")
        embedding_oov = embeddings[embedding_oov_id]
        new_words: List[str] = list(
            union_vocabulary - embeddings[embedding_oov_id].set_words
        )
        print(
            f"We will generate {len(new_words)} words for embedding {embedding_oov_id}"
        )
        new_vectors: np.ndarray = np.zeros(
            (len(new_words), dims),
            dtype=embeddings[embedding_oov_id].embedding_numpy_type,
        )

        if len(new_words) == 0:
            return new_words, new_vectors

        new_words2index: Dict[str, int] = {w: i for i, w in enumerate(new_words)}

        new_vectors_distance: np.ndarray = np.full(
            (len(new_words)),
            fill_value=float("-inf") if metric == "cosine" else float("inf"),
            dtype=embeddings[embedding_oov_id].embedding_numpy_type,
        )

        for embedding_index_id in range(len(embeddings)):
            if embedding_index_id == embedding_oov:
                continue

            embedding_index = embeddings[embedding_index_id]
            words2generate: List[str] = list(
                embedding_index.set_words - embedding_oov.set_words
            )
            if len(words2generate) == 0:
                continue

            intersection_vocab: np.ndarray = np.asarray(
                list(embedding_index.set_words.intersection(embedding_oov.set_words))
            )

            if len(intersection_vocab) == 0:
                continue

            embedding_index_true_centroids: np.ndarray = embedding_index.words2matrix(
                words2generate
            )
            intersection_vocab_vectors: np.ndarray = embedding_index.words2matrix(
                intersection_vocab
            )
            if metric == "cosine":
                if normalize:
                    knn, knn_distances = matrix_math.knn_dot(
                        embedding_index_true_centroids,
                        intersection_vocab_vectors,
                        k=max_k_value,
                        return_distances=True,
                        ordered=True,
                        show_progress=True,
                        batch_size=batch_size,
                    )
                else:
                    knn, knn_distances = matrix_math.knn_cosine(
                        embedding_index_true_centroids,
                        intersection_vocab_vectors,
                        k=max_k_value,
                        return_distances=True,
                        ordered=True,
                        show_progress=True,
                        batch_size=batch_size,
                    )

            else:
                knn, knn_distances = matrix_math.knn_euclidean_distance(
                    embedding_index_true_centroids,
                    intersection_vocab_vectors,
                    k=max_k_value,
                    return_distances=True,
                    ordered=True,
                    show_progress=True,
                    batch_size=batch_size,
                )
                knn_distances = 1 / knn_distances

            for i in tqdm(range(len(words2generate)), desc=f"Generating OOV words: "):
                oov_word_id = new_words2index[words2generate[i]]
                candidates = matrix_math.rolling_average(
                    a=intersection_vocab_vectors[knn[i]],
                    weights=knn_distances[i] + 1,
                )[min_k_value - 1 : max_k_value]

                if metric == "cosine":

                    d: np.ndarray = matrix_math.cosine_similarity(
                        candidates, embedding_index_true_centroids[i][np.newaxis, :]
                    )
                    best_k = np.argmax(d)

                else:
                    if normalize:
                        candidates = matrix_math.length_normalize(candidates)

                    d: np.ndarray = matrix_math.euclidean_distance(
                        candidates, embedding_index_true_centroids[i][np.newaxis, :]
                    )

                    best_k = np.argmin(d)

                d = d[best_k][0]
                best_k = best_k + min_k_value

                if (metric == "cosine" and d > new_vectors_distance[oov_word_id]) or (
                    metric == "euclidean" and d < new_vectors_distance[oov_word_id]
                ):
                    new_vector = np.average(
                        embedding_oov.words2matrix(intersection_vocab[knn[i][:best_k]]),
                        weights=knn_distances[i][:best_k] + 1,
                        axis=0,
                    )
                    if normalize:
                        new_vector /= np.linalg.norm(new_vector)

                    prev_d = new_vectors_distance[oov_word_id]
                    new_vectors[oov_word_id] = new_vector
                    new_vectors_distance[oov_word_id] = d

                    if debug:
                        print(
                            f"Generating word {words2generate[i]} "
                            f"with nns {intersection_vocab[knn[i][:best_k]]}. "
                            # f"matrix: {embedding_oov.words2matrix(intersection_vocab[knn[i][:k]])}. "
                            # f"weights: {knn_distances[i][:k]}"
                            # f"v: {v[:10]}. "
                            f"Distance: {d}. "
                            f"Prev distance: "
                            f"{prev_d}"
                        )

        save_meta_embedding(
            embedding=embedding_oov,
            new_words=new_words,
            new_vectors=new_vectors,
            save_path=os.path.join(
                save_dir, os.path.basename(embeddings_paths[embedding_oov_id])
            ),
        )

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--embeddings",
        nargs="+",
        type=str,
        required=True,
        help="Path to the embeddings that we will use to generate a meta-embeddings",
    )

    parser.add_argument(
        "--output_dir",
        type=str,
        required=True,
        help="Directory where we will store the resulting embedding",
    )

    parser.add_argument(
        "--metric",
        type=str,
        choices=["cosine", "euclidean"],
        default="cosine",
        help="use cosine similarity or euclidean distance to compute nearest nearest neighbours",
    )

    parser.add_argument(
        "--k_min_value",
        type=int,
        default=2,
        help="Min value to use in the k estimator grid search",
    )

    parser.add_argument(
        "--k_max_value",
        type=int,
        default=50,
        help="Max value to use in the k estimator grid search",
    )

    parser.add_argument(
        "--fp",
        type=int,
        choices=[16, 32, 64],
        default=32,
        help="FP precision to use for the computations",
    )

    parser.add_argument(
        "--batch_size",
        type=int,
        default=2000,
        help="Batch size to use in matrix operations",
    )

    parser.add_argument(
        "--min_freq_words",
        type=int,
        default=1,
        help="Min number of embeddings that need to have representation "
        "for a word to be included in the meta-embedding vocabulary",
    )

    parser.add_argument(
        "--filter_words",
        action="store_true",
        help="Remove uppercase words if a lowercase version exists and words that include symbols",
    )

    parser.add_argument(
        "--normalize_method",
        type=str,
        choices=["l2", "none", "scale", "scale_old"],
        default="none",
        help="Embedding length normalization method.\n"
        "l2: Normalize length using l2 norm\n"
        "scale: Scale embeddings (embedding * scale_factor) to match the embedding with max average vector length\n"
        "none: Do not normalize embeddings",
    )

    parser.add_argument(
        "--debug",
        action="store_true",
        help="Print the knn that we will use to generate each OOV word",
    )

    args = parser.parse_args()

    if not os.path.exists(args.output_dir):
        os.makedirs(args.output_dir)

    oov_generator(
        embeddings_paths=args.embeddings,
        save_dir=args.output_dir,
        min_k_value=args.k_min_value,
        max_k_value=args.k_max_value,
        metric=args.metric,
        batch_size=args.batch_size,
        min_freq_words=args.min_freq_words,
        normalize_method=args.normalize_method,
        fp=args.fp,
        filter_words=args.filter_words,
        debug=args.debug,
    )


import datetime
from model.dataframe import Dataframe
from model.recommend import Recommend
from bottle import Bottle, route, run, request, response

app = Bottle()
@app.hook("after_request")
def enable_cors():
    """
    You need to add some headers to each request.
    Don"t use the wildcard "*" for Access-Control-Allow-Origin in production.
    """
    response.headers["Access-Control-Allow-Origin"] = "*"
    response.headers["Access-Control-Allow-Methods"] = "PUT, GET, POST, DELETE, OPTIONS"
    response.headers["Access-Control-Allow-Headers"] = "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token"

@app.route("/test", method=["OPTIONS", "POST"])
def hello():
    return "O servidor está online! Data/hora atual: " + str(datetime.datetime.now())

@app.route("/smartbuy", method=["POST"])
def getRecommendation():
    try:
        clientId = int(request.params.get("clientId"))
        month = int(request.params.get("month"))

        if (month < 1 or month > 12):
            return "Erro: mês inválido. Valor deve estar entre 1 e 12."

        if (clientId < 1):
            return "Erro: categoria inválida. Valor deve ser maior que zero."

        rc = Recommend("tcc", "processed")
        return rc.get(clientId, month)

    except Exception as ex:
        print(ex)
        return str(ex)

if __name__ == "__main__":
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("--host", dest="host", default="localhost",
                      help="hostname or ip address", metavar="host")
    parser.add_option("--port", dest="port", default=8080,
                      help="port number", metavar="port")
    (options, args) = parser.parse_args()
    run(app, host=options.host, port=int(options.port))

# run(host="localhost", port=8080, debug=True)


# -*- coding: utf-8 -*-
"""
  Name     : c7_03_random_OLS.py
  Book     : Python for Finance (2nd ed.)
  Publisher: Packt Publishing Ltd.
  Author   : Yuxing Yan
  Date     : 6/6/2017
  email    : yany@canisius.edu
             paulyxy@hotmail.com
"""

import numpy as np
import scipy as sp
import statsmodels.regression.linear_model as sm

n=100
sp.random.seed(12345)

y=[1,2,3,4,2,3,4]
x1=range(1,8)
x2=[4,2,-1,4,2,3,5]
x3=[0,2,3,4,2,4,-1]
x=[x1,x2,x3]
x=sm.add_constant(x)
#est = sm.OLS(formula='Sales ~ TV + Radio', data=df_adv).fit()
results=sm.OLS(y,x1).fit()
print(results.summary())


# code Sample from the tutorial at https://learncodeshare.net/2015/06/02/basic-crud-operations-using-cx_oracle/
# The following resets the data to the initial state for the tutorial.

import cx_Oracle
import os
connectString = os.getenv('DB_CONNECT') # The environment variable for the connect string: DB_CONNECT=user/password@database
con = cx_Oracle.connect(connectString)
cur = con.cursor()

# Delete rows
statement = 'delete from lcs_pets'
cur.execute(statement)

# Delete rows
statement = 'delete from lcs_people'
cur.execute(statement)

# Reset Identity Coulmn
statement = 'alter table lcs_people modify id generated BY DEFAULT as identity (START WITH 3)'
cur.execute(statement)

# Insert default rows
rows = [{'id':1, 'name':'Bob', 'age':35, 'notes':'I like dogs'}, {'id':2, 'name':'Kim', 'age':27, 'notes':'I like birds'}]
cur.bindarraysize = 2
cur.executemany("insert into lcs_people(id, name, age, notes) values (:id, :name, :age, :notes)", rows)
con.commit()

# Insert default rows
rows = [{'id':1, 'name':'Duke', 'owner':1, 'type':'dog'}, {'id':2, 'name':'Pepe', 'owner':2, 'type':'bird'}]
cur.bindarraysize = 2
cur.executemany("insert into lcs_pets (id, name, owner, type) values (:id, :name, :owner, :type)", rows)
con.commit()

cur.close()


'''A module for testing sum_of_sums module'''
import unittest
from sum_of_sums import sum_of_sums

class TestSumOfSums(unittest.TestCase):
    '''A class for testing sum_of_sums module'''
    def test_three_element_list(self):
        '''Test a list of three numbers'''
        nums = [3, 5, 2]
        overall_sum = sum_of_sums(nums)
        self.assertEqual(overall_sum, 21)

    def test_four_element_list(self):
        '''Test a list of four numbers'''
        nums = [1, 5, 7, 3]
        overall_sum = sum_of_sums(nums)
        self.assertEqual(overall_sum, 36)

    def test_single_element_list(self):
        '''Test a list of one number'''
        nums = [4]
        overall_sum = sum_of_sums(nums)
        self.assertEqual(overall_sum, 4)

    def test_five_element_list(self):
        '''Test a list of five numbers'''
        nums = [1, 2, 3, 4, 5]
        overall_sum = sum_of_sums(nums)
        self.assertEqual(overall_sum, 35)

if __name__ == '__main__':
    unittest.main()


import pygame
import sys
import os
import numpy as np
from PIL import Image
from notpi.stick import Stick

class NotPi:
    def __init__(self, scale=16, init_pygame=True):
        self.pixels = [[0, 0, 0]] * 64
        self.scale = scale
        self.size = width, height = 8*self.scale, 8*self.scale
        self.stick = Stick(pygame.event)
        self.low_light = False
        self.init_pygame = init_pygame
        if self.init_pygame:
            pygame.init()
            self.screen = pygame.display.set_mode(self.size)
        self.update()

    def __index_to_xy(self, index):
        """
        A helper function which returns (x, y) from an index.
        """
        return (index % 8, index // 8)

    def clear(self, *args):
        """
        Clears the LED matrix with a single colour, default is black / off
        e.g. ap.clear()
        or
        ap.clear(r, g, b)
        or
        colour = (r, g, b)
        ap.clear(colour)
        """

        black = [0, 0, 0]  # default

        if len(args) == 0:
            colour = black
        elif len(args) == 1:
            colour = list(args[0])
        elif len(args) == 3:
            colour = list(args)
        else:
            raise ValueError('Pixel arguments must be given as (r, g, b) or r, g, b')

        self.set_pixels([colour] * 64)

    def show_message(
        self,
        text_string,
        scroll_speed=.1,
        text_colour=[255, 255, 255],
        back_colour=[0, 0, 0]
        ):
        """
        Prints a string of text to the terminal. This is a simple replacement
        for sense-hat's implementation which scrolls a string of text across
        the LED matrix using the specified speed and colours
        """
        print(text_string)

    def show_letter(
        self,
        s,
        text_colour=[255, 255, 255],
        back_colour=[0, 0, 0]
        ):

        """
        Prints a single text character to the terminal. This is a simple
        replacement for sense-hat's implementation which displays a single
        text character on the LED matrix using the specified colours
        """

        if len(s) > 1:
            raise ValueError('Only one character may be passed into this method')

        print(s)

    def set_pixels(self, pixels):
        """
        Accepts a list containing 64 smaller lists of [R,G,B] pixels and
        updates the LED matrix. R,G,B elements must intergers between 0
        and 255
        """
        if len(pixels) != 64:
            raise ValueError('The pixel list must be 64 elements long.')

        self.pixels = pixels
        self.update()

    def get_pixels(self):
        """
        Returns a list containing 64 smaller lists of [R,G,B] pixels
        representing what is currently displayed on the LED matrix
        """

        return self.pixels

    def set_rotation(self, r=0, redraw=True):
        """
        Does nothing except update the screen. The sense-hat implementation
        sets the LED matrix rotation for viewing, adjust if the Pi is upside
        down or sideways. 0 is with the Pi HDMI port facing downwards
        """
        self.update()

    def set_pixel(self, x, y, *args):
        """
        Updates the single [R,G,B] pixel specified by x and y on the LED matrix
        Top left = 0,0 Bottom right = 7,7
        e.g. ap.set_pixel(x, y, r, g, b)
        or
        pixel = (r, g, b)
        ap.set_pixel(x, y, pixel)
        """
        pixel_error = 'Pixel arguments must be given as (r, g, b) or r, g, b'

        if len(args) == 1:
            pixel = args[0]
            if len(pixel) != 3:
                raise ValueError(pixel_error)
        elif len(args) == 3:
            pixel = args
        else:
            raise ValueError(pixel_error)

        if x > 7 or x < 0:
            raise ValueError('X position must be between 0 and 7')

        if y > 7 or y < 0:
            raise ValueError('Y position must be between 0 and 7')

        for element in pixel:
            if element > 255 or element < 0:
                raise ValueError('Pixel elements must be between 0 and 255')

        self.pixels[x + y*8] = list(pixel)
        self.update()

    def get_pixel(self, x, y):
        """
        Returns a list of [R,G,B] representing the pixel specified by x and y
        on the LED matrix. Top left = 0,0 Bottom right = 7,7
        """

        if x > 7 or x < 0:
            raise ValueError('X position must be between 0 and 7')

        if y > 7 or y < 0:
            raise ValueError('Y position must be between 0 and 7')

        return self.pixels[x + y*8]

    def load_image(self, file_path, redraw=True):
        """
        Accepts a path to an 8 x 8 image file and updates the LED matrix with
        the image
        """

        if not os.path.exists(file_path):
            raise IOError('%s not found' % file_path)

        img = Image.open(file_path).convert('RGB')
        pixel_list = list(map(list, img.getdata()))

        if redraw:
            self.set_pixels(pixel_list)

        return pixel_list

    def flip_v(self, redraw=True):
        """
        Flips the image on the canvas vertically
        """
        pixel_list = self.get_pixels()
        flipped = []
        for i in reversed(range(8)):
            offset = i * 8
            flipped.extend(pixel_list[offset:offset + 8])

        self.pixels = flipped
        if redraw:
            self.update()

    def flip_h(self, redraw=True):
        """
        Flips the image on the canvas horizontally
        """
        pixel_list = self.get_pixels()
        flipped = []
        for i in range(8):
            offset = i * 8
            flipped.extend(reversed(pixel_list[offset:offset + 8]))
        self.pixels = flipped
        if redraw:
            self.update()

    def update(self):
        """
        Updates the NotPi screen according to the state of self.pixels
        """
        if not self.init_pygame:
            return

        rects = []
        for idx, pixel in enumerate(self.pixels):
            x, y = self.__index_to_xy(idx)

            rects.append((pygame.Rect(x*self.scale, y*self.scale, self.scale, self.scale), pixel))

        self.screen.fill((0,0,0))
        for rect, color in rects:
            pygame.draw.rect(self.screen, color, rect)

        pygame.display.flip()


from app import Trie, cheapest_operator
import pytest

operators = Trie()
operators.insert({"1":0.92})

@pytest.mark.parametrize("test_input,expected", [("1",0.92)])
def test_trie_insert(test_input, expected):
    """test insert function """
    assert operators.lookup('1734567890') == expected

@pytest.mark.parametrize("test_input,expected", [("1",0.92)])
def test_trie_lookup(test_input, expected):
    # trie.insert({'1': 0.92})
    assert operators.lookup(test_input) == expected
    assert operators.lookup(test_input) is expected
    # assert trie.lookup('') is None

def test_cheapest_operator():
    operators_all = {
            'operatorA': operators,
        }
    number = '234567890'
    cheapest = cheapest_operator(number=number, operators=operators_all)
    assert cheapest[0] is None


import pywhatkit
import time
import webbrowser as web
from selenium import webdriver
from simon.accounts.pages import LoginPage
from simon.chat.pages import ChatPage
from simon.chats.pages import PanePage
from simon.header.pages import HeaderPage

from simon.accounts.pages import LoginPage
from simon.header.pages import HeaderPage
from simon.pages import BasePage
#poetry run pytest tests/test_whatsapp.py::test_send_msg -s
def test_send_msg():
    pywhatkit.sendwhatmsg_instantly("+56975858852", "Este mensaje viene del pasado y se proyecta en el futuro", 10, True, 5)

#poetry run pytest tests/test_whatsapp.py::test_read_msg  -s
def test_read_msg():
    # Creating the driver (browser)
    driver = webdriver.Chrome()
    driver.maximize_window()

    # 1. Login
    #       and uncheck the remember check box
    #       (Get your phone ready to read the QR code)
    login_page = LoginPage(driver)
    login_page.load()
    login_page.remember_me = True
    time.sleep(7)

    # 2. The base page is inherited by all pages
    #       and here you can check whether any
    #       page is available on the screen of
    #       the browser.

    # we don't need to load the pages as whatsapp
    # web works as one-page web app
    base_page = BasePage(driver)
    base_page.is_title_matches()
    base_page.is_welcome_page_available()
    base_page.is_nav_bar_page_available()
    base_page.is_search_page_available()
    base_page.is_pane_page_available()
    # chat is only available after you open one
    base_page.is_chat_page_available()

    # 3. Logout
    header_page = HeaderPage(driver)
    header_page.logout()

    # Close the browser
    driver.quit()

#https://github.com/Fantaso/whatsapp-web
#poetry run pytest tests/test_whatsapp.py::test_base -s
def test_base():
    # Creating the driver (browser)
    driver = webdriver.Chrome()
    browser = web.get('MacOSX')
    driver.maximize_window()
    # 1. Login
    #       and uncheck the remember check box
    #       (Get your phone ready to read the QR code)
    login_page = LoginPage(driver)
    login_page.load()
    login_page.remember_me = False
    time.sleep(7)