'''
===RNN Network in TensorFlow

* This is a Recurrent Neural Network for sarcasm detection
* Author: DATX02-25
'''
# Importing dependencies for file handling
import collections
import math
import random
import os
import glob
import json
import pickle
import csv
import sys
import tempfile
from common_funs import *

sys.path.append("../identifiera-sarkasm/")
sys.path.append("../identifiera-sarkasm/tfnetworks/")

import tfnetworks
from tfnetworks.networks import calc_seqlenth

import time

import random
# Importing tensorflow
import tensorflow as tf
# Importing rnn framework
from tensorflow.contrib import rnn
from tensorflow.contrib import learn
# Importing NumPy
import numpy as np

# Importing Settings
from config import Config

#This is 'fulhack'
class EarlyStoppingHelper:

    def __init__(self, epoch_threshold = 3, avg_limit_percent = 1.1):
        self.epochs_since_best = 0
        self.last_avg_loss = 0
        self.last_best_acc = 0
        self.last_best_f1 = 0
        self.losses = []
        self.epoch_threshold = epoch_threshold
        self.avg_limit_percent = avg_limit_percent

    def test_scorings(self,loss,accuracy,f1_score = 0):
        flags = {'update': False , 'passing': False, 'early_stop': False}
        # First Check accuracy scoring, if better, than reset counter and set new scores
        if f1_score and f1_score > self.last_best_f1 :
            self.epochs_since_best = 0
            self.last_best_f1 = f1_score
            flags['update'] = True
            return flags

        if not f1_score and accuracy > self.last_best_acc :
            self.epochs_since_best = 0
            self.last_best_acc = accuracy
            flags['update'] = True
            return flags

        self.epochs_since_best += 1

        if self.epochs_since_best > self.epoch_threshold :
            flags['early_stop'] = True
            return flags
        if len(self.losses) < self.epoch_threshold:
            avg_loss = loss
        else:
            avg_loss = sum(self.losses[self.epoch_threshold:])/self.epoch_threshold
            avg_loss = round(avg_loss,3)

        loss_limit = self.avg_limit_percent * avg_loss
        if loss > loss_limit:
            flags['early_stop'] = True
        else:
            self.losses.append(loss)
            flags['passing'] = True
        return flags

cfg = Config()
cfg = Config()
cfg.print_debug = True
cfg.predictions_filename = 'predictions.pickle'
cfg.model_save_name = None
n_classes = 2
chunk_size = cfg.embedding_size
n_chunks = cfg.max_sequence
roundform = "{0:.5f}"
stop_reason = None

data_placeholder = tf.placeholder(dtype=tf.int32,shape=[None,cfg.max_sequence])
predict_placeholder = tf.placeholder(dtype=tf.float32,shape=[None,None])
labels_placeholder = tf.placeholder(dtype=tf.float32, shape=[None,n_classes])
keep_prob_placeholder = tf.placeholder('float')
f1_score_placeholder = tf.placeholder('float')

trainable_embeddings = False
logs_path = os.path.join(tempfile.gettempdir() ,"tfnetwork")
shuffle_training = False
date_stamp = time.strftime("%d%b-%H%M")
slizing = False
monitor_f1 = False
stop_reason = None

dynamic = True

def _arg_callback_pt():
	cfg.print_test = True

def _arg_callback_ds(ds_name):
    """
    Select dataset
    """
    cfg.dataset_name = ds_name
    print("<Using dataset: {}>".format(ds_name))

def _arg_callback_eval(model_name=None):
    cfg.save_the_model = False
    #cfg.pretrained_model = True
    cfg.training_mode = "evaluate"
    cfg.pretrained_file = model_name if model_name != None else None
    global run_id
    run_id = model_name
    if model_name != None:
        print("<Using pretrained model " + model_name + " for results only >")

def _arg_callback_train(nr_epochs=1, count=1, batchsize=30):
    cfg.epochs = int(nr_epochs)
    cfg.run_count = int(count)
    cfg.batch_size = int(batchsize)
    cfg.training_mode = "training"
    print("<Training for, epochs: {}, runs:{}, batchsize: {}>".format(nr_epochs, count, batchsize))

def _arg_callback_net(name):
    cfg.network_name = name
    print("<Using network: {}>".format(name))

def _arg_callback_in(file_name):
    """
    Take preprocessed samples from the selected file
    """
    cfg.ps_file_name = file_name
    print("<Using processed samples from: {}>".format(cfg.samples_path))

def _arg_callback_trainemb(trainable = True):
    global trainable_embeddings
    if type(trainable) is str:
        trainable = trainable == "True"
    trainable_embeddings = trainable

def _arg_callback_eshuffle(truth = True):
	global shuffle_training
	if type(truth) is str:
		truth = truth == "True"
	shuffle_training = truth

def _arg_callback_slicing(slicing = True):
    global slizing
    if type(slicing) is str:
        slicing = slicing == "True"
    slizing = slicing

def _arg_callback_usef1(f1 = True):
    global monitor_f1
    if type(f1) is str:
        f1 = f1 == "True"
    monitor_f1 = f1

def _arg_callback_dynseq(dyn = False):
    global dynamic
    if type(dyn) is str:
        dyn = dyn == "True"
    dynamic = dyn
#def _arg_callback_ss(s_step = None, s_epoch = 'False'):
#	"""
#	Set the snapshot step
#	"""
#	global snapshot_step, snapshot_epoch
#	if isinstance(s_step, str) and s_step.lower() == 'none':
#		s_step = None
#	snapshot_step = int(s_step) if s_step is not None else None
#	snapshot_epoch = True if s_epoch.lower() == 'true' else False
#	print("<Snapshot step: {}, Snaphot epoch end: {}>".format(s_step, s_epoch))
## Create the embedding variable
def init_embedding(vocabulary_size,embedding_size,trainable = False):
    with tf.device("/cpu:0"):
        W = tf.Variable(tf.constant(0.0, shape = [vocabulary_size, embedding_size]),
                        trainable = trainable,
                        name = "W")
        embedding_placeholder = tf.placeholder(dtype = tf.float32,
                                               shape = [vocabulary_size, embedding_size]
                                               )
        embedding_init = W.assign(embedding_placeholder)
        return embedding_init, W, embedding_placeholder

# Setting the word embeddings
def set_embedding(sess,init,placeholder,embeddings):
	with tf.device("/cpu:0"):
		sess.run(init, feed_dict={placeholder: embeddings})
#Word embedding layer
def word_embedding_layer(word,embedding_tensor):
	with tf.device("/cpu:0"):
		embedding_layer = tf.nn.embedding_lookup(embedding_tensor,word, validate_indices=False)
	#	shape = [-1] + embedding_layer.get_shape().as_list()[1:3] + [1]
	#	embedding_layer.seq_length = calc_seqlenth(tf.reshape(word,shape))
		return embedding_layer #Not sure if this is done yet

#Defining and building the Neural Network

def train_neural_network(ps,emb_init,W,emb_placeholder,network_name,log_run,perflog):
	# Defining all the operations
	params = {'n_chunks': n_chunks}
	es_handler = EarlyStoppingHelper(epoch_threshold = 3, avg_limit_percent = 1.05)
	embeddings = word_embedding_layer(data_placeholder,W)
	network = tfnetworks.fetch_network(network_name,n_classes,params)
	print(network.get_name())
	prediction = network.feed_network(embeddings,keep_prob_placeholder,chunk_size,n_chunks,dynamic)
	accuracy = test_accuracy(tf.nn.softmax(prediction),labels_placeholder)
	val_accuracy_op = test_accuracy(predict_placeholder,labels_placeholder)
	cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = prediction, labels = labels_placeholder))

	val_cost_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = predict_placeholder, labels = labels_placeholder))
	l2_loss = network.calc_l2_loss()
	if trainable_embeddings:
		l2_loss = l2_loss + tf.nn.l2_loss(W)

	optimizer = tf.train.AdamOptimizer(learning_rate=0.0005).minimize(cost + 0.01 * l2_loss)

	#Defining the summaries
	train_acc_sum = tf.summary.scalar("Accuracy:", accuracy)
	#tf.summary.scalar("Optimizer", optimizer)
	train_loss_sum = tf.summary.scalar("Loss:", cost)
	val_acc_sum = tf.summary.scalar("val_accuracy:", val_accuracy_op)
	val_f1_sum = tf.summary.scalar("val_f1_score:", f1_score_placeholder)
	date = time.strftime("%b%d%y-%H%M%S")
	train_summary_op = tf.summary.merge([train_acc_sum,train_loss_sum])
	val_summary_op = tf.summary.merge([val_acc_sum,val_f1_sum])
	sess = tf.Session()
	width, _ = ps.train.xs.shape
	slice_size = width//cfg.epochs if cfg.epochs > 1 else width//10
	#xs_split,ys_split = split_chunks(ps.train.xs,cfg.batch_size, np.array(ps.train.ys))
	with sess.as_default():
		print("Current Network Model id: %s" % run_id )
		sess.run(tf.global_variables_initializer())
		set_embedding(sess,emb_init,emb_placeholder,emb)
		early_stop = False
		writer = tf.summary.FileWriter(os.path.join(logs_path,run_id),sess.graph)
		os.makedirs(os.path.join(".","models",run_id))
		print("Tensorboard log path:",logs_path)
		training_flags = {}
		for epoch in range(cfg.epochs):
			train_set,train_test = batch_slice(ps.train.xs,epoch*slice_size , slice_size, slizing)
			train_lab,train_labtest = batch_slice(np.array(ps.train.ys),epoch*slice_size,slice_size, slizing)
			if shuffle_training:
				train_set,train_lab = shuffle_data(train_set,train_lab)

			xs_split,ys_split = split_chunks(train_set,cfg.batch_size,train_lab)
			epoch_loss = 0
			loops = len(xs_split)
			#print(indices)
			print("\n=== BEGIN EPOCH",epoch+1, "===\n")
			for batch_i in range(loops):
				batch_x = xs_split[batch_i]
				batch_y = ys_split[batch_i]
				_, c,train_acc ,summary = sess.run([optimizer,cost,accuracy,train_summary_op], feed_dict = {data_placeholder: batch_x, labels_placeholder: batch_y, keep_prob_placeholder: 0.5})
				writer.add_summary(summary, epoch*loops + batch_i)
				epoch_loss += c

				print("Optimizer:",optimizer.name, "|", batch_i + 1, "batches completed out of:", loops)
				print("current loss:",roundform.format(epoch_loss),"| Accuracy :",roundform.format(train_acc), "",end=" \033[A\r",flush=True)

			print("\033[2B")
			print("VALIDATING TRAINING:...")
			val_labels = np.array(ps.valid.ys)
			val_predict = batchpredict(batch_size,ps.valid.xs,prediction)
			val_loss = val_cost_op.eval(feed_dict={predict_placeholder: val_predict, labels_placeholder: val_labels})
			val_f1_score = calc_f1_score(val_predict,val_labels)
			val_accuracy,summary = sess.run([val_accuracy_op,val_summary_op],feed_dict={predict_placeholder: val_predict,labels_placeholder: val_labels, keep_prob_placeholder: 1.0,f1_score_placeholder: val_f1_score})
			writer.add_summary(summary, epoch*loops)
			print('Epoch', epoch+1, 'completed out of', cfg.epochs, 'loss:', roundform.format(epoch_loss), '| Accuracy:', roundform.format(val_accuracy),'| F1-score:',roundform.format(val_f1_score))
			if monitor_f1:
				training_flags = es_handler.test_scorings(val_loss,val_accuracy,val_f1_score)
			else:
				training_flags = es_handler.test_scorings(val_loss,val_accuracy)
			if training_flags['update']:
				saver = tf.train.Saver()
				save_path = saver.save(sess, os.path.join(".","models",run_id,"Checkpoint.ckpt"))
				print("Checkpoint file saved in %s" % save_path )

			elif training_flags['early_stop']:
				print("Training will stop because of EarlyStopping")
				print("last save path is %s" % save_path)
				early_stop = True
				break

		stop_reason = "epochs_end" if not early_stop else "early_stop"
		saver = tf.train.Saver()
		if training_flags['update']:
			saver_path = saver.save(sess, os.path.join(".","models",run_id,"final.ckpt"))
			print("Model saved at %s" % saver_path )
		elif training_flags['passing']:
			print("Latest Checkpoint is the best")
		run_test_print_cm(ps,prediction,perflog,log_run)
		sess.close()

def split_chunks(xs,size,ys=None):
    xh,xw = xs.shape
    n_batches = xh//size
    ys_split = None
    if ys != None:
        yh,yw = ys.shape
        if xh != yh :
            raise ValueError("Sizes don't match")
        ys_split = split_chunks_helper(ys,n_batches,size)
    xs_split = split_chunks_helper(xs,n_batches,size)
    return xs_split,ys_split

def split_chunks_helper(xs,n_batches,size):
	xh, _ = xs.shape
	#print(n_batches)
	overflow = xh - (n_batches * size)
	#print(overflow)
	xs_overflow = xs[-overflow:]
	    #print(xs_overflow.shape)
	xs_notoverflow = xs[:xh - overflow]
	xs_split = np.split(xs_notoverflow,n_batches)
	xs_split.append(np.array(xs_overflow))
	return xs_split

# Method for validating network in training
def test_accuracy(prediction,labels):
    correct = tf.equal(tf.argmax(prediction,1), tf.argmax(labels,1))
    accuracy = tf.reduce_mean(tf.cast(correct,'float'))
    return accuracy

def test_network(ps,W,network_name,perflog,model=None):
    if model == None :
        print("No model selected")
        print("Your network %s have following matching models:" % network_name)
        print_matching_models(network_name)
    elif network_name not in model:
        print("Your selected model %s is not for your selected network: %s" % (path, network_name))
    else:
        path = os.path.join(".","models", model)
        run_test(ps,W,path,perflog,network_name)

def run_test(ps,W,model,perflog,network_name):
    params = {'n_chunks': n_chunks}
    network = tfnetworks.fetch_network(network_name,n_classes,params)
    test_data = ps.test.xs
    test_labels = np.array(ps.test.ys)
    embeddings = word_embedding_layer(data_placeholder,W)
    output = network.feed_network(embeddings,keep_prob_placeholder,chunk_size,n_chunks,dynamic)
    path = determine_model(model)
    with tf.Session().as_default() as sess:
        saver = tf.train.Saver()
        saver.restore(sess, path)
        run_test_print_cm(ps,output,perflog,log_run)

def determine_model(model_path):
    files = os.listdir(model_path)
    append = "final.ckpt" if "final.ckpt" in files else "Checkpoint.ckpt"
    return os.path.join(model_path,append)

def batch_slice(arr,start,size,diff = False):
    arr_slice = arr[start:start + size]
    if diff:
        diff_left = arr[:start]
        diff_right = arr[start+size:]
        arr = np.concatenate((diff_left,diff_right))

    return arr, arr_slice

def shuffle_data(xs,ys):
    if (len(xs) != len(ys)):
        raise ValueError("Dimensions must match")
    indices = list(range(len(xs)))
    random.shuffle(indices)
    shuffled_xs = []
    shuffled_ys = []
    for i in indices:
        shuffled_xs.append(xs[i])
        shuffled_ys.append(ys[i])

    return np.array(shuffled_xs),np.array(shuffled_ys)

def run_test_print_cm(ps,network_op,perflog,log_run):

    stop_reason = "Evaluation"
    sess = tf.get_default_session()
    cm = Binary_confusion_matrix()
    pred1 = batchpredict(90,ps.train.xs,network_op)
    cm.calc(ps.train.ids , pred1, ps.train.ys, 'training-set')
    pred2 = batchpredict(90,ps.valid.xs,network_op)
    cm.calc(ps.valid.ids , pred2, ps.valid.ys, 'validation-set')
    if cfg.print_test:
        pred3 = batchpredict(90,ps.test.xs,network_op)
        cm.calc(ps.test.ids , pred3, ps.test.ys, 'test-set')
        perflog.log(
        test_acc = cm.metrics['test-set']['accuracy'],
        test_f1 = cm.metrics['test-set']['f1_score']
        )

    cm.print_tables()
    cm.save_predictions(predictions_filename,
                        directory = 'logs',
                        sets=['training-set','validation-set','test-set'],
                        update = True)

    cm.save(run_id + '.res', content='metrics')
    log_run.log(cm.metrics, logname="metrics", aslist = False)

    perflog.log(
		time = time.strftime('%Y-%m-%d %H:%M', time.localtime()),
		network = cfg.network_name,
		dataset = cfg.dataset_name,
		samples_file = cfg.ps_file_name,
		val_acc = cm.metrics['validation-set']['accuracy'],
		val_f1 = cm.metrics['validation-set']['f1_score'],
		run_id = run_id
	)

	# for troublemaker analysis
    cm.save_predictions(cfg.predictions_filename,
                        directory = cfg.logs_path,
                        sets=['training-set','validation-set','test-set'],
                        update = True
                        )

def batchpredict(batch_size,data,network_op):

	sess = tf.get_default_session()
	data_batches, _ = split_chunks(data,batch_size)
	results = np.array([np.zeros(n_classes)])

	for batch in data_batches:
		pred = sess.run(network_op,feed_dict={data_placeholder: batch, keep_prob_placeholder: 1.0})
		results = np.concatenate((results,pred))
	results = np.delete(results, (0), axis = 0)
	return results

def print_matching_models(network_name):
    for file in os.listdir("models"):
        if network_name in file:
            print(file,end="\n")
# Here starts the program
def calc_f1_score(predictions,labels):
    predictions = np.argmax(predictions,1)
    labels = np.argmax(labels,1)
    tp = fp = tn = fn = 0
    for a,b in zip(predictions,labels):
        if a == 1:
            if a == b:
                tp += 1
            else:
                fp += 1
        else:
            if a == b:
                tn += 1
            else:
                fn += 1

    precision = ( tp / (tp + fp) ) if (tp + fp) > 0 else 0
    recall = ( tp / (tp + fn) ) if (tp + fn) > 0 else 0
    f1_score = 2*((precision * recall) / (precision + recall )) if (precision + recall) > 0 else 0
    return f1_score

#Argument handling, Copy paste from tflearn_rnn.py
arghandler = Arg_handler()
arghandler.register_flag('in', _arg_callback_in, ['input', 'in-file'], "Which file to take samples from. args: <filename>")
arghandler.register_flag('net', _arg_callback_net, ['network'], "Which network to use. args: <network name>")
arghandler.register_flag('train', _arg_callback_train, helptext = "Use settings for training. Args: <epochs> <run_count> <batch size>")
#arghandler.register_flag('ss', _arg_callback_ss, ['snapshot'], helptext = "Set snapshots. No arguments means no snapshots. Args: <snapshot step> <epoch end>")
arghandler.register_flag('eval', _arg_callback_eval, ['model_path'], "Evaluate the network performance of a pre-trained model specified by the name of the argument. args: <path>")
arghandler.register_flag('ds', _arg_callback_ds, ['select-dataset', 'dataset'], "Which dataset to use. Args: <dataset-name>")
arghandler.register_flag('pt', _arg_callback_pt, ['print-test'], "Produce results on test-partition of dataset.")
print("\n")
arghandler.register_flag('trainemb', _arg_callback_trainemb, ['trainable'], "Set trainable embeddings")
arghandler.register_flag('eshuffle', _arg_callback_eshuffle, ['truth'], "Want to shuffle per epoch?")
arghandler.register_flag('slicing', _arg_callback_slicing, ['slicing'], "Slice out the training accuracy set from the data")
arghandler.register_flag('usef1', _arg_callback_usef1, ['f1'], "Use F1 as validation matric instead of accuracy")
arghandler.register_flag('dynseq', _arg_callback_dynseq, ['dyn'], "Use dynamic sequencing. If setting this to false, recommend to run without --trainemb args: <dyn>")
arghandler.consume_flags()
predictions_filename = 'predictions.pickle'

batch_size = cfg.batch_size

network_name = cfg.network_name

run_id = date_stamp + "-" + network_name
perflog = DB_backed_log(cfg.sqlite_file, 'training_performance')

with open(cfg.samples_path, 'rb') as handle:
    pd = pickle.load( handle )

ps = pd.dataset #Processed Samples

if cfg.use_embeddings:
    emb = np.array(pd.embeddings[:pd.vocab_size], dtype=np.float32)
else:
    emb = np.random.randn(pd.vocab_size, pd.emb_size).astype(np.float32)

emb_init, W, emb_placeholder = init_embedding(pd.vocab_size, pd.emb_size, trainable_embeddings)

log_run = Logger()

log_run.log(network_name, logname='network_name', aslist = False)
log_run.log(cfg.ps_file_name, logname='Dataset', aslist = False)

if cfg.training_mode != "evaluate":
    train_neural_network(ps,emb_init,W,emb_placeholder,network_name,log_run,perflog)
else:
    test_network(ps,W,network_name,perflog,cfg.pretrained_file)

log_run.save(run_id + '.log')
if len(perflog.peek()) > 0:
    perflog.log(status = stop_reason)
    perflog.flush()

print ("=== Code ran Successfully ===")


from django.http import HttpRequest
from django.shortcuts import render, redirect
from .models import Listings
from .forms import ListingForm
from .filters import ListingFilter

def index(request: HttpRequest):
    return render(request, "listings/index.html")

def all_listings(request: HttpRequest):
    all_listings = Listings.objects.order_by("-list_date")

    my_filter = ListingFilter(request.GET, queryset=all_listings)
    all_listings = my_filter.qs

    context = {"all_listings": all_listings, "my_filter": my_filter}
    return render(request, "listings/all_listings.html", context=context)

def new_listing(request: HttpRequest):
    if request.method != "POST":
        form = ListingForm()
    else:
        form = ListingForm(request.POST, request.FILES)
        if form.is_valid():
            instance: Listings = form.save(commit=False)
            instance.user = request.user
            instance.save()
            return redirect("listings:my_listings")

    context = {"form": form}
    return render(request, "listings/new_listing.html", context)

def detail(request: HttpRequest, detail_id: int):
    listing = Listings.objects.get(pk=detail_id)
    context = {"detail": listing}
    return render(request, "listings/detail.html", context=context)

def my_listings(request: HttpRequest):
    if request.user.is_authenticated:
        listings = Listings.objects.filter(user=request.user).order_by("-list_date")
        context = {"listings": listings}
        return render(request, "listings/my_listings.html", context=context)
    return redirect("users:login")

def edit_listing(request: HttpRequest, edit_id: int):
    listing = Listings.objects.get(id=edit_id)

    if request.method != "POST":
        form = ListingForm(instance=listing)
    else:
        form = ListingForm(request.POST, request.FILES, instance=listing)
        if form.is_valid():
            form.save()
            return redirect("listings:my_listings")

    context = {"listing": listing, "form": form}
    return render(request, "listings/edit_listing.html", context)

def delete_listing(request: HttpRequest, delete_id: int):
    listing = Listings.objects.get(id=delete_id)
    if request.method == "POST":
        listing.delete()
        return redirect("listings:my_listings")
    context = {"listing": listing}
    return render(request, "listings/delete_listing.html", context)


import json
from channels.generic.websocket import WebsocketHandler
from asgiref.sync import async_to_sync

class ChatConsumer(WebsocketHandler):
    async def connect(self):
        self.room_group_name = 'test'

        await async_to_sync(self.channel_layer.group_add)(
            self.room_group_name,
            self.channel_name
        )
        await self.accept()

    async def disconnect(self, close_code):
        await self.channel_layer.group_discard(
            self.roomGroupName ,
            self.channel_layer
        )
    async def receive(self, text_data):
        text_data_json = json.loads(text_data)
        message = text_data_json["message"]
        username = text_data_json["username"]
        time = text_data_json["time"]
        await self.channel_layer.group_send(
            self.roomGroupName,{
                "type" : "sendMessage" ,
                "message" : message ,
                "username" : username ,
                "time": time
            })
    async def sendMessage(self , event) :
        message = event["message"]
        username = event["username"]
        time = event["time"]
        await self.send(text_data = json.dumps({"message":message ,"username":username, "time": time}))


from keras.models import Sequential
from keras.layers import Dense
import numpy as np
import sys, os, string

characters = string.printable
char_indices = dict((c, i) for i, c in enumerate(characters))
indices_char = dict((i, c) for i, c in enumerate(characters))

INPUT_VOCAB_SIZE = len(characters)
WINDOW_SIZE = 3

def encode_one_hot(line):
    line = " " + line + " "
    x = np.zeros((len(line), INPUT_VOCAB_SIZE))
    for i, c in enumerate(line):
        index = char_indices[c] if c in characters else char_indices[' ']
        x[i][index] = 1
    return x

def decode_one_hot(x):
    s = []
    for onehot in x:
        one_index = np.argmax(onehot)
        s.append(indices_char[one_index])
    return ''.join(s)

def prepare_for_window(x):
    # All slices of size WINDOW_SIZE, sliding through x
    ind = [np.array(np.arange(i, i+WINDOW_SIZE)) for i in range(x.shape[0] - WINDOW_SIZE + 1)]
    ind = np.array(ind, dtype=np.int32)
    x_window = x[ind]
    # Reshape it back to a 2-d tensor
    return x_window.reshape(x_window.shape[0], x_window.shape[1]*x_window.shape[2])

def normalization_layer_set_weights(n_layer):
    wb = []
    w = np.zeros((WINDOW_SIZE*INPUT_VOCAB_SIZE, INPUT_VOCAB_SIZE))
    b = np.zeros((INPUT_VOCAB_SIZE))
    # Let lower case letters go through
    for c in string.ascii_lowercase:
        i = char_indices[c]
        w[INPUT_VOCAB_SIZE+i, i] = 1
    # Map capitals to lower case
    for c in string.ascii_uppercase:
        i = char_indices[c]
        il = char_indices[c.lower()]
        w[INPUT_VOCAB_SIZE+i, il] = 1
    # Map all non-letters to space
    sp_idx = char_indices[' ']
    non_letters = [c for c in list(characters) if c not in list(string.ascii_letters)]
    for c in non_letters:
        i = char_indices[c]
        w[INPUT_VOCAB_SIZE+i, sp_idx] = 1
    # Map single letters to space
    for c in non_letters:
        i = char_indices[c]
        w[i, sp_idx] = 0.75
        w[INPUT_VOCAB_SIZE*2+i, sp_idx] = 0.75

    wb.append(w)
    wb.append(b)
    n_layer.set_weights(wb)
    return n_layer

def build_model():
    # Normalize characters using a dense layer
    model = Sequential()
    model.add(Dense(INPUT_VOCAB_SIZE,
                    input_shape=(WINDOW_SIZE*INPUT_VOCAB_SIZE,),
                    activation='softmax'))
    return model

model = build_model()
model.summary()
normalization_layer_set_weights(model.layers[0])

with open(sys.argv[1]) as f:
    for line in f:
        if line.isspace(): continue
        batch = prepare_for_window(encode_one_hot(line))
        preds = model.predict(batch)
        normal = decode_one_hot(preds)
        print(normal)


__author__ = 'e0d'

"""
Retrieves AWS Auto-scaling lifecycle messages from and SQS queue and processes them.  For
the LifeCycleTransition type of autoscaling:EC2_INSTANCE_TERMINATING, ec2 instances are inspected
for an ok_to_retire tag.  If that tag exists, the termination state transition is continued, if not, the
lifecycle timeout is extended.

Because the lifecycle commands are not yet available in boto, these commands are, unfortunately,
run via a subprocess call to the awscli.  This should be fixed when boto is updated.

This script is meant to be run periodically via some process automation, say, Jenkins.

It relies on some component applying the proper tags and performing pre-retirement activities.

./sqs.py -q autoscaling-lifecycle-queue -b /home/you/.virtualenvs/aws/bin --hook MyLifeCycleHook
"""

import argparse
import boto3
import json
import subprocess
import logging
import os
from distutils import spawn

class MissingHostError(Exception):
    pass

class LifecycleHandler:

    INSTANCE_TERMINATION = 'autoscaling:EC2_INSTANCE_TERMINATING'
    TEST_NOTIFICATION = 'autoscaling:TEST_NOTIFICATION'
    NUM_MESSAGES = 10
    WAIT_TIME_SECONDS = 1
    VISIBILITY_TIMEOUT = 10

    def __init__(self, region, queue, hook, dry_run, bin_directory=None):
        logging.basicConfig(level=logging.INFO)
        self.queue = queue
        self.hook = hook
        self.region = region
        if bin_directory:
            os.environ["PATH"] = bin_directory + os.pathsep + os.environ["PATH"]
        self.aws_bin = spawn.find_executable('aws')
        self.python_bin = spawn.find_executable('python')

        self.base_cli_command ="{python_bin} {aws_bin} ".format(
            python_bin=self.python_bin,
            aws_bin=self.aws_bin)

        if self.region:
            self.base_cli_command += f"--region {self.region} "

        self.dry_run = args.dry_run
        self.ec2_con = boto3.client('ec2',region_name=self.region)
        self.sqs_con = boto3.client('sqs',region_name=self.region)

    def process_lifecycle_messages(self):
        queue_url = self.sqs_con.get_queue_url(QueueName=self.queue)['QueueUrl']
        queue = boto3.resource('sqs', region_name=self.region).Queue(queue_url)

        for sqs_message in self.sqs_con.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=LifecycleHandler.NUM_MESSAGES, VisibilityTimeout=LifecycleHandler.VISIBILITY_TIMEOUT,
                                              WaitTimeSeconds=LifecycleHandler.WAIT_TIME_SECONDS).get('Messages', []):
            body = json.loads(sqs_message['Body'])
            as_message = json.loads(body['Message'])
            logging.info(f"Proccessing message {as_message}.")

            if 'LifecycleTransition' in as_message and as_message['LifecycleTransition'] \
                    == LifecycleHandler.INSTANCE_TERMINATION:
                # Convenience vars, set here to avoid messages that don't meet the criteria in
                # the if condition above.
                instance_id = as_message['EC2InstanceId']
                asg = as_message['AutoScalingGroupName']
                token = as_message['LifecycleActionToken']

                try:

                    if self.verify_ok_to_retire(as_message['EC2InstanceId']):

                        logging.info("Host is marked as OK to retire, retiring {instance}".format(
                            instance=instance_id))

                        self.continue_lifecycle(asg, token, self.hook)

                        self.delete_sqs_message(queue, sqs_message, as_message, self.dry_run)

                    else:
                        logging.info("Recording lifecycle heartbeat for instance {instance}".format(
                            instance=instance_id))

                        self.record_lifecycle_action_heartbeat(asg, token, self.hook)
                except MissingHostError as mhe:
                    logging.exception(mhe)
                    # There is nothing we can do to recover from this, so we
                    # still delete the message
                    self.delete_sqs_message(queue, sqs_message, as_message, self.dry_run)

            # These notifications are sent when configuring a new lifecycle hook, they can be
            # deleted safely
            elif as_message['Event'] == LifecycleHandler.TEST_NOTIFICATION:
                self.delete_sqs_message(queue, sqs_message, as_message, self.dry_run)
            else:
                raise NotImplemented("Encountered message, {message_id}, of unexpected type.".format(
                    message_id=as_message['MessageId']))

    def delete_sqs_message(self, queue, sqs_message, as_message, dry_run):
        if not dry_run:
            logging.info(f"Deleting message with body {as_message}")
            self.sqs_con.delete_message(QueueUrl=queue.url, ReceiptHandle=sqs_message['ReceiptHandle'])
        else:
            logging.info(f"Would have deleted message with body {as_message}")

    def record_lifecycle_action_heartbeat(self, asg, token, hook):

        command = self.base_cli_command + "autoscaling record-lifecycle-action-heartbeat " \
                  "--lifecycle-hook-name {hook} " \
                  "--auto-scaling-group-name {asg} " \
                  "--lifecycle-action-token {token}".format(
            hook=hook,asg=asg,token=token)

        self.run_subprocess_command(command, self.dry_run)

    def continue_lifecycle(self, asg, token, hook):
        command = self.base_cli_command + "autoscaling complete-lifecycle-action --lifecycle-hook-name {hook} " \
                  "--auto-scaling-group-name {asg} --lifecycle-action-token {token} --lifecycle-action-result " \
                  "CONTINUE".format(
                hook=hook, asg=asg, token=token)

        self.run_subprocess_command(command, self.dry_run)

    def run_subprocess_command(self, command, dry_run):

        message = f"Running command {command}."

        if not dry_run:
            logging.info(message)
            try:
                output = subprocess.check_output(command.split(' '))
                logging.info(f"Output was {output}")
            except Exception as e:
                logging.exception(e)
                raise  e
        else:
            logging.info(f"Dry run: {message}")

    def get_ec2_instance_by_id(self, instance_id):
        """
        Simple boto call to get the instance based on the instance-id
        """
        reservations = self.ec2_con.describe_instances(InstanceIds=[instance_id]).get('Reservations', [])
        instances = []
        if len(reservations) == 1:
            instances = reservations[0].get('Instances', [])
        if len(instances) == 1:
            return self.ec2_con.describe_instances(InstanceIds=[instance_id])['Reservations'][0]['Instances'][0]
        else:
            return None

    def verify_ok_to_retire(self, instance_id):
        """
        Ensure that the safe_to_retire tag has been added to the instance in question
        with the value 'true'
        """
        instance = self.get_ec2_instance_by_id(instance_id)
        tags_dict = {}

        if instance:
            tags_dict = {}
            for t in instance['Tags']:
                tags_dict[t['Key']] = t['Value']
            if 'safe_to_retire' in tags_dict and tags_dict['safe_to_retire'].lower() == 'true':
                logging.info(f"Instance with id {instance_id} is safe to retire.")
                return True
            else:
                logging.info(f"Instance with id {instance_id} is not safe to retire.")
                return False
        else:
            # No instance for id in SQS message this can happen if something else
            # has terminated the instances outside of this workflow
            message = "Instance with id {id} is referenced in an SQS message, but does not exist.".\
                format(id=instance_id)
            raise MissingHostError(message)

if __name__=="__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-r', '--region',
                        help='The aws region to use '
                             'per line.',default='us-east-1')
    parser.add_argument('-b', '--bin-directory', required=False, default=None,
                        help='The bin directory of the virtual env '
                             'from which to run the AWS cli (optional)')
    parser.add_argument('-q', '--queue', required=True,
                        help="The SQS queue containing the lifecyle messages")

    parser.add_argument('--hook', required=True,
                        help="The lifecyle hook to act upon.")

    parser.add_argument('-d', "--dry-run", dest="dry_run", action="store_true",
                        help='Print the commands, but do not do anything')
    parser.set_defaults(dry_run=False)
    args = parser.parse_args()

    lh = LifecycleHandler(args.region, args.queue, args.hook, args.dry_run, args.bin_directory)
    lh.process_lifecycle_messages()


import six
if six.PY2:
    from urllib2 import urlopen
else:
    from urllib.request import urlopen, HTTPError, URLError
import logging
import os
import signal
import http.server
import socketserver
import time

logger = logging.getLogger(__name__)
logger.setLevel(logging.CRITICAL)

class SimpleServerHandler(http.server.SimpleHTTPRequestHandler):
    """
    The handler for the web server. It has the same functionality as the
    server for protractor's test app.
    """

    def send_text_response(self, text):
        self.send_response(200)
        self.send_header("Content-type", 'text/plain')
        self.send_header("Content-Length", len(text))
        self.end_headers()
        self.wfile.write(text.encode('utf-8'))

    def log_message(self, msg_format, *args):
        """Use python logging to avoid lots of output during testing."""
        logger.info("TESTSERVER: %s - - [%s] %s\n" %
                    (self.client_address[0],
                     self.log_date_time_string(),
                     msg_format % args))

class SimpleWebServerProcess(object):
    """
    A simple webserver for serving pages for testing.
    """
    HOST = '0.0.0.0'
    PORT = 9999
    APP_DIR = 'ng-app/app/'
    _pid = None

    def get_module_path(self):
        return os.path.dirname(__file__)

    def get_handler(self):
        return SimpleServerHandler

    def get_application_path(self):
        return os.path.join(self.get_module_path(), self.APP_DIR)

    def run(self):
        self._pid = os.fork()
        if self._pid == 0:
            self.start_server()
            exit() #ensure the child process exits instead of running parent's program
        else:
            logger.debug('Started webserver child as pid {} on'
                         ' port {}'.format(self._pid, self.PORT))
            # wait 5 seconds for server to start
            connected = False
            elapsed = 0
            while (not connected) and (elapsed < 5):
                try:
                    urlopen('http://localhost:{}/'.format(self.PORT), timeout=5)
                    connected = True
                except HTTPError:
                    connected = True
                except URLError:
                    time.sleep(0.1)
                    elapsed += 0.1
            if not connected:
                self.stop()
                raise Exception('webserver: fork failed')

    def start_server(self):
        server_path = self.get_application_path()
        logger.debug('Starting webserver for path {} on'
                     ' port {}'.format(server_path, self.PORT))
        os.chdir(server_path)
        handler = self.get_handler()
        socketserver.TCPServer.allow_reuse_address = True
        httpd = socketserver.TCPServer((self.HOST, self.PORT), handler)
        httpd.serve_forever()

    def stop(self):
        if self._pid != 0:
            logger.debug('Sending SIGTERM to webserver child with'
                         ' pid {}'.format(self._pid))
            os.kill(self._pid, signal.SIGTERM)
            os.waitpid(self._pid, 0)


from flask import Flask, render_template, url_for, request, redirect, flash, jsonify
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from database_setup import Restaurant, Base, MenuItem

app = Flask(__name__)

#set up to read the data base
engine = create_engine('sqlite:///restaurantmenu.db', connect_args={'check_same_thread': False}, echo=True)
Base.metadata.bind = engine
DBsession = sessionmaker(bind = engine)
session = DBsession()

#home page, list of menu
@app.route('/', methods = ['GET', 'POST'])
def HomePage():
	if request.method == 'POST':
		option = request.form['option']
		if option == 'restaurants':
			return redirect(url_for('ShowRestaurants'))
		else:
			return redirect(url_for('ShowMenu'))

	else:
		return render_template('homePage.html')

@app.route('/restaurants')
def ShowRestaurants():
	restaurants = session.query(Restaurant).all()
	return render_template('restaurants.html', restaurants = restaurants)

@app.route('/menus')
def ShowMenu():
	menus = session.query(MenuItem).all()
	return render_template('menu.html', menus = menus)

#show menu of a restaurant
@app.route('/restaurants/<int:restaurantID>/menu')
def ShowRestaurantMenu(restaurantID):
	restaurant = session.query(Restaurant).filter_by(id = restaurantID).one()
	menu_items = session.query(MenuItem).filter_by(restaurant_id = restaurantID).all()
	return render_template('restaurant_menu.html',
		restaurant = restaurant, menus = menu_items)

#add new menu item
@app.route('/menu/new/<int:restaurant_id>', methods = ['GET', 'POST'])
def NewMenu(restaurant_id):
	restaurant = session.query(Restaurant).filter_by(id = restaurant_id).one()
	if request.method == 'POST':
		menu_name = request.form['name']
		new_menu = MenuItem(name = menu_name, restaurant = restaurant)
		session.add(new_menu)
		session.commit()
		flash('New menu item is created. Add more?')
		return redirect(url_for('ShowRestaurantMenu', restaurantID = restaurant.id))

	# GET request
	else:
		return render_template('addMenu.html', restaurant = restaurant)

#edit menu item
@app.route('/menu/edit/<int:restaurant_id>/<int:menu_id>', methods = ['GET', 'POST'])
def EditMenu(restaurant_id, menu_id):
	restaurant = session.query(Restaurant).filter_by(id = restaurant_id).one()
	menu = session.query(MenuItem).filter_by(id = menu_id).one()
	if request.method == 'POST':
		new_name = request.form['new_name']
		menu.name = new_name
		session.add(menu)
		session.commit()
		return redirect(url_for('ShowRestaurantMenu', restaurantID = restaurant.id))

	else:
		return render_template('edit.html', restaurant = restaurant, menu = menu)

#delete menu item
@app.route('/menu/delete/<int:restaurant_id>/<int:menu_id>', methods = ['POST', 'GET'])
def DeleteMenu(restaurant_id, menu_id):
	restaurant = session.query(Restaurant).filter_by(id = restaurant_id).one()
	menu = session.query(MenuItem).filter_by(id = menu_id).one()
	if request.method == 'POST':
		option = request.form['option']
		if option == 'yes':
			session.delete(menu)
			session.commit()
			flash('Item has been deleted!')
		return redirect(url_for('ShowRestaurantMenu', restaurantID = restaurant.id))

	else:
		return render_template('delete.html', restaurant = restaurant, menu = menu)

#making API endpoint for GET request
@app.route('/restaurants/<int:restaurant_id>/menu/JSON')
def menuJSON(restaurant_id):
	restaurant = session.query(Restaurant).filter_by(id = restaurant_id).one()
	menu_items = session.query(MenuItem).filter_by(restaurant_id = restaurant_id).all()
	return jsonify(MenuItems = [i.serialize for i in menu_items])

@app.route('/restaurants/<int:restaurant_id>/menu/<int:menu_id>/JSON')
def itemJSON(restaurant_id, menu_id):
	restaurant = session.query(Restaurant).filter_by(id = restaurant_id).one()
	item = session.query(MenuItem).filter_by(id = menu_id).one()
	return jsonify(Item = item.serialize)

if __name__ == '__main__':
	app.secret_key = 'strong_password'
	app.debug = True
	app.run(host = '', port = 6969)


from django.urls import path

from api.views.tasks_api_view import TaskSimpleView, TaskUpdateView, TaskDeleteSimpleView

from api.views.project_api_view import ProjectSimpleView, ProjectUpdateView, ProjectDeleteSimpleView

urlpatterns =[
    path('tasks/detail/<int:pk>', TaskSimpleView.as_view(), name="tasks_detail"),
    path('tasks/resources/<int:pk>', TaskUpdateView.as_view(), name="tasks_update"),
    path('tasks/delete/<int:pk>', TaskDeleteSimpleView.as_view(), name="tasks_delete"),
    path('project/detail/<int:pk>', ProjectSimpleView.as_view(), name="project_detail"),
    path('project/resources/<int:pk>', ProjectUpdateView.as_view(), name="project_update"),
    path('project/delete/<int:pk>', ProjectDeleteSimpleView.as_view(), name="project_delete"),
]


import copy
from typing import Callable

import d20
import draconic
from d20.utils import TreeType

import aliasing.api.statblock
from cogs5e.models.sheet.statblock import StatBlock

def maybe_alias_statblock(target):
    """Returns the AliasStatBlock for the target if applicable."""
    if not isinstance(target, (StatBlock, str, type(None))):
        raise ValueError("target must be a statblock, str, or None")
    if isinstance(target, StatBlock):
        return aliasing.api.statblock.AliasStatBlock(target)

    return aliasing.api.statblock.AliasStatBlock(StatBlock(name=target or "Target"))

def upcast_scaled_dice(effect, autoctx, dice_ast):
    """Scales the dice of the cast to its appropriate amount (handling cantrip scaling and higher level addition)."""
    if effect.cantripScale:
        level = autoctx.caster.spellbook.caster_level
        if level < 5:
            level_dice = 1
        elif level < 11:
            level_dice = 2
        elif level < 17:
            level_dice = 3
        else:
            level_dice = 4
        level_dice = autoctx.args.last("cantripdice", default=level_dice, type_=int)

        def mapper(node):
            if isinstance(node, d20.ast.Dice):
                node.num = level_dice
            return node

        dice_ast = d20.utils.tree_map(mapper, dice_ast)

    if effect.higher:
        higher = effect.higher.get(str(autoctx.get_cast_level()))
        if higher:
            higher_ast = d20.parse(higher)
            dice_ast.roll = d20.ast.BinOp(dice_ast.roll, "+", higher_ast.roll)

    return dice_ast

def mi_mapper(minimum: int) -> Callable[[d20.ast.Node], d20.ast.Node]:
    """Returns a function that maps Dice AST objects to OperatedDice with miX attached."""

    def mapper(node: d20.ast.Node):
        if isinstance(node, d20.ast.Dice):
            miX = d20.ast.SetOperator("mi", [d20.ast.SetSelector(None, int(minimum))])
            return d20.ast.OperatedDice(node, miX)
        return node

    return mapper

def max_mapper(node: d20.ast.Node):
    """A function that maps Dice AST objects to OperatedDice that set their values to their maximum."""
    if isinstance(node, d20.ast.Dice):
        miX = d20.ast.SetOperator("mi", [d20.ast.SetSelector(None, node.size)])
        return d20.ast.OperatedDice(node, miX)
    return node

def max_add_crit_mapper(node: d20.ast.Node):
    """A function that adds the maximum value of each Dice AST node as a literal

    :return: A tuple containing the node and a bool for if the tree mapper should continue.
    :rtype: tuple(node, bool)
    """
    if isinstance(node, d20.ast.OperatedDice):
        return d20.ast.BinOp(node, "+", d20.ast.Literal(node.value.num * node.value.size)), False
    if isinstance(node, d20.ast.Dice):
        return d20.ast.BinOp(node, "+", d20.ast.Literal(node.num * node.size)), False
    return node, True

def crit_mapper(node: d20.ast.Node):
    """A function that doubles the number of dice for each Dice AST node."""
    if isinstance(node, d20.ast.Dice):
        return d20.ast.Dice(node.num * 2, node.size)
    return node

def double_dice_crit_mapper(node: d20.ast.Node):
    """A function that replaces each Dice AST node with itself multiplied by 2.

    :return: A tuple containing the node and a bool for if the tree mapper should continue.
    :rtype: tuple(node, bool)
    """
    if isinstance(node, (d20.ast.OperatedDice, d20.ast.Dice)):
        return d20.ast.BinOp(node, "*", d20.ast.Literal(2)), False
    return node, True

def crit_dice_gen(dice_ast: d20.ast.Node, critdice: int):
    """A function that finds the size of left most Dice AST node and generates crit dice based on that, for
    crit types that double all original dice, but not any additional crit dice. By finding the left most dice,
    we do our best to ensure its based on the weapon/source and not any other additional bonuses."""
    left = d20.utils.leftmost(dice_ast)

    # if we're at the bottom of the branch and it's the dice, add *critdice*
    if isinstance(left, d20.ast.Dice):
        return d20.ast.Dice(critdice, left.size)

def critdice_tree_update(dice_ast: d20.ast.Node, critdice: int):
    """
    Modifies the AST by adding *critdice* dice to any leftmost Dice, branching recursively on any Set.

    .. note::
        This mutates the AST, so it should be copied before calling to avoid mutating the cached AST.
    """
    left = dice_ast
    while left.children:
        # if we encounter a set going down the left branch, branch and run recursively on all children
        if isinstance(left, d20.ast.NumberSet):
            for child in left.children:
                critdice_tree_update(child, critdice)
            return
        # otherwise continue down the left branch
        left = left.children[0]

    # if we're at the bottom of the branch and it's the dice, add *critdice*
    if isinstance(left, d20.ast.Dice):
        left.num += critdice

def stringify_intexpr(evaluator, expr):
    """
    For use in str builders - use the given evaluator to return the result of the intexpr, or nan if any exception is
    caught

    :rtype: int or float
    """
    if isinstance(expr, (int, float)):
        return expr

    try:
        return int(evaluator.eval(str(expr)))
    except (TypeError, ValueError, draconic.DraconicException):
        return float("nan")

def target_hp_or_default(target, default):
    """Returns the target's hp if defined, otherwise default"""
    if isinstance(target, StatBlock) and target.hp is not None:
        return target.hp
    else:
        return default

def tree_map_prefix(func: Callable[[TreeType], tuple[TreeType, bool]], node: TreeType) -> TreeType:
    """
    Returns a copy of the tree, with each node replaced with ``func(node)[0]``.

    :param func: A transformer function that takes a node and returns a tuple (replacement,
                 continue_operations_on_children).
    :param node: The root of the tree to transform.
    """
    copied = copy.copy(node)
    operated, continue_operations_on_children = func(copied)
    if not continue_operations_on_children:
        # we still recurse on the children so that it satisfies the "returns a copy of the tree" property
        # so make the function a no-op
        func = lambda x: (x, True)  # noqa E731
    for i, child in enumerate(copied.children):
        copied.set_child(i, tree_map_prefix(func, child))
    return operated


import requests
import pandas as pd

class PseAPI:

    def get_data(self, ticker, start_date, end_date):
        ticker = ticker.upper()
        try:
            latest = requests.get(f'http://phisix-api.appspot.com/stocks/{ticker}.json', timeout=10).json()
            last_price = latest['stock'][0]['price']['amount']
            my_request = f'https://pselookup.vrymel.com/api/stocks/{ticker}/history/{start_date}/{end_date}'
            print(my_request)
            data = requests.get(my_request, timeout=10).json()[
                'history']
            data[-1]['close'] = last_price
            return pd.json_normalize(data)
        except Exception as e:
            raise Exception(f'Error occurred while processing {ticker}, {e}')


# coding=utf-8
import cv2
import pytesseract
import os
import textdistance
import numpy as np
import pandas as pd
from operator import itemgetter, attrgetter

# 路径配置
ROOT_PATH = os.getcwd()
IMAGE_PATH = os.path.join(ROOT_PATH, 'images/ktp6.png')
LINE_REC_PATH = os.path.join(ROOT_PATH, 'data/ID_CARD_KEYWORD.csv')
CITIES_REC_PATH = os.path.join(ROOT_PATH, 'data/CITIES.csv')
RELIGION_REC_PATH = os.path.join(ROOT_PATH, 'data/RELIGION.csv')
MARRIAGE_REC_PATH = os.path.join(ROOT_PATH, 'data/MARRIAGE.csv')
NEED_COLON = [3, 4, 6, 8, 10, 11, 12, 13, 14, 15, 17, 18, 19, 21]
NEXT_LINE = 9
ID_NUMBER = 3

# 对身份证上的信息进行初步OCR
def ocr_raw(image_path):
    # (1) Read
    img_raw = cv2.imread(image_path)
    image = cv2.resize(img_raw, (50 * 16, 500))
    img_gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    id_number = return_id_number(image, img_gray)
    # img_gray = cv2.cvtColor(img_raw, cv2.COLOR_BGR2GRAY)
    # (2) Threshold
    cv2.fillPoly(img_gray, pts=[np.asarray([(540, 150), (540, 499), (798, 499), (798, 150)])], color=(255, 255, 255))
    th, threshed = cv2.threshold(img_gray, 127, 255, cv2.THRESH_TRUNC)
    # (3) Detect
    result_raw = pytesseract.image_to_string(threshed, lang="ind")
    return result_raw, id_number

# 去除result_raw中的无效信息
def strip_op(result_raw):
    result_list = result_raw.split('\n')
    new_result_list = []
    for tmp_result in result_list:
        if tmp_result.strip(' '):
            new_result_list.append(tmp_result)
    return new_result_list

# KTP_ number
# NIK_NUMBER模版匹配识别
# 从左到右进行排列识别
def sort_contours(cnts, method="left-to-right"):
    reverse = False
    i = 0
    if method == "right-to-left" or method == "bottom-to-top":
        reverse = True
    if method == "top-to-bottom" or method == "bottom-to-top":
        i = 1
    boundingBoxes = [cv2.boundingRect(c) for c in cnts]  # 用一个最小的矩形，把找到的形状包起来x,y,h,w
    (cnts, boundingBoxes) = zip(*sorted(zip(cnts, boundingBoxes),
                                        key=lambda b: b[1][i], reverse=reverse))
    return cnts, boundingBoxes

def return_id_number(image, gray):
    # ret, thresh1 = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)#二值处理
    # 定义核函数
    rectKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (15, 15))  # 返回矩形核函数
    # sqKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (15, 15))
    tophat = cv2.morphologyEx(gray, cv2.MORPH_TOPHAT, rectKernel)  # 顶帽
    gradX = cv2.Sobel(tophat, ddepth=cv2.CV_32F, dx=1, dy=0, ksize=-1)
    # ksize=-1相当于用3*3的 sobel过滤器，由于 Sobel 函数求导数后会有负值，还会有大于 255 的值，而原图像是 uint8 ，
    # 所以 Sobel 建立的图像位数不够，会有截断。因此要使用 16 位有符号的数据类型，即 cv2.CV_16S
    gradX = np.absolute(gradX)
    (minVal, maxVal) = (np.min(gradX), np.max(gradX))
    gradX = (255 * ((gradX - minVal) / (maxVal - minVal)))
    gradX = gradX.astype("uint8")  # Sobel之后无法显示，将其转回原来的 uint8 形式，否则无法显示图像
    # print (np.array(gradX).shape)
    # 通过闭操作（先膨胀，再腐蚀）将数字连在一起
    gradX = cv2.morphologyEx(gradX, cv2.MORPH_CLOSE, rectKernel)
    # THRESH_OTSU会自动寻找合适的阈值，适合双峰，需把阈值参数设置为0
    thresh = cv2.threshold(gradX, 0, 255,
                           cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
    # 再来一个闭操作
    thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, rectKernel)  # 再来一个闭操作
    threshCnts, hierarchy = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)  # 寻找轮廓
    cnts = threshCnts
    cur_img = image.copy()
    cv2.drawContours(cur_img, cnts, -1, (0, 0, 255), 3)
    copy = image.copy()
    locs = []
    # 遍历轮廓
    for (i, c) in enumerate(cnts):
        # 计算矩形
        (x, y, w, h) = cv2.boundingRect(c)
        ar = w / float(h)
        # 选择合适的区域，根据实际任务来，这里的基本都是四个数字一组
        # if ar >10:
        # if (w > 40 ) and (h > 10 and h < 20):
        # 符合的留下来
        if h > 10 and w > 100 and x < 300:
            img = cv2.rectangle(copy, (x, y), (x + w, y + h), (0, 255, 0), 2)
            locs.append((x, y, w, h, w * h))
    # 将符合的轮廓按照面积从大到小排序
    locs = sorted(locs, key=itemgetter(1), reverse=False)

    # print(locs[1][0])
    nik = image[locs[1][1] - 10:locs[1][1] + locs[1][3] + 10, locs[1][0] - 10:locs[1][0] + locs[1][2] + 10]
    # cv_show('nik', nik)

    # print(nik)
    text = image[locs[2][1] - 10:locs[2][1] + locs[2][3] + 10, locs[2][0] - 10:locs[2][0] + locs[2][2] + 10]

    # 读取一个模板图像
    img = cv2.imread("images/module.jpeg")
    # 灰度图
    ref = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # 二值图像
    ref = cv2.threshold(ref, 66, 255, cv2.THRESH_BINARY_INV)[1]
    # 计算轮廓
    # cv2.findContours()函数接受的参数为二值图，即黑白的（不是灰度图）,cv2.RETR_EXTERNAL只检测外轮廓，cv2.CHAIN_APPROX_SIMPLE只保留终点坐标
    # 返回的list中每个元素都是图像中的一个轮廓
    refCnts, hierarchy = cv2.findContours(ref.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    # cv2.drawContours(img, refCnts, -1, (0, 0, 255), 3)#上颜色
    # cv2.imshow('888', img)
    # cv2.waitKey(0)
    # cv2.destroyAllWindows()

    refCnts = sort_contours(refCnts, method="left-to-right")[0]  # 排序，从左到右，从上到下
    digits = {}

    # 遍历每一个轮廓
    for (i, c) in enumerate(refCnts):
        # 计算外接矩形并且resize成合适大小
        (x, y, w, h) = cv2.boundingRect(c)
        # if
        roi = ref[y:y + h, x:x + w]
        roi = cv2.resize(roi, (57, 88))
        # 每一个数字对应每一个模板
        digits[i] = roi
    # cv_show('digits[i]', digits[i])
    # nik = np.clip(nik, 0, 255)
    # nik = np.array(nik,np.uint8)
    # NIK识别
    gray_nik = cv2.cvtColor(nik, cv2.COLOR_BGR2GRAY)
    # gray_nik = cv2.GaussianBlur(gray_nik, (3, 3), 0)
    # ret_nik, thresh_nik = cv2.threshold(gray_nik, 127, 255, cv2.THRESH_BINARY)
    # cv2.imshow('8', gray_nik)
    # cv2.waitKey(0)
    group = cv2.threshold(gray_nik, 127, 255, cv2.THRESH_BINARY_INV)[1]
    # cv2.imshow('9', group)

    # 计算每一组的轮廓
    digitCnts, hierarchy_nik = cv2.findContours(group.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

    nik_r = nik.copy()

    cv2.drawContours(nik_r, digitCnts, -1, (0, 0, 255), 3)

    # 定位NIK的位置
    gX = locs[1][0]
    gY = locs[1][1]
    gW = locs[1][2]
    gH = locs[1][3]

    ctx = sort_contours(digitCnts, method="left-to-right")[0]

    locs_x = []
    # 遍历轮廓
    for (i, c) in enumerate(ctx):
        # 计算矩形
        (x, y, w, h) = cv2.boundingRect(c)

        # 选择合适的区域，根据实际任务来，这里的基本都是四个数字一组

        if h > 10 and w > 10:
            img = cv2.rectangle(nik_r, (x, y), (x + w, y + h), (0, 255, 0), 2)
            locs_x.append((x, y, w, h))

    # digitCnts = sort_contours(digitCnts, method="left-to-right")[0]
    output = []
    groupOutput = []
    # 计算每一组中的每一个数值
    for c in locs_x:
        # 找到当前数值的轮廓，resize成合适的的大小
        (x, y, w, h) = c
        roi = group[y:y + h, x:x + w]
        roi = cv2.resize(roi, (57, 88))
        # cv_show('roi',roi)

        # 计算匹配得分
        scores = []

        # 在模板中计算每一个得分
        for (digit, digitROI) in digits.items():
            # 模板匹配
            result = cv2.matchTemplate(roi, digitROI,
                                       cv2.TM_CCOEFF)
            (_, score, _, _) = cv2.minMaxLoc(result)
            scores.append(score)

        # 得到最合适的数字

        groupOutput.append(str(np.argmax(scores)))

    # 画出来
    cv2.rectangle(image, (gX - 5, gY - 5),
                  (gX + gW + 5, gY + gH + 5), (0, 0, 255), 1)
    cv2.putText(image, "".join(groupOutput), (gX, gY - 15),
                cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
    # 图像，文字内容， 坐标 ，字体，大小，颜色，字体厚度

    # 得到结果
    output.extend(groupOutput)
    return ''.join(output)

def main():
    raw_df = pd.read_csv(LINE_REC_PATH, header=None)
    # 读入城市字典
    cities_df = pd.read_csv(CITIES_REC_PATH, header=None)
    # 读入宗教字典
    religion_df = pd.read_csv(RELIGION_REC_PATH, header=None)
    # 读入婚姻字典
    marriage_df = pd.read_csv(MARRIAGE_REC_PATH, header=None)
    result_raw, id_number = ocr_raw(IMAGE_PATH)
    result_list = strip_op(result_raw)

    # test
    # result_list = ['—————— ————— —&——— ——————', 'PROVINSI DAEGAH ISTIMAWA YOGYEKARTA', 'KABUPATEN SLEMQN', 'NIK : 347LL40209790001', 'Nama :RIYANTO. SE', 'Tempat/Tgi Lahir : GROBOGAN. 02-09-1979', 'Jenis Kelamin : LAKI-LAKI Gol Darah :O', 'Alamat PRM PURI DOMAS D-3. SEMPU', 'RTARW 001: 024', 'Kel/Desa : WEDOMARTANI', 'Kecamatan : NGEMPLAK', 'Agama :ISPAM', 'Status Perkawman: KAWAN PE', 'Pekerjaan : PEDAGANG 05-06-2012', 'Kewarganegaraan: WNI SA—', 'Berlaku Hingga  :02-09-2017 N EA']
    print('-------origin rec-------')
    print(result_list)
    loc2index = dict()
    for i, tmp_line in enumerate(result_list):
        for j, tmp_word in enumerate(tmp_line.split(' ')):
            tmp_sim_list = [textdistance.damerau_levenshtein.normalized_similarity(tmp_word_, tmp_word.strip(':')) for
                            tmp_word_ in
                            raw_df[0].values]
            tmp_sim_np = np.asarray(tmp_sim_list)
            arg_max = np.argmax(tmp_sim_np)
            if tmp_sim_np[arg_max] >= 0.6:
                loc2index[(i, j)] = arg_max  ##最高相似度

    print('--------processed------------')
    # 数据处理
    last_result_list = []
    useful_info = False
    for i, tmp_line in enumerate(result_list):
        tmp_list = []
        for j, tmp_word in enumerate(tmp_line.split(' ')):
            tmp_word = tmp_word.strip(':')
            if (i, j) in loc2index:
                useful_info = True
                if loc2index[(i, j)] == NEXT_LINE:  ##换行
                    last_result_list.append(tmp_list)
                    tmp_list = []
                tmp_list.append(raw_df[0].values[loc2index[(i, j)]])
                if loc2index[(i, j)] in NEED_COLON:
                    tmp_list.append(':')
            elif tmp_word == ':' or tmp_word == '':
                continue
            else:
                tmp_list.append(tmp_word)
        if useful_info:
            if len(last_result_list) > 2 and ':' not in tmp_list:
                last_result_list[-1].extend(tmp_list)
            else:
                last_result_list.append(tmp_list)
    # print(last_result_list)

    for tmp_data in last_result_list:
        if '—' in tmp_data:
            tmp_data.remove('—')
        # 对前两行进行处理
        if 'PROVINSI' in tmp_data or 'KABUPATEN' in tmp_data or 'KOTA' in tmp_data:
            for tmp_index, tmp_word in enumerate(tmp_data[1:]):
                # 计算相似度
                tmp_sim_list = [textdistance.damerau_levenshtein.normalized_similarity(tmp_word, tmp_word_) for
                                tmp_word_ in cities_df[0].values]
                tmp_sim_np = np.asarray(tmp_sim_list)
                # 获得相似度最大单词索引
                arg_max = np.argmax(tmp_sim_np)
                if tmp_sim_np[arg_max] >= 0.6:
                    tmp_data[tmp_index + 1] = cities_df[0].values[arg_max]

        # 对身份证一行进行处理
        if 'NIK' in tmp_data:
            if len(id_number) != 16:
                id_number = tmp_data[2]
                if "D" in id_number:
                    id_number = id_number.replace("D", "0")
                if "?" in id_number:
                    id_number = id_number.replace("?", "7")
                if "L" in id_number:
                    id_number = id_number.replace("L", "1")
                while len(tmp_data) > 2:
                    tmp_data.pop()
                tmp_data.append(id_number)
            else:
                while len(tmp_data) > 3:
                    tmp_data.pop()
                tmp_data[2] = id_number

        # 对宗教一行进行处理
        if 'Agama' in tmp_data:
            for tmp_index, tmp_word in enumerate(tmp_data[1:]):
                # 计算相似度
                tmp_sim_list = [textdistance.damerau_levenshtein.normalized_similarity(tmp_word, tmp_word_) for
                                tmp_word_ in religion_df[0].values]
                tmp_sim_np = np.asarray(tmp_sim_list)
                # 获得相似度最大单词索引
                arg_max = np.argmax(tmp_sim_np)
                if tmp_sim_np[arg_max] >= 0.6:
                    tmp_data[tmp_index + 1] = religion_df[0].values[arg_max]

        # 对婚姻一行进行处理
        if 'Status' in tmp_data or 'Perkawinan' in tmp_data:
            for tmp_index, tmp_word in enumerate(tmp_data[2:]):
                # 计算相似度
                tmp_sim_list = [textdistance.damerau_levenshtein.normalized_similarity(tmp_word, tmp_word_) for
                                tmp_word_ in marriage_df[0].values]
                tmp_sim_np = np.asarray(tmp_sim_list)
                # 获得相似度最大单词索引
                arg_max = np.argmax(tmp_sim_np)
                if tmp_sim_np[arg_max] >= 0.6:
                    tmp_data[tmp_index + 2] = marriage_df[0].values[arg_max]
        if 'Alamat' in tmp_data:
            for tmp_index in range(len(tmp_data)):
                if "!" in tmp_data[tmp_index]:
                    tmp_data[tmp_index] = tmp_data[tmp_index].replace("!", "I")
                if "1" in tmp_data[tmp_index]:
                    tmp_data[tmp_index] = tmp_data[tmp_index].replace("1", "I")
                if "i" in tmp_data[tmp_index]:
                    tmp_data[tmp_index] = tmp_data[tmp_index].replace("i", "I")
    for tmp_data in last_result_list:
        print(' '.join(tmp_data))

if __name__ == '__main__':
    main()


# import pytest

# import gym
from qwertyenv.ensure_valid_action_pz import EnsureValidAction

from tianshou.data import Collector
from tianshou.env import DummyVectorEnv, PettingZooEnv
from tianshou.policy import MultiAgentPolicyManager, RandomPolicy

from qwertyenv.collect_coins_pz import CollectCoinsEnv

from pettingzoo.test import api_test

def test_evaluate():

  env = CollectCoinsEnv(pieces=['rock', 'rock'])

  action = None

  def another_action_taken(action_taken):
    nonlocal action
    action = action_taken

  # Wrapping the original environment as to make sure a valid action will be taken.
  env = EnsureValidAction(
      env,
      env.check_action_valid,
      env.provide_alternative_valid_action,
      another_action_taken
  )

  api_test(env, num_cycles=10, verbose_progress=False)

def test_with_tianshou():

  action = None

  # env =  gym.make('qwertyenv/CollectCoins-v0', pieces=['rock', 'rock'])

  env = CollectCoinsEnv(pieces=['rock', 'rock'], with_mask=True)

  def another_action_taken(action_taken):
    nonlocal action
    action = action_taken

  # Wrapping the original environment as to make sure a valid action will be taken.
  env = EnsureValidAction(
      env,
      env.check_action_valid,
      env.provide_alternative_valid_action,
      another_action_taken
  )

  env = PettingZooEnv(env)

  policies = MultiAgentPolicyManager([RandomPolicy(), RandomPolicy()], env)

  env = DummyVectorEnv([lambda: env])

  collector = Collector(policies, env)

  result = collector.collect(n_step=200, render=0.1)


#**************************************************
# Adapted from:
# Source: https://github.com/EvamariaTG/EvaMaria/blob/master/plugins/pm_filter.py
#**************************************************/

from pyrogram.types import InlineKeyboardButton
from bot.core.get_vars import get_val
from pyrogram.types import InlineKeyboardMarkup
from bot.utils.bot_utils.menu_utils import menu_maker_for_rclone, next_page_results

async def next_page_leech(client, callback_query):
    _, next_offset, data_back_cb= callback_query.data.split(" ")
    list_info = get_val("JSON_RESULT_DATA")
    btn= []
    total = len(list_info)
    next_offset = int(next_offset)
    prev_offset = next_offset - 10

    list_info, _next_offset= next_page_results(list_info, next_offset)

    btn.append([InlineKeyboardButton(f" ✅ Select this folder", callback_data= f"leechmenu^start_leech_folder")])

    menu_maker_for_rclone(list_info, btn, data_callback="list_dir_leech_menu")

    if next_offset == 0:
        btn.append(
            [InlineKeyboardButton(f"🗓 {round(int(next_offset) / 10) + 1} / {round(total / 10)}", callback_data="setting pages"),
             InlineKeyboardButton("NEXT ⏩", callback_data= f"n_leech {_next_offset} {data_back_cb}")
            ])
    elif next_offset >= total:
        btn.append(
             [InlineKeyboardButton("⏪ BACK", callback_data=f"n_leech {prev_offset} {data_back_cb}"),
              InlineKeyboardButton(f"🗓 {round(int(next_offset) / 10) + 1} / {round(total / 10)}",
                                   callback_data="setting pages")])
    elif next_offset + 10 > total:
        btn.append(
             [InlineKeyboardButton("⏪ BACK", callback_data=f"n_leech {prev_offset} {data_back_cb}"),
              InlineKeyboardButton(f"🗓 {round(int(next_offset) / 10) + 1} / {round(total / 10)}",
                                   callback_data="setting pages")])
    else:
        btn.append([InlineKeyboardButton("⏪ BACK", callback_data=f"n_leech {prev_offset} {data_back_cb}"),
             InlineKeyboardButton(f"🗓 {round(int(next_offset) / 10) + 1} / {round(total / 10)}", callback_data="setting pages"),
             InlineKeyboardButton("NEXT ⏩", callback_data=f"n_leech {_next_offset} {data_back_cb}")
            ])

    btn.append(
            [InlineKeyboardButton("⬅ Back", f"leechmenu^{data_back_cb}")]
        )
    btn.append(
            [InlineKeyboardButton("✘ Close Menu", f"leechmenu^selfdest")]
        )

    message= callback_query.message
    default_drive= get_val("DEFAULT_RCLONE_DRIVE")
    base_dir= get_val("BASE_DIR")
    await message.edit(
        f"Select folder or file that you want to leech\n\nPath:`{default_drive}:{base_dir}`",
        reply_markup= InlineKeyboardMarkup(btn))


from PySide2.QtWidgets import QMainWindow, QFileDialog, QMessageBox, QTableWidgetItem, QGraphicsScene, QDialog
from ui.ui_editorial import Ui_Dialog
from PySide2.QtCore import Slot
from models.editorial import Editorial

class EditorialWindow(QDialog):
    def __init__(self, editorial=None):
        super(EditorialWindow, self).__init__()
        self.ui = Ui_Dialog()
        self.ui.setupUi(self)
        self.editorial = editorial

        self.ui.push_cancelar.clicked.connect(self.onClose)
        self.ui.push_guardar.clicked.connect(self.onSave)

        if self.editorial is not None:
            self.ui.edit_nombre.setText(self.editorial['nombre'])
            self.ui.edit_pais.setText(self.editorial['paisorigen'])

    @Slot()
    def onClose(self):
        self.done(0)

    @Slot()
    def onSave(self):
        nombre = self.ui.edit_nombre.text()
        pais = self.ui.edit_pais.text()
        editorial = Editorial(nombre, pais)
        if self.editorial is None:
            editorial.save()
        else:
            editorial._key = self.editorial['nombre']
            editorial.update()
        self.done(0)


##
##
##

from __future__ import annotations

import abc
import importlib
from dataclasses import dataclass
from typing import Any, Generic, Iterable, TypeVar

import torch
from torch.nn import Parameter
from typing_extensions import Self

from deepsight.data.structs import RECInput, RECOutput
from deepsight.utils.protocols import Moveable

from ._config import Config
from ._criterion import Criterion
from ._model import Model
from ._postprocessor import PostProcessor
from ._preprocessor import PreProcessor

Input = TypeVar("Input", bound=Moveable)
Output = TypeVar("Output", bound=Moveable)
ModelInput = TypeVar("ModelInput")
ModelOutput = TypeVar("ModelOutput")

@dataclass(frozen=True, slots=True)
class Pipeline(abc.ABC, Moveable, Generic[Input, Output, ModelInput, ModelOutput]):
    """A pipeline is a sequence of steps that are executed in order to perform a
    specific task."""

    name: str
    preprocessor: PreProcessor[Input, Output, ModelInput]
    model: Model[ModelInput, ModelOutput]
    postprocessor: PostProcessor[ModelOutput, Output]
    criterion: Criterion[ModelOutput, Output]

    @classmethod
    @abc.abstractmethod
    def new_for_rec(
        cls, config: Config
    ) -> Pipeline[RECInput, RECOutput, ModelInput, ModelOutput]:
        """Creates a new pipeline given a `config` for the Referring Expression
        Comprehension task.

        Calling this method on a subclass of `Pipeline` will return an instance of
        that subclass. Calling it on `Pipeline` will return an instance of a subclass
        of `Pipeline` depending on the `module` field of the `config`. The `module`
        field should be a valid module path to a subclass of `Pipeline`.

        Parameters
        ----------
        config : Config
            The configuration for the pipeline.

        Returns
        -------
        Pipeline[RECInput, RECOutput, ModelInput, ModelOutput]
            The pipeline.
        """

        module = importlib.import_module(config.module)
        sub_cls = getattr(module, cls.__name__)
        return sub_cls.new_for_rec(config)  # type: ignore

    def state_dict(self) -> dict[str, dict[str, Any]]:
        """Returns the state of the pipeline as a dict.

        It contains the state of the preprocessor, model, postprocessor and criterion.

        Returns
        -------
        dict[str, dict[str, Any]]
            The state of the pipeline.
        """

        return {
            "preprocessor": self.preprocessor.state_dict(),
            "model": self.model.state_dict(),
            "postprocessor": self.postprocessor.state_dict(),
            "criterion": self.criterion.state_dict(),
        }

    def load_state_dict(self, state_dict: dict[str, dict[str, Any]]) -> None:
        """Loads the pipeline state from the given `state_dict`.

        Parameters
        ----------
        state_dict : dict[str, dict[str, Any]]
            The state of the pipeline.
        """

        self.preprocessor.load_state_dict(state_dict["preprocessor"])
        self.model.load_state_dict(state_dict["model"])
        self.postprocessor.load_state_dict(state_dict["postprocessor"])
        self.criterion.load_state_dict(state_dict["criterion"])

    def to(self, device: torch.device | str) -> Self:
        """Moves the pipeline to the given `device`.

        Parameters
        ----------
        device : torch.device | str
            The device to move the pipeline to.

        Returns
        -------
        Self
            The pipeline moved to the given `device`.
        """

        return self.__class__(
            name=self.name,
            preprocessor=self.preprocessor.to(device),
            model=self.model.to(device),
            postprocessor=self.postprocessor.to(device),
            criterion=self.criterion.to(device),
        )

    def train(self) -> None:
        """Sets the whole pipeline to training mode."""

        self.preprocessor.train()
        self.model.train()
        self.postprocessor.train()
        self.criterion.train()

    def eval(self) -> None:
        """Sets the whole pipeline to evaluation mode."""

        self.preprocessor.eval()
        self.model.eval()
        self.postprocessor.eval()
        self.criterion.eval()

    def parameters(self) -> Iterable[Parameter]:
        """Returns an iterator over the pipeline parameters.

        Returns
        -------
        Iterable[Parameter]
            An iterator over the pipeline parameters.
        """

        yield from self.preprocessor.parameters()
        yield from self.model.parameters()
        yield from self.postprocessor.parameters()
        yield from self.criterion.parameters()

    def named_parameters(self) -> Iterable[tuple[str, Parameter]]:
        """Returns an iterator over the pipeline named parameters.

        Returns
        -------
        Iterable[tuple[str, Parameter]]
            An iterator over the pipeline named parameters.
        """

        for name, param in self.preprocessor.named_parameters():
            yield f"preprocessor.{name}", param

        for name, param in self.model.named_parameters():
            yield f"model.{name}", param

        for name, param in self.postprocessor.named_parameters():
            yield f"postprocessor.{name}", param

        for name, param in self.criterion.named_parameters():
            yield f"criterion.{name}", param


from collections import Counter
import random
import matplotlib.pyplot as plt
from numpy.random import randint
import math
import csv

def simulated_annealing(initial_state, jugadores):
    """Peforms simulated annealing to find a solution"""
    initial_temp = 500
    final_temp = .1
    alpha = 0.0001

    current_temp = initial_temp
    results = []
    # Start by initializing the current state with the initial state
    current_state = initial_state
    solution = current_state
    iteracion = 1
    while current_temp > final_temp:
        neighbor = random.choice(get_neighbors(current_state, jugadores))

        # Check if neighbor is best so far
        cost_neighbor = get_cost(neighbor, jugadores)
        cost_current = get_cost(current_state, jugadores)
        cost_diff = cost_current - cost_neighbor

        # if the new solution is better, accept it
        if cost_diff < 0:
            current_state = neighbor
            solution = current_state
            print("New Best: ", solution)
        # if the new solution is not better, accept it with a probability of e^(-cost/temp)
        else:
            if random.uniform(0, 1) < math.exp(-cost_diff / current_temp):
                current_state = neighbor
                solution = current_state
                print("Worst accepted: ", solution)
        # decrement the temperature
        current_temp /= (1+(alpha*current_temp))
        results.append((get_cost(current_state, jugadores), iteracion))
        iteracion += 1

    return solution, results

def get_cost(state, jugadores):
    """Calculates cost of the argument state for your solution."""
    puntos, coste = 0, 0
    posiciones = []
    for gen in state:
        puntos += jugadores[gen]["puntos"]
        coste += jugadores[gen]["precio"]
        posiciones.append(jugadores[gen]["tipo"])
    puntos /= len(state)
    contador = Counter(posiciones)
    if coste <= 60000 and contador[0] == contador[1] == contador[2] == contador[3] and contador[4] == 1:
        return puntos
    else:
        res = -math.log(coste)
        l = [-1 for x in range(4) if contador[x] != 2]
        if contador[4] != 1:
            l.append(-1)
        return res+sum(l)

def get_neighbors(state, jugadores):
    """Returns neighbors of the argument state for your solution."""
    res = []
    for _ in range(10):
        new_state = [x for x in state]
        new_state[randint(0, 9)] = randint(0, len(jugadores)-1)
        res.append(new_state)
    return res

def cargarJugadores():
	res = {}
	with open("jugadores.csv") as f:
		reader = csv.reader(f, delimiter=';')
		for row in reader:
			res[int(row[0])] = {"nombre": row[1], "tipo": int(row[2]), "puntos": float(row[3]), "precio": int(row[4])}
	return res

jugadores = cargarJugadores()
init_state = [random.randint(0, len(jugadores)-1) for _ in range(9)]
best, resultados = simulated_annealing(init_state, jugadores)
print("Done", best)
for j in best:
    print(jugadores[j])
print('Coste: %i' % (sum([jugadores[j]["precio"] for j in best])))
print('Puntuacion: %i' % (get_cost(best, jugadores)))
x = [i[1] for i in resultados]
y = [i[0] for i in resultados]

plt.plot(x,y)
plt.title("Enfriamiento simulado")
plt.xlabel("Iteracion")
plt.ylabel("Fitness")
plt.show()


import logging

from horizons.network.common import *
from horizons import network
from horizons.network import packets, find_enet_module

enet = find_enet_module()

MAX_PEERS = 4095
CONNECTION_TIMEOUT = 500

logging.basicConfig(level = logging.DEBUG)

class Server(object):
	def __init__(self, hostname, port):
		self.host = None
		self.hostname = hostname
		self.port = port
		self.callbacks = {
			'connect':    [ self.onconnect ],
			'disconnect': [ self.ondisconnect ],
			'receive':    [ self.onreceive ],
			packets.cmd_error:                 [ self.onerror ],
			packets.cmd_fatalerror:            [ self.onfatalerror ],
			packets.client.cmd_creategame:     [ self.oncreategame ],
			packets.client.cmd_listgames:      [ self.onlistgames ],
			packets.client.cmd_joingame:       [ self.onjoingame ],
			packets.client.cmd_leavegame:      [ self.onpktleavegame ],
			packets.client.cmd_chatmsg:        [ self.onchat ],
			packets.client.cmd_holepunchingok: [ self.onholepunchingok ],
			'holepunching':   [ self.onholepunching ],
			'startgame':      [ self.onstartgame ],
			'leavegame':      [ self.onleavegame ],
			'deletegame':     [ self.ondeletegame ],
		}
		self.games = []
		pass

	def register_callback(self, type, callback, prepend = False):
		if type in self.callbacks:
			if prepend:
				self.callbacks[type].insert(0, callback)
			else:
				self.callbacks[type].append(callback)
		else:
			raise TypeError("Unsupported type")

	def call_callbacks(self, type, *args):
		if not type in self.callbacks:
			return
		for callback in self.callbacks[type]:
			callback(*args)

	def run(self):
		logging.info("Starting up server on %s:%d" % (self.hostname, self.port))
		try:
			self.host = enet.Host(enet.Address(self.hostname, self.port), MAX_PEERS, 0, 0, 0)
		except (IOError, MemoryError):
			raise network.NetworkException("Unable to create network structure. Maybe invalid or irresolvable server address.")

		logging.debug("Entering the main loop...")
		while True:
			event = self.host.service(CONNECTION_TIMEOUT)
			if event.type == enet.EVENT_TYPE_NONE:
				continue
			elif event.type == enet.EVENT_TYPE_CONNECT:
				self.call_callbacks("connect", event)
			elif event.type == enet.EVENT_TYPE_DISCONNECT:
				self.call_callbacks("disconnect", event);
			elif event.type == enet.EVENT_TYPE_RECEIVE:
				self.call_callbacks("receive", event);
			else:
				logging.warning("Invalid packet (%u)" % (event.type))

	def send(self, peer, packet, channelid = 0):
		if self.host is None:
			raise network.NotConnected("Server is not running")
		packet.send(peer, channelid)
		self.host.flush()

	def disconnect(self, peer, later = False):
		logging.debug("Disconnecting client %s" % (peer.address))
		try:
			if later:
				peer.disconnect_later()
			else:
				peer.disconnect()
		except IOError:
			peer.reset()

	def onconnect(self, event):
		logging.debug("[CONNECT] New Client from %s" % (event.peer.address))

	def ondisconnect(self, event):
		logging.debug("[DISCONNECT] Client %s disconnected" % (event.peer.address))
		game = player = None
		for _game in self.games:
			for _player in _game.players:
				if _player.address == event.peer.address:
					game = _game
					player = _player
					break
		if game is None or player is None:
			return
		self.call_callbacks("leavegame", game, player)

	def onreceive(self, event):
		#logging.debug("[RECEIVE] Got data from %s" % (event.peer.address))
		packet = packets.unserialize(event.packet.data)
		if packet is None:
			logging.warning("Unknown packet from %s!" % (event.peer.address))
			self.send(event.peer, packets.cmd_fatalerror("Unknown packet. Maybe old client?"))
			self.disconnect(event.peer, True)
			return
		if not packet.__class__ in self.callbacks:
			logging.warning("Unhandled network packet from %s!" % (event.peer.address))
			return
		self.call_callbacks(packet.__class__, event.peer, packet)

	def onerror(self, peer, packet):
		# we shouldn't receive any errors from client
		# so ignore them all
		logging.debug("[ERROR] Client Message: %s" % (packet.errorstr))

	def onfatalerror(self, peer, packet):
		logging.debug("[FATAL] Client Message: %s" % (packet.errorstr))
		self.disconnect(peer, True)

	def oncreategame(self, peer, packet):
		game = None
		for _game in self.games:
			for _player in _game.players:
				if _player.address == peer.address:
					game = _game
					break
		if game is not None:
			self.send(peer, packets.cmd_error("You can't create a game while in another game"))
			return
		game = Game(packet, peer)
		logging.debug("[CREATE] uuid=%s, maxplayers=%d" % (game.uuid, game.maxplayers))
		self.games.append(game)
		self.send(peer, packets.server.data_gamestate(game))

		if game.playercnt == game.maxplayers:
			game.gamestarts = True
			self.call_callbacks("holepunching", game)

	def ondeletegame(self, game):
		logging.debug("[REMOVE] Game %s removed" % (game.uuid))
		self.games.remove(game)

	def onlistgames(self, peer, packet):
		logging.debug("[LIST]")
		gameslist = packets.server.data_gameslist()
		for _game in self.games:
			if _game.gamestarts:
				continue
			if _game.maxplayers == len(_game.players):
				continue
			if packet.clientversion != -1 and packet.clientversion != _game.clientversion:
				continue
			if packet.mapname and packet.mapname != _game.mapname:
				continue
			if packet.maxplayers and packet.maxplayers != _game.maxplayers:
				continue
			gameslist.addgame(_game);
		self.send(peer, gameslist)

	def onjoingame(self, peer, packet):
		logging.debug("[JOIN] name=%s, uuid=%s" % (packet.playername, packet.uuid))
		game = None
		for _game in self.games:
			for _player in _game.players:
				if _player.address == peer.address:
					game = _game
					break
		if game is not None:
			self.send(peer, packets.cmd_error("You can't join a game while in another game"))
			return

		game = None
		for _game in self.games:
			if (packet.clientversion != _game.clientversion):
				continue
			if (packet.uuid != _game.uuid):
				continue
			game = _game
			break
		if game is None:
			self.send(peer, packets.cmd_error("Unknown game"))
			return
		if game.gamestarts:
			self.send(peer, packets.cmd_error("Game has already started. No more joining"))
			return
		elif game.maxplayers == len(game.players):
			self.send(peer, packets.cmd_error("Game is full"))
			return

		playerunique = True
		for _player in game.players:
			if _player.name == packet.playername:
				playerunique = False
				break
			elif _player.address == peer.address:
				playerunique = False
				break
		if not playerunique:
			self.send(peer, packets.cmd_error("Your name or address is already known to this game"))
			return

		game.addplayer(Player(packet.playername, peer))
		game.playercnt += 1
		for _player in game.players:
			self.send(_player.peer, packets.server.data_gamestate(game))

		# if game is full ... lets start the game
		if game.playercnt == game.maxplayers:
			game.gamestarts = True
			self.call_callbacks("holepunching", game)

	def onpktleavegame(self, peer, packet):
		game = player = None
		for _game in self.games:
			for _player in _game.players:
				if _player.address == peer.address:
					game = _game
					player = _player
					break
		if game is None or player is None:
			self.send(peer, packets.cmd_error("You are an unknown player"))
			return
		self.call_callbacks("leavegame", game, player)
		self.send(peer, packets.cmd_ok())

	def onleavegame(self, game, player):
		logging.debug("[LEAVE] Player %s leaves game %s" % (player.name, game.uuid))
		game.players.remove(player)
		game.playercnt -= 1
		if not game.gamestarts:
			for _player in game.players:
				self.send(_player.peer, packets.server.data_gamestate(game))
		if game.playercnt <= 0:
			self.call_callbacks('deletegame', game)

	def onholepunching(self, game):
		logging.debug("[HOLEPUNCHING] %s" % (game.uuid))
		for _player in game.players:
			self.send(_player.peer, packets.server.cmd_holepunching())

	def onholepunchingok(self, peer, packet):
		game = player = None
		for _game in self.games:
			for _player in _game.players:
				if _player.address == peer.address:
					game = _game
					player = _player
					break
		if game is None:
			self.send(peer, packets.cmd_fatalerror("Unable to find the game you're in"))
			self.disconnect(peer, True)
			return
		player.ready = True
		logging.debug("[HOLEPUNCHING] [%s] Player %s (%s) is ready" % (game.uuid, player.name, player.address))

		playersready = True
		for _player in game.players:
			if not _player.ready:
				playersready = False
				break
		if playersready:
			self.call_callbacks("startgame", game)

	def onstartgame(self, game):
		logging.debug("[START] %s" % (game.uuid))
		for _player in game.players:
			self.send(_player.peer, packets.server.cmd_startgame())

	def onchat(self, peer, packet):
		game = player = None
		for _game in self.games:
			for _player in _game.players:
				if _player.address == peer.address:
					game = _game
					player = _player
					break
		if game is None or player is None:
			self.send(peer, packets.cmd_error("You are an unknown player"))
			self.disconnect(peer, True)
			return
		logging.debug("[CHAT] [%s] %s: %s" % (game.uuid, player.name, packet.chatmsg))
		# don't send packets to already started games
		if game.gamestarts:
			return
		for _player in game.players:
			self.send(_player.peer, packets.server.cmd_chatmsg(player.name, packet.chatmsg))


from django.contrib import admin
from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static

def trigger_error(request):
    division_by_zero = 1 / 0

urlpatterns = [
    # Django admin
    path("backside/", admin.site.urls),
    # Sentry error testing
    path("sentry-debug/", trigger_error),
    # User management
    path("accounts/", include("allauth.urls")),
    path("accounts/", include("users.urls")),
    # 3rd party
    path("djga/", include("google_analytics.urls")),
    # Local apps
    path("", include("pages.urls")),
    path("questions/", include("questions.urls")),
    path("q/", include("questions.urls")),
]

if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import operator
import math
import random
import itertools
from tqdm import tqdm

symbolToNumber = {'A': 0, 'T': 1, 'C': 2, 'G': 3}
numberToSymbol = {0: 'A', 1: 'T', 2: 'C', 3: 'G'}

def window(s, k):
    for i in range(len(s) - k + 1):
        yield s[i: i+k]

def Hamming(a, b):
    return sum([1 for i, j in zip(a, b) if i != j])

def ComputeProfile(motifs):
    len_motif = len(motifs[0])
    profile = {s : [0] * len_motif for s in 'ACTG'}
    profile['len'] = len(motifs)
    for i in range(len_motif):
        current = {'A': 0, 'T': 0, 'C': 0, 'G': 0}
        for motif in motifs:
            current[motif[i]] += 1
        for s in 'ACTG':
            profile[s][i] = current[s] + 1 / (len(motifs))
    return profile

def ConsensusString(motifs):
    len_motif = len(motifs[0])
    answer = ''
    for i in range(len_motif):
        current = {'A': 0, 'T': 0, 'C': 0, 'G': 0}
        for motif in motifs:
            current[motif[i]] += 1
        answer += max(current.items(), key=operator.itemgetter(1))[0]
    return answer

def ProfileProbability(profile, str):
    probability = 1
    for i, s in enumerate(str):
        probability *= profile[s][i]
    return probability

def GibbsMostProbable(profile, text, k):
    probabilities = []
    for kmer in window(text, k):
        probabilities.append(ProfileProbability(profile, kmer))
    probability_sum = sum(probabilities)
    probabilities = [p / probability_sum for p in probabilities]
    accumulated_probs = list(itertools.accumulate(probabilities))
    toss = random.random()
    for i in range(len(accumulated_probs)):
        if toss < accumulated_probs[i]:
            return text[i:i + k]

def Score(motifs):
    score = 0
    consensus = ConsensusString(motifs)
    for motif in motifs:
        score += Hamming(motif, consensus)
    return score

def GetRandomMotifs(text, k):
    motifs = []
    for string in text:
        ind = random.randint(0, len(string) - k)
        motifs.append(string[ind : ind + k])
    return motifs

def GibbsSampler(text, k, t, N):
    motifs = GetRandomMotifs(text, k)
    best_motifs = motifs
    best_score = math.inf
    for _ in range(N):
        removed_sequence = random.randrange(t)
        current_motifs = motifs[:removed_sequence] + motifs[removed_sequence + 1:]
        profile = ComputeProfile(current_motifs)
        computed_motif = GibbsMostProbable(profile, text[removed_sequence], k)
        motifs = motifs[:removed_sequence] + [computed_motif] + motifs[removed_sequence + 1:]
        if Score(motifs) < best_score:
            best_score = Score(motifs)
            best_motifs = motifs
    return best_motifs

def RunMultipleTimesGibbsSampler(dna_list, k, t, N):
    best_score = math.inf
    best_motifs = None
    for _ in tqdm(range(20)):
        motifs = GibbsSampler(dna_list, k, t, N)
        if Score(motifs) < best_score:
            best_motifs = motifs
            best_score = Score(motifs)
    return best_motifs

if __name__ == "__main__":
    k, t, N = map(int, input().split())
    text = [input() for _ in range(t)]
    for motif in RunMultipleTimesGibbsSampler(text, k, t, N):
        print(motif)


#!/usr/bin/env python
# -*- coding:utf-8 -*-

#
# python ./script-apitest-auto-analyse/report-analyse.py ../out/apitest/hilife-activity-ApiTest/builds/61/archive/target/jmeter/html_report/ApiTest_activity-20210225-0208.html
#

import csv,os,sys
import re
from matplotlib.pyplot import switch_backend, table
import requests
import json
import json,time
from bs4 import BeautifulSoup
reload(sys)
sys.setdefaultencoding('utf-8')

def get_ApitestCount(HtmlReport):
    with open(HtmlReport, 'r') as strf:
        str = strf.read()
    html = BeautifulSoup(str, "html.parser")
    tables = html.findAll("table")
    header = []
    data_list = []
    for index, table in enumerate(tables):
        if(index == 1):
            for idx, tr in enumerate(table.find_all('tr')):
                if idx == 0:
                    ths = tr.find_all('th')
                    for th in ths:
                        header.append(th.string)
                elif idx == 1:
                    tds = tr.find_all('td')
                    for tb in tds:
                        data_list.append(tb.string)
        if(index > 1):
            break
    return header,data_list

if __name__ == '__main__':
    print("文件路径,请求数量,成功,失败,成功率,平均响应时间,最短时间,最长时间,RT(≤0.5s),(0.5~1s),RT(>1s),90% Line,QPS")

    switch = {
        "请求数量" : 0,
        "# Samples" : 0,
        "成功" : 1,
        "Success" : 1,
        "失败" : 2,
        "Failures" : 2,
        "成功率" : 3,
        "Success Rate" : 3,
        "平均响应时间" : 4,
        "Average Time" : 4,
        "最短时间" : 5,
        "Min Time" : 5,
        "最长时间" : 6,
        "Max Time" : 6,
        "RT(≤0.5s)" : 7,
        "(0.5~1s)" : 8,
        "RT(>1s)" : 9,
        "90% Line" : 10,
        "QPS" : 11
    }
    print(switch)
    IFS = ","
    for htmlReportPath in sys.argv[1:]:
        if(os.path.exists(htmlReportPath)):
            header,data_list = get_ApitestCount(htmlReportPath)
            if(len(header) != len(data_list)):
                print(htmlReportPath + "," + "错误")
            else:
                values = ["","","","","","","","","","","",""]
                for idx, head in enumerate(header):
                    # if("# Samples" == header[idx]):
                    #     header[idx] = "请求数量"
                    # if("Success" == header[idx]):
                    #     header[idx] = "成功"
                    # if("Failures" == header[idx]):
                    #     header[idx] = "失败"
                    # if("Success Rate" == header[idx]):
                    #     header[idx] = "成功率"
                    # if("Average Time" == header[idx]):
                    #     header[idx] = "平均响应时间"
                    # if("Min Time" == header[idx]):
                    #     header[idx] = "最短时间"
                    # if("Max Time" == header[idx]):
                    #     header[idx] = "最长时间"

                    if("请求数量" == header[idx]):
                        index = 0
                    elif("# Samples" == header[idx]):
                        index = 0
                    elif("成功" == header[idx]):
                        index = 1
                    elif("Success" == header[idx]):
                        index = 1
                    elif("失败" == header[idx]):
                        index = 2
                    elif("Failures" == header[idx]):
                        index = 2
                    elif("成功率" == header[idx]):
                        index = 3
                    elif("Success Rate" == header[idx]):
                        index = 3
                    elif("平均响应时间" == header[idx]):
                        index = 4
                    elif("Average Time" == header[idx]):
                        index = 4
                    elif("最短时间" == header[idx]):
                        index = 5
                    elif("Min Time" == header[idx]):
                        index = 5
                    elif("最长时间" == header[idx]):
                        index = 6
                    elif("Max Time" == header[idx]):
                        index = 6
                    elif("RT(≤0.5s)" == header[idx]):
                        index = 7
                    elif("RT(≤0.5s) " == header[idx]):
                        index = 7
                    elif("(0.5~1s)" == header[idx]):
                        index = 8
                    elif("RT(0.5~1s)" == header[idx]):
                        index = 8
                    elif("RT(>1s)" == header[idx]):
                        index = 9
                    elif("90% Line" == header[idx]):
                        index = 10
                    elif("QPS" == header[idx]):
                        index = 11
                    else:
                        index = -1

                    if(index != -1):
                        values[index] = data_list[idx]
                    else:
                        print(header[idx])
                value = IFS.join(values)
                print(htmlReportPath + "," + value)