'''Using VAE on keyword occurrence submissions dataset using MLP

The VAE has a modular design. The encoder, decoder and VAE
are 3 models that share weights. After training the VAE model,
the encoder can be used to generate latent vectors.

The decoder can be used to generate eyword occurrence submissions
by sampling the latent vector from a Gaussian distribution
with mean = 0 and std = 1.

[1] Kingma, Diederik P., and Max Welling.
"Auto-Encoding Variational Bayes."
https://arxiv.org/abs/1312.6114

Code based on keras example:
https://blog.keras.io/building-autoencoders-in-keras.html
'''

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from keras.layers import Lambda, Input, Dense
from keras.models import Model
from keras.losses import mse
from keras.utils import plot_model
from keras import backend as K
from keras import metrics
from matplotlib import colors
from matplotlib import cm

import argparse
import math
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import seaborn as sns
import sys
from scipy import stats

exp_ints = {
	'no_response' : 0,
	'absolutely_none': 1,
	'other_language' : 2,
	'know_python' : 3,
	'veteran' : 4
	}

# reparameterization trick
# instead of sampling from Q(z|X), sample epsilon = N(0,I)
# z = z_mean + sqrt(var) * epsilon
def sampling(args):
	'''
	Reparameterization trick by sampling from an isotropic unit Gaussian.
	# Arguments
		args (tensor): mean and log of variance of Q(z|X)
	# Returns
		z (tensor): sampled latent vector
	'''
	z_mean, z_log_var = args
	batch = K.shape(z_mean)[0]
	dim = K.int_shape(z_mean)[1]
	# by default, random_normal has mean = 0 and std = 1.0
	epsilon = K.random_normal(shape=(batch, dim))
	return z_mean + K.exp(0.5 * z_log_var) * epsilon

def pad(array, reference_shape, offsets):
    '''
    Pads the array with zeros to fit the reference shape at the offsets given
    [0, 0] to align with top left
    :param array: Array to be padded
    :param reference_shape: tuple of size of ndarray to create
    :param offsets: list of offsets (number of elements must be equal to the dimension of the array)
    will throw a ValueError if offsets is too big and the reference_shape cannot handle the offsets
    '''
    result = np.zeros(reference_shape)
    insertHere = [slice(offsets[dim], offsets[dim] + array.shape[dim]) for dim in range(array.ndim)]
    result[insertHere] = array
    return result

def plot_training_loss(training_history, epochs, unit, problem):
	'''
	Plots the losses during training given the History object,
	number of epochs, and keyword list. Plots total, reconstruction,
	KL divergence, and feature-specific losses.
	:param training_history: the training history returned from model.fit
	:param epochs: number of epochs to train
	:param unit: course unit
	:param problem: course problem
	'''
	l = np.array(training_history.history['loss'])
	r_l = np.array(training_history.history['mse'])
	k_l = np.array(training_history.history['kl'])
	v_l = np.array(training_history.history['val_loss'])
	plt.plot(np.linspace(1, epochs, epochs), l, 'b+')
	plt.plot(np.linspace(1, epochs, epochs), r_l, 'c.')
	plt.plot(np.linspace(1, epochs, epochs), k_l, 'g.')
	plt.plot(np.linspace(1, epochs, epochs), v_l, 'r+')
	plt.legend(['training loss', 'reconstruction loss', 'KL divergence', 'validation loss'])
	plt.xlabel('epoch')
	plt.ylabel('loss')
	plt.title('Training Loss MSE+KL for Problem {}-{}'.format(unit, problem))
	os.makedirs('loss_plots', exist_ok=True)
	plt.savefig('loss_plots/flattraj_training_loss_{}_{}'.format(unit, problem))
	plt.close()

def plot_mean_results(models,
				 x_test,
				 y_test,
				 y1_test,
				 unit,
				 problem,
				 batch_size=128):
	'''
	Plots labels and vis of samples
		as a function of the 2D latent vector
	# Arguments
		models (tuple): encoder and decoder models
		x_test: test data
		y_test: first set of labels (experience)
		y1_test: second set of labels (video engagement)
		unit (int): course unit
		problem (int): course problem
		batch_size (int): prediction batch size
	'''
	encoder, decoder = models

	os.makedirs('means', exist_ok=True)

	# display a 2D plot of the experience classes in the latent space
	z_mean, _, _ = encoder.predict(x_test, batch_size=batch_size)

	for this_y in [y_test, y1_test]:
		plt.figure(figsize=(12, 10))
		sns.scatterplot(x=z_mean[:, 0], y=z_mean[:, 1], hue=this_y)
		plt.xlabel('z[0]')
		plt.ylabel('z[1]')
		if this_y is y_test:
			filename = 'means/vae_flattraj_mean_{}_{}_ytest.png'.format(unit, problem)
		else:
			filename = 'means/vae_flattraj_mean_{}_{}_y1test.png'.format(unit, problem)
		plt.savefig(filename)
		plt.close()

	# plot a density map in the latent space to contextualize the scatterplots
	filename = 'means/vae_flattraj_mean_{}_{}_kde.png'.format(unit, problem)
	plt.figure(figsize=(12, 10))
	cmap = sns.dark_palette("purple", as_cmap=True)
	ax = sns.kdeplot(z_mean[:, 0], z_mean[:, 1], n_levels=30, cmap=cmap)
	plt.xlabel('z[0]')
	plt.ylabel('z[1]')
	plt.savefig(filename)

	# mutate all y data to integers
	y_test = np.array([exp_ints[exp] for exp in y_test])
	y1_test = np.array([0 if vid==False else 1 for vid in y1_test])

	for this_y in [y_test, y1_test]:
		if this_y is y_test:
			print('EXPERIENCE')
		else:
			print('VIDEO ENGAGEMENT')
		_, _, r2, _, _ = stats.linregress(z_mean[:, 0], this_y)
		print('r-squared from regression z0 and this y:', r2)
		_, _, r2, _, _ = stats.linregress(z_mean[:, 1], this_y)
		print('r-squared from regression z1 and this y:', r2)

		pear = stats.pearsonr(z_mean[:, 0], this_y)
		print('r-squared from pearson correlation z0 and this y:', pear)
		pear = stats.pearsonr(z_mean[:, 1], this_y)
		print('r-squared from pearson correlation z1 and this y:', pear)

	return

def plot_latent_sampled(models,
				 x_test,
				 y_test,
				 y1_test,
				 unit,
				 problem):
	'''
	Plots vis of samples from latent space
	as a function of the 2D latent vector
	:param models (tuple): trained encoder and decoder networks
	:param x_test: test data
	:param y_test: labels
	:param unit: course unit
	:param problem: course problem
	'''
	encoder, decoder = models

	# display a 2D manifold of submissions
	n = 10
	num_keywords = 10
	max_subs = 30
	figure = np.zeros((1 * n, num_keywords * max_subs * n))
	latent_l1 = np.zeros((n, n))
	latent_l2 = np.zeros((n, n))

	# linearly spaced coordinates corresponding to
	# 2D plot of digit classes in latent space
	grid_x = np.linspace(-4, 4, n)
	grid_y = np.linspace(-4, 4, n)[::-1]

	for i, yi in enumerate(grid_y):
		for j, xi in enumerate(grid_x):
			z_sample = np.array([[xi, yi]])
			x_decoded = decoder.predict(z_sample)
			digit = x_decoded[0].reshape(1, num_keywords*max_subs)
			figure[i : (i + 1), j * num_keywords*max_subs: (j + 1) * num_keywords*max_subs] = np.rint(30*digit)
			latent_l1[i:i+1, j:j+1] = np.sum(np.rint(30*digit))
			latent_l2[i:i+1, j:j+1] = np.linalg.norm(np.rint(30*digit))

	viridis = cm.get_cmap('viridis', 256)

	os.makedirs('sample_vis', exist_ok=True)
	for norm in ['l1', 'l2']:
		filename = 'sample_vis/flattraj_subs_over_latent_{}_{}_{}.png'.format(norm, unit, problem)

		fig, ax = plt.subplots()
		if norm == 'l2':
			psm = ax.pcolormesh(latent_l2, cmap=viridis, rasterized=True)
		else:
			psm = ax.pcolormesh(latent_l1, cmap=viridis, rasterized=True)
		fig.colorbar(psm, ax=ax)

		ax.grid(which='major', axis='both', linestyle='-', color='k', linewidth=2)
		ax.set_xticks([])
		ax.set_yticks([])

		plt.xlabel('z[0]')
		plt.ylabel('z[1]')
		if norm == 'l2':
			latent_title = 'L2 norm of submissions generated \n from sampled 2D latent space for Problem {}-{}'
		else:
			latent_title = 'L1 norm of submissions generated \n from sampled 2D latent space for Problem {}-{}'
		plt.title(latent_title.format(unit, problem), fontsize=10)
		plt.savefig(filename)
	return

# specify a save name based on the features you're using
# this will be added to output filenames of generated figures and plots
savename = 'vae_flattraj_kw_occ'

# specify problem to consider
unit = 2
pb = 3

# load keyword occurrence submissions dataset
sub_data_csv = '../../data/keyword_occurrence/trajectory/unit{}_pb{}_kw_occ_traj.csv'
sub_data = pd.read_csv(sub_data_csv.format(unit, pb), index_col=None)

sub_data['kw_occ_matrix'] = sub_data['kw_occ_matrix'].apply(lambda x:eval(x))
x_data = np.array([pad(np.array(row), (30, 10), [0, 0]) for row in sub_data['kw_occ_matrix'].to_numpy()])
# flatten trajectory matrices
original_dim = x_data[0].shape[0] * x_data[0].shape[1]
x_data = np.reshape(x_data, [-1, original_dim])

# in this case, y is experience level
exp_data = sub_data['user_exp'].to_numpy()
y_data = exp_data

# in this case, y1 is video engagement
video_data = sub_data['video_engaged'].to_numpy()
y1_data = video_data

# split train / test
train_pct = 0.7
i = math.ceil(0.7*x_data.shape[0])
x_train, x_test = x_data[0:i, :], x_data[i:, :]
y_train, y_test = y_data[0:i], y_data[i:]
y1_train, y1_test = y1_data[0:i], y1_data[i:]

# normalize
largest = x_data.max()
print('largest', largest)
x_train, x_test = x_train.astype('float32') / largest, x_test.astype('float32') / largest

print(x_train.shape, x_test.shape)
print(y_train.shape, y_test.shape)
print(y1_train.shape, y1_test.shape)

# Keyword occurrence network params
input_shape = (original_dim, )
batch_size = 50
intermediate_dim = 100
latent_dim = 2
epochs = 75

# VAE model = encoder + decoder
# build encoder model
inputs = Input(shape=input_shape, name='encoder_input')
print('input matrix is ', inputs)

x = Dense(intermediate_dim, activation='relu')(inputs)
z_mean = Dense(latent_dim, name='z_mean')(x)
z_log_var = Dense(latent_dim, name='z_log_var')(x)

# use reparameterization trick to push the sampling out as input
# note "output_shape" isn't necessary with the TensorFlow backend
z = Lambda(sampling, output_shape=(latent_dim,), name='z')([z_mean, z_log_var])

# instantiate encoder
encoder = Model(inputs, [z_mean, z_log_var, z], name='encoder')
encoder.summary()
plot_model(encoder, to_file='{}_encoder.png'.format(savename), show_shapes=True)

# build decoder
latent_inputs = Input(shape=(latent_dim,), name='z_sampling')
x = Dense(intermediate_dim, activation='relu')(latent_inputs)
outputs = Dense(original_dim, activation='sigmoid')(x)

# instantiate decoder
decoder = Model(latent_inputs, outputs, name='decoder')
decoder.summary()
plot_model(decoder, to_file='{}_decoder.png'.format(savename), show_shapes=True)

# instantiate VAE
outputs = decoder(encoder(inputs)[2])
vae = Model(inputs, outputs, name='vae_flattraj')

if __name__ == '__main__':

	print('inputs \n', inputs)
	print('input shape \n', inputs.shape)

	parser = argparse.ArgumentParser()

	help_ = "Load h5 model trained weights"
	parser.add_argument("-w", "--weights", help=help_)

	args = parser.parse_args()

	models = (encoder, decoder)

	def recon_loss(inp, out, in_dim):
		reconstruction_loss = mse(inp, out)
		reconstruction_loss *= in_dim
		return reconstruction_loss

	def kl_div_loss(mean, log_var):
		kl_loss = 1 + log_var - K.square(mean) - K.exp(log_var)
		kl_loss = K.sum(kl_loss, axis=-1)
		kl_loss *= -0.5
		return kl_loss

	reconstruction_loss = recon_loss(inputs, outputs, original_dim)
	kl_loss = kl_div_loss(z_mean, z_log_var)
	vae_loss = K.mean(reconstruction_loss + kl_loss)
	vae.add_loss(vae_loss)

	vae.compile(loss=None, optimizer='adam')
	# https://github.com/keras-team/keras/issues/9526
	vae.summary()
	plot_model(vae, to_file='{}.png'.format(savename), show_shapes=True)

	# add custom metrics
	# https://github.com/keras-team/keras/issues/9459
	vae.metrics_tensors.append(K.mean(reconstruction_loss))
	vae.metrics_names.append("mse")

	vae.metrics_tensors.append(K.mean(kl_loss))
	vae.metrics_names.append("kl")

	if args.weights:
		vae.load_weights(args.weights)
	else:
		# train autoencoder
		train_h = vae.fit(x_train,
				epochs=epochs,
				batch_size=batch_size,
				validation_data=(x_test, None))

		# inspect training losses
		plot_training_loss(train_h, epochs, unit, pb)

		# save trained weights
		os.makedirs('weights', exist_ok=True)
		w_filename = 'weights/{}_{}_{}.h5'.format(savename, unit, pb)
		vae.save_weights(w_filename)

	# show means of test set in latent space
	plot_mean_results(models,
				 x_test,
				 y_test,
				 y1_test,
				 unit,
				 pb,
				 batch_size=batch_size)

	# visualize generated vectors from sampling the latent space
	plot_latent_sampled(models, x_test, y_test, y1_test, unit, pb)


from flask import Blueprint, request, jsonify
from sqlalchemy import text
from ..utils import db

bp = Blueprint("records", __name__, url_prefix="/records")

@bp.route("/")
def records():
    conn = db.get_db()
    page = request.args["page"] if "page" in request.args else 0
    number = request.args["number"] if "number" in request.args else 10
    field = request.args["sortField"] if "sortField" in request.args else "disease"
    order = request.args["sortOrder"] if "sortOrder" in request.args else "asc"
    author = request.args["author"] if "author" in request.args else None

    if field == "country":
        field = "R.cname"
    elif field == "disease":
        field = "D.description"
    elif field == "discover_country":
        field = "DS.cname"
    elif field == "deaths":
        field = "total_deaths"
    elif field == "cured":
        field = "total_patients"

    query = f"""SELECT
                    R.cname as country,
                    R.email,
                    R.disease_code,
                    D.description as disease,
                    first_enc_date,
                    DS.cname as discover_country,
                    pathogen,
                    population,
                    total_deaths as deaths,
                    total_patients as cured,
                    name,
                    surname
                FROM Record R
                JOIN Country C ON R.cname = C.cname
                JOIN Disease D ON R.disease_code = D.disease_code
                JOIN DiseaseType DT ON D.id = DT.id
                JOIN Discover DS ON R.disease_code = DS.disease_code
                JOIN Users U ON R.email = U.email """
    filter = f"WHERE R.email = '{author}' " if author is not None else " "
    sorting = f"""ORDER BY {field} {order}
                  OFFSET {number} * {page}
                  LIMIT {number}"""

    rows = conn.execute(text(query + filter + sorting)).all()

    query = f"""SELECT COUNT(*)
                FROM Record R """
    count = conn.execute(text(query + filter)).one()[0]

    records = []
    for r in rows:
        records.append(r._asdict())

    res = { "count": count, "records": records }
    return jsonify(res)


import os

from setuptools import find_packages, setup

def read(rel_path):
    here = os.path.abspath(os.path.dirname(__file__))
    # intentionally *not* adding an encoding option to open, See:
    #   https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690
    with open(os.path.join(here, rel_path), "r") as fp:
        return fp.read()

def get_version(rel_path):
    for line in read(rel_path).splitlines():
        if line.startswith("__version__"):
            delim = '"' if '"' in line else "'"
            return line.split(delim)[1]
    raise RuntimeError("Unable to find version string.")

setup(
    name="em-loader",
    version=get_version("em_loader/__init__.py"),
    description="",
    long_description=read("README.md"),
    long_description_content_type="text/markdown",
    author="UC San Diego, Kevin Anderson, Peter Gerstoft, Luke Wood",
    author_email="lukewoodcs@gmail.com",
    url="",
    license="Apache License 2.0",
    install_requires=[
        "keras_cv",
        "termcolor",
        "distinctipy",
        "matplotlib",
        "numpy",
        "Pillow",
        "tabulate",
        "tensorflow-datasets>=4.2",
        "tqdm",
        "requests",
        "ipyplot",
        "tensorflow_addons",
        "pycm",
        "plotly",
        "prawn",
        "luketils",
        "tensorflowjs",
    ],
    extras_require={
        "dev": [
            "flake8",
            "mypy",
            "pytest",
            "pytype",
            "setuptools",
            "twine",
            "types-tabulate",
            "wheel",
            "yapf",
        ],
    },
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Console",
        "License :: OSI Approved :: Apache Software License",
        "Intended Audience :: Science/Research",
        "Programming Language :: Python :: 3",
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
    ],
    packages=find_packages(),
)


from django.conf.urls import patterns, include, url
from web.views import *
#from django.contrib import admin
from django.conf import settings

urlpatterns = patterns('',
    # Examples:
    url(r'^$',login),
    url(r'^login/', login),
    url(r'^person/$',person),
    url(r'^mission/$',mission),
    url(r'^report/$',daily_report),
    url(r'^load/$',load),
    (r'^static/(?P<path>.*)$','django.views.static.serve',{'document_root':settings.STATIC_URL}),
    #url(r'^admin/', include(admin.site.urls)),
)


#!/usr/bin/env python

#import necessary libraries
#pip install flask
#export FLASK_APP=flask-app
#flask run
from flask import Flask, json, render_template, request, jsonify

import os

#create instance of Flask app
app = Flask(__name__)

#decorator
@app.route("/")
def home_page():
    welcome_msg = '
Nobel Data
 \

/all for all years
 \

/all/year for one year
 \

/add_data for input data function, returns data
'
    return welcome_msg

@app.route("/all")
def all():
    json_url = os.path.join(app.static_folder,"","nobel.json")
    data_json = json.load(open(json_url))
    #render_template is always looking in templates folder
    return render_template('index.html',data=data_json)

@app.route("/all/<year>")
def get_year(year):
    json_url = os.path.join(app.static_folder,"","nobel.json")
    data_json = json.load(open(json_url))
    data = data_json['prizes']
    year = request.view_args['year']
    output_data = [x for x in data if x['year']==year]
    #render_template is always looking in templates folder
    return render_template('index.html',data=output_data)

@app.route("/add_data", methods=["POST", "GET"])
def add_data():
    if request.method == "POST":
        # data in a diction
        data = request.form
        # file path
        filenm = "./static/nobel.json"

        with open(filenm,'r+') as file:
            # First we load existing data into a dict.
            file_data = json.load(file)
            # Join new_data with file_data inside prizes
            file_data["prizes"].append(data)
            # Sets file's current position at offset. not understanding this step
            file.seek(0)
            # convert back to json.
            json.dump(file_data, file, indent = 4)
        # I'd like to return a statement that it was added and the data
        # but I don't know how do that, possibly a new html page
        # way beyond my current capabilities
        return '
Data added
'

    else:
        return render_template("add_data.html")

if __name__ == "__main__":
    app.run(debug=True)


# adapted from https://github.com/SoloChe/ddib/blob/main/scripts/synthetic_sample.py

import argparse
import os
import sys
sys.path.append(os.path.realpath('./'))

import pathlib

import numpy as np
import torch as th
import torch.distributed as dist

from common import read_model_and_diffusion
from guided_diffusion import dist_util, logger
from guided_diffusion.script_util import  model_and_diffusion_defaults, add_dict_to_argparser

from torchvision.utils import make_grid, save_image

def sample(model, diffusion, num_classes=None,
           w=None, sample_shape=None, name=None):
    samples_for_each_cls = 5 # default
    if num_classes is not None: # for clf-free
        samples_for_each_cls = sample_shape[0] // num_classes
        y = th.ones(samples_for_each_cls, dtype=th.long) *\
            th.arange(start = 0, end = num_classes).reshape(-1, 1)
        y = y.reshape(-1,1).squeeze().to(dist_util.dev())
        model_kwargs = {'y': y}
    else:
        model_kwargs = {}

    samples = diffusion.ddim_sample_loop(model,
                                         sample_shape,
                                         clip_denoised=False,
                                         w=w,
                                         model_kwargs=model_kwargs,
                                         device=dist_util.dev())

    if name == 'cifar10':
        samples = (samples + 1) / 2 # normalize it back
    return samples, samples_for_each_cls

def main():
    args = create_argparser().parse_args()

    dist_util.setup_dist()
    logger.configure()
    logger.log(f"args: {args}")
    logger.log("starting to sample.")

    args.num_classes = int(args.num_classes) if args.num_classes is not None else None

    image_folder = args.image_dir
    pathlib.Path(image_folder).mkdir(parents=True, exist_ok=True)

    logger.log(f"reading models ...")
    model, diffusion = read_model_and_diffusion(args, args.model_path)

    all_samples = []

    for n in range(args.n_batches):
        logger.log("sampling in progress.")
        logger.log(f"on batch {n}, device: {dist_util.dev()}")

        # save samples

        samples, samples_for_each_cls = sample(model, diffusion, num_classes=args.num_classes,
                         w=args.w, sample_shape=args.sample_shape, name=args.name)

        if args.save_image:
            grid = make_grid(samples, nrow=samples_for_each_cls)
            path = os.path.join(image_folder, f'batch_{n}_rank_{dist.get_rank()}.png')
            save_image(grid, path)

        if args.save_data_numpy:
            gathered_samples = [th.zeros_like(samples) for _ in range(dist.get_world_size())]
            dist.all_gather(gathered_samples, samples)
            all_samples.extend([sample.cpu().numpy() for sample in gathered_samples])

    if args.save_data_numpy:
        samples = np.concatenate(all_samples, axis=0)
        points_path = os.path.join(image_folder, f"all_samples.npy")
        np.save(points_path, samples)

    dist.barrier()
    logger.log(f"sampling synthetic data complete\n\n")

def create_argparser():
    defaults = dict(
        n_batches=4,
        model_path="",
        image_dir="",
        name="",
        save_data_numpy=True, # save data in numpy format
        save_image=True # save image as in png format
    )
    defaults.update(model_and_diffusion_defaults())
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--sample_shape",
        type=int,
        nargs="+",
        help="sample shape for a batch"
    )
    parser.add_argument(
        "--w",
        type=float,
        help="weight for clf-free samples",
        default=-1. # disabled in default
    )
    add_dict_to_argparser(parser, defaults)
    return parser

if __name__ == "__main__":
    main()


import re
import os
import shutil
import logging
import subprocess
from wave_train.graphics.helper import movie
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

class Animation:
    """
    Class for handling creation of animated output and eventually saving the
    animation as a .mp4 file. The motivation to develop this class has been to
    solve the underlying problem of the FuncAnimation class, namely that creation
    of animation output and visualization are executed in order and not in parallel.

    1) If image_file is not None, last snaphot is saved, but previous snapshots are removed.
    2) If image_file is not None and snapshot is True, all snapshots are saved.
       If image_file == None and snapshot == True, a default snapshot name will be assigned
    3) If image_file is not None, movie_file is not None, and snapshot is True,
       a movie is created from the snapshot and snapshots are not removed
    """
    __slots__ = [
        "dynamics",     # dynamics object used to update plot
        "movie_file",   # output name of movie
        "movie_suffix", # file suffix of output movie
        "image_file",   # name of the image file for output of last figure state
        "image_suffix", # suffix of the image file
        "snapshots",    # avoid cleanup of files created
        "save",         # variable holding the logical state if output movie is generated
        "fig",          # figure object holding the plot
        "update",       # update function called in FuncAnimation loop
        "init",         # init function called in FuncAnimation setup
        "frames",       # frames in the animation
        "pause",        # pause between frames
        "frame_rate",   # number of frames per second
        "frame_count"   # counter of frames/snaphots in animation
    ]

    movie_formats = ['mp4']
    image_formats = ['png']

    def __init__(self, dynamics, movie_file, image_file, snapshots, figure, update,
                            init=None, frames=None, blit=False, pause=200, frame_rate=10):
        """
        Parameters
        ------------
        dynamics: Instance of Dyanmics class
            The instance holding the dynamics object, which is provided
            via the fargs option to the update function
        movieFile: str
            Name of the movie file saving the animation output
        imageFile: str
            Name of the image file snapshotting the last frame
        figure: Figure object
            Figure instance for animated output
        update: Callable
            Update function for new frame
        init: Callable
            Initializer function for defining plot parameters
        frames: Iterable
            Number of frames to display
        blit: bool
            Use of blitting
        pause: uint
            Pause between each frame in visualization
        frame_rate:  uint
            Number of frames in animation output
        """
        self.dynamics       = dynamics
        self.movie_file     = movie_file
        self.image_file     = image_file
        self.snapshots      = snapshots
        self.fig            = figure
        self.update         = update
        self.init           = init
        self.frames         = frames
        self.pause          = pause
        self.frame_rate     = frame_rate
        self.frame_count    = 0
        self.save           = True

        if not (isinstance(movie_file, str) or movie_file is None):
            raise TypeError("Movie file argument must be an instance of 'str' or 'NoneType'")
        if not (isinstance(image_file, str) or image_file is None):
            raise TypeError("Image file argument must be an instance of 'str' or 'NoneType'")

        if self.image_file is None:
            if snapshots and self.movie_file is not None:
                self.image_file = "snaphot.png"
                logging.warning("Name for snapshots was not provided. Defaulting to 'snapshot.png'")
            elif self.movie_file is not None:
                # indicate that all snaphots must be removed
                self.image_file = "snapshot_removable.png"
            else:
                self.image_file = ""

        if self.movie_file is None:
            self.movie_file = ""

        self.image_file, self.image_suffix = os.path.splitext(self.image_file)
        self.movie_file, self.movie_suffix = os.path.splitext(self.movie_file)

    def start(self):
        try:
            animation = FuncAnimation(self.fig, self.update, init_func=lambda: None, frames=self.frames,
                                      fargs=(self.fig, self.dynamics, self, self.save),
                                      repeat=False)
            plt.show()
        except Exception as AnimationError:
            print(AnimationError)
        finally:
            movie_files = self._get_temp_files()

            if len(movie_files) == 0:
                return

            if self.movie_file != "":
                self._export_as_mp4(movie_files)

            if not self.snapshots and self.image_file != "":
                # if the snapshot is removable, remove all snapshots
                if not self.image_file == "snapshot_removable":
                    movie_files.pop(-1)

                self._cleanup(movie_files)

            if self.image_file == "" and self.snapshots == False:
                self._cleanup(movie_files)

    def _export_as_mp4(self, movie_files):
        print("""
------------------------
Saving animation to file
------------------------

Filename : {}
Frames   : {} per second
        """.format(self.movie_file, str(self.frame_rate)))

        subprocess.call(["ffmpeg", "-loglevel", "quiet", "-y", "-r", str(self.frame_rate - 9), "-i",
                         f"{self.image_file}_%06d{self.image_suffix}", f"output{self.movie_suffix}"])

        subprocess.call(["ffmpeg", "-loglevel", "quiet", "-y", "-i", f"output{self.movie_suffix}", "-c:v", "libx264", "-preset",
                         "slow", "-crf", "22", "-pix_fmt", "yuv420p", "-c:a",
                         "libvo_aacenc", "-b:a", "128k", f"{self.movie_file}{self.movie_suffix}"])

        os.remove(f"output{self.movie_suffix}")

    def _get_temp_files(self):
        tmp_files = re.compile("%s_\d{6}%s" % (self.image_file, self.image_suffix))
        movie_files = []

        for root, dir, files in os.walk("."):
            for tmp in files:
                if re.match(tmp_files, tmp):
                    movie_files.append(tmp)

        return sorted(movie_files)

    def _cleanup(self, movie_files):
        """
        Delete all temporary files that were created during the
        animation run. Ideally, the movie_files argument should
        be provided as the return of the _get_temp_files function.
        """
        for movie_file in movie_files:
            os.remove(movie_file)

    def save_as_image(self):
        if self.image_file != "":
            file_name = f"{self.image_file}_{str(self.frame_count).zfill(6)}{self.image_suffix}"
            self.fig.savefig(file_name)
            self.frame_count += 1


# This file is part of sner4 project governed by MIT license, see the LICENSE.txt file.
"""
scheduler.views.queue tests
"""

import json
from http import HTTPStatus
from pathlib import Path

from flask import url_for

from sner.server.scheduler.models import Job, Queue

def test_queue_list_route(cl_operator):
    """queue list route test"""

    response = cl_operator.get(url_for('scheduler.queue_list_route'))
    assert response.status_code == HTTPStatus.OK

def test_queue_list_json_route(cl_operator, queue):
    """queue list_json route test"""

    response = cl_operator.post(
        url_for('scheduler.queue_list_json_route'),
        {'draw': 1, 'start': 0, 'length': 1, 'search[value]': queue.name}
    )
    assert response.status_code == HTTPStatus.OK
    response_data = json.loads(response.body.decode('utf-8'))
    assert response_data['data'][0]['name'] == queue.name

    response = cl_operator.post(
        url_for('scheduler.queue_list_json_route', filter=f'Queue.name=="{queue.name}"'),
        {'draw': 1, 'start': 0, 'length': 1}
    )
    assert response.status_code == HTTPStatus.OK
    response_data = json.loads(response.body.decode('utf-8'))
    assert response_data['data'][0]['name'] == queue.name

    response = cl_operator.post(url_for('scheduler.queue_list_json_route', filter='invalid'), {'draw': 1, 'start': 0, 'length': 1}, status='*')
    assert response.status_code == HTTPStatus.BAD_REQUEST

def test_queue_add_route(cl_operator, queue_factory):
    """queue add route test"""

    aqueue = queue_factory.build()

    form = cl_operator.get(url_for('scheduler.queue_add_route')).forms['queue_form']
    form['name'] = aqueue.name
    form['config'] = aqueue.config
    form['group_size'] = aqueue.group_size
    form['priority'] = aqueue.priority
    response = form.submit()
    assert response.status_code == HTTPStatus.FOUND

    tqueue = Queue.query.filter(Queue.name == aqueue.name).one()
    assert tqueue.name == aqueue.name

def test_queue_add_route_config_validation(cl_operator, queue_factory):
    """queue add route test"""

    aqueue = queue_factory.build()

    form = cl_operator.get(url_for('scheduler.queue_add_route')).forms['queue_form']
    form['name'] = aqueue.name
    form['config'] = ''
    form['group_size'] = aqueue.group_size
    form['priority'] = aqueue.priority
    response = form.submit()
    assert response.status_code == HTTPStatus.OK
    assert response.lxml.xpath('//div[@class="invalid-feedback" and contains(text(), "Invalid YAML")]')

    form = response.forms['queue_form']
    form['config'] = "module: 'notexist'"
    response = form.submit()
    assert response.status_code == HTTPStatus.OK
    assert response.lxml.xpath('//div[@class="invalid-feedback" and text()="Invalid module specified"]')

    form = response.forms['queue_form']
    form['config'] = "module: 'dummy'\nadditionalKey: 'value'\n"
    response = form.submit()
    assert response.status_code == HTTPStatus.OK
    assert response.lxml.xpath('//div[@class="invalid-feedback" and contains(text(), "Invalid config")]')

def test_queue_edit_route(cl_operator, queue):
    """queue edit route test"""

    form = cl_operator.get(url_for('scheduler.queue_edit_route', queue_id=queue.id)).forms['queue_form']
    form['name'] = f'{form["name"].value} edited'
    response = form.submit()
    assert response.status_code == HTTPStatus.FOUND

    assert Queue.query.get(queue.id).name == form['name'].value

def test_queue_enqueue_route(cl_operator, queue, target_factory):
    """queue enqueue route test"""

    atarget = target_factory.build(queue=queue)

    form = cl_operator.get(url_for('scheduler.queue_enqueue_route', queue_id=queue.id)).forms['queue_enqueue_form']
    form['targets'] = f'{atarget.target}\n \n '
    response = form.submit()
    assert response.status_code == HTTPStatus.FOUND

    tqueue = Queue.query.get(queue.id)
    assert len(tqueue.targets) == 1
    assert tqueue.targets[0].target == atarget.target

def test_queue_flush_route(cl_operator, target):
    """queue flush route test"""

    queue_id = target.queue_id

    form = cl_operator.get(url_for('scheduler.queue_flush_route', queue_id=target.queue_id)).form
    response = form.submit()
    assert response.status_code == HTTPStatus.FOUND

    assert not Queue.query.get(queue_id).targets

def test_queue_prune_route(cl_operator, job_completed):
    """queue flush route test"""

    form = cl_operator.get(url_for('scheduler.queue_prune_route', queue_id=job_completed.queue_id)).form
    response = form.submit()
    assert response.status_code == HTTPStatus.FOUND

    assert not Job.query.filter(Job.queue_id == job_completed.queue_id).all()
    assert not Path(job_completed.output_abspath).exists()

def test_queue_prune_route_runningjob(cl_operator, job):
    """queue flush route test with running job; should fail, delete running job would corrupt heatmap"""

    form = cl_operator.get(url_for('scheduler.queue_prune_route', queue_id=job.queue_id)).form
    response = form.submit(expect_errors=True)
    assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR

    assert len(Job.query.filter(Job.queue_id == job.queue_id).all()) == 1

def test_queue_delete_route(cl_operator, job_completed):
    """queue delete route test"""

    tqueue = Queue.query.get(job_completed.queue_id)
    assert Path(tqueue.data_abspath)

    form = cl_operator.get(url_for('scheduler.queue_delete_route', queue_id=tqueue.id)).form
    response = form.submit()
    assert response.status_code == HTTPStatus.FOUND

    assert not Queue.query.get(tqueue.id)
    assert not Path(tqueue.data_abspath).exists()

def test_queue_delete_route_runningjob(cl_operator, job):
    """queue delete route test with running job; should fail as deleting queue with running job would corrupt heatmap"""

    form = cl_operator.get(url_for('scheduler.queue_delete_route', queue_id=job.queue_id)).form
    response = form.submit(expect_errors=True)
    assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR

    assert len(Job.query.filter(Job.queue_id == job.queue_id).all()) == 1


# import files for servo motion and distance sensor
# import libraries
import numpy as np
import random
import pandas as pd
from itertools import product

class QLearning_advanced:

    def __init__(self, servos):
    ### init matrices and parameters ###
        self.servos = servos

        # Q-matrix
        self.q_matrix = np.zeros((81,6))
        self.actions = ['0_','1_','2_','_0','_1','_2']
        self.states = np.array(['00','01','11',
                        '10','02','22',
                        '20','12','21'])

        self.statesWithLastState = list(product(self.states, repeat=2))
        self.q_matrixDF = pd.DataFrame(self.q_matrix, columns=self.actions, index=self.statesWithLastState)
        self.q_matrixDF

        self.possibleStates = []

        for i in range(len(self.statesWithLastState)):
            if self.statesWithLastState[i][0][0] == self.statesWithLastState[i][1][0] or self.statesWithLastState[i][0][1] == self.statesWithLastState[i][1][1]:
                self.possibleStates.append(self.statesWithLastState[i])

        # removing impossible states from q matrix -> 45 possible states left
        self.q_matrix = np.zeros((45,6))
        self.q_matrixDF = pd.DataFrame(self.q_matrix, columns=self.actions, index=self.possibleStates)

        # discount rate
        self.gamma = 0.9

        # learning rate
        self.alpha = 0.1

        # decides if Q-value is used todetermine the action or take a random sample of the action space
        self.epsilon = 1.0
        self.epsilon_min = 0.01
        self.epsilon_decay = 0.95 #0.9

    ### methods for Q-Learning algorithm ###

    # get reward for movement
    def getDistanceReward(self, dist1, dist2):
        distanceReward = 0
        if dist2 < dist1:
            if dist1 - dist2 > 3:
                distanceReward = 10
            else:
                distanceReward = 5
        elif dist2 == dist1:
            distanceReward = -2
        elif dist1 - dist2 < -1:
            distanceReward = -10
        else:
            distanceReward = -5
        return distanceReward

    # get random action
    def getValidRandomAction(self, state):
        randomAction = state.sample(n=1, axis=1)
        print(f"random action:{randomAction.columns.values[0]}")
        return randomAction.columns.values[0]

    # get the next state for taken action
    def getNextState(self, state, action):
        if action.startswith('_'):
            temp = list(state.index[0])
            temp[1] = temp[0]
            temp[0] = temp[0][0] + action[1]
            nextState = tuple(temp)
        else:
            temp = list(state.index[0])
            temp[1] = temp[0]
            temp[0] = action[0] + temp[0][1]
            nextState = tuple(temp)
        return nextState

    # method to get all possible actions (not None) from a state row
    def getAllPossibleActions(self,stateRow):
        allPossibleActions = []
        for i in range(len(stateRow.columns)):
          allPossibleActions.append(stateRow.columns.values[i])
        return allPossibleActions

    def setStartingState(self,state,leg):
        if state == '00':
            print('00')
            self.servos.moveUpperLegDown(leg)
            self.servos.moveLowerLegDown(leg)
        elif state == '01':
            print('01')
            self.servos.moveUpperLegDown(leg)
            self.servos.moveLowerLegMid(leg)
        elif state == '02':
            print('02')
            self.servos.moveUpperLegDown(leg)
            self.servos.moveLowerLegUp(leg)
        elif state =='10':
            print('10')
            self.servos.moveUpperLegMid(leg)
            self.servos.moveLowerLegDown(leg)
        elif state =='11':
            print('11')
            self.servos.moveUpperLegMid(leg)
            self.servos.moveLowerLegMid(leg)
        elif state =='12':
            print('12')
            self.servos.moveUpperLegMid(leg)
            self.servos.moveLowerLegUp(leg)
        elif state =='20':
            print('20')
            self.servos.moveUpperLegUp(leg)
            self.servos.moveLowerLegDown(leg)
        elif state =='21':
            print('21')
            self.servos.moveUpperLegUp(leg)
            self.servos.moveLowerLegMid(leg)
        elif state =='22':
            print('22')
            self.servos.moveUpperLegUp(leg)
            self.servos.moveLowerLegUp(leg)

    def doAction(self, action, leg):
        if action == "2_":
            self.servos.moveUpperLegUp(leg)
        elif action == "0_":
            self.servos.moveUpperLegDown(leg)
        elif action == "_2":
            self.servos.moveLowerLegUp(leg)
        elif action == "_0":
            self.servos.moveLowerLegDown(leg)
        elif action == "1_":
            self.servos.moveUpperLegMid(leg)
        elif action == "_1":
            self.servos.moveLowerLegMid(leg)

    # function to find best action for current state
    def getBestActionFromQTable(self, state):
        maxValue = state.values[0][0]
        maxAction = state.columns.values[0]
        for i in range(len(state.columns)):
            if state.values[0][i] >= maxValue:
                maxAction = state.columns.values[i]
                maxValue = state.values[0][i]
            else:
                continue
        return maxAction

    def getBestNextValueFromQTable(self, state):
        maxValue = state.values[0][0]
        for i in range(len(state.columns)):
            if state.values[0][i] >= maxValue:
                maxValue = state.values[0][i]
            else:
                continue
        return maxValue


output_dir = './'
model_to_save = model.module if hasattr(model, 'module') else model
model_to_save.save_pretrained(output_dir)
tokenizer.save_pretrained(output_dir)

from transformers import BertTokenizer,BertForSequenceClassification
import torch
# Load the BERT tokenizer.
print('Loading BERT tokenizer...')
output_dir = './'
tokenizer = BertTokenizer.from_pretrained(output_dir)
model_loaded = BertForSequenceClassification.from_pretrained(output_dir)

test_text = test.tweet.values
input_ids = []
attention_mask = []
for i in tqdm(test_text):
    encoded_data = tokenizer.encode_plus(
    str(i),
    add_special_tokens=True,
    max_length= 64,
    pad_to_max_length = True,
    return_attention_mask= True,
    return_tensors='pt')
    input_ids.append(encoded_data['input_ids'])
    attention_mask.append(encoded_data['attention_mask'])
input_ids = torch.cat(input_ids,dim=0)
attention_mask = torch.cat(attention_mask,dim=0)

testset = TensorDataset(input_ids,attention_mask)
test_dl = DataLoader(testset, batch_size=1)

def evaluate2(dataloader_test):
    model_loaded =  BertForSequenceClassification.from_pretrained(output_dir)
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model_loaded = model_loaded.to(device)
    predictions = []

    for batch in dataloader_test:
        batch = tuple(b.to(device) for b in batch)
        inputs = {
            'input_ids':batch[0],
            'attention_mask': batch[1]
        }
        with torch.no_grad():
            outputs = model_loaded(**inputs)
        logits = outputs[0]
        logits = logits.detach().cpu().numpy()
        predictions.append(logits)
    predictions = np.concatenate(predictions,axis=0)
    result = np.argmax(predictions, axis=1).flatten()

    return result

test_result = evaluate2(test_dl)

measure_accuracy = test
measure_accuracy['bert'] = test_result
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split

print('f1 score', f1_score(measure_accuracy.sentiment, measure_accuracy.bert, average=None))
print(confusion_matrix(measure_accuracy.sentiment, measure_accuracy.bert))
print(precision_recall_fscore_support(measure_accuracy.sentiment, measure_accuracy.bert, average='weighted'))
print('accuarcy', accuracy_score(measure_accuracy.sentiment, measure_accuracy.bert))


import timeit
import sys, os
import importlib
import shutil

class TestHandler:

    def __init__(self, filename):
        self.originalFilname = filename
        if "/" in filename or "\\" in filename:
            shutil.copy(filename, os.path.join(os.path.dirname(__file__), "testFile.py"))
            filename = os.path.join(os.path.dirname(__file__), "testFile.py")
        self.filename = filename
        self.fileHandler = open(filename, "r")
        self.executable = importlib.import_module(filename.split("/")[-1].split("\\")[-1][:-3])

    def processFile(self):
        if self.originalFilname != self.filename:
            try:
                shutil.copy(self.originalFilname, "testFile.py")
            except shutil.SameFileError:
                pass
        importlib.reload(self.executable)
        self.fileHandler = open(self.filename, "r")
        #__import__(self.filename)
        self.testCases = {}
        inTest = False
        currentTestCategory = ""
        currentTestName = ""
        for line in self.fileHandler:
            line = line.rstrip()
            if line.startswith("endTest'''"):
                inTest = False
            if inTest:
                if line.startswith("testCategory"):
                    currentTestCategory = line.split("testCategory ")[1]
                    if currentTestCategory not in self.testCases:
                        self.testCases[currentTestCategory] = {}

                if line.startswith("testName"):
                    currentTestName = line.split("testName ")[1]
                    if currentTestName not in self.testCases[currentTestCategory]:
                        self.testCases[currentTestCategory][currentTestName] = {}

                if line.startswith("maxtime"):
                    maxtime = int(line.split("maxtime ")[1])
                    self.testCases[currentTestCategory][currentTestName]["maxtime"] = maxtime

                if line.startswith("expectedResult"):
                    expectedResult = eval(line.split("expectedResult ")[1])
                    self.testCases[currentTestCategory][currentTestName]["expectedResult"] = expectedResult

                if line.startswith("call"):
                    call = line.split("call ")[1]
                    self.testCases[currentTestCategory][currentTestName]["call"] = "self.executable." + call

            if line.startswith("'''testStart"):
                inTest = True

    def executeTests(self):
        allok = True
        for category in self.testCases:
            catok = True
            for test in self.testCases[category]:
                try:
                    expected = self.testCases[category][test]["expectedResult"]
                    start_time = timeit.default_timer()
                    result = eval(self.testCases[category][test]["call"])
                    elapsed = timeit.default_timer() - start_time
                    if result != expected or elapsed > self.testCases[category][test]["maxtime"]:
                        allok = False
                        catok = False
                        self.testCases[category][test]["testOK"] = False
                        errorstring = "   " + " ERROR " + test
                        rstring = self.testCases[category][test]["call"].split("self.executable.")[1]
                        if len(rstring) > 70:
                            rstring = rstring[:70] + "..."
                        if (result != expected):
                            errorstring = errorstring + "\n      " + "Falsches Ergebnis"
                        else:
                            errorstring = errorstring + "\n      " + "Zu langsam"
                        errorstring = errorstring + "\n      " + rstring
                        errorstring = errorstring + "\n      " + "expected: " + str(expected) + ", got: " + str(result)
                        errorstring = errorstring + "\n      " + "runtime: " + str(elapsed) + " seconds "
                        errorstring = errorstring + "\n"
                        self.testCases[category][test]["resultOutput"] = errorstring
                    else:
                        self.testCases[category][test]["testOK"] = True
                        self.testCases[category][test]["resultOutput"] = "   " + test + " OK"
                except Exception as inst:
                    allok = False
                    catok = False
                    self.testCases[category][test]["testOK"] = False
                    errorstring = "   " + " ERROR " + test
                    errorstring = errorstring + "\n      " + "Programmierfehler"
                    rstring = self.testCases[category][test]["call"].split("self.executable.")[1]
                    if len(rstring) > 70:
                        rstring = rstring[:70] + "..."
                    errorstring = errorstring + "\n      " + rstring
                    errorstring = errorstring + "\n      " + inst
                    exc_type, exc_obj, exc_tb = sys.exc_info()
                    fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
                    errorstring = errorstring + "\n      " + str(exc_type) + " " + str(fname) + " " + str(exc_tb.tb_lineno)
                    errorstring = errorstring + "\n"
                    self.testCases[category][test]["resultOutput"] = errorstring
            if catok:
                print("OK " + category)
            else:
                print("ERROR " + category)
            for test in self.testCases[category]:
                print(self.testCases[category][test]["resultOutput"])
        return allok


import asyncio
from fastapi import APIRouter, Depends, HTTPException
from fastapi_cache.decorator import cache
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from database import get_async_session
from operations.models import Operation
from operations.schemas import OperationCreate, OperationRead, OperationUpdate
from schemas import ErrorDetails

OPERATIONS_PER_PAGE = 3

router = APIRouter()

# Get operations with specific type
@router.get("/")
@cache(expire=5)
async def get_specific_operations(
    operation_type: str, page: int = 1, session: AsyncSession = Depends(get_async_session)
) -> list[OperationRead]:
    query = select(Operation).where(Operation.type == operation_type) \
                .limit(OPERATIONS_PER_PAGE).offset(OPERATIONS_PER_PAGE * (page - 1))
    result = await session.scalars(query)
    return result.all()

# Long operation
@router.get("/long-operation")
@cache(expire=30)
async def process_long_operations() -> str:
    await asyncio.sleep(5)
    return "results"

# Create operation
@router.post(
    "/",
    status_code=201
)
async def add_specific_operation(
    new_operation: OperationCreate,
    session: AsyncSession = Depends(get_async_session)
) -> OperationRead:
    op = Operation(**new_operation.model_dump())
    session.add(op)
    await session.commit()
    return op

# Create or replace operation
@router.put(
    "/{operation_id}",
    responses={
        440: {
            "description": "Error. An operation with this id doesn't exists",
            "model": ErrorDetails
        }
    }
)
async def replace_specific_operation(
    operation_id: int,
    operation_data: OperationCreate,
    session: AsyncSession = Depends(get_async_session)
) -> OperationRead:
    op = await session.get(Operation, operation_id)
    if op:
        op.update_from_dict(operation_data.model_dump())
    else:
        raise HTTPException(
            status_code=440,
            detail=f"Error occurred. An operation with id={operation_id} " \
                    "doesn't exists"
        )

    await session.commit()
    return op

# Edit operation
@router.patch(
    "/{operation_id}",
    responses={
        440: {
            "description": "Error. An operation with this id doesn't exists",
            "model": ErrorDetails
        }
    }
)
async def update_specific_operation(
    operation_id: int,
    operation_data: OperationUpdate,
    session: AsyncSession = Depends(get_async_session)
) -> OperationRead:
    op = await session.get(Operation, operation_id)
    if op:
        op.update_from_dict(operation_data.model_dump())
        await session.commit()
        return op
    else:
        raise HTTPException(
            status_code=440,
            detail=f"Error occurred. An operation with id={operation_id} " \
                    "doesn't exists"
        )


import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
import sys
from bs4 import BeautifulSoup
sys.path.append('../')
import pandas as pd
import re
import collections
from nltk.corpus import stopwords
import random
import pickle
import argparse
import math

def clean(review, remove_freq = True):
    """Given a review, cleans it by removing html tags and punctation.
    Also filters out very common words if the remove_freq flag is given.
    Params:
    review - a single review from the dataset
    remove_freq: True if we remove frequent words (enabled by default)
    """
    # remove html
    text = BeautifulSoup(review, "html5lib").get_text()
    # regexp matching to extract letters only
    letters_only = re.sub("[^a-zA-Z]", " ", text)
    words = letters_only.lower().split()
    stops = set(stopwords.words("english"))
    meaningful_words = [w for w in words if not w in stops] if remove_freq else words
    return (" ".join(meaningful_words))

def extract_and_clean_data(path_to_file, get_labels = True):
    """Takes in a path to the reviews dataset and returns cleaned reviews along
    with their sentiments (binary labels)
    Params:
    path_to_file: the path to the data file
    Returns:
    clean_reviews: The cleaned reviews (punctation and common words stripped)
    y: the labels (sentiments for each review)
    """
    data = pd.read_csv(path_to_file, header = 0, delimiter='\t', quoting=3)
    clean_reviews = [clean(data['review'][i])
                     for i in range(len(data['review']))]
    if not get_labels:
        return clean_reviews
    y = data['sentiment']
    return clean_reviews, y

def build_vocab(cleaned_reviews, keep_dups = False):
    """Takes in a bunch of reviews and returns a list V of all the words
    (no dups)
    Params: cleaned_reviews
    Returns: vocab: a list of all of the words
    """
    li = []
    for review in cleaned_reviews:
        a = review.split()
        for item in a:
            li.append(item)
    return list(set(li)) if not keep_dups else li

def word_one_hot(word, vocab):
    idx = vocab.index(word)
    if(idx < 0):
        raise ValueError("vocab must contain word")
    v = np.zeros(len(vocab))
    v[idx] = 1
    return np.array(v)

def create_training_examples_one_hot(review, vocab, window_size = 1):
    """Creates the word pairs for the skip-gram model.
    Params: review - a single (cleaned) review
    window_size: how many context words to use.
    Default = 1 - ie, use the word to the left and to the right as the context
    Returns:
    data - a bunch of training data
    """
    words = review.split()
    data, labels = [], []
    for i in range(len(words)):
        left = [words[i-j] for j in range(1, window_size + 1) if i-j >= 0]
        right = [words[i+j] for j in range(1, window_size + 1) if i+j < len(words)]
        neighbors = left + right
        for item in neighbors:
            data.append(word_one_hot(words[i], vocab))
            labels.append(word_one_hot(item, vocab))
    return data, labels

def build_dataset(words, vocabulary_size):
    # create a counter for the msot common words
  count = [['UNK', -1]]
  count.extend(collections.Counter(words).most_common(vocabulary_size - 1))
  dictionary = dict()
  for word, _ in count:
    dictionary[word] = len(dictionary)
  data = list()
  unk_count = 0
  # get word indices
  for word in words:
    if word in dictionary:
      index = dictionary[word]
    else:
      index = 0  # dictionary['UNK']
      unk_count += 1
    data.append(index)
  count[0][1] = unk_count

  reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
  return data, count, dictionary, reverse_dictionary

# Step 3: Function to generate a training batch for the skip-gram model.
def generate_batch(batch_size, num_skips, skip_window):
  global data_index
  assert batch_size % num_skips == 0
  assert num_skips <= 2 * skip_window
  batch = np.ndarray(shape=(batch_size), dtype=np.int32)
  labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
  span = 2 * skip_window + 1  # [ skip_window target skip_window ]
  buffer = collections.deque(maxlen=span)
  for _ in range(span):
    buffer.append(data[data_index])
    data_index = (data_index + 1) % len(data)
  for i in range(batch_size // num_skips):
    target = skip_window  # target label at the center of the buffer
    targets_to_avoid = [skip_window]
    for j in range(num_skips):
      while target in targets_to_avoid:
        target = random.randint(0, span - 1)
      targets_to_avoid.append(target)
      batch[i * num_skips + j] = buffer[skip_window]
      labels[i * num_skips + j, 0] = buffer[target]
    buffer.append(data[data_index])
    data_index = (data_index + 1) % len(data)
  # Backtrack a little bit to avoid skipping words in the end of a batch
  data_index = (data_index + len(data) - span) % len(data)
  return batch, labels

    # Visualize
def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):
    assert low_dim_embs.shape[0] >= len(labels), "More labels than embeddings"
    plt.figure(figsize=(18, 18))  # in inches
    for i, label in enumerate(labels):
        x, y = low_dim_embs[i, :]
        plt.scatter(x, y)
        plt.annotate(label,
                    xy=(x, y),
                    xytext=(5, 2),
                    textcoords='offset points',
                    ha='right',
                    va='bottom')

    plt.savefig(filename)
    return 1
     # plt.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-l", "--load", help="load data structures from a file",
                        action = "store_true")
    args = parser.parse_args()
    if args.load:
        print("loading data from file data/data.pik")
        with open('data/data.pik', 'rb') as f:
            test_clean_reviews, clean_reviews, y, data, count, dictionary, reverse_dictionary, vocabulary_size = pickle.load(f)
            vocab = build_vocab(clean_reviews, keep_dups = False)
    else:
        clean_reviews, y = extract_and_clean_data('../data/labeledTrainData.tsv')
        test_clean_reviews = extract_and_clean_data('../data/unlabeledTrainData.tsv',
                                                    get_labels = False)
        vocab = build_vocab(clean_reviews, keep_dups = False)
        words, vocabulary_size = vocab, len(vocab)
        data, count, dictionary, reverse_dictionary = build_dataset(words, vocabulary_size)

        del words # hint to Python to reduce memory
        with open('data/data.pik', 'wb') as f:
            pickle.dump([test_clean_reviews, clean_reviews, y, data, count, dictionary,
                         reverse_dictionary, vocabulary_size], f, -1)

    print("cleaned reviews len: {}".format(len(clean_reviews)))
    print('Most common words (+UNK)', count[:5])
    print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])
    data_index = 0
    batch_size = 8
    batch, labels = generate_batch(batch_size=batch_size, num_skips=2, skip_window=1)
    print(batch.shape)
    print(labels.shape)
    train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
    train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
    valid_window = 100  # Only pick dev samples in the head of the distribution.
    valid_size = 16
    embedding_size = 128
    valid_examples = np.random.choice(valid_window, valid_size, replace=False)
    valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
        # creates a vocab_size * embedding_size matrix. For the ith word in the vocab,
    # the vector embeddings[i] is the corresponding embedding.
    print(vocabulary_size)
    print(embedding_size)
    embeddings = tf.Variable(
        tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
    embed = tf.nn.embedding_lookup(embeddings, train_inputs)

    # treats the embedding matrix as a lookup table so we can get the embeddings

    # "Xavier" init
    nce_weights = tf.Variable(tf.truncated_normal(
        [vocabulary_size, embedding_size],
        stddev=1.0/math.sqrt(embedding_size)))
    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
    num_sampled = 64
    # Compute the average NCE loss for the batch.
    # tf.nce_loss automatically draws a new sample of the negative labels each
    # time we evaluate the loss.
    losses = ['nce', 'sampled', ]
    loss_fun = tf.reduce_mean(
        tf.nn.nce_loss(weights=nce_weights,
                       biases=nce_biases,
                       labels=train_labels,
                       inputs=embed,
                       num_sampled=num_sampled,
                       num_classes=vocabulary_size))

    # Construct the SGD optimizer using a learning rate of 1.0.
    optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss_fun)

    # Compute the cosine similarity between minibatch examples and all embeddings.
    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
    normalized_embeddings = embeddings / norm
    valid_embeddings = tf.nn.embedding_lookup(
        normalized_embeddings, valid_dataset)
    similarity = tf.matmul(
        valid_embeddings, normalized_embeddings, transpose_b=True)

    init = tf.global_variables_initializer()
    skip_window = 1       # How many words to consider left and right.
    num_skips = 2         # How many times to reuse an input to generate a label.
    nsteps = 500000
    with tf.Session() as sess:
        sess.run(init)
        avg_loss = 0.
        for step in range(nsteps):
            batch_inputs, batch_labels = generate_batch(batch_size,
                                                        num_skips, skip_window) # 2 = num skips, 1 = skip window
            feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
            _, loss = sess.run([optimizer, loss_fun], feed_dict = feed_dict)
            avg_loss+=loss
            if step % 10000 == 0:
                if step != 0: avg_loss /= 10000
                print("Average loss at epoch {}: {}".format(step, avg_loss))
                avg_loss = 0.
        final_embeddings = normalized_embeddings.eval()
    try:
         from sklearn.manifold import TSNE
         tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
         plot_only = 500
         low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
         labels = [reverse_dictionary[i] for i in range(plot_only)]
         plot_with_labels(low_dim_embs, labels)
    except ImportError:
        print("please install sklearn, matplotlib, and scipy for tsne embeddings")
    print(final_embeddings.shape)
    print(len(reverse_dictionary.values()))
    with open('data/embeddings.txt', 'wb') as f:
        print("dumping embeddings")
        pickle.dump([final_embeddings, reverse_dictionary], f, -1)
    with open('data/data.txt', 'wb') as f:
        print("dumping reviews, vocab, and labels")
        train_cleaned_reviews = clean_reviews
        pickle.dump([train_cleaned_reviews, test_clean_reviews, vocab, y], f, -1)


from kivy.lang import Builder
from kivymd.app import MDApp
from kivymd.uix.picker import MDTimePicker

class MainApp(MDApp):
	def build(self):
		self.theme_cls.theme_style = "Light"
		self.theme_cls.primary_palette = "BlueGray"
		return Builder.load_file('time.kv')

	# Get Time
	def get_time(self, instance, time):
		self.root.ids.time_label.text = str(time)

	# Cancel
	def on_cancel(self, instance, time):
		self.root.ids.time_label.text = "You Clicked Cancel!"

	def show_time_picker(self):
		from datetime import datetime

		# Define default time
		default_time = datetime.strptime("4:20:00", '%H:%M:%S').time()

		time_dialog = MDTimePicker()
		# Set default Time
		time_dialog.set_time(default_time)
		time_dialog.bind(on_cancel=self.on_cancel, time=self.get_time)
		time_dialog.open()

MainApp().run()


from django.shortcuts import render
from django.views import generic

from ml.models import UserProfile, Publication, Resource

class GroupView(generic.ListView):
    template_name = 'group.html'
    context_object_name = 'users'

    def get_queryset(self):
        users = dict()
        try:
            users['leader'] = UserProfile.objects.get(membership='L')
        except UserProfile.DoesNotExist:
            pass
        users['members'] = list(UserProfile.objects.filter(
            membership='M'))
        users['exmembers'] = list(UserProfile.objects.filter(
            membership='X'))
        return users

def get_profile(request, username):
    context = {'user': UserProfile.objects.get(user__username=username)}
    return render(request, 'user.html', context)

class PublicationDetailView(generic.DetailView):
    template_name = 'publ_detail.html'
    model = Publication

class PublicationListView(generic.ListView):
    template_name = 'publ_list.html'
    context_object_name = 'publications'

    def get_queryset(self):
        return Publication.objects.order_by('-year')

class ResourceDetailView(generic.DetailView):
    template_name = 'res_detail.html'
    model = Resource

class ResourcesView(generic.ListView):
    template_name = 'res_list.html'
    context_object_name = 'resources'

    def get_queryset(self):
        return Resource.objects.order_by('name')

class IndexView(generic.TemplateView):
    template_name = 'ml_home.html'


from collections import deque

class PieceInfo :
    def __init__(self, num, locaArr):
        self.num = num
        self.loca = locaArr

class PuzzleGame :
    def __init__(self, gameBoard, table):
        self.answer = 0
        self.gameBoard = gameBoard
        self.table = table
        self.N = len(gameBoard)

        self.gmaeDict = dict()
        self.tableDict = dict()

        self.gameVisted = set()
        self.tableVisted = set()

        self.dx = [-1,0,1,0]
        self.dy = [0,1,0,-1]

    def findPiece(self, target, board, x, y, visted) :
        visted.add((x,y))
        que = deque([(x,y)])
        arr = [(x,y)]
        while que :
            qx, qy = que.popleft()
            for i in range(4) :
                nx = qx + self.dx[i]
                ny = qy + self.dy[i]
                if nx < 0 or nx >= self.N or ny < 0 or ny >= self.N :
                    continue
                if board[nx][ny] != target :
                    continue
                if (nx,ny) in visted :
                    continue
                visted.add((nx, ny))
                que.append((nx, ny))
                arr.append((nx,ny))
        return arr

    def getTablePiece(self) :
        for i in range(self.N) :
            for j in range(self.N) :
                if (i,j) in self.tableVisted or self.table[i][j] == 0:
                    continue
                arr = self.findPiece(1, self.table, i,j, self.tableVisted)
                newPiece = PieceInfo(len(arr), arr)
                if len(arr) not in self.tableDict:
                    self.tableDict[len(arr)] = deque([])
                self.tableDict[len(arr)].append(newPiece)

    def getBoardPiece(self) :
        for i in range(self.N) :
            for j in range(self.N) :
                if (i,j) in self.gameVisted or self.gameBoard[i][j] == 1:
                    continue
                arr = self.findPiece(0, self.gameBoard, i,j, self.gameVisted)
                newPiece = PieceInfo(len(arr), arr)
                if len(arr) not in self.gmaeDict:
                    self.gmaeDict[len(arr)] = deque([])
                self.gmaeDict[len(arr)].append(newPiece)

    def moveToZero(self, arr) :
        result = []
        leftArr = sorted(arr, key = lambda x : x[0])
        topArr = sorted(arr, key = lambda x: x[1])

        left = leftArr[0][0]
        top = topArr[0][1]
        for x,y in arr :
            result.append((x-left, y - top))
        return sorted(result)

    def rotate90(self, arr) :
        retsut = []
        for x ,y in arr :
            nx = y + self.N -1
            ny = -x + self.N -1
            retsut.append((nx,ny))
        return self.moveToZero(retsut)

    def getAnswer(self) :
        self.getBoardPiece()
        self.getTablePiece()
        for k in self.gmaeDict.keys() :
            if k not in self.tableDict :
                continue
            if k == 2 :
                self.answer += min(len(self.gmaeDict[k]), len(self.tableDict[k]))*2
                continue
            elif k == 1 :
                self.answer += min(len(self.gmaeDict[k]), len(self.tableDict[k]))
                continue

            while self.gmaeDict[k] :
                piece = self.gmaeDict[k].pop()
                piece.loca = self.moveToZero(piece.loca)
                bf = False
                for tp in self.tableDict[k] :
                    target = self.moveToZero(tp.loca)
                    for _ in range(4) :
                        if target == piece.loca :
                            self.answer += k
                            bf = True
                            break
                        else :
                            target = self.rotate90(target)
                    if bf :
                        self.tableDict[k].remove(tp)
                        break
        return self.answer

def solution(game_board, table):
    answer = -1
    g = PuzzleGame(game_board, table)
    answer = g.getAnswer()
    return answer

gameBoard = [[1,1,0,0,1,0],[0,0,1,0,1,0],[0,1,1,0,0,1],[1,1,0,1,1,1],[1,0,0,0,1,0],[0,1,1,1,0,0]]
table = [[1,0,0,1,1,0],[1,0,1,0,1,0],[0,1,1,0,1,1],[0,0,1,0,0,0],[1,1,0,1,1,0],[0,1,0,0,0,0]]
print(solution(gameBoard, table))


import cloudpickle
from typing import Callable

from api_challenge import SETTINGS

class ModelReader:
    def __init__(self, model_name):
        self.model_name = model_name
        self._model = None

    @property
    def model(self) -> Callable:
        if self._model is None:
            try:
                with open(
                    SETTINGS.MODEL_ENVIRONMENTS_PATH / f"{self.model_name}.pkl", "rb"
                ) as file:
                    model = cloudpickle.load(file)
                self._model = model
            except Exception as exc:
                print("No es posible cargar el ambiente de los modelos")
                raise exc
        return self._model

MODELS = {
    "logistic_model": ModelReader("logistic_model").model,
    "xgboost_model": ModelReader("xgboost_model").model,
    "catboost_model": ModelReader("catboost_model").model,
    "xgboost_fI_model": ModelReader("xgboost_fI_model").model,
    "catboost_fI_model": ModelReader("catboost_fI_model").model,
}


import socket
import json
import pygame
from TicTacToeView import TicTacToeView

class TicTacToeClient:
    def __init__(self, host, port):
        self.host = host
        self.port = port
        self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.client.connect((self.host, self.port))

        pygame.init()
        self.view = TicTacToeView()
        self.clock = pygame.time.Clock()
        self.view.run()
        self.server_data = None

    def receive_data(self):
        while True:
            data = self.client.recv(1024)
            if not data:
                break
            self.server_data = json.loads(data.decode())

    def run(self):
        running = True
        while running:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    self.client.close()
                    pygame.quit()
                    running = False

            self.send_data()
            self.receive_data()

            self.view.update_board(self.server_data)
            self.view.update_screen()

            self.clock.tick(30)

    def send_data(self):
        while True:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    self.client.close()
                    pygame.quit()
                    return
                elif event.type == pygame.MOUSEBUTTONDOWN and event.button == 1:
                    if self.view.is_my_turn():
                        cell_row, cell_col = self.view.get_cell_index(*pygame.mouse.get_pos())
                        if self.view.board_states[cell_row][cell_col] == "":
                            self.view.handle_mouse_click(0, cell_row, cell_col)
                            self.send_board()
            self.clock.tick(30)
            self.send_board()

    def send_board(self):
        board_data = json.dumps(self.view.board_states)
        self.client.sendall(board_data.encode())

if __name__ == "__main__":
    client = TicTacToeClient("127.0.0.1", 12345)
    client.run()


from dash import Dash, dcc, html, Input, Output
import plotly.express as px
import pandas as pd
import datetime
import numpy as np
import datetime
from dateutil.relativedelta import relativedelta
monthDiff = 2
today = datetime.date.today()
monthsAgo = today - relativedelta(months=monthDiff)
slopeID = 0
sensorID = 0

df = pd.read_csv('pivotFlexBarrier.csv')
a,b,c,d,e = df.iloc[0], df.iloc[1], df.iloc[2], df.iloc[3], df.iloc[4]
df2 = df
df2.iloc[0] = d
df2.iloc[1] = a
df2.iloc[2] = c
df2.iloc[3] = e
df2.iloc[4] = b
SensorIDs = df['deviceID'].tolist()
for i in range(0, len(SensorIDs)):
    SensorIDs[i] = int(SensorIDs[i])
SensorIDs = [str(i) for i in SensorIDs]
df = df.drop('Unnamed: 0', 1)
df = df.drop('Unnamed: 0.1', 1)
df = df.drop('deviceID', 1)
z = df.to_numpy()

dates = list(df)
for i, date in enumerate(dates):
    #dates[i] = datetime.datetime.strptime(date, '%d/%m/%Y').date()
    dates[i] = datetime.datetime.strptime(date, '%Y-%m-%d').date()
dates = np.array(dates)

def reloadData():
    df = pd.read_csv('pivotFlexBarrier.csv')
    a, b, c, d, e = df.iloc[0], df.iloc[1], df.iloc[2], df.iloc[3], df.iloc[4]
    df2 = df
    df2.iloc[0] = d
    df2.iloc[1] = a
    df2.iloc[2] = c
    df2.iloc[3] = e
    df2.iloc[4] = b
    SensorIDs = df['deviceID'].tolist()
    for i in range(0, len(SensorIDs)):
        SensorIDs[i] = int(SensorIDs[i])
    SensorIDs = [str(i) for i in SensorIDs]
    df = df.drop('Unnamed: 0', 1)
    df = df.drop('Unnamed: 0.1', 1)
    df = df.drop('deviceID', 1)
    z = df.to_numpy()

    dates = list(df)
    for i, date in enumerate(dates):
        #dates[i] = datetime.datetime.strptime(date, '%d/%m/%Y').date()
        dates[i] = datetime.datetime.strptime(date, '%Y-%m-%d').date()
    dates = np.array(dates)
    return SensorIDs, dates, z

# base = datetime.datetime.today()
# dates = base - np.arange(90) * datetime.timedelta(days=1)
# for i in range(0, len(dates)):
#      dates[i] = dates[i].date()
#
# dfWork = pd.DataFrame([
#     dict(Task="Job A", Start=dates[len(dates)-2], Finish=dates[len(dates)-1],Description='JobA'),
#     dict(Task="Job B", Start=dates[len(dates)-10], Finish=dates[len(dates)-8],Description='JobB'),
#     dict(Task="Job C", Start=dates[0], Finish=dates[1],Description='JobC')
# ])
dfWork = pd.read_csv('Task_Info_Flex.csv')
for i in range(0, len(dfWork)):
    dfWork['Start'][i] = datetime.datetime.strptime(dfWork['Start'][i], '%d/%m/%Y').date()
for i in range(0, len(dfWork)):
    dfWork['Finish'][i] = datetime.datetime.strptime(dfWork['Finish'][i], '%d/%m/%Y').date()
dfWork['SlopeID'] = dfWork['SlopeID'].fillna(slopeID)
dfWork['SensorID'] = dfWork['SensorID'].fillna(sensorID)
dfWork = dfWork.loc[(dfWork['SlopeID'] == slopeID) | (dfWork['SlopeID'] == 'ALL'), :]
dfWork = dfWork.loc[(dfWork['SensorID'] == sensorID) | (dfWork['SensorID'] == 'ALL'), :]
dfWork = dfWork[~(dfWork['Start'] < monthsAgo)].reset_index(drop=True)
if len(dfWork) == 0:
    today = datetime.date.today()
    yesterday = today - relativedelta(days=1)
    task = 'Place Holder'
    description = 'None'
    slopeID = 'None'
    sensorID = 'None'
    data = [{'Task': task,
             'Start': yesterday,
             'Finish': today,
             'Description': description,
             'SlopeID': slopeID,
             'SensorID': sensorID
             }]
    dfWork = pd.DataFrame(data)
def reloadWork(slopeID, sensorID):
    dfWork = pd.read_csv('Task_Info_Flex.csv')
    today = datetime.date.today()
    monthsAgo = today - relativedelta(months=monthDiff)
    for i in range(0, len(dfWork)):
        dfWork['Start'][i] = datetime.datetime.strptime(dfWork['Start'][i], '%d/%m/%Y').date()
    for i in range(0, len(dfWork)):
        dfWork['Finish'][i] = datetime.datetime.strptime(dfWork['Finish'][i], '%d/%m/%Y').date()
    toDrop = list()
    for i in range(0, len(dfWork)):
        if dfWork.iloc[i]['Start'] < monthsAgo:
            toDrop.append(i)
    dfWork = dfWork.drop(toDrop, axis=0).reset_index(drop=True)
    dfWork['SlopeID'] = dfWork['SlopeID'].fillna(slopeID)
    dfWork['SensorID'] = dfWork['SensorID'].fillna(sensorID)
    dfWork = dfWork.loc[(dfWork['SlopeID'] == slopeID) | (dfWork['SlopeID'] == 'ALL'), :]
    dfWork = dfWork.loc[(dfWork['SensorID'] == sensorID) | (dfWork['SensorID'] == 'ALL'), :]
    if len(dfWork) == 0:
        today = datetime.date.today()
        yesterday = today - relativedelta(days=1)
        task = 'Place Holder'
        description = 'None'
        slopeID = 'None'
        sensorID = 'None'
        data = [{'Task': task,
                'Start': yesterday,
                'Finish': today,
                'Description': description,
                'SlopeID': slopeID,
                'SensorID': sensorID
                }]
        dfWork = pd.DataFrame(data)
    return dfWork
# Color = ['rgb(0, 0, 0)','rgb(0, 0, 0)','rgb(0, 0, 0)']
# Jobs = ['JobA','JobB','JobC']
app = Dash(__name__)

app.layout = html.Div([
        html.H4('TCK928 - Monitoring Slope No.: 11SE-D/ND9', style={'font-size': '40px', 'textAlign': 'center'}),
        dcc.Graph(id="graph", responsive=True),
        dcc.Interval(id="trigger", interval=1000),
        html.H4('Location of the Slope', style={'font-size': '40px', 'textAlign': 'center'}),
        html.Div(html.Img(src=app.get_asset_url('slope.jpg')),style={'width': '70%',  'textAlign': 'center', 'display': 'inline-block'}),
        html.Div([
            html.P("Sensor A SN: 890771058921"),
            html.P("Sensor B SN: 890771058913"),
            html.P("Sensor C SN: 890771058918"),
            html.P("Sensor D SN: 890771059014"),
            html.P("Sensor E SN: 890771058915"),
        ],style={'width': '30%', 'display': 'inline-block'})
])

# html.Img(src=app.get_asset_url('slope.jpg'))
@app.callback(Output("graph", "figure"), Input('trigger', 'interval'))
def filter_heatmap(cols):
    import plotly.graph_objects as go
    import numpy as np
    import plotly.figure_factory as ff
    from plotly.subplots import make_subplots
    import pandas as pd
    # df = pd.read_csv('pivotFlexBarrier.csv')
    # SensorIDs = df['deviceID'].tolist()
    # df = df.drop('Unnamed: 0', 1)
    # df = df.drop('deviceID', 1)
    # z = df.to_numpy()
    slopeID = 0
    sensorID = 0
    SensorIDs, dates, z = reloadData()
    dfWork = reloadWork(slopeID, sensorID)
    labels = SensorIDs
    hovertext = list()
    for yi, yy in enumerate(labels):
        hovertext.append(list())
        for xi, xx in enumerate(dates):
            hovertext[-1].append('Date: {}
Sensor ID: {}
Maximum Reading: {}'.format(xx, yy, z[yi][xi]))
    hovertextWork = list()
    for i in range(0, len(dfWork)):
        hovertextWork.append(list())
        hovertextWork[-1].append('Date: {}
Event: {}
'.format(dfWork['Start'][i], dfWork['Description'][i]))
    # for i in range(0, len(z)):
    #     if i == 0 or i == 13 or i == 29:
    #         labels.append('Post -' + str(i))
    #     else:
    #         labels.append('Barrier -' + str(i))
    fig = make_subplots(rows=3, cols=1, shared_xaxes=True,specs=[
        [{'rowspan':2}], [None],
        [{}],
    ])
    fig.add_trace(go.Heatmap(
        colorbar={"title": 'Unit of Measure: m/s
(approximated)
(assuming stationary)'},
        z=z,
        x=dates,
        y=labels,
        colorscale='Viridis', colorbar_y=0.7,colorbar_len=0.6, hoverinfo='text', text=hovertext))
    figg = ff.create_gantt(dfWork, group_tasks=True, index_col='Description', reverse_colors=False, show_colorbar=False)
    #figg.update_traces(hoverinfo='text', text=hovertextWork)
    for trace in figg.data:
        fig.add_trace(trace, row=3, col=1)
    fig.update_layout(title={"text": "Motion Over Time and Works scheduled", "xanchor": "center", "x": 0.53},
                           yaxis={"title": "Barrier Posts", "autorange": "reversed"},
                           width=1800,
                           height=1000,
                           xaxis={"title": 'Time', "tickangle": 45})
    fig.update_layout(xaxis_showgrid=False, yaxis_showgrid=False, autosize=False)
    fig.update_yaxes(showticklabels=False, row=3, col=1, tickson='boundaries')
    fig.update_xaxes(tickson='boundaries')
    fig['layout'].update(margin=dict(l=0, r=0, b=0, t=0))
    fig['layout']['yaxis2']['title'] = 'Weather & Work Records'
    fig.update_xaxes(dtick=86400000.0)
    fig.update_layout(font={"size": 13})
    return fig

PORT = 8055
ADDRESS = "0.0.0.0"
if __name__ == '__main__':
    app.run_server(host=ADDRESS,port=PORT, debug=True, threaded=True)

# http://127.0.0.1:8055/


import requests
# If you are using a Jupyter Notebook, uncomment the following line.
# %matplotlib inline
import matplotlib.pyplot as plt
import json
from PIL import Image
from io import BytesIO

# Add your Computer Vision subscription key and endpoint to your environment variables.
# if 'COMPUTER_VISION_SUBSCRIPTION_KEY' in os.environ:
#     subscription_key = os.environ['COMPUTER_VISION_SUBSCRIPTION_KEY']
# else:
#     print("\nSet the COMPUTER_VISION_SUBSCRIPTION_KEY environment variable.\n**Restart your shell or IDE for changes to take effect.**")
#     sys.exit()

# if 'COMPUTER_VISION_ENDPOINT' in os.environ:
#     endpoint = os.environ['COMPUTER_VISION_ENDPOINT']

subscription_key = '' #ah yeah i deleted the key um go put your own or just text me for it
endpoint = 'https://bigredhackslogodetection.cognitiveservices.azure.com/'

analyze_url = endpoint + "vision/v3.1/analyze"

# Set image_url to the URL of an image that you want to analyze.
image_url = input("Gimme jpg url: ")

headers = {'Ocp-Apim-Subscription-Key': subscription_key}
params = {'visualFeatures': 'Brands'}
data = {'url': image_url}
response = requests.post(analyze_url, headers=headers,
                         params=params, json=data)

# headers = {'Ocp-Apim-Subscription-Key': subscription_key, 'Content-Type':'application/octet-stream'}
# with open('C:\\Users\\spike\\Documents\\Hackathons\\BigRedHacks\\gray-shirt-logo.jpg', 'rb') as f:
#     data = f.read()
# response = requests.post(analyze_url, headers=headers,
#                          params=params, data=data)

response.raise_for_status()

# The 'analysis' object contains various fields that describe the image. The most
# relevant caption for the image is obtained from the 'description' property.
analysis = response.json()
print(json.dumps(response.json()))
# image_caption = analysis["description"]["captions"][0]["text"].capitalize()

# Display the image and overlay it with the caption.
image = Image.open(BytesIO(requests.get(image_url).content))
plt.imshow(image)
plt.axis("off")
# _ = plt.title(size="x-large", y=-0.1)
plt.show()


from django.db.models import Count

from interfaces.models import Interfaces
from testsuits.models import Testsuits

def get_count_by_project(datas):
    """
    1. 通过项目中的接口、用例、配置、套件的数量
    2. 对时间进行格式化
    :param datas:
    :return:
    """
    datas_list = []
    for item in datas:
        # create_time格式化
        create_time_list = item.get('create_time').split('T')
        first_part = create_time_list[0]
        second_part = create_time_list[1].split('.')[0]
        item['create_time'] = first_part + ' ' + second_part

        # 获取项目id值
        project_id = item['id']
        interfaces_testcases_objs = Interfaces.objects.values('id').annotate(testcases=Count('testcases')).\
            filter(project_id=project_id)
        # 获取接口总数
        interfaces_count = interfaces_testcases_objs.count()
        # 设置用例总数初始值为0
        testcases_count = 0
        for one_dict in interfaces_testcases_objs:
            testcases_count += one_dict['testcases']

        interfaces_configures_objs = Interfaces.objects.values('id').annotate(configures=Count('configures')). \
            filter(project_id=project_id)

        # 设置配置总数初始值为0
        configures_count = 0
        for one_dict in interfaces_configures_objs:
            configures_count += one_dict['configures']

        # 获取套件总数
        testsuits_count = Testsuits.objects.filter(project_id=project_id).count()

        item['interfaces'] = interfaces_count
        item['testsuits'] = testsuits_count
        item['testcases'] = testcases_count
        item['configures'] = configures_count

        datas_list.append(item)
    return datas_list


"""empty message

Revision ID: d2752f2c5479
Revises:
Create Date: 2022-11-04 18:35:51.337359

"""
from alembic import op
import sqlalchemy as sa
import ormar

# revision identifiers, used by Alembic.
revision = 'd2752f2c5479'
down_revision = None
branch_labels = None
depends_on = None

def upgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('black_list_signs',
    sa.Column('uuid', ormar.fields.sqlalchemy_uuid.CHAR(32), nullable=True),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
    sa.Column('name', sa.String(length=512), nullable=False),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('id')
    )
    op.create_index(op.f('ix_black_list_signs_name'), 'black_list_signs', ['name'], unique=True)
    op.create_table('black_lists',
    sa.Column('uuid', ormar.fields.sqlalchemy_uuid.CHAR(32), nullable=True),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
    sa.Column('dt', sa.Date(), nullable=False),
    sa.Column('name', sa.String(length=512), nullable=False),
    sa.Column('inn', sa.String(length=512), nullable=False),
    sa.Column('address', sa.String(length=512), nullable=False),
    sa.Column('sites', sa.JSON(none_as_null=True), nullable=True),
    sa.Column('sign', sa.Integer(), nullable=True),
    sa.Column('closed', sa.Boolean(), nullable=True),
    sa.ForeignKeyConstraint(['sign'], ['black_list_signs.id'], name='fk_black_lists_black_list_signs_id_sign'),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('id')
    )
    op.create_index(op.f('ix_black_lists_inn'), 'black_lists', ['inn'], unique=False)
    op.create_index(op.f('ix_black_lists_name'), 'black_lists', ['name'], unique=True)
    # ### end Alembic commands ###

def downgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_black_lists_name'), table_name='black_lists')
    op.drop_index(op.f('ix_black_lists_inn'), table_name='black_lists')
    op.drop_table('black_lists')
    op.drop_index(op.f('ix_black_list_signs_name'), table_name='black_list_signs')
    op.drop_table('black_list_signs')
    # ### end Alembic commands ###


import openai
from fastapi import FastAPI, Path
import os
from fastapi.middleware.cors import CORSMiddleware

app = FastAPI()

origins = ["*"]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

def open_file(filepath):
    with open(filepath, 'r', encoding='utf-8') as infile:
        return infile.read()

openai.api_key = os.environ.get('OPENAIKEY')
FINE_TUNED_MODEL = 'davinci:ft-personal-2022-12-15-22-49-18'
TEMP = 0.7

def gpt3_completion(prompt, engine='text-davinci-003', temp=TEMP, top_p=1.0, tokens=400, freq_pen=0.0, pres_pen=0.0, stop=['END']):
    prompt = prompt.encode(encoding='ASCII',errors='ignore').decode()
    response = openai.Completion.create(
        model=FINE_TUNED_MODEL,
        # engine=engine,
        prompt=prompt,
        temperature=temp,
        max_tokens=tokens,
        top_p=top_p,
        frequency_penalty=freq_pen,
        presence_penalty=pres_pen,
        stop=stop)
    text = response['choices'][0]['text'].strip()
    return text

@app.get('/')
def home():
    return {"Data" : "Testing"}

@app.get("/about")
def about():
    return {"Data" : "About"}

@app.get("/get-item/{item_id}")
def get_item(item_id: int = Path(None, description="The Id of the item you'd  like to view", gt=0, lt=2)):
    return {"data" : item_id}

@app.get("/get-tweet")
def get_item(topic: str):
    # user_input = input('Write A Tweet about: ')
    prompt = open_file('prompt_chat.txt').replace('<<BLOCK>>', topic)
    response = gpt3_completion(prompt)
    return {"data" : response}


# encoding:utf-8
from django.core.management.base import BaseCommand, CommandError
from django.template import Template, Context
import smtplib
from malla.models import Requerimiento, Contratista, ContratistaInfoContacto, Lista
import csv
from datetime import datetime

class Command(BaseCommand):
    def handle(self, fi, ff, *args, **options):

        fecha_inicial = fi
        fecha_final = ff

        fecha_inicial = datetime.strptime(fecha_inicial, "%Y-%m-%d")
        fecha_final = datetime.strptime(fecha_final, "%Y-%m-%d")

        archivo = open("ReporteListaContratistas.csv","w")
        writer = csv.writer(archivo, delimiter=';')

        requerimientos = Requerimiento.objects.filter(fecha_eje__range=[fecha_inicial,fecha_final])

        string = ""
        arreglo = []

        arreglo.append("Nombres")
        arreglo.append("Apellidos")
        arreglo.append("No. Documento")
        arreglo.append("Celular")
        arreglo.append("Direccion".encode('latin-1'))

        for i in requerimientos:

            arreglo.append(i.descripcion_req.encode('latin-1'))

        arreglo.append("Total Horas")
        # columnas
        writer.writerow(arreglo)

        contratistas = Contratista.objects.all()
        for contratista in contratistas:

            nombres = str(contratista.nombre1) +" "+ str(contratista.nombre2)
            apellidos = str(contratista.apellido1) +" "+ str(contratista.apellido2)
            numero_doc = contratista.numero_documento

            celular = ContratistaInfoContacto.objects.get(monitor = contratista).celularppal
            direccion = contratista.direccion

            arreglo = []
            arreglo.append(nombres.encode('latin-1'))
            arreglo.append(apellidos.encode('latin-1'))
            arreglo.append(numero_doc)
            arreglo.append(celular)
            arreglo.append(direccion.encode('latin-1'))

            global requerimientos
            x = len(requerimientos)

            for i in requerimientos:
                try:
                    lista = Lista.objects.get(contratista=contratista,  requerimiento= i).horas
                    arreglo.append(lista)
                except:
                    lista = 0
                    arreglo.append(lista)
                    continue
            total = 0
            for i in range(5, len(arreglo)):
                total = int(arreglo[i])+total

            arreglo.append(total)

            writer.writerow(arreglo)


# -*- coding: utf-8 -*-
from wox import Wox, WoxAPI
import socket
import pyperclip

# 主逻辑方法
def main_function(query):
    nums = query.split("%")
    answer = int(nums[0]) % int(nums[1])
    return answer

class Main(Wox):
    def query(self, query):
        answer = main_function(query)

        results = list()
        results.append({
            "Title": answer,
            "SubTitle": "click can copy",
            "IcoPath": "Images/ico.ico",
            "JsonRPCAction": {
                "method": "copy",
                "parameters": [answer],
                "dontHideAfterAction": False
            }
        })
        return results

    # 允许copy
    def copy(self, content):
        pyperclip.copy(content)

if __name__ == "__main__":
    Main()


"""
General helper functions that don't fit neatly under any given category.

They provide some useful string and conversion methods that might
be of use when designing your own game.

"""

import os
import shutil
from datetime import datetime
from muddery.launcher import utils as launcher_utils

def compare_version(ver1, ver2):
    """
    Compare two version.

    Args:
        ver1: (tuple) version number's list 1.
        ver2: (tuple) version number's list 2.

    Returns:
        -1: ver1 < ver2
        0: ver1 == ver2
        1: ver1 > ver2
    """
    if not ver1 or not ver2:
        return 0
    if ver1[0] < ver2[0]:
        return -1
    if ver1[0] > ver2[0]:
        return 1
    return compare_version(ver1[1:], ver2[1:])

def make_backup(game_dir):
    """
    Backup game dir.
    """
    # back up to a new dir
    base_backup_dir = game_dir + "_bak_" + datetime.now().date().isoformat()
    backup_dir = base_backup_dir
    count = 1
    while os.path.exists(backup_dir):
        backup_dir = "%s(%s)" % (base_backup_dir, count)
        count += 1

    # copy game dir to backup dir
    try:
        copy_tree(game_dir, backup_dir)
    except Exception as e:
        print("Can not create backup dir: %s" % e)
        raise(Exception)

def get_temp_dir_name(game_dir):
    """
    Move game dir to temp dir.
    """
    base_temp_dir = game_dir + "_temp_" + datetime.now().date().isoformat()
    temp_dir = base_temp_dir
    count = 1
    while os.path.exists(temp_dir):
        temp_dir = "%s(%s)" % (base_temp_dir, count)
        count += 1

    return temp_dir

def copy_path(src_dir, dest_dir, path):
    """
    Copy a file from the source dir to the destination dir.

    Args:
        src_dir:
        dest_dir:
        path:

    Returns:
        None
    """
    src_path = os.path.join(src_dir, path)
    dest_path = os.path.join(dest_dir, path)

    if os.path.isdir(src_path):
        copy_tree(src_path, dest_path)
    else:
        shutil.copy2(src_path, dest_path)

def copy_tree(source, destination):
    """
    copy file tree
    """
    if not os.path.exists(destination):
        # If does not exist, create one.
        os.mkdir(destination)

    # traverse files and folders
    names = os.listdir(source)
    for name in names:
        srcname = os.path.join(source, name)
        dstname = os.path.join(destination, name)
        try:
            if os.path.isdir(srcname):
                # If it is a folder, copy it recursively.
                copy_tree(srcname, dstname)
            else:
                # Copy file.
                shutil.copy2(srcname, dstname)
        except Exception as e:
            print("Can not copy file:%s to %s for %s." % (srcname, dstname, e))
            raise(Exception)

def get_settings(game_dir, setting_list):
    """
    Get values in setting dict from settings file.
    """
    setting_dict = {}
    settings_path = os.path.join(game_dir, "server", "conf", "settings.py")
    with open(settings_path, 'r') as f:
        lines = f.readlines()
        for line in lines:
            contents = line.split("=")
            if len(contents) == 2:
                key = contents[0].strip()
                if key in setting_list:
                    setting_dict[key] = contents[1].strip()
    return setting_dict

def comment_out_class(file_path, class_name):
    """
    Comment out specified class in the file.

    Args:
        file_path: (string) file's path
        class_name: (string) the name of the class to comment out

    Returns:
        None
    """
    lines = []
    with open(file_path, "r") as f:
        lines = f.readlines()

    first_line = True
    comment_out = False
    for i, line in enumerate(lines):
        if line[:5] == "class" and class_name in line:
            comment_out = True

        if comment_out:
            if len(line.strip()) > 0:
                lines[i] = "# " + line

            if not first_line and line[0] != " ":
                break

            first_line = False

    with open(file_path, "w+") as f:
        f.writelines(lines)

def comment_out_lines(file_path, comment_lines):
    """
    Comment out specified class in the file.

    Args:
        file_path: (string) file's path
        comment_lines: (set) lines to append

    Returns:
        None
    """
    lines = []
    with open(file_path, "r") as f:
        lines = f.readlines()

    for i, line in enumerate(lines):
        if line.strip() in comment_lines:
            lines[i] = "# " + line

    with open(file_path, "w+") as f:
        f.writelines(lines)

def file_append(file_path, lines):
    """
    Append lines to the file.

    Args:
        file_path: (string) file's path
        lines: (list) lines to append

    Returns:
        None
    """
    with open(file_path, "a") as f:
        f.writelines(lines)
        f.flush()


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches

def plot_bbox(
    image: np.ndarray,
    scores: np.ndarray,
    classname: np.ndarray,
    coordinates: np.ndarray,
    min_score_threshold: float,
    normalised_coordiantes: bool,
):
    width, height, channels = image.shape

    if normalised_coordiantes:
        coordinates[:, 0] = (coordinates[:, 0] * width).astype(int)
        coordinates[:, 1] = (coordinates[:, 1] * height).astype(int)
        coordinates[:, 2] = (coordinates[:, 2] * width).astype(int)
        coordinates[:, 3] = (coordinates[:, 3] * height).astype(int)

    assert (
        len(scores) == len(classname) == len(coordinates)
    ), "Length of scores, classes and coordinates are not equal"

    fig, ax = plt.subplots()
    ax.imshow(image)

    for idx, i in enumerate(coordinates):
        if scores[idx] > min_score_threshold:
            rect = patches.Rectangle(
                (i[1], i[0]), i[3], i[2], edgecolor="c", linewidth=1, fill=False
            )
            plt.text(
                i[1],
                i[0],
                classname[idx],
                bbox=dict(fill=True, facecolor="c", edgecolor="c", linewidth=1),
            )
            ax.add_patch(rect)
    plt.show()