import matplotlib.pyplot as plt
import numpy as np
from drawnow import drawnow
from matplotlib import pyplot
from matplotlib.animation import FuncAnimation

import time
import math

figure = pyplot.figure(figsize=(12,15))
ax1 = figure.add_subplot(1,1,1)
ideas={}
axes={}

def initIdeas():
    f=open("../ProjectFiles/graphs/ideas.txt",'r+')
    lines=f.readlines()
    nbrColors=len(lines)
    incrementValue=math.floor((2**24)/nbrColors)
    color=0

    for line in lines:
        #Init Data
        line=line.replace("\n","")
        details=line.split(":")
        ideas[details[0]]=(int(details[1]),'%06X' % color)
        axes[details[0]]=np.array([int(details[1])])

       # axes[details[0]]=(np.array([0],[int(details[1])]))

        color+=incrementValue

    print(ideas)

def getIdeaColor(idea):
    ideaInformations=ideas.get(idea)
    return ideaInformations[1]

def updateFigure():

    for idea,XYaxe in axes.items():
        print(XYaxe)
        plt.plot(XYaxe,'o-',color="#"+getIdeaColor(idea),label=idea,linewidth=2,figure=figure)
        plt.axis(xmin=0)

def decNbrIdea(idea):
    details=ideas[idea]

    ideas[idea]=(details[0]-1,details[1])

    #details[0]+=1
    #ideas[idea]=details

def incNbrIdea(idea):
    details=ideas[idea]

    ideas[idea]=(details[0]+1,details[1])

def addNewPointPerIdea(cpt):

    for idea,XYaxe in axes.items():

        XYaxe=np.append(XYaxe,[ideas[idea][0]])
        print (XYaxe)
        axes[idea]=XYaxe
       #np.append(XYaxe,[ideas[idea][0]],axis=1)

      #  Xaxe=np.array(XYaxe[0])
      #  print(Xaxe)
       # Xaxe.append(np.array(cpt))
        #Yaxe=XYaxe[1]
        #Yaxe.append(ideas[idea][0])
        #axes[idea]=(Xaxe,Yaxe)

def main():
    cpt=0

   # addNewPointPerIdea(cpt)
    drawnow(updateFigure)

    with open('../ProjectFiles/logs/SimulationLog.txt',"r+") as f:
        for line in f:
            line.replace("\n","")
            element=line.split(";")

            decNbrIdea(element[2])
            incNbrIdea(element[3].replace("\n",""))

            cpt+=1
            print(cpt)
            if cpt% 100==0:
                addNewPointPerIdea(cpt)
                drawnow(updateFigure)

initIdeas()
main()


from olc_webportalv2.geneseekr import views
from django.conf.urls import url, include

urlpatterns = [
    # GeneSeekr Stuff
    url(r'^geneseekr_home/', views.geneseekr_home, name='geneseekr_home'),
    url(r'^geneseekr_query/', views.geneseekr_query, name='geneseekr_query'),
    url(r'^geneseekr_processing/(?P<geneseekr_request_pk>\d+)/$', views.geneseekr_processing, name='geneseekr_processing'),
    url(r'^geneseekr_results/(?P<geneseekr_request_pk>\d+)/$', views.geneseekr_results, name='geneseekr_results'),

    # Tree stuff
    url(r'^tree_home/', views.tree_home, name='tree_home'),
    url(r'^tree_request/', views.tree_request, name='tree_request'),
    url(r'^tree_result/(?P<parsnp_request_pk>\d+)/$', views.tree_result, name='tree_result'),
]


import random
import json

from locust import HttpLocust, TaskSet, task

NAME = "mapresty"
TOTAL = 3000000
LIMIT = 2000

DATABASE = "sdb"
LAYER_ID = "country_point"

jsonEncoder = json.JSONEncoder()
jsonDecoder = json.JSONDecoder()

def check_response(response):
    if response.content is None:
        response.failure("empty content")
    elif response.content.rfind('"success":false') > 0:
        r = jsonDecoder.decode(response.content)
        response.failure(r.get("error") or "success: false")
    else:
        response.success()

class Query(TaskSet):

    @task
    def query_by_condition(self):
        samples = random.sample(xrange(1, TOTAL), LIMIT)
        where = "gid in (" + ",".join(str(v) for v in samples) + ")"
        data = {
            "page": 0,
            "count": LIMIT,
            "condition": where
        }
        with self.client.post(
                "/rest/sdb/databases/%(db)s/layers/%(layerid)s/data?op=query" %
                {"db": DATABASE, "layerid": LAYER_ID},
                data,
                catch_response=True,
                name="(%s) query by condition" % NAME) as response:
            check_response(response)

    @task
    def query_by_spatial(self):
        spatial_filter = {
            "relation": 0,
            "geometry": {
                "type": "Polygon",
                "coordinates": [[
                    [110.588, 32.575],
                    [112.492, 32.118],
                    [113.862, 31.166],
                    [113.234, 29.929],
                    [111.445, 30.290],
                    [110.170, 28.235]
                ]]
            }
        }
        data = {
            "page": 0,
            "count": LIMIT,
            "resultCRS": "WGS84",
            "spatialFilter": jsonEncoder.encode(spatial_filter)
        }
        with self.client.post(
                "/rest/sdb/databases/%(db)s/layers/%(layerid)s/data?op=query" %
                {"db": DATABASE, "layerid": LAYER_ID},
                data,
                catch_response=True,
                name="(%s) query by spatial" % NAME) as response:
            check_response(response)

class QueryUser(HttpLocust):
    task_set = Query
    host = "http://maptalks-server:11215"


import os,csv,sys, operator
from django.conf import settings
from django import template

class CSVFile:

	def __init__(self, file_name, data, header=False):
		self.filename = file_name
		self.data = data
		self.type = [[0]*3]*len(data[0])
		self.validation_check()

	def validation_check(self):
		for j in range(len(self.data[0])):
			column = self.data[:][j]
			for i in range(len(column)):
				data_type = classify(column[i])
				self.type[j][data_type] += 1

def csvparse():
	files = []
	for filename in os.listdir(settings.BASE_DIR+'/static/datastash'):
		csvfile = open(settings.BASE_DIR+'/static/datastash/'+filename, 'rU')
		reader = csv.reader(csvfile)
		data = []
		for row in reader:
			data.append(row)
		file_obj = CSVFile(filename, data)
		files.append(file_obj)
	return files

register = template.Library()

def classify(s):
	try:
		int(s)
		return 0
	except:
		try:
			float(s)
			return 1
		except:
			return 2

@register.filter
def get_type(value):
    return type(value)


from mylib import utils
from mylib.crypto import MyCrypto

import argparse
import numpy as np
from PIL import Image

# Arguments
parser = argparse.ArgumentParser(description="Image Encryption")
parser.add_argument("--file", "-f", type=str, default="encrypted_image.jpg", help="Path to encrypted image file.")
parser.add_argument("--key", "-k", type=str, default="DefaultKey", help="Encryption Key.")
parser.add_argument("--save", "-s", type=str, default="decrypted_image.jpg", help="File name for output image.")
args = parser.parse_args()

CROP_WIDTH = 16
CROP_HEIGHT = 8

if __name__ == '__main__':
    mycrypto = MyCrypto(args.key)

    # Load image
    encrypted_image = Image.open(args.file)
    encrypted_image = encrypted_image.convert('1')

    img_width, img_height = encrypted_image.size

    decrypted_crop_imgs = []
    for crop_img in utils.split_image(encrypted_image, CROP_HEIGHT, CROP_WIDTH):
        # Transform cropped image to bits format
        img_np = np.array(crop_img).flatten()
        img_np = img_np.astype('uint8')
        img_bit = ''.join([str(bit) for bit in img_np])

        # Transform bits to Bytes format
        img_bytes = []
        for i in range(0, len(img_bit), 8):
            img_bytes.append(int(img_bit[i:i+8], 2))
        img_bytes = bytes(img_bytes)

        # Decrypt Crop image
        decrypted_img_bytes = mycrypto.decrypt(img_bytes)

        # Transform Bytes to bits format
        decrypted_img_bits = ''
        for data_byte in decrypted_img_bytes:
            data_bits = bin(data_byte)[2:].zfill(8)
            decrypted_img_bits += data_bits

        # Transform bits to Numpy format
        decrypted_img_bits_np = np.array([int(bit) for bit in decrypted_img_bits])
        decrypted_img_bits_np = decrypted_img_bits_np.reshape((CROP_HEIGHT, CROP_WIDTH))
        decrypted_crop_imgs.append(decrypted_img_bits_np.tolist())

    # Combine crop images
    decrypted_crop_imgs = np.array(decrypted_crop_imgs)
    decrypted_img = utils.combine_image(decrypted_crop_imgs, img_width, img_height)

    # Numpy to PIL
    decrypted_image = Image.fromarray(decrypted_img*255)
    decrypted_image.show()
    decrypted_image.convert('RGB').save(args.save)


import json

from django.http import HttpResponse, JsonResponse
from ResModel.models import HubUser

def changeHead(request):
    if request.method == "POST":
        userEmail = request.POST.get("userId")
        userHead =  request.POST.get("url")
        if userEmail is not None and userHead is not None:
            user = HubUser.objects.filter(UserEmail=userEmail).update(UserImage=userHead)
            return JsonResponse({
                "status": 1,
                "message": "更换头像成功",
            }, safe=False)
        else:
            return JsonResponse({
                "status": 2,
                "message": "请求参数错误"
            })
    else:
        return JsonResponse({
          "status": 3,
          "message": "请求方法错误"
        })


""""
===================================================
梯度下降作业(代码)：
目标函数：
y = x**2 + b * x + c
需求：求解最小值对应的x和y(这里的最小值指的是函数y在所有数据上的一个累计最小值)
要求：写代码
数据：
b: 服从均值为-1，方差为10的服从高斯分布随机数
c：服从均值为0，方差为1的服从高斯分布随机数
假定b、c这样的数据组合总共1、2、10、100、10w、100w条数据,求解在现在的数据情况下，目标函数的取最小值的时候，x和y分别对应多少？
===================================================
"""""

import numpy as np
import matplotlib as mlt
import matplotlib.pyplot as plt

# 解决画图中文问题
plt.rcParams['font.sans-serif'] = ['SimHei']    # 步骤一（替换sans-serif字体）
plt.rcParams['axes.unicode_minus'] = False      # 步骤二（解决坐标轴负数的负号显示问题）

np.random.seed(18)

n = 8
# 产生均值为loc，标准差为scale的随机数列组成的数组，这个数组的形式为size
# size = (x, y): x行, y列
# size = n: n条数据
b_values = np.random.normal(loc=-1.0, scale=10, size=n)
print(b_values)


from cltk.stem.sanskrit.code import configuration
from cltk.stem.sanskrit.code.apply import *
from cltk.stem.sanskrit.code.iast_dev import *
from cltk.stem.sanskrit.code.dev_iast import *
from git import Repo
import os

config = configuration.config
class SandhiSplitter():
    def __init__(self, text, isIAST=False):
        self.text = text
        self.isiast = isIAST
        if self.isiast == False:
            self.text = dviast(self.text)

    def getSandhi(self):
        if os.path.exists("../data/input/additional-data-0-128.json") == False:
            print("Trained model not found, Downloading....")
            Repo.clone_from("https://github.com/sainimohit23/SanskritModel", "../data")
        preds = split_sandhi(self.text, config)
        textSplitted = self.text.split()
        predsSplitted = preds.split()
        d = {}
        for i in range(len(predsSplitted)):
            li = []
            for word in predsSplitted[i].split('-'):
                if len(word)>0:
                    li.append(iastdv(word))
            d[iastdv(textSplitted[i])] = li

        return d


#!/usr/bin/env python
import sys
import socket
import threading
import subprocess
import argparse

def main():
    if not args.listen:
        buffer = input()
        buffer += '\n'
        client_sender(buffer)
    else:
        server_loop()

def client_sender(buf):
    client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    try:
        # connect to our target host
        client.connect((args.target, args.port))
        if len(buf):
            client.sendall(buf.encode())

        # now wait for the data back
        while True:
            recv_len = 1
            response = ""

            while recv_len:
                data = client.recv(4096).decode()
                recv_len = len(data)
                response += data
                if recv_len < 4096:
                    break

            # wait for more input
            buf = input()
            buf += '\n'

            # send if off
            client.sendall(buf.encode())

    except Exception as e:
        print(f"[*] Exception! Exiting. {e}")

        # tear down the connection
        client.close()

def server_loop():

    # if no target defined we listen on all interfaces
    if not args.target:
        args.target = '0.0.0.0'

    server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    server.bind((args.target, args.port))

    # maximum 5 concurrent connections
    server.listen(5)

    while True:
        client_socket, addr = server.accept()

        # spin off a thread to handle our new client
        client_thread = threading.Thread(
            target=client_handler, args=(client_socket,))
        client_thread.start()

def run_command(command):

    # trim the new line
    command = command.rstrip()

    # run the command and get the output back
    try:
        output = subprocess.check_output(
            command, stderr=subprocess.STDOUT, shell=True).decode()
    except:
        output = f"Failed to execute command {command}"

    # send the output back to the client
    return output

def client_handler(client_socket):

    print(f"Handling client {client_socket}")

    if args.upload:
        # read in all the bytes and write to our destination
        file_buffer = ""

        # keep reading data until none is available
        while True:
            data = client_socket.recv(1024).decode()

            if not data:
                break
            else:
                file_buffer += data

            # now we take these bytes and try to write them out
            try:
                fd = open(args.upload, "wb")
                fd.write(file_buffer)
                fd.close()

                # acknowledge that we wrote the file out
                client_socket.send(
                    f"Successfully saved file to {args.upload}\r\n".encode())
            except:
                client_socket.send(
                    f"Failed saved file to {args.upload}\r\n".encode())

    # check for command execution
    if args.execute:
        # run this command
        output = run_command(args.execute)
        client_socket.send(output.encode())

    if args.command:

        while True:
            # show a simple prompt

            # now we receive until we see a linefeed (enter key)
            cmd_buffer = ""
            while "\n" not in cmd_buffer:
                cmd_buffer += client_socket.recv(1024).decode()

            # send back the command output, response is bytes
            response = run_command(cmd_buffer).rstrip('\n')
            response += '\n> Command:'
            client_socket.sendall(response.encode())

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-t", "--target", help="target host", type=str)
    parser.add_argument("-p", "--port", help="target port",
                        required=True, type=int)
    parser.add_argument("-v", "--verbose", help="increase output verbosity",
                        action="count", default=0)
    parser.add_argument("-l",
                        "--listen", help="listen on [host]:[port] for incoming connections", action='store_true')
    parser.add_argument("-e",
                        "--execute", help="execute the given file upon receiving a connection", type=str)
    parser.add_argument('-c', '--command',
                        help='initialize a command shell', action='store_true')
    parser.add_argument(
        '-u', '--upload', help='upon receiving connection upload a file and write to [UPLOAD]', type=str)
    args = parser.parse_args()
    print(args)
    main()


import json

from django.core.paginator import Paginator
from django.shortcuts import render
from django.http import HttpResponse
from .procedures import UsersDB

# Create your views here.
def users_list(request):
    udb = UsersDB()
    if request.method == 'GET':
        order_by = 'name'

        if request.GET.get('limit', False):
            limit = int(request.GET.get('limit'))
            limit = limit if limit >= 10 and limit <= 20 else 10
        else:
            limit = 10

        if request.GET.get('page', False):
            page = int(request.GET.get('page'))
        else:
            page = 1

        if request.GET.get('search', False):
            users_filter = request.GET.get('search')
        else:
            users_filter = None

        if users_filter:
            users_list = udb.search_users([users_filter, order_by])
        else:
            users_list = udb.get_users_list([order_by])
        paginator = Paginator(users_list, limit)
        users_list = paginator.page(page)
        del udb
        return render(request, 'users_list.html',
                      {'users': users_list, 'limit': limit,
                       'users_filter': users_filter})

    if request.method == 'POST' and request.is_ajax():
        if request.POST.get('remove_user', False):
            user_id = request.POST.get('remove_user')
            udb.remove_user([user_id])
            del udb
            return HttpResponse(status=204)

def courses_list(request):
    udb = UsersDB()
    courses_list = udb.get_courses_list()
    del udb
    return render(request, 'courses_list.html', {'courses': courses_list})

def create_user(request):
    if request.method == 'POST' and request.is_ajax():
        if request.POST.get('name', False)\
           and request.POST.get('email', False):
            user_name = request.POST.get('name')
            user_email = request.POST.get('email')

            user_phone = request.POST.get('phone_no')
            user_phone = user_phone if user_phone else None

            user_mobile = request.POST.get('mobile_no')
            user_mobile = user_mobile if user_mobile else None

            user_status = request.POST.get('status')
            user_status = True if user_status == 'active' else False

            udb = UsersDB()
            udb.add_user([user_name, user_email, user_status,
                          user_phone, user_mobile])
            del udb
            return HttpResponse(status=204)
    return render(request, 'create_user.html')

def change_user(request, uid):
    udb = UsersDB()
    if request.method == 'POST' and request.is_ajax():
        if request.POST.get('name', False)\
           and request.POST.get('email', False):
            user_name = request.POST.get('name')
            user_email = request.POST.get('email')

            user_phone = request.POST.get('phone_no')
            user_phone = user_phone if user_phone else None

            user_mobile = request.POST.get('mobile_no')
            user_mobile = user_mobile if user_mobile else None

            user_status = request.POST.get('status')
            user_status = True if user_status == 'active' else False
            udb.change_user([uid, user_name, user_email, user_status,
                             user_phone, user_mobile])
            del udb
            return HttpResponse(status=204)

        if request.POST.get('1'):
            udb.remove_users_course([uid])
            for i in request.POST.items():
                if request.POST.get(i[0]) == 'true':
                    udb.add_users_course([uid, i[0]])

            return HttpResponse(status=204)
        return HttpResponse(status=204)

    if request.method == 'GET':
        user = udb.get_user([uid])
        courses = udb.get_courses_list()
        user_courses = udb.get_users_courses([uid])

        show_courses = []

        courses_status = []

        for course in courses:
            if course not in user_courses\
               and course not in show_courses:
                course_status = list(course)
                course_status.append(False)
                courses_status.append(course_status)
                show_courses.append(course)
            elif course not in show_courses:
                course_status = list(course)
                course_status.append(True)
                courses_status.append(course_status)
        del udb
        return render(request, 'change_user.html',
                      {'user': user, 'courses': show_courses,
                       'user_courses': user_courses,
                       'js_courses': json.dumps(courses_status)})


import pandas as pd
import numpy as np
import requests
from scipy.optimize import leastsq
from bs4 import BeautifulSoup

from EnergyIntensityIndicators.get_census_data import Asm
from EnergyIntensityIndicators.get_census_data import Econ_census

class Mfg_prices:
    # def __init__(self):
    #     """
    #     Class for importing and interpolating historical energy prices for
    #     the manufacturing sector.
    #
    #     Historical Manufacturing Energy Consumption Survey (MECS) data are
    #     based on prior work from the Pacific Northwest National Laboratory
    #     (PNNL).
    #     """
    #     # Historical MECS data with missing observations estimated
    #     # ad-hoc by PNNL.
    #     self.mecs_historical_prices = pd.read_csv('mecs_historical_prices.csv')

    @staticmethod
    def import_mecs_historical(file_path):
        """
        Import and format csv of historical fuel prices from Manufacturing
        Energy Consumption Survey (MECS). MECS was conducted every three
        years from 1985 - 1994 and every four years since.

        Parameters
        ----------
        file_path : str
            File path of csv.

        Returns
        -------
        mecs_prices : dataframe

        """

        mecs_prices = pd.read_csv(file_path)

        mecs_prices = pd.melt(mecs_prices, id_vars=['NAICS', 'fuel'],
                              var_name='year', value_name='mecs_price')

        return mecs_prices

    @staticmethod
    def check_recent_mecs(latest_year, last_historical_year):
        """
        MECS has been conducted at four-year intervals since 1994. New data
        will need to be manually downloaded, formatted, and added to the
        historical price file, 'mfg_mecs_energy_prices.csv'.

        Parameters
        ----------
        latest_year : int
            Latest year of historical decomposition.

        last_historical_year : int
            Lastest year of price data available in
            'mfg_mecs_energy_prices.csv'.

        Returns
        -------

        """

        if latest_year not in range(2014, 2040, 4):
            print('{} is not a MECS year.'.format(latest_year))

            return

        if latest_year <= last_historical_year:

            print("Historical MECS prices are latest available")

            return

        else:

            check_url = 'https://www.eia.gov/consumption/manufacturing/' + \
                        'data/{}/xls/table7_2.xlsx'.format(str(latest_year))

            r = requests.get(check_url)

            soup = BeautifulSoup(r.text, 'html.parser')

            # Check if updated MECS are available by requesting Excel file.
            # If no Excel file exists, requests will return the
            # Consumption and Efficiency page. If the Excel file does exist,
            # there will be an exception, which prompts the user to download
            # and format the data.
            try:
                title = soup.title.name

            except AttributeError:
                print("Updated mecs data are now available.\n" +
                      "Please download Tables 3.1, 3.2, 7.2, and 7.6\nand update" +
                      "relevant csv files")

            else:
                print("Updated MECS data are not yet available")

            finally:
                return

    # @staticmethod
    # def import_asm_historical(file_path):
    #     """"
    #     Prices in $/MMBtu
    #     """
    #
    #
    # @staticmethod
    # def get_census_prices(latest_year, start_year=1983):
    #     """
    #     Get fuel prices from Census Bureau's Annual Survey of
    #     Manufacturers and Economic Census (years ending in 2 and 7)
    #
    #     Parameters
    #     ----------
    #     latest_year : int
    #         Most recent year of historical LMDI analysis.
    #
    #     start_year : int, default 1983
    #         Beginning year of price data.
    #
    #     Returns
    #     -------
    #     asm_prices : pandas.Series
    #         Pandas series of ASM price data from YYYY - latest_year
    #     """
    #     year_range = range(start_year, latest_year + 1)
    #
    #     asm_prices = pd.DataFrame()
    #
    #
    #
    #     return asm_prices

    @staticmethod
    def price_func(asm_prices, *params):
        """
        Calculates predicted fuel price in terms of current and lagged fuel
        prices.
        """
        a, b = params

        for index_, price in enumerate(asm_prices):
            if index_ == 0:
                predicted_price_series = np.array([0])
            else:
                predicted_price = a*price + b*asm_prices[index_-1]
                predicted_price_series = np.vstack([predicted_price_series,
                                                    predicted_price])

        early_mecs = predicted_price_series.flatten()[2:12:3]
        later_mecs = predicted_price_series.flatten()[15:len(asm_prices):4]
        final_mecs = np.append(early_mecs, later_mecs)
        # Align with available MECS
        final_mecs = final_mecs[[x[0] for x in enumerate(mecs_prices)]]
        return final_mecs

    @staticmethod
    def residuals(params, asm_prices, mecs_prices, price_func):

        return mecs_prices - price_func(asm_prices, *params)

    @staticmethod
    def calc_predicted_coeffs(asm_prices, mecs_prices, start_params):
        """
        Parameters
        ----------
        asm_prices : numpy.array
            Array of ASM prices

        mecs_prices : numpy.array
            Array of MECS prices, including nan values

        start_params : list
            starting parameters for optimization

        Returns
        -------
        coeff : np.array
            parameters of price prediction equation

        Examples
        --------
        @calmc successully ran this using data from the Gas-311 tab (
        copying cells C6:E42 to clipboard) in Ind_Gas_Prices_123019.xlsx.
        >>>price_data = pd.read_clipboard()
        >>># Using read_clipboard introduces a column of nans
        >>>price_data.dropna(axis=1, how='all', inplace=True)
        >>>price_data.columns = ['year', 'asm_prices', 'mecs_prices']
        >>>price_data.head()
           year  asm_prices  mecs_prices
        0  1983        4.05          NaN
        1  1984        4.10          NaN
        2  1985        3.83         4.21
        3  1986        3.14          NaN
        4  1987        2.85          NaN

        >>>asm_prices = price_data.asm_prices.values
        >>>mecs_prices = price_data.mecs_prices.dropna().values
        >>>coeff = predict_prices(asm_prices, mecs_prices, [1, 1])
        >>>coeff
        array([0.91430597, 0.1074011 ])
        """

        coeff, flag = leastsq(Mfg_prices.residuals, start_params,
                              args=(asm_prices, mecs_prices,
                                    Mfg_prices.price_func))

        return coeff

    @staticmethod
    def calc_predicted_prices(coeff, asm_prices):
        """Calculate predicted price with leastsq coeffs"""
        return Mfg_prices.price_func(asm_prices, *coeff)

    @staticmethod
    def resid_filler(price_df):
        """
        Apply to a dataframe that includes year and residual.

        price_df.columns = ['year', 'asm_price', 'MECS_price', 'predicted',
        'residual']

        Parameters
        ----------
        residual : np.array
            Array of residuals, including nan values.
        """

        increment_years = price_df.dropna()['year'].values

        # Lines 165 - 173 should be placed in separate method.
        predicted_prices = Mfg_prices.calc_predicted_prices(coeff, asm_prices)
        predicted_prices = pd.DataFrame(predicted_prices,
                                        columns=['predicted'],
                                        index=increment_years)

        price_df.set_index('year', inplace=True)
        price_df = pd.concat([price_df, predicted_prices], axis=1)
        price_df['residual'] = price_df.MECS_price - price_df.predicted

        price_df['interp_resid'] = np.nan

        for index, y_ in enumerate(increment_years):
            if index > 0:
                year_before = increment_years[index - 1]
                num_years = y_ - year_before
                resid_year_before = price_df.xs(year_before)['residual']
                resid_y_ = price_df.xs(y_)['residual']
                increment = 1 / num_years
                for delta in range(num_years):
                    value = resid_year_before * (1 - increment * delta) + \
                        resid_y_ * (increment * delta)
                    year = year_before + delta
                    price_df.loc[year, 'interp_resid'] = value

        # Fill in remaining missing values
        price_df.loc[increment_years[-1], 'interp_resid'] = \
            price_df.xs(increment_years[-1])['residual']
        price_df.interp_resid.fillna(method='ffill', inplace=True)

        price_df['calibrated_prediction'] = \
            price_df.predicted + price_df.interp_resid

        price_df.reset_index(inplace=True)

        return price_df

    def calc_calibrated_predicted_price(self, latest_year, fuel_type, naics):
        """
        Return the calibrated prediced prices, which is calculated as the

        """
        # Get asm prices from separate method?

        mecs_prices = self.mecs_historical_prices(fuel_type, naics)

        fit_coeffs = Mfg_prices.calc_predicted_coeffs()
        predicted = Mfg_prices.calc_predicted_prices()

        price_df['calibrated_prediction'] = \
            price_df.interp_resid + price_df.predicted

        return price_df

    # @staticmethod
    # def interpolate_residuals(predicted_prices, price_df):
    #     """Interpolate residuals"""
    #
    #     price_df_updated = price_df.copy(deep=True)
    #     price_df_updated['predicted_price'] = predicted_prices
    #     price_df_updated['residual'] = price_df_updated.mecs.subtract(
    #         price_df_updated.predicted_price
    #         )
    #
    #     fill = price_df_updated.dropna(subset=['residual'], axis=0).year.diff()
    #     # fill.fillna(0, inplace=True)
    #
    #     price_df_updated['fill'] = fill
    #
    #
    #     return interpolated_resid
    #

    # Predicted prices are use where? Check documentation to figure out [Search NAICS]


import numpy as np
import pandas as pd
import re
from scipy import stats

from . import utils
from .chart import Dashboard
from .models import arima

class CoinSeer:
    """
    The core Coin Seer class that provides a simple interface for managing a
    predictive model of a cryptocurrency time series.

    Parameters
    ----------
    ts : pandas.Series
        Datetime-indexed cryptocurrency time series
    currency_pair : str
        The currency pair for the time series, e.g., 'btc_usd', 'eth-xrp';
        must contain at least one separator character
    freq : pandas.tseries.offsets or None, optional, default: None
        The frequency of the time series; if None, inferred automatically
    preprocess : bool, optional, default: True
        Whether to preprocess the time series
    logret : int, optional, default: 1
        The k-step log return; if 0, do not calculate log returns

    Attributes
    ----------
    ts : pandas.Series
        Datetime-indexed cryptocurrency time series
    self.bc, self.qc : str
        Base and quote (counter) currencies
    self.logret : int
        The k-step log return; if 0, not modelling the log returns
    self.model : coin-seer model
        A coin-seer model, e.g., arima.Rolling
    self.statistics : dict
        Descriptive statistics for the time series or its log returns
    """
    def __init__(self, ts, currency_pair, freq=None,
                 preprocess=True, logret=1):
        self.ts = ts
        currency_pair = currency_pair.upper()
        self.bc, self.qc = re.split(r'[ ,-./:;_|]+', currency_pair)
        self.logret = logret
        self.model = None

        if preprocess:
            if isinstance(self.ts.index.dtype, np.object):
                self.ts.index = pd.to_datetime(self.ts.index)
            if freq is None:
                if self.ts.index.freq is None:
                    freq = pd.infer_freq(self.ts.index)
                else:
                    freq = self.ts.index.freq
            if logret > 0:
                _ts = np.where(self.ts == 0, np.nan, self.ts)
                self.ts = pd.Series(_ts, index=self.ts.index)
            self.ts = utils.preprocess(self.ts, freq=freq)

        if logret > 0:
            self.ts = utils.log_returns(self.ts, logret)

        self.statistics = {
            'Length': len(self.ts),
            'Number of null data': self.ts.isnull().sum(),
            'Minimum': self.ts.min(),
            'Maximum': self.ts.max(),
            'Mean': self.ts.mean(),
            'Median': self.ts.median(),
            'Interquartile range': stats.iqr(self.ts.values),
            'Standard deviation': self.ts.std(),
            'Skewness': stats.skew(self.ts.values),
            'Kurtosis': stats.kurtosis(self.ts.values)
        }

    def describe(self):
        """Retrieve a collection of descriptive statistics."""
        return self.statistics

    def dashboard(self, save=False, save_to=''):
        """
        Generate a dashboard containing the following elements:
        - a plot of the base time series or its k-step log returns;
        - a box plot, showing distribution characteristics;
        - autocorrelation function (ACF), partial ACF, and quantile-quantile
          plots; and
        - a power spectral density plot computed using Welch's method for
          identifying seasonality.

        Parameters
        ----------
        save : bool, optional, default: False
            Whether to export the dashboard in scalable vector graphics format
        save_to : str, optional, default: ''
            The directory to which to export; used only if `save` is True
        """
        db = Dashboard(self)
        db.make(save, save_to)

    def select(self, model, **hyperparameters):
        """
        Specify the model.

        Parameters
        ----------
        model : str {'arima', 'rolling'}
        hyperparameters : dict, optional
        """
        hyperparameters.update({
            'bc': self.bc,
            'qc': self.qc,
            'logret': self.logret
        })
        models = {'arima': arima.ARIMA, 'rolling': arima.Rolling}
        self.model = models[model](self.ts, **hyperparameters)

    def run(self, **run_args):
        """
        Prepare forecasts according to the given model and hyperparameters.

        Parameters
        ----------
        run_args : dict, optional
            Arguments to be passed to the model's `run` method
        """
        self.model.run(**run_args)

    def log(self):
        """Retrieve the forecast log."""
        if len(self.model.log) == 0:
            print('No log available. '
                  'Refit the model(s) with logging enabled.')
            return None
        return self.model.log

    def evaluate(self):
        """Compute and retrieve evaluation metrics."""
        return self.model.evaluate()

    def forecasts(self):
        """Retrieve the forecast(s)."""
        return self.model.fc

    def display_results(self, save=False, save_to=''):
        """
        Display the forecast results graphically.

        Parameters
        ----------
        save : bool, optional, default: False
            Whether to export the dashboard in scalable vector graphics format
        save_to : str, optional, default: ''
            The directory to which to export; used only if `save` is True
        """
        self.model.display_results(save, save_to)


###################################
#  application for ecYeast
###################################
from cobra.io import load_matlab_model, read_sbml_model
import os
import sys
import matplotlib.pyplot as plt # For python 3.6: sudo apt-get install python3.6-tk   TO INSTALL Tkinter
import pandas as pd
from cobra.flux_analysis import pfba
from cobra.flux_analysis import (
    single_gene_deletion, single_reaction_deletion, double_gene_deletion,
    double_reaction_deletion)

from cobra.manipulation import remove_genes
from cobra.manipulation import *
from cobra.flux_analysis.moma import moma

sys.path.append(r"/Users/luho/PycharmProjects/model/cobrapy/code")
sys.path.append(r"/Users/luho/PycharmProjects/model/strain_design/code")
os.chdir('/Users/luho/PycharmProjects/model/strain_design/code')

# import self function
from mainFunction import *
from in_silico_strain_design_scripts import *

# load the ecYeast model
ecYeast = load_matlab_model("../data/ec3HP.mat")

"""knock-out"""
# pFBA method is used
# run the gene deletion with pFBA method, minimization of proteins pool
# in this simulation, 3HP production is as the objective function
# qs is unconstrait
def getModelWithRemoveGene (model0, gene_remove0):
    model1 = model0.copy()
    remove_genes(model1, gene_remove0, remove_reactions=True) # this script can't be used
    return model1

def solveEcModel(model, obj_id, target_id, glucose_uptake ='r_1714_REV'):
    model.objective = obj_id  # we put it in the mutant model
    solution = model.optimize()  # we put it in the mutant model
    # mutModel.summary() # we put it in the mutant model
    solution_value0 = solution.objective_value  # we put it in the mutant model
    model.reactions.get_by_id(obj_id).bounds = (0.999 * solution_value0, solution_value0)  # we put it in the mutant model
    model.objective = {model.reactions.prot_pool_exchange: -1}
    solution2 = model.optimize()
    yield_p = solution2.fluxes[target_id]/solution2.fluxes[glucose_uptake]
    return  yield_p

# pipeline

# prepare the geneID
geneID = []
for x in ecYeast.genes:
    print(x.id)
    geneID.append(x.id)

# prepare ecModel for calculation
model2 = ecYeast.copy()
model2 = ecYeastMinimalMedia(model2)
model2.reactions.get_by_id("r_1714_REV").bounds = (0,1000)  # D-glucose exchange (reversible)

# using pfba
yield_p_ref0 =[]
yield_p_mutant0 =[]
for i, x in enumerate(geneID):
    print(i)
    gene_remove = []
    gene_remove.append(x)
    mutModel = getModelWithRemoveGene(model0=model2, gene_remove0=gene_remove)

    try:
        yield_p_ref = solveEcModel(model=model2, obj_id = 'newRxn3', target_id = 'newRxn3')
        yield_p_mutant = solveEcModel(model=mutModel, obj_id='newRxn3', target_id='newRxn3')
        print(yield_p_ref)
        print(yield_p_mutant)
        yield_p_ref0.append(yield_p_ref)
        yield_p_mutant0.append(yield_p_mutant)
    except:
        yield_p_ref = None
        yield_p_mutant = None
        yield_p_ref0.append(yield_p_ref)
        yield_p_mutant0.append(yield_p_mutant)
pfba_result = pd.DataFrame({'gene':geneID, 'yield_p_ref0': yield_p_ref0, 'yield_p_mutant0': yield_p_mutant0 })
saveExcel(pfba_result, "../result/pfba_result.xlsx")


import datetime
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import RandomizedSearchCV
from joblib import dump, load
from . import utils
import os

def createFeatures(dfTripsPerDay):

    ###Feature creation
    #Get the date of yesterday and one week ago
    dfTripsPerDay['date_yesterday'] = pd.DatetimeIndex(dfTripsPerDay['date']) - pd.DateOffset(1)
    dfTripsPerDay['date_oneWeekAgo'] = pd.DatetimeIndex(dfTripsPerDay['date']) - pd.DateOffset(7)

    #Get the number of trips for this dates
    tripsLastDay = []
    tripsOneWeekAgo = []

    #Save unique dates
    lastDay = list(dfTripsPerDay['date_yesterday'].unique())
    oneWeekEarlier = list(dfTripsPerDay['date_oneWeekAgo'].unique())

    #Try/except is necessary, because of the outlier drop
    for date in lastDay:
        try:
            valueDay = dfTripsPerDay[dfTripsPerDay['date'] == date]['tripsPerDay'].values[0]
        except:
            valueDay = None

        tripsLastDay.append(valueDay)

    for date in oneWeekEarlier:
        try:
            valueWeek = dfTripsPerDay[dfTripsPerDay['date'] == date]['tripsPerDay'].values[0]
        except:
            valueWeek = None

        tripsOneWeekAgo.append(valueWeek)

    #Fill in created features
    dfTripsPerDay['tripsLastDay'] = tripsLastDay
    dfTripsPerDay['tripsOneWeekAgo'] = tripsOneWeekAgo

    #Get the mean for fill of nans
    avgTripsPerDay = dfTripsPerDay['tripsPerDay'].mean()

    #Fill nans with mean
    dfTripsPerDay['tripsLastDay'].fillna(avgTripsPerDay,inplace=True)
    dfTripsPerDay['tripsOneWeekAgo'].fillna(avgTripsPerDay,inplace=True)

    #Drop Dates and Features with a low correlation
    dfTripsPerDay['isTerm'] = dfTripsPerDay.apply(isTerm, axis=1)
    dfTripsPerDay.dropna(inplace=True)
    dfTripsPerDay.drop(['date','date_yesterday','date_oneWeekAgo'],axis=1,inplace=True)
    dfTripsPerDay.drop(['day','month'],axis=1,inplace=True)
    dfTripsPerDay.drop(['temperatureMIN','temperatureMAX'],axis=1,inplace=True)

    return dfTripsPerDay

# Trains a model to predict the number of trips on the next day
def retrainModel_NumberOfTrips(dfTripsPerDay, optimalHyperparameterTest):

    ###First filter for outliers
    #Get standart deviation of trips per day
    std = dfTripsPerDay['tripsPerDay'].std()

    #For each month
    for i in range(1,13):
        #Get the average number of trips per month
        df = dfTripsPerDay[dfTripsPerDay['month']==i]
        avg = df['tripsPerDay'].mean()

        #Only use days where the average is within the standartdeviation
        row = df[(df['tripsPerDay'] <= avg+std)&(df['tripsPerDay'] >= avg-std)]

        #Combine the data for the months again
        if i == 1:
            dfTripsPerDayFilterd = row
        else:
            dfTripsPerDayFilterd = pd.concat([dfTripsPerDayFilterd,row])

    #Create features
    dfTripsPerDayFilterd = createFeatures(dfTripsPerDayFilterd)

    #Make a train test split
    Y = dfTripsPerDayFilterd['tripsPerDay']
    X = dfTripsPerDayFilterd.drop('tripsPerDay',axis=1).values
    x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, random_state=20)

    #Scale the inputdata
    sscaler = StandardScaler()
    x_train = sscaler.fit_transform(x_train)
    x_test = sscaler.transform(x_test)

    #Sacle the output data
    sscalerY = StandardScaler()
    y_train = sscalerY.fit_transform(y_train.values.reshape(-1, 1))
    y_test = sscalerY.transform(y_test.values.reshape(-1, 1))
    #Find the optimal Hyperparameter by testing multiple combinations using RandomizedSearchCV
    if optimalHyperparameterTest == True:

        n_estimators = [int(x) for x in np.linspace(start = 100, stop = 1500, num = 10)]
        max_features = ['auto', 'sqrt']
        max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]
        max_depth.append(None)
        min_samples_split = [2, 5, 10]
        min_samples_leaf = [1, 2, 4]
        bootstrap = [True, False]
        random_grid = {'n_estimators': n_estimators,
                'max_features': max_features,
                'max_depth': max_depth,
                'min_samples_split': min_samples_split,
                'min_samples_leaf': min_samples_leaf,
                'bootstrap': bootstrap}

        rf = RandomForestRegressor()
        rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 150, cv = 3, verbose=2, random_state=45, n_jobs = -1)
        rf_random.fit(x_train,y_train)

        #Take the best values
        reg = RandomForestRegressor(n_jobs=-1,
                            n_estimators = rf_random.best_params_['n_estimators'],
                            min_samples_split = rf_random.best_params_['min_samples_split'],
                            min_samples_leaf = rf_random.best_params_['min_samples_leaf'],
                            max_features = rf_random.best_params_['max_features'],
                            max_depth = rf_random.best_params_['max_depth'],
                            bootstrap = rf_random.best_params_['bootstrap']
                            )

    else:
        #Else use default
        reg = RandomForestRegressor(n_jobs=-1,n_estimators=1000)

    #Train the model
    reg.fit(x_train,y_train)

    #Predict the test data and rescale them
    y_pred_test = reg.predict(x_test)
    y_pred_test = sscalerY.inverse_transform(y_pred_test)
    y_test = sscalerY.inverse_transform(y_test)

    #Evaluate the model based on test set
    errTest = mean_absolute_error(y_test, y_pred_test)
    err_mseTest = mean_squared_error(y_test, y_pred_test)
    err_r2Test = r2_score(y_test, y_pred_test)

    #Predict the train data and rescale them
    y_pred_train = reg.predict(x_train)
    y_pred_train = sscalerY.inverse_transform(y_pred_train)
    y_train = sscalerY.inverse_transform(y_train)

    #Evaluate the model based on train set
    errTrain = mean_absolute_error(y_train, y_pred_train)
    err_mseTrain = mean_squared_error(y_train, y_pred_train)
    err_r2Train = r2_score(y_train, y_pred_train)

    #Print the results
    print("Test  :  ","MAE: ",errTest,"MSE: ",err_mseTest,"R^2: ",err_r2Test)
    print("Train :  ","MAE: ",errTrain,"MSE: ",err_mseTrain,"R^2: ",err_r2Train)

    #Visualize the Prediction
    day = []
    for i in range(len(y_test)):
        day.append(i)

    plt.plot(day, y_test,label="Number of trips")
    plt.plot(day,y_pred_test,label="Predicted number of trips")
    plt.title("Visualization of the Prediction")
    plt.xlabel("Date")
    plt.ylabel("Number of Trips")
    plt.legend(loc="upper left")

    plt.tight_layout()
    plt.show()

    #Retrain the model on all available data

    #Scale the Data
    sscaler = StandardScaler()
    xScaled = sscaler.fit_transform(X)
    sscalerY = StandardScaler()
    yScaled = sscalerY.fit_transform(Y.values.reshape(-1, 1))

    reg.fit(xScaled,yScaled)
    #Save the model
    path = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTrips.pkl")
    dump(reg,path)

    # Save the scaler
    pathScaler = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTripsScaler.bin")
    dump(sscaler,pathScaler)

    pathScalerY = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTripsScalerY.bin")
    dump(sscalerY,pathScalerY)

#Loads the saved ml-model and scaler
def loadModel_NumberOfTrips():
    #Load random forest regressor
    path = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTrips.pkl")
    rfr = load(path , mmap_mode ='r')
    #Load scaler for input data
    pathScaler = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTripsScaler.bin")
    sscaler = load(pathScaler)
    #Load scaler for target feature
    pathScalerY = os.path.join(utils.get_ml_path(), "numberOfTrips/randomForestRegressor_NumberOfTripsScalerY.bin")
    sscalerY = load(pathScalerY)

    return rfr,sscaler, sscalerY

#Predicts the number of trips for the next day
def predict_NumberOfTrips(dfInput, model, sscaler, sscalerY):

    #Mean values for compare
    meanOfAllValues = 1672
    meanOfPrevoiusMonth = 2921

    #Create inputs dataframe and scale it
    df = dfInput.copy()
    #Create necessary features
    features = createFeatures(df)
    #Save the true values to evaluate later
    goal = features['tripsPerDay']
    features.drop('tripsPerDay',inplace=True,axis=1,errors='ignore')
    featureValues = features.values
    xScaled = sscaler.transform(featureValues)

    #Predict and inverse transformation
    prediction = model.predict(xScaled)
    prediction = sscalerY.inverse_transform(prediction)

    # Save data
    path = os.path.join(utils.get_prediction_path(), "output/NumberOfTripPrediction.csv")
    df['tripsPerDay'] = prediction
    df.to_csv(path, index=False)

    print("Prediction is saved to csv --> output/NumberOfTripPrediction.csv")

    #Evaluate the prediction
    errTest = mean_absolute_error(goal, prediction)
    err_mseTest = mean_squared_error(goal, prediction)
    err_r2Test = r2_score(goal, prediction)

    print("Test  :  ","MAE: ",errTest,"MSE: ",err_mseTest,"R^2: ",err_r2Test)

    #Compare with mean methods
    allPreviousAvg = [meanOfAllValues] * len(goal)
    errAllPreviousAvg = mean_absolute_error(goal, allPreviousAvg)
    err_mseAllPreviousAvg = mean_squared_error(goal, allPreviousAvg)

    previousMonthAvg = [meanOfPrevoiusMonth] * len(goal)
    errPreviousMonthAvg = mean_absolute_error(goal, previousMonthAvg)
    err_msePreviousMonthAvg = mean_squared_error(goal, previousMonthAvg)

    print("Compare to prediction by average (last 6 months):  ","MAE: ",errAllPreviousAvg," MSE: ", err_mseAllPreviousAvg)
    print("Compare to prediction by average (last month):  ","MAE: ",errPreviousMonthAvg," MSE: ", err_msePreviousMonthAvg)

    # Plot data
    # TODO Fix legend and axis
    plt.plot(dfInput['date'],prediction, label="Predicted number of trips")
    plt.plot(dfInput['date'],goal, label="Number of trips")
    plt.title("Visualization of the Prediction")
    plt.xlabel("Date")
    plt.xticks(rotation=45)
    plt.ylabel("Number of Trips")
    plt.legend(loc="upper left")

    plt.tight_layout()
    plt.show()

    return df

#Checks if uni is open
def isTerm(row):
    value = -1
    if datetime.datetime(2019, 1, 7) <= row['date'] <= datetime.datetime(2019, 2, 15):
        value = 1
    elif datetime.datetime(2019, 4, 15) <= row['date'] <= datetime.datetime(2019, 7, 19):
        value = 1
    elif datetime.datetime(2019, 10, 14) <= row['date'] <= datetime.datetime(2019, 12, 20):
        value = 1
    else:
        value = 0
    return value


import datetime
import os
from decimal import Decimal

from boto3.dynamodb.conditions import Attr, Key

from aws_utils.utils import random_id

class DynamoDb:
    def __init__(self, session):
        self.client = session.client('dynamodb')
        self.resource = session.resource('dynamodb')

    @staticmethod
    def convert_types(item):
        for k in dict(item):
            if isinstance(item[k], datetime.datetime):
                item[k] = item[k].isoformat()
            elif isinstance(item[k], float):
                item[k] = Decimal(str(item[k]))
            elif isinstance(item[k], dict):
                item[k] = DynamoDb.convert_types(item[k])
        return item

    @staticmethod
    def projection_string(kwargs):
        if 'ProjectionExpression' in kwargs:
            names = {}
            counter = 1
            attributes = kwargs['ProjectionExpression'].split(',')
            for a_index, attribute in enumerate(attributes):
                sub_attributes = attribute.split('.')
                for sa_index, sub_attribute in enumerate(sub_attributes):
                    place_holder = '#attr' + str(counter)
                    names[place_holder] = sub_attribute
                    sub_attributes[sa_index] = place_holder
                    counter += 1
                attribute = '.'.join(sub_attributes)
                attributes[a_index] = attribute
            kwargs['ProjectionExpression'] = ','.join(attributes)
            kwargs['ExpressionAttributeNames'] = names
        return kwargs

    def post_item(self, table, key, item, **kwargs):
        item['created_on'] = datetime.datetime.utcnow().isoformat()
        item.update(key)
        item = DynamoDb.convert_types(item)
        table = self.resource.Table(os.environ[table])
        response = table.put_item(Item=item, ReturnValues='ALL_OLD', **kwargs)
        response['Key'] = key
        return response

    def get_item(self, table, key, **kwargs):
        kwargs = DynamoDb.projection_string(kwargs)
        table = self.resource.Table(os.environ[table])
        response = table.get_item(Key=key, **kwargs)
        if 'Item' in response and response['Item']:
            return response['Item']
        return {}

    def delete_item(self, table, key, **kwargs):
        table = self.resource.Table(os.environ[table])
        response = table.delete_item(Key=key, ReturnValues='ALL_OLD', **kwargs)
        response['Key'] = key
        return response

    def query(self, table, partition_key, sort_key=None, **kwargs):
        kwargs = DynamoDb.projection_string(kwargs)
        key1, val1 = partition_key.popitem()
        key_exp = Key(key1).eq(val1)
        if sort_key:
            key_exp = key_exp & sort_key
        table = self.resource.Table(os.environ[table])
        response = table.query(KeyConditionExpression=key_exp, **kwargs)
        if 'Items' in response and response['Items']:
            return {k: v for k, v in response.items() if k in ['Items', 'Count', 'ScannedCount', 'LastEvaluatedKey']}
        return {}

    def update_item(self, table, key, updates=None, deletes=None, adds=None, appends=None,**kwargs):
        exp = ''
        names = {}
        values = {}
        updates = updates or {}
        deletes = deletes or []
        updates = DynamoDb.convert_types(updates)
        updates['updated_on'] = datetime.datetime.utcnow().isoformat()

        def add_attribute(attribute):
            if '.' not in attribute:
                attr_placeholder = '#attr' + random_id()
                names[attr_placeholder] = attribute
                return attr_placeholder
            attributes = attribute.split('.')
            for _idx, _val in enumerate(attributes):
                attr_placeholder = '#attr' + random_id()
                attributes[_idx] = attr_placeholder
                names[attr_placeholder] = _val
            return '.'.join(attributes)

        def add_value(value):
            val_placeholder = ':val' + random_id()
            values[val_placeholder] = value
            return val_placeholder
        if appends:
            for k1, v1 in dict(appends).items():
                appends[add_attribute(k1)] = add_value(v1)
                del appends[k1]
            exp += 'SET '
            exp += ', '.join('{0}=list_append({0},{1})'.format(k2, v2) for (k2, v2) in appends.items())
            exp += ' '
        if updates:
            for k1, v1 in dict(updates).items():
                updates[add_attribute(k1)] = add_value(v1)
                del updates[k1]
            exp += 'SET '
            exp += ', '.join('{}={}'.format(k2, v2) for (k2, v2) in updates.items())
            exp += ' '
        if deletes:
            for k3, v3 in enumerate(deletes):
                deletes[k3] = add_attribute(v3)
            exp += 'REMOVE '
            exp += ', '.join(deletes)
            exp += ' '
        if adds:
            for k4, v4 in dict(adds).items():
                adds[add_attribute(k4)] = add_value(v4)
                del adds[k4]
            exp += 'ADD '
            exp += ', '.join('{} {}'.format(k5, v5) for (k5, v5) in adds.items())
            exp += ' '
        table = self.resource.Table(os.environ[table])

        # ensure key exist or reject
        key_exist_conditions = None
        for k, v in key.items():
            key_exist_conditions = key_exist_conditions & Attr(k).eq(v) if key_exist_conditions else Attr(k).eq(v)

        if 'ConditionExpression' in kwargs:
            kwargs['ConditionExpression'] = kwargs['ConditionExpression'] & key_exist_conditions
        else:
            kwargs['ConditionExpression'] = key_exist_conditions

        response = table.update_item(
            Key=key, UpdateExpression=exp,
            ExpressionAttributeNames=names,
            ExpressionAttributeValues=values, **kwargs
        )
        response['Key'] = key
        return response


import warnings
warnings.filterwarnings("ignore")

from tensorflow.python.util import deprecation
deprecation._PRINT_DEPRECATION_WARNINGS = False

import tensorflow as tf
tf.get_logger().setLevel('INFO')

import logging, os
os.environ['KMP_WARNINGS'] = '0'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
logging.disable(logging.WARNING)

from os.path import join
import sys
import io
import numpy as np
import torch
import argparse
import time
from pathlib import Path

import json
from scipy.io.wavfile import write

import sacremoses
import subword_nmt
from unidecode import unidecode
#from fairseq.models.transformer import TransformerModel

project_name = 'tacotron2'
sys.path.append(project_name)
sys.path.append(join(project_name, 'waveglow/'))

from hparams import create_hparams
from model import Tacotron2
from text import text_to_sequence
from glow import WaveGlow

# setting
torch.set_grad_enabled(False)

tacotron_model='model/checkpoint_11500'
waveglow_model='model/waveglow/waveglow_256channels_universal_v5.pt'

class Synthesizer:

    def load(self, tacotron_model, waveglow_model):
        # setting
        self.project_name = 'tacotron2'
        sys.path.append(self.project_name)
        sys.path.append(join(self.project_name, 'waveglow/'))

        # initialize Tacotron2
        self.hparams = create_hparams()
        self.hparams.sampling_rate = 22050
        self.hparams.max_decoder_steps = 1000
        self.hparams.fp16_run = True

        self.tacotron = Tacotron2(self.hparams)
        self.tacotron.load_state_dict(torch.load(tacotron_model)['state_dict'])
        _ = self.tacotron.cuda().eval()

        self.waveglow = torch.load(waveglow_model)['model']
        self.waveglow = self.waveglow.remove_weightnorm(self.waveglow)
        _ = self.waveglow.cuda().eval()
        for k in self.waveglow.convinv:
            k.float()

    def __init__(self, tacotron, waveglow, text):
        self.tacotron = tacotron
        self.waveglow = waveglow
        self.text = text

    def read_text(self):
        with open(self.text, 'r') as rf:
            return rf.readlines()

    def run(self):
        self.load(self.tacotron, self.waveglow)
        for i, text in enumerate(self.read_text(),start=1):
            self.synthesize(text, i)

    def synthesize(self, text, i):

        sequence = np.array(text_to_sequence(text, ['transliteration_cleaners']))[None, :]
        sequence = torch.from_numpy(sequence).to(device='cuda', dtype=torch.int64)
        sequence = sequence.cuda()

        with torch.no_grad():
            _, mel, _, _ = self.tacotron.inference(sequence)
            audio = self.waveglow.infer(mel, sigma=0.666)
            audio = audio[0].data.cpu().numpy()
            audio *= 32767 / max(0.01, np.max(np.abs(audio)))

        self.out = io.BytesIO()
        write('wavs/%02d.wav' %i, self.hparams.sampling_rate, audio.astype(np.int16))
        print('%02d.wav' %i)

        return self.out.getvalue()

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--tacotron2', default=tacotron_model, help='Full path to tacotron model')
    parser.add_argument('--waveglow', default=waveglow_model, help='Full path to waveglow model')
    parser.add_argument('--port', type=int, default=80)
    parser.add_argument('--hparams', default='tacotron2/hparams.py',
                        help='Hyperparameter overrides as a comma-separated list of name=value pairs')
    args = parser.parse_args()

    os.environ['KMP_WARNINGS'] = '0'
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    synthesizer = Synthesizer(args.tacotron, args.waveglow, 'txt/test.txt')
    synthesizer.run()


import pandas as pd
import numpy as np
from gluonts_forecasts.model_config_registry import ModelConfigRegistry
from gluonts_forecasts.gluon_dataset import remove_unused_external_features
from gluonts_forecasts.utils import (
    concat_timeseries_per_identifiers,
    concat_all_timeseries,
    add_row_origin,
    quantile_forecasts_series,
)
from dku_constants import (
    METRICS_DATASET,
    METRICS_COLUMNS_DESCRIPTIONS,
    TIMESERIES_KEYS,
    ROW_ORIGIN,
    CUSTOMISABLE_FREQUENCIES_OFFSETS,
)
from gluonts.model.forecast import QuantileForecast
from safe_logger import SafeLogger

logger = SafeLogger("Forecast plugin")

class TrainedModel:
    """
    Wrapper class to make forecasts using a GluonTS Predictor and a training GluonTS ListDataset, and to output a well formatted forecasts dataframe

    Attributes:
        predictor (gluonts.model.predictor.Predictor)
        gluon_dataset (gluonts.dataset.common.ListDataset): GluonTS ListDataset generated by the GluonDataset class (with extra fields to name timeseries)
        prediction_length (int): Number of time steps to predict (at most the prediction length used in training)
        quantiles (list): List of forecasts quantiles to compute in the forecasts_df
        include_history (bool): True to append to the forecasts dataframe the training data
        history_length_limit (int): Maximum number of values to retrieve from historical data per timeseries. Default to None which means all.
        time_column_name (str): Time column name used in training
        identifiers_columns (list): List of timeseries identifiers column names used in training.
        forecasts_df (DataFrame): Dataframe with the different quantiles forecasts and the training data if include_history is True
        model_name (str, optional)
    """

    def __init__(
        self,
        predictor,
        gluon_dataset,
        prediction_length,
        quantiles,
        include_history,
        history_length_limit=None,
        model_name=None,
    ):
        self.predictor = predictor
        self.gluon_dataset = gluon_dataset
        self.prediction_length = predictor.prediction_length if prediction_length == -1 else prediction_length
        self.quantiles = quantiles
        self.include_history = include_history
        self.time_column_name = None
        self.identifiers_columns = None
        self.forecasts_df = None
        self.frequency = gluon_dataset.process.trans[0].freq
        self.history_length_limit = history_length_limit
        self.model_name = model_name
        self._check()

    def predict(self):
        """
        Use the gluon dataset of training to predict future values and
        concat all forecasts timeseries of different identifiers and quantiles together
        """
        model_config = ModelConfigRegistry().get_model(self.model_name)
        if (
            self.model_name
            and not model_config.can_use_external_feature()
            and TIMESERIES_KEYS.FEAT_DYNAMIC_REAL in self.gluon_dataset.list_data[0]
        ):
            # remove external features from the ListDataset used for predictions if the model cannot use them
            gluon_dataset_without_external_features = remove_unused_external_features(
                self.gluon_dataset, self.frequency
            )
            forecasts = self.predictor.predict(gluon_dataset_without_external_features)
        else:
            forecasts = self.predictor.predict(self.gluon_dataset)

        forecasts_list = list(forecasts)

        forecasts_timeseries = self._compute_forecasts_timeseries(forecasts_list)

        multiple_df = concat_timeseries_per_identifiers(forecasts_timeseries)

        self.forecasts_df = concat_all_timeseries(multiple_df)

        self.time_column_name = self.gluon_dataset.list_data[0][TIMESERIES_KEYS.TIME_COLUMN_NAME]
        self.identifiers_columns = (
            list(self.gluon_dataset.list_data[0][TIMESERIES_KEYS.IDENTIFIERS].keys())
            if TIMESERIES_KEYS.IDENTIFIERS in self.gluon_dataset.list_data[0]
            else []
        )

        if self.include_history:
            self.forecasts_df = self._include_history(self.frequency, history_length_limit=self.history_length_limit)

        self.forecasts_df = add_row_origin(self.forecasts_df, both=ROW_ORIGIN.FORECAST, left_only=ROW_ORIGIN.HISTORY)

        self.forecasts_df = self.forecasts_df.rename(columns={"index": self.time_column_name})

    def _include_history(self, frequency, history_length_limit=None):
        """Include the historical data on which the model was trained to the forecasts dataframe.

        Args:
            frequency (str): Used to reconstruct the date range (because a gluon ListDataset only store the start date).
            history_length_limit (int): Maximum number of values to retrieve from historical data per timeseries. Default to None which means all.

        Returns:
            DataFrame containing both the historical data and the forecasted values.
        """
        history_timeseries = self._retrieve_history_timeseries(frequency, history_length_limit)
        multiple_df = concat_timeseries_per_identifiers(history_timeseries)
        history_df = concat_all_timeseries(multiple_df)
        return history_df.merge(self.forecasts_df, on=["index"] + self.identifiers_columns, how="left", indicator=True)

    def _generate_history_target_series(self, timeseries, frequency, history_length_limit=None):
        """Creates a pandas time series from the past target values with Nan values for the prediction_length future dates.

        Args:
            timeseries (dict): Univariate timeseries dictionary created with the GluonDataset class.
            frequency (str): Used in pandas.date_range.
            history_length_limit (int): Maximum number of values to retrieve from historical data per timeseries. Default to None which means all.

        Returns:
            Series with DatetimeIndex.
        """
        target_series = pd.Series(
            np.append(timeseries[TIMESERIES_KEYS.TARGET], np.repeat(np.nan, self.prediction_length)),
            name=timeseries[TIMESERIES_KEYS.TARGET_NAME],
            index=pd.date_range(
                start=timeseries[TIMESERIES_KEYS.START],
                periods=len(timeseries[TIMESERIES_KEYS.TARGET]) + self.prediction_length,
                freq=frequency,
            ),
        )
        if history_length_limit:
            target_series = target_series.iloc[-(history_length_limit + self.prediction_length) :]
        return target_series

    def _generate_history_external_features_dataframe(self, timeseries, frequency, history_length_limit=None):
        """Creates a pandas time series from the past and future external features values.

        Args:
            timeseries (dict): Univariate timeseries dictionary created with the GluonDataset class.
            frequency (str): Used in pandas.date_range.
            history_length_limit (int): Maximum number of values to retrieve from historical data per timeseries. Default to None which means all.

        Returns:
            DataFrame with DatetimeIndex.
        """
        external_features_df = pd.DataFrame(
            timeseries[TIMESERIES_KEYS.FEAT_DYNAMIC_REAL].T[
                : len(timeseries[TIMESERIES_KEYS.TARGET]) + self.prediction_length
            ],
            columns=timeseries[TIMESERIES_KEYS.FEAT_DYNAMIC_REAL_COLUMNS_NAMES],
            index=pd.date_range(
                start=timeseries[TIMESERIES_KEYS.START],
                periods=len(timeseries[TIMESERIES_KEYS.TARGET]) + self.prediction_length,
                freq=frequency,
            ),
        )
        if history_length_limit:
            external_features_df = external_features_df.iloc[-(history_length_limit + self.prediction_length) :]
        return external_features_df

    def _retrieve_history_timeseries(self, frequency, history_length_limit=None):
        """Reconstruct the history timeseries from the gluon_dataset object and fill the dates to predict with Nan values.

        Args:
            frequency (str)
            history_length_limit (int): Maximum number of values to retrieve from historical data per timeseries. Default to None which means all.

        Returns:
            Dictionary of list of timeseries by identifiers (None if no identifiers)
        """
        history_timeseries = {}
        for i, timeseries in enumerate(self.gluon_dataset.list_data):
            if TIMESERIES_KEYS.IDENTIFIERS in timeseries:
                timeseries_identifier_key = tuple(sorted(timeseries[TIMESERIES_KEYS.IDENTIFIERS].items()))
            else:
                timeseries_identifier_key = None

            target_series = self._generate_history_target_series(timeseries, frequency, history_length_limit)

            if TIMESERIES_KEYS.FEAT_DYNAMIC_REAL_COLUMNS_NAMES in timeseries:
                assert (
                    timeseries[TIMESERIES_KEYS.FEAT_DYNAMIC_REAL].shape[1]
                    >= len(timeseries[TIMESERIES_KEYS.TARGET]) + self.prediction_length
                )
                if timeseries_identifier_key not in history_timeseries:
                    external_features_df = self._generate_history_external_features_dataframe(
                        timeseries, frequency, history_length_limit
                    )
                    history_timeseries[timeseries_identifier_key] = [external_features_df]

            if timeseries_identifier_key in history_timeseries:
                history_timeseries[timeseries_identifier_key] += [target_series]
            else:
                history_timeseries[timeseries_identifier_key] = [target_series]
        return history_timeseries

    def _compute_forecasts_timeseries(self, forecasts_list):
        """Compute all forecasts timeseries for each quantile.

        Args:
            forecasts_list (list): List of gluonts.model.forecast.Forecast (objects storing the predicted distributions as samples).

        Returns:
            Dictionary of list of forecasts timeseries by identifiers (None if no identifiers)
        """
        forecasts_timeseries = {}
        for i, sample_forecasts in enumerate(forecasts_list):
            if TIMESERIES_KEYS.IDENTIFIERS in self.gluon_dataset.list_data[i]:
                timeseries_identifier_key = tuple(
                    sorted(self.gluon_dataset.list_data[i][TIMESERIES_KEYS.IDENTIFIERS].items())
                )
            else:
                timeseries_identifier_key = None

            if i == 0 and isinstance(sample_forecasts, QuantileForecast):
                self.quantiles = self._round_to_existing_quantiles(sample_forecasts)

            for quantile in self.quantiles:
                forecasts_label_prefix = "forecast"
                if quantile < 0.5:
                    forecasts_label_prefix += "_lower"
                elif quantile > 0.5:
                    forecasts_label_prefix += "_upper"

                forecasts_series = (
                    quantile_forecasts_series(sample_forecasts, quantile, self.frequency)
                    .rename(f"{forecasts_label_prefix}_{self.gluon_dataset.list_data[i][TIMESERIES_KEYS.TARGET_NAME]}")
                    .iloc[: self.prediction_length]
                )
                if timeseries_identifier_key in forecasts_timeseries:
                    forecasts_timeseries[timeseries_identifier_key] += [forecasts_series]
                else:
                    forecasts_timeseries[timeseries_identifier_key] = [forecasts_series]
        return forecasts_timeseries

    def _reorder_forecasts_df(self):
        """Reorder columns with timeseries identifiers columns right after time column"""
        forecasts_columns = [
            column for column in self.forecasts_df if column not in [self.time_column_name] + self.identifiers_columns
        ]
        self.forecasts_df = self.forecasts_df[[self.time_column_name] + self.identifiers_columns + forecasts_columns]

    def get_forecasts_df(self, session=None):
        """Add the session timestamp and model label to the forecasts dataframe. Sort timeseries in revert order to display predictions on top.

        Args:
            session (Timstamp, optional)

        Returns:
            Forecasts DaaFrame
        """
        if TIMESERIES_KEYS.IDENTIFIERS in self.gluon_dataset.list_data[0]:
            self._reorder_forecasts_df()
        if session:
            self.forecasts_df[METRICS_DATASET.SESSION] = session
        if self.model_name:
            self.forecasts_df[METRICS_DATASET.MODEL_COLUMN] = (
                ModelConfigRegistry().get_model(self.model_name).get_label()
            )

        self.forecasts_df = self.forecasts_df.sort_values(
            by=self.identifiers_columns + [self.time_column_name],
            ascending=[True] * len(self.identifiers_columns) + [False],
        )

        return self.forecasts_df

    def create_forecasts_column_description(self):
        """Explain the meaning of the forecasts columns"""
        column_descriptions = METRICS_COLUMNS_DESCRIPTIONS
        confidence_interval = self._retrieve_confidence_interval()
        for column in self.forecasts_df.columns:
            if "forecast_lower_" in column:
                column_descriptions[
                    column
                ] = f"Lower bound of the {confidence_interval}% forecasts confidence interval."
            elif "forecast_upper_" in column:
                column_descriptions[
                    column
                ] = f"Upper bound of the {confidence_interval}% forecasts confidence interval."
            elif "forecast_" in column:
                column_descriptions[column] = "Median of probabilistic forecasts"
        return column_descriptions

    def _check(self):
        """Raises ValueError if the selected prediction_length is higher than the one used in training"""
        if self.predictor.prediction_length < self.prediction_length:
            raise ValueError(
                f"Please choose a forecasting horizon lower or equal to the one chosen when training: {self.predictor.prediction_length}"
            )

    def _round_to_existing_quantiles(self, sample_forecasts):
        """Find the quantiles that exists in sample_forecasts that are closest to the selected quantiles.
        QuantileForecast cannot predict all quantiles but only a list predifined during training.

        Args:
            sample_forecasts (QuantileForecast)

        Returns:
            List of quantiles that exists in the sample_forecasts
        """
        new_quantiles = []
        possible_quantiles = list(map(float, sample_forecasts.forecast_keys))
        for quantile in self.quantiles:
            new_quantiles += [min(possible_quantiles, key=lambda x: abs(x - quantile))]
        return new_quantiles

    def _retrieve_confidence_interval(self):
        """Retrieve the confidence interval percentage from the minimum and maximum quantiles.
        If they are not symetric around 0.5, log a warning.

        Returns:
            Integer representing the percentage of the confidence interval.
        """
        lower_quantile, upper_quantile = min(self.quantiles), max(self.quantiles)
        confidence_interval = round((upper_quantile - lower_quantile) * 100)
        if round((1 - upper_quantile) * 100, 2) != round(lower_quantile * 100, 2):
            logger.warning(
                f"The output confidence interval is not centered around the median. Lower and upper quantiles are [{lower_quantile}, {upper_quantile}]"
            )
        return confidence_interval


# This file is part of Radicale Server - Calendar Server
# Copyright © 2008 Nicolas Kandel
# Copyright © 2008 Pascal Halter
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2008-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
import math
import sys
from hashlib import md5
from random import getrandbits

import vobject

from radicale import pathutils
from radicale.item import filter as radicale_filter

def predict_tag_of_parent_collection(vobject_items):
    if len(vobject_items) != 1:
        return ""
    if vobject_items[0].name == "VCALENDAR":
        return "VCALENDAR"
    if vobject_items[0].name in ("VCARD", "VLIST"):
        return "VADDRESSBOOK"
    return ""

def predict_tag_of_whole_collection(vobject_items, fallback_tag=None):
    if vobject_items and vobject_items[0].name == "VCALENDAR":
        return "VCALENDAR"
    if vobject_items and vobject_items[0].name in ("VCARD", "VLIST"):
        return "VADDRESSBOOK"
    if not fallback_tag and not vobject_items:
        # Maybe an empty address book
        return "VADDRESSBOOK"
    return fallback_tag

def check_and_sanitize_items(vobject_items, is_collection=False, tag=None):
    """Check vobject items for common errors and add missing UIDs.

    ``is_collection`` indicates that vobject_item contains unrelated
    components.

    The ``tag`` of the collection.

    """
    if tag and tag not in ("VCALENDAR", "VADDRESSBOOK"):
        raise j.exceptions.Value("Unsupported collection tag: %r" % tag)
    if not is_collection and len(vobject_items) != 1:
        raise j.exceptions.Value("Item contains %d components" % len(vobject_items))
    if tag == "VCALENDAR":
        if len(vobject_items) > 1:
            raise j.exceptions.Base("VCALENDAR collection contains %d " "components" % len(vobject_items))
        vobject_item = vobject_items[0]
        if vobject_item.name != "VCALENDAR":
            raise j.exceptions.Value("Item type %r not supported in %r " "collection" % (vobject_item.name, tag))
        component_uids = set()
        for component in vobject_item.components():
            if component.name in ("VTODO", "VEVENT", "VJOURNAL"):
                component_uid = get_uid(component)
                if component_uid:
                    component_uids.add(component_uid)
        component_name = None
        object_uid = None
        object_uid_set = False
        for component in vobject_item.components():
            # https://tools.ietf.org/html/rfc4791#section-4.1
            if component.name == "VTIMEZONE":
                continue
            if component_name is None or is_collection:
                component_name = component.name
            elif component_name != component.name:
                raise j.exceptions.Value(
                    "Multiple component types in object: %r, %r" % (component_name, component.name)
                )
            if component_name not in ("VTODO", "VEVENT", "VJOURNAL"):
                continue
            component_uid = get_uid(component)
            if not object_uid_set or is_collection:
                object_uid_set = True
                object_uid = component_uid
                if not component_uid:
                    if not is_collection:
                        raise j.exceptions.Value("%s component without UID in object" % component_name)
                    component_uid = find_available_uid(component_uids.__contains__)
                    component_uids.add(component_uid)
                    if hasattr(component, "uid"):
                        component.uid.value = component_uid
                    else:
                        component.add("UID").value = component_uid
            elif not object_uid or not component_uid:
                raise j.exceptions.Value("Multiple %s components without UID in " "object" % component_name)
            elif object_uid != component_uid:
                raise j.exceptions.Value(
                    "Multiple %s components with different UIDs in object: "
                    "%r, %r" % (component_name, object_uid, component_uid)
                )
            # vobject interprets recurrence rules on demand
            try:
                component.rruleset
            except Exception as e:
                raise j.exceptions.Value("invalid recurrence rules in %s" % component.name) from e
    elif tag == "VADDRESSBOOK":
        # https://tools.ietf.org/html/rfc6352#section-5.1
        object_uids = set()
        for vobject_item in vobject_items:
            if vobject_item.name == "VCARD":
                object_uid = get_uid(vobject_item)
                if object_uid:
                    object_uids.add(object_uid)
        for vobject_item in vobject_items:
            if vobject_item.name == "VLIST":
                # Custom format used by SOGo Connector to store lists of
                # contacts
                continue
            if vobject_item.name != "VCARD":
                raise j.exceptions.Value("Item type %r not supported in %r " "collection" % (vobject_item.name, tag))
            object_uid = get_uid(vobject_item)
            if not object_uid:
                if not is_collection:
                    raise j.exceptions.Value("%s object without UID" % vobject_item.name)
                object_uid = find_available_uid(object_uids.__contains__)
                object_uids.add(object_uid)
                if hasattr(vobject_item, "uid"):
                    vobject_item.uid.value = object_uid
                else:
                    vobject_item.add("UID").value = object_uid
    else:
        for i in vobject_items:
            raise j.exceptions.Value(
                "Item type %r not supported in %s collection" % (i.name, repr(tag) if tag else "generic")
            )

def check_and_sanitize_props(props):
    """Check collection properties for common errors."""
    tag = props.get("tag")
    if tag and tag not in ("VCALENDAR", "VADDRESSBOOK"):
        raise j.exceptions.Value("Unsupported collection tag: %r" % tag)

def find_available_uid(exists_fn, suffix=""):
    """Generate a pseudo-random UID"""
    # Prevent infinite loop
    for _ in range(1000):
        r = "%016x" % getrandbits(128)
        name = "%s-%s-%s-%s-%s%s" % (r[:8], r[8:12], r[12:16], r[16:20], r[20:], suffix)
        if not exists_fn(name):
            return name
    # something is wrong with the PRNG
    raise j.exceptions.Base("No unique random sequence found")

def get_etag(text):
    """Etag from collection or item.

    Encoded as quoted-string (see RFC 2616).

    """
    etag = md5()
    etag.update(text.encode("utf-8"))
    return '"%s"' % etag.hexdigest()

def get_uid(vobject_component):
    """UID value of an item if defined."""
    return vobject_component.uid.value if hasattr(vobject_component, "uid") else None

def get_uid_from_object(vobject_item):
    """UID value of an calendar/addressbook object."""
    if vobject_item.name == "VCALENDAR":
        if hasattr(vobject_item, "vevent"):
            return get_uid(vobject_item.vevent)
        if hasattr(vobject_item, "vjournal"):
            return get_uid(vobject_item.vjournal)
        if hasattr(vobject_item, "vtodo"):
            return get_uid(vobject_item.vtodo)
    elif vobject_item.name == "VCARD":
        return get_uid(vobject_item)
    return None

def find_tag(vobject_item):
    """Find component name from ``vobject_item``."""
    if vobject_item.name == "VCALENDAR":
        for component in vobject_item.components():
            if component.name != "VTIMEZONE":
                return component.name or ""
    return ""

def find_tag_and_time_range(vobject_item):
    """Find component name and enclosing time range from ``vobject item``.

    Returns a tuple (``tag``, ``start``, ``end``) where ``tag`` is a string
    and ``start`` and ``end`` are POSIX timestamps (as int).

    This is intened to be used for matching against simplified prefilters.

    """
    tag = find_tag(vobject_item)
    if not tag:
        return (tag, radicale_filter.TIMESTAMP_MIN, radicale_filter.TIMESTAMP_MAX)
    start = end = None

    def range_fn(range_start, range_end, is_recurrence):
        nonlocal start, end
        if start is None or range_start < start:
            start = range_start
        if end is None or end < range_end:
            end = range_end
        return False

    def infinity_fn(range_start):
        nonlocal start, end
        if start is None or range_start < start:
            start = range_start
        end = radicale_filter.DATETIME_MAX
        return True

    radicale_filter.visit_time_ranges(vobject_item, tag, range_fn, infinity_fn)
    if start is None:
        start = radicale_filter.DATETIME_MIN
    if end is None:
        end = radicale_filter.DATETIME_MAX
    try:
        return tag, math.floor(start.timestamp()), math.ceil(end.timestamp())
    except ValueError as e:
        if str(e) == ("offset must be a timedelta representing a whole " "number of minutes") and sys.version_info < (
            3,
            6,
        ):
            raise j.exceptions.Base("Unsupported in Python < 3.6: %s" % e) from e
        raise

class Item:
    def __init__(
        self,
        collection_path=None,
        collection=None,
        vobject_item=None,
        href=None,
        last_modified=None,
        text=None,
        etag=None,
        uid=None,
        name=None,
        component_name=None,
        time_range=None,
    ):
        """Initialize an item.

        ``collection_path`` the path of the parent collection (optional if
        ``collection`` is set).

        ``collection`` the parent collection (optional).

        ``href`` the href of the item.

        ``last_modified`` the HTTP-datetime of when the item was modified.

        ``text`` the text representation of the item (optional if
        ``vobject_item`` is set).

        ``vobject_item`` the vobject item (optional if ``text`` is set).

        ``etag`` the etag of the item (optional). See ``get_etag``.

        ``uid`` the UID of the object (optional). See ``get_uid_from_object``.

        ``name`` the name of the item (optional). See ``vobject_item.name``.

        ``component_name`` the name of the primary component (optional).
        See ``find_tag``.

        ``time_range`` the enclosing time range.
        See ``find_tag_and_time_range``.

        """
        if text is None and vobject_item is None:
            raise j.exceptions.Value("at least one of 'text' or 'vobject_item' must be set")
        if collection_path is None:
            if collection is None:
                raise j.exceptions.Value("at least one of 'collection_path' or " "'collection' must be set")
            collection_path = collection.path
        assert collection_path == pathutils.strip_path(pathutils.sanitize_path(collection_path))
        self._collection_path = collection_path
        self.collection = collection
        self.href = href
        self.last_modified = last_modified
        self._text = text
        self._vobject_item = vobject_item
        self._etag = etag
        self._uid = uid
        self._name = name
        self._component_name = component_name
        self._time_range = time_range

    def serialize(self):
        if self._text is None:
            try:
                self._text = self.vobject_item.serialize()
            except Exception as e:
                raise j.exceptions.Base(
                    "Failed to serialize item %r from %r: %s" % (self.href, self._collection_path, e)
                ) from e
        return self._text

    @property
    def vobject_item(self):
        if self._vobject_item is None:
            try:
                self._vobject_item = vobject.readOne(self._text)
            except Exception as e:
                raise j.exceptions.Base(
                    "Failed to parse item %r from %r: %s" % (self.href, self._collection_path, e)
                ) from e
        return self._vobject_item

    @property
    def etag(self):
        """Encoded as quoted-string (see RFC 2616)."""
        if self._etag is None:
            self._etag = get_etag(self.serialize())
        return self._etag

    @property
    def uid(self):
        if self._uid is None:
            self._uid = get_uid_from_object(self.vobject_item)
        return self._uid

    @property
    def name(self):
        if self._name is None:
            self._name = self.vobject_item.name or ""
        return self._name

    @property
    def component_name(self):
        if self._component_name is not None:
            return self._component_name
        return find_tag(self.vobject_item)

    @property
    def time_range(self):
        if self._time_range is None:
            self._component_name, *self._time_range = find_tag_and_time_range(self.vobject_item)
        return self._time_range

    def prepare(self):
        """Fill cache with values."""
        orig_vobject_item = self._vobject_item
        self.serialize()
        self.etag
        self.uid
        self.name
        self.time_range
        self.component_name
        self._vobject_item = orig_vobject_item


#-*-coding: utf-8-*-
import discord
from discord.ext import commands
from os import chdir, getcwd
from subprocess import run, PIPE
from pyautogui import screenshot
from time import strftime

bot = commands.Bot(command_prefix=".")

@bot.event
async def on_ready():
	print("[+]Ready.")

@bot.command(name="sh")
async def sh(ctx, *args):
	if len(args) == 0:
		return 0
	command = " ".join(args)
	print("[+]Command sent: {}".format(command))
	if args[0] == "cd":
		try:
			chdir(args[1])
		except:
			pass
		finally:
			output = getcwd()

	#elif args[0] == "screenshot":
	#	filename = "ss{}.jpg".format(strftime("%y%m%d%H%M%S"))
	#	screenshot(filename)
	#	await ctx.send(file=discord.File('{}\\{}'.format(getcwd(), filename)))
	#	run("del {}".format(filename), shell=True)
	#	return 1

	else:
		o = run(command, shell=True, stdout=PIPE, stderr=PIPE)
		output = o.stdout.decode("latin1") + o.stderr.decode("latin1")

	await ctx.send(output)

bot.run("Nzk4NDU3ODczOTYwOTI3MjUz.X_1T1w.Ru68vQhOyWQk6OJW6SjWX7yakUU")


import sys
import data.preprocess_data as pr
import data.functions as fun
import tensorflow as tf
from tensorflow.python.ops.rnn_cell import BasicRNNCell
from tensorflow.python.ops import rnn_cell
from datetime import datetime
sys.path.append('F:\\Program\\Python')

plan_path = 'save_plan_D'
K = 641
num_hidden = 300
num_layer = 1
# step_size : the input length of every time step, it must be the divisor of 640(K-1)
output_size = K
input_size = K

learning_rate = 0.2
need_init = False
batch_size = 200
incorrect_path = plan_path+'/incorrect.txt'

train_input, train_output, test_input, test_output, BOLD_range, neural_range = pr.preprocess_data(batch_size=batch_size)
test_input = test_input[0:batch_size, :]
test_output = test_output[0:batch_size, :]
# input_size=fun.get_BOLD_length_by_neural_length(output_size,BOLD_range,range_type='cross')
print('Data preprocessed.')

neural_length = train_output.shape[1]

input_place = tf.placeholder(tf.float32, [batch_size, input_size])
output_place = tf.placeholder(tf.float32, [batch_size, output_size])

rnn_neuron = BasicRNNCell(num_hidden)
rnn_neuron = rnn_cell.MultiRNNCell([rnn_neuron] * num_layer)

initial_state=rnn_neuron.zero_state(batch_size=batch_size, dtype=tf.float32)

val,final_state=rnn_neuron(input_place, initial_state)

weight = tf.Variable(tf.truncated_normal([num_hidden, output_size]),name='weight')
bias = tf.Variable(tf.constant(0.1, shape=[output_size]),name='bias')

prediction = tf.nn.sigmoid(tf.matmul(val, weight) + bias,name='prediction')

# Optimizer
residual=tf.square(tf.sub(output_place,prediction))
loss_mse=tf.reduce_mean(residual, name='loss_mse')
optimizer = tf.train.GradientDescentOptimizer(learning_rate,name='optimizer')
minimize = optimizer.minimize(loss_mse,name='minimize')

# Accuracy
mistakes = tf.not_equal(output_place, tf.round(prediction), name='mistakes')
error = tf.reduce_mean(tf.cast(mistakes, tf.float32),name='error')

saver=tf.train.Saver(tf.all_variables())
init_op = tf.initialize_all_variables()

restore_epoch=0
with tf.Session() as sess:

    if need_init:
        sess.run(init_op)
        file_incorrect = open(incorrect_path, 'w')
        print('Initialize successful.')
    else:
        restore_epoch=28300
        file_incorrect=open(incorrect_path,'a')
        saver.restore(sess,plan_path+'/model'+str(restore_epoch)+'.ckpt')
        print('Model restored.')

    no_of_batches = int(len(train_input) / batch_size)
    incorrect = 1
    epoch = restore_epoch

    time_indexs = range(0, len(neural_range)-K+1, K)

    while incorrect > 0.1:
        state=sess.run(rnn_neuron.zero_state(batch_size=batch_size, dtype=tf.float32))
        ptr = 0

        for batch in range(no_of_batches):
            # batch_start_time=datetime.now()
            for time in time_indexs:
                # inp = train_input[ptr:ptr+batch_size,time:time+input_size].reshape(batch_size,input_size)
                # out = fun.get_output_range_by_input_range(range(time, time + input_size), neural_range, train_output,
                #                                          batch_size, ptr, 'wide')
                out = train_output[ptr:ptr+batch_size, time:time+output_size].reshape(batch_size, output_size)
                inp = fun.get_output_range_by_input_range(range(time,time+output_size),BOLD_range,train_input,
                                                            batch_size, ptr, 'plan_D')

                _, state = sess.run([minimize, final_state], {initial_state:state, input_place:inp, output_place : out})
            '''batch_end_time = datetime.now()
            time_consume = (batch_end_time- batch_start_time).microseconds
            print('Epoch - {epoch} , Batch - {batch} , Time consume(per sample) - {time} '
                  .format(epoch=epoch+1, batch=batch+1, time=time_consume/batch_size))
            # print('Time consume - {time}'.format(time = time_consume))'''
            ptr += batch_size

        if (epoch + 1) % 1 == 0:
            incorrect = 0
            loss = 0
            ptr=0
            for time in time_indexs:
                # inp = test_input[:, time:time + input_size].reshape(batch_size, input_size)
                # out = fun.get_output_range_by_input_range(range(time, time + input_size), neural_range, test_output,
                #                                         batch_size, ptr, 'wide')
                out = test_output[ptr:ptr + batch_size, time:time + output_size].reshape(batch_size, output_size)
                inp = fun.get_output_range_by_input_range(range(time, time + output_size), BOLD_range, test_input,
                                                           batch_size, ptr, 'plan_D')
                los, err = sess.run([loss_mse, error], {input_place:inp, output_place: out})
                incorrect += err
                loss += los
            incorrect /= len(time_indexs)
            error_string = 'Epoch - {:2d} error {:2.3f}% loss {:f}'.format(epoch + 1, 100 * incorrect,loss)
            print(error_string)
            file_incorrect=open(incorrect_path,'a')
            file_incorrect.write(error_string+'\n')
            file_incorrect.close()
            if (epoch+1) % 100 == 0:
                save_path = saver.save(sess, plan_path+'/model' + str(epoch + 1) + '.ckpt')
                print('Model saved in file', save_path)

        epoch += 1