import fileSelectFunc as fs
import regionFunc as rf
from tkinter.constants import W
import numpy as np
import matplotlib.pyplot as plt
import sqlite3
import pandas as pd
import tkinter.ttk as ttk
from tkinter import messagebox
import matplotlib as mpl
from matplotlib import pylab
import seaborn as sns

def calculFeature(feature) :
     """
     Calculates the position and length of each feature and retrieves the sum of all the features present in the chromosome
     """

     con = sqlite3.connect(fs.dbName)
     cur = con.cursor()

     featureT= cur.execute("SELECT end-start from features WHERE featuretype = '%s' and seqid='%s' and start >=%s and end<=%s"%(feature,rf.chrSelected,rf.startSelected,rf.endSelected)).fetchall()

     featureTInt = []
     for featureLength in featureT :
          featureTInt.append(featureLength[0])

     featurePositions = [*range(1,len(featureT)+1)]

     sumFeatures = 0
     for g in featureTInt :
          sumFeatures+= g

     con.commit()
     cur.close()
     con.close()
     return featureTInt,featurePositions,sumFeatures

def calculIntergenes() :
     """
     Calculates the position and length of each intergenes and retrieves the sum of all the intergenes present in the chromosome
     """

     con = sqlite3.connect(fs.dbName)
     cur = con.cursor()

     startGenePlus = cur.execute("SELECT start from features WHERE featuretype ='gene' and strand = '+' and seqid='%s' and start >=%s and end<=%s"%(rf.chrSelected,rf.startSelected,rf.endSelected)).fetchall()

     endGenePlus= cur.execute("SELECT end from features WHERE featuretype ='gene' and strand = '+' and seqid='%s' and start >=%s and end<=%s"%(rf.chrSelected,rf.startSelected,rf.endSelected)).fetchall()

     interArrayPlus = np.column_stack((startGenePlus,endGenePlus))

     #determines the length of intergenes from start and end positions of genes
     interListPlus = []
     for inter in range (1,len(interArrayPlus)) :
          if interArrayPlus[inter,0] - interArrayPlus[inter-1,1] < 0 :
               continue
          else :
               interListPlus.append(interArrayPlus[inter,0] - interArrayPlus[inter-1,1])

     startGeneMinus= cur.execute("SELECT start from features WHERE featuretype ='gene' and strand = '-' and seqid='%s' and start >=%s and end<=%s"%(rf.chrSelected,rf.startSelected,rf.endSelected)).fetchall()

     endGeneMinus= cur.execute("SELECT end from features WHERE featuretype ='gene' and strand = '-' and seqid='%s' and start >=%s and end<=%s"%(rf.chrSelected,rf.startSelected,rf.endSelected)).fetchall()

     interArrayMinus = np.column_stack((startGeneMinus,endGeneMinus))

     interListMinus = []
     for inter in range (1,len(interArrayMinus)) :
          if interArrayMinus[inter,0] - interArrayMinus[inter-1,1] < 0 :
               continue
          else :
               interListMinus.append(interArrayMinus[inter,0] - interArrayMinus[inter-1,1])

     interListBoth = interListPlus + interListMinus

     sumInter = 0
     for v in interListBoth :
          sumInter+= v

     interPositionsBoth = [*range(1,len(interListBoth)+1)]

     con.commit()
     cur.close()
     con.close()
     return interListBoth,sumInter,interPositionsBoth

def generateGraphGene(show,save):
     """
     Generates a barplot representing the distribution of gene sizes inside the chromosome region
     """

     plt.figure(figsize=(7, 5))
     geneTInt,genePositions,sumGenes=calculFeature("gene")
     mpl.rcParams['axes.spines.right'] = False
     mpl.rcParams['axes.spines.top'] = False
     fig2 = pylab.gcf()
     fig2.canvas.manager.set_window_title('Graphes des genes')
     plt.bar(genePositions,geneTInt,label='Gene',color='olive')
     plt.xlabel('Position')
     plt.ylabel('Taille (acide nucleique)')
     plt.title('Distribution de taille des genes sur les 2 brins',fontsize=13)
     plt.legend(ncol=3,loc='upper right')
     if show == 1 :
          plt.show()
     if save == 1 :
          plt.savefig("./barGene.png")
          plt.close()
     return

def generateGraphInter(show,save):
     """
     Generates a barplot representing the distribution of intergene sizes inside the chromosome region
     """

     plt.figure(figsize=(7, 5))
     interListBoth,sumInter,interPositionsBoth=calculIntergenes()
     mpl.rcParams['axes.spines.right'] = False
     mpl.rcParams['axes.spines.top'] = False
     fig1 = pylab.gcf()
     fig1.canvas.manager.set_window_title('Graphes des régions intergéniques')
     plt.bar(interPositionsBoth,interListBoth,label='Régions Intergéniques',color='yellowgreen')
     plt.xlabel('Position')
     plt.ylabel('Taille (acide nucleique)')
     plt.title('Distribution de taille des régions intergéniques sur les 2 brins',fontsize=11)
     plt.legend(ncol=4,loc='upper right')
     if show == 1 :
          plt.show()
     if save == 1 :
          plt.savefig("./barInter.png")
          plt.close()
     return

def generateGraphExon (show,save):
     """
     Generates a barplot representing the distribution of exon sizes inside the chromosome region
     """

     plt.figure(figsize=(7, 5))
     exonTInt,exonPositions,sumExons=calculFeature("exon")
     mpl.rcParams['axes.spines.right'] = False
     mpl.rcParams['axes.spines.top'] = False
     fig1 = pylab.gcf()
     fig1.canvas.manager.set_window_title('Graphes des exons')
     plt.bar(exonPositions,exonTInt,label='exon',color='darksalmon')
     plt.xlabel('Position')
     plt.ylabel('Taille (acide nucleique)')
     plt.title('Distribution de taille des exons sur les 2 brins',fontsize=13)
     plt.legend(ncol=3,loc='upper right')

     if show == 1 :
          plt.show()
     if save == 1 :
          plt.savefig("./barExon.png")
          plt.close()
     return

def generateGraphIntron(show,save):
     """
     Generates a barplot representing the distribution of intron sizes inside the chromosome region
     """

     plt.figure(figsize=(7, 5))
     intronTInt,intronPositions,sumIntrons=calculFeature("intron")
     mpl.rcParams['axes.spines.right'] = False
     mpl.rcParams['axes.spines.top'] = False
     fig3 = pylab.gcf()
     fig3.canvas.manager.set_window_title('Graphes des introns')
     plt.bar(intronPositions,intronTInt,label='Intron',color='firebrick')
     plt.xlabel('Position')
     plt.ylabel('Taille (acide nucleique)')
     plt.title('Distribution de taille des introns sur les 2 brins',fontsize=13)
     plt.legend(ncol=3,loc='upper right')
     if show == 1 :
          plt.show()
     if save == 1 :
          plt.savefig("./barIntron.png")
          plt.close()
     return

def generatePiechartGenesIntergenes (show,save):
     """
     Generates a piechart representing the size percentages of genes and intergenes inside the chromosome region
     """

     plt.figure(figsize=(7, 5))
     geneTInt,genePositions,sumGenes=calculFeature("gene")
     interListBoth,sumInter,interPositionsBoth=calculIntergenes()
     values1 = [sumGenes,sumInter]
     Names1 = ["Genes","Intergenes"]
     col=['olive','yellowgreen']
     figP = pylab.gcf()
     figP.canvas.manager.set_window_title('Distributions des genes et des intergenes')
     plt.pie(values1,labels=Names1,autopct="%.1f%%",wedgeprops={'edgecolor':'white', 'linewidth':2},colors=col)
     plt.title('Pourcentages des genes et intergenes')
     if show == 1 :
          plt.show()
     if save == 1 :
          plt.savefig("./pieGeneInter.png")
          plt.close()
     return

def generatePiechartExonsIntrons (show,save):
     """
     Generates a piechart representing the size percentages of exons and introns inside the chromosome region
     """

     plt.figure(figsize=(7, 5))
     intronTInt,intronPositions,sumIntrons=calculFeature("intron")
     exonTInt,exonPositions,sumExons=calculFeature("exon")
     values = [sumIntrons,sumExons]
     Names = ["Introns","Exons"]
     col=['firebrick','darksalmon']
     figP = pylab.gcf()
     figP.canvas.manager.set_window_title('Distributions des exons et des intron')
     plt.pie(values,labels=Names,autopct="%.1f%%",wedgeprops={'edgecolor':'white', 'linewidth':2},colors=col)
     plt.title('Pourcentages des exons et introns')
     if show == 1 :
          plt.show()
     if save == 1 :
          plt.savefig("./pieExonsIntrons.png")
          plt.close()
     return

def generateBoxplot1(show,save) :
     """
     Generates a boxplot comparing the size distribution of genes and intergenes inside the chromosome region
     """

     plt.figure(figsize=(7, 5))
     geneTInt,genePositions,sumGenes=calculFeature("gene")
     interListBoth,sumInter,interPositionsBoth=calculIntergenes()
     figB1 = pylab.gcf()
     figB1.canvas.manager.set_window_title('Boxplot des genes et des intergenes')
     a = pd.DataFrame({ '' : np.repeat('Genes',len(geneTInt)), 'Taille (acide nucleique)': geneTInt })
     b = pd.DataFrame({ '' : np.repeat('Intergenes',len(interListBoth)), 'Taille (acide nucleique)': interListBoth })
     df=a.append(b)
     my_pal2 = {"Genes": "olive", "Intergenes": "yellowgreen"}
     sns.boxplot(x='', y='Taille (acide nucleique)', data=df,showmeans=True,palette=my_pal2)
     plt.title('Comparaison de taille entre genes et intergenes')
     if show == 1 :
          plt.show()
     if save == 1 :
          plt.savefig("./box1.png")
          plt.close()
     return

def generateBoxplot2(show,save) :
     """
     Generates a boxplot comparing the size distribution of exons and introns inside the chromosome region
     """

     plt.figure(figsize=(7, 5))
     exonTInt,exonPositions,sumExons=calculFeature("exon")
     intronTInt,intronPositions,sumIntrons=calculFeature("intron")
     figB2 = pylab.gcf()
     figB2.canvas.manager.set_window_title('Boxplot des exons et des introns')
     c = pd.DataFrame({ '' : np.repeat('Exons',len(exonTInt)), 'Taille (acide nucleique)': exonTInt })
     d = pd.DataFrame({ '' : np.repeat('Introns',len(intronTInt)), 'Taille (acide nucleique)':intronTInt })
     df=c.append(d)
     my_pal1 = {"Exons": "darksalmon", "Introns": "firebrick"}
     sns.boxplot(x='', y='Taille (acide nucleique)', data=df,showmeans=True,palette=my_pal1)
     plt.title('Comparaison de taille entre exons et introns')
     if show == 1 :
          plt.show()
     if save == 1 :
          plt.savefig("./box2.png")
          plt.close()
     return

def generateGraphFunc (window,resultsFrame,selectedRegion):
     """
     Creates the graph interface inside the resultsFrame with all graph buttons
     """

     if selectedRegion.cget("text") == "Aucune région sélectionnée" or selectedRegion.cget("text") == "" :
          messagebox.showwarning("Selection de région","Veuillez sélectionner une région")
     else :
          for widget in resultsFrame.winfo_children() :
               widget.destroy()

          window.geometry("730x550+350+0")

          mainTitle = ttk.Label(resultsFrame,text="Generation des graphes",foreground="black")
          mainTitle.grid(column=0,row=0,pady=15,columnspan=3)

          barTitle = ttk.Label(resultsFrame,text="Distributions des tailles",foreground="black")
          barTitle.grid(column=0,row=1,padx=20,pady=5)

          pieTitle = ttk.Label(resultsFrame,text="Proportions",foreground="black")
          pieTitle.grid(column=1,row=1,padx=20,pady=5)

          boxTitle = ttk.Label(resultsFrame,text="Comparaison des distributions",foreground="black")
          boxTitle.grid(column=2,row=1,padx=20,pady=5)

          gene_Button = ttk.Button(resultsFrame,text="Genes",command= lambda  : generateGraphGene(1,0))
          gene_Button.grid(column=0,row=2,pady=10,padx=25)

          inter_Button = ttk.Button(resultsFrame,text="Intergenes",command= lambda : generateGraphInter(1,0))
          inter_Button.grid(column=0,row=3,pady=10,padx=25)

          exon_Button = ttk.Button(resultsFrame,text="Exons",command= lambda : generateGraphExon(1,0))
          exon_Button.grid(column=0,row=4,pady=10,padx=25)

          intron_Button = ttk.Button(resultsFrame,text="Introns",command= lambda : generateGraphIntron(1,0))
          intron_Button.grid(column=0,row=5,pady=10,padx=25)

          pie2_Button = ttk.Radiobutton(resultsFrame,text='Genes/Intergenes',command= lambda : generatePiechartGenesIntergenes(1,0))
          pie2_Button.grid(column=1,row=2,padx=30,ipady= 25,sticky=W,rowspan=2)

          pie1_Button = ttk.Radiobutton(resultsFrame,text='Exons/Introns',command= lambda :generatePiechartExonsIntrons(1,0))
          pie1_Button.grid(column=1,row=4,padx=30,ipady= 25,sticky=W,rowspan=2)

          box1_Button = ttk.Button(resultsFrame,text='Boxplot G/I',command= lambda : generateBoxplot1(1,0))
          box1_Button.grid(column=2,row=2,padx=20,pady=5,rowspan=2)

          box2_Button = ttk.Button(resultsFrame,text='Boxplot E/I',command=lambda : generateBoxplot2(1,0))
          box2_Button.grid(column=2,row=4,padx=20,pady=5,rowspan=2)
     return


import keyword
import os
import re
from datetime import date, datetime
from inspect import getmodule
from pathlib import Path
from typing import Any, Dict, Set, Tuple, Type, Union, List

from uuid import UUID

import inflect
from pydantic import BaseModel, create_model
from pydantic.datetime_parse import StrBytesIntFloat, parse_date, parse_datetime

inflection_engine = inflect.engine()

TYPE_PREFERENCE_ORDER = [str,
                         bool,
                         float,
                         int,
                         date,
                         datetime]

def is_date(value: Union[date, StrBytesIntFloat]) -> bool:
    try:
        parse_date(value)
        return isinstance(value, (str, date))
    except:
        return False

def is_datetime(value: Union[datetime, StrBytesIntFloat]) -> bool:
    try:
        parse_datetime(value)
        return isinstance(value, (str, datetime))
    except:
        return False

def is_uuid(value) -> bool:
    if isinstance(value, UUID):
        return True
    try:
        if isinstance(value, str):
            UUID(value)
            return True
    except ValueError:
        pass
    return False

class ModelProperty(BaseModel):
    number_of_times_seen: int = 0
    observed_values: Set[Any] = set()
    type: Any
    is_list: bool = False

class Model(BaseModel):
    number_of_times_seen: int = 0
    keys: Dict[str, ModelProperty] = {}
    name: str

class JsonToModelParser(object):

    def __init__(self, root_name: str = "Root"):
        self.root_name: str = root_name
        self.models: Dict[str, Model] = {}

    def _get_type(self, name: str, value: Any) -> type:
        if isinstance(value, dict):
            _type = self.parse_dict(value, name=name)
        else:
            if str(value).replace(".", "", 1).replace("-", "", 1).isdigit():
                _type = type(value)
            elif is_datetime(value):
                _type = datetime
            elif is_date(value):
                _type = date
            elif is_uuid(value):
                _type = UUID
            else:
                _type = type(value)
        return _type

    def _get_dependencies_order(self, model: Model) -> List[str]:
        dependencies = []
        for key, property in model.keys.items():
            if isinstance(property.type, Model):
                dependencies.extend(self._get_dependencies_order(self.models.get(property.type.name)))
        dependencies.append(model.name)
        return dependencies

    def _normalize_key(self, key: str) -> str:
        return key.replace("_", " ").replace("-", " ").title().replace(" ", "")

    def parse_dict(self, model_dict: dict, name: str = None) -> Model:
        if name is None:
            name = self.root_name
        _model = self.models.get(name, Model(name=name))
        _model.number_of_times_seen += 1
        for key, value in model_dict.items():
            is_list = False
            if isinstance(value, list):
                is_list = True
                if inflection_engine.singular_noun(key):
                    singular_key = inflection_engine.singular_noun(key)
                else:
                    singular_key = key
                singular_key = self._normalize_key(singular_key)
                if len(value) > 0:
                    for _value in value:
                        _type = self._get_type(name + singular_key, _value)
                else:
                    _type = None
            else:
                _type = self._get_type(name + self._normalize_key(key), value)

            model_property = _model.keys.get(key, ModelProperty(type=_type))

            if value is not None:
                model_property.number_of_times_seen += 1
            model_property.is_list = is_list
            _model.keys[key] = model_property
            try:
                model_property.observed_values.add(value)
            except TypeError:
                # Error on non hashable types, we don't care
                pass

            if model_property.type is not _type and model_property.type != _type:
                print("Mismatch: Property '{name}' was type '{old_type}' but was now found to be '{new_type}'".format(name=key,
                                                                                old_type=model_property.type,
                                                                                new_type=_type))
                if model_property.type is None:
                    model_property.type = _type
                elif _type is not None:
                    if _type in TYPE_PREFERENCE_ORDER:
                        new_type_preference = TYPE_PREFERENCE_ORDER.index(_type)
                    else:
                        new_type_preference = len(TYPE_PREFERENCE_ORDER) + 1

                    if model_property.type in TYPE_PREFERENCE_ORDER:
                        old_type_preference = TYPE_PREFERENCE_ORDER.index(model_property.type)
                    else:
                        old_type_preference = len(TYPE_PREFERENCE_ORDER) + 1

                    model_property.type = _type if new_type_preference <= old_type_preference else model_property.type
                print("Resolved type to value '{new_type}'".format(new_type=model_property.type))

        self.models[name] = _model
        return _model

    def generate_models(self) -> Dict[str, Type[BaseModel]]:
        generated_models: Dict[Model, BaseModel] = {}
        for model_name in self._get_dependencies_order(self.models.get(self.root_name)):
            model = self.models.get(model_name)
            keys: Dict[str, Tuple[Any, Any]] = {}
            for key, model_property in model.keys.items():
                property_type = model_property.type

                if isinstance(property_type, Model):
                    property_type = generated_models.get(property_type.name)

                if model_property.is_list:
                    property_type = List[property_type]

                keys[key] = (property_type, ...)

            generated_models[model_name] = create_model(model_name, **keys)
        return generated_models

    def is_enum(self, property: ModelProperty) -> bool:
        if property.number_of_times_seen > 20 and property.type == str and float(len(property.observed_values)) / float(property.number_of_times_seen) < .5:
            return True
        return False

    def output_models_to_package(self, package: str, module_name: str) -> None:
        dependencies = {"from pydantic import BaseModel, Field", "from typing import Optional"}
        class_string = []
        for model_name in self._get_dependencies_order(self.models.get(self.root_name)):
            model = self.models.get(model_name)
            class_string.append("\n\nclass {model_name}(BaseModel):".format(model_name=model_name))
            for key, value in model.keys.items():
                if value.number_of_times_seen == model.number_of_times_seen:
                    optional = False
                else:
                    optional = True
                if isinstance(value.type, Model):
                    type = value.type.name
                else:
                    if value.type is None:
                        type = str("Any")
                        value.type = Any
                    else:
                        if self.is_enum(value):
                            print("{key} IS AN ENUM: {values}".format(key=key, values=value.observed_values))
                        type = str(value.type.__name__)
                    module = getmodule(value.type).__name__
                    if module != "builtins":
                        dependencies.add("from {module} import {type}".format(module=module, type=type))

                if value.is_list:
                    dependencies.add("from typing import List")
                    type = "List[{type}]".format(type=type)

                if optional:
                    type = "Optional[{type}]".format(type=type)

                if not key.isidentifier() or keyword.iskeyword(key):
                    cleaned_key = re.sub('\W|^(?=\d)', '_', key)
                    if keyword.iskeyword(cleaned_key):
                        cleaned_key = cleaned_key + "_"
                    default_value = "None" if optional else "..."
                    class_string.append("    {cleaned_key}: {type} = Field({default_value}, alias=\"{key}\")".format(key=key,
                                                                                type=type,
                                                                                default_value=default_value,
                                                                                cleaned_key=cleaned_key))
                else:
                    class_string.append("    {key}: {type}".format(key=key, type=type))
        directory = os.path.join(*package.split("."))
        file = os.path.join(directory, "{module_name}.py".format(module_name=module_name))
        Path(directory).mkdir(parents=True, exist_ok=True)

        with open(file, "w") as file:
            file.write("\n".join(dependencies))
            file.write("\n")
            file.write("\n".join(class_string))


import requests
import json
import datetime
from fam_pay_main.models import VideoInformation
from django.conf import settings
from fam_pay_test.celery import app

@app.task()
def fetch_youtube_data():
	current_start_date = VideoInformation.objects.all().first()

	API_KEY_LIST = settings.API_KEY

	if current_start_date:
		current_start_date = current_start_date.published_at
	else:
		current_start_date = datetime.datetime.now() - datetime.timedelta(days=10)

	end_date = datetime.datetime.now()

	query = "football"

	kwargs = {'part': 'snippet', 'maxResults': 50, 'q': query, 'order': 'date',
	          'publishedAfter': current_start_date.strftime("%Y-%m-%dT%H:%M:%SZ"),
	          'publishedBefore': end_date.strftime("%Y-%m-%dT%H:%M:%SZ"),
	          'type': 'video'}

	index = 0

	while True:
		if index == len(API_KEY_LIST):
			break

		url = "https://www.googleapis.com/youtube/v3/search?key={}".format(API_KEY_LIST[index])
		payload={}
		headers = {}
		response = requests.request("GET", url, headers=headers, params=kwargs)
		if response.status_code == 200:
			results = json.loads(response.text)
			count = 1

			while True:
				count = count + 1
				data = results['items']
				for item in data:
					video_data = {}
					video_data['video_id'] = item['id']['videoId']
					video_data['channel_id'] = item['snippet']['channelId']
					video_data['title'] = item['snippet']['title']
					video_data['description'] = item['snippet']['description']
					video_data['published_at'] = item['snippet']['publishedAt']
					video_data['thumbnail_url'] = item['snippet']['thumbnails']['default']['url']
					VideoInformation.objects.create(**video_data)

				if results.get('nextPageToken') is None:
					break

				kwargs['pageToken'] = results['nextPageToken']
				response = requests.request("GET", url, headers=headers, params=kwargs)
				results = json.loads(response.text)
			break

		elif response.status_code == 403:
			index = index + 1


from django.db import models
from django.urls import reverse
from django.template.defaultfilters import slugify

STATUS = (
        ('ONGOING', "Ongoing"),
        ('COMPLETED', "Completed"),
    )

FEATURED = (
    (0, 'No'),
    (1, 'Everywhere'),
    (2, 'Front Page')
)
# Create your models here.
class Novel(models.Model):

    name = models.CharField(max_length=200, null=False)
    author = models.CharField(max_length=200)
    picture = models.ImageField(upload_to='static/images')
    description = models.TextField(null=True)
    genre = models.ManyToManyField('Genre', blank=True)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    status = models.CharField(max_length= 40, choices=STATUS)
    featured = models.IntegerField(choices= FEATURED, default=0)

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        slug = slugify(f"{self.name}")
        return reverse("novel", kwargs={"id": self.id, "slug": slug})

class Chapter(models.Model):
    id = models.AutoField(primary_key=True, null=False)
    novel = models.ForeignKey(Novel, on_delete=models.CASCADE, null=False)
    name = models.CharField(max_length=1000)
    chapter = models.IntegerField(null=False)
    chapter_content = models.TextField(null=True)

    slug = models.SlugField(unique=True, null=False)

    def __str__(self):
        return self.name

    def save(self, *args, **kwargs):
        self.slug = slugify(f"{self.novel.name}")
        super(Chapter, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse("chapter", kwargs={"slug": self.slug, "chapter":self.chapter})

class Genre(models.Model):
    name = models.CharField(max_length=500, null=False)
    description = models.TextField()
    created_at = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.name


import nrrd
import numpy as np
import pandas as pd
import json
from treelib import Tree

def read_nrrd(path):
    anno, _ = nrrd.read(path)
    print(anno.shape)
    return anno

def find_region(coords, annotataion, resolution):
    return annotataion[int(coords[0]/resolution), int(coords[1]/resolution), int(coords[2]/resolution)]

def find_unique_regions(annotation):
    region_list = list(np.unique(annotation))
    # if 0 in region_list:
    #     region_list.remove(0)
    return region_list

def _recover_tree(json_file, tree=Tree()):
    for node in json_file:
        if isinstance(node, dict):
            if node['parent_structure_id']==None:
                tree.create_node(identifier=node['id'], data={'acronym':node['acronym'], 'name':node['name']})
            else:
                tree.create_node(identifier=node['id'], data={'acronym':node['acronym'], 'name':node['name']},
                                 parent=node['parent_structure_id'])
            for item in node.keys():
                if isinstance(node[item], list) and len(node[item])>0:
                    _recover_tree(node[item], tree=tree)
    return tree

def allen_brain_tree(json_path):
    info = json.load(open(json_path, 'rb'))['msg']
    return _recover_tree(info, tree=Tree())

def acronym_dict(json_path):
    stl_acro_dict = {}
    tree = allen_brain_tree(json_path)
    for node in tree.nodes.keys():
        stl_acro_dict[node] = tree.nodes[node].data['acronym']
    return stl_acro_dict

def find_name_by_id(node_id, json_path):
    tree = allen_brain_tree(json_path)
    return tree.nodes[node_id].data['name']

def find_acronym_by_id(node_id, json_path):
    tree = allen_brain_tree(json_path)
    return tree.nodes[node_id].data['acronym']

def find_id_by_acronym(acro, json_path):
    tree = allen_brain_tree(json_path)
    for node in tree.nodes.keys():
        if tree.nodes[node].data['acronym'] == acro:
            return node
    return ValueError('Invalid acronym.')

def find_parent(key, layer, json_path):
    tree = allen_brain_tree(json_path)
    if isinstance(key, int):
        id = key
    elif isinstance(key, str):
        id = find_id_by_acronym(key, json_path)
    while tree.depth(id)>=layer:
        id = tree.parent(id).identifier
    if isinstance(key, int):
        return id
    elif isinstance(key, str):
        return find_acronym_by_id(id, json_path)


from typing import Any, Dict, Iterable, List, Tuple

from ..common import (
    IDENTITY_FACTORY,
    IS_NOT_CALLABLE,
    IS_STR,
    VALUE,
    Predicate,
    Transform,
)
from ..services import Scoped
from .middleware import APPLY_INTO, Context, Middleware, WithMiddlewares

class Name(Middleware, WithMiddlewares):
    @classmethod
    def verify(cls, value: Any) -> None:
        super().verify(value)

        if IS_NOT_CALLABLE(value):
            raise TypeError(f"{value} should be either a callable or a string")

    @classmethod
    def _get_simplifications(cls) -> List[Tuple[Predicate, Transform]]:
        simplifications = super()._get_simplifications()

        simplifications.append((IS_STR, IDENTITY_FACTORY))
        return simplifications

    def _init_middleware_types(self) -> None:
        super()._init_middleware_types()

        from .action import Action

        self.middleware_types.append(Action)

    def _init_middleware_apply_methods(self) -> None:
        super()._init_middleware_apply_methods()
        self.middleware_apply_methods.append(APPLY_INTO)

    def create_middlewares_from(self, value: Dict[str, Any]) -> Iterable[Middleware]:
        for middleware_type, middleware_name in zip(
            self.middleware_types, self.get_middleware_names()
        ):
            # assert middleware_type is Action
            new_value = {middleware_name: value}
            yield middleware_type(new_value)

    def _do_parse(self, value: Dict[str, Any]) -> Dict[str, List[Middleware]]:
        self.verify(value)

        return {"middlewares": list(self.create_middlewares_from(value))}

    async def get_register_name(self, context: Context) -> str:
        coroutine = WithMiddlewares.apply(self, context.replace_with_void_next())
        await coroutine()

        return context.scoped.getmagic(VALUE)

    async def _do_apply(self, context: Context) -> Scoped:
        name = await self.get_register_name(context)

        context.scoped.setmagic(NAME, name, 1)

        await context.next()
        return context.scoped

NAME = Name.get_name()


from formspawner import FormSpawner
from batchspawner import MoabSpawner
from wtforms import BooleanField, StringField, validators, IntegerField, SelectField, SelectMultipleField, Form
from wtforms.fields.html5 import EmailField

class MoabFormSpawner(FormSpawner, MoabSpawner):
    @property
    def options_form(self):
        self.form.project.choices = [('ipm-500-aa', 'ipm-500-aa')]
        return super().options_form

class MoabForm(Form):
    project   = SelectField("Project")
    runtime   = IntegerField('Runtime (hours)', [validators.NumberRange(min=1, max=10)])
    email_addr = EmailField('Email address', [validators.Email()])

c.JupyterHub.spawner_class = MoabFormSpawner
c.MoabFormSpawner.form_class = MoabForm
c.MoabFormSpawner.req_nprocs = '8'
c.MoabFormSpawner.batch_submit_cmd = 'cat'
c.MoabFormSpawner.batch_script = """#!/bin/sh
#PBS -l walltime={user_options[runtime]}:00:00
#PBS -l nodes=1:ppn={nprocs}
#PBS -N jupyterhub-singleuser
#PBS -v {keepvars}
{cmd}
"""


from pytz import utc
from django.urls import reverse
from django.test import TestCase
from tests.contexts import ActTechInfoContext
from tests.factories.gbe_factories import (
    ActFactory,
    BioFactory,
    ProfileFactory,
    TechInfoFactory,
)
from tests.functions.gbe_functions import (
    grant_privilege,
    login_as,
)
from tests.functions.scheduler_functions import get_or_create_bio
from django.core.files.uploadedfile import SimpleUploadedFile
from django.utils.formats import date_format

class TestReviewActTechInfo(TestCase):
    '''Tests for index view'''
    view_name = 'act_techinfo_detail'

    def set_the_basics(self):
        self.context.act.tech.introduction_text = "intro text"
        self.context.act.tech.feel_of_act = "feel my act"
        self.context.act.tech.read_exact = True
        self.context.act.tech.prop_setup = "[u'I have props I will need ' + \
            'set before my number', u'I will leave props or set pieces ' + \
            'on-stage that will need to be cleared']"
        self.context.act.tech.crew_instruct = "crew instruct"
        self.context.act.performer.pronouns = "these are my pronouns"
        self.context.act.tech.primary_color = "reds"
        self.context.act.tech.secondary_color = "black"
        self.context.act.tech.track = SimpleUploadedFile(
            "file.mp3",
            b"file_content")
        self.context.act.tech.save()
        self.context.act.performer.save()

    @classmethod
    def setUpTestData(cls):
        cls.context = ActTechInfoContext(schedule_rehearsal=True)
        cls.profile = ProfileFactory()
        grant_privilege(cls.profile, 'Tech Crew')
        cls.url = reverse(cls.view_name,
                          urlconf='gbe.reporting.urls',
                          args=[cls.context.act.id])

    def test_review_act_techinfo_fail(self):
        '''review_act_techinfo view should load for Tech Crew
           and fail for others
        '''
        profile = ProfileFactory()
        login_as(profile, self)
        response = self.client.get(self.url, follow=True)
        self.assertRedirects(response, reverse('home', urlconf="gbe.urls"))

    def test_review_act_techinfo_bad_act(self):
        '''review_act_techinfo view should load for Tech Crew
           and fail for others
        '''
        login_as(self.profile, self)
        response = self.client.get(reverse(self.view_name,
                                           urlconf='gbe.reporting.urls',
                                           args=[self.context.act.id+1]),
                                   follow=True)
        self.assertEqual(response.status_code, 404)

    def test_review_act_techinfo_the_basics(self):
        '''review_act_techinfo view should load for Tech Crew
           and fail for others
        '''
        self.set_the_basics()
        login_as(self.profile, self)
        response = self.client.get(self.url)
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, self.context.act.b_title)
        self.assertContains(response, str(self.context.act.performer))
        self.assertContains(response, str(
            self.context.act.performer.contact.user_object.email))
        self.assertContains(response, self.context.sched_event.title)
        self.assertContains(response, self.context.act.tech.introduction_text)
        self.assertContains(response, self.context.act.tech.feel_of_act)
        self.assertContains(response, "Read this exacty")
        self.assertContains(
            response,
            'I have props I will need set before my number')
        self.assertContains(
            response,
            'I will leave props or set pieces on-stage that will need to ' +
            'be cleared')
        self.assertContains(response, self.context.act.tech.crew_instruct)
        self.assertContains(response, self.context.act.performer.pronouns)
        self.assertContains(response, self.context.act.tech.primary_color)
        self.assertContains(response, self.context.act.tech.secondary_color)
        self.assertContains(response, self.context.act.tech.track.url)
        self.assertContains(response, 'Follow Spot: OFF
')
        self.assertContains(response, date_format(
            self.context.rehearsal.start_time,
            "DATETIME_FORMAT"))
        self.assertContains(response,
                            "Role: %s" % self.context.order.role)

    def test_review_act_techinfo_advanced(self):
        '''review_act_techinfo view should load for Tech Crew
           and fail for others
        '''
        self.set_the_basics()
        self.context.act.tech.mic_choice = "I own a mic"
        self.context.act.tech.background_color = "blue"
        self.context.act.tech.wash_color = "green"
        self.context.act.tech.follow_spot_color = "purple"
        self.context.act.tech.special_lighting_cue = "this is my cue"
        self.context.act.tech.start_blackout = True
        self.context.act.tech.end_blackout = True
        self.context.act.tech.save()

        login_as(self.profile, self)
        response = self.client.get(self.url)
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, self.context.act.tech.mic_choice)
        self.assertContains(response, self.context.act.tech.background_color)
        self.assertContains(response, self.context.act.tech.wash_color)
        self.assertContains(response, self.context.act.tech.follow_spot_color)
        self.assertContains(response,
                            self.context.act.tech.special_lighting_cue)
        self.assertContains(response, "Starting Lights: Blackout")
        self.assertContains(response, "Lighting at End: Blackout")

    def test_review_act_techinfo_incomplete(self):
        '''review_act_techinfo view should load for Tech Crew
           and fail for others
        '''
        login_as(self.profile, self)
        response = self.client.get(self.url)
        self.assertContains(response, "Act Tech is NOT complete")

    def test_review_act_techinfo_complete_w_no_rehearsal(self):
        '''review_act_techinfo view should load for Tech Crew
           and fail for others
        '''
        self.context = ActTechInfoContext()
        self.context.act.tech = TechInfoFactory(
            confirm_no_music=True,
            confirm_no_rehearsal=True,
            prop_setup="[u'I have props I will need ' + \
            'set before my number', u'I will leave props or set pieces ' + \
            'on-stage that will need to be cleared']",
            starting_position="Onstage",
            primary_color="text",
            feel_of_act="text",
            introduction_text="text")
        self.context.act.accepted = 3
        self.context.act.save()
        login_as(self.profile, self)
        response = self.client.get(reverse(self.view_name,
                                           urlconf='gbe.reporting.urls',
                                           args=[self.context.act.id]))
        self.assertContains(response, self.context.act.b_title)
        self.assertNotContains(response, "Act Tech is NOT complete")
        self.assertContains(response, "Not Attending")

    def test_withdrawn_act(self):
        act = ActFactory(accepted=4)
        login_as(self.profile, self)
        response = self.client.get(reverse(self.view_name,
                                           urlconf='gbe.reporting.urls',
                                           args=[act.id]))
        self.assertContains(response, "Act state is Withdrawn")

    def test_review_act_techinfo_order(self):
        '''review_act_techinfo view should load for Tech Crew
           and fail for others
        '''
        self.context.order_act(self.context.act, "3")
        login_as(self.profile, self)
        response = self.client.get(self.url)
        self.assertContains(response, "3

Show Order
")

    def test_review_act_techinfo_troupe_member(self):
        '''review_act_techinfo view should load for Tech Crew
           and fail for others
        '''
        troupe = BioFactory(multiple_performers=True)
        people = get_or_create_bio(troupe)
        member = ProfileFactory()
        people.users.add(member.user_object)
        self.context.act.bio = troupe
        self.context.act.save()
        people.commitment_class_name = self.context.act.__class__.__name__
        people.commitment_class_id = self.context.act.pk
        people.save()
        self.set_the_basics()
        login_as(member, self)
        response = self.client.get(self.url)
        self.assertContains(response, self.context.act.b_title)
        self.assertContains(response, str(self.context.act.bio))
        self.assertContains(response, self.context.sched_event.title)
        self.assertContains(response, self.context.act.tech.introduction_text)


################################################################################
# Geocode data using Google and GeoNames
# Benjamin P. Stewart, May, 2013
# Purpose: Geocoding is something we need to do frequently, so this script will do that.
#   It leverages GeoNames (http://www.geonames.org/) primarily, and then tries to use
#   Google if GeoNames does not use. The reason for this is the request limit for Google
####  Completed by Hamza Haloui, September 2018
################################################################################

import urllib, xlrd, unicodedata, time, json

###USER Variables
inputExcelDocument = r"C:\Users\WB512163\Documents\test.xls"
outputDoc = open(r"C:\Users\WB512163\Documents\test_geocoded.tsv", 'w')
#Define the columns in the inputExcelDocument that describe the place to geocode.
#   The values will be separated by a comma to create a single string
locColumns = [0,0]
sheetIdx = 0 #Zer0-Based!!! If the data is not on the first sheet, you can change it here

#This bit makes the function emulate a firefox request...I really don't know if it is necessary
userAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 GTB7.1 (.NET CLR 3.5.30729)"
headers = { 'User-Agent' : userAgent }
values = { 's' : 'nothing' }
data = urllib.parse.urlencode(values).encode('utf8')

#Open the excel input dictionary, get the first sheet
dataDict = xlrd.open_workbook(inputExcelDocument)
projects = dataDict.sheet_by_index(sheetIdx)
outputDoc.write("OBJECTID\tPostal\tLat\tLng\n")
print("There are %s projects to geocode" % projects.nrows)

#Wrapper for the geo location tools
def geoCode(self, data, headers):
    geogLoc = getGeogLocation(self.rawLocation, data, headers)
    return(geogLoc)

def getJSONresponse(url, data, headers):
    urlReq = urllib.request.Request(url, data, headers)
    req = urllib.request.urlopen(urlReq)
    return(json.loads(req.read()))

# This is where the geolocation action actually happens
def getGeogLocation (location, data, headers):
    location2 = urllib.parse.quote(location) # get the internet version of the location string
    #Perform the geonames.org geolocation
    geoNamesUrl = "http://api.geonames.org/searchJSON?q=" + location2 + "&maxRows=3&username=petie_stewart&featureClass=P"
    jsonResponse = getJSONresponse(geoNamesUrl, data, headers)
    ret = dict(lat=-999, lng=-999)
    # If the geonames result has a length
    geoCodeSource = "nothing"
    if (jsonResponse['totalResultsCount'] > 0):
        res = jsonResponse['geonames'][0]
        ret = dict(lat=res['lat'], lng=res['lng'])
        geoCodeSource = "geonames"
    else:
        googleUrl = "http://maps.googleapis.com/maps/api/geocode/json?address=" + location2 + "&sensor=false"
        jsonResponse = getJSONresponse(googleUrl, data, headers)
        if (jsonResponse['status'] == "OK"):
            res = jsonResponse['results'][0]['geometry']['location']
            ret = dict(lat=res['lat'], lng=res['lng'])
            geoCodeSource = "google"
        else:
            print(location + " was not found")
    return({'location':ret, 'source':geoCodeSource})

class projectLocation:
    def __init__(self, curProject, columns):
        try:
            locString = ",".join([str(curProject[k]) for k in columns])
            self.rawLocation = locString #unicodedata.normalize('NFKD', locString)
        except:
            self.source = "StringError"
            self.geogLoc = dict(lat=-999, lng=-999)
            self.rawLocation = ''
        geogCodeResult = geoCode(self, data, headers)
        self.geogLoc = geogCodeResult['location']
        self.source = geogCodeResult['source']

googleCnt = 0
geoNames = 0
missed = 0

for rownum in range(1, projects.nrows):
    curProject = projects.row_values(rownum)
    if rownum < 5:
        print(curProject)
    projLocation = projectLocation(curProject, locColumns)
    if projLocation.source == "StringError":
        print("Error with row %s" % rownum)
        print(curProject)
    geoNames = (geoNames + 1 if projLocation.source == "geonames" else geoNames)
    googleCnt = (googleCnt + 1 if projLocation.source == "google" else googleCnt)
    missed = (missed + 1 if projLocation.source == "nothing" else missed)
    outputDoc.write("%s\t%s\t%s\t%s\t%s\n" % (rownum, projLocation.rawLocation, projLocation.geogLoc['lat'], projLocation.geogLoc['lng'], projLocation.source))
    time.sleep(1)

outputDoc.close()
print("Of %s projects, %s were found in GeoNames, %s were found in Google, %s were not found" %(projects.nrows, geoNames, googleCnt, missed))


import os
from flask import Flask, request
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = os.getenv("MYSQL_URL")
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

db = SQLAlchemy(app)

from controller.user.UserController import *
from controller.userdata.controller import *
from middleware.user.UserMiddleware import *

# route for user data
@app.route('/list-user-data', methods=['GET'])
@UserMiddleware
def userDataList():
    controller = UserController(request)
    return controller.ListUserData()

@app.route('/insert-user-data', methods=['POST'])
@UserMiddleware
def addUserData():
    controller = UserController(request)
    return controller.AddUserData()

@app.route('/update-user-data/<int:id>', methods=['PUT'])
@UserMiddleware
def updateUserData(id):
    controller = UserController(request)
    return controller.UpdateUserData(id)

@app.route('/delete-user-data/<int:id>', methods=['DELETE'])
@UserMiddleware
def deleteUserData(id):
    controller = UserController(request)
    return controller.DeleteUserData(id)

@app.route('/login', methods=['POST'])
def login():
    controller = UserDataController(request)
    return controller.login()

@app.route('/insert-user-profile', methods=['POST'])
@UserMiddleware
def addUserProfile():
    controller = UserDataController(request)
    return controller.addUserProfileData()

if __name__ == "__main__":
    app.run(debug=True, host='0.0.0.0', port=8080)


import json
from flask import request
import bson
import pymongo
from loguru import logger
from bson import json_util
from flasgger import swag_from
from flask_restful import Resource

from luizalabs_project.modules.databases import CustomerDB
from luizalabs_project.api.api_utils import Response, verify_auth

class CustomersListAPI(Resource):

    @swag_from('docs/customersListAPI_get.yml')
    @verify_auth
    def get(self):
        logger.info("Recovering information from all customer.")

        try:
            database = CustomerDB()

            return json.loads(json_util.dumps(database.show_all()))
        except (bson.errors.InvalidId, pymongo.errors.OperationFailure):
            logger.exception("Database Failure !!!")

            return Response.error()

        return Response.error()

class CustomerListAPI(Resource):
    @swag_from('docs/customerListAPI_get.yml')
    @verify_auth
    def get(self, customer_id):
        logger.info("Recovering information from a customer.")

        try:
            database = CustomerDB()

            return json.loads(json_util.dumps(database.show_one(customer_id)))
        except (bson.errors.InvalidId, pymongo.errors.OperationFailure):
            logger.exception("Database Failure !!!")

            return Response.error()

        return Response.error()

class CustomerAPI(Resource):

    @swag_from('docs/customerAPI_post.yml')
    @verify_auth
    def post(self):
        logger.info("Adding new customer.")

        try:
            req = request.json
            name, email = req['name'], req['email']

        except (KeyError, TypeError):
            logger.exception("Parameters error !!!")

            return Response.parameters_error()

        try:
            database = CustomerDB()

            result = database.add(name, email)

            if result:
                return Response.custom_success(
                    dict(message="Customer added with success")
                )

            return Response.custom_error(dict(message="Email already in use"))

        except pymongo.errors.OperationFailure:
            logger.exception("Database Failure !!!")

            return Response.error()

        return Response.error()

    @swag_from('docs/customerAPI_put.yml')
    @verify_auth
    def put(self):
        logger.info("Updating a customer.")

        try:
            req = request.json
            customer_id = req['customer_id']
            name = req['name'] if 'name' in req.keys() else None
            email = req['email'] if 'email' in req.keys() else None

            if (not name) and (not email):
                return Response.custom_error(
                    dict(message='Please, update need name or '
                         'email as parameters'))

        except (KeyError, TypeError):
            logger.exception("Parameters error !!!")

            return Response.parameters_error()

        try:
            database = CustomerDB()

            result = database.update(customer_id, name, email)

            if result:
                return Response.custom_success(
                    dict(message="Customer updated with success")
                )

            return Response.custom_error(dict(message="Couldn't update "
                                              "the user, email already in use")
                                         )

        except pymongo.errors.OperationFailure:
            logger.exception("Database Failure !!!")

            return Response.error()

        return Response.error()

    @swag_from('docs/customerAPI_delete.yml')
    @verify_auth
    def delete(self):
        logger.info("Deleting customer.")

        try:
            req = request.json

            customer_id = req['customer_id']

        except (KeyError, TypeError):
            logger.exception("Parameters error !!!")

            return Response.parameters_error()

        try:
            database = CustomerDB()
            result = database.remove(customer_id)

            if result:
                return Response.custom_success(
                    dict(message="Customer removed with success")
                )

            return Response.custom_error(dict(message="Couldn't remove "
                                              "the user")
                                         )

        except pymongo.errors.OperationFailure:
            logger.exception("Database Failure !!!")

            return Response.error()

        return Response.error()


import torch
from torch.utils.data import Dataset
from my_imfolder import ImageFolder
from torchvision.transforms import ToTensor
from torch.autograd import Variable
import torch.autograd as autograd
import numpy as np
import argparse
import cv2
from tqdm import tqdm
from model import GeneratorNet, DiscriminatorNet

def set_requires_grad(net, switch):
    for param in net.parameters():
        param.requires_grad = switch
    return

def to_var(x, requires_grad=True):
    if torch.cuda.is_available():
        x = x.cuda()
    return Variable(x, requires_grad= requires_grad)

def dcgan_train(data_loader, generator, discriminator, optimizer_G, optimizer_D, criterion, epoch):
    Tensor = torch.FloatTensor
    for e in range(epoch):
        desc = 'Epoch {}'.format(e)
        d_losses = []
        g_losses = []
        t = tqdm(data_loader)
        for i, im in enumerate(t):
            optimizer_G.zero_grad()
            optimizer_D.zero_grad()
            batch_size = im[0].size(0)
            #valid_label = to_var(Tensor(batch_size, 1).fill_(1.0), requires_grad=False)
            valid_label = to_var(Tensor(np.random.uniform(0.7, 1.2, batch_size)), requires_grad=False)
            fake_valid_label = to_var(Tensor(round(batch_size), 1).fill_(1.0), requires_grad=False)
            #fake_label = to_var(Tensor(round(batch_size), 1).fill_(0.0), requires_grad=False)
            fake_label = to_var(Tensor(np.random.uniform(0, 0.3, batch_size)), requires_grad=False)

            # Create Fake Image
            set_requires_grad(discriminator, False)
            noise = to_var(Tensor(np.random.normal(0, 1, (round(batch_size), opts.latent_size))))
            im_fake = generator(noise)
            g_loss = criterion(discriminator(im_fake), fake_valid_label)
            g_losses.append(g_loss.data[0])
            #print('g_loss = {}'.format(g_loss.data))
            if (i+1) % 5 == 0 or e > 30:
                g_loss.backward()
                optimizer_G.step()

            # Discriminate Valid Image
            set_requires_grad(discriminator, True)
            im_valid = to_var(im[0])
            valid_loss = criterion(discriminator(im_valid), valid_label)
            #print('valid_loss = {}'.format(valid_loss))
            fake_loss = criterion(discriminator(im_fake.detach()), fake_label)
            #print('fake_loss = {}'.format(fake_loss))
            d_loss = (valid_loss+fake_loss)/2
            d_losses.append(d_loss.data[0])
            d_loss.backward()
            optimizer_D.step()
            t.set_description('Epoch{},g_loss={:.4f}'.format(e+1, g_loss.data[0]))
        im_fake_output = np.clip(im_fake.cpu().data.numpy(), 0, 1)
        im_fake_output0 = np.round(im_fake_output[0]*255)
        im_fake_output1 = np.round(im_fake_output[1]*255)
        im_fake_output2 = np.round(im_fake_output[2]*255)
        im_fake_output = np.transpose(im_fake_output0 , (1, 2, 0))
        im_fake_output1 = np.transpose(im_fake_output1 , (1, 2, 0))
        im_fake_output2 = np.transpose(im_fake_output2 , (1, 2, 0))
        print(im_fake_output.shape)

        torch.save(generator.state_dict(), 'model/generator_epoch{}.pt'.format(e+1))
        torch.save(discriminator.state_dict(), 'model/discriminator_epoch{}.pt'.format(e+1))
        cv2.imwrite('epoch{}-{}.jpg'.format(e+1, 0), im_fake_output)
        cv2.imwrite('epoch{}-{}.jpg'.format(e+1, 1), im_fake_output1)
        cv2.imwrite('epoch{}-{}.jpg'.format(e+1, 2), im_fake_output2)
        print('d_loss = {}'.format(np.average(d_losses)))
        print('g_loss = {}'.format(np.average(g_losses)))

def compute_gradient_penalty(D, real_samples, fake_samples):
    # Random weight term for interpolation between real and fake samples
    Tensor = torch.FloatTensor
    if torch.cuda.is_available():
        Tensor = torch.cuda.FloatTensor
    alpha = Tensor(np.random.random((real_samples.size(0), 1, 1, 1)))
    # Get random interpolation between real and fake samples
    interpolates = (alpha * real_samples + ((1 - alpha) * fake_samples)).requires_grad_(True)
    d_interpolates = D(interpolates)
    fake = Variable(Tensor(real_samples.shape[0], 1).fill_(1.0), requires_grad=False)
    # Get gradient w.r.t. interpolates
    gradients = autograd.grad(outputs=d_interpolates, inputs=interpolates,
                              grad_outputs=fake, create_graph=True, retain_graph=True,
                              only_inputs=True)[0]
    gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
    return gradient_penalty

def wgan_train(data_loader, generator, discriminator, optimizer_G, optimizer_D, epoch, clip_val=0.01, LAMBDA=0.1, use_GP=False):
    Tensor = torch.FloatTensor
    for e in range(epoch):
        desc = 'Epoch {}'.format(e)
        for i, im in enumerate(tqdm(data_loader, desc= desc)):
            optimizer_G.zero_grad()
            optimizer_D.zero_grad()
            batch_size = im[0].size(0)
            im_valid = to_var(im[0])

            noise = to_var(Tensor(np.random.normal(0, 1, (batch_size, opts.latent_size))))
            set_requires_grad(discriminator, False)
            im_fake = generator(noise)
            g_loss = -torch.mean(discriminator(im_fake))
            if (i+1) % 7 == 0 or e > 30:
                g_loss.backward()
                optimizer_G.step()

            set_requires_grad(discriminator, True)
            if use_GP:
                gradient_penalty = compute_gradient_penalty(discriminator, im_valid.data, im_fake.data)
                d_loss = -torch.mean(discriminator(im_valid)) + torch.mean(discriminator(im_fake.detach())) + LAMBDA * gradient_penalty
            else:
                d_loss = -torch.mean(discriminator(im_valid)) + torch.mean(discriminator(im_fake.detach()))
            d_loss.backward()
            optimizer_D.step()
            #Weight Clipping
            if not use_GP:
                for param in discriminator.parameters():
                    param.data.clamp_(-clip_val, clip_val)
        im_fake_output = np.clip(im_fake.cpu().data.numpy(), 0, 1)
        im_fake_output = np.round(im_fake_output[0]*255)
        im_fake_output = np.transpose(im_fake_output , (1, 2, 0))
        cv2.imwrite('wgan_epoch{}.jpg'.format(e+1), im_fake_output)
        torch.save(generator.state_dict(), 'model/wgan/generator_epoch{}.pt'.format(e+1))

def main(opts):
    CHANNEL = 3
    mode = 'RGB'
    if opts.use_gray:
        CHANNEL = 1
        mode = 'L'
    im_folder = ImageFolder(root=opts.input, transform= ToTensor(), mode= mode)

    data_loader = torch.utils.data.DataLoader(im_folder,
                                              batch_size=opts.batch_size,
                                              shuffle=True)
    USE_WGAN = opts.use_wgan

    discriminator = DiscriminatorNet(use_WGAN=USE_WGAN, channel=CHANNEL)
    generator = GeneratorNet(input_size=100, channel= CHANNEL)
    criterion_dcgan = torch.nn.BCELoss()

    if torch.cuda.is_available():
        print('Using CUDA')
        discriminator.cuda()
        generator.cuda()
    optimizer_G = torch.optim.Adam(generator.parameters(), lr= 0.0002)
    #optimizer_G = torch.optim.RMSprop(generator.parameters(), lr= 0.0002)
    optimizer_D = torch.optim.Adam(discriminator.parameters(), lr= 0.0002)
    #optimizer_D = torch.optim.RMSprop(discriminator.parameters(), lr= 0.0002)

    if opts.use_wgan or opts.use_wgan_gp:
        print('WGAN Training')
        wgan_train(data_loader,generator,discriminator,optimizer_G,optimizer_D, opts.epoch, use_GP=opts.use_wgan_gp)
    else:
        print('DCGAN Training')
        dcgan_train(data_loader,generator,discriminator,optimizer_G,optimizer_D, criterion_dcgan, opts.epoch)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', type=str, default= 'resized_data', dest='input', help='Animate Face Input File Path')
    parser.add_argument('-b', type=int, default= 128, dest='batch_size', help='Batch Number')
    parser.add_argument('-l', type=int, default= 100, dest='latent_size', help='Latent Number')
    parser.add_argument('-e', type=int, default= 1, dest='epoch', help='Epoch Number')
    parser.add_argument('-WGAN', action= 'store_true', default= False, dest= 'use_wgan', help='use WGAN Weight Clipping')
    parser.add_argument('-WGAN-GP', action= 'store_true', default= False, dest= 'use_wgan_gp', help='use WGAN Weight Clipping')
    parser.add_argument('-GRAY', action= 'store_true', default= False, dest='use_gray', help='use Gray Scale Data Set')
    opts = parser.parse_args()
    main(opts)


"""
https://contest.yandex.ru/contest/24810/run-report/77259556/

In this task, it is necessary to implement heap sort. In this case, the heap must be implemented independently;
it is impossible to use the implementations available in the language.
First, it is recommended to solve problems about sifting down and up.

Timofey decided to organize a sports programming competition to find talented interns.
Tasks are selected, participants are registered, tests are written.
It remains to figure out how the winner will be determined at the end of the competition.

Each participant has a unique login. When the competition ends, two indicators will be attached to it:
the number of solved problems Pi and the size of the penalty Fi.
The penalty is calculated for unsuccessful attempts and time spent on the task.

Timofey decided to sort the table of results in the following way: when comparing two participants,
the one with more problems solved will go higher. If the number of solved problems is equal,
the participant with the lowest penalty goes first. If the penalties are the same, then the first
one will be the one whose login comes earlier in alphabetical (lexicographical) order.

Timofey ordered sweatshirts for the winners and went to the store to pick them up the day before.
In his absence, he commissioned you to implement a Heapsort algorithm for the results table.

Input Example:
5
alla 4 100
gena 6 1000
gosha 2 90
rita 2 90
timofey 4 80

Output Example:
gena
timofey
alla
gosha
rita

Time Complexity: O(nlogn)
Space Complexity:O(logn)
"""

from dataclasses import dataclass

@dataclass
class User:
    """
    We are using here dataclass  which helps us to generate class code.
    More info - https://habr.com/ru/post/415829/

    When we create objects for classes, it requires memory and the attribute are stored in the form of a dictionary.
    In case if we need to allocate thousands of objects, it will take a lot of memory space.
    slots provide a special mechanism to reduce the size of objects.It is a concept of memory optimisation on objects.
    More info - https://wiki.python.org/moin/UsingSlots
    """

    __slots__ = ['username', 'solved', 'errors']
    username: str
    solved: int
    errors: int

    def key(self):
        return -self.solved, self.errors, self.username

    def __gt__(self, other):
        # "Greater than" - means below in the list.
        #  This user has less Pi, greater Fi, and lesser alphabetical name.

        return self.key() > other.key()

    def __lt__(self, other):
        # "Lesser than" - means higher in the list.
        #  This user has greater Pi, less Fi, and greater alphabetical name.

        return self.key() < other.key()

def heapify(arr, n, i):
    # Find largest among root and children
    largest = i
    left = 2 * i + 1
    right = 2 * i + 2

    if left < n and arr[i] < arr[left]:
        largest = left

    if right < n and arr[largest] < arr[right]:
        largest = right

    # If root is not largest, swap with largest and continue heapifying
    if largest != i:
        arr[i], arr[largest] = arr[largest], arr[i]
        heapify(arr, n, largest)

def heapsort(arr: list):
    n = len(arr)

    for user in range(n // 2, -1, -1):
        # To build a max-heap from any tree, we can thus start heapifying each sub-tree from the bottom up and
        # end up with a max-heap after the function is applied to all the elements including the root element.
        # In the case of a complete tree, the first index of a non-leaf node is given by n/2 - 1.
        # All other nodes after that are leaf-nodes and thus don't need to be heapified.

        heapify(arr, n, user)

    for user in range(n - 1, 0, -1):
        # Remove the root element and put at the end of the array (nth position)
        # Put the last item of the tree (heap) at the vacant place.
        arr[user], arr[0] = arr[0], arr[user]

        # Heapify the root element again so that we have the highest element at root.
        heapify(arr, user, 0)

def read_input_and_create_users():
    n = int(input())
    users = []

    for _ in range(n):
        username, solved, errors = input().split()
        users.append(User(username, int(solved), int(errors)))

    return users

if __name__ == '__main__':
    users = read_input_and_create_users()
    heapsort(users)

    for user in users:
        print(user.username)


from glob import glob
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib

matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42

sns.set_style("ticks")
sns.set_palette("hls", 8)

RUN = 100
GENS = 1000

run_directory = "/home/sam/Archive/skriegma/XENO_3/run_{}/".format(RUN)
all_of_gen_files = glob(run_directory + "allIndividualsData/Gen_*.txt")
sorted_all_of_gen_files = sorted(all_of_gen_files, reverse=False)

line_hist = []
gen_age_fit_dict = {}

for gen in range(GENS+1):

    gen_age_fit_dict[gen] = {0: 0.0}

    with open(sorted_all_of_gen_files[gen], 'r') as infile:

        next(infile)  # skip header

        for line in infile:
            # this_gen = int(line.split()[0])
            # print this_gen == gen
            # this_id = int(line.split()[1])
            # parent_id = int(line.split()[3])
            # variation_type = line.split()[4]
            this_fit = float(line.split()[5])
            this_age = int(line.split()[7])

            if this_age not in gen_age_fit_dict[gen] or this_fit > gen_age_fit_dict[gen][this_age]:  # ord by fit anyway
                gen_age_fit_dict[gen][this_age] = this_fit  # most fit at each age level

            # if gen > 0 and this_age > 0 and gen_age_fit_dict[gen-1][this_age-1] > gen_age_fit_dict[gen][this_age]:
            #     print "asdf"

    if gen > 0:

        for age in gen_age_fit_dict[gen-1]:

            if age+1 not in gen_age_fit_dict[gen] or gen == 1000:  # extinction

                this_line = []
                n = 0
                while age-n > -1:
                    this_line += [gen_age_fit_dict[gen-1-n][age-n] * 6 / 8.0]
                    n += 1

                # if len(this_line) > 1 and this_line[1] > this_line[0]:
                #     print this_line

                pre_fill = [None]*(gen-age)
                # post_fill = [None]*(GENS-gen)
                line_hist += [pre_fill + list(this_line[::-1])]

fig, ax = plt.subplots(1, 1, figsize=(4, 3))

for line in line_hist:
    ax.plot(range(len(line)), line, linewidth=0.8)

ax.set_xlim([-10, GENS])
ax.set_ylim([-0.1, 8])
ax.text(50, 7, "Run {}".format(RUN), fontsize=20)
# ax.set_ylabel('Displacement')
# ax.set_xlabel('Generation')
# ax.set_xticklabels([0, 0, 200/50, 400/50, 600/50, 800/50, 1000/50], fontsize=12)
ax.set_xticklabels([0, 0, 200, 400, 600, 800, 1000], fontsize=12)
ax.set_yticklabels([0, 0, 2, 4, 6, 8], fontsize=12)

sns.despine()
plt.tight_layout()
plt.savefig("plots/XENO_3_rainbow_waterfall_run_{}.png".format(RUN), bbox_inches='tight', dpi=600)


from os import system, name, remove, mkdir, rmdir, listdir, environ
from os.path import exists
from sys import argv

import sqlite3

def clearConsole():
    system("cls" if name in ("nt", "dos") else "clear")

def create_config(token, main_admin_id, config_path="config.ini"):
    DEFAULT_CONFIG_TEXT = f"""[main_settings]
token = {token}
mainadminid = {main_admin_id}
debug = 0

[shop_settings]
name = Название магазина
greeting = Добро пожаловать!
refundpolicy = Текст для вкладки "Политика возврата"
contacts = Текст для вкладки "Контакты"
enableimage = 1
enablesticker = 0
enablephonenumber = 0
enabledelivery = 0
delivery_price = 0.0
enablecaptcha = 1

[stats_settings]
barcolor = 3299ff
borderwidth = 1
titlefontsize = 20
axisfontsize = 12
tickfontsize = 8
"""
    with open(config_path, "w") as config:
        config.write(DEFAULT_CONFIG_TEXT)

CREATE_CATS_TEXT = """
CREATE TABLE "cats" (
	"id" INTEGER,
	"name" TEXT NOT NULL,
	PRIMARY KEY("id")
)
"""
CREATE_ITEMS_TEXT = """
CREATE TABLE "items" (
	"id" INTEGER,
	"name" TEXT NOT NULL,
	"price" FLOAT NOT NULL,
	"cat_id" INTEGER NOT NULL,
	"desc" TEXT,
	"active" INTEGER,
	"amount" INTEGER,
	"image_id" INTEGER,
    "hide_image" INTEGER,
	PRIMARY KEY("id")
)
"""
CREATE_ORDERS_TEXT = """
CREATE TABLE "orders" (
	"order_id" INTEGER,
	"user_id" INTEGER,
	"item_list" TEXT,
	"email_adress" TEXT,
	"phone_number" TEXT,
	"home_adress" TEXT,
	"additional_message" TEXT,
	"date" TEXT,
    "status" INTEGER
    )
"""
CREATE_USERS_TEXT = """
CREATE TABLE "users" (
	"user_id" INTEGER NOT NULL,
	"is_admin" INTEGER,
	"is_manager" INTEGER,
	"notification" INTEGER,
	"date_created" TEXT,
    "cart" TEXT,
    "cart_delivery" INTEGER
)
"""
CREATE_COMMANDS_TEXT = """
CREATE TABLE "commands" (
    "id" INTEGER NOT NULL,
    "command" TEXT,
    "response" TEXT,
    PRIMARY KEY("id")
)
"""

def create_db():
    conn = sqlite3.connect("data.db")
    c = conn.cursor()
    c.execute(CREATE_CATS_TEXT)
    c.execute(CREATE_ITEMS_TEXT)
    c.execute(CREATE_ORDERS_TEXT)
    c.execute(CREATE_USERS_TEXT)
    c.execute(CREATE_COMMANDS_TEXT)
    conn.commit()
    conn.close()

def do_files_exist():
    return any(list(map(exists, ["config.ini", "images", "data.db"])))

if __name__ == "__main__":
    if "--nointeract" in argv:
        if do_files_exist():
            exit(0)

        token = environ.get("TELEGRAM_TOKEN")
        main_admin_id = environ.get("MAIN_ADMIN_ID")
        if token is None or main_admin_id is None:
            print("Не указаны переменные окружения TELEGRAM_TOKEN или MAIN_ADMIN_ID")
            exit(1)
        create_config(token, main_admin_id)

        create_db()
        [mkdir(name) for name in ["backups", "images"]]

        exit(0)

    clearConsole()
    if do_files_exist():
        while True:
            confirmation = input("Вы уверены, что хотите повторно запустить процесс установки? Все данные будут утеряны! (y/N) ")
            if confirmation.lower() in ["y", "yes", "n", "no", ""]:
                break
    else:
        confirmation = "y"

    if confirmation.lower() in ["y", "yes"]:
        print("Вы можете узнать как получить токен бота, перейдя по ссылке: https://youtu.be/fyISLEvzIec")
        token = input("Введите токен бота: ")
        print("Вы можете получить ваш ID, написав \"/start\" боту @userinfobot")
        main_admin_id = input("Введите ID главного администратора: ")
        if main_admin_id.isalnum():
            if exists("data.db"):
                remove("data.db")
                print("База данных была удалена.")
            create_db()
            print("База данных была создана.")
            if exists("config.ini"):
                remove("config.ini")
                print("Файл настроек был удален.")
            create_config(token, main_admin_id)
            print("Файл настроек был создан.")
            if exists("images"):
                for file in listdir("images"):
                    remove("images/" + file)
                rmdir("images")
                print("Папка \"images\" была удалена.")
            mkdir("images")
            print("Папка \"images\" была создана.")
            if exists("backups"):
                for folder in listdir("backups"):
                    for file in listdir("backups/" + folder):
                        remove(f"backups/{folder}/{file}")
                    rmdir(f"backups/{folder}")
                rmdir("backups")
                print("Папка \"backups\" была удалена.")
            mkdir("backups")
            print("Папка \"backups\" была создана.")
        else:
            print("Неверный ID главного администратора.")
    else:
        print("Установка была отменена.")

    input("Нажмите ENTER, чтобы продолжить...")


# -*- coding: utf-8 -*-
"""
author SparkByExamples.com
"""

from pyspark.sql import SparkSession
from pyspark.sql.types import StringType, ArrayType,StructType,StructField
spark = SparkSession.builder \
                    .appName('SparkByExamples.com') \
                    .getOrCreate()

arrayCol = ArrayType(StringType(),False)

data = [
 ("James,,Smith",["Java","Scala","C++"],["Spark","Java"],"OH","CA"),
 ("Michael,Rose,",["Spark","Java","C++"],["Spark","Java"],"NY","NJ"),
 ("Robert,,Williams",["CSharp","VB"],["Spark","Python"],"UT","NV")
]

schema = StructType([
    StructField("name",StringType(),True),
    StructField("languagesAtSchool",ArrayType(StringType()),True),
    StructField("languagesAtWork",ArrayType(StringType()),True),
    StructField("currentState", StringType(), True),
    StructField("previousState", StringType(), True)
  ])

df = spark.createDataFrame(data=data,schema=schema)
df.printSchema()
df.show()

from pyspark.sql.functions import explode
df.select(df.name,explode(df.languagesAtSchool)).show()

from pyspark.sql.functions import split
df.select(split(df.name,",").alias("nameAsArray")).show()

from pyspark.sql.functions import array
df.select(df.name,array(df.currentState,df.previousState).alias("States")).show()

from pyspark.sql.functions import array_contains
df.select(df.name,array_contains(df.languagesAtSchool,"Java")
    .alias("array_contains")).show()


import typer
from .oracle import Oracle
from .extractive import Extractive, ExtractivePreAbstractive
from .abstractive import Abstractive
from datasets import load_dataset, load_metric
from transformers import AutoTokenizer
from nltk.tokenize import sent_tokenize, word_tokenize
import numpy as np

app = typer.Typer()

@app.command()
def extractive_oracle(data_file: str, output_file: str):
    dataset = load_dataset("json", data_files={"split": data_file}, split="split")
    oracle = Oracle()
    new_dataset = oracle.add_oracle_sources_text_to_dataset(dataset)
    new_dataset.to_json(output_file)
    typer.secho("Done", fg=typer.colors.GREEN, bold=True)

@app.command()
def extractive_bert(
    data_file: str,
    output_file: str,
    model_checkpoint: str = "distilbert-base-multilingual-cased",
    pre_abstractive: bool = False,
    abstractive_model_checkpoint: str = "google/mt5-small",
):
    dataset = load_dataset("json", data_files={"split": data_file}, split="split")
    ext = (
        Extractive(model_checkpoint=model_checkpoint)
        if not pre_abstractive
        else ExtractivePreAbstractive(
            extractive_model_checkpoint=model_checkpoint,
            abstractive_model_checkpoint=abstractive_model_checkpoint,
        )
    )
    typer.echo(f"Type of Extractive: {type(ext)}")
    new_dataset = ext.add_extractive_summary_to_dataset(dataset)
    new_dataset.to_json(output_file)
    typer.secho("Done", fg=typer.colors.GREEN, bold=True)

@app.command()
def abstractive_mt5(
    data_file: str,
    output_file: str,
    model_checkpoint: str,
    device: int = -1,
    sources_text: str = "oracle_sources_text",
    min_length: int = 200,
    max_length: int = 512,
    num_beams: int = 5,
    no_repeat_ngram_size: int = 3,
):
    dataset = load_dataset("json", data_files={"split": data_file}, split="split")
    abs = Abstractive(
        model_checkpoint=model_checkpoint,
        device=device,
        sources_text=sources_text,
        min_length=min_length,
        max_length=max_length,
        num_beams=num_beams,
        no_repeat_ngram_size=no_repeat_ngram_size,
    )
    new_dataset = abs.add_abstractive_summary_to_dataset(dataset)
    new_dataset.to_json(output_file)
    typer.secho("Done", fg=typer.colors.GREEN, bold=True)

@app.command()
def tokenize(
    data_file: str,
    output_file: str,
    max_input_length: int = 512,
    max_target_length: int = 512,
    model_checkpoint: str = "google/mt5-small",
    source: str = "oracle_sources_text",
    target: str = "news",
):
    dataset = load_dataset("json", data_files={"split": data_file}, split="split")

    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)

    def preprocess_function(examples):
        model_inputs = tokenizer(
            examples[source], max_length=max_input_length, truncation=True
        )
        # Set up the tokenizer for targets
        with tokenizer.as_target_tokenizer():
            labels = tokenizer(
                examples[target], max_length=max_target_length, truncation=True
            )

        model_inputs["labels"] = labels["input_ids"]
        return model_inputs

    tokenized_dataset = dataset.map(preprocess_function, batched=True)
    tokenized_dataset.to_json(output_file)
    typer.secho("Done", fg=typer.colors.GREEN, bold=True)

@app.command()
def stats(
    data_file: str,
    output_file: str,
    model_checkpoint: str = "google/mt5-small",
    sources: str = "sources",
    sources_text: str = "sources_text",
    target: str = "news",
):
    dataset = load_dataset("json", data_files={"split": data_file}, split="split")

    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)

    rouge_score = load_metric("rouge")

    def stats_function(example):
        example_sources = example[sources]
        example_sources_text = example[sources_text]
        example_target = example[target]
        nb_of_sources = len(example_sources)
        nb_of_words_sources = len(word_tokenize(example_sources_text))
        sources_text_sent_tokenized = sent_tokenize(example_sources_text)
        nb_of_sentences_sources = len(sources_text_sent_tokenized)
        nb_of_tokens_sources = len(tokenizer(example_sources_text)["input_ids"])
        nb_of_words_target = len(word_tokenize(example_target))
        target_sent_tokenized = sent_tokenize(example_target)
        nb_of_sentences_target = len(target_sent_tokenized)
        nb_of_tokens_target = len(tokenizer(example_target)["input_ids"])

        rouge_score_sources = rouge_score.compute(
            references=["\n".join(target_sent_tokenized)],
            predictions=["\n".join(sources_text_sent_tokenized)],
        )

        if "stats" not in example.keys():
            example["stats"] = {}
        example["stats"][f"{sources_text}"] = {
            "nb": nb_of_sources,
            "nb_sentences": nb_of_sentences_sources,
            "nb_words": nb_of_words_sources,
            "nb_tokens": nb_of_tokens_sources,
            "rouge_score": rouge_score_sources,
        }
        example["stats"][f"{target}"] = {
            "nb_sentences": nb_of_sentences_target,
            "nb_words": nb_of_words_target,
            "nb_tokens": nb_of_tokens_target,
        }
        return example

    new_dataset = dataset.map(stats_function)
    new_dataset.to_json(output_file)
    typer.secho("Done", fg=typer.colors.GREEN, bold=True)

@app.command()
def print_stats(data_file: str, language: str = None):
    dataset = load_dataset("json", data_files={"split": data_file}, split="split")

    if language:

        def filter_by_language(example):
            return example["language"] == language

        dataset = dataset.filter(filter_by_language)

    stats = dataset["stats"]
    results = {}
    for stat in stats:
        for key, value in stat.items():
            if key not in results.keys():
                results[key] = {}
                for k, v in value.items():
                    if k == "rouge_score":
                        results[key]["R1_P"] = [round(v["rouge1"][1][0] * 100, 2)]
                        results[key]["R1_R"] = [round(v["rouge1"][1][1] * 100, 2)]
                        results[key]["R1_F"] = [round(v["rouge1"][1][2] * 100, 2)]
                        results[key]["R2_P"] = [round(v["rouge2"][1][0] * 100, 2)]
                        results[key]["R2_R"] = [round(v["rouge2"][1][1] * 100, 2)]
                        results[key]["R2_F"] = [round(v["rouge2"][1][2] * 100, 2)]
                        results[key]["RL_P"] = [round(v["rougeL"][1][0] * 100, 2)]
                        results[key]["RL_R"] = [round(v["rougeL"][1][1] * 100, 2)]
                        results[key]["RL_F"] = [round(v["rougeL"][1][2] * 100, 2)]
                        results[key]["RLsum_P"] = [round(v["rougeLsum"][1][0] * 100, 2)]
                        results[key]["RLsum_R"] = [round(v["rougeLsum"][1][1] * 100, 2)]
                        results[key]["RLsum_F"] = [round(v["rougeLsum"][1][2] * 100, 2)]
                    else:
                        results[key][k] = [v]
            else:
                for k, v in value.items():
                    if k == "rouge_score":
                        results[key]["R1_P"].append(round(v["rouge1"][1][0] * 100, 2))
                        results[key]["R1_R"].append(round(v["rouge1"][1][1] * 100, 2))
                        results[key]["R1_F"].append(round(v["rouge1"][1][2] * 100, 2))
                        results[key]["R2_P"].append(round(v["rouge2"][1][0] * 100, 2))
                        results[key]["R2_R"].append(round(v["rouge2"][1][1] * 100, 2))
                        results[key]["R2_F"].append(round(v["rouge2"][1][2] * 100, 2))
                        results[key]["RL_P"].append(round(v["rougeL"][1][0] * 100, 2))
                        results[key]["RL_R"].append(round(v["rougeL"][1][1] * 100, 2))
                        results[key]["RL_F"].append(round(v["rougeL"][1][2] * 100, 2))
                        results[key]["RLsum_P"].append(
                            round(v["rougeLsum"][1][0] * 100, 2)
                        )
                        results[key]["RLsum_R"].append(
                            round(v["rougeLsum"][1][1] * 100, 2)
                        )
                        results[key]["RLsum_F"].append(
                            round(v["rougeLsum"][1][2] * 100, 2)
                        )
                    else:
                        results[key][k].append(v)
    for key, value in results.items():
        print(f"===== {key.upper()} =====\n")
        for k, v in value.items():
            print(f"+++ {k} +++")
            print(f"Mean:             {round(np.mean(v), 2)}")
            print(f"99th percentile:  {round(np.percentile(v, 99), 2)}")
            print(f"95th percentile:  {round(np.percentile(v, 95), 2)}")
            print(f"75th percentile:  {round(np.percentile(v, 75), 2)}")
            print(f"50th percentile:  {round(np.percentile(v, 50), 2)}")
            print(f"25th percentile:  {round(np.percentile(v, 25), 2)}")
            print(f"5th percentile:   {round(np.percentile(v, 5), 2)}")
            print(f"1th percentile:   {round(np.percentile(v, 1), 2)}")
            print()
        print("\n\n")

if __name__ == "__main__":
    app()


import torch
import torch.nn as nn
from torch.autograd import Function
from torchvision.models.utils import load_state_dict_from_url

'''
Very easy template to start for developing your AlexNet with DANN
Has not been tested, might contain incompatibilities with most recent versions of PyTorch (you should address this)
However, the logic is consistent
'''

class ReverseLayerF(Function):
    # Forwards identity
    # Sends backward reversed gradients
    @staticmethod
    def forward(ctx, x, alpha):
        ctx.alpha = alpha

        return x.view_as(x)

    @staticmethod
    def backward(ctx, grad_output):
        output = grad_output.neg() * ctx.alpha

        return output, None

class RandomNetworkWithReverseGrad(nn.Module):
    #def __init__(self, **kwargs):
    def __init__(self, NUM_CLASSES=1000, NUM_DOMAINS=10):
        super(RandomNetworkWithReverseGrad, self).__init__()
        # adding conv parameters, took from AlexNet self.features = nn.Sequentia(etc.
        self.features = nn.Sequential(
        	nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(64, 192, kernel_size=5, padding=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(192, 384, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(384, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
        )
        # adding fc parameters, took from AlexNet self.classifier = nn.Sequential( etc)
        self.classifier = nn.Sequential(
        	nn.Dropout(),
            nn.Linear(256 * 6 * 6, 4096),
            nn.ReLU(inplace=True),
            nn.Dropout(),
            nn.Linear(4096, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, NUM_CLASSES),
        )
        # same architecture of AlexNet's FC but adding a new densely connected branch with 2 output neurons (TO FINISH)
        self.avgpool = nn.AdaptiveAvgPool2d((6, 6))
        self.domain_classifier = nn.Sequential(
        	nn.Dropout(),
            nn.Linear(256 * 6 * 6, 4096),
            nn.ReLU(inplace=True),
            nn.Dropout(),
            nn.Linear(4096, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, NUM_DOMAINS),
        )

    def update(self):
        self.domain_classifier[1].weight.data = self.classifier[1].weight.data
        self.domanin_classifier[1].bias.data = self.classifier[1].bias.data

        self.domain_classifier[4].weight.data = self.classifier[4].weight.data
        self.domain_classifier[4].bias.data = self.classifier[4].bias.data

    def forward(self, x, alpha=None):
        features = self.features(x);
        features = self.avgpool(features)
        features = torch.flatten(features, 1)
        # If we pass alpha, we can assume we are training the discriminator
        if alpha is not None:
            # gradient reversal layer (backward gradients will be reversed)
            reverse_feature = ReverseLayerF.apply(features, alpha)
            discriminator_output = self.domain_classifier(features);
            return discriminator_output
        # If we don't pass alpha, we assume we are training with supervision
        else:
            # do something else
            class_outputs = self.classifier(features);
            return class_outputs

    def DANN_AlexNet(pretrained=False, progress=True, **kwargs):
        r"""AlexNet model architecture from the
        `"One weird trick..." <https://arxiv.org/abs/1404.5997>`_ paper.
        Args:
            pretrained (bool): If True, returns a model pre-trained on ImageNet
            progress (bool): If True, displays a progress bar of the download to stderr
    """
        model = RandomNetworkWithReverseGrad(**kwargs)
        if pretrained:
            state_dict = load_state_dict_from_url(model_urls['alexnet'],
                                                    progress = progress
                                                    )
            state_dict.popitem("classifier.6.weight")
            state_dict.popitem("classifier.6.bias")

            model.load_state_dict(state_dict, strict=False)
            model.update()
        return model


from typing import Optional

import requests

from django.conf import settings

def make_ghost_request(
        method: str,
        path: str,
        files: Optional[dict] = None,
        data: Optional[dict] = None,
        json: Optional[dict] = None,
        headers: Optional[dict] = None,
        params: Optional[dict] = None,
) -> requests.Response:
    params = params or {}
    params['key'] = settings.GHOST_API_KEY

    request = requests.Request(
        method=method,
        url=f'{settings.GHOST_URL}/ghost/api/v4/content/{path}',
        files=files,
        data=data,
        json=json,
        headers=headers,
        params=params,
    ).prepare()

    return requests.Session().send(request)

def get_post(post_id: str, include: Optional[list[str]] = None):
    response = make_ghost_request(
        'get',
        f'posts/{post_id}/',
        params={
            'include': include,
        },
    )

    if response.status_code != 200:
        return

    try:
        return response.json().get('posts')[0]
    except (KeyError, IndexError):
        return

def browse_posts(include: Optional[list[str]] = None, request_filter: Optional[str] = None):
    response = make_ghost_request(
        'get',
        f'posts/',
        params={
            'include': include,
            'filter': request_filter,
        },
    )

    if response.status_code != 200:
        return

    try:
        return response.json().get('posts')
    except (KeyError, IndexError):
        return


from ..common_imports import *
from .utils import DepKey, hash_dict
from .model import (
    Node,
    CallableNode,
    GlobalVarNode,
    TerminalNode,
)

from .shallow_versions import DAG

class Version:
    """
    Model of a "deep" version of a component that includes versions of its
    dependencies.
    """

    def __init__(
        self,
        component: DepKey,
        dynamic_deps_commits: Dict[DepKey, str],
        memoized_deps_content_versions: Dict[DepKey, Set[str]],
    ):
        ### raw data from the trace
        # the component whose dependencies are traced
        self.component = component
        # the content hashes of the direct dependencies
        self.direct_deps_commits = dynamic_deps_commits
        # pointers to content hashes of versions of memoized calls
        self.memoized_deps_content_versions = memoized_deps_content_versions

        ### cached data. These are set against a dependency state
        self._is_synced = False
        # the expanded set of dependencies, including all transitive
        # dependencies. Note this is a set of *content* hashes per dependency
        self._content_expansion: Dict[DepKey, Set[str]] = None
        # a hash uniquely identifying the content of dependencies of this version
        self._content_version: str = None
        # the semantic hashes of all dependencies for this version;
        # the system enforces that the semantic hash of a dependency is the same
        # for all commits of a component referenced by this version
        self._semantic_expansion: Dict[DepKey, str] = None
        # overall semantic hash of this version
        self._semantic_version: str = None

    @property
    def presentation(self) -> str:
        return f'Version of "{self.component[1]}" from module "{self.component[0]}" (content: {self.content_version}, semantic: {self.semantic_version})'

    @staticmethod
    def from_trace(
        component: DepKey, nodes: Dict[DepKey, Node], strict: bool = True
    ) -> "Version":
        dynamic_deps_commits = {}
        memoized_deps_content_versions = defaultdict(set)
        for dep_key, node in nodes.items():
            if isinstance(node, (CallableNode, GlobalVarNode)):
                dynamic_deps_commits[dep_key] = node.content_hash
            elif isinstance(node, TerminalNode):
                terminal_data = node.representation
                pointer_dep_key = terminal_data.dep_key
                version_content_hash = terminal_data.call_content_version
                memoized_deps_content_versions[pointer_dep_key].add(
                    version_content_hash
                )
            else:
                raise ValueError(f"Unexpected node type {type(node)}")
        return Version(
            component=component,
            dynamic_deps_commits=dynamic_deps_commits,
            memoized_deps_content_versions=dict(memoized_deps_content_versions),
        )

    ############################################################################
    ### methods for setting cached data from a versioning state
    ############################################################################
    def _set_content_expansion(self, all_versions: Dict[DepKey, Dict[str, "Version"]]):
        result = defaultdict(set)
        for dep_key, content_hash in self.direct_deps_commits.items():
            result[dep_key].add(content_hash)
        for (
            dep_key,
            memoized_content_versions,
        ) in self.memoized_deps_content_versions.items():
            for memoized_content_version in memoized_content_versions:
                referenced_version = all_versions[dep_key][memoized_content_version]
                for (
                    referenced_dep_key,
                    referenced_content_hashes,
                ) in referenced_version.content_expansion.items():
                    result[referenced_dep_key].update(referenced_content_hashes)
        self._content_expansion = dict(result)

    def _set_content_version(self):
        self._content_version = hash_dict(
            {
                dep_key: tuple(sorted(self.content_expansion[dep_key]))
                for dep_key in self.content_expansion
            }
        )

    def _set_semantic_expansion(
        self,
        component_dags: Dict[DepKey, DAG],
        all_versions: Dict[DepKey, Dict[str, "Version"]],
    ):
        result = {}
        # from own deps
        for dep_key, dep_content_hash in self.direct_deps_commits.items():
            dag = component_dags[dep_key]
            semantic_hash = dag.commits[dep_content_hash].semantic_hash
            result[dep_key] = semantic_hash
        # from pointers
        for (
            dep_key,
            memoized_content_versions,
        ) in self.memoized_deps_content_versions.items():
            for memoized_content_version in memoized_content_versions:
                dep_version_semantic_hashes = all_versions[dep_key][
                    memoized_content_version
                ].semantic_expansion
                overlap = set(result.keys()).intersection(
                    dep_version_semantic_hashes.keys()
                )
                if any(result[k] != dep_version_semantic_hashes[k] for k in overlap):
                    raise ValueError(
                        f"Version {self} has conflicting semantic hashes for {overlap}"
                    )
                result.update(dep_version_semantic_hashes)
        self._semantic_expansion = result
        self._semantic_version = hash_dict(result)

    def sync(
        self,
        component_dags: Dict[DepKey, DAG],
        all_versions: Dict[DepKey, Dict[str, "Version"]],
    ):
        """
        Set all the cached things in the correct order
        """
        self._set_content_expansion(all_versions=all_versions)
        self._set_content_version()
        self._set_semantic_expansion(
            component_dags=component_dags, all_versions=all_versions
        )
        self.set_synced()

    @property
    def content_version(self) -> str:
        assert self._content_version is not None
        return self._content_version

    @property
    def semantic_version(self) -> str:
        assert self._semantic_version is not None
        return self._semantic_version

    @property
    def semantic_expansion(self) -> Dict[DepKey, str]:
        assert self._semantic_expansion is not None
        return self._semantic_expansion

    @property
    def content_expansion(self) -> Dict[DepKey, Set[str]]:
        assert self._content_expansion is not None
        return self._content_expansion

    @property
    def support(self) -> Iterable[DepKey]:
        return self.content_expansion.keys()

    @property
    def is_synced(self) -> bool:
        return self._is_synced

    def set_synced(self):
        # it can only go from unsynced to synced
        if self._is_synced:
            raise ValueError("Version is already synced")
        self._is_synced = True

    def __repr__(self) -> str:
        return f"""
Version(
    dependencies={['.'.join(elt) for elt in self.support]},
)"""


from django.db import models

class LearningLectureStat(models.Model):
    id = models.AutoField(primary_key=True)
    user = models.ForeignKey('users.User', related_name='lecture_stat_user', on_delete=models.CASCADE)
    lecture = models.ForeignKey('curricula.LearningLecture', related_name='lecture_stat', on_delete=models.CASCADE)
    lecture_status = models.PositiveSmallIntegerField(default=0)
    practice_status = models.PositiveSmallIntegerField(default=0)
    created = models.DateField(auto_now_add=True)

    class Meta:
        db_table = 'curricula_learning_lecture_stat'
        default_permissions = ()
        permissions = (
            ("view_learninglecturestat", "View learning lecture stat"),
            ("list_learninglecturestat", "List learning lecture stat")
        )


# File_name: stop_elasticache.py
# Purpose: delete elasticache clusters
# Problem:
# Author: Søren	Wandrup-Bendixen
# Email: soren.wandrup-Bendixen@cybercom.com
# Created: 2019-07-01
# Called from lambda_function.py

import boto3

# instance_type	= 'elasticache'
def	delete_clusters(instance_type,region_name_,RunningInstances) :
	client	= boto3.client(instance_type, region_name=region_name_)
	clusters = client.describe_cache_clusters(  )['CacheClusters']
	for cluster in clusters:
		RunningInstances.append(instance_type	+ '	'	+ region_name_ + '	CacheCluster	   ' + cluster['CacheClusterId'])
		# auto remove previos snapshot
		snapshot_identifier = 'auto-stop-all-' + instance_id
		try:
			old_snapshots = client.describe_snapshots( SnapshotName= snapshot_identifier)
			if len(old_snapshots) > 0:
				response = client.delete_snapshot( SnapshotName= snapshot_identifier)
		except client.exceptions.SnapshotNotFoundFault:
			print('No snapshot found with this id ' + snapshot_identifier )

		response = client.create_snapshot(    CacheClusterId=cluster['CacheClusterId'],    SnapshotName= snapshot_identifier)
		response = client.delete_cache_cluster( CacheClusterId=cluster['CacheClusterId']	)

	return


import logging

from flask import Flask, jsonify
from flask_cors import cross_origin
from flask_jsontools import DynamicJSONEncoder, jsonapi
from flask_swagger import swagger

# setup the configuration for the application
from habt.config import Config
from habt.database import session
from habt.manager import PackageManager

# Configure the logger
config = Config()
config.setup_logger()
log = logging.getLogger(__name__)

app = Flask(__name__)
app.debug = config.debug
# uses the <obj>.__json__() method to encode json
app.json_encoder = DynamicJSONEncoder

@app.teardown_appcontext
def shutdown_session(exception=None):
    """
        Destroy the Database session at the end of a request
    """
    session.remove()

@app.route("/spec")
@cross_origin()
def spec():
    """
        returns:
         Api specification
    """
    swag = swagger(app)
    # api information
    swag["info"]["version"] = "0.0.0"
    swag["info"]["title"] = "habt API"
    # api prefix
    swag["basePath"] = "/api"
    return jsonify(swag)

@app.route("/search/<string:query>")
@cross_origin()
@jsonapi
def search(query):
    """
         Returns the details of a debian package
        ---
        tags:
          - search

        parameters:
          - name: query
            description: Search query
            in: path
            type: string

        responses:
          200:
            description: Search was successful
        """
    return PackageManager().search_packages(query)

@app.route("/package/<string:name>")
@cross_origin()
@jsonapi
def package(name):
    """
       Returns the details of a debian package
       ---
       tags:
         - package
       parameters:
         - name: name
           description: Debian package name
           in: path
           type: string
       responses:
           200:
               description: Datail successfully loaded
    """
    return PackageManager().get_package(name)

@app.route("/package/<string:name>/version/<string:version>")
@cross_origin()
@jsonapi
def version(name, version):
    """
       Returns the details of a debian package version
       ---
       tags:
         - version
       parameters:
        - name: name
          description: Debian package name
          in: path
          type: string
        - name: version
          description: Package version
          in: path
          type: string
       responses:
           200:
               description: Datail successfully loaded
    """
    return PackageManager().get_package_version(name, version)

if __name__ == "__main__":
    app.run("0.0.0.0", port=8000)