from statsmodels.stats.diagnostic import het_white
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import RFE
from sklearn.linear_model import LinearRegression
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn.metrics import r2_score

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.api as sm

import seaborn as sns

def plot_lin_reg(x, y, model, name, title):
    # plot and save:
    fig, ax = plt.subplots(figsize=(8, 4))
    ax.scatter(x, y, alpha=0.5, color='orange')
    fig.suptitle(title)
    fig.tight_layout(pad=2)
    ax.grid(True)

    x_pred = np.linspace(x.min(), x.max(), 50)
    x_predM = sm.add_constant(x_pred)
    y_pred = model.predict(x_predM)
    ax.plot(x_pred, y_pred, '-', color='blue', linewidth=2)

    fig.savefig(name, dpi=150)

sns.set()

data = pd.read_csv("CarDekho_cardata.csv")

# Eliminar filas con valores null:
data = data.dropna()

# Quitar unidades de columnas con datos numéricos:
data['mileage'] = data['mileage'].str.extract('(\d+)', expand=False)
data['mileage'] = data['mileage'].astype(int)

data['engine'] = data['engine'].str.extract('(\d+)', expand=False)
data['engine'] = data['engine'].astype(int)

data['max_power'] = data['max_power'].str.extract('(\d+)', expand=False)
data['max_power'] = data['max_power'].astype(int)

data['torque'] = data['torque'].str.extract('(\d+)', expand=False)
data['torque'] = data['torque'].astype(int)

# Separar la marca del modelo en columnas distintas:
data['manufacturer'] = np.nan
data['model'] = np.nan

data['manufacturer'] = data['name'].str.split(" ", 1, expand=True)
data['model'] = data['name']

data.drop(data.columns[0], axis=1, inplace=True)

print(data.columns)

# Describir los datos:
pd.set_option('display.max_columns', 20)
print(data.describe(include='all'))

# Comprobación de linealidad y variables escogidas:
# Variable dependiente:
price = data['selling_price']

# Variables independientes:
year = data['year']
mileage = data['km_driven']
fuel_efficiency = data['mileage']
engine_size = data['engine']
max_hp = data['max_power']
torque = data['torque']
seats = data['seats']

# Variables categóricas:
seller_type = data['seller_type']
owner = data['owner']
transmission = data['transmission']
manufacturer = data['manufacturer']
model = data['model']
fuel_type = data['fuel']

# Linealidad:
plt.scatter(price, year)  # (x, y)
plt.ylabel('Year of production', fontsize=20)
plt.xlabel('Vehicle Price', fontsize=20)
plt.show()

plt.scatter(price, mileage)  # (x, y)
plt.ylabel('Mileage', fontsize=20)
plt.xlabel('Vehicle Price', fontsize=20)
plt.show()

plt.scatter(price, fuel_efficiency)  # (x, y)
plt.ylabel('Fuel efficiency (km/l)', fontsize=20)
plt.xlabel('Vehicle Price', fontsize=20)
plt.show()

plt.scatter(price, engine_size)  # (x, y)
plt.ylabel('Engine Size (CC)', fontsize=20)
plt.xlabel('Vehicle Price', fontsize=20)
plt.show()

plt.scatter(price, max_hp)  # (x, y)
plt.ylabel('Max Horse Power (bhp)', fontsize=20)
plt.xlabel('Vehicle Price', fontsize=20)
plt.show()

plt.scatter(price, torque)  # (x, y)
plt.ylabel('Max Torque (Nm)', fontsize=20)
plt.xlabel('Vehicle Price', fontsize=20)
plt.show()

plt.scatter(price, seats)  # (x, y)
plt.ylabel('Number of Seats', fontsize=20)
plt.xlabel('Vehicle Price', fontsize=20)
plt.show()

# Regresión Lineal Simple:
y = price

# Price-year:

x1 = sm.add_constant(year)
print(x1)

resultados1 = sm.OLS(y, x1).fit()

print(resultados1.summary())

plot_lin_reg(year, y, resultados1, 'price_year_lin.png', 'Price-Year Simple Linear:')

# Kilometraje-precio:
x2 = sm.add_constant(mileage)
print(x2)

resultados2 = sm.OLS(y, x2).fit()

print(resultados2.summary())

plot_lin_reg(mileage, y, resultados2, 'price_mileage_lin.png', 'Price-Mileage Simple Linear:')

# Caballos de fuerza - precio:
x3 = sm.add_constant(max_hp)
print(x3)

resultados3 = sm.OLS(y, x3).fit()

print(resultados3.summary())

plot_lin_reg(max_hp, y, resultados3, 'price_hp_lin.png', 'Price-HorsePower Simple Linear:')

# Torque-precio:
x4 = sm.add_constant(torque)
print(x4)

resultados4 = sm.OLS(y, x4).fit()

print(resultados4.summary())

plot_lin_reg(torque, y, resultados4, 'price_torque_lin.png', 'Price-Torque Simple Linear:')

# Regresión Lineal Múltiple:
# Obtener variables dummy (numéricas) para las variables categóricas:
fuel_d = pd.get_dummies(fuel_type)
transmission_d = pd.get_dummies(transmission, drop_first=True)
seller_d = pd.get_dummies(seller_type, drop_first=True)
manufacturer_d = pd.get_dummies(manufacturer, drop_first=True)
owner_d = pd.get_dummies(owner, drop_first=True)

fuel_d.drop('CNG', axis=1, inplace=True)  # Se elimina porque hay muy pocos registros de este combustible.
data2 = data.copy()
data3 = data.copy()
data2 = pd.concat([data2, fuel_d, transmission_d, seller_d, manufacturer_d, owner_d], axis=1)
data3 = pd.concat([data3, fuel_d, transmission_d, seller_d, owner_d], axis=1)
print(data2.head())

# Eliminamos todas las columnas que no vamos a usar:
data2.drop(['fuel', 'seller_type', 'transmission', 'mileage', 'seats', 'manufacturer', 'model', 'owner'], axis=1, inplace=True)
data3.drop(['fuel', 'seller_type', 'transmission', 'mileage', 'seats', 'manufacturer', 'model', 'owner'], axis=1, inplace=True)
print(data2.head())
print(data2.columns)

# Usamos sklearn para tratar las variables numéricas y categóricas igual:

np.random.seed(0)  # Para tener una muestra realmente aleatoria.
train, test = train_test_split(data2, test_size=0.3, random_state=100)
train3, test3 = train_test_split(data3, test_size=0.3, random_state=100)

numeric_vars = ['selling_price', 'km_driven', 'max_power', 'engine', 'year']

scaler = StandardScaler()  # Para estandarizar las variables entre su min, media y media y max.
train[numeric_vars] = scaler.fit_transform(train[numeric_vars])
test[numeric_vars] = scaler.transform(test[numeric_vars])

train3[numeric_vars] = scaler.fit_transform(train3[numeric_vars])
test3[numeric_vars] = scaler.transform(test3[numeric_vars])

# Dividimos el dataset:
y_train = train.pop('selling_price')
X_train = train

X_train = X_train.drop('Diesel', axis=1)
X_train = X_train.drop('Petrol', axis=1)
X_train = X_train.drop('Maruti', axis=1)
X_train = X_train.drop('Hyundai', axis=1)
X_train = X_train.drop('torque', axis=1)
X_train = X_train.drop('Mahindra', axis=1)
X_train = X_train.drop('Tata', axis=1)
X_train = X_train.drop('Manual', axis=1)
X_train = X_train.drop('Toyota', axis=1)
X_train = X_train.drop('Honda', axis=1)

y_train3 = train3.pop('selling_price')
X_train3 = train3

# Definimos el modelo:
linear_model = LinearRegression()
linear_model.fit(X_train, y_train)

# Definimos las caracter[isticas y la cantidad de caracter[isticas utilizando RFE (Recursive Feature Elimination):
rfe = RFE(linear_model, 15)  # Aquí se definen la cantidad de características a seleccionar.
rfe = rfe.fit(X_train, y_train)

# Característica seleccionadas por RFE:
print(list(zip(X_train.columns, rfe.support_, rfe.ranking_)))

col = X_train.columns[rfe.support_]
print(col)

X_train_rfe = X_train[col]

X_train_rfe = sm.add_constant(X_train_rfe)

# Realizamos la regresión lineal múltiple:
multi_linear_model = sm.OLS(y_train, X_train_rfe).fit()
print("\nSELECTED---------------------------------------------")
print(multi_linear_model.summary())

# Regresión con todas las variables:
linear_model2 = LinearRegression()
linear_model2.fit(X_train, y_train)

rfe2 = RFE(linear_model, 36)
rfe2 = rfe2.fit(X_train, y_train)

col2 = X_train.columns[rfe2.support_]
print(col2)

X_train_rfe2 = X_train[col2]

X_train_rfe2 = sm.add_constant(X_train_rfe2)

multi_linear_model2 = sm.OLS(y_train, X_train_rfe2).fit()
print("\nSELECTED---------------------------------------------")
print(multi_linear_model2.summary())

# Regresión lineal sin considerar marcas:
linear_model3 = LinearRegression()
linear_model3.fit(X_train3, y_train3)

rfe3 = RFE(linear_model, 16)
rfe3 = rfe3.fit(X_train3, y_train3)

col3 = X_train3.columns[rfe3.support_]
print(col3)

X_train_rfe3 = X_train3[col3]

X_train_rfe3 = sm.add_constant(X_train_rfe3)

multi_linear_model3 = sm.OLS(y_train3, X_train_rfe3).fit()
print(multi_linear_model3.summary())

# Regresión lineal sin marcas (Optimizado):
linear_model4 = LinearRegression()
linear_model4.fit(X_train3, y_train3)

rfe4 = RFE(linear_model, 10)
rfe4 = rfe4.fit(X_train3, y_train3)

col4 = X_train3.columns[rfe4.support_]
print(col4)

X_train_rfe4 = X_train3[col4]

X_train_rfe4 = sm.add_constant(X_train_rfe4)

multi_linear_model4 = sm.OLS(y_train3, X_train_rfe4).fit()
print(multi_linear_model4.summary())

# Buscar heteroscedasticity por medio del test de White:
white_test1 = het_white(multi_linear_model.resid,  multi_linear_model.model.exog)  # Modelo 15 variables.
white_test2 = het_white(multi_linear_model2.resid,  multi_linear_model2.model.exog)  # Modelo 46 variables.

labels = ['LM Statistic', 'LM-Test p-value', 'F-Statistic', 'F-Test p-value']

test_results1 = dict(zip(labels, white_test1))
test_results2 = dict(zip(labels, white_test2))

print(white_test1)
print()
print(white_test2)

p1 = test_results1.get('F-Test p-value')
p2 = test_results2.get('F-Test p-value')

print("p1: " + str(p1))
print("p2: " + str(p2))

if p1 < 0.05:

    print("Model 1 is heteroskedastic")

if p2 < 0.05:

    print("Model 2 is heteroskedastic")

print("\nTest No Multicolinealidad Modelo 1 y 2:")
X_train_new = X_train
vif = pd.DataFrame()
X = X_train_new
vif['Features'] = X.columns
vif['VIF'] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]
vif['VIF'] = round(vif['VIF'], 2)
vif = vif.sort_values(by="VIF", ascending=False)

print(vif)

# Testing:
y_test = test.pop('selling_price')  # Objetivos
X_test = test  # Características

X_test_predict = X_test[X_train.columns]
X_test_predict1 = X_test[X_train_rfe.drop('const', axis=1).columns]
X_test_predict = sm.add_constant(X_test_predict)
X_test_predict1 = sm.add_constant(X_test_predict1)

y_predict1 = multi_linear_model.predict(X_test_predict1)
y_predict2 = multi_linear_model2.predict(X_test_predict)

results1 = pd.DataFrame({'Actual': y_test, "Predicted": y_predict1})
print(results1.head())

results2 = pd.DataFrame({'Actual': y_test, "Predicted": y_predict2})
print(results2.head())

score1 = r2_score(y_test, y_predict1)
score2 = r2_score(y_test, y_predict2)

print("Score of model 1: " + str(score1))
print("Score of model 2: " + str(score2))


#날짜/시간과 관련된 기능 가져오기
import datetime

#현재 날짜/시간 구하기
now = datetime.datetime.now()

#오전 구분
if now.hour < 12 :
    print("현재 시각은 {}시로 오전입니다.".format(now.hour))

#오후 구분
if now.hour >=12 :
    print("현재 시각은 {}시로 오후입니다.".format(now.hour))


from .component import Component
from ..astline import  ASTLine

from argparse import ArgumentParser

class ContributorDocumentation(Component):
    parse_group_description = 'Contributor Documentation parameters'
    name = 'contributordoc'

    @classmethod
    def cmdline_documentation(cls, parser: ArgumentParser):
        parser_group = super().cmdline_documentation(parser)

        parser_group.add_argument(
            f'--{cls.name}.group-by',
            action="store",
            default="section",
            help='How the contributor documentation will group the different elements. "section" or "type"'
        )

        parser_group.add_argument(
            f'--{cls.name}.hide-uncommented',
            action="store_true",
            help='(flag) hide non-commented or non-defined variable, rules...'
        )

        parser_group.add_argument(
            f'--{cls.name}.hide-code',
            action="store_true",
            help='(flag) hide source code'
        )

    def __init__(self, builder, parameters) -> None:
        super().__init__(builder, parameters)

    def _include_code(self, astline: ASTLine):
        if not self.parameters[self.name]['hide_code']:
            return super()._include_code(astline)

    def check_parameters(self):
        super().check_parameters()
        if not 'group_by' in  self.parameters[self.name]:
            self.parameters[self.name]['group_by'] = 'section'

        if not 'hide_uncommented' in self.parameters[self.name]:
            self.parameters[self.name]['hide_uncommented'] = False

        if not 'hide_code' in self.parameters[self.name]:
            self.parameters[self.name]['hide_code'] = False

    def _factory(self, astline: ASTLine):
        if self.parameters[self.name]['hide_uncommented'] and not astline.comments:
            return False

        self.document.h4(f"{self._get_name(astline)}")
        self.document.newline()

        if 'predicate' in self.builder.all_directives:
            for directive in self.builder.all_directives['predicate']:

                if directive.parameters[0] == astline.identifier:
                    self.document.content(f'{directive.parameters[1]} -> {directive.description}')
                    self.document.newline()

        if astline.dependencies:
            self.document.h5(f"Dependencies:")
            d_done = []
            for d in astline.dependencies:
                if d.signature not in d_done:
                    self.document.li(d.signature)
                    d_done.append(d.signature)
            self.document.newline()

        self.document.content(
            f"Location: Line {self._get_location(astline)}")
        self.document.newline()
        self._include_code(astline)
        self._include_comments(astline)
        self.document.newline()
        return True

    def _build_variable_table(self):
        self.document.h2('Variables')
        self.document.newline()
        vars = self.builder.all_directives.get('var')
        data = []
        if vars:
            for var in vars:
                data.append([var.parameters[0],var.description])

        self.document.table(['Variable', 'Definition'], data=data)
        self.document.newline()

    def build_rst_file(self):

        self.document.title("Contributor Documentation")
        self.document.newline()
        self.document.content("*Understand the encoding*")
        self.document.newline()
        self.document.table_of_contents('Contents', depth=3)
        self.document.newline()
        self._build_variable_table()
        self.document.h2(f'Encoding decomposition')

        if self.parameters[self.name]['group_by'] == "type":
            self.document.content("*Ordered by type*")
            self.document.newline()
            types = {}
            for east in self.builder.easts:
                for astline in east.ast_lines:
                        if astline.type not in types:
                            types[astline.type] = [astline]
                        else:
                            types[astline.type].append(astline)

            for t in types:
                self.document.h3(f'Type: {t}')
                self.document.newline()
                for astline in types[t]:
                    self._factory(astline)

        elif self.parameters[self.name]['group_by'] == "section":
            self.document.content("*Ordered by section*")
            self.document.newline()

            sections = {}
            no_section = []
            for east in self.builder.easts:
                for astline in east.ast_lines:
                    if astline.section != None:
                        for key in sections:
                            if key.parameters[0] == astline.section.parameters[0]:
                                sections[key] .append(astline)
                                break
                        else:
                            sections[astline.section] = [astline]
                    else:
                        no_section.append(astline)

            if not sections:
                print('Warning: No section detected and using group by "section"')

            for section in sections:
                self.document.h3(f'Section: {section.parameters[0]}')
                self.document.newline()
                if section.description:

                    self.document.content(section.description)
                    self.document.newline()
                for astline in sections[section]:
                    self._factory(astline)

            if no_section:
                self.document.h1('No section')
                self.document.newline()
                for astline in no_section:
                    self._factory(astline)

        else:
            raise ValueError(f"invalid groupby value: {self.groupby}")


from flask import session
from models.users import Users
from flask.json import jsonify
from auth.auth import is_authenticated, is_admin
from models.users import db

def get_dashboard_data():

    user_id = session.get("user_id")
    # Check if session exist
    if not is_authenticated(user_id):
        return jsonify({"error": "Unauthorized"}), 401

    # Check if role is admin
    if not is_admin(user_id):
        return jsonify({"error": "Unauthorized"}), 401

    totalUser = Users.query.filter_by(role="user", is_deleted=0).count()
    distinctRole = db.session.query(Users.role).distinct().count()
    deletedUsers = Users.query.filter_by(is_deleted=1, role="user").count()
    updatedUsers = Users.query.filter(
        Users.updated_at.isnot(None)).filter(Users.role == "user").filter(Users.is_deleted == 0).count()

    return jsonify({
        "totalUsers": totalUser,
        "distinctRoles": distinctRole,
        "deletedUsers": deletedUsers,
        "updatedUsers": updatedUsers
    })


import csv

from django.core.management.base import BaseCommand
from recipes.models import Ingredient

class Command(BaseCommand):
    DATA = {
        Ingredient: "ingredients.csv",
    }

    def add_arguments(self, parser):
        parser.add_argument("--data_directory", type=str)

    def handle(self, *args, **options):
        dirname = options["data_directory"] or './data/'
        for model, filename in self.DATA.items():
            with open(
                f"{dirname}/{filename}",
                newline="",
                encoding="utf-8"
            ) as csvfile:
                reader = csv.reader(csvfile, delimiter=",")
                for row in reader:
                    model.objects.update_or_create(
                        name=row[0],
                        measurement_unit=row[1]
                    )
            print(f"{filename} successfully added to DB")
        print("DB successfully filled")


#!/usr/bin/env python
# coding: utf-8

# In[2]:

import warnings
warnings.filterwarnings('ignore')
import os
import sys
import json
import datetime
import numpy as np
import skimage.draw
import cv2
import random
import math
import re
import time
import tensorflow as tf
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import matplotlib.image as mpimg

os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
from mrcnn import utils
from mrcnn import visualize
from mrcnn.visualize import display_images
from mrcnn.visualize import display_instances
import mrcnn.model as modellib
from mrcnn.model import log
from mrcnn.config import Config
from mrcnn import model as modellib, utils

#import custom

# Root directory of the project
ROOT_DIR = "/users/pgrad/saxenac/Downloads"

DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "logs")

MODEL_DIR = os.path.join(ROOT_DIR, "logs")

WEIGHTS_PATH = "/users/pgrad/saxenac/Downloads/mask_rcnn_coco.h5"

# In[5]:

class CustomConfig(Config):
    """Configuration for training on the custom  dataset.
    Derives from the base Config class and overrides some values.
    """
    # Give the configuration a recognizable name
    NAME = "object"

    IMAGES_PER_GPU = 1

    NUM_CLASSES = 1 + 3  # Background + Car and truck

    # Number of training steps per epoch
    STEPS_PER_EPOCH = 10

    # Skip detections with < 90% confidence
    DETECTION_MIN_CONFIDENCE = 0.9

# In[22]:

class CustomDataset(utils.Dataset):

    def load_custom(self, dataset_dir, subset):

        self.add_class("object", 1, "0")
        self.add_class("object", 2, "1")
        self.add_class("object", 3, "2")

        assert subset in ["train", "val"]
        dataset_dir = os.path.join(dataset_dir, subset)
        annotations1 = json.load(open('/users/pgrad/saxenac/Downloads/via_project_9Jan2023_0h19m_json.json'))

        annotations = list(annotations1.values())  # don't need the dict keys

        annotations = [a for a in annotations if a['regions']]

        # Add images
        for a in annotations:

            polygons = [r['shape_attributes'] for r in a['regions']]
            objects = [s['region_attributes']['name'] for s in a['regions']]
            print("objects:",objects)
            name_dict = {"0": 0,"1": 1,"2": 2}
            num_ids = [name_dict[a] for a in objects]

            print("numids",num_ids)
            image_path = os.path.join(dataset_dir, a['filename'])
            image = skimage.io.imread(image_path)
            height, width = image.shape[:2]
            self.add_image(
                "object",
                image_id=a['filename'],  # use file name as a unique image id
                path=image_path,
                width=width, height=height,
                polygons=polygons,
                num_ids=num_ids
                )

    def load_mask(self, image_id):

        image_info = self.image_info[image_id]
        if image_info["source"] != "object":
            return super(self.__class__, self).load_mask(image_id)

        info = self.image_info[image_id]
        if info["source"] != "object":
            return super(self.__class__, self).load_mask(image_id)
        num_ids = info['num_ids']
        mask = np.zeros([info["height"], info["width"], len(info["polygons"])],
                        dtype=np.uint8)
        for i, p in enumerate(info["polygons"]):

        	rr, cc = skimage.draw.polygon(p['all_points_y'], p['all_points_x'])

        	mask[rr, cc, i] = 1

        num_ids = np.array(num_ids, dtype=np.int32)
        return mask, num_ids #np.ones([mask.shape[-1]], dtype=np.int32)

    def image_reference(self, image_id):
        """Return the path of the image."""
        info = self.image_info[image_id]
        if info["source"] == "object":
            return info["path"]
        else:
            super(self.__class__, self).image_reference(image_id)

# In[23]:

# cd = CustomDataset()
# cd.load_custom('/users/pgrad/saxenac/Downloads/',"train")

# In[24]:

# image_ids = [-1,-2,-3]
# for image_id in image_ids:
#     image = cd.load_image(image_id)
# #     print(cd.load_mask(image_id))
#     mask, class_ids = cd.load_mask(image_id)
#     visualize.display_top_masks(image, mask, class_ids,['unripe','pripe','ripe',])

# In[25]:

def train(model):
    """Train the model."""
    # Training dataset.
    dataset_train = CustomDataset()
    dataset_train.load_custom('/users/pgrad/saxenac/Downloads/',"train")
    dataset_train.prepare()

    # Validation dataset
    dataset_val = CustomDataset()
    dataset_val.load_custom('/users/pgrad/saxenac/Downloads/',"val")
    dataset_val.prepare()

    # *** This training schedule is an example. Update to your needs ***
    # Since we're using a very small dataset, and starting from
    # COCO trained weights, we don't need to train too long. Also,
    # no need to train all layers, just the heads should do it.
    print("Training network heads")
    model.train(dataset_train, dataset_val,
                learning_rate=config.LEARNING_RATE,
                epochs=5,
                layers='heads')

# In[ ]:

config = CustomConfig()
model = modellib.MaskRCNN(mode="training", config=config,
                                  model_dir=DEFAULT_LOGS_DIR)

model.load_weights(WEIGHTS_PATH, by_name=True, exclude=[
            "mrcnn_class_logits", "mrcnn_bbox_fc",
            "mrcnn_bbox", "mrcnn_mask"])

train(model)

# In[ ]:


import random
import heapq
import datetime
import networkx as nx
import math
import argparse
import matplotlib.pyplot as plt
import time
import pickle
import numpy as np
import operator

save_dir = '../datasets/Flickr/'
def featureUniform(dimension, scale):
	vector = np.array([random.random() for i in range(dimension)])
	l2_norm = np.linalg.norm(vector, ord =2)

	vector = vector/l2_norm

	gau = np.random.normal(0.5, 0.5, 1)[0]
	while gau < 0 or gau > 1:
		gau = np.random.normal(0.5, 0.5, 1)[0]

	vector = vector / scale * gau * 1.5

	return vector

dimension = 4
nodeDic = {}
edgeDic = {}
degree = []
G = pickle.load(open(save_dir+'Small_Final_SubG.G', 'rb'))
for u in G.nodes():
	s = len(G.edges(u))
	nodeDic[u] = [featureUniform(dimension, 1), featureUniform(dimension, s)]
for u in G.nodes():
	d = 0
	for v in G[u]:
		prob = np.dot(nodeDic[u][1], nodeDic[v][0]) * 4
		if prob > 1:
			prob = 1
		if prob < 0:
			prob = 0
		edgeDic[(u,v)] = prob
		d += prob
	degree.append(d)
pickle.dump(nodeDic, open(save_dir+'Small_nodeFeatures.dic', "wb" ))
pickle.dump(edgeDic, open(save_dir+'Probability.dic', "wb" ))

plt.hist(degree)
plt.show()


from config.config import ALPHABET, Wiring
from config.pygame_config import (
    COMPONENT_BORDER_RADIUS,
    COMPONENT_LETTERS_COLOR,
    COMPONENT_BORDER_COLOR,
    COMPONENT_BORDER_WIDTH,
)
import pygame

class Reflector:
    def __init__(self, wiring: Wiring, name: str) -> None:
        self.left = ALPHABET
        self.right = wiring.wiring
        self.name = name

    def reflect(self, signal: int) -> int:
        letter = self.right[signal]
        return self.left.find(letter)

    def draw(self, screen: pygame.surface.Surface, x: int, y: int, w: int, h: int, font: pygame.font.Font):
        r = pygame.Rect(x, y, w, h)
        pygame.draw.rect(screen, COMPONENT_BORDER_COLOR, r, width=COMPONENT_BORDER_WIDTH, border_radius=COMPONENT_BORDER_RADIUS)
        for i in range(26):
            letter = font.render(self.left[i], True, COMPONENT_LETTERS_COLOR)
            text_box = letter.get_rect(center=(x + w / 4, y + (i + 1) * h / 27))

            screen.blit(letter, text_box)

            letter = font.render(self.right[i], True, COMPONENT_LETTERS_COLOR)
            text_box = letter.get_rect(center=(x + w * 3 / 4, y + (i + 1) * h / 27))
            screen.blit(letter, text_box)


import torch

import numpy as np
import torch.nn as nn

from code.config import Hyper

class SampleDropout:
    def __init__(self, hyper: Hyper):
        self.p = hyper.prob
        self.dropout_ratio = nn.Embedding(hyper.n, 1)
        self.dropout_ratio = self.dropout_ratio.cuda(hyper.gpu)
        self._init_dropout()

    def _init_dropout(self):
        np_prob = np.array(self.p)
        np_prob = np_prob.reshape(-1, 1)
        weight = torch.from_numpy(np_prob)
        self.dropout_ratio.weight.data.copy_(weight)
        self.dropout_ratio.weight.requires_grad = False

    def __call__(self, x, label):
        ratio, mask = self._generate_mask(x, label)
        x *= mask

        amplification = torch.reciprocal(1 -  ratio)
        amplification = amplification.detach()
        x *= amplification
        return x

    def _generate_mask(self, input, label):
        ratio = self.dropout_ratio(label)

        mask = torch.rand_like(input)
        mask = mask > ratio
        mask = mask.float()
        mask.requires_grad = False
        return ratio, mask


from django import VERSION as DJANGO_VERSION
from django.core.exceptions import ValidationError
from django.db import DataError, models, transaction
from django.test import TestCase, override_settings
from django.utils.translation import override

from modeltrans.fields import TranslationField

from .app.models import (
    Article,
    Blog,
    Challenge,
    ChallengeContent,
    ChildArticle,
    NullableTextModel,
    TextModel,
)
from .utils import CreateTestModel

class TranslatedFieldTest(TestCase):
    def test_get_active_language(self):
        m = Blog(title="Falcon", i18n={"title_nl": "Valk", "title_de": "Falk"})

        with override("nl"):
            # value for the active language
            self.assertEqual(m.title_i18n, "Valk")

            self.assertEqual(m.title_en, "Falcon")
            self.assertEqual(m.title_de, "Falk")

        with override("de"):
            self.assertEqual(m.title_i18n, "Falk")

    def test_get_has_no_translation(self):
        m = Blog(title="Falcon", i18n={"title_nl": "Valk", "title_de": "Falk"})

        # Fallback to base langauge
        with override("fr"):
            self.assertEqual(m.title_i18n, "Falcon")

        # other translations are still there.
        self.assertEqual(m.title_nl, "Valk")
        self.assertEqual(m.title_de, "Falk")

    def test_get_non_translatable_field(self):
        m = Blog(title="Falcon")

        with self.assertRaisesMessage(AttributeError, "'Blog' object has no attribute 'foo'"):
            m.foo

    def test_set_translatable_field(self):
        m = Blog.objects.create(title="Toad")

        m.title_nl = "Pad"
        m.save()

        self.assertEqual(Blog.objects.get(title="Toad").title_nl, "Pad")

    def test_set_translatable_field_active_language(self):
        m = Blog.objects.create(title="Toad")

        with override("nl"):
            m.title_i18n = "Pad"
        m.save()

        self.assertEqual(Blog.objects.get(title="Toad").title_nl, "Pad")

    def test_set_default_langauge(self):
        m = Blog.objects.create(title="Toad 123")

        m.title_en = "Toad"
        m.save()

        self.assertEqual(m.title, "Toad")

    def test_set_None_doesnt_result_in_null_keys(self):
        m = Blog.objects.create(title="Couch")
        m.title_nl = None
        m.save()

        m = Blog.objects.get(title="Couch")
        self.assertEqual(m.i18n, {})

        m.title_nl = "Bank"
        m.save()
        self.assertEqual(m.i18n, {"title_nl": "Bank"})

        m.title_nl = None
        m.save()
        self.assertEqual(m.i18n, {})

    def test_fallback_getting_CharField(self):
        m = Blog.objects.create(title="Falcon")
        with override("de"):
            self.assertEqual(m.title_i18n, "Falcon")

        # this empty string in title_fr might be the result of an admin edit
        m = Blog.objects.create(title="Falcon", title_fr="")
        with override("fr"):
            self.assertEqual(m.title_i18n, "Falcon")

        # should also fallback if a value is None
        m = Blog.objects.create(title="Falcon", title_fr=None)
        with override("fr"):
            self.assertEqual(m.title_i18n, "Falcon")

        # should not fallback with string 'False'
        m = Blog.objects.create(title="Falcon", title_fr="False")
        with override("fr"):
            self.assertEqual(m.title_i18n, "False")

    def test_fallback_getting_TextField(self):
        DESCRIPTION = "Story about Falcon"
        m = TextModel(title="Falcon", description_en=DESCRIPTION)
        with override("fr"):
            self.assertEqual(m.description_i18n, DESCRIPTION)

        m = NullableTextModel.objects.create(description=DESCRIPTION, description_fr="")
        with override("fr"):
            self.assertEqual(m.description_i18n, DESCRIPTION)

    def test_creating_using_virtual_default_language_field(self):
        m = Blog.objects.create(title_en="Falcon")

        self.assertEqual(m.title, "Falcon")

    def test_creationg_prevents_double_definition(self):
        expected_message = (
            'Attempted override of "title" with "title_en". Only ' "one of the two is allowed."
        )
        with self.assertRaisesMessage(ValueError, expected_message):
            Blog.objects.create(title="Foo", title_en="Bar")

    def test_creating_with_nonexisting_field(self):
        """
        Blogs have titles, not names, so trying to add something with a name
        should raise an eror.
        """
        if DJANGO_VERSION < (4, 1):
            expected_message = "Blog() got an unexpected keyword argument 'name'"
        else:
            expected_message = "Blog() got unexpected keyword arguments: 'name'"

        with self.assertRaisesMessage(TypeError, expected_message):
            Blog.objects.create(name="Falcon")

        if DJANGO_VERSION < (4, 1):
            expected_message = "Blog() got an unexpected keyword argument 'name_nl'"
        else:
            expected_message = "Blog() got unexpected keyword arguments: 'name_nl'"
        with self.assertRaisesMessage(TypeError, expected_message):
            Blog.objects.create(title="Falcon", name_nl="Valk")

    def test_clean_required_languages_list(self):
        """
        Blog has required_languages=('nl', ), so this should raise an error
        if `title_nl` is not set.
        """
        m = Blog(title="Horse", body="Horses are nice")

        with self.assertRaises(ValidationError) as e:
            m.full_clean()

        self.assertEqual(
            {(field, tuple(errors)) for field, errors in e.exception},
            {
                ("title_nl", ("This field cannot be null.",)),
                ("body_nl", ("This field cannot be null.",)),
            },
        )

        # With an added `title_nl`, it should validate.
        m.title_nl = "Paard"
        m.body_nl = "foo"
        m.full_clean()

    def test_clean_required_languages_dict(self):
        class Model(models.Model):
            title = models.CharField(max_length=10)
            i18n = TranslationField(fields=["title"], required_languages={"title": ["nl", "de"]})

            class Meta:
                app_label = "test"

        with CreateTestModel(Model, translate=True):
            m = Model(title_nl="foo", title="bar")

        with self.assertRaises(ValidationError) as e:
            m.full_clean()

        self.assertEqual(
            {(field, tuple(errors)) for field, errors in e.exception},
            {("title_de", ("This field cannot be null.",))},
        )

    def test_textfield(self):
        """
        Constrains on the original field should also be enforced on the
        translated virtual fields (except for null/blank).

        Note that the database contraints are not enforced on the virtual fields,
        because those are ignored by Django.
        """

        expected_message = "value too long for type character varying(50)"

        short_str = "bla bla"
        long_str = "bla" * 40

        with transaction.atomic():
            with self.assertRaisesMessage(DataError, expected_message):
                TextModel.objects.create(title=long_str)

        with self.assertRaises(ValidationError) as e:
            b = TextModel.objects.create(title=short_str, title_nl=long_str)
            b.full_clean()

        self.assertEqual(
            sorted(list(e.exception), key=lambda v: v[0]),
            [
                ("description", ["This field cannot be blank."]),
                ("title_nl", ["Ensure this value has at most 50 characters (it has 120)."]),
            ],
        )

        TextModel.objects.create(title=short_str, description=long_str)

        m = TextModel.objects.create(
            title=short_str, description=short_str, description_nl=long_str, description_de=long_str
        )
        self.assertEqual(m.description_nl, long_str)

    @override_settings(
        MODELTRANS_AVAILABLE_LANGUAGES=("fr", "fy", "nl"),
        MODELTRANS_FALLBACK={"default": ("en",), "fy": ("nl", "en")},
    )
    def test_fallback_chain(self):
        """
        Testing the fallback chain setting for model
        """
        b = Blog.objects.create(
            title="Buzzard",
            i18n={"title_fy": "Mûzefalk", "title_nl": "Buizerd", "title_fr": "Buse"},
        )

        with override("nl"):
            self.assertEqual(b.title_i18n, "Buizerd")
        with override("fr"):
            self.assertEqual(b.title_i18n, "Buse")
        with override("fy"):
            self.assertEqual(b.title_i18n, "Mûzefalk")

        b = Blog.objects.create(title="Buzzard", i18n={"title_nl": "Buizerd", "title_fr": "Buse"})
        with override("fy"):
            self.assertEqual(b.title_i18n, "Buizerd")

        b = Blog.objects.create(title="Buzzard", i18n={"title_fr": "Buse"})
        with override("fy"):
            self.assertEqual(b.title_i18n, "Buzzard")
        with override("fr"):
            self.assertEqual(b.title_i18n, "Buse")

    def test_defer_i18n(self):
        Blog.objects.create(title="Buzzard", title_nl="Buizerd")

        qs = Blog.objects.defer("i18n")

        blog = qs[0]
        self.assertEqual(blog.get_deferred_fields(), {"i18n"})

        with self.assertRaises(ValueError):
            blog.title_i18n

class CustomFallbackLanguageTest(TestCase):
    def test_instance_fallback(self):
        instance = Challenge(default_language="nl", title="Hurray", i18n={"title_nl": "Hoera"})

        with override("de"):
            self.assertEqual(instance.title_i18n, "Hoera")
        with override("en"):
            self.assertEqual(instance.title_i18n, "Hurray")
        with override("nl"):
            self.assertEqual(instance.title_i18n, "Hoera")

    def test_empty_original_field(self):
        instance = Challenge(default_language="nl", title="", i18n={"title_nl": "Hoera"})

        with override("de"):
            self.assertEqual(instance.title_i18n, "Hoera")
        with override("en"):
            self.assertEqual(instance.title_i18n, "Hoera")

    def test_instance_fallback_follow_relation(self):
        challenge = Challenge.objects.create(default_language="nl", title="Hurray")
        content = ChallengeContent(
            challenge=challenge, content="Congratulations", i18n={"content_nl": "Gefeliciteerd"}
        )

        with override("de"):
            self.assertEqual(content.content_i18n, "Gefeliciteerd")
        with override("en"):
            self.assertEqual(content.content_i18n, "Congratulations")
        with override("nl"):
            self.assertEqual(content.content_i18n, "Gefeliciteerd")

    def test_instance_fallback_without_related_instance(self):
        content = ChallengeContent(content="Congratulations", i18n={"content_nl": "Gefeliciteerd"})
        self.assertEqual(content.content_nl, "Gefeliciteerd")
        self.assertEqual(content.content_en, "Congratulations")

        with override("de"):
            self.assertEqual(content.content_i18n, "Congratulations")
        with override("en"):
            self.assertEqual(content.content_i18n, "Congratulations")
        with override("nl"):
            self.assertEqual(content.content_i18n, "Gefeliciteerd")

class TranslatedFieldInheritanceTest(TestCase):
    def test_child_model_i18n_fields(self):
        self.assertFalse(hasattr(Article, "child_title_nl"))
        self.assertTrue(hasattr(ChildArticle, "child_title_nl"))

    def test_child_model_required_languages(self):
        self.assertTrue(Article._meta.get_field("title_nl").blank)
        self.assertFalse(ChildArticle._meta.get_field("title_nl").blank)

    def test_diff_i18n_parent_child_models_instances(self):
        """
        Test different behavior of Article and ChildArticle instances
        """
        article = Article(title="Title")
        article.full_clean()
        article.save()
        child_article = ChildArticle(title="Title", child_title="Child title")
        with self.assertRaises(ValidationError):
            child_article.full_clean()
        child_article.title_nl = "Title NL"
        child_article.child_title_nl = "Child title NL"
        child_article.full_clean()
        child_article.save()
        self.assertFalse("child_title_nl" in article.i18n)
        self.assertTrue("child_title_nl" in child_article.i18n)

class RefreshFromDbTest(TestCase):
    def test_refresh_from_db(self):
        b = Blog.objects.create(title="Falcon", i18n={"title_nl": "Valk", "title_de": "Falk"})
        Blog.objects.filter(title="Falcon").update(title="Falcon II")

        b.refresh_from_db()
        self.assertEqual(b.title, "Falcon II")
        self.assertEqual(b.title_nl, "Valk")

class CreatingInstancesTest(TestCase):
    def test_manager_create(self):
        b = Blog.objects.create(title="Falcon", title_nl="Valk")

        self.assertEqual(b.title, "Falcon")
        self.assertEqual(b.title_nl, "Valk")

    def test_manager_create_override(self):
        b = Blog.objects.create(title="Falcon", title_nl="Valk", i18n={"title_nl": "foo"})

        self.assertEqual(b.title_nl, "Valk")

    def test_model_constructor(self):
        b = Blog(title="Falcon", title_nl="Valk")
        b.save()

        self.assertEqual(b.title, "Falcon")
        self.assertEqual(b.title_nl, "Valk")

    def test_get_or_create(self):
        kwargs = dict(title="Falcon", title_nl="Valk")

        a = Blog.objects.create(**kwargs)
        b, created = Blog.objects.get_or_create(**kwargs)
        self.assertFalse(created)

        self.assertEqual(a, b)
        c, created = Blog.objects.get_or_create(title="Falcon")
        self.assertEqual(c, a)

        kwargs = dict(title="Buzzard", title_nl="Buizerd")
        a, _ = Blog.objects.get_or_create(**kwargs)
        b, created = Blog.objects.get_or_create(**kwargs)
        self.assertEqual(a, b)

    def test_update_or_create(self):
        a = Blog.objects.create(title="Falcon")

        defaults = dict(title_nl="Valk", title_de="Falk")
        b, created = Blog.objects.update_or_create(defaults=defaults, title="Falcon")
        self.assertFalse(created)
        self.assertEqual(a, b)

        a.refresh_from_db()
        self.assertEqual(a.title_de, defaults["title_de"])


from __future__ import absolute_import, division, print_function

import os
import skimage.transform
import numpy as np
import PIL.Image as pil

from .mono_dataset import MonoDataset

class NYUv2Dataset(MonoDataset):
	def __init__(self, *args, **kwargs):
		super(NYUv2Dataset, self).__init__(*args, **kwargs)

		self.K = np.array([[0.8107, 0, 0.5087, 0],
                           [0, 1.0822, 0.5286, 0],
                           [0, 0, 1, 0],
                           [0, 0, 0, 1]], dtype=np.float32)

		self.full_res_shape = (640, 480)

	def get_image_path(self, folder, frame_index, side):
		f_str = "img_{}{}".format(frame_index, self.img_ext)
		image_path = os.path.join(
			self.data_path,
			folder,
			f_str)
		return image_path

	def get_depth(self, folder, frame_index, side, do_flip):
		f_str = "gt_{}{}".format(frame_index, self.img_ext)
		depth_path = os.path.join(
			self.data_path,
			"depths/",
			f_str)

		depth_gt = pil.open(depth_path)
		depth_gt = depth_gt.resize(self.full_res_shape, pil.NEAREST)
		depth_gt = np.array(depth_gt).astype(np.float32) / 256

		if do_flip:
		    depth_gt = np.fliplr(depth_gt)

		return depth_gt

	def get_color(self, folder, frame_index, side, do_flip):
		color = self.loader(self.get_image_path(folder, frame_index, side))

		if do_flip:
			color = color.transpose(pil.FLIP_LEFT_RIGHT)

		return color

	def check_depth(self):
		line = self.filenames[0].split()
		scene_name = line[0]
		frame_index = int(line[1])

		depth_filename = os.path.join(
			self.data_path,
			"depths/",
			"gt_{}.jpg".format(int(frame_index)))


import numpy as np
import matplotlib.pyplot as plt
import mopc as mop

'''choose halo redshift and mass [Msun]
'''
z = 0.57
m = 2e13

'''choose radial range (x = r/rvir)
'''
x = np.logspace(np.log10(0.1),np.log10(10),100)

'''choose angular range [eg. arcmin]
'''
theta = np.arange(100)*0.05 + 0.5
theta = np.linspace(0.5,5.5,50)
sr2sqarcmin = 3282.8 * 60.**2

'''
choose observing frequency [GHz]
'''
nu = 150.

'''
read beam profile
'''
theta_rad, beam = np.loadtxt('data/beam_example.txt',unpack=True)
def f_beam(tht):
    tht_in = theta_rad
    return np.interp(tht, tht_in, beam, period=np.pi)

##########################################################################################
'''project a gNFW density profile [cgs unit] from Battaglia 2016
into a T_kSZ profile [muK arcmin^2]
'''
rho0 = np.log10(4e3 * (m/1e14)**0.29 * (1+z)**(-0.66))
xc = 0.5
bt = 3.83 * (m/1e14)**0.04 * (1+z)**(-0.025)
a2h= 1.1
par_rho = [rho0,xc,bt,a2h]
rho_gnfw = mop.rho_gnfw1h(x,m,z,par_rho[:3])
temp_ksz_gnfw = mop.make_a_obs_profile_sim_rho(theta,m,z,par_rho,f_beam)

plt.figure(0,figsize=(10, 4))
plt.subplot(1, 2, 1)
plt.loglog(x, rho_gnfw, lw=3, label='gNFW density')
plt.xlabel(r'x',size=14)
plt.ylabel(r"$\rho_{gas}(x) [g/cm^3]$ ",size=14)
plt.subplot(1, 2, 2)
plt.plot(theta, temp_ksz_gnfw*sr2sqarcmin,lw=3)
plt.xlabel(r'$\theta$ [arcmin]',size=14)
plt.ylabel(r'$T_{kSZ} [\mu K \cdot arcmin^2]$',size=14)
plt.tight_layout()
plt.savefig('fig/rho_gnfw.pdf')

##########################################################################################
'''project a gNFW profile of the thermal Pressure [cgs unit] from Battaglia et al. 2012
into a T_tSZ profile [muK arcmin^2]
'''

P0 = 18.1 * (m/1e14)**0.154 * (1+z)**(-0.758)
al = 1.
bt = 4.35 * (m/1e14)**0.0393 * (1+z)**0.415
a2h = 0.7
par_pth = [P0, al,bt,a2h]
pth_gnfw = mop.Pth_gnfw1h(x,m,z,par_pth[:3])
temp_tsz_gnfw = mop.make_a_obs_profile_sim_pth(theta,m,z,par_pth,nu,f_beam)

plt.figure(1,figsize=(10, 4))
plt.subplot(1, 2, 1)
plt.loglog(x, pth_gnfw,lw=3, label='gNFW pressure')
plt.xlabel(r'x',size=14)
plt.ylabel(r"$P_{th}(x) [dyne/cm^2]$",size=14)
plt.subplot(1, 2, 2)
plt.plot(theta, temp_tsz_gnfw*sr2sqarcmin,lw=3)
plt.xlabel(r'$\theta$ [arcmin]',size=14)
plt.ylabel(r'$T_{tSZ} [\mu K \cdot arcmin^2]$',size=14)
plt.tight_layout()
plt.savefig('fig/pth_gnfw.pdf')


import functools
import glob
import json
import os

import mock
import pytest

from google.protobuf import text_format
from google.cloud.firestore_v1beta1.proto import firestore_pb2
from google.cloud.firestore_v1beta1.proto import test_pb2
from google.cloud.firestore_v1beta1.proto import write_pb2

_UNIMPLEMENTED_FEATURES = [
    #  tests having to do with the ArrayUnion, ArrayRemove, and Delete
    # transforms
    'create-all-transforms.textproto',
    'create-arrayremove-multi.textproto',
    'create-arrayremove-nested.textproto',
    'create-arrayremove-noarray-nested.textproto',
    'create-arrayremove-noarray.textproto',
    'create-arrayremove.textproto',
    'create-arrayunion-multi.textproto',
    'create-arrayunion-nested.textproto',
    'create-arrayunion-noarray-nested.textproto',
    'create-arrayunion-noarray.textproto',
    'create-arrayunion.textproto',
    'set-all-transforms.textproto',
    'set-arrayremove-multi.textproto',
    'set-arrayremove-nested.textproto',
    'set-arrayremove-noarray-nested.textproto',
    'set-arrayremove-noarray.textproto',
    'set-arrayremove.textproto',
    'set-arrayunion-multi.textproto',
    'set-arrayunion-nested.textproto',
    'set-arrayunion-noarray-nested.textproto',
    'set-arrayunion-noarray.textproto',
    'set-arrayunion.textproto',
    'set-del-merge-alone.textproto',
    'set-del-merge.textproto',
    'set-del-mergeall.textproto',
    'set-del-nomerge.textproto',
    'update-all-transforms.textproto',
    'update-arrayremove-alone.textproto',
    'update-arrayremove-multi.textproto',
    'update-arrayremove-nested.textproto',
    'update-arrayremove-noarray-nested.textproto',
    'update-arrayremove-noarray.textproto',
    'update-arrayremove.textproto',
    'update-arrayunion-alone.textproto',
    'update-arrayunion-multi.textproto',
    'update-arrayunion-nested.textproto',
    'update-arrayunion-noarray-nested.textproto',
    'update-arrayunion-noarray.textproto',
    'update-arrayunion.textproto',
    'update-del-alone.textproto',
    'update-del-dot.textproto',
    'update-del-nested.textproto',
    'update-del-noarray-nested.textproto',
    'update-del-noarray.textproto',
    'update-del.textproto',
    ]

def _load_testproto(filename):
    with open(filename, 'r') as tp_file:
        tp_text = tp_file.read()
    test_proto = test_pb2.Test()
    text_format.Merge(tp_text, test_proto)
    shortname = os.path.split(filename)[-1]
    test_proto.description = (
        test_proto.description + ' (%s)' % shortname
        )
    return test_proto

_UNIMPLEMENTED_FEATURE_TESTPROTOS = [
    _load_testproto(filename) for filename in sorted(
        glob.glob('tests/unit/testdata/*.textproto'))
    if os.path.split(filename)[-1] in _UNIMPLEMENTED_FEATURES
]

IMPLEMENTED_FEATURE_TESTPROTOS = [
    _load_testproto(filename) for filename in sorted(
        glob.glob('tests/unit/testdata/*.textproto'))
    if not os.path.split(filename)[-1] in _UNIMPLEMENTED_FEATURES
]

_CREATE_TESTPROTOS = [
    test_proto for test_proto in IMPLEMENTED_FEATURE_TESTPROTOS
    if test_proto.WhichOneof('test') == 'create']

_GET_TESTPROTOS = [
    test_proto for test_proto in IMPLEMENTED_FEATURE_TESTPROTOS
    if test_proto.WhichOneof('test') == 'get']

_SET_TESTPROTOS = [
    test_proto for test_proto in IMPLEMENTED_FEATURE_TESTPROTOS
    if test_proto.WhichOneof('test') == 'set']

_UPDATE_TESTPROTOS = [
    test_proto for test_proto in IMPLEMENTED_FEATURE_TESTPROTOS
    if test_proto.WhichOneof('test') == 'update']

_UPDATE_PATHS_TESTPROTOS = [
    test_proto for test_proto in IMPLEMENTED_FEATURE_TESTPROTOS
    if test_proto.WhichOneof('test') == 'update_paths']

_DELETE_TESTPROTOS = [
    test_proto for test_proto in IMPLEMENTED_FEATURE_TESTPROTOS
    if test_proto.WhichOneof('test') == 'delete']

_LISTEN_TESTPROTOS = [
    test_proto for test_proto in IMPLEMENTED_FEATURE_TESTPROTOS
    if test_proto.WhichOneof('test') == 'listen']

def _mock_firestore_api():
    firestore_api = mock.Mock(spec=['commit'])
    commit_response = firestore_pb2.CommitResponse(
        write_results=[write_pb2.WriteResult()],
    )
    firestore_api.commit.return_value = commit_response
    return firestore_api

def _make_client_document(firestore_api, testcase):
    from google.cloud.firestore_v1beta1 import Client
    from google.cloud.firestore_v1beta1.client import DEFAULT_DATABASE
    import google.auth.credentials

    _, project, _, database, _, doc_path = testcase.doc_ref_path.split('/', 5)
    assert database == DEFAULT_DATABASE

    # Attach the fake GAPIC to a real client.
    credentials = mock.Mock(spec=google.auth.credentials.Credentials)
    client = Client(project=project, credentials=credentials)
    client._firestore_api_internal = firestore_api
    return client, client.document(doc_path)

def _run_testcase(testcase, call, firestore_api, client):
    if getattr(testcase, 'is_error', False):
        # TODO: is there a subclass of Exception we can check for?
        with pytest.raises(Exception):
            call()
    else:
        call()
        firestore_api.commit.assert_called_once_with(
            client._database_string,
            list(testcase.request.writes),
            transaction=None,
            metadata=client._rpc_metadata)

@pytest.mark.parametrize('test_proto', _CREATE_TESTPROTOS)
def test_create_testprotos(test_proto):
    testcase = test_proto.create
    firestore_api = _mock_firestore_api()
    client, document = _make_client_document(firestore_api, testcase)
    data = convert_data(json.loads(testcase.json_data))
    call = functools.partial(document.create, data)
    _run_testcase(testcase, call, firestore_api, client)

@pytest.mark.parametrize('test_proto', _GET_TESTPROTOS)
def test_get_testprotos(test_proto):
    testcase = test_proto.get
    # XXX this stub currently does nothing because no get testcases have
    # is_error; taking this bit out causes the existing tests to fail
    # due to a lack of batch getting
    try:
        testcase.is_error
    except AttributeError:
        return
    else:  # pragma: NO COVER
        testcase = test_proto.get
        firestore_api = _mock_firestore_api()
        client, document = _make_client_document(firestore_api, testcase)
        call = functools.partial(document.get, None, None)
        _run_testcase(testcase, call, firestore_api, client)

@pytest.mark.parametrize('test_proto', _SET_TESTPROTOS)
def test_set_testprotos(test_proto):
    testcase = test_proto.set
    firestore_api = _mock_firestore_api()
    client, document = _make_client_document(firestore_api, testcase)
    data = convert_data(json.loads(testcase.json_data))
    if testcase.HasField("option"):
        merge = convert_set_option(testcase.option)
    else:
        merge = False
    call = functools.partial(document.set, data, merge=merge)
    _run_testcase(testcase, call, firestore_api, client)

@pytest.mark.parametrize('test_proto', _UPDATE_TESTPROTOS)
def test_update_testprotos(test_proto):
    testcase = test_proto.update
    firestore_api = _mock_firestore_api()
    client, document = _make_client_document(firestore_api, testcase)
    data = convert_data(json.loads(testcase.json_data))
    if testcase.HasField("precondition"):
        option = convert_precondition(testcase.precondition)
    else:
        option = None
    call = functools.partial(document.update, data, option)
    _run_testcase(testcase, call, firestore_api, client)

@pytest.mark.skip(
    reason="Python has no way to call update with a list of field paths.")
@pytest.mark.parametrize('test_proto', _UPDATE_PATHS_TESTPROTOS)
def test_update_paths_testprotos(test_proto):  # pragma: NO COVER
    pass

@pytest.mark.parametrize('test_proto', _DELETE_TESTPROTOS)
def test_delete_testprotos(test_proto):
    testcase = test_proto.delete
    firestore_api = _mock_firestore_api()
    client, document = _make_client_document(firestore_api, testcase)
    if testcase.HasField("precondition"):
        option = convert_precondition(testcase.precondition)
    else:
        option = None
    call = functools.partial(document.delete, option)
    _run_testcase(testcase, call, firestore_api, client)

@pytest.mark.skip(reason="Watch aka listen not yet implemented in Python.")
@pytest.mark.parametrize('test_proto', _LISTEN_TESTPROTOS)
def test_listen_paths_testprotos(test_proto):  # pragma: NO COVER
    pass

@pytest.mark.skip(reason="Feature not yet implemented in Python.")
@pytest.mark.parametrize('test_proto', _UNIMPLEMENTED_FEATURE_TESTPROTOS)
def test_unimplemented_features_testprotos(test_proto):  # pragma: NO COVER
    pass

def convert_data(v):
    # Replace the strings 'ServerTimestamp' and 'Delete' with the corresponding
    # sentinels.
    from google.cloud.firestore_v1beta1 import SERVER_TIMESTAMP, DELETE_FIELD

    if v == 'ServerTimestamp':
        return SERVER_TIMESTAMP
    elif v == 'Delete':
        return DELETE_FIELD
    elif isinstance(v, list):
        return [convert_data(e) for e in v]
    elif isinstance(v, dict):
        return {k: convert_data(v2) for k, v2 in v.items()}
    else:
        return v

def convert_set_option(option):
    from google.cloud.firestore_v1beta1 import _helpers

    if option.fields:
        return [
            _helpers.FieldPath(*field.field).to_api_repr()
            for field in option.fields
        ]

    assert option.all
    return True

def convert_precondition(precond):
    from google.cloud.firestore_v1beta1 import Client

    if precond.HasField('exists'):
        return Client.write_option(exists=precond.exists)

    assert precond.HasField('update_time')
    return Client.write_option(last_update_time=precond.update_time)


# -*- coding: utf-8 -*-
"""
@ Instituto Tecnologico y de Estudios Superiores de Occidente
@author: Adrian Ramos Perez

"""
from matplotlib import pyplot as plt
import numpy as np
from scipy import signal as sig

from sklearn.datasets import make_blobs
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.metrics import r2_score,mean_squared_error,make_scorer

# Audio library imports:
import librosa as lb
import librosa.display
import soundfile as sf
from IPython.display import Audio

# Deep Learning imports:
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.optimizers import Adam, SGD
#from TDNN_Layer import TDNNLayer

#%% Lectura de archivos de audio
samples, sampling_rate= lb.load("../Data/Train/ADS_train_audio_44kHz.wav", sr=None, mono=True,offset=0.0,duration=None)
# Duración en segundos:
duration = len(samples)/sampling_rate
# Escuchar el audio:
Audio("../Data/Train/ADS_train_audio_22kHz.wav")
# Cargar respuesta al impulso:
ir_samples, ir_sampling_rate= lb.load("../Data/Train/ir_golpe22_short.wav", sr=None, mono=True,offset=0.0,duration=None)
ir_duration = len(ir_samples)/ir_sampling_rate

ts_samples, ts_sampling_rate = lb.load("../Data/Train/QueenWatch1.wav", sr=22050, mono=True,offset=0.0,duration=None)
ts_duration = len(ts_samples)/ts_sampling_rate
#%% Plot Audio Signals

# Original Speech Signal:
# fig = plt.figure(figsize=(20,10))
# plt.title(f'ADS Speech Signal (original) sr={sampling_rate}',fontsize=30)
# plt.grid()
# lb.display.waveplot(y=samples,sr=sampling_rate)
# plt.show()
# fig.savefig(f'../Figures/Original_signal_sr={sampling_rate}')

# Impulse Response Signal:
fig = plt.figure(figsize=(20,10))
plt.title(f'Impulse Response sr={ir_sampling_rate}', fontsize=30)
plt.grid()
lb.display.waveplot(y=ir_samples,sr=ir_sampling_rate)
plt.show()
#fig.savefig(f'../Figures/Impulse_Response_sr={ir_sampling_rate}')

# Test Audio Signal:
# fig = plt.figure(figsize=(20,10))
# plt.title(f'Test Audio Signal sr={ts_sampling_rate}', fontsize=30)
# plt.grid()
# lb.display.waveplot(y=ts_samples,sr=ts_sampling_rate)
# plt.show()
# Audio("../Data/QueenWatch1.wav")
# fig.savefig(f'../Figures/Test_audio_signal_Queen_sr={ts_sampling_rate}')
#%% PREPROCESSING

# Scaling
# samples = preprocessing.scale(samples)
# ir_samples = preprocessing.scale(ir_samples)

#%% Convolution between train signals and impulse response.

conv_signal = np.convolve(samples, ir_samples, mode='same')
# Plot the resulting singal:
fig = plt.figure(figsize=(20,10))
plt.title('Convolved signal (Impulse Response * Original Speech Signal)',fontsize=20)
plt.grid()
lb.display.waveplot(y=conv_signal,sr=sampling_rate)
plt.show()
#fig.savefig(f'../Figures/Convolved_signal')
# signal.astype(np.int16).tofile('../Data/conv_signal.wav')
#sf.write('../Data/Train/convolved_signal.wav',conv_signal, sampling_rate, subtype=None)
#%% Delayed Signal (Lags) Matrix generation

n_lag = 5
n_lag_range = np.array([i for i in range(0,n_lag)])
dim_conv_signal = np.shape(conv_signal)
# Lag matrix of dimmensions: convolved signal - lag number (rows) x lag number (columns)
lag_matrix = np.zeros((dim_conv_signal[0]-n_lag, n_lag))
for i in range(n_lag):
        lag_matrix[:,i] = conv_signal[i : dim_conv_signal[0]-n_lag+i]
# del_sig = conv_signal[2:-1]
# del_sig2 = conv_signal[1:-2]
# del_sig3 = conv_signal[:-3]

# del_sig = conv_signal[:-7]
# del_sig2 = conv_signal[1:-6]
# del_sig3 = conv_signal[2:-5]
# del_sig4 = conv_signal[3:-4]
# del_sig5 = conv_signal[4:-3]
# del_sig6 = conv_signal[5:-2]
# del_sig7 = conv_signal[6:-1]
# matrix = np.c_[del_sig, del_sig2, del_sig3]
# matrix7 = np.c_[del_sig, del_sig2, del_sig3, del_sig4, del_sig5, del_sig6, del_sig7]

# Random plot one of the lagged signals:
r_int = np.random.randint(0,n_lag)
fig = plt.figure(figsize=(20,10))
plt.title(f'{r_int} lags sample of the Lagged Signals Matrix : ',fontsize=20)
plt.grid()
lb.display.waveplot(y=lag_matrix[r_int],sr=sampling_rate)
plt.show()

#%% Train dataset and Test dataset split
X_train = lag_matrix
Y_train = samples[n_lag:]

#%% Time-Delay Neural Network model

# Hyperparameters:
Epochs = 200
Learning_rate = 0.05
Decay_rate = Learning_rate/Epochs
Momentum = 0.8
Batch_size = 200

# Model definition:
model = Sequential()

nn_i, nn_h1, nn_h2 = 10,5,5
# Layer(s) definition:
model.add(Dense(nn_i, input_shape=(n_lag,), activation="tanh", name="input_layer_2"))
model.add(Dense(nn_h1, activation="tanh", name="hidden_layer_1"))
# model.add(Dense(nn_h2, activation="tanh", name="hidden_layer_2"))
# model.add(TDNNLayer([-5,2],True, input_shape=(45053,1), activation="tanh"))
model.add(Dense(1, activation='linear', name="output_layer"))

# Compile model:
model.compile(optimizer=Adam(learning_rate =Learning_rate), loss='MeanSquaredError', metrics=['mse'])
xtrain = lag_matrix
# xtrain = preprocessing.scale(xtrain)
ytrain = samples[n_lag:]

# Train model:
m_hist = model.fit(xtrain, ytrain, epochs=Epochs, batch_size=Batch_size, verbose=1)

#%% Plot the loss function and the metrics
fig, axs = plt.subplots(2, 1, figsize=(20,12))
fig.subplots_adjust(hspace=0.3)
axs[0].set_title(f'Loss: {n_lag} lags, Neurons: {nn_i}i, {nn_h1}h, {nn_h2}h2, Epochs:{Epochs}, Batch:{Batch_size}, LR:{Learning_rate}', fontsize=20, fontname="Times New Roman", fontweight='bold')
axs[0].plot(m_hist.history['loss'], 'r', label='train')
# ax.plot(m_hist.history['val_loss'], 'b' ,label='val')
axs[0].set_xlabel(r'Epoch', fontsize=20)
axs[0].set_ylabel(r'Loss', fontsize=20)
axs[0].grid(True)
axs[0].legend()
axs[0].tick_params(labelsize=20)

# Plot the Mean-Square Error
# fig, ax = plt.subplots(2, 2, figsize=(10,6))
# ax.plot(np.sqrt(m_hist.history['mean_squared_error']), 'r', label='train')
# ax.plot(np.sqrt(m_hist.history['val_mean_squared_error']), 'b' ,label='val')
axs[1].set_title(f'MSE: {n_lag} lags, Neurons: {nn_i}i, {nn_h1}h, {nn_h2}h2, Epochs:{Epochs}, Batch:{Batch_size}, LR:{Learning_rate}', fontsize=20, fontname="Times New Roman", fontweight='bold')
axs[1].plot(np.sqrt(m_hist.history['mse']), 'r', label='train')
# ax.plot(np.sqrt(m_hist.history['val_mse']), 'b' ,label='val')
axs[1].set_xlabel(r'Epoch', fontsize=20)
axs[1].set_ylabel(r'MSE', fontsize=20)
axs[1].grid(True)
axs[1].legend()
axs[1].tick_params(labelsize=20)
fig.savefig(f'../Figures/Loss_MSE_{n_lag}lags_{nn_i}i_{nn_h1}h1_{nn_h2}h2_{Epochs}e_{Batch_size}bs_{Learning_rate}lr.png')
#%% View model
from keras.utils.vis_utils import plot_model
plot_model(model, to_file='../Figures/model.png', show_shapes = True)
#%% Prediction

y_pred = model.predict(xtrain)

# Plotting original signal:
fig, axs = plt.subplots(2, 1, figsize=(30,12))
fig.subplots_adjust(hspace=0.4)
# plt.tight_layout(pad=0.7)
axs[0].set_title(f'Original Signal', fontsize=30, fontname="Times New Roman", fontweight='bold')
axs[0].plot(samples,c='b')
axs[0].set_xlabel('Samples', fontsize=30)
axs[0].set_ylabel('Amplitude', fontsize=30)
axs[0].grid(True)

# Plotting predicted signal (y hat):
# fig = plt.figure(figsize=(20,10))
axs[1].set_title(f'Y_hat: {n_lag} lags, Neurons: {nn_i}i, {nn_h1}h, {nn_h2}h2, Epochs:{Epochs}, Batch:{Batch_size}, LR:{Learning_rate}', fontsize=30, fontname="Times New Roman", fontweight='bold')
axs[1].plot(y_pred,c='g')
axs[1].set_xlabel('Samples', fontsize=30)
axs[1].set_ylabel('Amplitude', fontsize=30)
axs[1].grid(True)
plt.show()
fig.savefig(f'../Figures/y_hat_{n_lag}lags_{nn_i}i_{nn_h1}h_{nn_h2}h2_{Epochs}e_{Batch_size}bs_{Learning_rate}lr.png', dpi=450 )

# Save to sound file:
sf.write(f'../Data/predicted_signal_{n_lag}lags_{nn_i}i_{nn_h1}h1_{nn_h2}h2_{Epochs}e_{Batch_size}bs_{Learning_rate}lr.wav',y_pred, sampling_rate, subtype=None)
#%% Evaluate performance:

R2_score = r2_score(ytrain,y_pred)
# Plotting R2_Score
xmin,xmax = min(ytrain),max(ytrain)
xline = np.linspace(xmin,xmax)
fig = plt.figure(figsize=(20,15))
plt.scatter(ytrain,y_pred,label='Estimation')
plt.plot(xline,xline,'k--',label='Perfect estimation')
plt.xlabel('Real output', fontsize=35, fontname="Times New Roman",)
plt.ylabel('Estimation output', fontsize=35, fontname="Times New Roman",)
plt.suptitle('R^2=%0.4f'%R2_score, fontsize=40, fontname="Times New Roman", fontweight='bold')
plt.title(f'{n_lag}lags_{nn_i}i_{nn_h1}h1_{nn_h2}h2_{Epochs}e_{Batch_size}bs_{Learning_rate}lr', fontsize=30,fontname="Times New Roman" )
plt.legend()
plt.grid()
plt.show()
fig.savefig(f'../Figures/R2_{n_lag}lags_{nn_i}i_{nn_h1}h1_{nn_h2}h2_{Epochs}e_{Batch_size}bs_{Learning_rate}lr.png', dpi=450)

#%%  Test

# Queen_y_pred = model.predict(ts_samples)
# sf.write('../Data/queen_predicted_signal.wav',Queen_y_pred, ts_sampling_rate, subtype=None)
# plt.show()
# plt.figure()
# plt.title('Queen Predicted Signal')
# plt.plot(Queen_y_pred,c='r')

# score = model.evaluate(matrix, samples[3:])
# print('Test loss:', score[0])
# print('Test mse:', score[1])

# corr = sig.correlate(y_pred, y_pred)

# plt.figure()
# plt.title('Signals Correlation')
# plt.plot(corr)
# plt.show()

#%% Hyperparameter tunning
from sklearn.model_selection import GridSearchCV

# RandomSearchCV

#%% XGBoost Model Approach

import xgboost as xgb
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import mean_squared_error, mean_absolute_error, mean_absolute_percentage_error, r2_score

xgb_model = xgb.XGBRegressor()
# parameter definitions
param_grid = {
    'n_estimators': range(100, 200, 10), # De 50 hasta 250 estimadores en aumentos de 50
    'learning_rate': [0.05, 0.1, 0.3, 1], # Eta
    'max_depth': range(2, 5, 1),
    'subsample': [0.2, 0.5, 0.7, 1] # Submuestreo aleatorio para prevenir overfitting
}

grid_search = GridSearchCV(estimator=xgb_model,
                  param_grid=param_grid,
                  #scoring='neg_mean_squared_error',
                  scoring='r2',
                  n_jobs=-1, # All procesors
                  cv=5,
                  verbose=4)

#grid_search.fit(X_train, Y_train)

print("Mejores parámetros:",grid_search.best_params_)

# Dictionary of best parameters:
best_params = {
    'n_estimators': 100, # De 50 hasta 250 estimadores en aumentos de 50
    'learning_rate': 0.1, # Eta
    'max_depth': 5,
    'subsample': 0.5
    }

# Create model with best params:
xgb_model_best_params = xgb.XGBRegressor(
                             n_estimators = best_params['n_estimators'],
                             max_depth = best_params['max_depth'],
                             eval_metric = 'r2',
                             learning_rate = best_params['learning_rate'],
                             verbosity = 4,
                             n_jobs = -1,
                             gamma = 1,)
#%%
xgb_best_model = grid_search.best_estimator_
# Save model for model persistence:
xgb_best_model.save_model("../Models/xgb_best_model.json")

# Realizar predicción sobre la señal original:
xgb_best_Yhat = xgb_best_model.predict(conv_signal)

# Calcular scores de la predicción del modelo:
xgb_best_rmse = (mean_squared_error(samples, xgb_best_Yhat)**0.5)
xgb_best_mae = mean_absolute_error(samples, xgb_best_Yhat)
xgb_best_mape = mean_absolute_percentage_error(samples, xgb_best_Yhat)
xgb_best_r2 = r2_score(samples, xgb_best_Yhat)
print('RMSE: {0:.3f}'.format(xgb_best_rmse))
print('MAE: {0:.3f}'.format(xgb_best_mae))
print('MAPE: {0:.3f}'.format(xgb_best_mape))
print('R^2: {0:.3f}'.format(xgb_best_r2))

# Graph and save the predicted signal:
xgb_best_Yhat = xgb_best_model.predict(conv_signal)

# Plotting original signal:
fig, axs = plt.subplots(2, 1, figsize=(30,12))
fig.subplots_adjust(hspace=0.4)
# plt.tight_layout(pad=0.7)
axs[0].set_title(f'Original Signal', fontsize=30, fontname="Times New Roman", fontweight='bold')
axs[0].plot(samples,c='b')
axs[0].set_xlabel('Samples', fontsize=30)
axs[0].set_ylabel('Amplitude', fontsize=30)
axs[0].grid(True)

# Plotting predicted signal (y hat):
# fig = plt.figure(figsize=(20,10))
axs[1].set_title(f'Y_hat: {n_lag} lags, Neurons: {nn_i}i, {nn_h1}h, {nn_h2}h2, Epochs:{Epochs}, Batch:{Batch_size}, LR:{Learning_rate}', fontsize=30, fontname="Times New Roman", fontweight='bold')
axs[1].plot(xgb_best_Yhat ,c='g')
axs[1].set_xlabel('Samples', fontsize=30)
axs[1].set_ylabel('Amplitude', fontsize=30)
axs[1].grid(True)
plt.show()
fig.savefig(f'../Figures/xgb_yhat_{n_lag}lags_{n_estimators}estimators_{max_depth}mdepth_{nn_h2}h2_{Epochs}e_{Batch_size}bs_{xgb_learning_rate}lr.png', dpi=450 )

# Save to sound file:
sf.write(f'../Data/Test/xgb_predicted_signal_{n_lag}lags_{nn_i}i_{nn_h1}h1_{nn_h2}h2_{Epochs}e_{Batch_size}bs_{Learning_rate}lr.wav',y_pred, sampling_rate, subtype=None)

#%% Save a model to JSON

# serialize model to JSON
model_json = model.to_json()

with open("../Figures/TDNN_model.json", "w") as json_file:
    json_file.write(model_json)

# save weights to HDF5
model.save_weights("../Figures/TDNN_model.h5")
print("Model saved")

#%% Retrieve the model: load json and create model
# from keras.models import model_from_json
# json_file = open('../Figures/TDNN_model.json', 'r')
# saved_model = json_file.read()
# close the file as good practice:
# json_file.close()
# model_ = model_from_json(saved_model)
# load weights into new model:
# model_.load_weights("../Figures/TDNN_model.h5")
# print("Model loaded")


"""setup script for yodl
"""

from setuptools import setup

with open('requirements.txt') as handle:
    REQUIREMENTS = handle.readlines()

setup(
    name='yodl',
    version='0.0.1',
    packages=['yodl'],
    description='Django models from YAML!',
    author='Bruno Lange',
    author_email='blangeram@gmail.com',
    url='https://github.com/brunolange/yodl',
    install_requires=REQUIREMENTS,
    python_requires='>=3',
)


from flask import render_template, Blueprint, jsonify, current_app
from flask_login import current_user

from noirart.notifications import push_follow_notification, push_collect_notification
from noirart.models import User, Notification, Photo

ajax_bp = Blueprint('ajax', __name__)

@ajax_bp.route('/profile/<int:user_id>')
def get_profile(user_id):
    user = User.query.get_or_404(user_id)
    return render_template('main/profile_popup.html', user=user)

@ajax_bp.route('/notifications-count')
def notifications_count():
    if not current_user.is_authenticated:
        return jsonify(message='Login required.'), 403

    count = Notification.query.with_parent(current_user).filter_by(is_read=False).count()
    return jsonify(count=count)

@ajax_bp.route('/<int:photo_id>/followers-count')
def collectors_count(photo_id):
    photo = Photo.query.get_or_404(photo_id)
    count = len(photo.collectors)
    return jsonify(count=count)

@ajax_bp.route('/collect/<int:photo_id>', methods=['POST'])
def collect(photo_id):
    current_app.logger.debug('ajax.py:collect()--->')
    if not current_user.is_authenticated:
        return jsonify(message='Login required.'), 403
    # if not current_user.confirmed:
    #     return jsonify(message='Confirm account required.'), 400
    if not current_user.can('COLLECT'):
        return jsonify(message='No permission.'), 403

    photo = Photo.query.get_or_404(photo_id)
    if current_user.is_collecting(photo):
        return jsonify(message='Already collected.'), 400

    current_user.collect(photo)
    if current_user != photo.author and photo.author.receive_collect_notification:
        push_collect_notification(collector=current_user, photo_id=photo_id, receiver=photo.author)
    return jsonify(message='Photo collected.')

@ajax_bp.route('/uncollect/<int:photo_id>', methods=['POST'])
def uncollect(photo_id):
    current_app.logger.debug('ajax.py uncollect()')
    if not current_user.is_authenticated:
        return jsonify(message='Login required.'), 403

    photo = Photo.query.get_or_404(photo_id)
    if not current_user.is_collecting(photo):
        return jsonify(message='Not collect yet.'), 400

    current_user.uncollect(photo)
    return jsonify(message='Collect canceled.')


# -*- coding: utf-8 -*-

import re
from datetime import timedelta

from pyload.core.utils.convert import to_bytes

from ..base.simple_downloader import SimpleDownloader

class FastshareCz(SimpleDownloader):
    __name__ = "FastshareCz"
    __type__ = "downloader"
    __version__ = "0.45"
    __status__ = "testing"

    __pattern__ = r"https?://(?:www\.)?fastshare\.cz/\d+/.+"
    __config__ = [
        ("enabled", "bool", "Activated", True),
        ("use_premium", "bool", "Use premium account if available", True),
        ("fallback", "bool", "Fallback to free download if premium fails", True),
        ("chk_filesize", "bool", "Check file size", True),
        ("max_wait", "int", "Reconnect if waiting time is greater than minutes", 10),
    ]

    __description__ = """FastShare.cz downloader plugin"""
    __license__ = "GPLv3"
    __authors__ = [
        ("Walter Purcaro", "vuolter@gmail.com"),
        ("GammaC0de", "nitzo2001[AT]yahoo[DOT]com"),
        ("ondrej", "git@ondrej.it"),
    ]

    URL_REPLACEMENTS = [("#.*", "")]

    COOKIES = [("fastshare.cz", "lang", "en")]

    NAME_PATTERN = r'<h2 title="(.+?)" class="section_title'
    SIZE_PATTERN = r' (?P<S>\d+) (?P[\w^_]+)'
    TEMP_OFFLINE_PATTERN = r"[^\w](503\s|[Mm]aint(e|ai)nance|[Tt]emp([.-]|orarily))"
    OFFLINE_PATTERN = r">(The file has been deleted|Requested page not found|This file is no longer available)"

    LINK_FREE_PATTERN = r'form .*target="iframe_dwn" .*action=([^>]+)'
    LINK_PREMIUM_PATTERN = r"(https?://\w+\.fastshare\.cz/download\.php\?id=\d+&)"

    SLOT_ERROR = "> 100% of FREE slots are full"
    CREDIT_ERROR = " credit for "

    def check_errors(self):
        if self.SLOT_ERROR in self.data:
            errmsg = self.info["error"] = self._("No free slots")
            self.retry(12, 60, errmsg)

        if self.CREDIT_ERROR in self.data:
            errmsg = self.info["error"] = self._("Not enough traffic left")
            self.log_warning(errmsg)
            self.restart(premium=False)

        self.info.pop("error", None)

    def handle_free(self, pyfile):
        m = re.search(self.LINK_FREE_PATTERN, self.data)
        if m is None:
            self.error(self._("LINK_FREE_PATTERN not found"))

        self.link = self.fixurl(m.group(1))

    def check_download(self):
        check = self.scan_download(
            {
                "parallel-dl": re.compile(
                    rb"<title>FastShare.cz</title>|


import time

import numpy as np
import pymongo
import matplotlib.pyplot as plt

from env.core import AircraftAgentSet
from env.load import load_data_set, load_and_split_data
from env.model import Routing, FlightPlan, Aircraft
from env.render import plot_line, make_random_color, make_color, draw
from env.utils import pnpoly

# --------------------------------------------
# -- Generate Continuous Conflict Scenarios --
# --------------------------------------------

"""
 1. 找到所有经过武汉扇区（vertices）的航路 → wh_routing_list；
 2. 截取wh_routing_list航路中在武汉扇区（vertices）里的航段；
 3. 随机抽取120个routing，构建飞行计划和AgentSet；
 4. 运行AgentSet，并进行冲突探测；
 5. 剔除冲突时间-起飞时间<=600的飞行计划，并重建AgentSet；
 6. 运行AgentSet，并进行冲突探测；
 7. 记录冲突信息和飞行计划；
 8. 记录各个冲突信息和飞行计划；
"""

shift = 360

vertices = ((109.51666666666667, 31.9),
            (110.86666666666666, 33.53333333333333),
            (114.07, 32.125),
            (115.81333333333333, 32.90833333333333),
            (115.93333333333334, 30.083333333333332),
            (114.56666666666666, 29.033333333333335),
            (113.12, 29.383333333333333),
            (109.4, 29.516666666666666),
            (109.51666666666667, 31.9),
            (109.51666666666667, 31.9))

data_set = load_data_set()
flight_level = {i - 20: i * 300.0 + int(i >= 29) * 200.0 for i in range(20, 41)}  # 6000~12500

# step 1和2
def search_routing_in_wuhan():
    import simplekml

    kml = simplekml.Kml()
    folder = kml.newfolder(name='Polygons')
    plot_line(folder,
              (p + (8100.0,) for p in vertices),
              name='border')

    folder = kml.newfolder(name='Lines')
    inner_routes, route_id = [], []
    for r_id, routing in data_set.rou_dict.items():
        wpt_list = routing.wpt_list

        plot_line(folder,
                  (wpt.location.tuple() + (8100.0,) for wpt in wpt_list),
                  name=r_id,
                  color=make_random_color())

        i, idx = 0, []
        for i, wpt in enumerate(wpt_list):
            if pnpoly(vertices, wpt.location.tuple()):
                idx.append(i)

        if len(idx) > 0:
            min_v, max_v = min(idx), max(idx)
            if min_v == 0:
                mode = 'start'
            elif max_v == i:
                mode = 'end'
            else:
                mode = 'part'

            wpt_list_part = wpt_list[max(0, min_v - 1): min(i + 1, max_v + 2)]
            inner_routes.append([r_id, wpt_list_part, mode])
            route_id.append(r_id)

            plot_line(folder,
                      (wpt.location.tuple() + (8100.0,) for wpt in wpt_list_part),
                      name=r_id + '_part',
                      color=make_color(255, 0, 0))
    kml.save('wuhan.kml')

    count = 0
    for fpl in data_set.fpl_dict.values():
        if fpl.routing.id in route_id:
            count += 1
    print(count)

    return inner_routes

# step 3
def get_fpl_random(routes, interval=30, number=100, max_time=43200):
    fpl_set = list(data_set.fpl_dict.values())
    np.random.shuffle(fpl_set)

    count = 0
    fpl_list, candi = [], {}
    for j in range(0, max_time, interval):
        np.random.shuffle(routes)
        candi[j] = []

        for [r_id, wpt_list, mode] in routes[:number]:
            fpl_template = fpl_set[count % len(fpl_set)]

            # routing
            routing = Routing(id=r_id, wpt_list=wpt_list)
            # aircraft
            aircraft = fpl_template.aircraft
            # min_alt
            if mode == 'start':  # 在扇区内起飞的航班，上升后平飞
                min_alt = 6000.0
            else:
                min_alt = flight_level[np.random.randint(0, len(flight_level) - 1)]
            # max_alt
            if mode == 'end':  # 在扇区内落地的航班，下降
                max_alt = 6000.0
            elif np.random.randint(0, 60) % 2 == 0:  # 1/2的航班起始高度等于目标高度
                max_alt = flight_level[np.random.randint(0, len(flight_level) - 1)]
            else:
                max_alt = min_alt
            # flight plan
            fpl = FlightPlan(id=str(count),
                             routing=routing,
                             aircraft=aircraft,
                             startTime=j,
                             min_alt=min_alt,
                             max_alt=max_alt)
            fpl_list.append(fpl)
            candi[j].append(fpl.id)
            count += 1

    return fpl_list, candi

# step 4
def run_scenario(fpl_list, candi):
    print('\t>>>', len(fpl_list))
    agent_set = AircraftAgentSet(fpl_list=fpl_list, candi=candi)

    all_conflicts, record = [], {'flow': [], 'conflict': []}
    shift_list, check_list = [], []
    start = time.time()
    while True:
        agent_set.step(duration=1)
        conflicts = []
        for c in agent_set.detect():
            c_id = c.id
            [a0_id, a1_id] = c_id.split('-')
            c_id_reverse = a1_id + '-' + a0_id
            if c_id in check_list or c_id_reverse in check_list:
                continue

            check_list.append(c_id)
            check_list.append(c_id_reverse)
            fpl0 = agent_set.agents[a0_id].fpl
            fpl1 = agent_set.agents[a1_id].fpl
            if c.time - fpl0.startTime < shift and a0_id not in shift_list:
                shift_list.append(a0_id)
            if c.time - fpl1.startTime < shift and a1_id not in shift_list:
                shift_list.append(a1_id)
            conflicts.append(c)

        if len(conflicts) > 0:
            all_conflicts += conflicts

        now = agent_set.time
        if now % 1000 == 0:
            print('\t>>>', now, len(agent_set.agent_id_en), len(all_conflicts))
        record['flow'].append([now, len(agent_set.agent_id_en)])
        record['conflict'].append([now, len(conflicts)])

        if agent_set.is_done():
            print('场景运行结束：', now, len(conflicts), time.time() - start)
            break

    return all_conflicts, record, shift_list

# Step 7
def write_in_db(name, conflict_info, fpl_info, col='scenarios_meta'):
    database = pymongo.MongoClient('localhost')['admin']
    collection = database[col]

    c_times = [c.time for c in conflict_info]
    conflicts = [c.to_dict() for c in conflict_info]
    fpl_list = [fpl.to_dict() for fpl in fpl_info]
    collection.insert_one(dict(id=name, start=min(c_times) - 301, conflicts=conflicts, fpl_list=fpl_list))

def analysis(record, new_record, candi, new_candi, name):
    fig, axes = plt.subplots(3, 1)
    x, y = [], []
    for [t, flow, *_] in record['flow']:
        x.append(t)
        y.append(flow)
    axes[0].plot(x, y, label='before')
    x, y = [], []
    for [t, flow, *_] in new_record['flow']:
        x.append(t)
        y.append(flow)
    axes[0].plot(x, y, label='after')
    axes[0].set_xlabel('Time Axis')
    axes[0].set_ylabel('Flow')
    axes[0].legend()

    x, y = [], []
    for [t, flow, *_] in record['conflict']:
        x.append(t)
        y.append(flow)
    axes[1].plot(x, y, label='before')
    x, y = [], []
    for [t, flow, *_] in new_record['conflict']:
        x.append(t)
        y.append(flow)
    axes[1].plot(x, y, label='after')
    axes[1].set_xlabel('Time Axis')
    axes[1].set_ylabel('Conflict')
    axes[1].legend()

    x, y = [], []
    for key in sorted(candi.keys()):
        x.append(key)
        y.append(len(candi[key]))
    axes[2].plot(x, y, label='before')
    x, y = [], []
    for key in sorted(new_candi.keys()):
        x.append(key)
        y.append(len(new_candi[key]))
    axes[2].plot(x, y, label='after')
    axes[2].set_xlabel('Time Axis')
    axes[2].set_ylabel('Candi')
    axes[2].legend()

    plt.subplots_adjust(hspace=0.5)
    fig.savefig('stats_{}.pdf'.format(name))
    # plt.show()

def make_random_scenario(num_start=0, num_end=10):
    # np.random.seed(1234)
    inner_routes = search_routing_in_wuhan()
    print('>>> 一共找到{}条经过武汉扇区的Routing（Step 1和2）'.format(len(inner_routes)))

    for i in range(num_start, num_end):
        print('No.{}/{}'.format(i+1, num_end))
        print('>>> 随机加载航空器（Step 3)')
        fpl_list, candi = get_fpl_random(inner_routes[:], interval=10, number=3, max_time=1000)

        print('>>> 开始运行场景，并进行冲突探测（Step 4和5）')
        conflicts, record, shift_list = run_scenario(fpl_list, candi)

        print(len(shift_list))
        new_fpl_list, new_candi = [], {}
        for fpl in fpl_list:
            if fpl.id not in shift_list:
                new_fpl_list.append(fpl)
                start = fpl.startTime
                if start in new_candi.keys():
                    new_candi[start].append(fpl.id)
                else:
                    new_candi[start] = [fpl.id]

        print('>>> 重新运行场景，并进行冲突探测（Step 5和6）')
        new_conflicts, new_record, shift_list = run_scenario(new_fpl_list, new_candi)
        assert len(shift_list) == 0
        analysis(record, new_record, candi, new_candi, str(i + 1))

        print('>>> 记录冲突信息和飞行计划（Step 7）\n')
        write_in_db(i, new_conflicts, new_fpl_list)

# ------------------------------------------
# -- Generate Episodic Conflict Scenarios --
# ------------------------------------------
def run_scenario_1(fpl_list, candi):
    print('fpl_list:', len(fpl_list))
    agent_set = AircraftAgentSet(fpl_list=fpl_list, candi=candi)

    check_list, shift_list = [], []
    conflicts, structure = [], {}
    while not agent_set.is_done():
        agent_set.step(1)
        for c in agent_set.detect():
            c_id = c.id
            [a0_id, a1_id] = c_id.split('-')
            c_id_reverse = a1_id + '-' + a0_id
            if c_id in check_list or c_id_reverse in check_list:
                continue
            check_list.append(c_id)
            check_list.append(c_id_reverse)

            if a0_id in structure.keys():
                structure[a0_id].append((c.time, a1_id))
            else:
                structure[a0_id] = [(c.time, a1_id)]

            if a1_id in structure.keys():
                structure[a1_id].append((c.time, a0_id))
            else:
                structure[a1_id] = [(c.time, a0_id)]

            fpl0 = agent_set.agents[a0_id].fpl
            fpl1 = agent_set.agents[a1_id].fpl
            if c.time - fpl0.startTime < shift and a0_id not in shift_list:
                shift_list.append(a0_id)
            if c.time - fpl1.startTime < shift and a1_id not in shift_list:
                shift_list.append(a1_id)

            # c.printf()
            conflicts.append(c)

        now = agent_set.time
        if now % 1000 == 0:
            print('\t>>>', now, len(agent_set.agent_id_en))

    print(len(shift_list) > 0, shift_list)

    shift_list = []
    for key, value in structure.items():
        if len(value) <= 1:
            continue

        print(key, value)
        base = None
        for v in value:
            if base is None:
                base = v
                continue

            if abs(v[0] - base[0]) <= 360:
                shift_list.append(v[1])
                print('\t>>>', base, v, shift_list)
            else:
                base = v
    print(list(set(shift_list)))
    return shift_list, conflicts

def split_individual_conflict():
    train_set, _ = load_and_split_data(col='scenarios_meta', ratio=1.0)
    count, s_name = 1, 1
    for info in train_set[100:]:
        fpl_list = info['fpl_list']
        shift_list, _ = run_scenario_1(fpl_list=fpl_list, candi=info['candi'])

        new_fpl_list, new_candi = [], {}
        for fpl in fpl_list:
            fpl_id, start = fpl.id, fpl.startTime
            if fpl_id not in shift_list:
                new_fpl_list.append(fpl)
                if start in new_candi.keys():
                    new_candi[start].append(fpl_id)
                else:
                    new_candi[start] = [fpl_id]

        new_shift_list, new_conflicts = run_scenario_1(fpl_list=new_fpl_list, candi=new_candi)
        assert len(new_shift_list) <= 0

        for c in new_conflicts:
            c.printf()
            write_in_db('{}_{}'.format(count, s_name), [c, ], new_fpl_list, col='scenarios_dqn')
            s_name += 1

        count += 1

# ---------------------------------------
# -- Check Episodic Conflict Scenarios --
# ---------------------------------------

def check_conflict(col='scenarios_gail'):
    train_set, _ = load_and_split_data(col=col, ratio=1.0)

    count = 0
    for info in train_set:
        print(info['id'])
        conflict_ac, start = info['conflict_ac'], info['start']
        print('{}/{}'.format(count + 1, len(train_set)), info['id'], conflict_ac, start)
        print(info['conflict'])

        agent_set = AircraftAgentSet(fpl_list=info['fpl_list'], candi=info['candi'])
        agent_set.step(duration=start, basic=True)
        end_time = agent_set.time + 601

        check_list, conflicts = [], []
        while not agent_set.is_done():
            agent_set.step(1)

            for c in agent_set.detect(search=conflict_ac):
                c_id = c.id
                [a0_id, a1_id] = c_id.split('-')
                c_id_reverse = a1_id + '-' + a0_id
                if c_id in check_list or c_id_reverse in check_list:
                    continue
                check_list.append(c_id)
                check_list.append(c_id_reverse)
                conflicts.append(c)
                c.printf()

            if end_time is not None and agent_set.time > end_time:
                break

if __name__ == '__main__':
    # make_random_scenario(num_start=100, num_end=200)
    # split_individual_conflict()
    check_conflict()