#*----------------------------------------------------------------------------*
#* Copyright (C) 2021 Politecnico di Torino, Italy                            *
#* SPDX-License-Identifier: Apache-2.0                                        *
#*                                                                            *
#* Licensed under the Apache License, Version 2.0 (the "License");            *
#* you may not use this file except in compliance with the License.           *
#* You may obtain a copy of the License at                                    *
#*                                                                            *
#* http://www.apache.org/licenses/LICENSE-2.0                                 *
#*                                                                            *
#* Unless required by applicable law or agreed to in writing, software        *
#* distributed under the License is distributed on an "AS IS" BASIS,          *
#* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   *
#* See the License for the specific language governing permissions and        *
#* limitations under the License.                                             *
#*                                                                            *
#* Author:  Matteo Risso                                                      *
#*----------------------------------------------------------------------------*

import numpy as np
import tensorflow as tf
import argparse
import json
from config import Config
import sys
import pdb

import math

# aliases
val_mae = 'val_mean_absolute_error'
mae = 'mean_absolute_error'

from tensorflow.keras.optimizers import Adam, SGD
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint

from sklearn.model_selection import LeaveOneGroupOut, GroupKFold
from sklearn.utils import shuffle

from scipy.io import loadmat

from custom_callbacks import SaveGamma, export_structure

from preprocessing import preprocessing_Dalia as pp

from trainer import train_TEMPONet, train_ResTCN

from models import build_TEMPONet, build_ResTCN

import utils

from custom_losses import NLL, accuracy

# PARSER
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('dataset', help='PPG_Dalia | Nottingham | JSB_Chorales | SeqMNIST | PerMNIST')
parser.add_argument('strength', help='Regularization Strength')
parser.add_argument('warmup', help='Number of warmup epochs')
args = parser.parse_args()

# Setup config
cf = Config(args.dataset)
cf.reg_strength = float(args.strength)
try:
    cf.warmup = int(args.warmup)
except:
    cf.warmup = args.warmup

# Common callbacks
save_gamma = SaveGamma()
exp_str = export_structure(cf)

#############
# PPG_Dalia #
#############
if args.dataset == 'PPG_Dalia':
    early_stop = EarlyStopping(monitor=val_mae, min_delta=0.01, patience=35, mode='min', verbose=1)

    # Load data
    X, y, groups, activity = pp.preprocessing(cf.dataset, cf)

    # organize data
    group_kfold = GroupKFold(n_splits=4)
    group_kfold.get_n_splits(X, y, groups)

    # Learn dil fact
    model = build_TEMPONet.TEMPONet_d1(1, cf.input_shape, cf, ofmap=[])
    del model
    model = build_TEMPONet.TEMPONet_d1(1, cf.input_shape, cf, trainable=False,
            ofmap=[])

    # save model and weights
    checkpoint = ModelCheckpoint(cf.saving_path+'PPG_Dalia/trained_weights_warmup'+str(cf.warmup)+'.h5',
                                 monitor=val_mae, verbose=1,
                                 save_best_only=True, save_weights_only=True, mode='min', period=1)
    #configure  model
    adam = Adam(lr=cf.lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
    model.compile(loss='logcosh', optimizer=adam, metrics=[mae])

    X_sh, y_sh = shuffle(X, y)

    ##########
    # Warmup #
    ##########
    if cf.warmup != 0:
        print('Train model for {} epochs'.format(cf.warmup))
        strg = cf.reg_strength
        cf.reg_strength = 0

        if cf.warmup == 'max':
            epochs_num = cf.epochs
        else:
            epochs_num = cf.warmup

        train_TEMPONet.warmup(model, epochs_num, X_sh, y_sh, early_stop, checkpoint, cf)
        cf.reg_strength = strg

    del model
    model = build_TEMPONet.TEMPONet_d1(1, cf.input_shape, cf, trainable=True, ofmap=[])
    model.compile(loss='logcosh', optimizer=adam, metrics=[mae])

    if cf.warmup != 0:
        tmp_model = build_TEMPONet.TEMPONet_d1(1, cf.input_shape, cf, trainable=False, ofmap=[])
        # load weights in temp model
        tmp_model.load_weights(cf.saving_path+'PPG_Dalia/trained_weights_warmup'+str(cf.warmup)+'.h5')
        utils.copy_weights(model, tmp_model, cf)

    ################
    # Train gammas #
    ################
    print('Train on Gammas')
    print('Reg strength : {}'.format(cf.reg_strength))
    train_TEMPONet.train_gammas(model, X_sh, y_sh, early_stop, save_gamma, exp_str, cf)

    ##############################
    # Retrain and cross-validate #
    ##############################
    tr_model, MAE = train_TEMPONet.retrain(groups, X, y, activity, checkpoint, early_stop, cf, ofmap=[])
    print(MAE)
    # Evaluate average MAE
    avg = 0
    for _, val in MAE.items():
        avg += val
        print("Average MAE : %f", avg/len(MAE))

    #######################
    # Create summary file #
    #######################
    f=open("summary_PPG_Dalia_warmup{}.txt".format(cf.warmup), "a+")
    f.write("regularization strength : {reg_str} \t threshold : {th} \t MAE : {mae} \t Model size : {size} \n".format(
           reg_str = cf.reg_strength,
           th = cf.threshold,
           mae = avg/len(MAE),
           size = tr_model.count_params()))
    f.close()

###############################
# Nottingham and JSB_Chorales #
###############################
elif args.dataset == 'Nottingham' or args.dataset == 'JSB_Chorales':
    early_stop = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=35, mode='min', verbose=1)

    # Load data
    if args.dataset == 'Nottingham':
        data = loadmat(cf.path_Nottingham)
    elif args.dataset == 'JSB_Chorales':
        data = loadmat(cf.path_JSB_Chorales)
    X_train = data['traindata'][0]
    X_valid = data['validdata'][0]
    X_test = data['testdata'][0]

    # build model
    model = build_ResTCN.ResTCN_d1(cf.n_classes, cf.n_channels, cf, cf.k, cf.dp, variant=args.dataset)
    del model
    model = build_ResTCN.ResTCN_d1(cf.n_classes, cf.n_channels, cf, cf.k, cf.dp,
                                   trainable=False, variant=args.dataset)
    opt = Adam(lr=cf.lr, clipvalue=0.5)
    model.compile(
             loss=NLL,
             optimizer=opt)

    ##########
    # Warmup #
    ##########
    if cf.warmup != 0:
        print('Train model for {} epochs'.format(cf.warmup))
        strg = cf.reg_strength
        cf.reg_strength = 0

        if cf.warmup == 'max':
            epochs_num = cf.epochs
        else:
            epochs_num = cf.warmup

        train_ResTCN.warmup_Nottingham(model, epochs_num, cf, X_train, X_valid, X_test)
        cf.reg_strength = strg

    del model
    # build new model with trainable gamma
    model = build_ResTCN.ResTCN_d1(cf.n_classes, cf.n_channels, cf, cf.k, cf.dp,
                                   trainable=True, variant=args.dataset)
    model.compile(
           loss=NLL,
           optimizer=opt)

    if cf.warmup != 0:
        tmp_model = build_ResTCN.ResTCN_d1(cf.n_classes, cf.n_channels, cf, cf.k, cf.dp,
                                           trainable=False, variant=args.dataset)
        # load weights in temp model
        tmp_model.load_weights(cf.saving_path+cf.dataset+'/trained_weights_warmup'+str(cf.warmup)+'.h5')
        utils.copy_weights(model, tmp_model, cf)

    ###################
    # train on gammas #
    ###################
    print('Train on Gammas')
    print('Reg strength : {}'.format(cf.reg_strength))
    train_ResTCN.train_gammas_Nottingham(model, cf.epochs, cf, X_train, X_valid, X_test)

    ###########
    # retrain #
    ###########
    # obtain conv #output filters from learned json structure
    with open(cf.saving_path+cf.dataset+'/learned_dil_'+'{:.1e}'.format(cf.reg_strength)+'_'+'{:.1e}'.format(cf.threshold)+'_{}'.format(cf.warmup)+'.json', 'r') as f:
        dil_list = [val for _,val in json.loads(f.read()).items()]

    del model
    model = build_ResTCN.ResTCN_learned(cf.n_classes, cf.n_channels, cf.k,cf.dp,
                                        dil_list=dil_list, variant=args.dataset)

    model.compile(
           loss=NLL,
           optimizer=opt)
    best_t_l = train_ResTCN.retrain_Nottingham(model, cf.epochs, cf, X_train, X_valid, X_test)

    ################
    # summary file #
    ################
    f=open("summary_"+args.dataset+"_warmup{}.txt".format(cf.warmup), "a+")
    f.write("regularization strength : {reg_str} \t threshold : {th} \t Loss : {loss} \t Model size : {size} \n".format(
                       reg_str = cf.reg_strength,
                       th = cf.threshold,
                       loss = best_t_l,
                       size = model.count_params()))
    f.close()

#########################
# SeqMNIST and PerMNIST #
#########################
elif args.dataset == 'SeqMNIST' or args.dataset == 'PerMNIST':
    early_stop = EarlyStopping(monitor='val_accuracy', min_delta=0.01, patience=35, mode='max', verbose=1)

    # Load data
    (X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()

    # Normalize
    mean = 0.1307
    std = 0.3081
    X_train = (X_train - mean) / std
    X_test = (X_test - mean) / std

    # Serialize
    X_train = X_train.reshape((X_train.shape[0], 1, X_train.shape[1]*X_train.shape[2]))
    X_test = X_test.reshape((X_test.shape[0], 1, X_test.shape[1]*X_test.shape[2]))

    if args.dataset == 'PerMNIST':
        perm = np.random.permutation(X_train.shape[-1])
        X_train = X_train[:,:,perm]
        X_test = X_test[:,:,perm]

    X_train, y_train = shuffle(X_train, y_train)

    # save model and weights
    checkpoint = ModelCheckpoint(cf.saving_path+cf.dataset+'/trained_weights_warmup'+str(cf.warmup)+'.h5',
                                 monitor='val_accuracy', verbose=1,
                                 save_best_only=True, save_weights_only=True, mode='max', period=1)

    # build model
    model = build_ResTCN.ResTCN_d1(cf.n_classes, cf.n_channels, cf, cf.k, cf.dp, variant=args.dataset)
    del model
    model = build_ResTCN.ResTCN_d1(cf.n_classes, cf.n_channels, cf, cf.k, cf.dp,
                                   trainable=False, variant=args.dataset)

    opt = Adam(lr=cf.lr)
    model.compile(
                 loss='sparse_categorical_crossentropy',
                 optimizer=opt,
                 metrics=[accuracy])
    ##########
    # Warmup #
    ##########
    if cf.warmup != 0:
        print('Train model for {} epochs'.format(cf.warmup))
        strg = cf.reg_strength
        cf.reg_strength = 0

        if cf.warmup == 'max':
            epochs_num = cf.epochs
        else:
            epochs_num = cf.warmup

        train_ResTCN.warmup_SeqMNIST(model, epochs_num, cf, X_train, y_train, early_stop, checkpoint)
        cf.reg_strength = strg

    del model
    # build new model with trainable gamma
    model = build_ResTCN.ResTCN_d1(cf.n_classes, cf.n_channels, cf, cf.k, cf.dp,
                                   trainable=True, variant=args.dataset)
    model.compile(
            loss='sparse_categorical_crossentropy',
            optimizer=opt,
            metrics=[accuracy])

    if cf.warmup != 0:
        tmp_model = build_ResTCN.ResTCN_d1(cf.n_classes, cf.n_channels, cf, cf.k, cf.dp,
                                           trainable=False, variant=args.dataset)
        # load weights in temp model
        tmp_model.load_weights(cf.saving_path+cf.dataset+'/trained_weights_warmup'+str(cf.warmup)+'.h5')
        utils.copy_weights(model, tmp_model, cf)

    ###################
    # train on gammas #
    ###################
    print('Train on Gammas')
    print('Reg strength : {}'.format(cf.reg_strength))

    train_ResTCN.train_gammas_SeqMNIST(model, cf, X_train, y_train, early_stop, save_gamma, exp_str)

    ###########
    # retrain #
    ###########
    # obtain conv #output filters from learned json structure
    with open(cf.saving_path+cf.dataset+'/learned_dil_'+'{:.1e}'.format(cf.reg_strength)+'_'+'{:.1e}'.format(cf.threshold)+'_{}'.format(cf.warmup)+'.json', 'r') as f:
        dil_list = [val for _,val in json.loads(f.read()).items()]

    del model
    model = build_ResTCN.ResTCN_learned(cf.n_classes, cf.n_channels, cf.k,cf.dp,
                                        dil_list=dil_list, variant=args.dataset)

    model.compile(
            loss='sparse_categorical_crossentropy',
            optimizer=opt,
            metrics=[accuracy])
    train_ResTCN.retrain_SeqMNIST(model, cf.epochs, cf, X_train, y_train, X_test,
                                             y_test, early_stop, checkpoint)
    best_t_l = model.evaluate(
            X_test.reshape(-1, 1, X_test.shape[-1], 1),
            y_test.reshape(y_test.shape[0], 1))

    ################
    # summary file #
    ################
    f=open("summary_"+args.dataset+"_warmup{}.txt".format(cf.warmup), "a+")
    f.write("regularization strength : {reg_str} \t threshold : {th} \t Acc : {loss} \t Model size : {size} \n".format(
                       reg_str = cf.reg_strength,
                       th = cf.threshold,
                       loss = best_t_l,
                       size = model.count_params()))
    f.close()


import dash
import pandas as pd, numpy as np, seaborn as sns, matplotlib.pyplot as plt
from matplotlib.pyplot import suptitle
import plotly.express as px, plotly.io as pio, plotly.offline as pyo
import plotly.graph_objs as go, plotly.subplots as sp
from plotly.subplots import make_subplots
import dash, dash_bootstrap_components as dbc
from dash import dash_table
from dash import dcc, html
from dash import Dash
from dash.dependencies import Input, Output, State
import os
from tkinter import Button
from dash import Dash
from enum import unique
from matplotlib.pyplot import autoscale
from datetime import datetime
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
from pyclbr import Class
pd.options.display.float_format = '{:,.2f}'.format
pd.set_option("display.max_rows", None)
pd.set_option("display.max_columns", None)
pd.set_option("display.width", 1000)
pio.templates.default = "plotly_dark"

from Functions_Spotify import DataManipulation

df_artist = DataManipulation.get_df_artist()
df_tracks = DataManipulation.get_df_tracks()

#####################################################################################################################################################################################################################

indicator1 = go.Indicator(
    mode = "gauge+number",
    value = sum(df_artist["Followers"]),
    domain = {"x" : [0, 0.5], "y" : [0, 1]},
    title = {"text" : f"Total Followers Top 100 Artist", "font" : {"size" : 14}},
    gauge = {'bar' : {'color' : "white"}})

indicator2 = go.Indicator(
    mode = "gauge+number",
    value = (df_artist["Artist_Name"].nunique()),
    domain = {"x" : [0.5, 1], "y" : [0, 1]},
    title = {"text" : f"Count of Artist", "font" : {"size" : 14}},
    gauge = {'bar' : {'color' : "white"}})

indicator3 = go.Indicator(
    mode = "gauge+number",
    value = (df_tracks["Album_Name"].nunique()),
    domain = {"x" : [0.5, 1], "y" : [0, 1]},
    title = {"text" : "Count of Album", "font" : {"size" : 14}},
    gauge = {'bar' : {'color' : "white"}})

Fig = make_subplots(rows = 1, cols = 3, specs = [[{"type" : "indicator"}, {"type" : "indicator"}, {"type" : "indicator"}]], horizontal_spacing = 0.08)
Fig.add_trace(indicator1, row = 1, col = 3)
Fig.add_trace(indicator2, row = 1, col = 2)
Fig.add_trace(indicator3, row = 1, col = 1)

Fig.update_layout(height = 300,
    xaxis_title_font = {'size' : 18, 'color' : 'white'},
    yaxis_title_font = {'size' : 18, 'color' : 'white'},
    title = {'text' : "General Information", 'x': 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font': { 'size' : 18, 'color' : 'white'}},
    plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)")

#####################################################################################################################################################################################################################

df_top5 = df_tracks.groupby('Year').apply(lambda x : x.nlargest(5, 'Popularity')).reset_index(drop = True)
df_top5 = df_tracks.groupby('Year').apply(lambda x : x.nlargest(5, 'Popularity')).reset_index(drop = True)
df_top5 = df_top5.sort_values(by = ["Year", "Popularity"], ascending = [True, False])
df_top5['Rank'] = df_top5.groupby('Year').cumcount() + 1
df_top5 = df_top5[["Rank", "Song_Name", "Album_Name", "Query_Genre", "Popularity", "Release_Date", "Year", "Album_Link"]]

#####################################################################################################################################################################################################################

df_artist_grouped = df_tracks.groupby(["Artist_Name"]) \
.agg({"Year" : "count"}) \
.reset_index().sort_values(by = "Year", ascending = False).rename(columns = {"Year" : "Sum of Track", "Artist_Name" : "Artist Name"}).reset_index(drop = True)

df_artist_grouped = df_artist_grouped.head(20)
df_artist_grouped["Artist_Name_Short"] = df_artist_grouped["Artist Name"].apply(lambda x : x[:5] if isinstance(x, str) else x)

color = ["white"]
Fig_1 = px.bar(df_artist_grouped, x = "Artist_Name_Short", y = "Sum of Track", text = "Sum of Track", color_discrete_sequence = color,
              hover_data = {"Artist_Name_Short" : False, "Artist Name" : True})

Fig_1.update_layout(
    xaxis_title = "Artist Name",
    yaxis_title = "Sum of Track",
    xaxis_title_font = {'size': 18, 'color': 'white'},
    yaxis_title_font = {'size': 18, 'color': 'white'},
    yaxis_gridcolor = 'white',
    xaxis_gridcolor = 'white',
    title = {'text': "Top 20 Artists by Number of Hits in the Top 100", 'x': 0.5, 'xanchor': 'center', 'y': 0.95, 'yanchor': 'top', 'font': { 'size': 18, 'color': 'white'}},
    xaxis_tickangle = -45,
    hovermode = "x unified",
    hoverlabel = dict(bgcolor = "white",  font_size = 16, font_family = "Rockwell", font_color = "black"),
    plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)",
    height = 600)

#####################################################################################################################################################################################################################

df_artist_followers_grouped = df_artist.groupby(["Query_Genre", "Artist_Name"]).agg({"Followers" : "sum"}).reset_index()

Fig_2 = px.treemap(df_artist_followers_grouped, path = ["Query_Genre", "Artist_Name"],
                  values = "Followers", color = "Query_Genre", height = 600, color_continuous_scale = 'Blues')

Fig_2.update_layout (title = {'text' : "Analysis of Followers Based on Music Genres and Artist Name",
                                 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}},
                        plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)")

Fig_2.update_traces(hoverlabel = dict(bgcolor = "white", font = dict(family = "Arial", color = "black", size = 20)), marker = {'line' : {'color' : 'black', 'width': 2}},
                       hovertemplate = 'Query_Genre :  %{parent}
 \
                       Top_Track_Album :  %{label}
 \
                       Followers :  %{value:,}<extra></extra>')

#####################################################################################################################################################################################################################

df_tracks_year = df_tracks.groupby(["Album_Name", "Year"]).agg({"Album_Link" : "count"}).reset_index().sort_values(by = "Album_Link", ascending = False)
df_tracks_year = df_tracks_year.groupby(["Year"]).agg({"Album_Link" : "sum"}).reset_index().rename(columns = {"Album_Link" : "Count of Album"})

Fig_3 = px.area(df_tracks_year, x = "Year", y = "Count of Album", color_discrete_sequence = ['white'])
Fig_3.update_layout(plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)")
Fig_3.update_yaxes(range = [0, 1600], gridcolor = "white")
Fig_3.update_xaxes(gridcolor = "white")
scatter = go.Scatter(x = df_tracks_year["Year"], y = df_tracks_year["Count of Album"], mode = 'markers', marker_color = 'black')
Fig_3.add_trace(scatter)
Fig_3.update_layout(hovermode = "x unified",
                    hoverlabel = dict(bgcolor = "white",  font_size = 16, font_family = "Rockwell", font_color = "black"), height = 600,
                    title = {'text' : "Albums Count by Published Year",
                                 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}})

#####################################################################################################################################################################################################################

df_tracks_heat_map = df_tracks.groupby(["Months_Cat", "Year"]).agg({"Day" : "count"}).reset_index().sort_values(by = "Year").rename(columns = {"Months_Cat" : "Months", "Day" : "Tracks"})

Fig_4 = px.density_heatmap(df_tracks_heat_map, x = "Year", y = "Months", z = "Tracks", color_continuous_scale = "Blues", text_auto = True)
Fig_4.update_xaxes(tickmode = 'array', tickvals = list(range(min(df_tracks_heat_map["Year"]), max(df_tracks_heat_map["Year"]) + 1, 10)), gridcolor = 'LightGray', gridwidth = 1)
Fig_4.update_layout(plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)", height = 600,
                     title = {'text' : "Tracks Count by Year and Month According to Release Date",
                                 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}})

#####################################################################################################################################################################################################################

df_tracks_mean = df_tracks.groupby(["Year", "Season", "Query_Genre"]).agg({"Popularity" : "mean"}).reset_index().sort_values(by = ["Year", "Popularity"])
bins = [1900, 1990, 2000, 2010, 2023]
labels = ["1970 - 1990", "1990 - 2000", "2000 - 2010", "2010 - 2023"]
df_tracks_mean["Years_Cat"] = pd.cut(df_tracks_mean["Year"], labels = labels, bins = bins)
df_tracks_mean2 = df_tracks_mean.groupby(["Years_Cat", "Query_Genre"]).agg({"Popularity" : "mean"}).reset_index()
df_tracks_mean2["Popularity"].fillna(0, inplace = True)
df_tracks_mean2["Average_Popularity of Years Category"] = df_tracks_mean2["Popularity"]
df_tracks_mean2 = df_tracks_mean2[["Average_Popularity of Years Category", "Years_Cat", "Query_Genre"]]
df_tracks_mean = pd.merge(df_tracks_mean, df_tracks_mean2, how = "left", on = ["Years_Cat", "Query_Genre"])

Fig_5 = px.area(df_tracks_mean, x = "Year", y = "Popularity", color_discrete_sequence = ['white'])

scatter = go.Scatter(x = df_tracks_mean["Year"], y = df_tracks_mean["Popularity"], mode = 'markers', marker_color = 'black')
Fig_5.add_trace(scatter)

Fig_5.update_layout(plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)", height = 600, hovermode = "x unified",
    hoverlabel = dict(bgcolor = "white",  font_size = 16, font_family = "Rockwell", font_color = "black"),
    legend = dict(x = 1, y = 0.5),
    title = {"text" : "Average popularity of the top 100 songs according to their release dates", 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}})

Fig_5.update_yaxes(gridcolor = "white", autorange = False, range = [df_tracks_mean["Popularity"].min(), df_tracks_mean["Popularity"].max() + 10])
Fig_5.update_xaxes(gridcolor = "white", autorange = False, range = [df_tracks_mean["Year"].min(), df_tracks_mean["Year"].max()])

#####################################################################################################################################################################################################################

df_artist_top_10 = df_artist.groupby(["Artist_Name"]).agg({"Followers" : "sum"}).reset_index().sort_values(by = "Followers", ascending = False).rename(columns = {"Artist_Name" : "Artist Name"})
df_artist_top_10 = df_artist_top_10.head(10)
Fig_6 = px.funnel(df_artist_top_10, x = "Followers", y = "Artist Name", color_discrete_sequence = ['white'])
Fig_6.update_layout(plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)", height = 600, hovermode = "y unified",
    hoverlabel = dict(bgcolor = "white",  font_size = 16, font_family = "Rockwell", font_color = "black"),
    legend = dict(x = 1, y = 0.5),
    title = {"text" : "Elite Artists : The Top 10 Groups by Followers", 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}})
Fig_6.update_yaxes(gridcolor = "white")

#####################################################################################################################################################################################################################

df_artist_popularity_global_mean = df_tracks.groupby(["Query_Genre"]).agg({"Popularity" : "mean"}).reset_index().rename(columns = {"Popularity" : "Popularity Mean", "Query_Genre" : "Query Genre"})

colors = plt.cm.viridis(np.linspace(0, 1, len(df_artist_popularity_global_mean["Query Genre"])))
colors_hex = [mcolors.to_hex(c) for c in colors]

Fig_7 = go.Figure(data = [go.Pie(labels = df_artist_popularity_global_mean["Query Genre"],
                                 values = df_artist_popularity_global_mean["Popularity Mean"],
                                 hole = 0.3,
                                 hoverinfo = "none",
                                 hovertemplate = 'Query Genre : %{label}
Popularity Mean : %{value}',
                                 textinfo = "value",
                                 textfont = dict(size = 20),
                                 marker = dict(colors = colors_hex))])

Fig_7.update_layout(
    font = dict(color = 'white', size = 18),
    height = 600,
    title_text = "Average Popularity by Query Genre",
    title_x = 0.5,
    title_font = dict(size = 24, color = 'white'),
    legend = dict(
    yanchor = "top",
    y = 0.97,
    xanchor = "left",
    x = 0.95,
    font = dict(color = 'white', size = 10)),
    annotations = [dict(text = 'Query Genre', x = 0.50, y = 0.5, font_size = 16, showarrow = False)],
    plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)")

Fig_7.update_traces(marker = {'line': {'color': 'white', 'width': 2}})

#####################################################################################################################################################################################################################

df_season_artist = df_artist.groupby(["Season", "Artist_Name"]).agg({"Popularity" : "mean"}).reset_index()
df_season_artist = df_season_artist.groupby("Season").apply(lambda x: x.nlargest(5, 'Popularity')).reset_index(drop = True)
df_season_artist["Popularity"] = df_season_artist["Popularity"].astype("int64")
df_season_artist["Artist_Name_Short"] = df_season_artist["Artist_Name"].apply(lambda x : x[:7] if isinstance(x, str) else x)

df_season_tracks = df_tracks.groupby(["Season", "Album_Name"]).agg({"Popularity" : "mean"}).reset_index()
df_season_tracks = df_season_tracks.groupby("Season").apply(lambda x: x.nlargest(5, 'Popularity')).reset_index(drop = True)
new_values = [1, 2, 4, 5, 7]
df_season_tracks.loc[[10, 11, 12, 13, 14], 'Popularity'] = new_values
df_season_tracks["Popularity"] = df_season_tracks["Popularity"].astype("int64")
df_season_tracks["Album_Name_Short"] = df_season_tracks["Album_Name"].apply(lambda x: x[:7] if isinstance(x, str) else x)

unique_categories = ["Spring", "Summer", "Autumn", "Winter"]
category_orders = {"Season" : unique_categories}

fig_artist = px.bar(df_season_artist, y = 'Artist_Name_Short', x = 'Popularity', color = 'Season',
                    hover_data = {"Artist_Name_Short" : False, "Artist_Name" : True}, category_orders = {"Season" : unique_categories})

fig_track = px.bar(df_season_tracks, y = 'Album_Name_Short', x = 'Popularity', color = 'Season',
                   hover_data = {"Album_Name_Short" : False, "Album_Name" : True}, category_orders = {"Season" : unique_categories})

Fig_8 = make_subplots(rows = 1, cols = 2, subplot_titles = ("ARTİST", "TRACK"))

for trace in fig_artist.data:
    Fig_8.add_trace(trace, row = 1, col = 1)

for trace in fig_track.data:
    Fig_8.add_trace(trace, row = 1, col = 2)

Fig_8.update_layout(height = 600,
                  plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)",
                  hovermode = "y unified", hoverlabel = dict(bgcolor = "white",  font_size = 16, font_family = "Rockwell", font_color = "black"),
                  title = {"text" : "Top 5 in popularity for tracks and artists, according to seasons", 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}})

#####################################################################################################################################################################################################################

df_artist_guery_genre_top5 = df_artist.groupby(["Query_Genre", "Artist_Name"]).agg({"Followers" : "mean"}).reset_index()
df_artist_guery_genre_top5 = df_artist_guery_genre_top5.groupby("Query_Genre").apply(lambda x : x.nlargest(5, "Followers")).reset_index(drop=True)
Fig_9 = px.bar(df_artist_guery_genre_top5, x = "Artist_Name", y = "Followers", color = "Query_Genre")

Fig_9.update_layout(height = 600,
                  plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)",
                  hovermode = "x unified", hoverlabel = dict(bgcolor = "white",  font_size = 16, font_family = "Rockwell", font_color = "black"),
                  title = {"text" : "Top 5 artists with the most followers, categorized by query genre", 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}})

Fig_9_Copy = Fig_9

#####################################################################################################################################################################################################################

df_tracks_query_genre_top_5 = df_tracks.groupby(["Query_Genre", "Artist_Name"]).agg({"Popularity" : "mean"}).reset_index()
df_tracks_query_genre_top_5 = df_tracks_query_genre_top_5.groupby("Query_Genre").apply(lambda x : x.nlargest(5, "Popularity")).reset_index(drop = True)

Fig_10 = px.bar(df_tracks_query_genre_top_5, x = "Artist_Name", y = "Popularity", color = "Query_Genre")
Fig_10.update_layout(height = 600,
                  plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)",
                  hovermode = "x unified", hoverlabel = dict(bgcolor = "white",  font_size = 16, font_family = "Rockwell", font_color = "black"),
                  title = {"text" : "Top 5 artists in popularity, categorized by query genre", 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}})
Fig_10_Copy = Fig_10

#####################################################################################################################################################################################################################

df_tracks_duration_count = df_tracks.groupby(["Query_Genre", "Duration_Cat"]).agg({"Song_Link" : "count"}).reset_index().rename(columns = {"Song_Link" : "Count_of_Song"})
df_tracks_duration_count["Count_of_Song"].fillna(0, inplace = True)
category_orders = {"Duration_Cat" : ["3 - 4", "2 - 3", "1 - 2", "0 - 1"]}

unique_duration_cats = df_tracks_duration_count['Duration_Cat'].unique()
buttons = [dict(label = 'All', method = 'update', args = [{'visible' : df_tracks_duration_count['Duration_Cat'].isin(unique_duration_cats)}, {'title' : 'Total Number of Hit Songs by Query Genre and Song Duration'}])]

for cat in unique_duration_cats:
    buttons.append(dict(label = cat, method = 'update', args = [{'visible' : df_tracks_duration_count['Duration_Cat'] == cat}, {'title' : f" {cat} Total Number of Hit Songs by Query Genre and Song Duration"}]))

Fig_11 = go.Figure()

for cat in unique_duration_cats:
    Fig_11.add_trace(go.Bar(
        x = df_tracks_duration_count[df_tracks_duration_count['Duration_Cat'] == cat]["Query_Genre"],
        y = df_tracks_duration_count[df_tracks_duration_count['Duration_Cat'] == cat]["Count_of_Song"],
        text = df_tracks_duration_count[df_tracks_duration_count['Duration_Cat'] == cat]["Count_of_Song"],
        textposition = 'outside',
        hovertemplate = 'Query Genre: %{x}'+
                        '
Duration Category: '+cat+
                        '
Count of Song: %{y}
',
        name = cat))

Fig_11.update_layout(updatemenus = [go.layout.Updatemenu(active = 0, buttons = buttons)])

Fig_11.update_layout(height = 600,
                  plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)",
                  hovermode = "x unified", hoverlabel = dict(bgcolor = "white",  font_size = 16, font_family = "Rockwell", font_color = "black"),
                  title = {"text" : "Total Number of Hit Songs by Query Genre and Song Duration", 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}})

#####################################################################################################################################################################################################################

df_tracks_duration_genre = df_tracks.groupby(["Query_Genre", "Duration_Cat"]).agg({"Popularity" : "mean"}).reset_index()
df_tracks_duration_genre["Popularity"].fillna(0, inplace = True)
category_orders = {"Duration_Cat" : ["3 - 4", "2 - 3", "1 - 2", "0 - 1"]}

unique_duration_cats = df_tracks_duration_genre['Duration_Cat'].unique()

buttons = [dict(label = 'All', method = 'update', args = [{'visible' : df_tracks_duration_genre['Duration_Cat'].isin(unique_duration_cats)}, {'title' : 'Average Popularity by Query Genre and Song Duration'}])]

for cat in unique_duration_cats:
    buttons.append(dict(label = cat, method = 'update', args = [{'visible' : df_tracks_duration_genre['Duration_Cat'] == cat}, {'title' : f"{cat} Average Popularity by Query Genre and Song Duration"}]))

Fig_12 = go.Figure()

for cat in unique_duration_cats:
    Fig_12.add_trace(go.Bar(
        x = df_tracks_duration_genre[df_tracks_duration_genre['Duration_Cat'] == cat]["Query_Genre"],
        y = df_tracks_duration_genre[df_tracks_duration_genre['Duration_Cat'] == cat]["Popularity"],
        text = round(df_tracks_duration_genre[df_tracks_duration_genre['Duration_Cat'] == cat]["Popularity"], 2),
        textposition = 'outside',
        hovertemplate = 'Query Genre: %{x}'+
                        '
Duration Category: '+cat+
                        '
Avg Popularity: %{y}
',
        name = cat))

Fig_12.update_layout(updatemenus = [go.layout.Updatemenu(active = 0, buttons = buttons)])

Fig_12.update_layout(height = 600,
                  plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)",
                  hovermode = "x unified", hoverlabel = dict(bgcolor = "white",  font_size = 16, font_family = "Rockwell", font_color = "black"),
                  title = {"text" : "Average Popularity by Query Genre and Song Duration", 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}})

#####################################################################################################################################################################################################################

df_tracks_duration = df_tracks.groupby(["Duration_Cat"]).agg({"Popularity" : "mean"}).reset_index().rename(columns = {"Popularity" : "Popularity Mean"})
Fig_13 = go.Figure(data = [go.Pie(labels = df_tracks_duration['Duration_Cat'],
                            values = df_tracks_duration['Popularity Mean'],
                            hovertemplate = 'Duration Category: %{label}'+
                                             '
Average Popularity: %{value}
',
                            textinfo = 'value')])

Fig_13.update_layout(height = 600,
                  plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)",
                  hovermode = "x unified", hoverlabel = dict(bgcolor = "white",  font_size = 16, font_family = "Rockwell", font_color = "black"),
                  title = {"text" : "Average Popularity by Duration Category", 'x' : 0.5, 'xanchor' : 'center', 'y' : 0.95, 'yanchor' : 'top', 'font' : { 'size' : 18, 'color' : 'white'}})

#####################################################################################################################################################################################################################

query_genre = {"k-pop": 1000, "j-pop": 1000, "japanese": 1000, "korean": 1000, "j-dance": 1000, "chinese": 1000, "j-idol": 999}

data = {
    "country": ["China", "Japan", "South Korea", "North Korea", "Mongolia", "Taiwan"],
    "value": [query_genre.get("chinese", 0),
              query_genre.get("japanese", 0) + query_genre.get("j-pop", 0),
              query_genre.get("korean", 0) + query_genre.get("k-pop", 0),
              0, 0, 0]}

colorscale = "RdYlBu"
df = pd.DataFrame(data)

background_world = go.Scattergeo(
    lon = [180, -180], lat = [90, -90], mode = "text", text = [""],
    showlegend = False, hoverinfo = "none",
    marker = dict(size = 0, cmin = 0, cmax = 0,
    colorscale = [[0, "rgba(28, 107, 160, 0.6)"], [1, "rgba(28, 107, 160, 0.6)"]], showscale = False,
    colorbar = dict(lenmode = "fraction", len = 0.1, yanchor = "top", y = 1, xanchor = "left", x = 0.01, bgcolor = None),))

main_map = go.Choropleth(
    locations = df["country"], z = df["value"], locationmode = "country names",
    colorscale = colorscale, colorbar_title = "Total Song Count", hoverinfo = "location+z",
    marker_line_color = "darkgray", marker_line_width = 1)

Fig_14 = go.Figure(data=[background_world, main_map])

Fig_14.update_layout(title = {"text" : "Highlighted Countries of Interest", "y" : 0.92, "x" : 0.5, "xanchor" : "center", "yanchor" : "top"},
    geo = dict(scope = "world",  showland = True, landcolor = "lightgray", showocean = True, oceancolor = "rgba(28, 107, 160, 0.6)"),
    autosize = False, height = 600, plot_bgcolor = "rgb(84,52,107)", paper_bgcolor = "rgb(84,52,107)")

#####################################################################################################################################################################################################################


import torch
import torch.nn as nn
import torch.nn.functional as F
from factory.embed_factory import NodeEmbedFactory
from pymodels.submodels.gnn_layers.gat_layer import GATLayer
from utils.pymodels_util import to_cuda, graph_readout

class PairwiseGATModel(nn.Module):
    def __init__(self, config):
        super(PairwiseGATModel, self).__init__()
        # Initialization of Attributes
        self.config = config
        self.forcing_ratio = 0.75
        self.use_cuda = self.config.use_cuda
        self.class_num = self.config.class_num
        self.use_nfeat = self.config.node_emb_layer['use_nfeature']
        self.graph_config = getattr(self.config, 'gat')
        self.in_dim = self.graph_config['in_dim']
        self.out_dim = self.graph_config['out_dim']
        self.vocab_len = self.config.token_vocab_dict.vocab_size()
        self.word_emb_dim = self.config.word_emb_dims

        # Embedding Configurations
        self.node_emb_layer = NodeEmbedFactory().get_node_embed_technique(self.config)(self.config)

        # GCN Layers
        self.gat_layers_one = nn.ModuleList([GATLayer(config) for _ in range(self.graph_config['layers'])])
        self.gat_layers_two = nn.ModuleList([GATLayer(config) for _ in range(self.graph_config['layers'])])
        self.fforward = nn.Linear(self.graph_config['out_dim'], self.config.class_num)

    def forward(self, batch_dict, running_mode, loss_fn):
        class_target = batch_dict['tgt']
        fnone_mean_feats = self.gcn_prop(batch_dict['graphs_one'], self.gat_layers_one)
        fntwo_mean_feats = self.gcn_prop(batch_dict['graphs_two'], self.gat_layers_two)

        euc_dist = (fnone_mean_feats - fntwo_mean_feats) ** 2
        dense_output = F.leaky_relu(self.fforward(euc_dist))

        loss = 0
        if running_mode in ['train', 'val']:
            tgt = to_cuda(torch.tensor(class_target, dtype=torch.long),
                          use_cuda=self.use_cuda)
            loss = loss_fn(dense_output, tgt)
        sm_mask_output = F.softmax(dense_output, dim=-1)
        return sm_mask_output, class_target, loss

    def gcn_prop(self, g, gcn_fn):
        h = to_cuda(g.ndata['node_feat'], self.use_cuda)
        node_len = g.ndata['node_len'].cpu().tolist()
        h = self.node_emb_layer(h, node_len)

        for gcn in gcn_fn:
            h = gcn(g, h)
            h = torch.mean(h, dim=1)

        g.ndata['h'] = h
        mean_feats = graph_readout(g, self.graph_config['graph_agg'])
        return mean_feats


# -*- coding: utf-8 -*-
"""
Created on Mon Apr 15 15:23:56 2019

@author: xulingjie

梯度下降法求线性回归（采用均方误差）
"""

import numpy as np

# 线性回归方程： y = b0 +b1*x1+b2*x2+...+bn*xn

# 梯度下降
def gradient_descent(p, m, sample, y, a):
    """
    使用梯度下降法求均方误差最小值，拟合线性回归函数

    参数：
    p--参数矩阵n*1(n-1元);
    m--样本数量;
    sample--样本矩阵m*(n-1);
    y--目标矩阵m*1;
    a--步长/学习率
    ---------------------------
    example:
    >>> gradient_descent(p, m, sample, y, a)
    """

    b = np.ones((m, 1)) # 设置值全为1的矩阵
    x = np.hstack((b, sample)) #矩阵拼接：行拼接vstack, 列拼接hstack
    h = (np.dot(x, p)-y)
    # 求梯度
    gradient = np.dot(x.T, h)
    # 求均方误差
    mse = 1/(2*m)*np.dot(h.T, h)
    n = 0
    #未知MSE最小值可以达到多少，因此因限制斜率接近于平稳
    while np.all(np.absolute(gradient) >= 1e-7):
        p = p - a*gradient
        h = (np.dot(x, p)-y)
        gradient = (1/m)*np.dot(x.T, h)
        mse = 1/(2*m)*np.dot(h.T, h)
        n = n + 1
    else:
            print("参数：" + str(p))
            print("梯度：" + str(gradient))
            print("均方误差：" + str(mse))
            print("迭代次数：" + str(n))
    return

########################### 一元散点线性回归
# 输入未知数的初始值
p = np.array([[1, 1]]).T
# 输入样本数
m = 11
# 输入样本x
sample = np.array([[0.3, -0.6, -0.1, 0.1, 0.2, 0.8, -0.3, -0.7, -0.4, 0.4, 0.6]]).T
# 输入y
y = np.array([[0.46, -0.9, -0.21, -0.2, 0.1, 0.6, -0.2, -0.82, -0.34, 0.2, 0.9]]).T
# 步长
a = 0.5

gradient_descent(p, m, sample, y, a)

########################### 波士顿房价
import pandas as pd
from sklearn.datasets import load_boston

# 这里得到的是 scikit-learn 中的 Bunch 对象
boston = load_boston()

# 特征集与目标变量
X = pd.DataFrame(data=boston.data, columns=boston.feature_names)
Y = boston.target

# 数据标准化
def transf_standard(dataframe):
    """
    对数据框格式的数据进行标准化

    参数：
    dataframe 需标准化的数据框
    ---------------------------
    example:
    >>> X_standard = transf_standard(X)
    """

    df = dataframe.copy()
    for ColName in df.columns:
        MinValue = min(df[ColName])
        MaxValue = max(df[ColName])
        df[ColName] = df[ColName].apply(lambda x: (x - MinValue)/(MaxValue-MinValue))
    return df

X_standard = transf_standard(X)

# 对这个数据集的文字描述
#print(boston.DESCR)

# 设置参数
# 初始值 几元+截距项
p = np.ones((len(X_standard.columns)+1, 1))
m = len(X_standard.index)
sample = np.array(X_standard)
# sample.shape
y = np.array(Y).reshape(m, 1)
a = 0.01

gradient_descent(p, m, sample, y, a)


from django.urls import path

from course import views

urlpatterns = [
    path("category/", views.CourseCategoryListAPIView.as_view()),
    path("list/", views.CourseListAPIView.as_view()),

    path("list_filter/", views.CourseFilterListAPIView.as_view()),
    path("retrieve/<str:id>/", views.CourseOneRetrieveAPIView.as_view()),
    path('chapter/', views.CourseChapterAPIvie.as_view()),

]


import uvicorn
from fastapi import FastAPI
from typing import List
from fastapi.middleware.cors import CORSMiddleware
from UserManager import UserDataHandler,User
app = FastAPI()

origins = ["*"]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

@app.post("/api/users")
def create_user(user: User):
    user_handler =UserDataHandler()
    user_handler.add_user(user)
    return f"Successfully created user {user.name}"

if __name__ == "__main__":
    uvicorn.run("main:app", port=8000, reload=True)


#!/usr/bin/python3
"""
Contains the TestUserDocs classes
"""

from datetime import datetime
import inspect
import models
from models import user
from models.base_model import BaseModel
import pep8
import unittest
User = user.User

class TestUserDocs(unittest.TestCase):
    """Tests to check the documentation and style of User class"""
    @classmethod
    def setUpClass(cls):
        """Set up for the doc tests"""
        cls.user_f = inspect.getmembers(User, inspect.isfunction)

    def test_pep8_conformance_user(self):
        """Test that models/user.py conforms to PEP8."""
        pep8s = pep8.StyleGuide(quiet=True)
        result = pep8s.check_files(['models/user.py'])
        self.assertEqual(result.total_errors, 0,
                         "Found code style errors (and warnings).")

    def test_pep8_conformance_test_user(self):
        """Test that tests/test_models/test_user.py conforms to PEP8."""
        pep8s = pep8.StyleGuide(quiet=True)
        result = pep8s.check_files(['tests/test_models/test_user.py'])
        self.assertEqual(result.total_errors, 0,
                         "Found code style errors (and warnings).")

    def test_user_module_docstring(self):
        """Test for the user.py module docstring"""
        self.assertIsNot(user.__doc__, None,
                         "user.py needs a docstring")
        self.assertTrue(len(user.__doc__) >= 1,
                        "user.py needs a docstring")

    def test_user_class_docstring(self):
        """Test for the City class docstring"""
        self.assertIsNot(User.__doc__, None,
                         "User class needs a docstring")
        self.assertTrue(len(User.__doc__) >= 1,
                        "User class needs a docstring")

    def test_user_func_docstrings(self):
        """Test for the presence of docstrings in User methods"""
        for func in self.user_f:
            self.assertIsNot(func[1].__doc__, None,
                             "{:s} method needs a docstring".format(func[0]))
            self.assertTrue(len(func[1].__doc__) >= 1,
                            "{:s} method needs a docstring".format(func[0]))

class TestUser(unittest.TestCase):
    """Test the User class"""
    def test_is_subclass(self):
        """Test that User is a subclass of BaseModel"""
        user = User()


import librosa
path='data/test/' #default
# path='data/people/'
def formataudio(path2):
    """Formats wav file into 16 different speeds and writes them to path2 directory

        Parameters
        ----------
        path2 : file name, wav format"""

    y, sr = librosa.load(path+path2+'.wav.ig')
    for i in range(16):
        if i == 4:
            librosa.output.write_wav('{}_{}_{}.wav'.format(path,path2,i+1), y, sr)
        else:
            y_stretch = librosa.effects.time_stretch(y, (i*2/16)+.5)  # .5 to 2.5 factor
            librosa.output.write_wav('{}_{}_{}.wav'.format(path,path2,i+1), y_stretch, sr)
if __name__ == '__main__':
    path2 = input("Please enter the file to format\n")
    formataudio(path2)


# -*- coding: utf-8 -*-

import itertools

N = int(input())
name_list = []
for i in range(N):
    name_str = input()
    # 1文字目がMARCHの名前だけリストに詰める
    if name_str[0] == "M" or name_str[0] == "A" or name_str[0] == "R" or name_str[0] == "C" or name_str[0] == "H":
        name_list.append(name_str)
# 3人を選ぶ組み合わせ
cmb_list = list(itertools.combinations(name_list, 3))

ans = 0
for names in cmb_list:
    # 各人の名前のイニシャルを取得
    init1 = names[0][0]
    init2 = names[1][0]
    init3 = names[2][0]
    # 重複がなければ条件を満たす
    if init1 != init2 and init1 != init3 and init2 != init3:
        ans += 1

print(ans)


#!/usr/bin/env python
from __future__ import print_function, unicode_literals
from netmiko import ConnectHandler
import threading
from my_devices import device_list as devices

def remote_cmd(a_device, cmd=""):
    '''
    Execute show version command using Netmiko
    '''
    remote_conn = ConnectHandler(**a_device)
    print()
    print('#' * 80)
    print(remote_conn.send_command(cmd))
    print('#' * 80)
    print()

def main():
    for a_device in devices:
        my_thread = threading.Thread(target=remote_cmd, args=(a_device, "show arp"))
        my_thread.start()

    main_thread = threading.currentThread()
    for some_thread in threading.enumerate():
        if some_thread != main_thread:
            some_thread.join()

if __name__ == "__main__":
    main()


import numpy as np
from python_speech_features import mfcc

def pad(sequence, max_len):
    if len(sequence) < max_len:
        sequence = np.append(sequence, [0] * (max_len - len(sequence)))
    else:
        sequence = sequence[:max_len]
    return sequence

def partition_sequence(sequence, rate, window_size):
    sequence = pad(sequence, 16000)
    time_len = len(sequence) / rate
    partition_len, partitions = int(rate * window_size), []
    for i in range(0, len(sequence), partition_len):
        start, end = i, min(i + partition_len, len(sequence))
        partitions.append(sequence[start:end])
    return np.vstack(partitions)

def mfcc_sequence(sequence, rate, window_size):
    sequence = pad(sequence, 16000)
    return mfcc(sequence, samplerate=rate, winlen=window_size)


#! /bin/python3
import _curses
import argparse
import curses
import json
import math
import time
import os, sys

vid = None
colored = None

###### VIDEO PARSING ######

def ParseVideo(file):
    with open(file, "r") as f:
        mov = json.loads(f.read())

    global colored
    colored = mov['isColored']
    return mov['name'], mov['FPS'], mov['frames']

###### PLAY VIDEO ######

def PlayVideo(fps, frames):
    ftime = CalcFrameTime(fps)
    for i in frames:
        os.system('clear')
        sys.stdout.write(i)
        time.sleep(ftime)

    """TODO: IMPLEMENT"""

###### OTHER ######

def CalcFrameTime(fps):
    frame_time = 1/fps
    return round_down(frame_time, 3)

def round_down(n, decimals=0):
    multiplier = 10 ** decimals
    return math.floor(n * multiplier) / multiplier

##### MAIN ######

def main():
    global vid, colored
    parser = argparse.ArgumentParser(description="ASCII movie player")
    parser.add_argument('--file', '-f', dest="file", required=True, help="File to play")

    args = parser.parse_args()

    file = args.file

    vid = ParseVideo(file)
    if colored:
        print(f"""
Name: {vid[0]}
FPS: {vid[1]}
Frame amount: {len(vid[2])}
""")
        input("Press enter to start playing.")
        PlayVideo(vid[1], vid[2])
        input("Finished, press enter to exit")
        os._exit(0)
    else:
        curses.wrapper(curses_main)

def curses_main(stdscr):
    global vid
    curses.curs_set(0)
    curses.init_pair(1, curses.COLOR_BLACK, curses.COLOR_WHITE)
    curses.noecho()
    curses.cbreak()
    h, w = stdscr.getmaxyx()
    name, fps, frames = vid
    stdscr.addstr(2, 2, f"Name: {name}")
    stdscr.addstr(4, 2, f"FPS: {fps}")
    stdscr.addstr(6, 2 , f"Frame amount: {len(frames)}")
    x = w//2 - len("Press anything to start playing")//2
    y = h//2
    stdscr.addstr(y, x, "Press anything to start playing")
    stdscr.refresh()
    ftime = CalcFrameTime(fps)
    if stdscr.getch():
        stdscr.clear()
        for i in frames:
            try:
                #if stdscr.getch() == curses.KEY_BACKSPACE:
                    #stdscr.clear()
                    #stdscr.addstr(h//2, w//2 - len("PAUSED"), "PAUSED")
                    #stdscr.refresh()
                    #stdscr.getch()
                stdscr.addstr(0, 0, i)
            except _curses.error:
                stdscr.clear()
                stdscr.addstr(0,0,"Term window is too small.")
                stdscr.getch()
                stdscr.clear()
                os._exit(0)

            time.sleep(ftime)
            stdscr.refresh()
    stdscr.clear()
    stdscr.addstr(h//2, w//2 - len("Finished. Press enter to exit"), "Finished. Press enter to exit")
    if stdscr.getch():
        stdscr.clear()
        os._exit(0)

if __name__ == '__main__':
    main()


import time
from datetime import datetime

from peewee import *
from playhouse.pool import PooledMySQLDatabase
from playhouse.shortcuts import ReconnectMixin

from inventory_srv.settings import settings

class ReconnectMysqlDatabase(ReconnectMixin, PooledMySQLDatabase):
    pass

class BaseModel(Model):
    add_time = DateTimeField(default=datetime.now, verbose_name="添加时间")
    is_deleted = BooleanField(default=False, verbose_name="是否删除")
    update_time = DateTimeField(default=datetime.now, verbose_name="更新时间")

    @classmethod
    def delete(cls, permanently=False):  # permanently表示是否永久删除
        if permanently:
            return super().delete()
        else:
            return super().update(is_deleted=True)

    def delete_instance(
        self, permanently=False, recursive=False, delete_nullable=False
    ):
        if permanently:
            return self.delete(permanently).where(self._pk_expr()).execute()
        else:
            self.is_deleted = True
            self.save()

    @classmethod
    def select(cls, *fields):
        return super().select(*fields).where(cls.is_deleted == False)

    def save(self, *args, **kwargs):
        # 判断这是一个新添加的数据还是更新的数据
        if self._pk is not None:
            # 这是一个新数据
            self.update_time = datetime.now()
        return super().save(*args, **kwargs)

    class Meta:
        database = settings.DB

class Inventory(BaseModel):
    # 商品的库存表
    goods = IntegerField(verbose_name="商品id", unique=True)
    stocks = IntegerField(verbose_name="库存数量", default=0)
    version = IntegerField(verbose_name="版本号", default=0)  # 分布式锁的乐观锁

import threading

R = threading.Lock()

def sell():
    goods_list = [(421, 99), (422, 20), (423, 30)]
    with settings.DB.atomic() as txn:
        for goods_id, num in goods_list:
            # 查询库存
            R.acquire()
            goods_inv = Inventory.get(Inventory.goods == goods_id)
            from random import randint

            time.sleep(randint(1, 3))
            if goods_inv.stocks < num:
                print(f"商品：{goods_id}库存不足")
                txn.rollback()
                break
            else:
                # 让数据库根据自己当前的值更新数据，这个语句不能处理并发的问题
                query = Inventory.update(stocks=Inventory.stocks - num).where(
                    Inventory.goods == goods_id
                )
                ok = query.execute()
                if ok:
                    print("更新成功")
                else:
                    print("更新失败")
                # goods_inv.stocks -= num
                # goods_inv.save()
            R.release()

def sell2():
    # 演示mysql分布式锁
    goods_list = [(421, 10), (422, 20), (423, 30)]
    with settings.DB.atomic() as txn:
        for goods_id, num in goods_list:
            # 查询库存
            while True:
                goods_inv = Inventory.get(Inventory.goods == goods_id)
                from random import randint

                time.sleep(randint(1, 3))
                if goods_inv.stocks < num:
                    print(f"商品：{goods_id}库存不足")
                    txn.rollback()
                    break
                else:
                    # 让数据库根据自己当前的值更新数据，这个语句不能处理并发的问题
                    # 我当时查询数据的时候版本号是goods_inv.version 乐观锁
                    query = Inventory.update(
                        stocks=Inventory.stocks - num, version=Inventory.version + 1
                    ).where(
                        Inventory.goods == goods_id,
                        Inventory.version == goods_inv.version,
                    )
                    ok = query.execute()
                    if ok:
                        print("更新成功")
                        break
                    else:
                        print("更新失败")
                    # goods_inv.stocks -= num
                    # goods_inv.save()

if __name__ == "__main__":
    import threading

    t1 = threading.Thread(target=sell2)
    t2 = threading.Thread(target=sell2)
    t1.start()
    t2.start()

    t1.join()
    t2.join()


import pycuda.driver as cuda
import pycuda.autoinit
from pycuda.compiler import SourceModule
import numpy as np

###  Need to add the following lines to environment
###  This will vary on different computers, particularly,
###      the 14.16.27023 folder name
###  These lines are not necessary on Linux, or on the Prince cluster
import os
if (os.system("cl.exe")):
    os.environ['PATH'] += ';'+r"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\bin\HostX64\x64"
if (os.system("cl.exe")):
    raise RuntimeError("cl.exe still not found, path probably incorrect")

#a = np.random.randn(4,4)
#a = a.astype(np.float32)
#a_gpu = cuda.mem_alloc(a.nbytes)
#cuda.memcpy_htod(a_gpu, a)

mod = SourceModule("""
 __device__ inline float sninvdev(double dseed[])
 {
	const double norm = 2.328306549295728e-10;
	const double m1 = 4294967087.0;
	const double m2 = 4294944443.0;
	const double a12 = 1403580.0;
	const double a13n = 810728.0;
	const double a21 = 527612.0;
	const double a23n = 1370589.0;
	int k;
	double p1, p2;
	float ans, p;
	p1 = a12*dseed[1] - a13n*dseed[2];
	k = p1 / m1;
	p1 -= k*m1;
	if (p1 < 0.0) p1 += m1;
	dseed[2] = dseed[1]; dseed[1] = dseed[0]; dseed[0] = p1;
	p2 = a21*dseed[3] - a23n*dseed[5];
	k = p2 / m2;
	p2 -= k*m2;
	if (p2 < 0.0) p2 += m2;
	dseed[5] = dseed[4]; dseed[4] = dseed[3]; dseed[3] = p2;
	if (p1 <= p2) p = ((p1 - p2 + m1)*norm);
	else p = ((p1 - p2)*norm);
	if (p <= 0.0) {
		ans = -100.0f;
	}
	else {
		if (p >= 1.0) ans = 100.0f;
		else ans = normcdfinvf(p);
	}
	return ans;
 }

 __global__ void SimulatePathGPU(int nSim, int nTimePeriods, float dt, float sigma,
        unsigned int *seeds, float *result)
 {
	int iSim = threadIdx.x + threadIdx.y;

	if (iSim < nSim) {
		int i;
		float sqrt_dt, z;
		double dseed[6];
		//	extract seeds for this path from seeds
		for (i = 0; i < 6; i++) dseed[i] = seeds[i + iSim * 6];

		sqrt_dt = sqrtf(dt);
		z = 0.0;
		for (i=0;i<nTimePeriods;i++) z += sigma * sqrt_dt * sninvdev(dseed);

		result[iSim] = z;
	}
 }
""")

nSim = 1028
nTimePeriods = 1000
dt = 1.0 / nTimePeriods
sigma = 0.20
nBlock = (1 + nSim / 128)
#dt = dt.astype(np.float32)
#sigma = sigma.astype(np.float32)
result = np.zeros(nSim).astype(np.float32)
seeds = np.zeros(nSim*6).astype(np.uint32)
seeds[0] = 39499
seeds[1] = 828388
seeds[2] = 129399
seeds[3] = 39499
seeds[4] = 828388
seeds[5] = 1295959
##  Need to apply skip ahead for seeds
for i in range(0,nSim-1):
    for j in range(0,6): seeds[(i+1)*6+j] = seeds[i*6+j]+13

d_seeds = cuda.mem_alloc(seeds.nbytes)
d_result  = cuda.mem_alloc(result.nbytes)
cuda.memcpy_htod(d_seeds, seeds)

func = mod.get_function("SimulatePathGPU")
#func(a_gpu, block=(nBlock,128))
func(np.int32(nSim), np.int32(nTimePeriods), np.float32(dt), np.float32(sigma), d_seeds, d_result, block=(1024,1,1))

cuda.memcpy_dtoh(result, d_result)
#print(a_doubled)
#print(a)

print(' length ', len(result), ' mean ', np.mean(result), np.std(result), np.var(result))

Sample Code/Lecture 5 Code/Pycuda Examples/PyCuda_simulation.py


import unittest
from solution import *
from typing import FrozenSet
from time import time

class PowerSetTest(unittest.TestCase):
    def setUp(self) -> None:
        self.sol = Solution1()

    def test_1(self) -> None:
        nums = [1, 2, 3]
        expected = [[], [1], [2], [1, 2], [3], [1, 3], [2, 3], [1, 2, 3]]
        expected = self.convert_to_frozenset(expected)
        actual = self.convert_to_frozenset(self.sol.power_set(nums))
        self.assertEqual(expected, actual)

    def test_2(self) -> None:
        nums = [1, 2, 3, 4, 5]
        expected = [[], [1], [2], [1, 2], [3], [1, 3], [2, 3], [1, 2, 3], [4], [1, 4], [2, 4], [1, 2, 4], [3, 4], [1, 3, 4], [2, 3, 4], [1, 2, 3, 4], [5], [1, 5], [2, 5], [1, 2, 5], [3, 5], [1, 3, 5], [2, 3, 5], [1, 2, 3, 5], [4, 5], [1, 4, 5], [2, 4, 5], [1, 2, 4, 5], [3, 4, 5], [1, 3, 4, 5], [2, 3, 4, 5], [1, 2, 3, 4, 5]]
        expected = self.convert_to_frozenset(expected)
        actual = self.convert_to_frozenset(self.sol.power_set(nums))
        self.assertEqual(expected, actual)

    def test_3(self) -> None:
        nums = [1]
        self.assertEqual([[], [1]], self.sol.power_set(nums))

    def test_4(self) -> None:
        nums = []
        self.assertEqual([[]], self.sol.power_set(nums))

    def test_5(self) -> None:
        nums = [i for i in range(18)]
        start = time()
        self.sol.power_set(nums)
        print(f'{time() - start:.3f} seconds')

    def convert_to_frozenset(self, arr: List[List[int]]) -> FrozenSet[FrozenSet[int]]:
        temp = arr.copy()
        for i in range(len(temp)):
            temp[i] = frozenset(temp[i])
        return frozenset(temp)


# 데이터 수집을 위한 프로그램

import mediapipe as mp
import numpy as np
import cv2
# import tkinter as tk
# from pynput import keyboard
import time

max_num_hands = 2

# 수집할 제스처
gesture = {
    0:'open', 1:'close', 2:'index_finger', 3:'Bunch'
}

mp_hands = mp.solutions.hands
mp_drawing = mp.solutions.drawing_utils
hands = mp_hands.Hands(
    max_num_hands = max_num_hands,
    min_detection_confidence = 0.5,
    min_tracking_confidence = 0.5
)

# 수집한 데이터가 저장될 위치
f = open('./Training/test.txt', 'w')

# 테스트 학습
file = np.genfromtxt('./Training/DataSet.txt', delimiter=',')

angleFile = file[:,:-1]
labelFile = file[:,-1]

angle = angleFile.astype(np.float32)
label = labelFile.astype(np.float32)

knn = cv2.ml.KNearest_create()
knn.train(angle, cv2.ml.ROW_SAMPLE, label)

cap = cv2.VideoCapture(0)

startTime = time.time()
prev_index = 0
sentence = ''
recognizeDelay = 1

while True:
    ret, img = cap.read()
    if not ret:
        continue
    imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    result = hands.process(imgRGB)

    pressed_key = cv2.waitKey(1) # 키 입력

    if result.multi_hand_landmarks is not None:
        for res in result.multi_hand_landmarks:
            joint = np.zeros((21, 3))
            for j, lm in enumerate(res.landmark):
                joint[j] = [lm.x, lm.y, lm.z]

            v1 = joint[[0, 1, 2, 3, 0, 5, 6, 7, 0,  9, 10, 11,  0, 13, 14, 15,  0, 17, 18, 19], :]
            v2 = joint[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], :]

            v = v2 - v1

            v = v / np.linalg.norm(v, axis=1)[:, np.newaxis]

            compareV1 = v[[0, 1, 2, 4, 5, 6, 7,  8,  9, 10, 12, 13, 14, 16, 17], :]
            compareV2 = v[[1, 2, 3, 5, 6, 7, 9, 10, 11, 13, 14, 15, 17, 18, 19], :]

            angle = np.arccos(np.einsum('nt,nt->n', compareV1, compareV2))

            angle = np.degrees(angle)

            # opencv창 활성화 상태에서 a키 입력시 test.txt에 저장
            if pressed_key == 97:
                for num in angle:
                    num = round(num, 6)
                    f.write(str(num))
                    f.write(',')
                f.write("3.000000") # change Intager part to dict_key which you want to get data
                f.write('\n')
                print("next")

            data = np.array([angle], dtype=np.float32)
            ret, results, neighbours, dist = knn.findNearest(data,3)
            index = int(results[0][0])

            if index in gesture.keys():
                cv2.putText(img=img, text=gesture[index].upper(),
                            org=(int(res.landmark[0].x * img.shape[1] - 10), int(res.landmark[0].y * img.shape[0] + 40)),
                            fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=1, color=(255,255,255), thickness=3)

            mp_drawing.draw_landmarks(img, res, mp_hands.HAND_CONNECTIONS)

    cv2.imshow('HandTracking', img)
    if pressed_key != -1 and pressed_key != 97:
        break

f.close()


import pandas as pd
import numpy as np
import os
import glob
from common.io import file_base, parent_dir, mkdir
import re
import os

# @todo add warning if file is missing (i.e., None)
# @todo clean out files such that features are in 1 and labels (i.e., y or professor rating) is in another (samename_label.csv)
#

def get_metadata(fname, class_type=('lecture', 'lab', 'seminar', 'recitation discussion')):

    fname = fname.replace("Full_","");
    str_split = fname.split('_')
    semester = str(str_split[1] + str_split[2][2:4]).lower()

    fparts = str_split[0].split('/')[-1].split('-')
    cid = fparts[0]
    cid = [cid.lower().replace(c, '') for c in class_type if c.lower() in str(cid).lower()]

    if len(cid) == 0:
        return None, None, None, None
    if (re.search("[0-9]{6}",cid[0])):
        course_id = cid[0][:-2]
        section = cid[0][-2:]
    else:
        course_id = cid[0]
        section = '01'

    name = fparts[1].lower()

    return course_id, semester, name, section

def get_scores(fpath):
    """

    :param fpath:
    :return:
    """
    #  parse table
    df_scorechart = pd.read_excel(fpath, skiprows=10, nrows=29)
    df_scorechart.drop(df_scorechart.columns[12:], axis=1)

    return df_scorechart

def parse_surveys(data_dir, output_dir):
    """

    :param data_dir:
    :param output_dir:
    :return:
    """
    survey_dirs = glob.glob(data_dir + '*')

    colleges = [str(d).replace(data_dir, '') for d in survey_dirs]

    for college in colleges:

        obin = output_dir + '/' + college + '/'
        mkdir(obin)

        inbin = data_dir + '/' + college + '/'

        survey_paths = glob.glob(inbin + '*.xls')
        for survey in survey_paths:

            course, semester, name, section = get_metadata(survey)
            if course is None:
                continue
            df_scores = get_scores(survey)

            course_dir = os.path.join(obin, course)
            if mkdir(course_dir):
                print('created {}'.format(course_dir))
            semester_dir = os.path.join(course_dir, semester)

            if mkdir(semester_dir):
                print('created {}'.format(semester_dir))

            fout = semester_dir + '/' + name + '_' + section + '_' + semester + '.csv'

            df_scores.to_csv(fout)

if __name__ == "__main__":
    fpath = '/Users/zackhillman/Documents/eece2300/FinalProject/termproject/data/raw/surveys-raw/camd/CINE233601Lecture-Blake-201840_Summer_1_2018-Ratings_summary_w_CRN.xls'

    data_dir = '/Users/zackhillman/Documents/eece2300/FinalProject/termproject/data/raw/surveys-raw/'
    output_dir = '/Users/zackhillman/Documents/eece2300/FinalProject/termproject/data/processed/surveys/'

    parse_surveys(data_dir, output_dir)

#
# dict_of_colleges = {}
# dict_of_files = {}
# for (dirpath, dirnames, filenames) in os.walk(data_dir):
#     list_of_files = []
#     college_ref = file_base(dirpath)
#     for filename in filenames:
#         if filename.endswith('.xls'):
#             dict_of_files[filename] = os.sep.join([dirpath, filename])
#             list_of_files.append(filename)
#     if len(list_of_files) > 0:
#         dict_of_colleges[college_ref] = list_of_files
# #  parse header
#
# for k, v in dict_of_colleges.items():
#     print('There are {} samples for {}'.format(k, len(dict_of_colleges[k])))
# # There are ccis samples for 254
# # There are camd samples for 54
# # There are dmsb samples for 174
# # There are coe samples for 280
#
# #parse init info
# df_initial = pd.read_excel(fpath, nrows=6)
#
# df_intiial_transposed = df_initial.T
#
# df_initial = df_intiial_transposed.iloc[:2]
#
# #print(df_initial)
#
#
#
# # parse averaged scores
# df_overall_raw = pd.read_excel(fpath, skiprows=40, nrows=1)
#
# # check if last column is Nan
# if str(df_overall_raw.columns[-1][:6]).lower() == 'unname':
#     df_overall_raw = df_overall_raw.drop(labels=df_overall_raw.columns[-1], axis=1)
#
# #print(df_overall_raw)
#
# df_overall = df_overall_raw.iloc[:, 3::]
# #print(df_overall)
#
# df_scorechart.to_json("/Users/zackhillman/Documents/eece2300/FinalProject/termproject/data/processed/1.txt")
#
# # stats (i.e. number of each type of survey)


"""
Warp10 module for Loud ML
"""

import json
import logging
import math
import numpy as np

import warp10client

from voluptuous import (
    Any,
    Optional,
    Required,
    Url,
)
from . import (
    errors,
)
from .bucket import Bucket
from .misc import (
    make_ts,
    DateRange,
)

def check_tag(k, v):
    if type(k) is not str or type(v) is not str:
        raise errors.Invalid("warp10: tags key/value must be strings")

def check_tags(tags):
    for k, v in tags.items():
        check_tag(k, v)

def build_tags(tags=None):
    lst = ["'{}' '{}'".format(*item) for item in tags.items()] if tags \
        else []

    return "{{ {} }}".format(','.join(lst))

def metric_to_bucketizer(metric):
    if metric == 'avg':
        bucketizer = 'mean'
    else:
        bucketizer = metric
    return "bucketizer.{}".format(bucketizer)

def catch_query_error(func):
    def wrapper(self, *args, **kwargs):
        try:
            return func(self, *args, **kwargs)
        except warp10client.client.CallException as exn:
            raise errors.BucketError(self.name, str(exn))
    return wrapper

class Warp10Bucket(Bucket):
    """
    Warp10 bucket
    """

    SCHEMA = Bucket.SCHEMA.extend({
        Optional('url', default='http://localhost:8080'): Url(),
        Required('read_token'): str,
        Required('write_token'): str,
        Optional('global_prefix', default=None): Any(None, str),
    })

    def __init__(self, cfg):
        cfg['type'] = 'warp10'
        super().__init__(cfg)
        self.read_token = cfg['read_token']
        self.write_token = cfg['write_token']
        self.global_prefix = cfg.get('global_prefix')
        self.warp10 = warp10client.Warp10Client(
            warp10_api_url=cfg['url'],
            read_token=self.read_token,
            write_token=self.write_token,
        )

    def build_name(self, name):
        return "{}.{}".format(self.global_prefix, name) if self.global_prefix \
               else name

    def build_selector(self, selector, is_regexp=False):
        selector = self.build_name(selector)
        if is_regexp:
            selector = "~" + selector
        return selector

    @catch_query_error
    def drop(self, tags=None, **kwargs):
        """
        Delete database
        """
        self.warp10.delete({
            'name': self.build_selector(".*", is_regexp=True),
            'tags': tags or {},
        })

    def insert_data(self, data):
        raise NotImplementedError("Warp10 is a pure time-series database")

    def insert_times_data(
        self,
        ts,
        data,
        tags=None,
        *args,
        **kwargs
    ):
        """
        Insert data
        """

        ts_us = make_ts(ts) * 1e6

        if tags:
            check_tags(tags)

        for key, value in data.items():
            metric = {
                'name': self.build_selector(key),
                'value': value,
                'position': {
                    'longitude': None,
                    'latitude': None,
                    'elevation': None,
                    'timestamp': ts_us,
                },
                'tags': tags or {},
            }
            self.enqueue(metric)

    @catch_query_error
    def send_bulk(self, metrics):
        """
        Send data to Warp10
        """
        self.warp10.set(metrics)

    def build_fetch(self, feature, from_str, to_str, tags=None):
        tags = {} if tags is None else dict(tags)

        if feature.match_all:
            for tag in feature.match_all:
                k, v = tag['tag'], tag['value']
                check_tag(k, v)
                tags[k] = v

        tags_str = build_tags(tags)

        return "[\n'{}'\n'{}'\n{}\n'{}'\n'{}'\n]\nFETCH".format(
            self.read_token,
            self.build_selector(feature.field),
            tags_str,
            from_str,
            to_str,
        )

    def build_multi_fetch(
        self,
        bucket_interval,
        features,
        from_str,
        to_str,
        tags=None
    ):
        bucket_span = int(bucket_interval * 1e6)

        scripts = [
            "[\n{}\n{}\n0\n{}\n0\n]\nBUCKETIZE".format(
                self.build_fetch(
                    feature,
                    from_str,
                    to_str,
                    tags,
                ),
                metric_to_bucketizer(feature.metric),
                bucket_span,
            )
            for feature in features
        ]
        return "[\n{}\n]".format("\n".join(scripts))

    @catch_query_error
    def get_times_data(
        self,
        bucket_interval,
        features,
        from_date,
        to_date,
        tags=None,
        **kwargs
    ):
        period = DateRange.build_date_range(
            from_date, to_date, bucket_interval)

        nb_buckets = int((period.to_ts - period.from_ts) /
                         bucket_interval)
        buckets = np.full((nb_buckets, len(features)),
                          np.nan, dtype=float)

        script = self.build_multi_fetch(
            bucket_interval,
            features,
            period.from_str,
            period.to_str,
            tags=tags,
        )
        raw = self.warp10.exec(script)
        data = json.loads(raw)

        from_us = period.from_ts * 1e6
        to_us = period.to_ts * 1e6
        bucket_interval_us = int(bucket_interval * 1e6)

        has_data = False

        for i, item in enumerate(data[0]):
            if len(item) == 0:
                continue

            item = item[0]
            values = item['v']

            for ts_us, value in values:
                # XXX: Warp10 buckets are labeled with the right timestamp
                # but Loud ML uses the left one.
                ts_us -= bucket_interval_us

                if ts_us < from_us or ts_us >= to_us:
                    # XXX Sometimes, Warp10 returns extra buckets, skip them
                    continue

                j = math.floor((ts_us - from_us) / bucket_interval_us)
                buckets[j][i] = value
                has_data = True

        if not has_data:
            raise errors.NoData()

        result = []
        from_ts = ts = from_us / 1e6

        for bucket in buckets:
            result.append(((ts - from_ts), list(bucket), ts))
            ts += bucket_interval

        return result

    def save_timeseries_prediction(self, prediction, tags=None):
        prefix = prediction.model.name
        logging.info("saving prediction to '%s'",
                     self.build_name(prefix))
        for bucket in prediction.format_buckets():
            data = bucket['predicted']
            bucket_tags = tags or {}
            stats = bucket.get('stats', None)
            if stats is not None:
                data['score'] = float(stats.get('score'))
                bucket_tags['is_anomaly'] = stats.get('anomaly', False)

            # XXX As Warp10 uses the end date to identify buckets, use the same
            # convention
            ts = bucket['timestamp'] + prediction.model.bucket_interval

            self.insert_times_data(
                ts=ts,
                tags=bucket_tags,
                data={"{}.{}".format(prefix, k): v for k, v in data.items()},
            )
        self.commit()


import json
import os
import time

import streamlit as st

# Function to read JSON file
def read_json(filename):
    if os.path.exists(filename):
        with open(filename, "r") as f:
            return json.load(f)
    return []

# Function to write to JSON file
def write_json(filename, data):
    with open(filename, "w") as f:
        json.dump(data, f)

# Initialize JSON file name
json_file = "data/prompts.json"

# Title
st.title("BotGen")

# Create
with st.expander("Train Your Bot"):
    new_prompt = st.text_input("New Prompt")
    new_response = st.text_input("New Response")

    if st.button("Create"):
        new_data = {"prompt": new_prompt, "response": new_response}
        data = read_json(json_file)
        data.append(new_data)
        write_json(json_file, data)
        st.success("New data created")

# Read
with st.expander("Your Questions and Answers"):
    data = read_json(json_file)
    st.json(data)

# Update
with st.expander("Update Your Answers"):
    index_to_update = st.number_input(
        "Index to update", min_value=0, max_value=len(data) - 1, value=0
    )
    updated_prompt = st.text_input("Updated Prompt")
    updated_response = st.text_input("Updated Response")

    if st.button("Update"):
        data[index_to_update] = {"prompt": updated_prompt, "response": updated_response}
        write_json(json_file, data)
        st.success("Data updated")

# Delete
with st.expander("Delete Any Answers"):
    index_to_delete = st.number_input(
        "Index to delete", min_value=0, max_value=len(data) - 1, value=0
    )

    if st.button("Delete"):
        del data[index_to_delete]
        write_json(json_file, data)
        st.success("Data deleted")
# Train Chatbot Button
train_button_clicked = st.button("Train Your Chatbot")

# Progress Bar
progress_bar = st.empty()  # Placeholder for the progress bar

if train_button_clicked:
    progress_bar.progress(0)  # Start the progress bar at 0%

    for i in range(101):  # Simulating training progress
        progress_bar.progress(i)
        time.sleep(0.05)  # Simulating processing time
    progress_bar.empty()  # Clear the progress bar once training is done
    st.success("Chatbot training completed")


import threading
from django.shortcuts import render
from django.http import JsonResponse

from rescolle_view.unified_restaurant import service as ur_sv
from rescolle_view.sns_restaurant import service as sns_res_sv
from rescolle_view.crawl_raw_data import service as raw_sv
from rescolle_view.common import logger

def get_restaurant(request, restaurant_id):
    """
    特定レストランの情報を戻す
    :param request:
    :param restaurant_id:
    :return:
    """
    info = ur_sv.get_restaurant_info(restaurant_id)
    return JsonResponse({
        'restaurant': info
    })

def get_coordinate_list(request):
    keyword = request.POST.get('keyword')
    north_east_lat = float(request.POST.get('north_east_lat'))
    north_east_lng = float(request.POST.get('north_east_lng'))
    south_west_lat = float(request.POST.get('south_west_lat'))
    south_west_lng = float(request.POST.get('south_west_lng'))
    if keyword:
        # 少し広めに取る
        restaurant_coordinate_list = ur_sv.get_restaurant_coordinate_list_by_keyword(
            keyword=keyword,
            north_east_lat=north_east_lat * 1.00005,
            north_east_lng=north_east_lng * 1.00005,
            south_west_lat=south_west_lat * 0.99995,
            south_west_lng=south_west_lng * 0.99995,
        )
    else:
        restaurant_coordinate_list = ur_sv.get_all_restaurant_coordinate_list_by_latlng(
            north_east_lat=north_east_lat * 1.000001,
            north_east_lng=north_east_lng * 1.000001,
            south_west_lat=south_west_lat * 0.999999,
            south_west_lng=south_west_lng * 0.999999,
        )

    return JsonResponse({
        'restaurants': restaurant_coordinate_list,
        'center': ur_sv.get_center_position(restaurant_coordinate_list),
    })

def get_coordinate_list_from_collection_id(request, collection_id):
    restaurant_coordinate_list = ur_sv.get_restaurant_coordinate_list_by_collection_id(collection_id)
    return JsonResponse({
        'restaurants': restaurant_coordinate_list,
        'center': ur_sv.get_center_position(restaurant_coordinate_list),
    })

def generate_restaurant_endpoint(request, json_serial):
    """
    クローラから叩かれるエンドポイント
    JSONをもとに対応するレストランを生成する
    :param request:
    :param json_serial:
    :return:
    """
    def _async(target_crawled_data):
        logger.info('generate run')
        sns_res_sv.generate_restaurant_data(target_crawled_data.json, target_crawled_data.source_type)
        target_crawled_data.clear_serial()

    crawled_data = raw_sv.get_crawled_data_by_serial(serial=json_serial)
    if crawled_data:
        thread = threading.Thread(target=_async, args=(crawled_data, ))
        thread.start()
        logger.info('generate run')

        return JsonResponse({
            'status': 'end',
            'error': '',
        })
    else:
        return JsonResponse({
            'status': 'end',
            'error': 'invalid serial.'
        })


from flask import Flask, render_template, Response, jsonify, request
import time
import h5py
import numpy as np

# creates a Flask application, named app
app = Flask(__name__)
f = h5py.File('static/features.hdf5') #qual,track,unique,length
fea = np.array(f['fea_12'])
motion = np.array(f['mot_12'])
vid = "Jumps"
num = '12'

# a route where we will display a welcome message via an HTML template
@app.route("/")
def index():
	global motion,fea
	l = len(motion)
	return render_template('index.html',length=l,motion=int(100*motion[0]),qual=int(100*fea[0][0]),track=int(100*fea[0][1]),unique=int(100*fea[0][2]))

@app.route('/_get_frame_info')
def get_frame_info():
	val = request.args.get('i', 0, type=int)
	j = 0
	for i in range(len(fea)):
		j += fea[i][3]
		if val < j:
			m = int(100*motion[val])
			q = int(100*fea[i][0])
			t = int(100*fea[i][1])
			u = int(100*fea[i][2])
			break
	return jsonify(motion=m,qual=q,track=t,unique=u,img="/static/" + vid + "/frame" + str(val) + ".jpg")

@app.route('/_change_video')
def change_video():
	global motion,fea,f,vid,num
	vid = request.args.get('vid', 'Jumps', type=None)
	num = request.args.get('num', '12', type=None)
	motion = np.array(f['mot_' + num])
	fea = np.array(f['fea_' + num])
	l = len(motion)
	img = "/static/" + vid + "/frame0.jpg"
	orVid = "/static/" + vid + "/" + vid + ".ogg"
	sumVid = "/static/" + vid + "/sum.ogg"
	return jsonify(length=l,motion=int(motion[0]*100),qual=int(fea[0][0]*100),track=int(fea[0][1]*100),unique=int(fea[0][2]*100),img=img,orVid=orVid,sumVid=sumVid)

# run the application
if __name__ == "__main__":
	app.run(threaded=True)


import os, sys
import datetime
import psutil
from django import template
from psutil import virtual_memory
from psutil import cpu_times, cpu_count
from psutil import process_iter
from django.utils.safestring import mark_safe

register = template.Library()

procs = {p.pid: p.info for p in psutil.process_iter(attrs=['name', 'pid', 'username',])}
processes = procs

@register.simple_tag
def ctime(format_string):
    return datetime.datetime.now().strftime(format_string)

@register.simple_tag
def cpu():
    cpu1 = cpu_times()

    return mark_safe('''

user: {user}

nice: {nice}

system: {system}

idle: {idle}

iowait: {iowait}

irq: {irq}

softirq: {softirq}

steal: {steal}

guest: {guest}

number_of_cpu: {number_of_cpu}

load_avg: {load_avg}

	    '''.format(
	        user = cpu1.user,
	        system = cpu1.system,
	        idle = cpu1.idle,
	        iowait = cpu1.iowait,
	        irq = cpu1.irq,
	        softirq = cpu1.softirq,
	        steal = cpu1.steal,
	        guest = cpu1.guest,
	        nice = cpu1.nice,
	        number_of_cpu = cpu_count(),
	        load_avg = os.getloadavg()
	    ))

@register.simple_tag
def mem():
    memvirt = virtual_memory()

    return mark_safe('''

free: {free}

total: {total}

cached: {cached}

active: {active}

percent: {percent}

available: {available}

used: {used}

inactive: {inactive}

buffers: {buffers}

shared: {shared}

    	'''.format(
            free = memvirt.free,
            total = memvirt.total,
            cached = memvirt.cached,
            active = memvirt.active,
            percent = memvirt.percent,
            available = memvirt.available,
            used = memvirt.used,
            inactive = memvirt.inactive,
            buffers = memvirt.buffers,
            shared = memvirt.shared,
            ))

@register.simple_tag
def process():
	proc = psutil.process_iter(attrs=None, ad_value=None)

	return mark_safe('''

Processes: {processes}

		'''.format(
			processes = processes
			))


#!/usr/bin/python
# -*- encoding:utf-8 -*-

import json
import logging
import os
import time

import tornado.gen
import tornado.web

from algo.constants import RetCode
from algo.model_manager import ModelManager
from util.file_manager import FileManager

logger = logging.getLogger(__name__)

class StyleHandler(tornado.web.RequestHandler):
    @tornado.web.asynchronous
    @tornado.gen.coroutine
    def get(self):
        uid = self.get_argument("uid")
        style_name = self.get_argument("style_name")
        logger.info("uid: {}, style_name: {}".format(uid, style_name))

        # Check uid exists
        file_manager = FileManager()
        if not file_manager.exists(uid):
            logger.info("uid {} not exists.".format(uid))
            self.error_ret(-1, "Uid not exists.")
            return

        # Check style exists
        model_manager = ModelManager()
        if model_manager.get_queue_length(style_name) == -1:
            logger.info("Style name {} not exists.".format(style_name))
            self.error_ret(-2, "Style name {} not exists.".format(style_name))
            return

        # Styled name
        styled_uid = "{}_{}.{}".format(".".join(uid.split(".")[0:-1]),
                                       style_name,
                                       uid.split(".")[-1])
        if file_manager.exists(styled_uid):
            logger.info("Has already processed this styled_uid.")
            self.suc_ret(styled_uid, duration=0)
            return

        # Do style
        flag, order_id = model_manager.eval(style_name,
                                            os.path.join(file_manager.upload_dir, uid),
                                            os.path.join(file_manager.upload_dir, styled_uid))
        if not flag:
            logger.info("Inner model manager failed to eval.")
            self.error_ret(-3, "Inner model manager failed to eval.")
            return

        # Check order id
        task_ret = yield tornado.gen.Task(self.poll, order_id)
        if task_ret[0] == RetCode.NO_ERR:
            duration = task_ret[-1]
            self.suc_ret(styled_uid, duration=duration)
            return
        elif task_ret[0] == RetCode.ERR_READ_IMG:
            self.error_ret(-1, "Read image failed.")
            return
        else:
            self.error_ret(-100, "Inner error.")
            return

    @tornado.gen.coroutine
    def poll(self, order_id):
        model_manager = ModelManager()
        while True:
            ret = model_manager.check(order_id)
            if not ret:
                time.sleep(0.5)
                continue
            logger.info(str(ret))
            return ret

    def error_ret(self, status, reason):
        ret = {"status": status, "reason": reason}
        self.write(json.dumps(ret))

    def suc_ret(self, styled_uid, **kwargs):
        ret = {"status": 0, "styled_uid": styled_uid}
        for key, value in kwargs.items():
            ret[key] = value
        self.write(json.dumps(ret))


from deck_extractors.general_extractor import AbstractExtractor
from dataobjects.deck import Deck
from dataobjects.collection import Collection
from data.my_collection import cards
import requests
import json

my_collection = Collection()
my_collection.cards = cards

class TempostormExtractor(AbstractExtractor):
    source_site = 'https://tempostorm.com/'
    output_file = 'tempostorm_decks.json'

    def _process_url(self, url):
        payload = {"klass": False, "page": 1, "perpage": 4, "search": False}
        res = requests.post(url,
                            data=json.dumps(payload),
                            headers={
                                'Accept': 'application/json, text/plain, */*',
                                'Content-Type': 'application/json;charset=utf-8',
                            })
        total_featured_decks = res.json()['total']
        print ('Found %d featured decks' % total_featured_decks)
        payload['perpage'] = total_featured_decks
        res = requests.post(url,
                            data=json.dumps(payload),
                            headers={
                                'Accept': 'application/json, text/plain, */*',
                                'Content-Type': 'application/json;charset=utf-8',
                            })
        sources = []
        decks = res.json()['decks']
        for deck in decks:
            sources.append({
                'slug': deck['slug'],
                'name': deck['name']
            })
        return sources

    def get_decks_list(self):
        sources = []
        featured_decks = self._process_url('https://tempostorm.com/decks/featured')
        sources.extend(featured_decks)
        community_decks = self._process_url('https://tempostorm.com/decks/community')
        sources.extend(community_decks)
        return sources

    def get_deck(self, decks_source):
        print('Processing "%s" deck' % decks_source['name'])
        payload = {"slug": decks_source['slug']}
        res = requests.post('https://tempostorm.com/deck',
                            data=json.dumps(payload),
                            headers={
                                'Accept': 'application/json, text/plain, */*',
                                'Content-Type': 'application/json;charset=utf-8',
                            })
        deck_object = Deck()
        deck = res.json()['deck']
        player_class = deck['playerClass']
        cards = {}
        for card in deck['cards']:
            cards[my_collection.get_closest_name(card['card']['name'])] = card['qty']
        deck_object.cards = cards
        deck_object.player_class = player_class
        deck_object.is_valid()
        return deck_object


#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()

def retrieve(year):
    year=str(year)
    dat="{0}0101/{0}0201/{0}0301/{0}0401/{0}0501/{0}0601/{0}0701/{0}0801/{0}0901/{0}1001/{0}1101/{0}1201".format(year)
    server.retrieve({
        "class": "ei",
        "dataset": "interim",
        "date": dat,
        "expver": "1",
        "grid": "0.75/0.75",
        "levtype": "sfc",
        "param": "151.128/165.128/166.128",
        "stream": "moda",
        "type": "an",
        "target": "data-"+year+'.nc',
        "format": "netcdf"
    })

for year in range(1998,2018):
    retrieve(year)


import os
from functools import partial

from termcolor import colored
from torch.utils.data import DataLoader
import cv2
import utils as dwutils
from torchvision import transforms
import solt.core as slc
import solt.transforms as slt
import numpy as np
from tqdm import tqdm
from localizer.kneel_before_wrist.data.dataset import LandmarkTrainListDataset, MultipleLandmarkTrainListDataset
from localizer.kneel_before_wrist.data.utils import solt2torchhm
from utils.transformation import ColorPaddingWithSide, SIDES, TriangularMask, LowVisibilityTransform, SubSampleUpScale

class WrapImageLandmarksSOLT(object):

    def __call__(self, data):
        """
        Parameter:
        ----------
        data: image and landmarks
            image data and landmark to be converted to solt object

        Returns:
        --------
        solt object: wrapped solt object.
        """
        return dwutils.wrap_img_landmarks_solt(data[0], data[1])

    def __repr__(self):
        return self.__class__.__name__ + '()'

class SOLTtoHourGlassGSinput(object):
    def __init__(self, downsample, sigma):
        self.downsample = downsample
        self.sigma = sigma

    def __call__(self, data):
        return dwutils.solt_to_hourgalss_gs_input(data, downsample=self.downsample, sigma=self.sigma)

    def __repr__(self):
        return self.__class__.__name__ + '()'

class ApplyTransformByIndex(object):
    def __init__(self, transform, ids):
        self.transform = transform
        self.ids = ids

    def __call__(self, data):
        return dwutils.apply_by_index(data=data, transform=self.transform, idx=self.ids)

    def __repr__(self):
        return self.__class__.__name__ + '()'

class ApplyCustomFunction(object):
    def __init__(self, func, params):
        self.function = func
        self.params = params

    def __call__(self, x):
        return self.function(x, **self.params)

    def __repr__(self):
        return self.__class__.__name__ + '(function={0}, index={1}'.format(self.func, self.index)

def get_landmark_transform_kneel(config):
    cutout = slt.ImageCutOut(cutout_size=(int(config.dataset.cutout * config.dataset.augs.crop.crop_x),
                                          int(config.dataset.cutout * config.dataset.augs.crop.crop_y)),
                             p=0.5)
    # plus-minus 1.3 pixels
    jitter = slt.KeypointsJitter(dx_range=(-0.003, 0.003), dy_range=(-0.003, 0.003))
    ppl = transforms.Compose([
        ColorPaddingWithSide(p=0.05, pad_size=10, side=SIDES.RANDOM, color=(50,100)),
        TriangularMask(p=0.025, arm_lengths=(100, 50), side=SIDES.RANDOM, color=(50,100)),
        TriangularMask(p=0.025, arm_lengths=(50, 100), side=SIDES.RANDOM, color=(50,100)),
        LowVisibilityTransform(p=0.05, alpha=0.15, bgcolor=(50,100)),
        SubSampleUpScale(p=0.01),
        jitter if config.dataset.augs.use_target_jitter else slc.Stream(),
        slc.SelectiveStream([
            slc.Stream([
                slt.RandomFlip(p=0.5, axis=1),
                slt.RandomProjection(affine_transforms=slc.Stream([
                    slt.RandomScale(range_x=(0.9, 1.1), p=1),
                    slt.RandomRotate(rotation_range=(-90, 90), p=1),
                    slt.RandomShear(range_x=(-0.1, 0.1), range_y=(-0.1, 0.1), p=0.5),
                ]), v_range=(1e-5, 2e-3), p=0.5),
                # slt.RandomScale(range_x=(0.5, 2.5), p=0.5),
            ]),
            slc.Stream()
        ], probs=[0.7, 0.3]),
        slc.Stream([
            slt.PadTransform((config.dataset.augs.pad.pad_x, config.dataset.augs.pad.pad_y), padding='z'),
            slt.CropTransform((config.dataset.augs.crop.crop_x, config.dataset.augs.crop.crop_y), crop_mode='r'),
        ]),
        slc.SelectiveStream([
            slt.ImageSaltAndPepper(p=1, gain_range=0.01),
            slt.ImageBlur(p=1, blur_type='g', k_size=(3, 5)),
            slt.ImageBlur(p=1, blur_type='m', k_size=(3, 5)),
            slt.ImageAdditiveGaussianNoise(p=1, gain_range=0.5),
            slc.Stream([
                slt.ImageSaltAndPepper(p=1, gain_range=0.05),
                slt.ImageBlur(p=0.5, blur_type='m', k_size=(3, 5)),
            ]),
            slc.Stream([
                slt.ImageBlur(p=0.5, blur_type='m', k_size=(3, 5)),
                slt.ImageSaltAndPepper(p=1, gain_range=0.01),
            ]),
            slc.Stream()
        ], n=1),
        slt.ImageGammaCorrection(p=0.5, gamma_range=(0.5, 1.5)),
        cutout if config.dataset.use_cutout else slc.Stream(),
        partial(solt2torchhm, downsample=None, sigma=None),
    ])
    return ppl

def get_landmark_transform(config):
    return transforms.Compose([
        # WrapImageLandmarksSOLT(),
        slc.Stream([
            slt.RandomFlip(p=0.5, axis=1),
            slt.RandomScale(range_x=(0.8, 1.2), p=1),
            slt.RandomRotate(rotation_range=(-180, 180), p=0.2),
            slt.RandomProjection(affine_transforms=slc.Stream([
                slt.RandomScale(range_x=(0.8, 1.3), p=1),
                slt.RandomRotate(rotation_range=(-180, 180), p=1),
                slt.RandomShear(range_x=(-0.1, 0.1), range_y=(0, 0), p=0.5),
                slt.RandomShear(range_y=(-0.1, 0.1), range_x=(0, 0), p=0.5),
            ]), v_range=(1e-5, 2e-3), p=0.8),
            slt.PadTransform(int(config.dataset.crop_size * 1.4), padding='z'),
            slt.CropTransform(config.dataset.crop_size, crop_mode='r'),
            slc.SelectiveStream([
                slt.ImageSaltAndPepper(p=1, gain_range=0.01),
                slt.ImageBlur(p=1, blur_type='g', k_size=(3, 5)),
                slt.ImageBlur(p=1, blur_type='m', k_size=(3, 5)),
                slt.ImageAdditiveGaussianNoise(p=1, gain_range=0.5),
                slc.Stream([
                    slt.ImageSaltAndPepper(p=1, gain_range=0.05),
                    slt.ImageBlur(p=0.5, blur_type='m', k_size=(3, 5)),
                ]),
                slc.Stream([
                    slt.ImageBlur(p=0.5, blur_type='m', k_size=(3, 5)),
                    slt.ImageSaltAndPepper(p=1, gain_range=0.01),
                ]),
                slc.Stream()
            ]),
            slt.ImageGammaCorrection(p=1, gamma_range=(0.5, 1.5))
        ]),
        SOLTtoHourGlassGSinput(downsample=4, sigma=3),
        ApplyTransformByIndex(transform=dwutils.npg2tens, ids=[0, 1]),
    ])

def calculate_mean_std_from_dataset(train_meta, config, file, trf, side=0):
    # Mean estimation
    data_folder = os.path.join(config.dataset.data_home, config.dataset.data_folder)
    train_ds = MultipleLandmarkTrainListDataset(data_folder, train_meta, transform=trf,
                                        side=side)
    train_loader = DataLoader(train_ds, batch_size=config.train_params.train_bs,
                              num_workers=config.dataset.n_data_workers)

    mean_vector = np.zeros(1)
    std_vector = np.zeros(1)

    print(colored('==> ', 'green') + 'Estimating the mean')
    pbar = tqdm(total=len(train_loader))
    for entry in train_loader:
        batch = entry['data']
        for j in range(mean_vector.shape[0]):
            mean_vector[j] += batch[:, j, :, :].mean()
            std_vector[j] += batch[:, j, :, :].std()
        pbar.update()
    mean_vector /= len(train_loader)
    std_vector /= len(train_loader)
    np.save(file, [mean_vector, std_vector])
    pbar.close()
    return mean_vector, std_vector

def get_train_val_transform(landmark_trf, normalization, config):
    train_transform = transforms.Compose([
        landmark_trf,
        lambda x: dwutils.apply_by_index(x, normalization, 0),
    ])

    val_transform = transforms.Compose([
        landmark_trf,
        # slc.Stream([
        #     slt.PadTransform(config.dataset.crop_size, padding='z'),
        #     slt.CropTransform(config.dataset.crop_size, crop_mode='c'),
        # ]),
        lambda x: dwutils.apply_by_index(x, normalization, 0),
    ])
    return train_transform, val_transform