import sys, os, inspect
import pandas as pd
import numpy as np
import glob
from clv.executor import CLV
from sqlalchemy import create_engine, MetaData

currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0, parentdir)

from customeranalytics.configs import default_es_port, default_es_host, max_elasticsearch_bulk_insert_bytes, default_es_bulk_insert_chunk_size
from customeranalytics.utils import *
from customeranalytics.data_storage_configurations.query_es import QueryES
from customeranalytics.data_storage_configurations.reports import Reports

engine = create_engine('sqlite://///' + join(abspath_for_sample_data(), "web", 'db.sqlite3'),
                       convert_unicode=True, connect_args={'check_same_thread': False})
metadata = MetaData(bind=engine)
con = engine.connect()

class CLVPrediction:
    """
    Customer Lifetime Value Prediction;
    For more information pls check; https://github.com/caglanakpinar/clv_prediction

    """
    def __init__(self,
                 temporary_export_path,
                 host=None,
                 port=None,
                 time_period='weekly',
                 download_index='downloads',
                 order_index='orders'):
        """
        ******* ******** *****
        Dimensional CLV Prediction:
        Descriptive Statistics must be created individually for dimensions.
        For instance, the Data set contains locations dimension.
        In this case, each location of 'orders' and 'downloads' indexes must be created individually.
        by using 'download_index' and 'order_index' dimension can be assigned in order to create the CLV Prediction.

        download_index; downloads_location1 this will be the location dimension of
                        parameters in order to query downloads indexes; 'location1'.
        download_index; orders_location1 this will be the location dimension of
                        parameters in order to query orders indexes; 'location1'.
        ******* ******** *****
        !!!

        :param host: elasticsearch host
        :param port: elasticsearch port
        :param download_index: elasticsearch port
        :param order_index: elasticsearch port
        """
        self.port = default_es_port if port is None else port
        self.host = default_es_host if host is None else host
        self.download_index = download_index
        self.order_index = order_index
        self.time_period = time_period
        self.clv = None
        self.query_es = QueryES(port=port, host=host)
        self.path = temporary_export_path
        self.temp_csv_file = join(temporary_export_path, "temp_data.csv")
        self.temp_folder = join(temporary_export_path, "temp_purchase_amount_results", "*.csv")
        self.results = join(temporary_export_path, "results_*.csv")
        self.clv_fields = ["client", "session_start_date", "payment_amount", "dimension"]
        self.result_query = "session_start_date == session_start_date and data_type != 'actual'"
        self.result_columns = ['session_start_date', 'client', 'payment_amount']
        self.clv_predictions = pd.DataFrame()
        self.client_dimensions = pd.DataFrame()
        self.collect_reports = Reports()
        self.has_prev_clv = False

    def get_customers_dimesions(self, _data):
        _data["dimension"] = _data["dimension"].fillna("main")
        if list(_data['dimension'].unique()) != 1:
            _data_pv = _data.groupby(["client", "dimension"]).agg({"session_start_date": "count"}).reset_index().rename(
                columns={"session_start_date": "order_count"})
            _data_pv['order_seq_num'] = _data_pv.sort_values(
                by=["client", "dimension", "order_count"], ascending=False).groupby(["client"]).cumcount() + 1
            _data_pv = _data_pv.query("order_seq_num == 1")
            _data_pv = _data_pv[["client", "dimension"]].groupby("client").agg({"dimension": "first"}).reset_index()
            self.client_dimensions = pd.merge(_data.drop('dimension', axis=1), _data_pv, on='client', how='left')

    def get_orders_data(self, end_date):
        """
        Purchased orders are collected from Orders Index.
        Session_start_date, client, payment_amount are needed for initializing CLv Prediciton.
        :param end_date: final date to start clv prediction
        :return:
        """
        self.query_es = QueryES(port=self.port, host=self.host)
        self.query_es.date_queries_builder({"session_start_date": {"lt": end_date}})
        self.query_es.query_builder(fields=self.clv_fields,
                                    boolean_queries=[{"term": {"actions.purchased": True}}])
        _data = pd.DataFrame(self.query_es.get_data_from_es())
        _data['session_start_date'] = _data['session_start_date'].apply(lambda x: str(convert_to_date(x)))
        self.get_customers_dimesions(_data)
        _data.query("session_start_date == session_start_date").to_csv(self.temp_csv_file, index=False)

    def check_for_previous_clv_prediction(self):
        """
        check if the last clv prediction is applied in 7 days.
        """
        try:

            _reports = self.collect_reports.collect_reports(self.port, self.host, 'main',
                                                            query={"report_name": "clv_prediction"})
            if len(_reports) != 0:
                try:
                    _directory = pd.read_sql("SELECT * FROM es_connection", con).to_dict('results')[-1]['directory']
                    print(_directory)
                    clv_report = pd.read_csv(join(_directory, "build_in_reports", "main", "daily_clv.csv"))
                    if len(clv_report) != 0:
                        clv_report = clv_report.query("data_type == 'prediction'")
                        clv_report['date'] = clv_report['date'].apply(lambda x: convert_to_day(x))
                        last_transaction_date = min(list(clv_report['date']))
                        print(last_transaction_date)
                except Exception as e:
                    print(e)
                    _reports['report_date'] = _reports['report_date'].apply(lambda x: convert_to_date(x))
                    last_transaction_date = max(list(_reports['report_date']))

                if abs(last_transaction_date - current_date_to_day()).total_seconds() / 60 / 60 / 24 < 7:
                    print("clv predictions will be updated!!!!!")
                    self.has_prev_clv = True
        except: pass

    def insert_into_reports_index(self, clv_predictions, time_period, date=None, index='orders'):
        """
        via query_es.py, each report can be inserted into the reports index with the given format.
        {"id": unique report id,
         "report_date": date or current date,
         "report_name": "clv_prediction",
         "index": "main",
         "report_types": {"time_period": weekly, monthly, daily },
         "data": segments.fillna(0.0).to_dict("results") -  dataframe to list of dictionary
         }
         !!! null values are assigned to 0.

        :param segments: data set, data frame
        :param index: dimensionality of data index orders_location1 ;  dimension = location1
        """

        iteration = int(len(clv_predictions) / default_es_bulk_insert_chunk_size) + 1
        print("number of bulk iteration :", iteration)
        print("bulk size :", default_es_bulk_insert_chunk_size)
        for i in range(iteration):
            _start = i * default_es_bulk_insert_chunk_size
            _end = ((i+1) * default_es_bulk_insert_chunk_size) if i != (iteration - 1) else len(clv_predictions) + 1
            _clv_predictions = clv_predictions.iloc[_start:_end]
            _list_of_obj = [{"id": np.random.randint(200000000),
                             "report_date": current_date_to_day().isoformat() if date is None else convert_to_day(
                                 date).isoformat(),
                             "report_name": "clv_prediction",
                             "index": get_index_group(index),
                             "report_types": {"time_period": time_period, "partition": i},
                             "data": _clv_predictions.fillna(0.0).to_dict("results")}]
            self.query_es.insert_data_to_index(_list_of_obj, index='reports')

    def convert_time_period(self, period):
        """
        clv_prediction library takes time-period as week, day, month.
        This converts the general time-period into the clv_prediction form.
        :param period: weekly or daily or month.
        :return: week or day or month
        """
        if period == 'weekly':
            return 'week'
        if period == 'daily':
            return 'day'
        if period == 'monthly':
            return 'month'
        if period == '6 months':
            return '6*month'

    def remove_temp_files(self):
        """
        Removing temp_data.csv, .csv files in temp_purchase_amount_results and results*.csv files
        """
        # remove temp_data.csv
        try:
            os.unlink(self.temp_csv_file)
        except Exception as e:
            print("no file is observed!!!")

        # remove results*.csv files
        for f in glob.glob(self.results):
            print(f)
            try:
                os.unlink(f)
            except Exception as e:
                print(e)
                print("no file is observed!!!")

        # remove .csv files in temp_purchase_amount_results
        for f in glob.glob(self.temp_folder):
            try:
                os.unlink(f)
            except Exception as e:
                print(e)
                print("no file is observed!!!")

    def execute_clv(self, start_date, job='train_prediction', time_period='6 months'):
        """
        1. train clv prediction models
           For more details about models pls check; https://github.com/caglanakpinar/clv_prediction
        2. predict users of feature payment amount via using built models.
        3. use 'train_prediction' for the job argument which allows us to train and predict the results.
           If time period is weekly and if there is a stored .json and .h5 trained models in last 7 days,
           skips the train process and uses the latest trained model.
        4. Each model and stored in the directory at elasticsearch folder. After it is used, it is removed.
        5. If last stored clv prediction is applied before 7 days, it is not triggered for prediction again.

        :param start_date: date of start for clv prediction
        :param job: train or prediction
        :param time_period: weekly, daily, monthly
        """
        start_date = str(current_date_to_day())[0:10] if start_date is None else start_date
        self.check_for_previous_clv_prediction()
        if not self.has_prev_clv:
            self.get_orders_data(end_date=start_date)
            self.clv = CLV(customer_indicator="client",
                           amount_indicator="payment_amount",
                           job=job,
                           date=start_date,
                           data_source='csv',
                           data_query_path=self.temp_csv_file,
                           time_period=self.convert_time_period(time_period),
                           time_indicator="session_start_date",
                           export_path=self.path)
            self.clv.clv_prediction()
            self.clv_predictions = self.clv.get_result_data().query(self.result_query)[self.result_columns]
            self.clv_predictions = self.clv_predictions.rename(columns={"session_start_date": "date"})
            self.clv_predictions['date'] = self.clv_predictions['date'].apply(lambda x: convert_to_iso_format(x))
            self.add_dimensions()
            self.clv_predictions = self.clv_predictions[['dimension', 'date', 'payment_amount', 'client']]
            self.insert_into_reports_index(self.clv_predictions,
                                           time_period=time_period,
                                           date=start_date,
                                           index=self.order_index)
            del self.clv
            del self.clv_predictions
            self.clv = None
            self.clv_predictions = pd.DataFrame()
            self.remove_temp_files()

    def add_dimensions(self):
        _prediction_start_date = min(self.clv_predictions['date'])
        if len(self.client_dimensions) != 0:
            _clv_predictions_newcomers = self.clv_predictions.query("client == 'newcomers'")
            print(_clv_predictions_newcomers.head())
            self.clv_predictions = pd.merge(self.clv_predictions.query("client != 'newcomers'"),
                                            self.client_dimensions[['client', 'dimension']], on="client", how="left")
            # add newcomers of dimensions
            _dimension_user_ratios = self.clv_predictions.groupby('dimension').agg(
                {'client': lambda x: len(np.unique(x))}).reset_index().rename(columns={'client': 'ratio'})
            _dimension_user_ratios['ratio'] = _dimension_user_ratios['ratio'] / len(
                self.clv_predictions['client'].unique())
            _clv_predictions_newcomers = _clv_predictions_newcomers.groupby("date").agg(
                {"payment_amount": "sum"}).reset_index().rename(columns={'payment_amount': 'pa'})
            _clv_predictions_newcomers['client'] = 'newcomers'
            dfs = []
            for _ratio in list(_dimension_user_ratios['ratio']):
                _clv_predictions_newcomers['payment_amount'] = _clv_predictions_newcomers['pa'] * _ratio
                dfs.append(_clv_predictions_newcomers)
            self.clv_predictions = pd.concat([self.clv_predictions.query("client != 'newcomers'")] + dfs)
        else: self.clv_predictions['dimension'] = 'main'
        self.clv_predictions['dimension'] = self.clv_predictions['dimension'].apply(lambda x: str(x))
        self.clv_predictions = self.clv_predictions.query("date > @_prediction_start_date")

    def fetch(self, end_date=None, time_period='weekly'):
        """
        Collect CLV Prediction results with the given date.
        :param end_date: final date to start clv prediction
        :param time_period: weekly or daily or month.
        :return:
        """

        boolean_queries, date_queries = [], []
        boolean_queries = [{"term": {"report_name": "clv_prediction"}},
                           {"term": {"report_types.time_period": time_period}},
                           {"term": {"index": get_index_group(self.order_index)}}]

        if end_date is not None:
            date_queries = [{"range": {"report_date": {"lt": convert_to_iso_format(end_date)}}}]

        self.query_es = QueryES(port=self.port,
                                host=self.host)
        self.query_es.query_builder(fields=None, _source=True,
                                    date_queries=date_queries,
                                    boolean_queries=boolean_queries)
        _res = self.query_es.get_data_from_es(index="reports")
        _data = pd.DataFrame()
        if len(_res) != 0:
            _data = pd.DataFrame(_res[0]['_source']['data'])
            if end_date is not None:
                _data[time_period] = _data["session_start_date"].apply(lambda x: convert_to_date(x))
        return _data


# calc.py
import argparse

parser = argparse.ArgumentParser(description = "CLI Calculator.")

subparsers = parser.add_subparsers(help = "sub-command help", dest="command")

add = subparsers.add_parser("add", help = "add integers")
add.add_argument("ints_to_sum", nargs='*', type=int)

sub = subparsers.add_parser("sub", help = "subtract integers")
sub.add_argument("ints_to_sub", nargs=2, type=int)

div = subparsers.add_parser("div", help = "divide integers")
div.add_argument("ints_div", nargs=2, type=int)

def aec_subtract(ints_to_sub):
  arg_1 = ints_to_sub[0]
  arg_2 = ints_to_sub[1]
  our_sub = arg_1 - arg_2
  if our_sub < 0:
    our_sub = 0
  print(f"The subtracted result is {our_sub}")
  return our_sub

def aec_divide(ints_div):
    try:
        our_div = ints_div[0] / ints_div[1]
        print(f"the divide of values is: {our_div}")
        return our_div
    except ZeroDivisionError:
        print(f"cannot divide {ints_div[0]} by {ints_div[1]}.")

if __name__ == '__main__':
    args = parser.parse_args()

    if args.command == "add":
        our_sum = sum(args.ints_to_sum)
        print(f"the sum of values is: {our_sum}")

    if args.command == "sub":
        aec_subtract(args.ints_to_sub)

    if args.command == "div":
        aec_divide(args.ints_div)
        # try:
        #     our_div = args.ints_div[0] / args.ints_div[1]
        #     print(f"the divide of values is: {our_div}")

        # except ZeroDivisionError:
        #     print(f"cannot divide {args.ints_div[0]} by {args.ints_div[1]}.")


#!/usr/bin/python

import os
import esm
import torch
import random
import argparse
import numpy as np
import pandas as pd
from tqdm import tqdm
import matplotlib.pyplot as plt

from utils.data import *
from utils.plot_protherm import *
from sequence_attack import SequenceAttack
from models.esm_embedding import EsmEmbedding
from models.msa_esm_embedding import MsaEsmEmbedding
from utils.protein_sequences import compute_cmaps_distance_df_protherm

print("\ntorch.cuda.is_available() =", torch.cuda.is_available(), "\ttorch version =", torch.version.cuda)

random.seed(0)
np.random.seed(0)
torch.manual_seed(0)

parser = argparse.ArgumentParser()
parser.add_argument("--data_dir", default='/scratch/external/gcarbone/', type=str, help="Datasets path.")
parser.add_argument("--out_dir", default='/fast/external/gcarbone/adversarial-protein-sequences_out/', type=str,
	help="Output data path.")
parser.add_argument("--model", default='ESM', type=str, help="Choose 'ESM' or 'ESM_MSA'")
# parser.add_argument("--max_tokens", default=None, type=eval,
#     help="Optionally cut sequences to maximum number of tokens. None does not cut sequences.")
parser.add_argument("--min_filter", default=30, type=int, help="Minimum number of sequences selected for the filtered MSA.")
parser.add_argument("--token_selection", default='max_attention', type=str,
	help="Method used to select most relevant token idxs. Choose 'max_attention', 'max_entropy' or 'min_entropy'.")
parser.add_argument("--target_attention", default='last_layer', type=str,
	help="Attention matrices used to choose target token idxs. Set to 'last_layer' or 'all_layers'. \
	Used only when `token_selection`=`max_attention")

parser.add_argument("--loss_method", default='max_masked_prob', type=str,
	help="Loss function used to compute gradients in the first embedding space. Choose 'max_masked_ce', max_masked_prob' \
	or 'max_tokens_repr'.")

parser.add_argument("--cmap_dist_lbound", default=0.2, type=int,
	help='Lower bound for upper triangular matrix of long range contacts.')
parser.add_argument("--cmap_dist_ubound", default=0.8, type=int,
	help='Upper bound for upper triangular matrix of long range contacts.')

parser.add_argument("--device", default='cuda', type=str, help="Device: choose 'cpu' or 'cuda'.")
parser.add_argument("--load", default=False, type=eval, help='If True load else compute.')
parser.add_argument("--verbose", default=True, type=eval)
args = parser.parse_args()
print("\n", args)

if args.model=="ESM":
	max_tokens = None
	out_filename = f"ProTherm_{args.model}_atk_max_toks={max_tokens}_{args.loss_method}_attn={args.target_attention}"

elif args.model=="ESM_MSA":
	max_tokens = 100
	out_filename = f"ProTherm_{args.model}_atk_max_toks={max_tokens}_minFilter={args.min_filter}_{args.loss_method}_attn={args.target_attention}"

else:
	raise NotImplementedError

out_plots_path = os.path.join(args.out_dir, "plots/", out_filename+"/")
out_data_path =  os.path.join(args.out_dir, "data/protherm/", out_filename+"/")
os.makedirs(os.path.dirname(out_plots_path), exist_ok=True)
os.makedirs(os.path.dirname(out_data_path), exist_ok=True)

if args.model=='ESM':
	perturbations_keys = ['protherm','max_dist','max_cos','max_cmap_dist', 'masked_pred']
	perturbations_keys_plots = ['protherm','max_dist','max_cos','max_cmap_dist']

elif args.model=='ESM_MSA':
	perturbations_keys = ['protherm','max_dist','max_cos','max_cmap_dist', 'max_entropy', 'masked_pred']
	perturbations_keys_plots = ['protherm','max_dist','max_cos','max_cmap_dist','max_entropy']

else:
	raise NotImplementedError

if args.load:

	atk_df = pd.read_csv(os.path.join(out_data_path, out_filename+"_atk.csv"), index_col=[0])
	cmap_df = pd.read_csv(os.path.join(out_data_path, out_filename+"_cmaps.csv"))
	distances_df_protherm = pd.read_csv(os.path.join(out_data_path, out_filename+"_distances_protherm.csv"))
	distances_df_adversarial = pd.read_csv(os.path.join(out_data_path, out_filename+"_distances_adversarial.csv"))

else:

	### load data
	protherm_df = pd.read_csv(f"{args.data_dir}ProTherm/processed_single_mutation.csv", sep=';')
	sequences_df = pd.read_csv(f"{args.data_dir}ProTherm/sequences_single_mutation.csv", sep=';')

	print("\nprotherm_df:\n\n", protherm_df.head())
	print("\nsequences_df:\n\n", sequences_df.head())
	print(f"\n{len(protherm_df)} ProTherm mutations")

	if args.model=='ESM':
		pretrained_model=esm.pretrained.esm1b_t33_650M_UR50S
		embedding_model=EsmEmbedding

	elif args.model=='ESM_MSA':
		pretrained_model=esm.pretrained.esm_msa1b_t12_100M_UR50S
		embedding_model=MsaEsmEmbedding

	else:
		raise NotImplementedError

	esm_model, alphabet = pretrained_model()
	batch_converter = alphabet.get_batch_converter()
	n_layers = esm_model.args.layers
	esm_model = esm_model.to(args.device)

	emb_model = embedding_model(original_model=esm_model, alphabet=alphabet).to(args.device)
	emb_model = emb_model.to(args.device)

	atk = SequenceAttack(original_model=esm_model, embedding_model=emb_model, alphabet=alphabet)

	### Dataframes

	atk_df = pd.DataFrame()
	cmap_df = pd.DataFrame()
	distances_df_protherm = pd.DataFrame()
	distances_df_adversarial = pd.DataFrame()

	for row_idx, row in tqdm(sequences_df.iterrows(), total=len(sequences_df)):

		original_sequence=row.SEQUENCE[:max_tokens]

		print(f"\n=== Mutations for sequence {row.FASTA} ===\n")
		print(original_sequence, "\n")

		mutations_df = protherm_df[(protherm_df.PFAM==row.PFAM) & (protherm_df.UNIPROT==row.UNIPROT)]

		if not mutations_df.empty:

			name = row.FASTA
			seq_filename = row.FASTA.replace('/','_')

			### compute first continuous embedding

			if args.model=='ESM_MSA':

				msa, max_tokens = load_msa(
					filepath=f"{args.data_dir}msa/hhfiltered/hhfiltered_{row.PFAM}_filter={args.min_filter}",
					filename=f"{row.PFAM}_{seq_filename}_no_gaps_filter={args.min_filter}",
					max_model_tokens=esm_model.args.max_tokens, max_tokens=max_tokens)

				### put current sequence on top of the msa
				msa = dict(msa)
				if name in msa.keys():
					msa.pop(name)
				msa = tuple(msa.items())
				msa = [(name, original_sequence)] + list(msa)

				batch_labels, batch_strs, batch_tokens = batch_converter(msa)
				batch_tokens = batch_tokens.to(args.device)

				with torch.no_grad():
					repr_layer_idx = 0
					results = esm_model(batch_tokens, repr_layers=[repr_layer_idx], return_contacts=True)
					first_embedding = results["representations"][repr_layer_idx].to(args.device)

			else:

				msa=None

				batch_labels, batch_strs, batch_tokens = batch_converter([(name, original_sequence)])
				batch_tokens = batch_tokens.to(args.device)

				with torch.no_grad():
					results = esm_model(batch_tokens, repr_layers=[0], return_contacts=True)
					first_embedding = results["representations"][0].to(args.device)

			### choose target positions

			target_token_idxs, target_tokens_attention = atk.choose_target_token_idxs(token_selection=args.token_selection,
				n_token_substitutions=len(original_sequence), msa=msa, batch_tokens=batch_tokens,
				target_attention=args.target_attention, verbose=False)

			if max_tokens is not None:
				mutations_df = mutations_df[mutations_df['POSITION']<=max_tokens]

			for _, mutation_row in mutations_df.iterrows():

				torch.cuda.empty_cache()

				pdb_start = mutation_row.PDB_START
				target_pdb_idxs = [idx+pdb_start for idx in target_token_idxs]
				protherm_token_idx = mutation_row.POSITION
				protherm_idx_rank = target_pdb_idxs.index(protherm_token_idx)/len(target_pdb_idxs)
				print(f"\nprotherm_token_idx {protherm_token_idx} found at position = {protherm_idx_rank}")

				### attack sequence at ProTherm mutant position

				for perturbation in perturbations_keys:

					target_token=mutation_row.MUTANT if perturbation=='protherm' else None

					# target_token_idx=protherm_token_idx-pdb_start
					target_token_idx=target_token_idxs[0]

					if mutation_row.WILD_TYPE==original_sequence[target_token_idx]:

						signed_gradient, loss = atk.compute_loss_gradient(original_sequence=original_sequence,
							batch_tokens=batch_tokens, target_token_idxs=[target_token_idx], first_embedding=first_embedding,
							loss_method=args.loss_method)

						row_dict = atk.attack_single_position(name=name, original_sequence=original_sequence,
							original_batch_tokens=batch_tokens, msa=msa, position=protherm_token_idx, pdb_start=pdb_start,
							target_token_idx=target_token_idx, first_embedding=first_embedding, signed_gradient=signed_gradient,
							perturbation=perturbation, verbose=args.verbose, target_token=target_token)

						row_dict['DDG'] = mutation_row.DDG
						row_dict['chose_mutant_token'] = bool(row_dict['target_token']==mutation_row.MUTANT)
						row_dict['protherm_idx_rank'] = protherm_idx_rank

						atk_df = atk_df.append(row_dict, ignore_index=True)

						dist_df_prot = atk.build_distances_df(name=name, original_sequence=original_sequence,
							original_batch_tokens=batch_tokens, msa=msa, first_embedding=first_embedding,
							target_token_idxs=[protherm_token_idx-pdb_start],
							signed_gradient=signed_gradient, verbose=args.verbose)

						dist_df_adv = atk.build_distances_df(name=name, original_sequence=original_sequence,
							original_batch_tokens=batch_tokens, msa=msa, first_embedding=first_embedding,
							target_token_idxs=[target_token_idx],
							signed_gradient=signed_gradient, verbose=args.verbose)

						distances_df_protherm = pd.concat([distances_df_protherm, dist_df_prot], ignore_index=True)
						distances_df_adversarial = pd.concat([distances_df_adversarial, dist_df_adv], ignore_index=True)

						### contact maps distances

						perturbed_sequences_dict = {perturbation:row_dict['perturbed_sequence']}

						df = compute_cmaps_distance_df_protherm(model=esm_model, alphabet=alphabet,
								perturbation=perturbation, ddg=mutation_row.DDG, original_sequence=original_sequence,
								sequence_name=name, perturbed_sequences_dict=perturbed_sequences_dict, verbose=args.verbose,
								cmap_dist_lbound=args.cmap_dist_lbound, cmap_dist_ubound=args.cmap_dist_ubound)

						cmap_df = pd.concat([cmap_df, df], ignore_index=True)

	atk_df.to_csv(os.path.join(out_data_path,  out_filename+"_atk.csv"))
	cmap_df.to_csv(os.path.join(out_data_path,  out_filename+"_cmaps.csv"))
	distances_df_protherm.to_csv(os.path.join(out_data_path,  out_filename+"_distances_protherm.csv"))
	distances_df_adversarial.to_csv(os.path.join(out_data_path,  out_filename+"_distances_adversarial.csv"))

print("\natk_df:\n", atk_df.keys(), "len =", len(atk_df))
print("\ncmap_df:\n", cmap_df.keys(), "len =", len(cmap_df))

### select rows with top DDG values

stabilization_ths=1.
atk_df = atk_df[(atk_df['DDG']>stabilization_ths) | (atk_df['DDG']<-stabilization_ths)]
cmap_df = cmap_df[(cmap_df['ddg']>stabilization_ths) | (cmap_df['ddg']<-stabilization_ths)]

# atk_df = atk_df[(atk_df['DDG']<-stabilization_ths)]
# atk_df = atk_df[(atk_df['DDG']>stabilization_ths)]
# idx = 0
# print(len(list(atk_df['original_sequence'])[idx]),list(atk_df['DDG'])[idx],list(atk_df['original_sequence'])[idx],list(atk_df['perturbed_sequence'])[idx])
# exit()

print(f"\nn. protherm mutations = {len(atk_df[atk_df['perturbation']=='protherm'])}")
print(f"\natk_df size = {len(atk_df)}")

### plots

plot_hist_position_ranks(atk_df, keys=perturbations_keys, filepath=out_plots_path, filename=out_filename)
plot_confidence(atk_df, keys=perturbations_keys, filepath=out_plots_path, filename=out_filename)
plot_embeddings_distances(atk_df, keys=perturbations_keys_plots, filepath=out_plots_path, filename=out_filename)
plot_blosum_distances(atk_df, keys=perturbations_keys_plots, filepath=out_plots_path, filename=out_filename)
plot_cmap_distances(cmap_df, R=20, keys=perturbations_keys_plots, filepath=out_plots_path, filename=out_filename)


from django import forms
from django.forms import ModelForm
from .models import Movie, Rating

class MovieForm(ModelForm):
    class Meta:
        model = Movie
        fields = ('title', 'description', 'release_date')
        labels = {
            'title': '',
            'description': '',
            'release_date': ''
        }
        widgets = {
            'title': forms.TextInput(attrs={'class':'form-control', 'placeholder': 'Enter Movie Title'}),
            'description': forms.Textarea(attrs={'class':'form-control', 'placeholder': 'Enter Description for Movie'}),
            'release_date': forms.DateInput(attrs={'type': 'date', 'class': 'form-control'}),
        }


# robo-advisor/test/advisor_test.py

import json

from app.robo_advisor import to_usd, to_one_decimal_perc, get_response, write_csv_file_name

def test_to_usd():
    result = to_usd(1500)
    assert result == "$1,500.00"

    result = to_usd(98.78384)
    assert result == "$98.78"

    result = to_usd(2.5)
    assert result == "$2.50"

def test_to_one_decimal_perc():
    result = to_one_decimal_perc(0.83724)
    assert result == "0.8%"

    result = to_one_decimal_perc(124.5)
    assert result == "124.5%"

# Python code for test_get_response obtained from prof-rossetti repository
# https://github.com/s2t2/robo-advisor-screencast/blob/v3-testing/test/advisor_test.py

def test_get_response():
    t = "MSFT"
    response = get_response(t)
    parsed_response = json.loads(response.text)

    assert isinstance(parsed_response, dict)
    assert "Meta Data" in parsed_response.keys()
    assert "Time Series (Daily)" in parsed_response.keys()
    assert parsed_response["Meta Data"]["2. Symbol"] == t

def test_write_csv_file_name():
    result = write_csv_file_name("TSLA")
    assert result == "prices_TSLA.csv"


import numpy as np
import matplotlib.pyplot as plt

N_th = 16
P_th = 40000

audio = np.memmap("./Anexo_38986931.pcm", dtype='h', mode='r')

#tamanho_eco = 8
#audio = np.zeros(100)
#audio[0] = 1

len_audio = len(audio)
vet_saida = np.zeros(len_audio)
vet_th = np.zeros(N_th)

multiplier_N_th = 1
current_N_th = N_th
media_atual = 0
for i in range(len(audio)):
    input_val = audio[i]
    if i >= (N_th*multiplier_N_th):
        multiplier_N_th += 1
        media_atual = 0
    for j in range((N_th*(multiplier_N_th-1)),(N_th*multiplier_N_th)):
        if j <= len(audio)-1:
            valor = audio[j]
            if audio[j] < 0:
                valor = valor*-1
            media_atual += (valor)/N_th
    if media_atual >= P_th:
        vet_saida[i] = input_val
    else:
        vet_saida[i] = 0
#plt.stem(np.arange(0, 100, 1), vet_saida, linefmt='r', markerfmt='ro', basefmt='r')
#plt.xlabel('x')
#plt.ylabel('y')
#plt.title(f'Impulso Unitário')
#plt.show()

plt.plot(audio)
plt.title('output')
plt.show()

plt.plot(vet_saida)
plt.title('output')
plt.show()

fp = np.memmap("./output_analise_potencia.pcm", dtype='h', mode='w+', shape=audio.shape)

for i in range(audio.shape[0]):
    fp[i] = vet_saida[i]


from django.shortcuts import render, redirect

from phones.models import Phone

def index(request):
    return redirect('catalog')

def show_catalog(request):
    template = 'catalog.html'
    if request.GET.get('sort') == 'min_price':
            sort = 'price'
    elif request.GET.get('sort') == 'max_price':
            sort = '-price'
    else:
        sort = '-name'
    phones_object = Phone.objects.all().order_by(sort)
    context = {'phones': phones_object}
    return render(request, template, context)

def show_product(request, slug):
    template = 'product.html'
    phone_obj = Phone.objects.filter(slug=slug)
    context = {'phone': phone_obj[0]}
    return render(request, template, context)


import sys

import numpy as np
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import floyd_warshall

I = np.array(sys.stdin.read().split(), dtype=np.int64)
n, m = I[:2]
abt = I[2:].reshape(-1, 3)

def main():
    G = np.zeros((n, n))

    for a, b, t in abt:
        G[a - 1][b - 1] = t
        G[b - 1][a - 1] = t

    dist = floyd_warshall(csgraph=csr_matrix(G), directed=False)

    ans = np.amin(np.amax(dist, axis=1))
    print(ans)

if __name__ == "__main__":
    main()


import pickle
import os
import numpy as np
import json
from keras.models import load_model
from tqdm import tqdm

def confidence(item):
    return item['confidence']

def countIncorrect(model, lb, image_list, label_list):
    result = model.predict(image_list)

    count = {}
    for clss in lb.classes_:
        count[clss] = 0
    for i in tqdm(range(len(result))):
        proba = result[i]
        proba = list(proba)
        for j in range(len(proba)):
            proba[j] = {
                'label': j,
                'confidence': proba[j]
            }
        proba.sort(key=confidence, reverse=True)
        correctClass = lb.inverse_transform(np.expand_dims(label_list[i], axis=0))[0]
        if lb.classes_[proba[0]['label']] != correctClass:
            count[correctClass] += 1

    return count

def main():
    labelbin_path = './labelbin/labelbin'
    model_path = './model/model'

    lb = pickle.loads(open(labelbin_path, 'rb').read())
    model = load_model(model_path)

    data = np.load('train_data.npy', allow_pickle=True)
    image_list = np.array(list(data[:,0])) # .reshape(-1, IMG_DIMS[0], IMG_DIMS[1], IMG_DIMS[2])
    class_list = np.array(list(data[:,1]))
    label_list = lb.transform(class_list)

    count = countIncorrect(model, lb, image_list, label_list)
    with open('./incorrectCount.json', 'w') as f:
        json.dump(count, f, indent=2)

if __name__ == "__main__":
    main()


import ipaddress
import logging
import random
import pytest
import os
import time
import json

from ptf.mask import Mask
import ptf.packet as scapy

from tests.common.fixtures.ptfhost_utils import remove_ip_addresses  # noqa F401
import ptf.testutils as testutils
from tests.common.helpers.assertions import pytest_require
from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer, LogAnalyzerError

logger = logging.getLogger(__name__)

pytestmark = [
    pytest.mark.topology("t0", "m0", "mx"),
    pytest.mark.disable_loganalyzer,  # Disable automatic loganalyzer, since we use it for the test
]

LOG_ERROR_INSUFFICIENT_RESOURCES = ".*SAI_STATUS_INSUFFICIENT_RESOURCES.*"

ACL_JSON_FILE_SRC = "acl/null_route/acl.json"
ACL_JSON_FILE_DEST = "/host/" + os.path.basename(ACL_JSON_FILE_SRC)

ACL_TABLE_NAME_V4 = "NULL_ROUTE_ACL_TABLE_V4"
ACL_TABLE_NAME_V6 = "NULL_ROUTE_ACL_TABLE_V6"

NULL_ROUTE_HELPER = "null_route_helper"

FORWARD = "FORWARD"
DROP = "DROP"

TEST_DATA = [
    # src_ip, action, expected_result
    ("1.2.3.4", "", FORWARD),  # Should be forwared in default
    ("fc03:1001::1", "", FORWARD),  # Should be forwared in default

    ("1.2.3.4", "block {} 1.2.3.4"
     .format(ACL_TABLE_NAME_V4), DROP),  # Verify block ipv4 without prefix len
    ("1.2.3.4", "unblock {} 1.2.3.4/32"
     .format(ACL_TABLE_NAME_V4), FORWARD),  # Verify unblock ipv4 with prefix len
    ("1.2.3.4", "block {} 1.2.3.4/32"
     .format(ACL_TABLE_NAME_V4), DROP),  # Verify block ipv4 with prefix len
    ("1.2.3.4", "block {} 1.2.3.4/32"
     .format(ACL_TABLE_NAME_V4), DROP),  # Verify double-block dosen't cause issue
    ("1.2.3.4", "unblock {} 1.2.3.4/32"
     .format(ACL_TABLE_NAME_V4), FORWARD),  # Verify unblock ipv4 with prefix len
    ("1.2.3.4", "unblock {} 1.2.3.4/32"
     .format(ACL_TABLE_NAME_V4), FORWARD),  # Verify double-unblock doesn't cause issue

    ("fc03:1000::1", "block {} fc03:1000::1"
     .format(ACL_TABLE_NAME_V6), DROP),  # Verify block ipv6 without prefix len
    ("fc03:1000::1", "unblock {} fc03:1000::1/128".
     format(ACL_TABLE_NAME_V6), FORWARD),  # Verify unblock ipv6 with prefix len
    ("fc03:1000::1", "block {} fc03:1000::1/128"
     .format(ACL_TABLE_NAME_V6), DROP),  # Verify block ipv6 with prefix len
    ("fc03:1000::1", "block {} fc03:1000::1/128"
     .format(ACL_TABLE_NAME_V6), DROP),  # Verify double-block dosen't cause issue
    ("fc03:1000::1", "unblock {} fc03:1000::1/128"
     .format(ACL_TABLE_NAME_V6), FORWARD),  # Verify unblock ipv4 with prefix len
    ("fc03:1000::1", "unblock {} fc03:1000::1/128"
     .format(ACL_TABLE_NAME_V6), FORWARD),  # Verify double-unblock doesn't cause issue
]

@pytest.fixture(scope="module", autouse=True)
def remove_dataacl_table(rand_selected_dut):
    """
    Remove DATAACL to free TCAM resources
    """
    TABLE_NAME = "DATAACL"
    lines = rand_selected_dut.shell(cmd="show acl table {}".format(TABLE_NAME))['stdout_lines']
    data_acl_existing = False
    for line in lines:
        if TABLE_NAME in line:
            data_acl_existing = True
            break
    if not data_acl_existing:
        yield
        return
    # Remove DATAACL
    logger.info("Removing ACL table {}".format(TABLE_NAME))
    rand_selected_dut.shell(cmd="config acl remove table {}".format(TABLE_NAME))
    yield
    # Recover DATAACL
    config_db_json = "/etc/sonic/config_db.json"
    output = rand_selected_dut.shell("sonic-cfggen -j {} --var-json \"ACL_TABLE\"".format(config_db_json))['stdout']
    try:
        entry = json.loads(output)[TABLE_NAME]
        cmd_create_table = "config acl add table {} {} -p {} -s {}"\
            .format(TABLE_NAME, entry['type'], ",".join(entry['ports']), entry['stage'])
        logger.info("Restoring ACL table {}".format(TABLE_NAME))
        rand_selected_dut.shell(cmd_create_table)
    except Exception as e:
        pytest.fail(str(e))

def remove_acl_table(duthost):
    """
    A helper function to remove ACL table for testing
    """
    cmds = [
        "config acl remove table {}".format(ACL_TABLE_NAME_V4),
        "config acl remove table {}".format(ACL_TABLE_NAME_V6)
    ]
    logger.info("Removing ACL table for testing")
    duthost.shell_cmds(cmds=cmds)

def get_neighbor_ports(mg_facts, neighbor_name):
    neighbor_ports = []
    for key, value in list(mg_facts["minigraph_neighbors"].items()):
        if neighbor_name in value["name"]:
            neighbor_ports.append(key)
    return neighbor_ports

@pytest.fixture(scope="module")
def create_acl_table(rand_selected_dut, tbinfo):
    """
    Create two ACL tables on DUT for testing.
    """
    mg_facts = rand_selected_dut.get_extended_minigraph_facts(tbinfo)
    if tbinfo["topo"]["type"] == "mx":
        neighbor_ports = get_neighbor_ports(mg_facts, "M0")
        ports = ",".join(neighbor_ports)
    else:
        # Get the list of LAGs
        ports = ",".join(list(mg_facts["minigraph_portchannels"].keys()))
    cmds = [
        "config acl add table {} L3 -p {}".format(ACL_TABLE_NAME_V4, ports),
        "config acl add table {} L3V6 -p {}".format(ACL_TABLE_NAME_V6, ports)
    ]
    logger.info("Creating ACL table for testing")
    loganalyzer = LogAnalyzer(ansible_host=rand_selected_dut, marker_prefix="null_route_helper")
    loganalyzer.match_regex = [LOG_ERROR_INSUFFICIENT_RESOURCES]

    # Skip test case if ACL table created failed due to insufficient resources
    try:
        with loganalyzer:
            rand_selected_dut.shell_cmds(cmds=cmds)
    except LogAnalyzerError:
        skip_msg = "ACL table creation failed due to insufficient resources, test case will be skipped"
        logger.error(skip_msg)
        remove_acl_table(rand_selected_dut)
        pytest.skip(skip_msg)

    yield

    remove_acl_table(rand_selected_dut)

@pytest.fixture(scope="module")
def apply_pre_defined_rules(rand_selected_dut, create_acl_table):
    """
    This is to apply some ACL rules as production does
    """
    rand_selected_dut.copy(src=ACL_JSON_FILE_SRC, dest=ACL_JSON_FILE_DEST)
    rand_selected_dut.shell("acl-loader update full " + ACL_JSON_FILE_DEST)
    # Wait 5 seconds for ACL rule creation
    time.sleep(5)
    yield
    # Clear ACL rules
    rand_selected_dut.shell('sonic-db-cli CONFIG_DB keys "ACL_RULE|{}*" | xargs sonic-db-cli CONFIG_DB del'
                            .format(ACL_TABLE_NAME_V4))
    rand_selected_dut.shell('sonic-db-cli CONFIG_DB keys "ACL_RULE|{}*" | xargs sonic-db-cli CONFIG_DB del'
                            .format(ACL_TABLE_NAME_V6))

@pytest.fixture(scope="module")
def setup_ptf(rand_selected_dut, ptfhost, tbinfo):
    """
    Add ipv4 and ipv6 address to a port on ptf.
    """
    dst_ports = {}
    vlan_name = ""
    mg_facts = rand_selected_dut.get_extended_minigraph_facts(tbinfo)
    vlans = {}
    for vlan_info in mg_facts["minigraph_vlan_interfaces"]:
        ip_ver = ipaddress.ip_network(vlan_info['addr'], False).version
        if vlan_info["attachto"] not in vlans:
            vlans[vlan_info["attachto"]] = {}
        vlans[vlan_info["attachto"]][ip_ver] = vlan_info

    for key, value in vlans.items():
        if len(value.keys()) == 2:
            vlan_name = key
            for ip_ver, value in value.items():
                dst_ports[ip_ver] = str(ipaddress.ip_address(value['addr']) + 1) + '/' + str(value['prefixlen'])
            break

    pytest_require(vlan_name != "", "Cannot get correct vlan")
    vlan_port = mg_facts['minigraph_vlans'][vlan_name]['members'][0]
    dst_ports['port'] = mg_facts['minigraph_ptf_indices'][vlan_port]

    logger.info("Setting up ptf for testing")
    ptfhost.shell("ifconfig eth{} {}".format(dst_ports['port'], dst_ports[4]))
    ptfhost.shell("ifconfig eth{} inet6 add {}".format(dst_ports['port'], dst_ports[6]))

    yield dst_ports
    ptfhost.shell("ifconfig eth{} 0.0.0.0".format(dst_ports['port']))
    ptfhost.shell("ifconfig eth{} inet6 del {}".format(dst_ports['port'], dst_ports[6]))

def generate_packet(src_ip, dst_ip, dst_mac):
    """
    Build ipv4 and ipv6 packets/expected_packets for testing.
    """
    if ipaddress.ip_network(src_ip.encode().decode(), False).version == 4:
        pkt = testutils.simple_ip_packet(eth_dst=dst_mac, ip_src=src_ip, ip_dst=dst_ip)
        exp_pkt = Mask(pkt)
        exp_pkt.set_do_not_care_scapy(scapy.Ether, "dst")
        exp_pkt.set_do_not_care_scapy(scapy.Ether, "src")
        exp_pkt.set_do_not_care_scapy(scapy.IP, "ttl")
        exp_pkt.set_do_not_care_scapy(scapy.IP, "chksum")
    else:
        pkt = testutils.simple_tcpv6_packet(eth_dst=dst_mac, ipv6_src=src_ip, ipv6_dst=dst_ip)
        exp_pkt = Mask(pkt)
        exp_pkt.set_do_not_care_scapy(scapy.Ether, "dst")
        exp_pkt.set_do_not_care_scapy(scapy.Ether, "src")
        exp_pkt.set_do_not_care_scapy(scapy.IPv6, "hlim")

    return pkt, exp_pkt

def send_and_verify_packet(ptfadapter, pkt, exp_pkt, tx_port, rx_port, expected_action):
    """
    Send packet with ptfadapter and verify if packet is forwarded or dropped as expected.
    """
    ptfadapter.dataplane.flush()
    testutils.send(ptfadapter, pkt=pkt, port_id=tx_port)
    if expected_action == FORWARD:
        testutils.verify_packet(ptfadapter, pkt=exp_pkt, port_id=rx_port, timeout=5)
    else:
        testutils.verify_no_packet(ptfadapter, pkt=exp_pkt, port_id=rx_port, timeout=5)

def test_null_route_helper(rand_selected_dut, tbinfo, ptfadapter, apply_pre_defined_rules, setup_ptf):
    """
    Test case to verify script null_route_helper.
    Some packets are generated as defined in TEST_DATA and sent to DUT,
    and verify if packet is forwarded or dropped as expected.
    """
    ptf_port_info = setup_ptf
    rx_port = ptf_port_info['port']
    router_mac = rand_selected_dut.facts["router_mac"]
    mg_facts = rand_selected_dut.get_extended_minigraph_facts(tbinfo)
    if tbinfo["topo"]["type"] == "mx":
        neighbor_ports = get_neighbor_ports(mg_facts, "M0")
        ptf_interfaces = [mg_facts['minigraph_ptf_indices'][port] for port in neighbor_ports]
    else:
        portchannel_members = []
        for _, v in list(mg_facts["minigraph_portchannels"].items()):
            portchannel_members += v['members']

        ptf_interfaces = []
        for port in portchannel_members:
            ptf_interfaces.append(mg_facts['minigraph_ptf_indices'][port])

    # Run testing as defined in TEST_DATA
    for test_item in TEST_DATA:
        src_ip = test_item[0]
        action = test_item[1]
        expected_result = test_item[2]
        ip_ver = ipaddress.ip_network(src_ip.encode().decode(), False).version
        logger.info("Testing with src_ip = {} action = {} expected_result = {}"
                    .format(src_ip, action, expected_result))
        pkt, exp_pkt = generate_packet(src_ip, ptf_port_info[ip_ver].split("/")[0], router_mac)
        if action != "":
            rand_selected_dut.shell(NULL_ROUTE_HELPER + " " + action)
            time.sleep(1)

        send_and_verify_packet(ptfadapter, pkt, exp_pkt, random.choice(ptf_interfaces), rx_port, expected_result)


from django.urls import path
from users.api.views import (
    UsersListAPIView,
    UsersCaptureAPIView,
    UsersCaptureDeleteAPIView,
    UserCapturesRetrieveAPIView,
    CapturesListAPIView,
)

app_name = "users-api"

urlpatterns = [
    path(
        '', UsersListAPIView.as_view(),
        name='users'
    ),
    path(
        'capture', UsersCaptureAPIView.as_view(),
        name='capture'
    ),
    path(
        'captures', CapturesListAPIView.as_view(),
        name='captures'
    ),
    path(
        'captures/<str:matrice>', UserCapturesRetrieveAPIView.as_view(),
        name='user-captures'
    ),
    path(
        'capture/<str:matrice>/<str:image_key>/delete',
        UsersCaptureDeleteAPIView.as_view(),
        name='delete-capture'
    ),
]


from django.core.management import call_command
from django.db import migrations

def load_initial_data(apps, schema_editor):
    initial_data_fixtures = [
        "demo/demo/fixture/knobs.json"
    ]
    for fixture in initial_data_fixtures:
        call_command("loaddata", fixture, app_label="demo")

def unload_initial_data(apps, schema_editor):
    model_names = [
        "KnobCatalog"
    ]
    for model_name in model_names:
        model = apps.get_model("demo", model_name)
        model.objects.all().delete()

class Migration(migrations.Migration):

    dependencies = [
        ('demo', '0001_initial'),
    ]

    operations = [
        migrations.RunPython(load_initial_data, unload_initial_data)
    ]


from flask import render_template, request, redirect, flash, url_for, session
from classes import *
from festival_is import app, login_manager
from forms import *
from werkzeug.security import generate_password_hash, check_password_hash
from flask_login import login_required, logout_user, current_user, login_user
from datetime import timedelta
from werkzeug.datastructures import MultiDict
import json, boto3
import os

@app.before_request
def make_session_permanent():
    session.permanent = True
    app.permanent_session_lifetime = timedelta(minutes=15)

@app.errorhandler(500)
def undef_error(e):
    flash("Undefined error is raised on client side. Please, contact admin.", "warning")
    flash(f"Debug {e}.", "warning")
    return redirect("/")

@app.errorhandler(404)
def undef_error(e):
    flash("Error 404 raised", "warning")
    flash(f"Debug {e}.", "warning")
    return redirect("/")

@app.errorhandler(Exception)
def undef_error(e):
    flash("Exception is raised", "warning")
    flash(f"Debug {e}.", "warning")
    return redirect("/")

@login_manager.user_loader
def user_loader(user_id):
    """Given *user_id*, return the associated User object.
    :param unicode user_id: user_id (email) user to retrieve
    """
    return User.query.get(user_id)

@app.route("/", methods=["GET", "POST"])
def home():

    recommendations = None
    styles_set, data = Festival.get_festivals_styles()
    list_of_dicts = [row for row in data]
    if current_user.is_authenticated:
        recommendations = list(current_user.get_recomendations())
        recommendations_count = len(recommendations)
        return render_template(
            "festivals.html",
            user_columns=current_user,
            fests=list_of_dicts,
            recommendations=recommendations,
            styles=styles_set,
            recommendations_count=recommendations_count,
        )
    return render_template("festivals.html", fests=list_of_dicts,styles=styles_set)

@app.route("/about")
def about():
    if current_user.is_authenticated:
        return render_template("about.html", user_columns=current_user, title="About")
    return render_template("about.html", title="About")

@app.route("/user")
def user():
    return render_template("festivals.html", title="User")

@app.route("/organizer")
def organizer():
    return render_template("festivals.html")

@app.route("/register", methods=["GET", "POST"])
def register():
    form = RegistrationForm()
    if form.validate_on_submit():
        perms = int(request.form["options"])
        email = form.email.data
        existing_user = User.find_by_email(email)
        if existing_user is None:
            new_user, msg, status = BaseUser.register(form, perms)
            print(msg, status)
            flash(msg, status)
            if new_user:
                return login(user=new_user)
        else:
            flash(f"Email {email} is already registered!", "danger")
            return render_template("register.html", title="Registration", form=form)
    return render_template("register.html", title="Registration", form=form)

@app.route("/login", methods=["GET", "POST"])
def login(user=None):
    form = LoginForm()
    if form.validate_on_submit():
        user = User.find_by_email(form.email.data)
        if user is not None:
            if not user.active:
                flash("Your user is removed by admin", "warning")
                return redirect("/")
            remember = True if request.form.get("remember") else False
            if user.check_passwd(form.password.data):
                user.is_authenticated = True
                if login_user(user, remember=remember):
                    user.is_active = True
                flash("You have been logged in!", "success")
                return redirect(url_for("protected"))
                # return redirect(url_for('home'))
            else:
                flash("Log in failed! Check email and password", "danger")
        else:
            flash("Account for this email doesn't exist", "warning")

    return render_template("login.html", title="Login", form=form)

# Listen for GET requests to yourdomain.com/account/
@login_required
@app.route("/account/", methods=["GET"])
def account():
    user = User.query.filter_by(user_id=current_user.user_id).first()
    return render_template("account.html", user_columns=user)

# Listen for POST requests to yourdomain.com/submit_form/
@login_required
@app.route("/submit-form/", methods=["POST"])
def submit_form():
    msg, status = current_user.change_account(request.form)
    print(msg, status)
    flash(msg, status)
    return redirect("/account")

# Listen for GET requests to yourdomain.com/sign_s3/
@app.route("/sign-s3/<folder>/<_id>/")
@login_required
def sign_s3(folder, _id):
    # Load necessary information into the application
    S3_BUCKET = os.environ.get("S3_BUCKET")

    # Load required data from the request
    folder_name = f"{folder}/{_id}"
    file_name = request.args.get("file-name")
    file_type = request.args.get("file-type")

    # Initialise the S3 client
    s3 = boto3.client("s3")
    # Generate and return the presigned URL
    presigned_post = s3.generate_presigned_post(
        Bucket=S3_BUCKET,
        Key=f"{folder_name}/{file_name}",
        Fields={"acl": "public-read", "Content-Type": file_type},
        Conditions=[{"acl": "public-read"}, {"Content-Type": file_type}],
        ExpiresIn=3600,
    )

    # Return the data to the client
    return json.dumps(
        {
            "data": presigned_post,
            "url": f"https://{S3_BUCKET}.s3.amazonaws.com/{folder_name}/{file_name}",
        }
    )

@app.route("/protected")
@login_required
def protected():
    return redirect("/")

@login_required
@app.route("/logout")
def logout():
    flash(f"User {current_user.user_id} is logged out", "success")
    logout_user()
    return redirect("/")

@app.route("/festival/<fest_id>", methods=["GET", "POST"])
def festival_page(fest_id):
    fest = Festival.get_festival(fest_id)
    anonim = current_user.is_anonymous
    perfs = Performance.query.filter( and_( Performance.fest_id == fest_id, Performance.canceled == False )).all()
    tags = ""
    for perf in perfs[0:5:]:
        tags += " ".join(perf.band.tags.split(";"))

    form = (
        TicketForm()
        if anonim
        else TicketForm(
            formdata=MultiDict(
                {
                    "user_name": current_user.name,
                    "user_surname": current_user.surname,
                }  # For autofill if user is logged in
            )
        )
    )

    if form.is_submitted():
        try:
            if anonim:
                BaseUser.reserve_ticket(form, fest_id)
            else:
                current_user.reserve_ticket(fest_id)
        except ValueError as e:
            flash(f"{e}", "warning")
            return redirect("/")
        flash("Ticket is successfully reserved", "success")
        return redirect("/")
    return render_template(
        "festival_page.html",
        fest=fest,
        user_columns=current_user,
        form=form,
        anonym=current_user.is_anonymous,
        perfs=perfs,
        tags=tags
    )

@login_required
@app.route("/my_tickets", methods=["GET", "POST"])
def my_tickets():
    tickets = current_user.get_tickets()
    if not tickets:
        return redirect("/")
    return render_template(
        "ticket_page.html",
        actual_tickets=tickets[0],
        outdated_tickets=tickets[1],
        user_columns=current_user,
    )

@login_required
@app.route("/my_tickets/<ticket_id>/cancel")
def cancel_ticket(ticket_id):
    current_user.cancel_ticket(ticket_id)
    return redirect("/my_tickets")

@login_required
@app.route("/my_festivals")
def my_festivals():
    planed_fests, outdated_fests = current_user.get_festivals()
    return render_template(
        "my_festivals.html",
        actual_fests=planed_fests,
        outdated_fests=outdated_fests,
        user_columns=current_user,
        source="/my_festivals",
    )
@login_required
@app.route("/<source>/<fest_id>/update_fest", methods=["POST"])
def update_fest(fest_id, source):
    msg, status = current_user.update_fest(request.form, fest_id)
    flash(msg, status)
    return redirect(f"/my_festivals/{fest_id}/edit")

@login_required
@app.route("/<source>/<fest_id>/edit", methods=["GET"])
def edit_festival(fest_id, source):
    seller_form = RoleForm()

    fest = Festival.query.filter_by(fest_id=fest_id).first()

    perfs = current_user.get_perf(fest_id=fest_id)
    sellers = current_user.get_sellers(fest_id)
    return render_template(
        "edit_festival.html",
        fest=fest,
        seller_form=seller_form,
        perfs=perfs,
        user_columns=current_user,
        sellers=sellers,
        org=None,
    )

@login_required
@app.route("/my_festivals/add", methods=["GET", "POST"])
def add_festival():
    seller_form = RoleForm()
    if request.method == "POST":
        form = request.form
        msg, status, fest = current_user.add_fest(form)
        flash(msg, status)
        if fest is None:
            return redirect("/my_festivals/add")

        if request.form["fest_logo"] == "https://festival-static.s3-eu-west-1.amazonaws.com/def_fest_logo.png":
            pass
        else:
            print(request.form["fest_logo"], flush=True)
            S3_BUCKET = os.environ.get("S3_BUCKET")

            s3 = boto3.resource("s3")
            copy_source = {
                "Bucket": S3_BUCKET,
                "Key": form["fest_logo"].split(".com/")[-1],
            }
            bucket = s3.Bucket(S3_BUCKET)
            bucket.copy(copy_source, f"fest/{fest.fest_id}/{fest.fest_id}.png")

            fest.fest_logo = f'{form["fest_logo"].split(".com/")[0]}.com/fest/{fest.fest_id}/{fest.fest_id}.png'

        db.session.commit()
        return redirect(f"/my_festivals/{fest.fest_id}/edit")
    return render_template(
        "edit_festival.html",
        fest=None,
        seller_form=seller_form,
        org=current_user,
        perfs=None,
        sellers=None,
        user_columns=current_user,
    )

@login_required
@app.route("/<source>/<fest_id>/cancel_festival")
def cancel_fest(fest_id, source):
    msg, status = current_user.cancel_fest(fest_id)
    flash(msg, status)
    return redirect(f"/{source}")

@login_required
@app.route("/<source>/<fest_id>/add_perf", methods=["POST"])
def fest_add_perf(fest_id, source):
    form = request.form
    msg, status = current_user.fest_add_perf(form, fest_id)
    flash(msg, status)
    return redirect(f"/my_festivals/{fest_id}/edit")

@login_required
@app.route("/<source>/<fest_id>/del_perf/<perf_id>")
def fest_del_perf(fest_id, perf_id, source):
    current_user.fest_del_perf(perf_id)
    flash(f"Performance {perf_id} canceled", "success")
    return redirect(f"/my_festivals/{fest_id}/edit")

@login_required
@app.route("/<source>/<fest_id>/add_seller", methods=["POST"])
def fest_add_seller(fest_id, source):
    form = request.form
    msg, status = current_user.fest_add_seller(form, fest_id)
    flash(msg, status)
    return redirect(f"/my_festivals/{fest_id}/edit")

@login_required
@app.route("/<source>/<fest_id>/del_seller/<seller_id>")
def fest_del_seller(fest_id, seller_id, source):
    current_user.fest_del_seller(fest_id, seller_id)
    flash(f"Seller {seller_id} removed from festival {fest_id}", "success")
    return redirect(f"/my_festivals/{fest_id}/edit")

@login_required
@app.route("/<source>/<fest_id>/create_seller", methods=["POST"])
def create_seller(fest_id, source):
    form = request.form
    msg, status = current_user.create_seller(form, fest_id)
    flash(msg, status)
    return redirect(f"/my_festivals/{fest_id}/edit")

@login_required
@app.route("/<source>/<fest_id>/manage_tickets")
def manage_tickets(fest_id, source):
    tickets, fest, actuality = current_user.get_sellers_tickets(fest_id)
    return render_template(
        "manage_tickets.html",
        tickets=tickets,
        user_columns=current_user,
        actuality=actuality,
        fest=fest,
    )

@login_required
@app.route(
    "/<source>/<fest_id>/manage_tickets/<ticket_id>/<action>",
    methods=["GET", "POST"],
)
def manage_ticket_seller(fest_id, ticket_id, action, source):
    reason = request.form["reason"]
    current_user.manage_ticket_seller(ticket_id, action, reason)
    return redirect(f"/{source}/{fest_id}/manage_tickets")

@login_required
@app.route("/manage_sellers")
def manage_sellers():
    form = RegistrationForm()
    if form.validate_on_submit():
        existing_user = User.find_by_email(form.email.data)
        if existing_user is None:
            current_user.add_seller(form)
            flash(f"Seller {form.username.data} successfully added!", "success")
        else:
            flash(f"Email {email} is already registered!", "danger")

    return render_template(
        "sellers_page.html", sellers=current_user.get_sellers(), form=form
    )

@login_required
@app.route("/manage_festivals")
def manage_festivals():
    planed_fests, outdated_fests = current_user.manage_festivals()
    return render_template(
        "my_festivals.html",
        actual_fests=planed_fests,
        outdated_fests=outdated_fests,
        user_columns=current_user,
        source="/manage_festivals",
    )

@login_required
@app.route("/manage_users")
def manage_users():
    users = current_user.get_all_users()
    admin_form = RoleForm()
    return render_template(
        "users_page.html", users=users, admin_form=admin_form, user_columns=current_user
    )

@login_required
@app.route("/manage_users/add_admin", methods=["POST"])
def add_admin():
    response, status = current_user.add_admin(request.form)
    flash(response, status)
    return redirect("/manage_users")

@login_required
@app.route("/manage_users/<user_id>/remove_role")
def remove_role(user_id):
    msg, status = current_user.remove_role(user_id)
    flash(msg, status)
    return redirect("/manage_users")

@login_required
@app.route("/manage_users/<user_id>/remove_user")
def remove_user(user_id):
    response, status = current_user.remove_user(user_id)
    flash(response, status)
    return redirect("/manage_users")

@login_required
@app.route("/manage_bands", methods=["GET", "POST"])
def manage_bands():
    form = BandForm()
    if form.validate_on_submit():
        current_user.add_band(form)
    bands = current_user.get_bands()
    return render_template(
        "bands_page.html", bands=bands, form=form, user_columns=current_user
    )

@login_required
@app.route("/manage_bands/add", methods=["POST"])
def add_band():
    msg, status, band = current_user.add_band(request.form)
    flash(msg, status)

    if request.form["band-logo"] == "https://festival-static.s3-eu-west-1.amazonaws.com/defaut_band_logo.png":
        pass
    else:
        S3_BUCKET = os.environ.get("S3_BUCKET")
        s3 = boto3.resource("s3")
        copy_source = {
            "Bucket": S3_BUCKET,
            "Key": request.form["band-logo"].split(".com/")[-1],
        }
        bucket = s3.Bucket(S3_BUCKET)
        bucket.copy(copy_source, f"band/{band.band_id}/{band.name}.png")

        band.logo = f'{request.form["band-logo"].split(".com/")[0]}.com/band/{band.band_id}/{band.name}.png'
    db.session.commit()

    return redirect("/manage_bands")

@login_required
@app.route("/manage_bands/<band_id>/delete")
def delete_band(band_id):
    current_user.delete_band(band_id)
    return redirect("/manage_bands")

@login_required
@app.route("/manage_stages")
def manage_stages():
    stages = current_user.get_all_stages()
    return render_template("stages_page.html", stages=stages, user_columns=current_user)

@login_required
@app.route("/manage_stages/add_stage", methods=["POST"])
def add_stage():
    form = request.form
    msg, status = current_user.add_stage(form)
    return redirect("/manage_stages")

@login_required
@app.route("/manage_stages/<stage_id>/delete", methods=["POST"])
def delete_stage(stage_id):
    msg, status = current_user.remove_stage(stage_id)
    flash(msg, status)
    return redirect("/manage_stages")


#!/usr/bin/env python3
#
# (c) 2019 Yoichi Tanibayashi
#
"""
IrSendClient
"""
__author__ = 'Yoichi Tanibayashi'
__date__   = '2019'

from TcpCmdClient import TcpCmdClient
from IrSendServer import IrSendServer
import json

from MyLogger import get_logger

class IrSendClient(TcpCmdClient):
    DEF_SVR_HOST = 'localhost'
    DEF_SVR_PORT = IrSendServer.DEF_PORT

    def __init__(self, host=DEF_SVR_HOST, port=DEF_SVR_PORT, debug=False):
        self._debug = debug
        self._logger = get_logger(__class__.__name__, self._debug)
        self._logger.debug('host=%s, port=%s', host, port)

        self._svr_host = host
        self._svr_port = port
        super().__init__(self._svr_host, self._svr_port, timeout=0.5,
                         debug=self._debug)

    def send_recv(self, arg_str):
        self._logger.debug

        if arg_str == '':
            arg_str = '@list'

        rep_str = super().send_recv(arg_str)

        try:
            rep_json = json.loads(rep_str)
        except json.decoder.JSONDecodeError:
            self._logger.warning('invalid json format')
            return rep_str

        return rep_json

class App:
    def __init__(self, arg, host, port, debug=False):
        self._debug = debug
        self._logger = get_logger(__class__.__name__, self._debug)
        self._logger.debug('arg=%s, host=%s, port=%d', arg, host, port)

        self._arg_str = ' '.join(list(arg))
        self._logger.debug('_arg_str=%a', self._arg_str)

        self._cl = IrSendClient(host, port, debug=self._debug)

    def main(self):
        self._logger.debug('')

        ret = self._cl.send_recv(self._arg_str)
        self._logger.debug('ret=%s', ret)

        if type(ret) != dict:
            self._logger.warning('invalid reply foramt')
            print(ret)
            return

        if 'rc' not in ret:
            self._logger.warning('invalid reply foramt')
            ret_str = json.dumps(ret, indent=2, ensure_ascii=False)
            print(ret_str)
            return

        rc = ret['rc']

        if 'data' not in ret:
            # only rc
            print(rc)
            return

        data = ret['data']

        if type(data) == str:
            # error message
            print(data)
            return

        if type(data) == list:
            # device list
            for d in data:
                print(d)
            return

        # button list
        if 'macro' in data:
            print('* macro')
            for m in data['macro']:
                print('%s: %s' % (m, data['macro'][m]))

        if 'buttons' in data:
            print('* button')
            for b in data['buttons']:
                print('%s: %s' % (b, data['buttons'][b]))

    def end(self):
        self._logger.debug('')
        self._cl.end()

import click
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

@click.command(context_settings=CONTEXT_SETTINGS,
               help='IrSendClient')
@click.argument('arg', type=str, nargs=-1)
@click.option('--svrhost', '-s', 'svrhost', type=str,
              default=IrSendClient.DEF_SVR_HOST,
              help='server hostname')
@click.option('--svrport', '--port', '-p', 'svrport', type=int,
              default=IrSendClient.DEF_SVR_PORT,
              help='server port nubmer')
@click.option('--debug', '-d', 'debug', is_flag=True, default=False,
              help='debug flag')
def main(arg, svrhost, svrport, debug):
    logger = get_logger(__name__, debug)
    logger.debug('arg=%s, svrhost=%s, svrport=%d', arg, svrhost, svrport)

    app = App(arg, svrhost, svrport, debug=debug)
    try:
        app.main()
    finally:
        logger.debug('finally')
        app.end()

if __name__ == '__main__':
    main()


from flask import render_template, redirect

# import z pliku __init__.py
from . import app, db

# import z pliku models z bieżącego katalogu
from .models import Client

# import z pliku forms z bieżącego katalogu
from .forms import ClientForm

@app.route('/')
def hello():
    # odczytanie zawartości tabeli clients
    clients = Client.query.all()

    # budujemy zmienne przekazywane do szablonu
    ctx = {'clients': clients, 'additional_param': 'Hello :-)'}

    # szablon zostanie uzupełniony zawartością zmiennych i przesłany do przeglądarki
    return render_template('clients.html', **ctx)

# przykład na obsługę metod GET i POST
@app.route('/add', methods=['GET', 'POST'])
def add():
    form = ClientForm()

    # wartość zwracana przez metodę validate_on_submit() jest True, gdy mamy do czynienia z metodą 'POST'
    if form.validate_on_submit():
        # kopiowanie przesłanych danych aby usunąć z nich pole 'csrf_token'
        validated_data = form.data.copy()
        validated_data.pop('csrf_token')

        # utworzenie obiektu typu Client (zdefiniowany w models.py)
        # dzięki zgodności nazw pól łatwiej jest zainicjalizować obiekt
        new_client = Client(**validated_data)

        # dodanie rekordu do bazy
        db.session.add(new_client)
        db.session.commit()

        # przeglądarka zostanie przekierowana na stronę główną
        return redirect('/')

    # dla metody GET protokołu HTTP zostanie zwrócony formularz z szablonu client_form.html
    return render_template('client_form.html', form=ClientForm())

# Dodatkowo: catch_all
@app.route('/', defaults={'path': ''})
@app.route('/<path:path>')
def catch_all(path):
    return 'Próba otwarcia nietypowej ścieżki: %s' % path


import datetime
import itertools

from absl.testing import absltest
from dm_c19_modelling.evaluation import constants
from dm_c19_modelling.evaluation import dataset_factory
from dm_c19_modelling.evaluation import evaluation
from dm_c19_modelling.evaluation import forecast_indexing
import numpy as np
import pandas as pd

_TEST_DATASET = "test_dataset"

def _get_forecasts(dates, sites):
  return pd.DataFrame({
      constants.DATE: np.repeat(dates, len(sites)),
      constants.SITE_ID: np.tile(sites, len(dates)),
      constants.PREDICTION: np.random.rand(len(dates) * len(sites)) * 10,
  })

def _get_entry(last_observation_date="2020-05-07",
               dataset_index_key="dset_index_1", cadence=1):
  return forecast_indexing.build_entry(
      forecast_id="",
      file_location="",
      dataset_name=_TEST_DATASET,
      creation_timestamp="",
      dataset_index_key=dataset_index_key,
      dataset_location="",
      last_observation_date=last_observation_date,
      cadence=cadence,
      extra_info={})

def _get_dataset(eval_dates, sites, target="new_confirmed", cadence=1):
  training_datetime = (
      datetime.datetime.strptime(eval_dates[0], "%Y-%m-%d") -
      datetime.timedelta(days=1))
  return dataset_factory.Dataset(
      training_targets=np.random.randint(0, 10, (1, len(sites), 1)),
      training_features=[],
      evaluation_targets=np.random.randint(0, 10,
                                           (len(eval_dates), len(sites), 1)),
      sum_past_targets=np.random.randint(0, 10, (len(sites), 1)),
      target_names=[target],
      feature_names=[],
      training_dates=[
          datetime.datetime.strftime(training_datetime, "%Y-%m-%d")
      ],
      evaluation_dates=eval_dates,
      sites=np.array(sites),
      dataset_index_key="test_dset_index_1",
      cadence=cadence)

class EvaluationTest(absltest.TestCase):

  def setUp(self):
    super().setUp()
    self._default_dates = ["2020-05-07", "2020-05-08", "2020-05-09"]
    self._default_sites = ["site_1", "site_2"]

  def test_comparable_forecasts_subset_sites(self):
    """Check that validation fails for forecasts with different sites."""
    forecasts_1 = _get_forecasts(self._default_dates, self._default_sites)
    forecasts_2 = _get_forecasts(self._default_dates, ["site_1", "site_3"])
    all_forecast_arrays = evaluation._convert_forecasts_to_arrays(
        [forecasts_1, forecasts_2])
    _, sites_to_eval, sites_to_drop, updated_data_arrays = (
        evaluation._get_forecast_spec_and_comparable_predictions(
            all_forecast_arrays))
    np.testing.assert_array_equal(sites_to_eval, ["site_1"])
    np.testing.assert_array_equal(sites_to_drop, ["site_2", "site_3"])
    np.testing.assert_array_equal(
        updated_data_arrays, np.array(
            [arr.data_array for arr in all_forecast_arrays])[:, :, 0:1, :])
    np.testing.assert_array_equal(updated_data_arrays[0].shape,
                                  updated_data_arrays[1].shape)

  def test_incomparable_forecasts_subset_dates(self):
    """Check that validation fails for forecasts with different start dates."""
    forecasts_1 = _get_forecasts(self._default_dates, self._default_sites)
    forecasts_2 = _get_forecasts(["2020-05-06", "2020-05-07"],
                                 self._default_sites)
    all_forecast_arrays = evaluation._convert_forecasts_to_arrays(
        [forecasts_1, forecasts_2])
    dates_to_eval, _, _, updated_data_arrays = (
        evaluation._get_forecast_spec_and_comparable_predictions(
            all_forecast_arrays))
    np.testing.assert_array_equal(dates_to_eval, ["2020-05-07"])
    np.testing.assert_array_equal(
        updated_data_arrays[0], all_forecast_arrays[0].data_array[0:1])
    np.testing.assert_array_equal(
        updated_data_arrays[1], all_forecast_arrays[1].data_array[1:2])
    np.testing.assert_array_equal(updated_data_arrays[0].shape,
                                  updated_data_arrays[1].shape)

  def test_valid_different_forecast_horizons(self):
    """Check that validation passes for forecasts with different horizons."""
    forecasts_1 = _get_forecasts(self._default_dates, self._default_sites)
    forecasts_2 = _get_forecasts(["2020-05-07", "2020-05-08"],
                                 self._default_sites)
    all_forecast_arrays = evaluation._convert_forecasts_to_arrays(
        [forecasts_1, forecasts_2])
    evaluation._get_forecast_spec_and_comparable_predictions(
        all_forecast_arrays)

  def test_badly_formatted_forecasts(self):
    """Checks that forecasts with unexpected format fail evaluation."""
    forecasts = _get_forecasts(self._default_dates, self._default_sites)
    forecasts["extra_column"] = ""
    all_forecast_arrays = evaluation._convert_forecasts_to_arrays([forecasts])
    with self.assertRaisesRegex(AssertionError,
                                "Unexpected columns in forecasts:*"):
      evaluation._get_forecast_spec_and_comparable_predictions(
          all_forecast_arrays)

  def test_incomparable_last_observation_dates(self):
    """Checks that validation fails for different last_observation_dates."""
    entry_1 = _get_entry(last_observation_date="2020-05-06")
    entry_2 = _get_entry(last_observation_date="2020-05-07")
    with self.assertRaisesRegex(
        ValueError,
        "Models can only be compared if they have the same "
        "last_observation_date. *"):
      evaluation._get_last_observation_date_and_validate_comparable(
          [entry_1, entry_2])

  def test_incomparable_forecast_cadences(self):
    """Checks that validation fails for forecasts with different cadences."""
    entry_1 = _get_entry(cadence=1)
    entry_2 = _get_entry(cadence=7)
    with self.assertRaisesRegex(
        ValueError,
        "Models can only be compared if they have the same forecast cadence *"):
      evaluation._get_last_observation_date_and_validate_comparable(
          [entry_1, entry_2])

  def test_incomparable_forecast_sources(self):
    """Checks validation fails for forecasts trained on different datasets."""
    entry_1 = _get_entry(dataset_index_key="dset_index_1")
    entry_2 = _get_entry(dataset_index_key="dset_index_2")
    with self.assertRaisesRegex(
        ValueError,
        "Models can only be compared if they were trained using the same "
        "dataset.*"):
      evaluation._get_last_observation_date_and_validate_comparable(
          [entry_1, entry_2])

  def test_incomparable_eval_dset_missing_sites(self):
    """Checks that validation fails when the dataset is missing sites."""
    dataset = _get_dataset(self._default_dates, ["site_1", "site_3"])
    with self.assertRaisesRegex(
        ValueError, "Not all of the sites in the forecasts are present in the "
        "evaluation dataset*"):
      evaluation._validate_eval_dataset_comparable(dataset, self._default_dates,
                                                   self._default_sites)

  def test_incomparable_eval_dset_missing_dates(self):
    """Checks that validation fails when the dataset is missing dates."""
    dataset = _get_dataset(["2020-05-08", "2002-05-09"], self._default_sites)
    with self.assertRaisesWithLiteralMatch(
        AssertionError, "Dates in forecasts differ from dates in evaluation "
        "dataset"):
      evaluation._validate_eval_dataset_comparable(dataset, self._default_dates,
                                                   self._default_sites)

  def test_calculate_metrics(self):
    """Checks that metric calculations yield the correct values."""
    predictions = np.array([[[1.], [2.], [3.]], [[1.], [2.], [2.]]])
    ground_truth = np.array([[[1.], [1.], [1.]], [[1.], [1.], [2.]]])
    metrics_df = evaluation._calculate_metrics("", predictions, ground_truth,
                                               "new_confirmed")
    self.assertAlmostEqual(
        1.,
        metrics_df.query("metric_name=='rmse'").metric_value.values[0])

  def test_convert_forecasts_to_arrays(self):
    """Checks all data is preserved when converting DataFrames to arrays."""
    forecasts_1 = _get_forecasts(self._default_dates, self._default_sites)
    forecasts_2 = _get_forecasts(self._default_dates, self._default_sites[::-1])
    forecast_dfs = [forecasts_1, forecasts_2]
    forecast_arrays = evaluation._convert_forecasts_to_arrays(forecast_dfs)
    for array, df in zip(forecast_arrays, forecast_dfs):
      np.testing.assert_array_equal(array.dates_array, self._default_dates)
      np.testing.assert_array_equal(array.sites_array, self._default_sites)
      np.testing.assert_array_equal(array.features_array,
                                    [constants.PREDICTION])
      for date, site in itertools.product(self._default_dates,
                                          self._default_sites):
        date_idx = list(array.dates_array).index(date)
        site_idx = list(array.sites_array).index(site)
        array_entry = array.data_array[date_idx][site_idx][0]
        df_entry = df.query(
            f"site_id=='{site}' and date=='{date}'").prediction.values[0]
        self.assertAlmostEqual(df_entry, array_entry)

if __name__ == "__main__":
  absltest.main()


import time

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait

ed = webdriver.Edge()
ed.get('https://www.baidu.com')
ed.maximize_window()
ed.implicitly_wait(10)

span=WebDriverWait(ed,5,1).until(lambda e1:ed.find_element(By.XPATH,'//*[@id="form"]/span[1]/span[1]'),message="没找到等待元素")
span.click()
inputmange=ed.find_element(By.XPATH,'//*[@id="form"]/div/div[2]/div[2]/input')
inputmange.send_keys(r'C:\Users\39884\Desktop\微信图片_20220728134709.jpg')
time.sleep(5)
ed.quit()


# encoding: utf-8

import sys, inspect

from workflow import Workflow, notify

from config import create_config
import request.device as device
from remo_client import RemoClient
import utils

class RemoCommand:
    def show_temperature_and_humidity(self, wf):
        req = device.ListRequest(create_config())
        dic = RemoClient.call(req)
        newest_events = dic[0]['newest_events']
        temp = newest_events['te']['val']
        humo = newest_events['hu']['val']
        text = u"室温: %0.2f度\\n湿度: %0.2f％" % (temp, humo)
        notify.notify(u'室温と湿度', text)
        wf.add_item(title = text, icon='icon.png', arg = text, valid = True)
        wf.send_feedback()

if __name__ == '__main__':
    func_dict = utils.create_instance_func_dict(RemoCommand)
    key = sys.argv[1]
    wf = Workflow()
    sys.exit(wf.run(func_dict[key]))


from django.shortcuts import render, get_list_or_404, get_object_or_404
from .models import Song

def index(request):
    songs_datas = Song.objects.order_by('-song_name').filter(is_posted=True)

    songs = {
        'songs': songs_datas
    }
    return render(request, 'index.html', songs)

# MÚSICAS
def song(request, song_id):
    song = get_object_or_404(Song, pk=song_id)

    song_to_display = {
        'song': song
    }

    return render(request, 'song.html', song_to_display)

# VOZES
def voice(request, song_id):
    song = get_object_or_404(Song, pk=song_id)

    song_to_display = {
        'song': song
    }

    return render(request, 'voice.html', song_to_display)

# INSTRUMENTAL
def instrumental(request, song_id):
    song = get_object_or_404(Song, pk=song_id)

    song_to_display = {
        'song': song
    }

    return render(request, 'instrumental.html', song_to_display)

# PESQUISA
def search(request):
    songs_datas_list = Song.objects.order_by('-song_name').filter(is_posted=True)

    if 'search' in request.GET:
        search_name = request.GET['search']
        if search:
            songs_datas_list = songs_datas_list.filter(song_name__icontains=search_name)

    data = {
        'songs': songs_datas_list
    }

    return render(request, 'search.html', data)


import urllib.request
import chardet
import os

def main():
    i = 0
    path = os.path.realpath(os.curdir)#获取当前目录的绝对路径
    path = os.path.join(path, "urls.txt")#加上文件名

    with open("urls.txt", "r") as f:
        # 读取待访问的网址
        # 由于urls.txt每一行一个URL
        # 所以按换行符'\n'分割
        urls = f.read().splitlines()

    for each_url in urls:
        response = urllib.request.urlopen(each_url)
        html = response.read()

        # 识别网页编码
        encode = chardet.detect(html)['encoding']
        if encode == 'GB2312':
            encode = 'GBK'

        i += 1
        filename = "url_%d.txt" % i

        with open(filename, "w", encoding=encode) as each_file:
            each_file.write(html.decode(encode, "ignore"))

if __name__ == "__main__":
    main()

code_python/python study/python basic/53/web save.py


"""
Where everything comes together
"""
import torch

from discriminators.isvoice_dtor import get_isvoice_discriminator
from discriminators.content_dtor import get_content_discriminator
from discriminators.identity_dtor import get_identity_discriminator

from transformer.mem_transformer import MemTransformer

from embedding import embeddings

import warnings

def get_transformer(config):
    """
    Returns a neural network which takes a stylevector (N x S) tensor
    and audiosample (N x 1 x T x M) and returns another (N x 1 x T x M) tensor
    representing the output mel. (Where audiosample i has been transformed to
    have style stylevector[i])

    Neural net should extend pytorch.module so it can be easily checkpointed?

    where S is the dimensionality of style vector and M is the number of
    mel-spectrogram channels
    """
    return MemTransformer(config)

def get_embedder_and_size(mel_size, path=None, cuda=False,
                          embedding_size=512):
    """
    Returns a embedding model captures the sytle of a speakers voice

    returns (Batch Size, style_size)

    where network which takes a transformation of a speakers utterances (Batch Size x 1 x Frames x Features)
    """

    if path != None:
        loaded_style_size, num_classes, num_features, num_frames = embeddings.parse_params(path)
        if mel_size != num_features:
            raise RuntimeError("Loaded embedder yields mel size of " + str(num_features)
                               + " but mel size of " + str(mel_size) + " requested")
        if embedding_size != loaded_style_size:
            raise RuntimeError("Loaded embedder yields style size of " + str(loaded_style_size)
                               + " but style size of " + str(embedding_size) + " requested")

        embedder = embeddings.load_embedder(
            checkpoint_path=path,
            embedding_size=embedding_size,
            num_classes=num_classes,
            num_features=num_features,
            frame_dim=num_frames,
            cuda=cuda
        )

    else:
        print("No Embedder , initializing random weights")
        embedder = embeddings.load_embedder(embedding_size=embedding_size,
                                            num_features=mel_size,
                                            cuda=cuda)

    return (embedder, embedding_size)

class ProjectModel(torch.nn.Module):
    def __init__(self, config, embedder_path, mel_size, style_size, identity_mode, cuda):
        """
        :style_size: The size of the stylevector produced by embedder
        :mel_size: The number of frequency channels in the mel-cepstrogram
        """
        super().__init__()

        self.mel_size = mel_size
        self.embedder, self.style_size = get_embedder_and_size(mel_size=mel_size,
                                                               path=embedder_path,
                                                               embedding_size=style_size,
                                                               cuda=cuda)

        config["d_model"] = self.mel_size
        config["d_style"] = self.style_size

        self.isvoice_dtor = get_isvoice_discriminator(self.mel_size)
        # self.content_dtor = get_content_discriminator(self.mel_size)
        self.identity_dtor = get_identity_discriminator(self.style_size,
                                                        identity_mode=identity_mode)
        self.transformer = get_transformer(config)

    def forward(self, source_mel, target_style):
        """
        :target_style: An (N x S) tensor
        :input_audio: A (N x 1 x T x M) tensor

        returns the tuple (out_audio, isvoice_prob,
        samecontent_prob, targetperson_prob)

        where the probabilities are scalar Tensors
        """

        transformed_mel = self.transformer(source_mel, target_style)
        transformed_style = self.embedder(transformed_mel)
        transformed_style = transformed_style.reshape(target_style.shape)

        isvoice_prob = self.isvoice_dtor(transformed_mel, None)
        # content_prob = self.content_dtor(torch.stack((source_mel,
        #                                               transformed_mel),
        #                                              dim=1), None)
        identity_prob = self.identity_dtor(torch.cat((target_style,
                                                        transformed_style),
                                                       dim=1), None)

        return transformed_mel, isvoice_prob, identity_prob


import pandas as pd
from datetime import datetime

#FEATURES_PATH = 'data/features.csv'
#DATA_TEST_PATH = 'data/data_test.csv'
#DATA_TEST_MERGE_PATH = 'data/data_test_merge.csv'

def preparing(path_data, path_features):

    df_features = pd.read_csv(path_features, chunksize=100000, iterator=True, delimiter='\\t')
    df_train = pd.read_csv(path_data, delimiter=',')
    df_train = df_train.drop(columns=['Unnamed: 0'])
    df_train = df_train.drop_duplicates()

    df_features_new = pd.DataFrame()  # пустой фрейм
    for chunk_iter in df_features:  # цикл по кускам dataframe
        filtr_id = chunk_iter['id'].isin(df_train.id)  # создаём фильтр по куску
        chunk = chunk_iter[filtr_id]  # фильтруем записи в куске
        df_features_new = pd.concat([df_features_new, chunk])

    df_features_new['buy_time'] = df_features_new.buy_time.apply(datetime.fromtimestamp)
    df_train['buy_time'] = df_train.buy_time.apply(datetime.fromtimestamp)

    df_features_new = df_features_new.sort_values(by=['buy_time'])
    df_train = df_train.sort_values(by=['buy_time'])

    dt_merge = pd.merge_asof(df_train, df_features_new, on='buy_time', by='id')

    dt_merge = dt_merge.dropna()

    return dt_merge

#data_test = preparing(DATA_TEST_PATH, FEATURES_PATH)
#def save(name = 'data/data_test_prep.csv', path_data, path_features):
#    data_test = preparing(path_data, path_features)
#data_test.to_csv(DATA_TEST_MERGE_PATH, index=False)


# -*- coding: utf-8 -*-
# @Author: jsgounot
# @Date:   2022-03-28 19:32:34
# @Last Modified by:   jsgounot
# @Last Modified time: 2022-03-29 11:32:42

from Bio import SeqIO

def main(fnames, outfile):
    idx = 1
    with open(outfile, 'w') as f:
        f.write('#Auto generated by snakemake pipeline\n')
        for sample, fname in fnames.items():
            fdata = SeqIO.parse(fname, 'fasta')
            for ridx, record in enumerate(fdata):
                line = f'chr - {record.id} {sample}_{ridx} 0 {len(record.seq)} chr{idx} \n'
                f.write(line)
                idx += 1

fnames = snakemake.params[0]
outfile = snakemake.output[0]
main(fnames, outfile)


# -*- coding: utf-8 -*-
"""Writing of VCF records
"""

import io
import textwrap

import pytest

import vcfpy
from vcfpy import parser, writer, header, record

__author__ = "Manuel Holtgrewe <manuel.holtgrewe@bihealth.de>"

# TODO: cleanup, refactor tests somewhat

MEDIUM_HEADER = """
##fileformat=VCFv4.3
##fileDate=20090805
##source=myImputationProgramV3.1
##reference=file:///seq/references/1000GenomesPilot-NCBI36.fasta
##contig=<ID=20,length=62435964,assembly=B36,md5=f126cdf8a6e0c7f379d618ff66beb2da,species="Homo sapiens",taxonomy=x>
##phasing=partial
##INFO=<ID=NS,Number=1,Type=Integer,Description="Number of Samples With Data">
##INFO=<ID=DP,Number=1,Type=Integer,Description="Total Depth">
##INFO=<ID=AF,Number=A,Type=Float,Description="Allele Frequency">
##INFO=<ID=AA,Number=1,Type=String,Description="Ancestral Allele">
##INFO=<ID=DB,Number=0,Type=Flag,Description="dbSNP membership, build 129">
##INFO=<ID=H2,Number=0,Type=Flag,Description="HapMap2 membership">
##INFO=<ID=ANNO,Number=.,Type=String,Description="Additional annotation">
##FILTER=<ID=q10,Description="Quality below 10">
##FILTER=<ID=s50,Description="Less than 50% of samples have data">
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=GQ,Number=1,Type=Integer,Description="Genotype Quality">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Read Depth">
##FORMAT=<ID=HQ,Number=2,Type=Integer,Description="Haplotype Quality">
##FORMAT=<ID=FT,Number=.,Type=String,Description="Call-wise filters">
#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tNA00001\tNA00002\tNA00003
""".lstrip()

@pytest.fixture(scope="function")
def header_samples():
    p = parser.Parser(stream=io.StringIO(MEDIUM_HEADER), path="<builtin>")
    p.parse_header()
    return (p.header, p.samples)

def test_write_minimal_record(header_samples, tmpdir_factory):
    O = vcfpy.OrderedDict
    # open temporary file and setup the Writer with header
    path = tmpdir_factory.mktemp("write_header").join("out.vcf")
    header, _ = header_samples
    w = writer.Writer.from_path(path, header)
    # construct record to write out from scratch
    r = record.Record(
        "20",
        100,
        [],
        "C",
        [record.Substitution(record.SNV, "T")],
        None,
        [],
        O(),
        ["GT"],
        [
            record.Call("NA00001", O(GT="0/1")),
            record.Call("NA00002", O(GT="0/0")),
            record.Call("NA00003", O(GT="1/1")),
        ],
    )
    # write out the record, close file to ensure flushing to disk
    w.write_record(r)
    w.close()
    # compare actual result with expected
    RESULT = path.read()
    LINE = "20\t100\t.\tC\tT\t.\t.\t.\tGT\t0/1\t0/0\t1/1\n"
    EXPECTED = MEDIUM_HEADER + LINE
    assert EXPECTED == RESULT

def test_write_annotated_record(header_samples, tmpdir_factory):
    O = vcfpy.OrderedDict
    S = record.Substitution
    # open temporary file and setup the Writer with header
    path = tmpdir_factory.mktemp("write_annotated_record").join("out.vcf")
    header, _ = header_samples
    w = writer.Writer.from_path(path, header)
    # construct record to write out from scratch
    r = record.Record(
        "20",
        100,
        ["rs333", "CSN42"],
        "C",
        [record.Substitution(record.SNV, "T"), record.Substitution(record.SNV, "G")],
        50,
        ["PASS"],
        O([("DP", 93), ("AF", [0.3, 0.2]), ("DB", True)]),
        ["GT", "DP", "GQ", "HQ"],
        [
            record.Call("NA00001", O(GT="0/1", DP=30, GQ=40, HQ=[1, 2])),
            record.Call("NA00002", O(GT="0/2", DP=31, GQ=41, HQ=[3, 4])),
            record.Call("NA00003", O(GT="1/2", DP=32, GQ=42, HQ=[5, 6])),
        ],
    )
    # write out the record, close file to ensure flushing to disk
    w.write_record(r)
    w.close()
    # compare actual result with expected
    RESULT = path.read()
    LINE = "20\t100\trs333;CSN42\tC\tT,G\t50\tPASS\tDP=93;AF=0.3,0.2;DB\tGT:DP:GQ:HQ\t0/1:30:40:1,2\t0/2:31:41:3,4\t1/2:32:42:5,6\n"
    EXPECTED = MEDIUM_HEADER + LINE
    assert EXPECTED == RESULT

def test_write_record_with_escaping(header_samples, tmpdir_factory):
    O = vcfpy.OrderedDict
    S = record.Substitution
    # open temporary file and setup the Writer with header
    path = tmpdir_factory.mktemp("write_header").join("out.vcf")
    header, _ = header_samples
    w = writer.Writer.from_path(path, header)
    # construct record to write out from scratch
    r = record.Record(
        "20",
        100,
        [],
        "C",
        [record.Substitution(record.SNV, "T")],
        None,
        [],
        O([("ANNO", ["Here,are%some chars", "%25"])]),
        ["GT", "FT"],
        [
            record.Call("NA00001", O(GT="0/1", FT=["%25", "FOO"])),
            record.Call("NA00002", O(GT="0/0", FT=[])),
            record.Call("NA00003", O(GT="1/1", FT=[])),
        ],
    )
    # write out the record, close file to ensure flushing to disk
    w.write_record(r)
    w.close()
    # compare actual result with expected
    RESULT = path.read()
    LINE = (
        "20\t100\t.\tC\tT\t.\t.\tANNO=Here%2Care%25some chars,"
        "%2525\tGT:FT\t0/1:%2525;FOO\t0/0:.\t1/1:.\n"
    )
    EXPECTED = MEDIUM_HEADER + LINE
    assert EXPECTED == RESULT

def test_write_record_no_samples(tmpdir_factory):
    O = vcfpy.OrderedDict
    # Create header without samples
    hdr = header.Header(
        lines=[header.HeaderLine("fileformat", "VCFv4.0")], samples=header.SamplesInfos([])
    )
    # construct record to write out from scratch
    r = record.Record("20", 100, [], "C", [record.Substitution(record.SNV, "T")], None, [], O())
    # Write out header and record
    path = tmpdir_factory.mktemp("write_header").join("out.vcf")
    w = writer.Writer.from_path(path, hdr)
    w.write_record(r)
    w.close()
    # Compare result
    RESULT = path.read()
    EXPECTED = textwrap.dedent(
        """
    ##fileformat=VCFv4.0
    #CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO
    20\t100\t.\tC\tT\t.\t.\t.
    """
    ).lstrip()
    assert RESULT == EXPECTED