import pytest

from keywords.MobileRestClient import MobileRestClient
from libraries.testkit.cluster import Cluster
from requests import Session
from keywords.utils import log_info
from keywords.SyncGateway import SyncGateway
from keywords.SyncGateway import sync_gateway_config_path_for_mode
from utilities.cluster_config_utils import persist_cluster_config_environment_prop, copy_to_temp_conf
from keywords.constants import RBAC_FULL_ADMIN
from requests.auth import HTTPBasicAuth

@pytest.mark.sanity
@pytest.mark.syncgateway
@pytest.mark.changes
@pytest.mark.oscertify
@pytest.mark.parametrize("sg_conf_name, x509_cert_auth", [
    ("sync_gateway_default", False),
])
def test_deleted_docs_from_changes_active_only(params_from_base_test_setup, sg_conf_name, x509_cert_auth):
    """
    https://github.com/couchbase/sync_gateway/issues/2955
    1. Create a document
    2. Delete the document
    3. Restart Sync Gateway (to force rebuild of cache from view)
    4. Issue an active_only=true changes request
    5. Issue an active_only=false changes request
    The deleted document was not being included in the result set in step 5.
    """
    cluster_config = params_from_base_test_setup["cluster_config"]
    topology = params_from_base_test_setup["cluster_topology"]
    need_sgw_admin_auth = params_from_base_test_setup["need_sgw_admin_auth"]
    sg_admin_url = topology["sync_gateways"][0]["admin"]
    sg_db = "db"
    num_docs = 10
    mode = params_from_base_test_setup["mode"]
    sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)

    disable_tls_server = params_from_base_test_setup["disable_tls_server"]
    if x509_cert_auth and disable_tls_server:
        pytest.skip("x509 test cannot run tls server disabled")
    if x509_cert_auth:
        temp_cluster_config = copy_to_temp_conf(cluster_config, mode)
        persist_cluster_config_environment_prop(temp_cluster_config, 'x509_certs', True)
        persist_cluster_config_environment_prop(temp_cluster_config, 'server_tls_skip_verify', False)
        cluster_config = temp_cluster_config

    cluster = Cluster(cluster_config)
    cluster.reset(sg_conf)
    client = MobileRestClient()

    auth = need_sgw_admin_auth and (RBAC_FULL_ADMIN['user'], RBAC_FULL_ADMIN['pwd']) or None
    # Add doc to SG
    added_doc = client.add_docs(
        url=sg_admin_url,
        db=sg_db,
        number=num_docs,
        id_prefix="test_changes",
        auth=auth
    )

    # Delete 1 doc
    doc_id = added_doc[0]["id"]
    log_info("Deleting {}".format(doc_id))
    doc = client.get_doc(url=sg_admin_url, db=sg_db, doc_id=doc_id, auth=auth)
    doc_rev = doc['_rev']
    client.delete_doc(sg_admin_url, sg_db, doc_id, doc_rev, auth=auth)

    # Restart SG
    sg_obj = SyncGateway()
    sg_obj.restart_sync_gateways(cluster_config)

    # Changes request with active_only=true
    session = Session()
    if auth:
        session.auth = HTTPBasicAuth(auth[0], auth[1])
    request_url = "{}/{}/_changes?active_only=true".format(sg_admin_url, sg_db)
    log_info("Issuing changes request {}".format(request_url))
    resp = session.get(request_url)
    resp.raise_for_status()
    resp_obj = resp.json()
    log_info("Checking that the deleted doc is not included in the active_only=true changes request")
    for d in resp_obj["results"]:
        assert doc_id not in d

    # Changes request with active_only=false
    request_url = "{}/{}/_changes?active_only=false".format(sg_admin_url, sg_db)
    log_info("Issuing changes request {}".format(request_url))
    resp = session.get(request_url)
    resp.raise_for_status()
    resp_obj = resp.json()
    doc_found = False
    for d in resp_obj["results"]:
        if doc_id != d["id"]:
            continue
        else:
            assert doc_id == d["id"]
            assert d["deleted"]
            doc_found = True
            break

    log_info("Checking that the deleted doc is included in the active_only=false changes request")
    assert doc_found


# This file will need to use the DataManager, FlightSearch, FlightData, NotificationManager
# classes to achieve the program requirements.

from data_manager import DataManager
from flight_search import FlightSearch
from notification_manager import NotificationManager
import dotenv
import os

dotenv.load_dotenv()

API_KEY = os.getenv("REACT_APP_API_KEY")

data_manager_obj = DataManager(API_KEY)
flight_search_obj = FlightSearch(API_KEY)
notification_manager_obj = NotificationManager()

city_dict = data_manager_obj.post_iata_to_google_sheet()

for city, code_low_price in city_dict.items():
    flight_details = flight_search_obj.search_flight(code_low_price)
    if flight_details is not None:
        from_city = "London-STN"
        to_city = f"{city}-{code_low_price[0]}"
        print(to_city)
        from_date = flight_details[0][0]
        print(from_date)
        to_date = flight_details[0][1]
        print(to_date)
        price = flight_details[1][0]
        # notification_manager_obj.send_sms(from_city, to_city, from_date, to_date, price)

# new_user = Users()
# new_user.add_user()


import os
import re
import glob
import argparse
from math import sqrt

from config import get_settings, write_test_log, on_finish_test
from utils import write_json
from label_wav_syllable_test import get_cdf
from phoneme import get_syllable

*_, settings = get_settings()
training_set_settings = settings['training_set_settings']
training_default_values = settings['training_default_values']
model_settings = settings['model_settings']
data_path = settings['data_path']
voice_data_path = settings['voice_data_path']

mode = ''
true_mode = ''
args = None
_mode_dict = {
    'all': '_0',
    'top': '_1',
    'mid': '_2',
    'bot': '_3',
    'pref': '_4',
    'suff': '_5',
    'aall': '_0',
    'atop': '_0',
    'amid': '_0',
    'abot': '_0',
}

def main():
    global true_mode
    os.environ['to_print_results'] = '0'
    from label_wav import AcousticModels

    steps = 0
    true_mode = mode.strip('a')
    if true_mode == 'll':
        true_mode = 'all'

    _mode = int(_mode_dict[true_mode][1])

    # _clip name
    _type_name = _mode_dict[mode]
    _name = args.model_name
    _result_suffix = '_' + args.output_file_suffix
    _test_dataset = args.test_set

    _common_path = model_settings['__common_path'] + _name

    # graph path
    graph = _common_path + '/model_labels/' + _type_name + '_model.pb'
    # labels path
    labels = _common_path + '/model_labels/' + _type_name + '_labels.txt'
    # load model
    model = AcousticModels([graph], [labels])
    # input name
    input_name = 'file'
    # output name
    output_name = 'labels_softmax:0'
    # how many labels that needs to show
    how_many_labels = [5, 4, 3, 2, 1]

    _type_name = _mode_dict[true_mode]
    _common_path = model_settings['__common_path'] \
                   + _name + '/' + _type_name

    os.makedirs(_common_path, exist_ok=True)
    # result file path , name
    result_path = [_common_path + '/result_' + str(i) + _result_suffix + '.txt'
                   for i in how_many_labels]

    tops = [2 * i - 1 for i in how_many_labels] if true_mode == 'all' \
        else how_many_labels

    # global count of labels
    global_counts = [0, 0, 0, 0, 0]

    partial = {}

    if not args.prepare_train_json:
        # create result files
        result_files = [open(p, 'w') for p in result_path]

    # partial results
    partial_results = [[], [], [], [], []]

    # dict count
    count_dicts = [{}, {}, {}, {}, {}]

    # dict wav number
    num_wav_dict = dict()

    wav_labels = [l for l in os.listdir(_test_dataset)
                  if os.path.isdir(_test_dataset + '/' + l) and
                  len(re.sub('[\u3105-\u3129]|no_label', '', l)) == 0]
    wav_labels = list(sorted(wav_labels))
    result_dict = {}
    my_files = []
    total = 0
    rankings = []

    log_t = '{0:s}\n{1:d}\n{2:d}'.format(str(os.getpid()), total, steps)
    write_test_log(_mode, log_t)

    for wav_label in wav_labels:
        # get wav file
        files = list(sorted(glob.glob(_test_dataset + '/' + wav_label + "/*.wav")))
        total += len(files)
        my_files.append(files)
    label_files = zip(wav_labels, my_files)

    trans_dict = {}
    if not true_mode == 'all':
        for k in set(map(get_syllable, wav_labels)):
            trans_dict[k] = {}
    my_rankings = []
    num_predicts = 0
    for wav_label, wav_files in label_files:
        if len(wav_files) == 0:
            continue
        # get 5, 3, 1 labels count
        counts = [0, 0, 0, 0, 0]
        wav_rankings = []
        if not args.prepare_train_json:
            # one line : all files
            for f in result_files:
                f.write("#############---[%s]---#############\n" % wav_label)

        actual = get_syllable(wav_label, true_mode)

        for _wav_file in wav_files:

            # wav
            wav = _wav_file
            # catch words and scores
            words, scores = model.label_wav(wav, input_name, output_name)
            sy_idx = {w: i for i, w in enumerate(words, 1)}
            num_predicts = len(sy_idx)
            ranking = sy_idx[actual] if actual in sy_idx else -1
            rankings.append(ranking)
            wav_rankings.append(ranking)

            steps += 1
            if args.steps:
                print(' ' * 80, end='\r')
                print(steps, '/', total, end='\r')
            if steps % 50 == 0 or steps == total:
                log_t = '{0:s}\n{1:d}\n{2:d}'.format(str(os.getpid()), total, steps)
                write_test_log(_mode, log_t)

            if args.prepare_train_json:
                # f = lambda s: s + _mode_dict[true_mode][1] if re.match('[_a-z]+', s) else s
                result_dict[wav.split('/')[-1]] = [(w, str(pr)) for w, pr in zip(words, scores)]
                continue

            for i in range(len(how_many_labels)):
                # write the 5, 3, 1 labels result in txt
                result_files[i].write("[%s]\n" % wav)
                pos = 0
                pos_c = 0
                counted = False
                for word, prob in zip(words[:tops[i]], scores[:tops[i]]):
                    pos_c += 1
                    predict = get_syllable(word, true_mode)
                    result_files[i].write("predict: %s: %.5f\n" % (predict, prob))
                    if not true_mode == 'all' and tops[i] == 1:
                        from_ = trans_dict[actual]
                        from_.setdefault(predict, 0)
                        to_ = from_[predict] + 1
                        from_[predict] = to_
                        trans_dict[actual] = from_

                    if predict == actual and not counted:
                        counts[i] += 1
                        pos = pos_c
                        counted = True
                result_files[i].write("%d / %d \n" % (pos, tops[i]))

        mu_ranking = sum(wav_rankings) / len(wav_files)
        std_dev_ranking = sqrt(
            sum(map(lambda r: pow(r - mu_ranking, 2), wav_rankings)) / len(wav_rankings))
        my_rankings.append((wav_label, round(mu_ranking, 2),
                            round(std_dev_ranking, 2),
                            get_cdf(wav_rankings, num_predicts, no_cdf=True),
                            len(wav_rankings)))
        # all len of wav_files
        num_of_files = len(wav_files)
        partial[wav_label] = {
            'amount': num_of_files,
            'correct_count':
                {'top ' + str(i): v
                 for i, v in zip(tops, counts)},
            'percentage':
                {'top ' + str(i): '{0:.2f}%'.format(v / num_of_files * 100)
                 for i, v in zip(tops, counts)}
        }

        # sum of wav_files
        if actual not in num_wav_dict:
            num_wav_dict[actual] = len(wav_files)
        else:
            num_wav_dict[actual] += len(wav_files)

        for i in range(len(how_many_labels)):
            # Global
            global_counts[i] += counts[i]

            # write partial results in file
            partial_results[i].append("Partial(" + wav_label + ") : "
                                      + str(counts[i]) + "/" + str(len(wav_files))
                                      + "\n")

            # top how_many_labels[i] syllable
            if actual not in count_dicts[i]:
                count_dicts[i][actual] = counts[i]
            else:
                count_dicts[i][actual] += counts[i]

    if not args.prepare_train_json:
        for i in range(len(how_many_labels)):
            result_files[i].write("-----------------------------------\n")
            # write partial at the end
            for partial_result in partial_results[i]:
                result_files[i].write(partial_result)

            # percentage
            percentage = round((global_counts[i] / total), 4) * 100

            # Total results
            result_files[i].write("Total : " + str(global_counts[i]) + "/" +
                                  str(total) + ", percentage : " +
                                  str(percentage) + "%")

    model.close()

    if args.prepare_train_json:
        write_json(result_dict, os.path.join(_test_dataset, _mode_dict[mode] + '_test.json'))
        exit(0)

    d = \
        {
            k: {
                'amount': num_wav_dict[k],
                'correct_count': {
                    'top ' + str(i): c_d[k]
                    for i, c_d in zip(tops, count_dicts)
                },
                'percentage': {
                    'top ' + str(i): '{0:.2f}%'.format(c_d[k] / num_wav_dict[k] * 100)
                    for i, c_d in zip(tops, count_dicts)
                }
            }
            for k in num_wav_dict.keys()
        }

    my_dict = \
        {
            'partial': partial,
            'total':
                {
                    'amount': total,
                    'num_of_categories': len(wav_labels),
                    'correct_count':
                        {'top ' + str(i): v
                         for i, v in zip(tops, global_counts)},
                    'accuracy':
                        {'top ' + str(i): '{0:.2f}%'.format(v / total * 100)
                         for i, v in zip(tops, global_counts)}
                }
        }

    if not true_mode.startswith('a'):
        my_dict[_mode_dict[true_mode]] = d
        my_dict['transfer'] = trans_dict

    my_rankings.sort(key=lambda it: (it[3], it[0]))
    max_ranking = max(rankings)
    _95, cdf = get_cdf(rankings, num_predicts)
    total_mu_ranking = sum(rankings) / total
    total_sigma_ranking = sqrt(sum(map(lambda it: pow(it - total_mu_ranking, 2), rankings)) / total)
    my_dict['ranking'] = \
        {
            'partial': my_rankings,
            'total':
                [
                    total_mu_ranking, total_sigma_ranking,
                    {
                        'num_of_files': total,
                        'max_ranking': max_ranking,
                        'K': _95,
                        'cdf': cdf
                    }
                ],
        }

    f_name = '/results.json' if _result_suffix == '_1' else '/results' + _result_suffix[1:] + '.json'
    write_json(my_dict, _common_path + f_name)

    for f in result_files:
        f.close()

    on_finish_test(_mode)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--steps', action='store_true', help="""\
                            show steps""")
    parser.add_argument('--prepare_train_json', action='store_true', help="""\
                            prepare_train_json""")
    parser.add_argument('-m', '--mode', default='all', help="""\
                        all, top, mid, bot, pref, suff, aall, atop, amid, abot""")
    parser.add_argument('-ts', '--test_set', default='', help="""\
                            path for test set""")
    parser.add_argument('-s', '--output_file_suffix', default='1', help="""\
                        suffix for output file e.g: xxx_(suffix).txt""")
    parser.add_argument('-d', '-n', '--model_name', default='', help="""\
                            model_settings['common_path']_(name)/model_labels/xxx.pb""")

    args, unknown = parser.parse_known_args()
    mode = args.mode
    if mode not in _mode_dict:
        exit(-1)
    # use settings.json for args default value
    if len(args.model_name) == 0:
        args.model_name = model_settings['name']
    if len(args.test_set) == 0:
        args.test_set = voice_data_path['testing_set']
    main()


#!/usr/bin/env python
# encoding=utf-8
# maintainer: rgaudin

import sys
import locale
import logging

from nosmsd.utils import import_path
from nosmsd.database import Inbox
from nosmsd.settings import settings as nosettings

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
logger.addHandler(handler)

if nosettings.NOSMSD_GETTEXT:
    locale.setlocale(locale.LC_ALL, nosettings.NOSMSD_GETTEXT_LOCALE)

def handle(*args, **options):

    # Message ID in DB is provided as first argument
    # multiple ID are passed for multipart.
    # we'll only work on firt one as Inbox retrieves other parts.
    if len(args) < 2:
        logger.error(u"No message ID provided")
        sys.exit(1)
    try:
        sql_id = int(args[1])
    except:
        sql_id = None

    if not isinstance(sql_id, int):
        logger.error(u"Provided ID (%s) is not an int." % sql_id)
        sys.exit(1)

    # retrieve message from DB
    try:
        message = Inbox.filter(ID=sql_id, Processed=Inbox.PROC_FALSE).get()
    except Inbox.DoesNotExist:
        logger.warning(u"No unprocessed row in DB for ID %d" % sql_id)
        return False

    # process handler
    try:
        handler_func = import_path(nosettings.NOSMSD_HANDLER)
    except AttributeError:
        message.status = Inbox.STATUS_ERROR
        message.save()
        logger.error(u"NO SMS_HANDLER defined while receiving SMS")
    except Exception as e:
        message.status = Inbox.STATUS_ERROR
        message.save()
        logger.error(u"Unbale to call SMS_HANDLER with %r" % e)
    else:
        try:
            handler_func(message)
        except Exception as e:
            message.status = Inbox.STATUS_ERROR
            message.save()
            logger.error(u"SMS handler failed on %s with %r"
                         % (message, e))

    message.status = Inbox.STATUS_PROCESSED
    message.Processed = Inbox.PROC_TRUE
    message.save()

if __name__ == '__main__':
    handle(*sys.argv)


# Create fastq files as simulated reads output
# Start from half-sampled reads coordinates
#BAMfile_halfsampled_coordinates.txt

## JOBS:
## For each referenced peak region, find shift length from true peak
## Write mates for each read, and fill in read end position
## Separate read1 and read2

## Input true and ref peak pos
## 1. Shift the read positions with shift_number
## 2. Find all reads mate
### Load libraries
import pandas as pd
# import pickle
import numpy as np
import csv
import collections
import time
import sys
pd.options.mode.chained_assignment = None  # default='warn'
import string
import random
from tqdm import tqdm
import pysam
from collections import defaultdict

def flatten(x):
    if isinstance(x, collections.Iterable):
        return [a for i in x for a in flatten(i)]
    else:
        return [x]

def cellbarcode_generator(length, size=10):
	chars = 'ACGT'
	cb_list = [''.join(random.choice(chars) for _ in range(size)) for cell in range(length)]
	return cb_list

def PerTruePeakEdition(peak_record, count_vec, read_lines, read_len, jitter_size, random_cellbarcode_list):
	# peak_record = peaks_assignments.loc[1,] # Input
	true_peak_concat = peak_record[0] + ":" + str(peak_record[1]) + "-" + str(peak_record[2])
	reads_cur = read_lines[read_lines['peak_name'] == true_peak_concat] # Input
	nread_cur= np.sum(count_vec).astype(int)
	# Add cell information
	nonempty_cell_ind = np.where(count_vec != 0)[0]
	random_umi_list = cellbarcode_generator(nread_cur, size=10)
	read_code_simu_cur = [random_cellbarcode_list[nonempty_cell_ind[ind]] + ":CellType1" + "CellNo" + str(nonempty_cell_ind[ind] + 1) + ":" + str(true_peak_concat) + "#" + str(count).zfill(4) for ind in range(len(nonempty_cell_ind)) for count in range(count_vec[nonempty_cell_ind[ind]])]
	read_code_simu_cur_withUMI = [read_code_simu_cur[read_id].split(":",1)[0] + random_umi_list[read_id] + ":" + read_code_simu_cur[read_id].split(":",1)[1] for read_id in range(len(read_code_simu_cur))]
# read_code_simu_cur = ["CellType1" + "CellNo" + str(nonempty_cell_ind[ind] + 1) + ":" + str(true_peak_concat) + "#" + str(count).zfill(4) for ind in range(len(nonempty_cell_ind)) for count in range(count_vec[nonempty_cell_ind[ind]])]
	# start = time.time()
	jitter_value_vec = np.random.random_integers(-jitter_size,jitter_size,size=np.shape(reads_cur)[0])  # nrow(reads_cur) should equal to nfrag_cur
	if sum(reads_cur['r1_start'].isna()) == 0:
		reads_cur['r1_start_shifted'] = reads_cur['r1_start'].astype(int) + jitter_value_vec
		reads_cur['r1_end_shifted'] = reads_cur['r1_start'].astype(int) + read_len + jitter_value_vec
		read_1_df = reads_cur[['chr','r1_start_shifted', 'r1_end_shifted']]
		read_1_df['read_name'] = read_code_simu_cur_withUMI
		read_1_df['read_length'] = read_len
		read_1_df['strand'] = '+'
	else:
		read_1_df = pd.DataFrame({'A' : []}) # Create an empty dataframe
	return read_1_df

def PerTruePeakEdition_UMIandRead(peak_record, count_vec, UMI_count_vec, read_lines, read_len, jitter_size, random_cellbarcode_list):
	# peak_record = peaks_assignments.loc[1,] # Input
	true_peak_concat = peak_record[0] + ":" + str(peak_record[1]) + "-" + str(peak_record[2])
	reads_cur = read_lines[read_lines['peak_name'] == true_peak_concat] # Input
	nread_cur= np.sum(count_vec).astype(int)
	# Add cell information
	nonempty_cell_ind = np.where(count_vec != 0)[0]
	# Use UMI_count_vec to guide UMI generation
	random_umi_list = [random.sample(cellbarcode_generator(UMI_count_vec[nonempty_cell_ind_cur], size=10)*count_vec[nonempty_cell_ind_cur], count_vec[nonempty_cell_ind_cur]) if UMI_count_vec[nonempty_cell_ind_cur] > 0 else cellbarcode_generator(1, size=10)*count_vec[nonempty_cell_ind_cur] for nonempty_cell_ind_cur in nonempty_cell_ind]
	random_umi_list = [x for xs in random_umi_list for x in xs]
	# random_umi_list = cellbarcode_generator(nread_cur, size=10)
	read_code_simu_cur = [random_cellbarcode_list[nonempty_cell_ind[ind]] + ":" + "CellNo" + str(nonempty_cell_ind[ind] + 1) + ":" + str(true_peak_concat) + "#" + str(count).zfill(4) for ind in range(len(nonempty_cell_ind)) for count in range(count_vec[nonempty_cell_ind[ind]])]
	read_code_simu_cur_withUMI = [read_code_simu_cur[read_id].split(":",1)[0]  + random_umi_list[read_id] + ":" + read_code_simu_cur[read_id].split(":",1)[1] for read_id in range(len(read_code_simu_cur))]
	# read_code_simu_cur = ["CellType1" + "CellNo" + str(nonempty_cell_ind[ind] + 1) + ":" + str(true_peak_concat) + "#" + str(count).zfill(4) for ind in range(len(nonempty_cell_ind)) for count in range(count_vec[nonempty_cell_ind[ind]])]
	# start = time.time()
	jitter_value_vec = np.random.random_integers(-jitter_size,jitter_size,size=np.shape(reads_cur)[0])  # nrow(reads_cur) should equal to nfrag_cur
	if sum(reads_cur['r1_start'].isna()) == 0:
		reads_cur['r1_start_shifted'] = reads_cur['r1_start'].astype(int) + jitter_value_vec
		reads_cur['r1_end_shifted'] = reads_cur['r1_start'].astype(int) + read_len + jitter_value_vec
		read_1_df = reads_cur[['chr','r1_start_shifted', 'r1_end_shifted']]
		read_1_df['read_name'] = read_code_simu_cur_withUMI
		read_1_df['read_length'] = read_len
		read_1_df['strand'] = '+'
	else:
		read_1_df = pd.DataFrame({'A' : []}) # Create an empty dataframe
	return read_1_df

def UMI_read_dict(bed_file,  UMI_count_mat_file, read1_bedfile, INPUT_bamfile, outdirectory, OUTPUT_cells_barcode_file, jitter_size=5, read_len=90):
	UMI_count_mat_df = pd.read_csv("%s/%s" % (outdirectory, UMI_count_mat_file), header=0, delimiter="\t")
	UMI_count_mat = UMI_count_mat_df.to_numpy()
	UMI_count_mat_cluster = UMI_count_mat_df.columns.to_numpy()
	n_cell = np.shape(UMI_count_mat)[1]
	samfile = pysam.AlignmentFile(INPUT_bamfile, "rb")
	with open(bed_file) as open_peak:
	    reader = csv.reader(open_peak, delimiter="\t")
	    open_peak = np.asarray(list(reader))
	peak_nonzero_id = np.nonzero(UMI_count_mat.sum(axis=1))[0]
	random.seed(2022)
	random_cellbarcode_list = cellbarcode_generator(n_cell, size=16)
	with open(OUTPUT_cells_barcode_file, 'w') as f:
			for item in random_cellbarcode_list:
			    f.write(item + "\n")
	cellbarcode_list_withclusters = np.vstack((random_cellbarcode_list, UMI_count_mat_cluster)).transpose()
	with open(OUTPUT_cells_barcode_file + ".withSynthCluster", 'w') as f:
			for item in cellbarcode_list_withclusters:
			    f.write("\t".join(item) + "\n")
	with open("%s/%s" % (outdirectory, read1_bedfile), 'w') as fp:
		pass
	start = time.time()
	for relative_peak_ind in tqdm(range(len(peak_nonzero_id))):
		peak_ind = peak_nonzero_id[relative_peak_ind]
		# peak_ind = 1
		rec = open_peak[peak_ind]
		rec_name = '_'.join(rec)
		UMI_read_dict_pergene = defaultdict(lambda: [])
		reads = samfile.fetch(rec[0], int(rec[1]), int(rec[2]))
		# TODO: create a dictionary recording UMI(key) to reads starting positions(values)
		for read in reads:
			if read.has_tag('UB:Z'):
				UMI = read.get_tag('UB:Z')
				start = read.reference_start
				if read.is_reverse==1:
					strand = "-"
				else:
					strand = "+"
				# strand = read.is_reverse
				UMI_read_dict_pergene[UMI].append(str(start) + ":" + str(strand))
		## Real data: Obtain the empirical distribution of # reads of UMI in the gene
		UMI_real_vec = list(UMI_read_dict_pergene.keys())
		nread_perUMI = [len(x) for x in UMI_read_dict_pergene.values()] # return a vector with the length of # UMIs in the gene
		# nread_perUMI_prob = nread_perUMI / np.sum(nread_perUMI)
		## Synthetic data
		# Sample syntheitc UMI's read number and assign real UMI to synthetic UMI (only for sampling reads)
		UMI_count_vec = UMI_count_mat[peak_ind,:] # Synthetic umi count
		nonzer_UMI_zero_id = np.nonzero(UMI_count_vec)[0]
		synthetic_nread_perUMI_percell = [np.random.choice(nread_perUMI, size=UMI_count_vec[i], replace=True) for i in nonzer_UMI_zero_id] # return a vector with the length of non-zero count cells, each entry idnicates the number of reads for each UMI
		nread_synthetic = sum(np.sum(x) for x in synthetic_nread_perUMI_percell) # total number of synthetic reads
		# Assign real UMI to synthetic UMI based on read number per UMI
		# Construct a dic with keys as number of reasd per UMI and values as the corresponding UMI strings
		nread_UMI_dic = defaultdict(lambda: [])
		for i,item in enumerate(nread_perUMI):
			nread_UMI_dic[item].append(UMI_real_vec[i])
		synthetic_nread_perUMI_percell_unlist = [xs for x in synthetic_nread_perUMI_percell for xs in list(x)]
		synthetic_realUMI_assign_array = np.array(synthetic_nread_perUMI_percell_unlist).astype(str)
		synthetic_nread_perUMI_percell_unlist_array = np.array(synthetic_nread_perUMI_percell_unlist)
		for i in set(synthetic_nread_perUMI_percell_unlist):
			tmp_ind = np.where(synthetic_nread_perUMI_percell_unlist_array == i)[0]
			synthetic_realUMI_assign_array[tmp_ind] = np.random.choice(nread_UMI_dic[i], size=len(tmp_ind), replace=True)
		synthetic_realUMI_assign_vec = list(synthetic_realUMI_assign_array) # a vectir sane length to synthetic_nread_perUMI_percell_unlist, synthetic UMI's number
		# synthetic_realUMI_assign_vec = [np.random.choice(nread_UMI_dic[nread], size=1, replace=True) for nread in synthetic_nread_perUMI_percell_unlist]
		#
		# Sample reads for each synthetic UMI
		synthetic_read_list = [np.random.choice(UMI_read_dict_pergene[synthetic_realUMI_assign_vec[id]], size=synthetic_nread_perUMI_percell_unlist[id], replace=True) for id in range(len(synthetic_realUMI_assign_vec))]
		synthetic_read_unlist = [xs for x in synthetic_read_list for xs in list(x)]
		synthetic_read_start_list = [int(xs.split(':', 1)[0]) - 1  for xs in synthetic_read_unlist]  # -1 accounts for the BAM coordinates is +1 compared with getfasta bedtools
		synthetic_read_strand_list = [str(xs.split(':', 1)[1]) for xs in synthetic_read_unlist]
		# Creat synthetic UMI string
		UMI_synthetic_uniq = cellbarcode_generator(np.sum(UMI_count_vec), size=10)
		UMI_synthetic_list = np.repeat(np.array(UMI_synthetic_uniq), synthetic_nread_perUMI_percell_unlist)
		# Create syntheitc CB string
		CB_synthetic_repeat = [np.sum(x) for x in synthetic_nread_perUMI_percell] # same length to nozero cells
		CB_synthetic_list = np.repeat(np.array(random_cellbarcode_list)[nonzer_UMI_zero_id], CB_synthetic_repeat)
		# Create syntheitc read name string
		read_name_remaining_list = ["CellNo" + str(nonzer_UMI_zero_id[ind] + 1) + ":" + str(rec_name) + "#" + str(count).zfill(4) for ind in range(len(CB_synthetic_repeat)) for count in range(CB_synthetic_repeat[ind])]
		read_name_list = [CB_synthetic_list[id] + UMI_synthetic_list[id] + ":" + read_name_remaining_list[id] for id in range(nread_synthetic)]
		# Create output table
		jitter_value_vec = np.random.randint(-jitter_size,jitter_size,size=nread_synthetic)  # nrow(reads_cur) should equal to nfrag_cur
		reads_cur = pd.DataFrame({
			'chr': rec[0],
			'r1_start_shifted': synthetic_read_start_list  + jitter_value_vec,
			'r1_end_shifted': synthetic_read_start_list + jitter_value_vec + read_len,
			'read_name': read_name_list,
			'read_length': read_len,
			'strand': synthetic_read_strand_list
			})
		reads_cur.to_csv("%s/%s" % (outdirectory, read1_bedfile), header=None, index=None, sep='\t', mode='a')
	end = time.time()

def main():
	# random.seed(2022)
	user_args = sys.argv[1:]
	bed_file,  UMI_count_mat_file, read1_bedfile, INPUT_bamfile, outdirectory, OUTPUT_cells_barcode_file = user_args
	jitter_size = 5
	read_len = 90
	UMI_read_dict(bed_file,  UMI_count_mat_file, read1_bedfile, INPUT_bamfile, outdirectory, OUTPUT_cells_barcode_file, jitter_size, read_len)

if __name__ == '__main__':
  main()

# # Analysis of intergenetic regions
# outdirectory = '/home/gayan/Projects/scATAC_Simulator/results/20221011_e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1_NONINPUT'
# INPUT_bamfile = "/home/gayan/Projects/scATAC_Simulator/data/10X_MOUSE_BRAIN_ATACandRNA/split.e18_mouse_brain_fresh_5k_gex_possorted_bam/e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1.bam"
# OUTPUT_cells_barcode_file = outdirectory + "/synthetic_cell_barcode.txt"
# bed_file = "/home/gayan/Projects/scATAC_Simulator/results/20221011_e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1_NONINPUT/e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1_peaks.COMPLE.bed"
# UMI_count_mat_file = "e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1.UMI.COMPLE.countmatrix.scDesign2Simulated.txt"
# read1_bedfile = "e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1.syntheticBAM.COMPLE.20221011_UMITransLevel2" + ".read.bed"

# # Analysis of genetic regions
# # outdirectory = '/home/gayan/Projects/scATAC_Simulator/results/20221011_e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1_NONINPUT'
# INPUT_bamfile = "/home/gayan/Projects/scATAC_Simulator/data/10X_MOUSE_BRAIN_ATACandRNA/split.e18_mouse_brain_fresh_5k_gex_possorted_bam/e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1.bam"
# OUTPUT_cells_barcode_file = outdirectory + "/synthetic_cell_barcode.txt"
# bed_file = "/home/gayan/Projects/scATAC_Simulator/results/20221011_e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1_NONINPUT/e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1_peaks.bed"
# UMI_count_mat_file = "e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1.UMI.countmatrix.scDesign2Simulated.txt"
# read1_bedfile = "e18_mouse_brain_fresh_5k_gex_possorted_bam_chr1.syntheticBAM.20221011_UMITransLevel2" + ".read.bed"


import argparse
import os
import time

import numpy as np
import torch
import torch.backends.cudnn as cudnn
from torch.cuda.amp import autocast
from torch.utils.data import DataLoader
from tqdm import tqdm

from checkpoints import EarlyStopping, load_checkpoint
from data import ClusterDataset, parse_data_structure
from hparams import hparams as hp
from logs import Logger
from model import AutoEncoder

def inference(model, dataset):
    image_data = []

    model.eval()
    with torch.no_grad():
        for _ in tqdm(range(hp.training.n_inference_examples), desc='Inference'):
            inputs, features, audio, M, file = next(iter(dataset))

            inputs = inputs.to(0, non_blocking=True).unsqueeze(0)
            features = dataset.normalize(features.unsqueeze(0)).to(0, non_blocking=True)
            M = M.to(0, non_blocking=True).unsqueeze(0)

            with autocast():
                y, encoding = model(inputs, features)

            image_data.append({
                'original_sample': {
                    'title': file.replace('../data', ''),
                    'type': 'spectrogram',
                    'data': M.detach().cpu().numpy().astype(np.float32).squeeze()
                },
                'reconstructed_sample': {
                    'title': 'Reconstructed Sample',
                    'type': 'spectrogram',
                    'data': y.detach().cpu().numpy().astype(np.float32).squeeze()
                },
                'audio': {
                    'title': 'Waveform',
                    'type': 'wave',
                    'data': audio.detach().cpu().numpy().astype(np.float32).squeeze()
                },
                'encoding': {
                    'title': 'Encoding',
                    'type': 'embedding',
                    'data': encoding.detach().cpu().numpy().astype(np.float32)
                }
            })

    return image_data

def validation(model, criterion, loader, dataset):
    valid_loss = []
    data = dict()

    model.eval()
    with torch.no_grad():
        with tqdm(desc='Valid', total=hp.training.n_validation_steps) as pbar:
            for inputs, features, _, M, file in loader:

                if pbar.n >= hp.training.n_validation_steps:
                    break

                inputs = inputs.to(0, non_blocking=True)
                features = dataset.normalize(features).to(0, non_blocking=True)
                M = M.to(0, non_blocking=True)

                with autocast():
                    y, encoding = model(inputs, features)
                    valid_loss.append(criterion(y, M).item())

                    data[file[0]] = {
                        'loss': valid_loss[-1],
                        'encoding': encoding.squeeze()
                    }

                pbar.set_postfix(loss=valid_loss[-1])
                pbar.update()

            valid_loss = np.mean(valid_loss)

            embedding = torch.stack([v['encoding'] for v in data.values()]).detach().cpu().numpy().astype(np.float32)
            return valid_loss, \
                   (embedding, list(data.keys())), \
                   {k: data[k]['loss'] for k in data}

def training(model, optimizer, scaler, criterion, loader, dataset, early_stopping, logger):
    global step

    with tqdm(desc='Train') as pbar:
        pbar.update(step)
        for inputs, features, _, M, _ in loader:

            inputs = inputs.to(0, non_blocking=True)
            features = dataset.normalize(features).to(0, non_blocking=True)
            M = M.to(0, non_blocking=True)

            model.train()
            with autocast():
                y, _ = model(inputs, features)
                loss = criterion(y, M) / hp.training.accumulation_steps
                scaler.scale(loss).backward()

                if (step + 1) % hp.training.accumulation_steps == 0:
                    scaler.step(optimizer)
                    scaler.update()
                    model.zero_grad()

            pbar.set_postfix(loss=loss.item(), patience=f'{early_stopping.counter}/{hp.training.patience}')
            pbar.update()
            step = pbar.n

            logger.log_training(pbar.n, {
                'training.loss': {'loss': loss.item()},
            })

            if pbar.n % hp.training.validation_every_n_steps == 0:
                valid_loss, embedding_data, datapoint_losses = validation(model, criterion, loader, dataset)
                image_data = inference(model, dataset)
                logger.log_validation(model,
                                      step=pbar.n,
                                      scalars={'validation.loss': {'loss': valid_loss}},
                                      image_data=image_data,
                                      embedding_data=embedding_data,
                                      datapoint_losses=datapoint_losses)
                early_stopping(valid_loss, model, optimizer, scaler, step)

                if early_stopping.early_stop:
                    break

if __name__ == '__main__':

    # Parse command line arguments
    parser = argparse.ArgumentParser(description='Train VAE')
    parser.add_argument('--checkpoint', default=None, type=str, help='Checkpoint file to continue training from.')
    args = parser.parse_args()

    # Initialize torch modules
    model = AutoEncoder()
    model.cuda()
    optimizer = torch.optim.Adam(model.parameters(), lr=hp.training.learning_rate)
    scaler = torch.cuda.amp.GradScaler()
    criterion = torch.nn.MSELoss()

    # Load parameters from checkpoint to resume training or start new training
    if args.checkpoint is not None:
        run_dir = os.path.dirname(os.path.dirname(args.checkpoint))
        model, optimizer, scaler, early_stopping_score, early_stopping_counter, running_mean, running_std, step, = \
            load_checkpoint(run_dir, model, optimizer, scaler, 'latest')
    else:
        now = time.strftime("%Y-%m-%d__%H_%M_%S", time.localtime())
        run_dir = os.path.join('..', 'runs', now)
        os.makedirs(os.path.join(run_dir, 'checkpoints'), exist_ok=True)
        os.makedirs(os.path.join(run_dir, 'logs'), exist_ok=True)
        early_stopping_score = None
        early_stopping_counter = 0
        step = 0

    # Initialize EarlyStopping
    early_stopping = EarlyStopping(step=step,
                                   run_dir=run_dir,
                                   best_score=early_stopping_score,
                                   counter=early_stopping_counter,
                                   verbose=False)

    # Initializing logger
    logger = Logger(os.path.join(run_dir, 'logs'))

    # Initialize data loaders
    dataset = ClusterDataset(files=parse_data_structure(hp.files),
                             feature_mean=np.load('mu.npy'),
                             feature_std=np.load('std.npy'))
    loader = DataLoader(dataset=dataset,
                        batch_size=hp.training.batch_size,
                        num_workers=hp.training.num_workers,
                        pin_memory=True)

    # Auto select best algorithm to maximize GPU utilization
    cudnn.benchmark = True

    # Execute train loop
    # try:
    training(model, optimizer, scaler, criterion, loader, dataset, early_stopping, logger)
    # except KeyboardInterrupt:
    #     pass
    # finally:
    # save_checkpoint(run_dir, model, optimizer, scaler, early_stopping_score, early_stopping_counter, dataset, step, 'latest')
    # print('Saved checkpoint.')


#!/bin/python3

import math
import os
import random
import re
import sys

# Complete the plusMinus function below.
def plusMinus(arr):
    negCount = 0
    posCount = 0
    zeroCount = 0
    n = len(arr)
    for i in arr:
        if i > 0: posCount += 1
        if i < 0: negCount += 1
        if i == 0: zeroCount += 1
    posCount /= n
    negCount /= n
    zeroCount /= n
    print("{0:.6f}".format(posCount))
    print("{0:.6f}".format(negCount))
    print("{0:.6f}".format(zeroCount))

if __name__ == '__main__':
    n = int(input())

    arr = list(map(int, input().rstrip().split()))

    plusMinus(arr)


import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import make_interp_spline

def sigmoid(x):
    """
    sigmoid函数
    Param:
        x: 输入
    Return:
        y: 对应的sigmoid函数值
    """
    y = 1./(1+np.exp(-x))
    return y

def mse(x, y):
    """
    计算均方误差
    Param:
        x, y: 输入
    Return:
        x与y的均方误差向量
    """
    MSEs = []
    for i in range(x.shape[1]):
        xi = x[:,i]                   # 列向量
        MSE = 0                       # 初始化
        for j in range(len(y)):
            square = (xi[j]-y[j])**2  # 平方和
            MSE += square
        MSE /= len(y)                 # 取平均
        MSEs.append(MSE)
    return MSEs

def creat_network(net_type, inputs, gts, epoch):
    """
    构造神经网络
    Param:
        net_type: 网络的类型
        inputs: 数据输入
        gts: 正确值
        epoch: 训练轮数
    Return:
        weight: 训练好的权重
    """
    weight = np.array([0,0,0]) # 初始化权重为[0,0,0]
    # weight = np.array([1,1,1]) # 初始化权重为[1,1,1]，行向量
    alpha = 0.9                # 参数
    if net_type == 'SGD':      # 随机梯度下降
        for i in range(epoch):
            for j in range(inputs.shape[0]):     # 每轮需要调整权重的次数
                # index = np.random.randint(inputs.shape[0]) # 随机取一行
                # input = inputs[index]
                input = inputs[j]                             # 依次选取
                output = sigmoid(np.dot(weight, input.T))  # 计算输出
                # e = gts[index] - output
                e = gts[j] - output                        # 输出误差
                delta = alpha*output*(1-output)*e*input   # 误差更新值
                weight = weight + delta                    # 更新权重
    elif net_type == 'Batch':
        deltas = []
        for i in range(epoch):
            for j in range(inputs.shape[0]):               # 每一行
                input = inputs[j]
                output = sigmoid(np.dot(weight, input.T)) # 计算输出
                e = gts[j] - output                        # 输出误差
                delta = alpha*output*(1-output)*e*input   # 误差更新值
                deltas.append(delta)                       # 保存
            final_delta = sum(deltas)/len(deltas)
            weight = weight + final_delta
    return weight

def main():
    dataset = np.array([[0,0,1,0],[0,1,1,0],[1,0,1,1],[1,1,1,1]])
    inputs = dataset[:,0:3]       # 输入
    gts = dataset[:,3:4]          # ground truth
    outputs_SGD = []              # 初始化输出，利于可视化
    outputs_Batch = []
    epochs = 1000
    for epoch in range(1, epochs):
        # 使用SGD构造神经网络
        nn_SGD = creat_network('SGD', inputs, gts, epoch)
        # 使用Bacth构造神经网络
        nn_Batch = creat_network('Batch', inputs, gts, epoch)
        output_SGD = sigmoid(np.dot(nn_SGD, inputs.T))     # SGD的结果
        output_Batch = sigmoid(np.dot(nn_Batch, inputs.T)) # Batch的结果
        outputs_SGD.append(output_SGD)                      # 保存结果
        outputs_Batch.append(output_Batch)
    outputs_SGD = np.array(outputs_SGD)
    outputs_Batch = np.array(outputs_Batch)
    print('Outputs of SGD:\n',outputs_SGD)
    print('Outputs of Batch:\n',outputs_Batch)
    MSE_SGD = mse(outputs_SGD.T, gts)                     # 均方误差
    MSE_Batch = mse(outputs_Batch.T, gts)
    epoch = np.linspace(1,epochs,epochs-1)
    model_SGD = make_interp_spline(epoch, MSE_SGD)       # 绘制平滑曲线
    model_Batch = make_interp_spline(epoch, MSE_Batch)   # 绘制平滑曲线
    y_SGD = model_SGD(epoch)
    y_Batch = model_Batch(epoch)
    l1, = plt.plot(epoch, y_SGD)
    l2, = plt.plot(epoch, y_Batch)
    plt.legend(handles=[l1,l2],labels=["SGD","Batch"],loc='best')
    plt.title('MSE')
    plt.show()

if __name__ == "__main__":
    main()


#----------------------------------------
#
# Author     Perry Evans
#            evansjp@mail.med.upenn.edu
# 2008
#
#----------------------------------------
"""
Functions here help with network loading,
printing, and searching.
"""
import sys
from collections import defaultdict

def getNodes(afile):
    """ Make a {} from a file.  Each line
        of the file is one item in the {}.

    @param afile: file w/ one item per line
    @return: {} of nodes
    """

    nodes = dict()
    f_nodes = open(afile)
    for line in f_nodes:
        nodes[ line.strip() ] = True
    f_nodes.close()
    return nodes

def dumpNodes(afile, node_dict):
    """ Output {} to file.  One item
    per line.

    @param afile: write to this file
    @param node_dict: {} with items for writing
    """

    f_out = open(afile, 'w')
    for node in node_dict.keys():
        f_out.write(node + '\n')
    f_out.close()

def getEdges(afile):
    """ Make edge {} from file.
    {} format is d[n1][n2] = True

    @param afile: tab-delimited file; one edge per line; edges are bi-directional
    @return: {} of edges
    """

    edges = dict()
    f_edges = open(afile)
    for line in f_edges:
        sp = [x.strip() for x in line.split('\t')]
        [node1, node2] = sp[0:2]
        if node1 != node2:
            if not edges.has_key(node1):
                edges[node1] = dict()
            if not edges.has_key(node2):
                edges[node2] = dict()
            edges[node1][node2] = True
            edges[node2][node1] = True
    f_edges.close()
    return edges

def getEdges_pubmed(afile):
    """ Make edge {} from file.
    {} format is d[n1][n2] = True

    @param afile: tab-delimited file; one edge per line; edges are bi-directional
    @return: {} of edges
    """

    edges = dict()
    f_edges = open(afile)
    for line in f_edges:
        [node1, node2, pubmed] = [x.strip() for x in line.split('\t')]
        if node1 != node2:
            if not edges.has_key(node1):
                edges[node1] = dict()
            if not edges.has_key(node2):
                edges[node2] = dict()
            pubmed_ls = []
            for paper in pubmed.split(','):
                paper_id = paper.strip()
                if paper_id != '':
                    pubmed_ls.append(paper_id)
            edges[node1][node2] = pubmed_ls
            edges[node2][node1] = pubmed_ls
    f_edges.close()
    return edges

def getEdges_forProteins(afile, use_proteins):
    """ Make edge {} from file.
    {} format is d[n1][n2] = True

    @param afile: tab-delimited file; one edge per line; edges are bi-directional
    @return: {} of edges
    """

    edges = dict()
    f_edges = open(afile)
    for line in f_edges:
        [node1, node2] = [x.strip() for x in line.split('\t')][0:2]
        if use_proteins.has_key(node1) and use_proteins.has_key(node2):
            if node1 != node2:
                if not edges.has_key(node1):
                    edges[node1] = dict()
                if not edges.has_key(node2):
                    edges[node2] = dict()
                edges[node1][node2] = True
                edges[node2][node1] = True
    f_edges.close()
    return edges

def dumpEdges(afile, edge_dict):
    """ Write edges to file.

    @param afile: output is tab-delimited; bi-directional edges are not duplicated
    @param edge_dict: {} format is d[n1][n2] = True
    """

    f_out = open(afile, 'w')
    seen = {}
    for gene1 in edge_dict.keys():
        for gene2 in edge_dict[gene1].keys():
            key1 = gene1+':'+gene2
            key2 = gene2+':'+gene1
            if not seen.has_key(key1) and not seen.has_key(key2):
                seen[key1] = True
                seen[key2] = True
                f_out.write(gene1 + '\t' + gene2 + '\t' + edge_dict[gene1][gene2] + '\n')
    f_out.close()

def mkNodesFromEdges(afile):
    """ Given an edge file, make a {} of nodes.

    @param afile: tab-delimted file of edges; one per line; edges are bi-directional
    @return: {} of nodes
    """

    nodes = {}
    f_edges = open(afile)
    for line in f_edges:
        splitup = line.strip().split('\t')
        nodes[splitup[0]] = True
        nodes[splitup[1]] = True
    f_edges.close()
    return nodes

def getConnected(ls_file, net_file):
    """ Find all the genes in the network
    that are one away from these genes.

    @param ls_file: file of genes to find connections to; one gene per line
    @param net_file: tab-delimited file for the network; one edge per line; edges are bi-directional
    @return: {} of genes
    """

    seed_ls = getNodes(ls_file)
    net = getEdges(net_file)
    second_level_ls = {}
    for gene in seed_ls.keys():
        if net.has_key(gene):
            for gene2 in net[gene].keys():
                if gene != gene2:
                    second_level_ls[gene2] = True
    return second_level_ls

def intersectLists(list_of_lists):
    """ Given a list of {}, return the intersection
    as a {}.

    @param list_of_lists: [] of {}
    @return: {} of genes
    """

    if len(list_of_lists) == 0: return {}
    same = {}
    for item in list_of_lists[0].keys():
        count = 0
        for alist in list_of_lists[1:]:
            if alist.has_key(item):
                count += 1
        if count == len(list_of_lists)-1:
            same[item] = True
    return same

def intersectFiles(list_of_files):
    """ Given a list of files, return the intersection
    as a {}.

    @param list_of_lists: [] of files
    @return: {} of genes
    """

    list_of_lists = []
    for afile in list_of_files:
        list_of_lists.append(getNodes(afile))
    return intersectLists(list_of_lists)

def unionLists(list_of_lists):
    """ Given a list of {}, return the union
    as a {}.

    @param list_of_lists: list of {}
    @return: {} of union
    """

    union = {}
    for alist in list_of_lists:
        for k in alist.keys():
            union[k] = True
    return union

def degree(gene, network):
    """ Find the degree of this node
    in this network.

    @param gene: node in the network
    @param network: edge {} format d[n1][n2] = True
    @return: deg of gene as int
    """

    return len(network[gene].keys())

def degree_ignoreSim(gene, network, sims, cutoff):
    """ Find the degree of this node
    in this network.

    @param gene: node in the network
    @param network: edge {} format d[n1][n2] = True
    @return: deg of gene as int
    """
    seen_genes = {}
    for neigh in network[gene]:
        found_match = False
        for seen in seen_genes:
            if neigh in sims[seen]:
                if sims[seen][neigh] < cutoff:
                    found_match = True
        if not found_match:
            seen_genes[neigh] = True
    return len(seen_genes.keys())

def avgConnectivity(gene_ls, network):
    """ Given a set of genes, return
    their average connectivity in the network.

    @param geneLs: {} of genes
    @param network: edge {} format d[n1][n2] = True
    @return: float for avg connectivity
    """

    [connectivity, total] = [0, 0]
    for gene in gene_ls.keys():
        if network.has_key(gene):
            connectivity += degree(gene, network)
            total += 1
    return float(connectivity) / float(total)

def getDegreeDistributionLs(gene_dict, network_edges):
    """ For these genes, return the degree distribution
        according to this network.

    @param gene_dict: {} w/ gene names
    @param network_edges {} of edges d[n1][n2] = True
    @return {} of degrees d[degree] = count
    """

    degreeDict = []
    for gene in gene_dict.keys():
        if network_edges.has_key(gene):
            degreeDict.append(degree(gene, network_edges))
            #if not degreeDict.has_key(deg):
            #    degreeDict[str(deg)] = 0
            #degreeDict[str(deg)] += 1
    return degreeDict

def getDegreeDistributionDict(gene_dict, network_edges):
    """ For these genes, return the degree distribution
        according to this network.

    @param gene_dict: {} w/ gene names
    @param network_edges {} of edges d[n1][n2] = True
    @return {} of degrees d[degree] = count
    """

    degreeDict = {}
    for gene in gene_dict.keys():
        if network_edges.has_key(gene):
            deg = degree(gene, network_edges)
            if not degreeDict.has_key(str(deg)):
                degreeDict[str(deg)] = 0
            degreeDict[str(deg)] += 1
    return degreeDict

def getProteinsWithXneighbors(network_edges, X):
    """ Return {} of proteins with X neighbors. """

    qualify = {}
    for protein in network_edges.keys():
        if degree(protein, network_edges) == X:
            qualify[protein] = True
    return qualify

def getProteinsWithAtLeastXneighbors(network_edges, X):
    """ Return {} of proteins with at least X neighbors. """

    qualify = {}
    for protein in network_edges.keys():
        if degree(protein, network_edges) >= X:
            qualify[protein] = True
    return qualify

def getTopXconnectedProteins(network_edges, X, bool_expand):
    """ Return {} of proteins in top X% connectivity.

        Rank by connectivity and take top X%.

        If bool_expand == True, the hub list will be
        expanded to include all hubs with the degree
        cutoff.  If False, the degree cutoff will
        be the lowest cutoff that keeps the node
        count below X%."""

    degree_protein = []
    for protein in network_edges.keys():
        deg = degree(protein, network_edges)
        degree_protein.append([deg, protein])

    degree_protein.sort(reverse=True)
    total_proteins = len(network_edges.keys())
    top_x = int(float(X) * float(total_proteins) / float(100))
    deg = 0
    proteins = {}
    counter = 0
    while counter < top_x:
        [d, protein] = degree_protein[counter]
        proteins[protein] = True
        deg = d
        counter += 1
    if bool_expand:
        #extend list
        if counter < len(degree_protein):
            current_d = degree_protein[counter][0]
            while current_d == deg and counter < len(degree_protein):
                [d, protein] = degree_protein[counter]
                proteins[protein] = True
                current_d = d
                counter += 1
    else:
        # shrink list
        if counter < len(degree_protein):
            if degree_protein[counter][0] == deg:
                counter -= 1
            while degree_protein[counter][0] == deg:
                del proteins[degree_protein[counter][1]]
                counter -= 1

    return proteins

def getTopXconnectedProteins_deg(network_edges, X):
    """ Get the proteins corresponding to the top X degrees """

    degree2protein = defaultdict(dict)
    for protein in network_edges:
        deg = degree(protein, network_edges)
        degree2protein[deg][protein] = True
    degrees = degree2protein.keys()
    degrees.sort(reverse=True)
    total_degrees = len(degrees)
    top_x = int(float(X) * float(total_degrees) / float(100))

    proteins = {}
    counter = 0
    while counter < top_x:
        for protein in degree2protein[ degrees[counter] ]:
            proteins[protein] = True
        counter += 1

    return proteins

def split_into_topConnect_other(network_edges, X, expand):
    """ Return [top connected, all others] """

    hubs = getTopXconnectedProteins(network_edges,
                                    X, expand)
    nonHubs = {}
    for gene in network_edges:
        if not gene in hubs:
            nonHubs[gene] = True
    return [hubs, nonHubs]

def getTopXconnectedProteins_ignoreSim(network_edges, sims, X, simCut):
    """ Return {} of proteins in top X% connectivity.

        Rank by connectivity and take top X%. """

    degree_protein = []
    for protein in network_edges.keys():
        deg = degree_ignoreSim(protein, network_edges, sims, simCut)
        degree_protein.append([deg, protein])

    degree_protein.sort(reverse=True)
    total_proteins = len(network_edges.keys())
    top_x = int(float(X) * float(total_proteins) / float(100))
    deg = 0
    proteins = {}
    counter = 0
    while counter < top_x:
        [d, protein] = degree_protein[counter]
        proteins[protein] = True
        deg = d
        counter += 1
    #for [d, protein] in degree_protein:
    #    print d, protein
    current_d = degree_protein[counter][0]
#    print '//', current_d
    while current_d == deg:
        [d, protein] = degree_protein[counter]
        proteins[protein] = True
        current_d = d
        counter += 1
    #print '//', current_d
    return proteins

def getNeighbors(genes, edges):
   neighbors = {}
   for gene in genes.keys():
       if edges.has_key(gene):
           for neighbor in edges[gene].keys():
               neighbors[neighbor] = True
   return neighbors

def countEdges(network):
    edges = {}
    for p1 in network.keys():
        for p2 in network[p1].keys():
            k1 = p1+':'+p2
            k2 = p2+':'+p1
            if not edges.has_key(k1) and not edges.has_key(k2):
                edges[k1] = True
    return len(edges.keys())


from django.db import models
from django.utils.dateformat import *
from django.utils import timezone
import re

# Create your models here.
class Block(models.Model):

    position = models.IntegerField(
        verbose_name='Позиция',
        null=True,
        blank=True
        )

    title = models.CharField(
        max_length=100,
        blank=True,
        null=True,
        verbose_name="Раздел",
        )

    text = models.TextField(
        max_length=1000,
        blank=True,
        null=True,
        verbose_name="Текст",
        )

    tag = models.CharField(
        max_length=10,
        blank=True,
        null=True,
        verbose_name="Тег",
        )

    class Meta:
        ordering = ('position', )
        verbose_name = 'Блок'
        verbose_name_plural = 'Блоки'

    def __str__(self):
        return self.title

class BlockImage(models.Model):
    block = models.ForeignKey(
        Block,
        on_delete=models.SET_NULL,
        related_name='block_image',
        verbose_name='Блок',
        null=True,
        blank=True
    )

    image = models.ImageField(upload_to='images/', null=True, blank=True)

    tag = models.CharField(
        max_length=100,
        blank=True,
        null=True,
        verbose_name="Тег",
        )

    class Meta:
        verbose_name = 'Изображение'
        verbose_name_plural = 'Изображения'

    def __str__(self):
        return self.block.title

class FeedBack(models.Model):
    phone = models.CharField(max_length=16, verbose_name="Телефон", null=True, blank=True, unique=True)
    name = models.CharField(max_length=50, verbose_name="Имя", null=True, blank=True, unique=True)
    email = models.CharField(max_length=80, verbose_name="email", null=True, blank=True, unique=True)
    text = models.CharField(max_length=500, verbose_name="Сообщение", null=True, blank=True)
    datetime_create = models.DateTimeField(default=timezone.now, verbose_name="Дата и время создания")
    complete = models.BooleanField(default=False, verbose_name="Отметка о выполнении")
    user_last_change = models.ForeignKey("BotUsers", null=True,  blank=True,  on_delete=models.SET_NULL)
    datetime_complete = models.DateTimeField(null=True, blank=True, verbose_name="Дата и время отработки заявки")

    def save_phone(self, phone):
        phone = re.sub(r'\+?[78](\d{3})(\d{3})(\d\d)(\d\d)', r'+7\1\2\3\4', phone)
        if phone[0] == '9' and len(phone) == 10:
            phone = '+7' + phone

        self.phone = re.sub(r'\+?[78](\d{3})(\d{3})(\d\d)(\d\d)', r'+7\1\2\3\4', phone)
        self.username = re.sub(r'\+?[78](\d{3})(\d{3})(\d\d)(\d\d)', r'+7\1\2\3\4', phone)

    class Meta:
        verbose_name = 'Заявка'
        verbose_name_plural = 'Заявки'

    def __str__(self):
        return self.name + " Дата и время создания: %s" % (format(self.datetime_create, "d.m.Y H:i"))

class BotUsers(models.Model):
     username = models.CharField(max_length=50, verbose_name="Имя пользователя")
     first_name = models.CharField(max_length=50, verbose_name="Имя", null=True, blank=True)
     last_name = models.CharField(max_length=50, verbose_name="Фамилия", null=True, blank=True)
     userid = models.IntegerField(verbose_name="user_id", unique=True)

     class Meta:
         verbose_name = 'User'
         verbose_name_plural = 'Адресаты рассылки'

     def __str__(self):
         return self.username


# Imports
import re
import numpy as np

def checkTimeValidity(inputTime):
	if(len(inputTime) != 3):
		print("Time format invalid")
		exit()
	if(int(inputTime[0]) < 0 or int(inputTime[0]) > 24):
		print("Time format invalid")
		exit()
	if(int(inputTime[1]) < 0 or int(inputTime[1]) > 59):
		print("Time format invalid")
		exit()
	if(int(inputTime[2]) < 0 or int(inputTime[2]) > 59):
		print("Time format invalid")
		exit()

def convertToSeconds(inputTime):
	return int(inputTime[0])*60*60+int(inputTime[1])*60+int(inputTime[2])

#Start and end times user input
startTime = input("Enter start time in HH:MM:SS\n")
startTime = re.split(':', startTime)
checkTimeValidity(startTime)

endTime = input("Enter end time in HH:MM:SS\n")
endTime = re.split(':', endTime)
checkTimeValidity(endTime)

#Find time difference by converting to seconds
startSeconds = convertToSeconds(startTime)
endSeconds = convertToSeconds(endTime)
diff = endSeconds-startSeconds
if(diff < 0):
	print("End time should be after start time")
	exit()

# File related
f = open("134.141.123.1.txt", 'r')
lines = f.readlines()

# Initialize metrics
success = 0
failure = 0
rtt = []

# Read line-by-line and split tokens
for line in lines:
	splitLine = re.split('\[|\]| ', line)
	inputTime = re.split(':', splitLine[2]) #Parse Time
	if(splitLine[3] == "PM"): # add 12 hours to the time if "PM" to convert to HH:MM:SS format
		inputTime[0] = int(inputTime[0]) + 12

	if(convertToSeconds(inputTime) >= startSeconds and convertToSeconds(inputTime)<endSeconds):
		if("Reply" in splitLine[6]):
			success = success+1
			value = int(splitLine[11])
			rtt.append(value)
		else:
			failure = failure+1
f.close()

if(success != 0):
	print("Success: " + str(success) + ", Failure: " + str(failure) + " (" + str("{:.2f}".format(failure*100/success))+"%)")
	print("RTT (avg/min/max/std) = (" + str("{:.2f}".format(np.mean(rtt))) + "/" + str(min(rtt)) + "/" + str(max(rtt)) + "/" + str("{:.2f}".format(np.std(rtt))) + ")")
else:
	print("Success: " + str(success) + ", Failure: " + str(failure))


import ast

from oslo_log import log
from oslo_utils import strutils
from oslo_utils import uuidutils
import six
import webob
from webob import exc

from manila.api import common
from manila.api.openstack import wsgi
from manila.api.views import shares as share_views
from manila import db
from manila import exception
from manila.i18n import _
from manila.i18n import _LI
from manila import share
from manila.share import share_types

LOG = log.getLogger(__name__)

class ShareController(wsgi.Controller):
    """The Shares API controller for the OpenStack API."""

    _view_builder_class = share_views.ViewBuilder

    def __init__(self):
        super(ShareController, self).__init__()
        self.share_api = share.API()

    def show(self, req, id):
        """Return data about the given share."""
        context = req.environ['manila.context']

        try:
            share = self.share_api.get(context, id)
        except exception.NotFound:
            raise exc.HTTPNotFound()

        return self._view_builder.detail(req, share)

    def delete(self, req, id):
        """Delete a share."""
        context = req.environ['manila.context']

        LOG.info(_LI("Delete share with id: %s"), id, context=context)

        try:
            share = self.share_api.get(context, id)

            # NOTE(ameade): If the share is in a consistency group, we require
            # it's id be specified as a param.
            if share.get('consistency_group_id'):
                consistency_group_id = req.params.get('consistency_group_id')
                if (share.get('consistency_group_id') and
                        not consistency_group_id):
                    msg = _("Must provide 'consistency_group_id' as a request "
                            "parameter when deleting a share in a consistency "
                            "group.")
                    raise exc.HTTPBadRequest(explanation=msg)
                elif consistency_group_id != share.get('consistency_group_id'):
                    msg = _("The specified 'consistency_group_id' does not "
                            "match the consistency group id of the share.")
                    raise exc.HTTPBadRequest(explanation=msg)

            self.share_api.delete(context, share)
        except exception.NotFound:
            raise exc.HTTPNotFound()
        except exception.InvalidShare as e:
            raise exc.HTTPForbidden(explanation=six.text_type(e))

        return webob.Response(status_int=202)

    @wsgi.Controller.api_version("2.5", None, True)
    @wsgi.action("os-migrate_share")
    def migrate_share(self, req, id, body):
        """Migrate a share to the specified host."""
        context = req.environ['manila.context']
        try:
            share = self.share_api.get(context, id)
        except exception.NotFound:
            msg = _("Share %s not found.") % id
            raise exc.HTTPNotFound(explanation=msg)
        params = body['os-migrate_share']
        try:
            host = params['host']
        except KeyError:
            raise exc.HTTPBadRequest(explanation=_("Must specify 'host'"))
        force_host_copy = params.get('force_host_copy', False)
        try:
            force_host_copy = strutils.bool_from_string(force_host_copy,
                                                        strict=True)
        except ValueError:
            raise exc.HTTPBadRequest(
                explanation=_("Bad value for 'force_host_copy'"))
        self.share_api.migrate_share(context, share, host, force_host_copy)
        return webob.Response(status_int=202)

    def index(self, req):
        """Returns a summary list of shares."""
        return self._get_shares(req, is_detail=False)

    def detail(self, req):
        """Returns a detailed list of shares."""
        return self._get_shares(req, is_detail=True)

    def _get_shares(self, req, is_detail):
        """Returns a list of shares, transformed through view builder."""
        context = req.environ['manila.context']

        search_opts = {}
        search_opts.update(req.GET)

        # Remove keys that are not related to share attrs
        search_opts.pop('limit', None)
        search_opts.pop('offset', None)
        sort_key = search_opts.pop('sort_key', 'created_at')
        sort_dir = search_opts.pop('sort_dir', 'desc')

        # Deserialize dicts
        if 'metadata' in search_opts:
            search_opts['metadata'] = ast.literal_eval(search_opts['metadata'])
        if 'extra_specs' in search_opts:
            search_opts['extra_specs'] = ast.literal_eval(
                search_opts['extra_specs'])

        # NOTE(vponomaryov): Manila stores in DB key 'display_name', but
        # allows to use both keys 'name' and 'display_name'. It is leftover
        # from Cinder v1 and v2 APIs.
        if 'name' in search_opts:
            search_opts['display_name'] = search_opts.pop('name')
        if sort_key == 'name':
            sort_key = 'display_name'

        common.remove_invalid_options(
            context, search_opts, self._get_share_search_options())

        shares = self.share_api.get_all(
            context, search_opts=search_opts, sort_key=sort_key,
            sort_dir=sort_dir)

        limited_list = common.limited(shares, req)

        if is_detail:
            shares = self._view_builder.detail_list(req, limited_list)
        else:
            shares = self._view_builder.summary_list(req, limited_list)
        return shares

    def _get_share_search_options(self):
        """Return share search options allowed by non-admin."""
        # NOTE(vponomaryov): share_server_id depends on policy, allow search
        #                    by it for non-admins in case policy changed.
        #                    Also allow search by extra_specs in case policy
        #                    for it allows non-admin access.
        return (
            'display_name', 'status', 'share_server_id', 'volume_type_id',
            'share_type_id', 'snapshot_id', 'host', 'share_network_id',
            'is_public', 'metadata', 'extra_specs', 'sort_key', 'sort_dir',
            'consistency_group_id', 'cgsnapshot_id'
        )

    def update(self, req, id, body):
        """Update a share."""
        context = req.environ['manila.context']

        if not body or 'share' not in body:
            raise exc.HTTPUnprocessableEntity()

        share_data = body['share']
        valid_update_keys = (
            'display_name',
            'display_description',
            'is_public',
        )

        update_dict = dict([(key, share_data[key])
                            for key in valid_update_keys
                            if key in share_data])

        try:
            share = self.share_api.get(context, id)
        except exception.NotFound:
            raise exc.HTTPNotFound()

        share = self.share_api.update(context, share, update_dict)
        share.update(update_dict)
        return self._view_builder.detail(req, share)

    @wsgi.Controller.api_version("2.4")
    def create(self, req, body):
        return self._create(req, body)

    @wsgi.Controller.api_version("1.0", "2.3")  # noqa
    def create(self, req, body):  # pylint: disable=E0102
        # Remove consistency group attributes
        body.get('share', {}).pop('consistency_group_id', None)
        share = self._create(req, body)
        return share

    def _create(self, req, body):
        """Creates a new share."""
        context = req.environ['manila.context']

        if not self.is_valid_body(body, 'share'):
            raise exc.HTTPUnprocessableEntity()

        share = body['share']

        # NOTE(rushiagr): v2 API allows name instead of display_name
        if share.get('name'):
            share['display_name'] = share.get('name')
            del share['name']

        # NOTE(rushiagr): v2 API allows description instead of
        #                display_description
        if share.get('description'):
            share['display_description'] = share.get('description')
            del share['description']

        size = share['size']
        share_proto = share['share_proto'].upper()

        msg = (_LI("Create %(share_proto)s share of %(size)s GB") %
               {'share_proto': share_proto, 'size': size})
        LOG.info(msg, context=context)

        availability_zone = share.get('availability_zone')

        if availability_zone:
            try:
                db.availability_zone_get(context, availability_zone)
            except exception.AvailabilityZoneNotFound as e:
                raise exc.HTTPNotFound(explanation=six.text_type(e))

        kwargs = {
            'availability_zone': availability_zone,
            'metadata': share.get('metadata'),
            'is_public': share.get('is_public', False),
            'consistency_group_id': share.get('consistency_group_id')
        }

        snapshot_id = share.get('snapshot_id')
        if snapshot_id:
            snapshot = self.share_api.get_snapshot(context, snapshot_id)
        else:
            snapshot = None

        kwargs['snapshot'] = snapshot

        share_network_id = share.get('share_network_id')

        if snapshot:
            # Need to check that share_network_id from snapshot's
            # parents share equals to share_network_id from args.
            # If share_network_id is empty than update it with
            # share_network_id of parent share.
            parent_share = self.share_api.get(context, snapshot['share_id'])
            parent_share_net_id = parent_share['share_network_id']
            if share_network_id:
                if share_network_id != parent_share_net_id:
                    msg = "Share network ID should be the same as snapshot's" \
                          " parent share's or empty"
                    raise exc.HTTPBadRequest(explanation=msg)
            elif parent_share_net_id:
                share_network_id = parent_share_net_id

        if share_network_id:
            try:
                self.share_api.get_share_network(
                    context,
                    share_network_id)
            except exception.ShareNetworkNotFound as e:
                raise exc.HTTPNotFound(explanation=six.text_type(e))
            kwargs['share_network_id'] = share_network_id

        display_name = share.get('display_name')
        display_description = share.get('display_description')

        if 'share_type' in share and 'volume_type' in share:
            msg = 'Cannot specify both share_type and volume_type'
            raise exc.HTTPBadRequest(explanation=msg)
        req_share_type = share.get('share_type', share.get('volume_type'))

        if req_share_type:
            try:
                if not uuidutils.is_uuid_like(req_share_type):
                    kwargs['share_type'] = \
                        share_types.get_share_type_by_name(
                            context, req_share_type)
                else:
                    kwargs['share_type'] = share_types.get_share_type(
                        context, req_share_type)
            except exception.ShareTypeNotFound:
                msg = _("Share type not found.")
                raise exc.HTTPNotFound(explanation=msg)
        elif not snapshot:
            def_share_type = share_types.get_default_share_type()
            if def_share_type:
                kwargs['share_type'] = def_share_type

        new_share = self.share_api.create(context,
                                          share_proto,
                                          size,
                                          display_name,
                                          display_description,
                                          **kwargs)

        return self._view_builder.detail(req, dict(six.iteritems(new_share)))

def create_resource():
    return wsgi.Resource(ShareController())


import boto3
import requests
import json
import pprint

SECRET_NAME = 'fitbit'
REGION_NAME = 'us-west-1'
DATE = '2021-12-04'  # I ran this day by using running mode, so GPS should have worked
PERIOD = '1d'
# RESOURCE = 'steps'
# RESOURCE = 'calories'  # Calories burned
RESOURCE = 'distance'
RESPONSE_JSON = f'data/activity_time_series_{RESOURCE}_{DATE}.json'

def get_secret(region_name: str, secret_name: str) -> dict:
    session = boto3.session.Session()
    client = session.client(service_name='secretsmanager', region_name=region_name)
    secret_value = client.get_secret_value(SecretId=secret_name)
    secret_string = secret_value['SecretString']
    secret = json.loads(secret_string)
    return secret

def main():

    # Get secret
    secret = get_secret(REGION_NAME, SECRET_NAME)
    access_token = secret['access-token']
    encoded_id = secret['encoded-id']

    # Get activity
    url = f'https://api.fitbit.com/1/user/{encoded_id}/activities/{RESOURCE}/date/{DATE}/{PERIOD}.json'
    headers = {'Authorization': f'Bearer {access_token}'}
    response = requests.request('GET', url, headers=headers)
    print('Response headers')
    pprint.pprint(response.headers)
    print()
    print('Response contents')
    pprint.pprint(response.json())
    print()

    # Save data
    with open(RESPONSE_JSON, 'w') as f:
        json.dump(response.json(), f, indent=2)
    print(f'Saved the data from fitbit API to {RESPONSE_JSON}')

if __name__ == '__main__':
    main()


from discord.ext.commands import Cog
from discord.ext.commands import command
from discord.utils import get
from discord import Embed

class Roles(Cog):
  def __init__(self, bot):
    self.bot = bot

  @Cog.listener()
  async def on_ready(self):
    if not self.bot.ready:
      self.bot.cogs_ready.ready_up("roles")

  @command(name="pais")
  async def pais(self, ctx):
    paises = await ctx.channel.send("Para agregarte un rol de acuerdo a tu nacionalidad/país de residencia, haz click en la bandera del país que corresponda.")
    banderas = ["🇦🇷", "🇧🇴", "🇧🇷", "🇨🇱", "🇨🇴", "🇪🇨", "🇺🇸", "🇲🇽", "🇵🇾", "🇵🇪", "🇻🇪", "🇫🇷"]
    for i in banderas:
      await paises.add_reaction(i)

  @command()
  async def notifs(self, ctx):
    await ctx.message.delete()
    notifs = await ctx.channel.send("Para agregarte el rol de <@&856666465145978890>, reacciona con la campanita abajo.")
    await notifs.add_reaction('🛎')

  @command()
  async def linerole(self, ctx):
    await ctx.message.delete()
    notifs = await ctx.channel.send("Para agregarte el rol de acuerdo a la línea que maineas, haz click en la reacción que corresponda (puedes agregarte más de uno).\n\n:hospital: : support main\n:dagger: : adc main\n:mage: : mid main\n:camping: : jungle main\n:shield: : top main")

    lineas = ["🏥", "🗡", "🧙", "🏕", "🛡"]
    for i in lineas:
        await notifs.add_reaction(i)

  @Cog.listener()
  async def on_raw_reaction_add(self, payload):
    user = payload.member

    if user == self.bot:
      return

    campanita = get(user.guild.roles, name="notificaciones")
    argentina = get(user.guild.roles, name="argentina")
    bolivia = get(user.guild.roles, name="bolivia")
    brasil = get(user.guild.roles, name="brasil")
    chile = get(user.guild.roles, name="chile")
    colombia = get(user.guild.roles, name="colombia")
    ecuador = get(user.guild.roles, name="ecuador")
    estados_unidos = get(user.guild.roles, name="estados unidos")
    mexico = get(user.guild.roles, name="méxico")
    paraguay = get(user.guild.roles, name="paraguay")
    peru = get(user.guild.roles, name="perú")
    venezuela = get(user.guild.roles, name="venezuela")
    spain = get(user.guild.roles, name="españa")
    francia = get(user.guild.roles, name="francia")
    korea = get(user.guild.roles, name="corea")
    supp = get(user.guild.roles, name="supp main")
    adc = get(user.guild.roles, name="adc main")
    top = get(user.guild.roles, name="top main")
    jg = get(user.guild.roles, name="jg main")
    mid = get(user.guild.roles, name="mid main")
    vip = get(user.guild.roles, name="vip")

    if payload.channel_id == 857388217946210314:
      if payload.emoji.name =='🛎':
        await user.add_roles(campanita)
      elif payload.emoji.name == '🇦🇷':
        await user.add_roles(argentina)
      elif payload.emoji.name == '🇧🇴':
        await user.add_roles(bolivia)
      elif payload.emoji.name == '🇧🇷':
        await user.add_roles(brasil)
      elif payload.emoji.name == '🇨🇱':
        await user.add_roles(chile)
      elif payload.emoji.name == '🇨🇴':
        await user.add_roles(colombia)
      elif payload.emoji.name == '🇪🇨':
        await user.add_roles(ecuador)
      elif payload.emoji.name == '🇺🇸':
        await user.add_roles(estados_unidos)
      elif payload.emoji.name == '🇲🇽':
        await user.add_roles(mexico)
      elif payload.emoji.name == '🇵🇾':
        await user.add_roles(paraguay)
      elif payload.emoji.name == '🇵🇪':
        await user.add_roles(peru)
      elif payload.emoji.name == '🇻🇪':
        await user.add_roles(venezuela)
      elif payload.emoji.name == '🇪🇸':
        await user.add_roles(spain)
      elif payload.emoji.name == '🇫🇷':
        await user.add_roles(francia)
      elif payload.emoji.name == '🇰🇷':
        await user.add_roles(korea)
      elif payload.emoji.name == '🏥':
        await user.add_roles(supp)
      elif payload.emoji.name == '🗡':
        await user.add_roles(adc)
      elif payload.emoji.name == '🛡':
        await user.add_roles(top)
      elif payload.emoji.name == '🏕':
        await user.add_roles(jg)
      elif payload.emoji.name == '🧙':
        await user.add_roles(mid)
    elif payload.channel_id == 862069303141466144:
      if payload.emoji.name == '☑':
          await user.add_roles(vip)

def setup(bot):
  bot.add_cog(Roles(bot))


#Import statements
from datetime import datetime
from block import Block
from transaction import Transaction
from transaction_list import TransactionList
from util import *
import hashlib
import json

#Blockchain object declaration
class Blockchain:
    def __init__(self):
        self.genesis_block = self.create_genesis_block()
        self.blockchain = []
        self.hash_list = []
        self.pending_transactions = []
        self.failures = None
        self.transaction_list = TransactionList()
        self.transactions_verified = False
        self.blockchain.append(self.genesis_block)
        self.init_time = datetime.now()
        self.count = 0

    def create_genesis_block(self):
        #Ideally these parameters should be kept in .env file and changed automatically to improve
        #cryptographic strength of genesis block hashs
        transaction = Transaction(
            datetime.now(),
            "genesis_recipient",
            "genesis_sender",
            "0"
        )
        genesis_block = Block("genesis_block_message", "", 0)
        genesis_block.mine_block(1)
        return genesis_block

    #Adds a block object to the top of the blockchain
    #Requires verification that all transaction data is present
    #in object cache before adding to blockchain
    #Instantiates block object from transaction data and hash of top block

    def handle_transaction(self, transaction, bundling=False, difficulty=3):
        printPurple('[###] Handling Transaction')
        if bundling is True:
            self.transaction_list.add_transaction(transaction, bundling=True)
        else:
            self.transaction_list.add_transaction(transaction)

        # printRed('\n[DBG] DEBUG INFO\n')
        # printRed('-[DBG] Added transaction to list')
        # printRed('--[DBG] Top block hash: ', self.get_last_block().current_hash)
        block = Block(self.get_last_block().current_hash,
            self.transaction_list.compile_json_dump(), self.count)
        # printRed('--[DBG] Instantiated block w/ hash: ', block.current_hash)
        self.count += 1
        self.hash_list.append(block.current_hash)
        block.mine_block(3)
        self = block.verify_transaction_data(self, return_failures=True)
        self.blockchain.append(block)
        return self

    def extract_block(self, idx):
        return self.blockchain[idx]

    def pull_fresh_hashes(self):
        self.transaction_list.hash_list = self.hash_list

    def purge_block(self, idx):
        self.blockchain.pop(idx)

    def get_last_block(self):
        return self.blockchain[-1]

    def get_first_block(self):
        return self.blockchain[0]

    def get_next_block(self, idx):
        return self.blockchain[idx+1]

    def stage_transaction(self, transaction):
        self.pending_transactions.append(transaction)

    def print_blockchain_status(self):
        printYellow(self.transaction_list.compile_json_dump())
        if self.transactions_verified is True:
            printGreen('[$$$] Successfully mined blocks with all transactions verified')
        else:
            printRed('[!!!] Transaction verification failed at some point')


from gooey import Gooey, GooeyParser
import tkinter
from tkinter import filedialog
import soupaccessories
from bs4 import BeautifulSoup

def change_button_text(soup, old_button_text):
    buttons = soupaccessories.find_button_by_tag(soup, old_button_text)
    if (buttons.count>1):
        print("Somehow you've selected more than one button.")
    return buttons

def main():
    soup = soupaccessories.SoupHandler()
    change_buttons = []
    first_buttons = soup.first_buttons()
    second_buttons = soup.second_buttons()
    third_buttons = soup.third_buttons()
    if (len(first_buttons) > 0 and first_buttons[0].text.strip() == "Mechanical"):
        change_buttons = first_buttons
    elif(len(second_buttons) > 0 and second_buttons[0].text.strip() == "Mechanical"):
        change_buttons = second_buttons
    elif(len(third_buttons) > 0 and third_buttons[0].text.strip() == "Mechanical"):
        change_buttons = third_buttons

    new_button_names = []

    # print("Change button length is " + str(len(change_buttons)))
    new_button = soup.create_new_side_button("../FireProtection/FireProtection.html", change_buttons[-1]['id'],change_buttons[-1]['style'],
                                "Commissioning",button_class=change_buttons[-1]['class'])

    new_button2 = soup.create_new_side_button("../FireProtection/FireProtection.html", change_buttons[-1]['id'],change_buttons[-1]['style'],
                                "Commissioning",button_class=change_buttons[-1]['class'])

    change_buttons[2].insert_after(new_button2)

    new_button.insert_after(new_button2)

    soup.save_file()

if __name__ == '__main__':
    main()


import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns
seed = 13

np.random.seed(seed)
import os

os.listdir('/kaggle/input/bitsf312-lab1')
df = pd.read_csv("/kaggle/input/bitsf312-lab1/train.csv", sep=",")

df_test = pd.read_csv("/kaggle/input/bitsf312-lab1/test.csv", sep=",")

print(df.shape)

print(df_test.shape)
df.info()
df['Class'].value_counts()
df.Class.value_counts().plot(kind="pie", autopct='%.1f%%', figsize=(8,8))
df.head()
for key in df.keys():

 print(key, ': ', pd.unique(df[key]))
for key in df_test.keys():

 print(key, ': ', pd.unique(df_test[key]))
df.dtypes
df_test.dtypes
df = df.replace({'?': None})
df.head()
plt.scatter(df['Class'],df['Score'])
# plt.plot(df_test['Score'])

print(df_test[df_test['Score']>5000].head())
df = df.drop(['ID'], axis = 1)

df_test = df_test.drop(['ID'], axis=1)

df = df[df.Score < 50000]
null_columns = df.columns[df.isnull().any()]

null_columns
df.drop_duplicates(inplace = True)
print(df.isnull().sum())
for i in range(df.shape[0]):

    if df.iloc[i]['Number of Quantities']==None:

        print(df.iloc[i]['Number of Insignificant Quantities'])

print("-------")

for i in range(df.shape[0]):

    if df.iloc[i]['Number of Insignificant Quantities']==None:

        print(df.iloc[i]['Number of Quantities'])
for i in range(df.shape[0]):

    if df.iloc[i]['Number of Special Characters']==None:

        print(df.iloc[i]['Number of Sentences'])

print("-------")

for i in range(df.shape[0]):

    if df.iloc[i]['Number of Sentences']==None:

        print(df.iloc[i]['Number of Special Characters'])

print("-------")
df = df.drop(['Number of Special Characters'], axis = 1)

df_test = df_test.drop(['Number of Special Characters'], axis=1)
df['Number of Quantities'] = df['Number of Quantities'].replace({None: 2})
df['Number of Insignificant Quantities'] = df['Number of Insignificant Quantities'].replace({None: '0'})
print(df.isnull().sum())
df = df.drop('Number of Quantities', axis=1)

df_test = df_test.drop('Number of Quantities', axis=1)

# df.dropna(inplace=True)
print(df.info())
print(df_test.info())
for key in df.keys():

 print(key, ': ', pd.unique(df[key]))
# Modify data encoding

# using function

df['Size'] = df['Size'].map({ 'Small': 0,'Medium': 1, 'Big':2, None:1})

df_test['Size'] = df_test['Size'].map({ 'Small': 0,'Medium': 1, 'Big':2, None:1})
# df = pd.get_dummies(df, columns=['Size'],prefix= ['Size'])

# df_test = pd.get_dummies(df_test, columns=['Size'],prefix= ['Size'])
df['Size']

df.dropna(inplace=True)
print(df.isnull().sum())
df.head()
df.dtypes
df['Number of Insignificant Quantities'] = df['Number of Insignificant Quantities'].astype(int)

df['Total Number of Words'] = df['Total Number of Words'].astype(int)

df['Difficulty'] = df['Difficulty'].astype(float)

# df['Size'] = df['Size'].astype(int)
df['Chars per Class'] = df['Total Number of Characters']/ df['Total Number of Words']

df_test['Chars per Class'] = df_test['Total Number of Characters']/ df_test['Total Number of Words']

df = df.drop('Total Number of Words', axis=1)

df_test = df_test.drop('Total Number of Words', axis=1)

df['New2'] = df['Total Number of Characters'] / df['Number of Sentences']

df_test['New2'] = df_test['Total Number of Characters'] / df_test['Number of Sentences']

# df = df.drop('Number of Sentences', axis=1)

# df_test = df_test.drop('Number of Sentences', axis=1)

# df['New2'] = df['Second Index'] / df['Total Number of Characters']

# df['New3'] = df['First Index'] / df['Total Number of Characters']

# df_test['New2'] = df_test['Second Index'] / df_test['Total Number of Characters']

# df_test['New3'] = df_test['First Index'] / df_test['Total Number of Characters']

# df['FS Ratio'] = df['First Index']/ df['Second Index']

# df_test['FS Ratio'] = df_test['First Index']/ df_test['Second Index']

# df = df.drop('First Index', axis=1)

# df_test = df_test.drop('First Index', axis=1)

# df = df.drop('Second Index', axis=1)

# df_test = df_test.drop('Second Index', axis=1)

# df = df.drop('Total Number of Characters', axis=1)

# df_test = df_test.drop('Total Number of Characters', axis=1)
df.shape
import seaborn as sns

corr = df.corr()

fig, ax = plt.subplots(figsize=(10,10))

sns.heatmap(corr, annot=True)
# for i in range(df.shape[0]):

#     print(i, ' ', df.iloc[i]['First Index'], ' ', df.iloc[i]['Second Index'])
print(df['Class'].value_counts())

df.Class.value_counts().plot(kind="pie", autopct='%.1f%%', figsize=(8,8))
# import matplotlib

# x = df['Class']

# y = df['Number of Sentences']

# label = df['Class']

# colors = ['red','green','blue','purple','black','yellow']

# fig = plt.figure(figsize=(20,8))

# plt.scatter(x, y, c=label, cmap=matplotlib.colors.ListedColormap(colors))

# cb = plt.colorbar()

# loc = np.arange(0,max(label),max(label)/float(len(colors)))

# cb.set_ticks(loc)

# cb.set_ticklabels(colors)
# x = df['Class']

# y = df['New2']

# label = df['Class']

# colors = ['red','green','blue','purple','black','yellow']

# fig = plt.figure(figsize=(20,8))

# plt.scatter(x, y, c=label, cmap=matplotlib.colors.ListedColormap(colors))

# cb = plt.colorbar()

# loc = np.arange(0,max(label),max(label)/float(len(colors)))

# cb.set_ticks(loc)

# cb.set_ticklabels(colors)
# x = df['Class']

# y = df['New3']

# label = df['Class']

# colors = ['red','green','blue','purple','black','yellow']

# fig = plt.figure(figsize=(20,8))

# plt.scatter(x, y, c=label, cmap=matplotlib.colors.ListedColormap(colors))

# cb = plt.colorbar()

# loc = np.arange(0,max(label),max(label)/float(len(colors)))

# cb.set_ticks(loc)

# cb.set_ticklabels(colors)
# x = df['Class']

# y = df['First Index']

# label = df['Class']

# colors = ['red','green','blue','purple','black','yellow']

# fig = plt.figure(figsize=(20,8))

# plt.scatter(x, y, c=label, cmap=matplotlib.colors.ListedColormap(colors))

# cb = plt.colorbar()

# loc = np.arange(0,max(label),max(label)/float(len(colors)))

# cb.set_ticks(loc)

# cb.set_ticklabels(colors)
# x = df['Class']

# y = df['Total Number of Characters'] / df['Number of Sentences']

# label = df['Class']

# colors = ['red','green','blue','purple','black','yellow']

# fig = plt.figure(figsize=(20,8))

# plt.scatter(x, y, c=label, cmap=matplotlib.colors.ListedColormap(colors))

# cb = plt.colorbar()

# loc = np.arange(0,max(label),max(label)/float(len(colors)))

# cb.set_ticks(loc)

# cb.set_ticklabels(colors)
# df['New 1'] = df['First Index'] / df['Second Index']

# df['New2'] = df['Second Index'] / df['Total Number of Characters']

# df['New3'] = df['First Index'] / df['Total Number of Characters']

# df.groupby("Class").mean()[['New2', 'New3', 'Total Number of Characters']]
# for key in df.keys():

#  print(key, ': ', pd.unique(df[key]))
# import matplotlib

# x = df['Second Index']

# y = df['First Index']

# label = df['Class']

# colors = ['red','green','blue','pink','black','yellow']

# fig = plt.figure(figsize=(20,8))

# plt.scatter(x, y, c=label, cmap=matplotlib.colors.ListedColormap(colors))

# cb = plt.colorbar()

# loc = np.arange(0,max(label),max(label)/float(len(colors)))

# cb.set_ticks(loc)

# cb.set_ticklabels(colors)
# import matplotlib

# x = df['Second Index'] / df['First Index']

# y = df['Total Number of Characters']

# label = df['Class']

# colors = ['red','green','blue','purple','black','yellow']

# fig = plt.figure(figsize=(20,8))

# plt.scatter(x, y, c=label, cmap=matplotlib.colors.ListedColormap(colors))

# cb = plt.colorbar()

# loc = np.arange(0,max(label),max(label)/float(len(colors)))

# cb.set_ticks(loc)

# cb.set_ticklabels(colors)
df_onehot = df.copy()

df_onehot = pd.get_dummies(df_onehot, columns=['Class'], prefix = ['Class'])

df_onehot.head()
# for i in range(df.shape[0]):

#     if df.iloc[i]['Number of Quantities']==3:

#         print(df.iloc[i])

#         print("------")
df_onehot.dropna(inplace=True)

df_onehot.drop_duplicates(inplace=True)
y = df_onehot[['Class_0', 'Class_1', 'Class_2', 'Class_3', 'Class_4', 'Class_5']]

X = df_onehot.drop(['Class_0', 'Class_1', 'Class_2', 'Class_3', 'Class_4', 'Class_5'], axis = 1)

X.head()
X_test = df_test
X_test.head()
print(X.shape)

print(y.shape)

print(X_test.shape)
from sklearn.model_selection import train_test_split

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.2, random_state = seed)
X_train = X_train.reset_index().drop(['index'], axis = 1)

X_val = X_val.reset_index().drop(['index'], axis = 1)

X_test = X_test.reset_index().drop(['index'], axis = 1)
numerical_columns = [

                     'First Index',

                     'Second Index',

                        'Number of Insignificant Quantities',

#                      'Total Number of Words',

                     'Chars per Class',

#                      'FS Ratio',

                    'New2',

#                     'New3',

                     'Total Number of Characters',

                     'Number of Sentences',

                     'Difficulty',

                    'Size',

                     'Score']

numerical_df_train = pd.DataFrame(X_train[numerical_columns])

rest_df_train = X_train.drop(numerical_columns, axis = 1)

numerical_df_val = pd.DataFrame(X_val[numerical_columns])

rest_df_val = X_val.drop(numerical_columns, axis = 1)

numerical_df_test = pd.DataFrame(X_test[numerical_columns])

rest_df_test = X_test.drop(numerical_columns, axis = 1)

print(numerical_df_train.shape)

print(rest_df_train.shape)

from sklearn.preprocessing import MinMaxScaler

from sklearn.preprocessing import StandardScaler

from sklearn.preprocessing import RobustScaler

# scaler = StandardScaler()

# scaler = MinMaxScaler()

scaler = RobustScaler()

numerical_df_train_scaled = pd.DataFrame(scaler.fit_transform(numerical_df_train),columns=numerical_columns)

# scaler = StandardScaler()

numerical_df_val_scaled = pd.DataFrame(scaler.fit_transform(numerical_df_val),columns=numerical_columns)

numerical_df_test_scaled = pd.DataFrame(scaler.fit_transform(numerical_df_test),columns=numerical_columns)

print(numerical_df_train.shape)

print(rest_df_train.shape)

X_train = pd.concat([numerical_df_train_scaled.reset_index(drop=True), rest_df_train.reset_index(drop=True)], axis=1)

X_train.reset_index()

X_val = pd.concat([numerical_df_val_scaled.reset_index(drop=True), rest_df_val.reset_index(drop=True)], axis=1)

X_val.reset_index()

X_test = pd.concat([numerical_df_test_scaled.reset_index(drop=True), rest_df_test.reset_index(drop=True)], axis=1)

X_test.reset_index()
X_train = X_train.reset_index().drop(['index'], axis = 1)

X_val = X_val.reset_index().drop(['index'], axis = 1)

X_test = X_test.reset_index().drop(['index'], axis = 1)
X_test.head()
X_train.head()
print(X_train.shape)

print(y_train.shape)

print(X_test.shape)
from keras.models import Sequential

from keras.layers import Dense, Dropout

from keras.optimizers import Adam

from keras import regularizers
y_train = y_train.reset_index().drop(['index'], axis = 1)

y_val = y_val.reset_index().drop(['index'], axis = 1)
y_train.head()
y_val.head()
import keras

keras.backend.clear_session()

# import regularizer

from keras.regularizers import l1_l2

model = Sequential()

model.add(Dense(20, input_dim=10, activation='relu', kernel_regularizer=l1_l2(0.001)))

model.add(Dropout(0.2, seed=seed))

model.add(Dense(40, activation='relu', kernel_regularizer=l1_l2(0.001)))

model.add(Dropout(0.2, seed=seed))

model.add(Dense(10, activation='relu', kernel_regularizer=l1_l2(0.001)))

# model.add(Dropout(0.2, seed=seed))

# model.add(Dense(25, activation='relu', kernel_regularizer=l1_l2(0.001)))

model.add(Dropout(0.1, seed=seed))

model.add(Dense(6, activation='softmax'))
# Set callback functions to early stop training and save the best model so far

# from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau

# callbacks = [EarlyStopping(monitor='val_accuracy', patience=5)]

# reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=4, min_lr=0.0001)
optimizer = Adam(lr=0.001)

model.compile(optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
history = model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=500, batch_size=2)
# plot history

plt.plot(history.history['accuracy'], label='train')

plt.plot(history.history['val_accuracy'], label='test')

plt.legend()

plt.show()
plt.plot(history.history['loss'], label='train_loss')

plt.plot(history.history['val_loss'], label='test_loss')

plt.legend()

plt.show()
# model.save_weights("model-v5.h5")

# model.load_weights("model-v5.h5")
y_prob = model.predict(X_test)

y_classes = y_prob.argmax(axis=-1)

print(y_classes)
df_submission = pd.read_csv('/kaggle/input/bitsf312-lab1/sample_submission.csv', sep=',')
df_submission.head()
for i in range(df_submission.shape[0]):

    df_submission.iloc[i]['Class'] = y_classes[i]
df_submission.head()
df_submission[df_submission['ID']==398]
print(df_submission['Class'].value_counts())

df_submission.Class.value_counts().plot(kind="pie", autopct='%.1f%%', figsize=(5,5))
from IPython.display import HTML

import pandas as pd

import numpy as np

import base64

def create_download_link(df, title = "Download CSV file", filename = "data.csv"):

    csv = df.to_csv(index=False)

    b64 = base64.b64encode(csv.encode())

    payload = b64.decode()

    html = '{title}'

    html = html.format(payload=payload,title=title,filename=filename)

    return HTML(html)

create_download_link(df_submission)