from django import forms
from django.db import transaction
from django.utils.translation import gettext as _

from ..models import Duplicate, HelperShift

class MergeDuplicatesForm(forms.Form):
    def __init__(self, *args, **kwargs):
        self.helpers = kwargs.pop("helpers")

        super(MergeDuplicatesForm, self).__init__(*args, **kwargs)

        self.fields["helpers_ignore"] = forms.ModelMultipleChoiceField(
            queryset=self.helpers,
            widget=forms.CheckboxSelectMultiple(attrs={"id": "helper_ignore"}),
            required=False,
            label="",
        )

        self.fields["helpers_selection"] = forms.ModelChoiceField(
            queryset=self.helpers,
            widget=forms.RadioSelect(attrs={"id": "helper_selection"}),
            empty_label=None,
            required=True,
            label="",
        )

    def clean(self):
        cleaned_data = super().clean()
        remaining_helper = cleaned_data["helpers_selection"]
        ignore_helpers = cleaned_data["helpers_ignore"]

        # remaining helpers must not be ignored (this makes no sense)
        if remaining_helper in ignore_helpers:
            raise forms.ValidationError(_("The remaining helper must not be ignored."))

        # check for overlapping shifts
        if not self.check_merge_possible(ignore_helpers):
            raise forms.ValidationError(_("Cannot merge helpers which have the same shift."))

    @transaction.atomic
    def merge(self):
        """
        Merge the helpers and keep the data selected in the form.
        """
        remaining_helper = self.cleaned_data["helpers_selection"]
        ignore_helpers = self.cleaned_data["helpers_ignore"]
        oldest_timestamp = remaining_helper.timestamp

        # we check this again inside the atomic code block to ensure that no change happends after the
        # validation and before the merge (= no new shifts were added)
        if not self.check_merge_possible(ignore_helpers):
            raise ValueError("Cannot merge helpers with same shifts")

        # and then to the merge
        for helper in self.helpers:
            if helper == remaining_helper or helper in ignore_helpers:
                continue
            # merge shifts
            for helpershift in HelperShift.objects.filter(helper=helper):
                helpershift.helper = remaining_helper
                helpershift.save()

            # merged coordinated jobs
            for job in helper.coordinated_jobs:
                job.coordinators.add(remaining_helper)

            # merge gifts
            if remaining_helper.event.gifts:
                remaining_helper.gifts.merge(helper.gifts)

            # then create the duplicate entry so that old links in mails still work
            Duplicate.objects.create(deleted=helper.id, existing=remaining_helper)

            # the overall timestamp of the helper should be the oldest one
            # (there are multiple timestamps saved: one per helper and one per shift)
            if helper.timestamp < oldest_timestamp:
                oldest_timestamp = helper.timestamp

            # and delete the old helper
            helper.delete()

        # update timestamp
        remaining_helper.timestamp = oldest_timestamp
        remaining_helper.save()

        return remaining_helper

    def check_merge_possible(self, ignore_helpers=None):
        """
        Check if the merge is possible.

        It is not possible when multiple helpers have the same shift. If we would merge those helpers,
        we would "loose" allocated seats and this is probably not intended.
        """
        shifts = []
        for helper in self.helpers:
            # if we have ignored_helpers, check that
            if ignore_helpers and helper in ignore_helpers:
                continue

            # compare all shifts
            for shift in helper.shifts.all():
                if shift in shifts:
                    return False
                else:
                    shifts.append(shift)
        return True


from django.test import TestCase
from users.forms import UserUpdateForm

class TestForms(TestCase):

    def test_update_form_valid_data(self):
        """Test for valid update form"""

        form = UserUpdateForm(data={
            'username': 'Praveen',
            'email': 'Praveen.t@gmail.com'
        })

        self.assertTrue(form.is_valid())

    def test_update_form_valid_data(self):
        """Test for invalid update form"""

        form = UserUpdateForm(data={ })

        self.assertFalse(form.is_valid())
        self.assertEquals(len(form.errors), 1)


import os
import sys
import math
from tqdm import tqdm
import pandas as pd
import numpy as np

sys.path.insert(1, os.path.join(sys.path[0], '..'))
from util import argparser

def permutation_test(df, column, n_permutations=100000, batch_size=1000):
    # Get actual batch size
    batch_size = min(batch_size, n_permutations)

    # Get real avg
    real_avg = df[column].mean().item()
    values = df[column].values
    values_exp = np.expand_dims(values, axis=1).repeat(batch_size, axis=1)

    # Make n_permutations divisible per batch_size
    n_batches = math.ceil(n_permutations / batch_size)
    n_permutations = n_batches * batch_size

    # Get number of permutations above
    n = 0
    for _ in range(n_batches):
        permut = np.random.randint(0, 2, size=(len(values), batch_size)) * 2 - 1

        random_avgs = np.mean(values_exp * permut, axis=0)
        n += (random_avgs >= real_avg).sum()

    return n / n_permutations, n_permutations

def remove_unused_cols(df):
    del df['item_id']
    del df['position']
    del df['family_size']
    del df['family_weight']
    del df['length']

    return df

def get_macroarea_counts(df):
    df_count = df[['macroarea', 'concept_id', 'token_idx']].groupby(['concept_id', 'token_idx']).agg('count').reset_index()
    df_count['macroarea_count'] = df_count['macroarea']
    del df_count['macroarea']
    df = pd.merge(df, df_count, left_on=['concept_id', 'token_idx'], right_on=['concept_id', 'token_idx'])

    return df

def get_tokens_means(df):
    df_new = df.groupby(['language_id', 'family', 'macroarea', 'concept_id', 'concept_name', 'token', 'token_idx']).agg('mean').reset_index()
    df_new = df_new.groupby(['family', 'macroarea', 'concept_id', 'concept_name', 'token', 'token_idx']).agg('mean').reset_index()
    df_new = df_new.groupby(['macroarea', 'concept_id', 'concept_name', 'token', 'token_idx']).agg('mean').reset_index()

    df_new = get_macroarea_counts(df_new)
    return df_new

def main():
    args = argparser.parse_args(csv_folder='cv')
    context = 'onehot'

    fname = os.path.join(args.rfolder_base, 'avg_seed_results_per_pos.tsv')
    df = pd.read_csv(fname, sep='\t')
    remove_unused_cols(df)

    # df_concepts = df.groupby(['concept_id', 'concept_name']).agg('mean').reset_index()
    df_tokens = get_tokens_means(df)
    df_tokens.set_index(['macroarea', 'concept_id', 'token_idx'], inplace=True)
    df_tokens = df_tokens.sort_index()
    df_tokens = df_tokens[df_tokens.macroarea_count == 4]

    df_tokens['p_value'] = -1
    df_tokens['n_permutations'] = -1
    df_tokens['n_instances'] = -1
    for macroarea, concept_id, token_idx in tqdm(df_tokens.index.unique(), desc='Concept--token permutation tests'):
        idx = (macroarea, concept_id, token_idx)
        df_temp = df[(df.macroarea == macroarea) & (df.concept_id == concept_id) & (df.token_idx == token_idx)]
        p_val, n_permutations = permutation_test(df_temp, 'mi-' + context, n_permutations=100000)
        # p_val, n_permutations = permutation_test_recursive(df_temp, 'mi-' + context, n_permutations=100000)

        df_tokens.loc[idx, 'p_value'] = p_val
        df_tokens.loc[idx, 'n_permutations'] = n_permutations
        df_tokens.loc[idx, 'n_instances'] = df_temp.shape[0]

    fname = os.path.join(args.rfolder_base, 'tokens_results.tsv')
    df_tokens.to_csv(fname, sep='\t')

if __name__ == '__main__':
    main()


import asyncio
import json
import logging
import logging.config
from dataclasses import dataclass

import yaml
from web3 import Web3
from web3._utils.filters import LogFilter

@dataclass
class FilterWrapper:
    event_filter: LogFilter
    pair_name: str
    oracle_address: str
    logger: logging.Logger

class BlockchainMonitor:

    def __init__(self):
        with open("spec.yaml", "r") as s:
            spec = yaml.safe_load(s)

        with open("logging.yaml", "r") as s:
            logging.config.dictConfig(yaml.safe_load(s))

        with open('abi.json', "r") as s:
            abi = json.load(s)

        w3 = Web3(Web3.HTTPProvider(spec['connection_settings']['alchemy_url']))

        self.filters = []
        pair_name_to_logger = {}

        for currency_pair in spec['currency_pairs']:
            for pair_name, pair_spec in currency_pair.items():
                for oracle_address in pair_spec['oracles_addresses']:
                    contract = w3.eth.contract(address=oracle_address, abi=abi)
                    pair_name_to_logger[pair_name] = pair_name_to_logger \
                        .get(pair_name, logging.getLogger(pair_name))
                    self.filters.append(FilterWrapper(
                        contract.events.AnswerUpdated.createFilter(fromBlock='latest'),
                        pair_name,
                        oracle_address,
                        pair_name_to_logger[pair_name]
                    ))

    @staticmethod
    def __handle_event(event, filter_wrapper):
        logger = filter_wrapper.logger
        logger.info("Price changes in pair {}. Oracle address: {}. Current price: {}, block number: {}, tx hash: {}"
                    .format(filter_wrapper.pair_name,
                            filter_wrapper.oracle_address,
                            event.args.current,
                            event.blockNumber,
                            event.transactionHash.hex()))

    async def __monitor(self, filter_wrapper, poll_interval):
        filter_wrapper.logger.info("Start monitor pair {}. Oracle address: {}".format(
            filter_wrapper.pair_name, filter_wrapper.oracle_address))
        while True:
            for AnswerUpdated in filter_wrapper.event_filter.get_new_entries():
                self.__handle_event(AnswerUpdated, filter_wrapper)
            await asyncio.sleep(poll_interval)

    def monitor(self):
        loop = asyncio.get_event_loop()
        try:
            for filter_wrapper in self.filters:
                asyncio.ensure_future(self.__monitor(filter_wrapper, 10))
            loop.run_forever()
        finally:
            loop.close()

if __name__ == "__main__":
    BlockchainMonitor().monitor()


import pandas as pd
import sqlite3

def connect_sqlite(db_file):
    with sqlite3.connect(db_file) as conn:
        conn.row_factory = sqlite3.Row
        cur = conn.cursor()
    return conn, cur

def get_dataframe(db_file, sql):
    conn, cur = connect_sqlite(db_file)
    df = pd.read_sql(sql,conn)
    if conn:
        conn.close()

    return df

def req_count(df):
    '''
        df : pandas dataframe
    '''
    df['datatime']

if __name__ == '__main__' :
    db_file = 'demklog_2017-01-21'
    sql = 'select * from demklog '
    df = get_dataframe(db_file, sql)
    #print df.head()

    df.plot()
#     df_tm = pd.TimeSeries( pd. to_datetime( df['timestamp'] ) )
#     print 'type df_tm=', type(df_tm),df_tm.head()

#     ddtest =  df['timestamp']
#
# #    dd1.resample('M')
#     print type(ddtest), ddtest [800:810]
#     print ddtest.resample('H')


"""
The `test.unit.sha_api.mybottle.sha_api_bottle_test` module provides unit tests for the `ShaApiBottle` class in
`sha_api.mybottle.sha_api_bottle`.

Classes:
    TestShaApiBottle: A unit test class for the `ShaApiBottle` class.
"""

import json
import tempfile
import unittest

from bottle import ConfigDict # pylint: disable=no-name-in-module
from mock import MagicMock, patch
from sha_api.mybottle.sha_api_bottle import global_config, ShaApiBottle
from sha_api.sha_apid import ROUTES

import sha_api

class TestShaApiBottle(unittest.TestCase):
    """
    A unit test class for the `ShaApiBottle` class.

    Methods:
        setUp: Unit test initialization.
        test_error_handler: Tests to ensure the error handler returns a JSON value.
        test_global_config: Tests to ensure we get the correct values from a `ConfigDict` instance.
        test_sha_api_constructor: Tests that `ShaApiBottle` instance can be properly instantiated without throwing any
                                  exceptions.
    """

    def setUp(self):
        """
        Initializes the unit test global configs
        """

        self.maxDiff = None # pylint: disable=invalid-name
        self.config_sample = tempfile.NamedTemporaryFile(delete=False)
        self.dbfile = tempfile.NamedTemporaryFile(delete=False)
        self.os_environ = {u'SHA_API_CONFIG': self.config_sample.name}

        self.configdict_ns = ConfigDict().load_dict(
            {
                u'sha_api': {
                    u'test_variable': u'test_value'
                },
                u'sqlite': {
                    u'dbfile': self.dbfile.name
                }
            }
        )

        with open(self.config_sample.name, 'w') as fout:
            fout.write(u"[sha_api]\ntest_variable = test_value\n[sqlite]\ndbfile = %s" % self.dbfile.name)

    def test_error_handler(self):
        """
        Tests to ensure the error handler returns a JSON value.
        """

        res = MagicMock()
        res_json = json.dumps({u'err_msg': u'Response Body'})

        res.body = u'Response Body'

        api = ShaApiBottle(ROUTES)

        self.assertEqual(api.default_error_handler(res), res_json)

    def test_global_config(self):
        """
        Tests to ensure we get the correct values from a `ConfigDict` instance.
        """

        self.assertEqual(global_config(self.configdict_ns, u'sqlite', u'dbfile', u'Not Found'), self.dbfile.name)

    def test_sha_api_constructor(self):
        """
        Tests that `ShaApiBottle` instance can be properly instantiated without throwing any exceptions.
        """

        # Branch 1: Nothing throws an error and all goes well
        try:
            api = ShaApiBottle(ROUTES) # pylint: disable=unused-variable
        except Exception as err: # pylint: disable=broad-except
            self.fail(u'ShaApiBottle sha_api instance failed to initialize: %s' % str(err))

        # Branch 2: When routes object is not a list we get a proper assert error

        with self.assertRaises(AssertionError) as err:
            api = ShaApiBottle(dict())

        self.assertEqual(str(err.exception), u'routes must be an array of route dicts to be passed to bottle.route')

        # Branch 3: When routes object is a list but it doesnt contain dict items we get a proper assert error

        with self.assertRaises(AssertionError) as err:
            api = ShaApiBottle([False])

        self.assertEqual(str(err.exception), u'route must be a dict that can be passed to bottle.route')

        # Branch 4: When environment variable specifies config file it is properly loaded.

        with patch.dict(u'sha_api.mybottle.sha_api_bottle.os.environ', values=self.os_environ):
            self.assertEqual(self.os_environ[u'SHA_API_CONFIG'],
                             sha_api.mybottle.sha_api_bottle.os.environ.get(u'SHA_API_CONFIG'))

            api = ShaApiBottle(ROUTES)

            self.assertEqual(api.config.get(u'sha_api.test_variable'), u'test_value')
            self.assertEqual(api.config.get(u'sqlite.dbfile'), self.dbfile.name)

        # Branch 5: When any portion of the db initialization fails it should just bubble up the exception

        with patch(u'sha_api.mybottle.sha_api_bottle.sqlite3.connect') as sqlite_connect:
            self.assertEqual(sqlite_connect, sha_api.mybottle.sha_api_bottle.sqlite3.connect)

            sqlite_connect.side_effect = Exception(u'sqlite exception')

            with self.assertRaises(Exception) as err:
                api = ShaApiBottle()

            self.assertEqual(str(err.exception), u'sqlite exception')


# --------------
#Importing header files
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
#Path of the file
path

#Code starts here

#Reading the file
data=pd.read_csv(path)

#Renaming a column
data.rename(columns={'Total':'Total_Medals'},inplace=True)

#Printing the first five columns
print(data.head(10))

#Code ends here

# --------------
#Code starts here
#Creating new column 'Better_Event'
data['Better_Event'] = np.where(data['Total_Summer']>data['Total_Winter'], 'Summer', 'Winter')
data['Better_Event'] =np.where(data['Total_Summer'] ==data['Total_Winter'],'Both',data['Better_Event'])

#Finding the value with max count in 'Better_Event' column
better_event=data['Better_Event'].value_counts().index.values[0]

#Printing the better event
print('Better_Event=', better_event)

# --------------
#Code starts here

#Subsetting the dataframe
top_countries=data[['Country_Name','Total_Summer', 'Total_Winter','Total_Medals']]

#Dropping the last column
top_countries=top_countries[:-1]

#Function for top 10
def top_ten(data, col):

    #Creating a new list
    country_list=[]

    #Finding the top 10 values of 'col' column
    country_list= list((data.nlargest(10,col)['Country_Name']))

    #Returning the top 10 list
    return country_list

#Calling the function for Top 10 in Summer
top_10_summer=top_ten(top_countries,'Total_Summer')
print("Top 10 Summer:\n",top_10_summer, "\n")

#Calling the function for Top 10 in Winter
top_10_winter=top_ten(top_countries,'Total_Winter')
print("Top 10 Winter:\n",top_10_winter, "\n")

#Calling the function for Top 10 in both the events
top_10=top_ten(top_countries,'Total_Medals')
print("Top 10:\n",top_10, "\n")

#Extracting common country names from all three lists
common=list(set(top_10_summer) & set(top_10_winter) & set(top_10))

print('Common Countries :\n', common, "\n")

#Code ends here

# --------------
#Code starts here
top_10_summer=top_ten(top_countries,'Total_Summer')
top_10_winter=top_ten(top_countries,'Total_Winter')
top_10=top_ten(top_countries,'Total_Medals')
df=pd.DataFrame([top_10_summer,top_10_winter,top_10])

summer_df= data[data['Country_Name'].isin(top_10_summer)]
winter_df= data[data['Country_Name'].isin(top_10_winter)]
top_df= data[data['Country_Name'].isin(top_10)]
summer_df.plot.bar('Country_Name','Total_Summer')
winter_df.plot.bar('Country_Name','Total_Winter')
top_df.plot.bar('Country_Name','Total_Medals')

# --------------
#Code starts here
summer_df= data[data['Country_Name'].isin(top_10_summer)]
summer_df['Golden_Ratio']=data['Gold_Summer']/data['Total_Summer']
summer_max_ratio=summer_df['Golden_Ratio'].max()
summer_country_gold=summer_df.loc[summer_df['Golden_Ratio'].idxmax(),'Country_Name']
winter_df= data[data['Country_Name'].isin(top_10_winter)]
winter_df['Golden_Ratio']=data['Gold_Winter']/data['Total_Winter']
winter_max_ratio=winter_df['Golden_Ratio'].max()
winter_country_gold=winter_df.loc[winter_df['Golden_Ratio'].idxmax(),'Country_Name']
top_df= data[data['Country_Name'].isin(top_10)]
top_df['Golden_Ratio']=data['Gold_Total']/data['Total_Medals']
top_max_ratio=top_df['Golden_Ratio'].max()
top_country_gold=summer_df.loc[top_df['Golden_Ratio'].idxmax(),'Country_Name']

# --------------
#Code starts here
data_1=data[:-1]
data_1['Total_Points']=(data['Gold_Total']*3)+(data['Silver_Total']*2)+(data['Bronze_Total'])
most_points=max(data_1['Total_Points'])
best_country=data_1.loc[data_1['Total_Points'].idxmax(),'Country_Name']
print(best_country)

# --------------
#Code starts here

#Subsetting the dataframe
best=data[data['Country_Name']==best_country]
best.reset_index(drop = True, inplace = True)
best=best[['Gold_Total','Silver_Total','Bronze_Total']]

#Plotting bar plot
best.plot.bar(stacked=True)

#Changing the x-axis label
plt.xlabel('United States')

#Changing the y-axis label
plt.ylabel('Medals Tally')

#Rotating the ticks of X-axis
plt.xticks(rotation=45)

#Updating the graph legend
l=plt.legend()
l.get_texts()[0].set_text('Gold_Total :' + str(best['Gold_Total'].values))
l.get_texts()[1].set_text('Silver_Total :' + str(best['Silver_Total'].values))
l.get_texts()[2].set_text('Bronze_Total :' + str(best['Bronze_Total'].values))

#Code ends here


import requests
import json
import urllib.parse
from django.conf import settings

def current_place():
    """
    現在地の緯度経度を取得する。

    Returns:
        int: 現在地の緯度、経度
    """
    geo_request_url = "https://get.geojs.io/v1/ip/geo.json"
    geo_data = requests.get(geo_request_url).json()
    # print(geo_data['latitude'])
    # print(geo_data['longitude'])
    return geo_data["latitude"], geo_data["longitude"]

def get_movie_theatre(latitude, longitude):
    """
    現在地の緯度、経度をインプットとして、付近の映画館リストを返す。

    Args:
        latitude (int): 現在地の緯度
        longitude (int): 現在地の経度
    """

    url = "https://maps.googleapis.com/maps/api/place/nearbysearch/json?language=ja&location=" + latitude + "," + longitude + "&radius=2000&type=movie_theater&key=" + settings.API_MAP
    # print(url)
    payload={}
    headers = {}

    response = requests.request("GET", url, headers=headers, data=payload)

    json_dict = response.json()

    movie_theatre = []

    if json_dict["status"] == "OK":
        for theatre in json_dict["results"]:
            movie_theatre.append((theatre["name"],theatre["name"]))
        print(movie_theatre)

    return movie_theatre

def search_theatre(search_text):
    """
    現在地の緯度、経度をインプットとして、付近の映画館リストを返す。

    Args:
        latitude (int): 現在地の緯度
        longitude (int): 現在地の経度
    """

    url = "https://maps.googleapis.com/maps/api/place/textsearch/json?language=ja&type=movie_theater&query=" + urllib.parse.quote(search_text) + "&key=" + settings.API_MAP
    # print(url)
    payload={}
    headers = {}

    response = requests.request("GET", url, headers=headers, data=payload)

    json_dict = response.json()

    movie_theatre = []

    if json_dict["status"] == "OK":
        for theatre in json_dict["results"]:
            movie_theatre.append((theatre["name"], theatre["name"]))
        # print(movie_theatre)
    # else:
    #     movie_theatre.append("Result nothing","Result nothing")

    return movie_theatre

if __name__ == "__main__":
    latitude, longitude = current_place()
    get_movie_theatre(latitude, longitude)
    search_theatre("TOHO 新宿")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import datetime

class Migration(migrations.Migration):

    dependencies = [
        ('article', '0002_remove_article_article_date'),
    ]

    operations = [
        migrations.AddField(
            model_name='article',
            name='article_date',
            field=models.DateTimeField(default=datetime.date(2014, 10, 21)),
            preserve_default=False,
        ),
    ]


import itertools
import tqdm
import subprocess

def add_to_seq(seq, feature, label):
    if label[0] == 'B':
        seq.append(feature)
    elif label[0] == 'I':
        if len(seq) > 0:
            seq[-1] += feature
        else:
            seq.append(feature)
    elif label[0] == 'S':
        seq.append(feature)

def cal(predict_seq, right_seq):
    r"""
    :return: 分词正确的个数，错误个数，总单词个数，该句子是否分词正确
    """
    # flag 表示全部正确
    flag = 1
    num_right_word = 0

    if len(predict_seq) != len(right_seq):
        flag = 0

    for feat in predict_seq:
        if feat in right_seq:
            num_right_word += 1
        else:
            flag = 0

    return num_right_word, len(predict_seq) - num_right_word, len(right_seq), flag

def evaluate(model, dataset_loader, idx2feature, idx2label, device, log_file):
    r"""
    计算数据集上的F1, P, R值

    :return: F1, accuracy
    """
    model.eval()

    num_sentence = 0
    num_right_sentence = 0

    num_all_word = 0
    num_right_word = 0
    num_error_word = 0

    for _, batch in tqdm.tqdm(enumerate(itertools.chain.from_iterable(dataset_loader))):
        batch = tuple(t.to(device) for t in batch)
        features, labels, masks = batch

        scores, paths = model(features, masks)

        num_sentence += features.shape[0]
        length = features.shape[1]
        for i, (sentence, label) in enumerate(zip(features, labels)):
            predict_seq = []
            right_seq = []
            for j, tensor_feat in enumerate(sentence):
                if j == length or masks[i][j] == 0:
                    # 会有一个<eof>标志
                    nums = cal(predict_seq, right_seq)

                    num_right_word += nums[0]
                    num_error_word += nums[1]
                    num_all_word += nums[2]
                    num_right_sentence += nums[3]

                    break
                else:
                    feature = idx2feature[tensor_feat.item()]
                    predict_label = idx2label[paths[i][j].item()]
                    right_label = idx2label[label[j].item()]

                    add_to_seq(predict_seq, feature, predict_label)
                    add_to_seq(right_seq, feature, right_label)

    P = num_right_word / (num_error_word + num_right_word)
    R = num_right_word / (num_all_word)
    F1 = (2 * P * R) / (P + R)
    ER = num_error_word / num_all_word

    print(
        '标准词数：%d个，词数正确率：%f个，词数错误率：%f' % (num_all_word, num_right_word / num_all_word, num_error_word / num_all_word))
    print('标准行数：%d，行数正确率：%f' % (num_sentence, num_right_sentence / num_sentence))
    print('Recall: %f' % (R))
    print('Precision: %f' % (P))
    print('F1 MEASURE: %f' % (F1))
    print('ERR RATE: %f' % (ER))

    with open(log_file, 'a') as f:
        f.write(
            '标准词数：%d个，词数正确率：%f个，词数错误率：%f\n' % (
                num_all_word, num_right_word / num_all_word, num_error_word / num_all_word))
        f.write('标准行数：%d，行数正确率：%f\n' % (num_sentence, num_right_sentence / num_sentence))
        f.write('Recall: %f\n' % (R))
        f.write('Precision: %f\n' % (P))
        f.write('F1 MEASURE: %f\n' % (F1))
        f.write('ERR RATE: %f\n\n\n' % (ER))

    return P, R, F1, ER

def evaluate_with_perl(gold_file, predict_file, log=None, epoch=0, loss=None, dev=True):
    r"""
    这个效率高

    :param gold_file:
    :param predict_file:
    :return:
    """
    perl_path = r'/home/yxu/Seg_ner_pos/icwb2-data/scripts/score'
    word_list = r'/home/yxu/Seg_ner_pos/icwb2-data/gold/pku_training_words.utf8'
    p = subprocess.Popen(['perl', perl_path, word_list, gold_file, predict_file], stdout=subprocess.PIPE)
    output = p.stdout.read()
    output = output.decode(encoding='utf8')
    outputs = output.split('\n')
    p.kill()
    res = outputs[-15:]
    dev_R, dev_P, dev_F1 = float(res[-8].split('\t')[-1]), float(res[-7].split('\t')[-1]), float(
        res[-6].split('\t')[-1])

    if log is not None:
        with open(log, 'a') as f:
            f.write('EPOCH : %d\n' % epoch)

            if dev:
                f.write('Dev\n')
            else:
                f.write('Train\n')

            if loss is not None:
                f.write('Epoch loss : %f\n' % loss)

            for j in res:
                print(j)
                f.write(j + '\n')
    else:
        for j in res:
            print(j)

    return dev_R, dev_P, dev_F1

def predict_write(model, dataset_loader, idx2feature, idx2label, device, tmp_file='./tmp', origin_texts=None):
    r"""
    返回一个临时的预测文件

    :param model:
    :param dataset_loader:
    :param idx2feature:
    :param idx2label:
    :param device:
    :param tmp_file:
    :return:
    """
    # !!
    model.eval()

    with open(tmp_file, 'w') as f:
        for idx, batch in tqdm.tqdm(enumerate(itertools.chain.from_iterable(dataset_loader))):
            batch = tuple(t.to(device) for t in batch)
            features, labels, masks = batch
            features_v, labels_v, masks_v = features.transpose(0, 1), labels.transpose(0, 1), masks.transpose(0, 1)
            scores, predict_labels = model.predict(features_v, masks_v)
            batch_size = labels.shape[0]
            # 原始文本内容，避免最终结果出现<unk>
            if origin_texts:
                origin_text = origin_texts[idx * batch_size:(idx + 1) * batch_size]

            for j in range(batch_size):
                if origin_texts:
                    origin_line = origin_text[j]

                feature, predict_label, mask = features[j], predict_labels[j], masks[j]
                line = ''
                length = feature.shape[0]
                for k in range(length):
                    if k + 1 == length or mask[k + 1].item() == 0:
                        break
                    else:
                        if origin_texts:
                            content = origin_line[k]
                        else:
                            content = idx2feature[feature[k].item()]

                        if idx2label[predict_label[k].item()][0] in ('B', 'S') and k != 0:
                            line += ' ' + content
                        else:
                            line += content
                f.write(line + '\n')

    return tmp_file

def read_line(f):
    '''
        读取一行，并清洗空格和换行
    '''
    line = f.readline()
    return line.strip()

def evaluate_by_file(real_text_file, pred_text_file, prf_file, epoch):
    file_gold = open(real_text_file, 'r', encoding='utf8')
    file_tag = open(pred_text_file, 'r', encoding='utf8')

    line1 = read_line(file_gold)
    N_count = 0  # 将正类分为正或者将正类分为负
    e_count = 0  # 将负类分为正
    c_count = 0  # 正类分为正
    e_line_count = 0
    c_line_count = 0

    while line1:
        line2 = read_line(file_tag)

        list1 = line1.split(' ')
        list2 = line2.split(' ')

        count1 = len(list1)  # 标准分词数
        N_count += count1
        if line1 == line2:
            c_line_count += 1  # 分对的行数
            c_count += count1  # 分对的词数
        else:
            e_line_count += 1
            count2 = len(list2)

            arr1 = []
            arr2 = []

            pos = 0
            for w in list1:
                arr1.append(tuple([pos, pos + len(w)]))  # list1中各个单词的起始位置
                pos += len(w)

            pos = 0
            for w in list2:
                arr2.append(tuple([pos, pos + len(w)]))  # list2中各个单词的起始位置
                pos += len(w)

            for tp in arr2:
                if tp in arr1:
                    c_count += 1
                else:
                    e_count += 1

        line1 = read_line(file_gold)

    R = float(c_count) / N_count
    P = float(c_count) / (c_count + e_count)
    F = 2. * P * R / (P + R)
    ER = 1. * e_count / N_count

    print("result:")
    print('标准词数：%d个，词数正确率：%f个，词数错误率：%f' % (N_count, c_count / N_count, e_count / N_count))
    print('标准行数：%d，行数正确率：%f，行数错误率：%f' % (c_line_count + e_line_count, c_line_count / (c_line_count + e_line_count),
                                         e_line_count / (c_line_count + e_line_count)))
    print('Recall: %f' % (R))
    print('Precision: %f' % (P))
    print('F MEASURE: %f' % (F))
    print('ERR RATE: %f' % (ER))

    # print P,R,F

    f = open(prf_file, 'a', encoding='utf-8')
    f.write('result-(epoch:%s):\n' % epoch)
    f.write('标准词数：%d，词数正确率：%f，词数错误率：%f \n' % (N_count, c_count / N_count, e_count / N_count))
    f.write('标准行数：%d，行数正确率：%f，行数错误率：%f \n' % (c_line_count + e_line_count, c_line_count / (c_line_count + e_line_count),
                                              e_line_count / (c_line_count + e_line_count)))
    f.write('Recall: %f\n' % (R))
    f.write('Precision: %f\n' % (P))
    f.write('F MEASURE: %f\n' % (F))
    f.write('ERR RATE: %f\n' % (ER))
    f.write('====================================\n')

    return P, R, F


import datetime
import os
import time

import xarray as xr
from app.dataprocessing.benchmark import Timer
from app.dataprocessing.datasource_interface import IDatasource
from app.dataprocessing.local.local_reader import LocalReader
from app.dataprocessing.remote.opendap_access_cas import OpendapAccessCAS
from app.datastructures.datastructure_interface import INode, IStructure, get_bounds
from app.datastructures.n_dimensional.kd_tree import KDTree
from app.datastructures.three_dimensional.octree import Octree
from app.datastructures.two_dimensional.quad_tree import QuadTree
from dotenv import load_dotenv
from sympy import im

class CachedDS:
    def __init__(self, node: INode):
        self.ds = node.ds
        self.bounds = get_bounds(node.ds)
        self.time_stamp = datetime.datetime.now()
        self.resolution = node.resolution

    def __str__(self) -> str:
        return f"\tBounds:{self.bounds}\n\tCreated:{self.time_stamp}\n\tResolution:{self.resolution:.2f}"

class DataHandler:
    """
    Defines the data source and selects proper data structure
    """

    def __init__(self) -> None:
        self.ds = None  # xarray.Dataset

        self.data_source: IDatasource = None

        self.data_structure: IStructure = None

        self.max_chunk_size = 50

        self.on_demand_data = False

        self.custom_rules = None

        self.cache: list[CachedDS] = []

    def set_max_chunk_size(self, chunk_size):
        self.max_chunk_size = chunk_size

    def get_cache(self):
        return self.cache

    def set_custom_rules(self, custom_rules):
        self.custom_rules = custom_rules

    def set_opendap_cas(
        self,
        cas_url,
        ds_url,
        username,
        password,
        file_size=None,
        constraints=None,
        struct=None,
    ):
        self.on_demand_data = True

        if username == None or password == None:
            print("please save credentials to .env")

        self.data_source = OpendapAccessCAS(
            username,
            password,
            ds_url,
            cas_url,
            file_size_MB=file_size,
            constraints=constraints,
        )

        self.ds = self.data_source.get_dataset()
        with Timer("Creating data structure"):
            self.data_structure = self.__set_data_structure(struct)

    def set_local_netcdf_reader(self, file_path, constraints=None, struct=None):
        self.data_source = LocalReader(file_path, constraints)

        with Timer("Loading dataset"):
            self.ds = self.data_source.get_dataset()
        with Timer("Creating data structure"):
            self.data_structure = self.__set_data_structure(struct)

    def get_inital_netcdf(self):
        ds, bounds, node = self.data_structure.get_initial_dataset()

        file_name = "tmp/nc/data_" + str(time.time()) + ".nc"  # TODO: revisit.

        ds.to_netcdf(file_name)

        if self.on_demand_data:
            self.__node_stream_to_local_src(node, file_name)

        return file_name

    def get_initial_ds(self):
        ds, bounds, node = self.data_structure.get_initial_dataset()

        return ds, bounds, node

    def request_data_netcdf(self, bounds, return_xr_chunk=False, fit_bounds=False):
        ds, bounds, node = self.data_structure.request_data_single_chunk(
            bounds, fit_bounds=fit_bounds
        )

        file_name = "tmp/nc/data_" + str(time.time())[-5:] + ".nc"  # TODO: revisit.

        ds.to_netcdf(file_name)

        if self.on_demand_data and fit_bounds == False:
            self.__node_stream_to_local_src(node, file_name)

        if return_xr_chunk:
            return file_name, bounds, node
        else:
            return file_name

    def get_file_size_MB(self, file_path):
        return os.path.getsize(file_path) / (1024 * 1024)

    def get_node_resolution(self, node):
        return self.data_structure.get_node_resolution(node) * 100

    def get_node_spatial_resolution(self, node) -> dict:
        return self.data_structure.get_node_spatial_resolution(node)

    def get_full_xr_ds(self) -> xr.Dataset:
        return self.data_structure.ds

    def __node_stream_to_local_src(self, node, file_path):
        # store cache in list

        node.ds = xr.open_dataset(file_path)
        self.cache.append(CachedDS(node))

    def __set_data_structure(self, custom):
        if custom:
            if custom == "KDTree":
                return KDTree(
                    self.ds,
                    full_file_size=self.data_source.get_file_size_MB(),
                    max_chunk_size=self.max_chunk_size,
                    custom_rules=self.custom_rules,
                )

        ds_dims = self.__get_num_dimensions()
        if ds_dims == 2:
            return QuadTree(
                self.ds, self.data_source.get_file_size_MB(), self.max_chunk_size
            )
        elif ds_dims == 3:
            return Octree(
                self.ds, self.data_source.get_file_size_MB(), self.max_chunk_size
            )
        elif ds_dims > 3:
            return KDTree(
                self.ds,
                full_file_size=self.data_source.get_file_size_MB(),
                max_chunk_size=self.max_chunk_size,
                custom_rules=self.custom_rules,
            )
        else:
            raise Exception("DataHandler: unsupported number of dimensions")

    def __get_num_dimensions(self):
        return len(self.ds.dims)


import os

from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives.hashes import SHA256
from cryptography.hazmat.primitives.asymmetric import padding

# save file helper
def save_file(filename, content):
    filepath = os.path.dirname(os.path.abspath(__file__))
    f = open(filepath + "/" + filename, "wb")
    f.write(content)
    f.close()

def create_key():
    # generate private key & write to disk
    private_key = rsa.generate_private_key(
        public_exponent=65537,
        key_size=4096,
        backend=default_backend()
    )

    pem = private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
        #encryption_algorithm=serialization.BestAvailableEncryption(b'mypassword')
    )

    save_file("private.pem", pem)

    # generate public key
    public_key = private_key.public_key()
    pem = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    )

    save_file("public.pem", pem)


from logging import getLogger
from os.path import join
from configparser import NoOptionError

from uchicagoldrtoolsuite import log_aware
from uchicagoldrtoolsuite.core.app.abc.cliapp import CLIApp
from ..lib.writers.filesystemstagewriter import FileSystemStageWriter
from ..lib.readers.filesystemstagereader import FileSystemStageReader
from ..lib.processors.generictechnicalmetadatacreator import \
    GenericTechnicalMetadataCreator
from ..lib.techmdcreators.fitscreator import FITsCreator
from ..lib.techmdcreators.apifitscreator import APIFITsCreator

__author__ = "Brian Balsamo"
__email__ = "balsamo@uchicago.edu"
__company__ = "The University of Chicago Library"
__copyright__ = "Copyright University of Chicago, 2016"
__publication__ = ""
__version__ = "0.0.1dev"

log = getLogger(__name__)

def launch():
    """
    entry point launch hook
    """
    app = TechnicalMetadataCreator(
            __author__=__author__,
            __email__=__email__,
            __company__=__company__,
            __copyright__=__copyright__,
            __publication__=__publication__,
            __version__=__version__
    )
    app.main()

class TechnicalMetadataCreator(CLIApp):
    """
    Creates technical metadata (FITs) for all the material suites in a stage.
    """
    @log_aware(log)
    def main(self):
        # Instantiate boilerplate parser
        self.spawn_parser(description="The UChicago LDR Tool Suite utility " +
                          "creating technical metadata for materials in " +
                          "a stage.",
                          epilog="{}\n".format(self.__copyright__) +
                          "{}\n".format(self.__author__) +
                          "{}".format(self.__email__))
        # Add application specific flags/arguments
        log.debug("Adding application specific cli app arguments")
        self.parser.add_argument("stage_id", help="The id of the stage",
                                 type=str, action='store')
        self.parser.add_argument("--skip_existing", help="Skip material " +
                                 "suites which already claim to have " +
                                 "technical metadata",
                                 action='store_true',
                                 default=False)
        self.parser.add_argument("--staging_env", help="The path to your " +
                                 "staging environment",
                                 type=str,
                                 default=None)
        self.parser.add_argument("--eq_detect", help="The equality " +
                                 "metric to use on writing, check " +
                                 "LDRItemCopier for supported schemes.",
                                 type=str, action='store',
                                 default="bytes")
        self.parser.add_argument("--fits_path", help="The path to the FITS " +
                                 "executable on this system. " +
                                 "Overrides any value found in configs.",
                                 type=str, action='store',
                                 default=None)
        self.parser.add_argument("--fits_api_url", help="The url of a FITS " +
                                 "Servlet examine endpoint. " +
                                 "Overrides any value found in configs.",
                                 type=str, action='store',
                                 default=None)
        self.parser.add_argument("--use_api", help="Use a FITS Servlet " +
                                 "instead of a local FITS install.",
                                 action="store_true",
                                 default=False)

        # Parse arguments into args namespace
        args = self.parser.parse_args()
        self.process_universal_args(args)

        # App code

        if args.staging_env:
            staging_env = args.staging_env
        else:
            staging_env = self.conf.get("Paths", "staging_environment_path")
        staging_env = self.expand_path(staging_env)

        dto = {}
        try:
            dto['fits_path'] = self.conf.get("Paths", "fits_path")
        except NoOptionError:
            pass
        try:
            dto['fits_api_url'] = self.conf.get("URLs", "fits_api_url")
        except NoOptionError:
            pass

        if args.fits_api_url is not None:
            dto['fits_api_url'] = args.fits_api_url
        if args.fits_path is not None:
            dto['fits_path'] = args.fits_path

        reader = FileSystemStageReader(staging_env, args.stage_id)
        stage = reader.read()
        log.info("Stage: " + join(staging_env, args.stage_id))

        log.info("Processing...")

        if args.use_api:
            techmd_processors = [APIFITsCreator]
        else:
            techmd_processors = [FITsCreator]

        techmd_creator = GenericTechnicalMetadataCreator(stage,
                                                         techmd_processors)
        techmd_creator.process(skip_existing=args.skip_existing,
                               data_transfer_obj=dto)

        log.info("Writing...")
        writer = FileSystemStageWriter(stage, staging_env,
                                       eq_detect=args.eq_detect)
        writer.write()
        log.info("Complete")

if __name__ == "__main__":
    s = TechnicalMetadataCreator()
    s.main()


"""
test cli module
"""
import subprocess
from typing import List, Tuple

def capture(command: List[str]) -> Tuple[bytes, bytes, int]:
    proc = subprocess.Popen(
        command,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    out, err = proc.communicate()
    return out, err, proc.returncode

def test_cli() -> None:
    """Test cli module"""
    command = ["sdwaddle"]
    out, err, exitcode = capture(command)
    assert exitcode == 0


from __future__ import division
from builtins import str
import numpy as np
import pandas as pd
import seaborn as sns
from .helpers import *
import matplotlib.pyplot as plt
import matplotlib as mpl

mpl.rcParams['pdf.fonttype'] = 42

def plot(results, subjgroup=None, subjname='Subject Group', listgroup=None,
         listname='List', subjconds=None, listconds=None, plot_type=None,
         plot_style=None, title=None, legend=True, xlim=None, ylim=None,
         save_path=None, show=True, ax=None, **kwargs):
    """
    General plot function that groups data by subject/list number and performs analysis.

    Parameters
    ----------
    results : quail.FriedEgg
        Object containing results

    subjgroup : list of strings or ints
        String/int variables indicating how to group over subjects.  Must be
        the length of the number of subjects

    subjname : string
        Name of the subject grouping variable

    listgroup : list of strings or ints
        String/int variables indicating how to group over list.  Must be
        the length of the number of lists

    listname : string
        Name of the list grouping variable

    subjconds : list
        List of subject hues (str) to plot

    listconds : list
        List of list hues (str) to plot

    plot_type : string
        Specifies the type of plot. If list (default), the list groupings (listgroup)
        will determine the plot grouping. If subject, the subject groupings
        (subjgroup) will determine the plot grouping. If split (currenty just
        works for accuracy plots), both listgroup and subjgroup will determine
        the plot groupings

    plot_style : string
        Specifies the style of the plot.  This currently works only for
        accuracy and fingerprint plots. The plot style can be bar (default for
        accruacy plot), violin (default for fingerprint plots) or swarm.

    title : string
        The title of the plot

    legend : bool
        If true (default), a legend is plotted.

    ylim : list of numbers
        A ymin/max can be specified by a list of the form [ymin, ymax]

    xlim : list of numbers
        A xmin/max can be specified by a list of the form [xmin, xmax]

    save_path : str
        Path to save out figure.  Include the file extension, e.g.
        save_path='figure.pdf'

    show : bool
        If False, do not show figure, but still return ax handle (default True).

    ax : Matplotlib.Axes object or None
        A plot object to draw to. If None, a new one is created and returned.

    Returns
    ----------
    ax : matplotlib.Axes.Axis
        An axis handle for the figure

    """

    def plot_acc(data, plot_style, plot_type, listname, subjname, **kwargs):

        # set defaul style to bar
        plot_style = plot_style if plot_style is not None else 'bar'
        plot_type = plot_type if plot_type is not None else 'list'

        if plot_style is 'bar':
            plot_func = sns.barplot
        elif plot_style is 'swarm':
            plot_func = sns.swarmplot
        elif plot_style is 'violin':
            plot_func = sns.violinplot

        if plot_type is 'list':
            ax = plot_func(data=data, x=listname, y="Accuracy", **kwargs)
        elif plot_type is 'subject':
            ax = plot_func(data=data, x=subjname, y="Accuracy", **kwargs)
        elif plot_type is 'split':
            ax = plot_func(data=data, x=subjname, y="Accuracy", hue=listname, **kwargs)

        return ax

    def plot_temporal(data, plot_style, plot_type, listname, subjname, **kwargs):

        # set default style to bar
        plot_style = plot_style if plot_style is not None else 'bar'
        plot_type = plot_type if plot_type is not None else 'list'

        if plot_style is 'bar':
            plot_func = sns.barplot
        elif plot_style is 'swarm':
            plot_func = sns.swarmplot
        elif plot_style is 'violin':
            plot_func = sns.violinplot

        if plot_type is 'list':
            ax = plot_func(data=data, x=listname, y="Temporal Clustering Score", **kwargs)
        elif plot_type is 'subject':
            ax = plot_func(data=data, x=subjname, y="Temporal Clustering Score", **kwargs)
        elif plot_type is 'split':
            ax = plot_func(data=data, x=subjname, y="Temporal Clustering Score", hue=listname, **kwargs)

        return ax

    def plot_fingerprint(data, plot_style, plot_type, listname, subjname, **kwargs):

        # set default style to violin
        plot_style = plot_style if plot_style is not None else 'violin'
        plot_type = plot_type if plot_type is not None else 'list'

        if plot_style is 'bar':
            plot_func = sns.barplot
        elif plot_style is 'swarm':
            plot_func = sns.swarmplot
        elif plot_style is 'violin':
            plot_func = sns.violinplot

        if plot_type is 'list':
            ax = plot_func(data=tidy_data, x="Feature", y="Clustering Score", hue=listname, **kwargs)
        elif plot_type is 'subject':
            ax = plot_func(data=tidy_data, x="Feature", y="Clustering Score", hue=subjname, **kwargs)
        else:
            ax = plot_func(data=tidy_data, x="Feature", y="Clustering Score", **kwargs)

        return ax

    def plot_fingerprint_temporal(data, plot_style, plot_type, listname, subjname, **kwargs):

        # set default style to violin
        plot_style = plot_style if plot_style is not None else 'violin'
        plot_type = plot_type if plot_type is not None else 'list'

        if plot_style is 'bar':
            plot_func = sns.barplot
        elif plot_style is 'swarm':
            plot_func = sns.swarmplot
        elif plot_style is 'violin':
            plot_func = sns.violinplot

        order = list(tidy_data['Feature'].unique())
        if plot_type is 'list':
            ax = plot_func(data=data, x="Feature", y="Clustering Score", hue=listname, order=order, **kwargs)
        elif plot_type is 'subject':
            ax = plot_func(data=data, x="Feature", y="Clustering Score", hue=subjname, order=order, **kwargs)
        else:
            ax = plot_func(data=data, x="Feature", y="Clustering Score", order=order, **kwargs)

        return ax

    def plot_spc(data, plot_style, plot_type, listname, subjname, **kwargs):

        plot_type = plot_type if plot_type is not None else 'list'

        if plot_type is 'subject':
            ax = sns.lineplot(data = data, x="Position", y="Proportion Recalled", hue=subjname, **kwargs)
        elif plot_type is 'list':
            ax = sns.lineplot(data = data, x="Position", y="Proportion Recalled", hue=listname, **kwargs)
        ax.set_xlim(0, data['Position'].max())

        return ax

    def plot_pnr(data, plot_style, plot_type, listname, subjname, position, list_length, **kwargs):

        plot_type = plot_type if plot_type is not None else 'list'

        if plot_type is 'subject':
            ax = sns.lineplot(data = data, x="Position", y='Probability of Recall: Position ' + str(position), hue=subjname, **kwargs)
        elif plot_type is 'list':
            ax = sns.lineplot(data = data, x="Position", y='Probability of Recall: Position ' + str(position), hue=listname, **kwargs)
        ax.set_xlim(0,list_length-1)

        return ax

    def plot_lagcrp(data, plot_style, plot_type, listname, subjname, **kwargs):

        plot_type = plot_type if plot_type is not None else 'list'

        if plot_type is 'subject':
            ax = sns.lineplot(data=data[data['Position']<0], x="Position", y="Conditional Response Probability", hue=subjname, **kwargs)
            if 'ax' in kwargs:
                del kwargs['ax']
            sns.lineplot(data=data[data['Position']>0], x="Position", y="Conditional Response Probability", hue=subjname, ax=ax, legend=False, **kwargs)
        elif plot_type is 'list':
            ax = sns.lineplot(data=data[data['Position']<0], x="Position", y="Conditional Response Probability", hue=listname, **kwargs)
            if 'ax' in kwargs:
                del kwargs['ax']
            sns.lineplot(data=data[data['Position']>0], x="Position", y="Conditional Response Probability", hue=listname, ax=ax, legend=False, **kwargs)
        ax.set_xlim(-5,5)

        return ax

    # if no grouping, set default to iterate over each list independently
    subjgroup = subjgroup if subjgroup is not None else results.data.index.levels[0].values
    listgroup = listgroup if listgroup is not None else results.data.index.levels[1].values

    if subjconds:
        # make sure its a list
        if type(subjconds) is not list:
            subjconds=[subjconds]

        # slice
        idx = pd.IndexSlice
        results.data = results.data.sort_index()
        results.data = results.data.loc[idx[subjconds, :],:]

        # filter subjgroup
        subjgroup = filter(lambda x: x in subjconds, subjgroup)

    if listconds:
        # make sure its a list
        if type(listconds) is not list:
            listconds=[listconds]

        # slice
        idx = pd.IndexSlice
        results.data = results.data.sort_index()
        results.data = results.data.loc[idx[:, listconds],:]

    # convert to tiny and format for plotting
    tidy_data = format2tidy(results.data, subjname, listname, subjgroup, analysis=results.analysis, position=results.position)

    if not ax==None:
        kwargs['ax']=ax

    #plot!
    if results.analysis=='accuracy':
        ax = plot_acc(tidy_data, plot_style, plot_type, listname, subjname, **kwargs)
    elif results.analysis=='temporal':
        ax = plot_temporal(tidy_data, plot_style, plot_type, listname, subjname, **kwargs)
    elif results.analysis=='fingerprint':
        ax = plot_fingerprint(tidy_data, plot_style, plot_type, listname, subjname, **kwargs)
    elif results.analysis=='fingerprint_temporal':
        ax = plot_fingerprint_temporal(tidy_data, plot_style, plot_type, listname, subjname, **kwargs)
    elif results.analysis=='spc':
        ax = plot_spc(tidy_data, plot_style, plot_type, listname, subjname, **kwargs)
    elif results.analysis=='pfr' or results.analysis=='pnr':
        ax = plot_pnr(tidy_data, plot_style, plot_type, listname, subjname, position=results.position, list_length=results.list_length,  **kwargs)
    elif results.analysis=='lagcrp':
        ax = plot_lagcrp(tidy_data, plot_style, plot_type, listname, subjname, **kwargs)
    else:
        raise ValueError("Did not recognize analysis.")

    # add title
    if title:
        plt.title(title)

    if legend is False:
        try:
            ax.legend_.remove()
        except:
            pass

    if xlim:
        plt.xlim(xlim)

    if ylim:
        plt.ylim(ylim)

    if save_path:
        mpl.rcParams['pdf.fonttype'] = 42
        plt.savefig(save_path)

    return ax


#!/usr/bin/env python

'''
对测试集数据进行测试，统计所有数据平均的RRMSE,SNR和CC值
'''

import argparse
import os
import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader
from utility.data import EEGData
from utility.conv_tasnet_v1 import TasNet
from utility.network import ResCNN, Novel_CNN2, Novel_CNN, fcNN
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import math
import scipy.io as io
from scipy.signal import butter, lfilter

EPS = 1e-8

parser = argparse.ArgumentParser('Evaluate separation performance using Conv-TasNet')
parser.add_argument('--model_path', type=str,
                    help='Path to model file created by training')
parser.add_argument('--data_dir', type=list, default=['./data/mixdata/foldtrain.txt',
                                                      './data/mixdata/foldtest.txt'],
                    help='directory of fold')
parser.add_argument('--use_cuda', type=int, default=1,
                    help='Whether use GPU')
parser.add_argument('--batch_size', default=128, type=int,
                    help='Batch size')
parser.add_argument('--num_workers', default=0, type=int,
                    help='Num_workers')

# Network architecture
parser.add_argument('--N', default=256, type=int,
                    help='Encode dim')
parser.add_argument('--B', default=64, type=int,
                    help='Feature dim')
parser.add_argument('--sr', default=512, type=int,
                    help='Sample rate')
parser.add_argument('--L', default=16, type=int,
                    help='Length of the filters in samples (16=16ms at 1kHZ)')
parser.add_argument('--X', default=6, type=int,
                    help='Number of convolutional blocks in each repeat')
parser.add_argument('--R', default=3, type=int,
                    help='Number of repeats')
parser.add_argument('--P', default=3, type=int,
                    help='Kernel size in convolutional blocks')
parser.add_argument('--C', default=1, type=int,
                    help='Number of speakers')

# 计算相关系数
def calc_corr(a, b):
    a_avg = sum(a) / len(a)
    b_avg = sum(b) / len(b)
    # 计算分子，协方差————按照协方差公式，本来要除以n的，由于在相关系数中上下同时约去了n，于是可以不除以n
    cov_ab = sum([(x - a_avg) * (y - b_avg) for x, y in zip(a, b)])
    # 计算分母，方差乘积————方差本来也要除以n，在相关系数中上下同时约去了n，于是可以不除以n
    sq = math.sqrt(sum([(x - a_avg) ** 2 for x in a]) * sum([(x - b_avg) ** 2 for x in b]))
    corr_factor = cov_ab / sq
    return corr_factor

def butter_lowpass(data, cutoff, fs, order=5):
    nyq = 0.5 * fs
    normal_cutoff = cutoff / nyq
    b, a = butter(order, normal_cutoff, btype='low', analog=False)
    y = lfilter(b, a, data)
    return y

def evaluate(args, snr_test=1):
    # Load model
    package = torch.load(args.model_path)
    model = TasNet(args.N, args.B, args.sr, args.L, args.X, args.R, args.P, args.C)
    # model = fcNN(lenth=1024)
    # model = Novel_CNN2(len_signal=1024)
    # model = ResCNN(1024)
    model.load_state_dict(package['model'])
    # print(model)
    # for name in model.state_dict():
    #     print(name)
    # print('encoder1:',model.state_dict()['encoder1.weight'])
    model.eval()
    if args.use_cuda:
        model.cuda()

    # Load data
    f_test = np.load('../eegdenoisenet/testdata512/test_eeg.npz')
    noiseEEG_test, EEG_test, SNRs_test = f_test['noiseEEG_test'], f_test['EEG_test'], f_test['SNRs_test']

    # 选择 信噪比
    idx = np.where(SNRs_test == snr_test)[0]
    # 不分档则以下2行注释
    noiseEEG_test = noiseEEG_test[idx]
    EEG_test = EEG_test[idx]

    evaluate_dataset = EEGData(noiseEEG_test, EEG_test)
    evaluate_loader = DataLoader(evaluate_dataset, batch_size=args.batch_size, shuffle=False,
                                 num_workers=args.num_workers)

    with torch.no_grad():
        Snr = 1.5
        RRMSEspec_total = []
        RRMSE_total = []
        CC_total = []
        SNR = []
        estimate_all = []
        for i, data in enumerate(evaluate_loader):
            # Get batch data

            eeg_mix = data['eeg_mix'].type(torch.float32)
            # emg = data['emg'].type(torch.float32)
            eeg_clean = data['eeg_clean'].type(torch.float32)

            lpeeg_mix = butter_lowpass(eeg_mix, 20, 500)
            # hpeeg_mix = butter_highpass(eeg_mix,100,500)

            eeg_mix = eeg_mix.type(torch.float32)

            lpeeg_mix = torch.from_numpy(lpeeg_mix).type(torch.float32)
            # hpeeg_mix = torch.from_numpy(hpeeg_mix).type(torch.float32)
            # lpeeg_clean = data['lpeeg_clean']

            # Forward
            if args.use_cuda:
                eeg_clean = eeg_clean.cuda()
                lpeeg_mix = lpeeg_mix.cuda()
                eeg_mix = eeg_mix.cuda()

            estimate_source1, estimate_source2 = model(lpeeg_mix, eeg_mix)
            # estimate_source2  = model(eeg_mix)
            e_noise = eeg_clean - estimate_source2.squeeze()
            snr = 10 * torch.log10(torch.sum(eeg_clean ** 2, dim=1) / (torch.sum(e_noise ** 2, dim=1) + EPS) + EPS)

            estimate2 = estimate_source2.cpu().numpy().squeeze()
            # emg = emg.numpy()
            eeg_clean = eeg_clean.cpu().numpy()
            estimate_all.append(estimate2)
            # eeg_mix = eeg_mix.numpy()

            for j in range(estimate2.shape[0]):
                eeg_snr = snr[j].item()
                SNR.append(eeg_snr)

                e_noise = eeg_clean[j, :] - estimate2[j, :]
                RRMSE = np.sqrt(np.sum(e_noise ** 2) / (np.sum(eeg_clean[j, :] ** 2) + EPS) + EPS)

                # TODO
                Pxx_eeg, _ = mlab.psd(eeg_clean[j, :], NFFT=1024)
                Pxx_estimate, _ = mlab.psd(estimate2[j, :], NFFT=1024)
                # Pxx_eeg = Pxx_eeg[:201]                # TODO
                # Pxx_estimate = Pxx_estimate[:201]      # TODO
                RRMSEspec = np.sqrt(np.sum((Pxx_estimate - Pxx_eeg) ** 2) / (np.sum(Pxx_eeg ** 2) + EPS) + EPS)

                eeg_RRMSE = RRMSE
                # 计算CC
                cc = calc_corr(eeg_clean[j, :], estimate2[j, :])
                RRMSEspec_total.append(RRMSEspec)
                RRMSE_total.append(eeg_RRMSE)
                CC_total.append(cc)
        estimate_all = np.vstack(estimate_all)
        io.savemat(os.path.join('result', f'result_{snr_test}.mat'),
                   {'estimate_EEG': estimate_all, 'eeg_mix': noiseEEG_test, 'eeg_clean': EEG_test})
        # plt.psd
        return RRMSE_total, CC_total, SNR, RRMSEspec_total

if __name__ == '__main__':
    args = parser.parse_args()
    args.model_path = '/mnt/DEV/han/eeg/DASTCN_grnFFT/checkpoint/EEGARNet_model/epoch89.pth.tar'
    out_dir = 'result'
    print(args)

    meanRRMSEs = []
    meanRRMSEspecs = []
    meanCCs = []
    meanSNRs = []
    for snr in np.linspace(-7, 2, 10):
        print(snr)
        RRMSE_total, CC_total, SNR, RRMSEspec_total = evaluate(args, snr_test=snr)

        meanRRMSE = np.round(np.mean(RRMSE_total), 4)
        meanRRMSEspec = np.round(np.mean(RRMSEspec_total), 4)
        meanSNR = np.round(np.mean(SNR), 4)
        varRRMSE = np.round(np.var(RRMSE_total), 4)
        meanCC = np.round(np.mean(CC_total), 4)
        varCC = np.round(np.var(CC_total), 4)

        print('meanRRMSEspec:', meanRRMSE)
        print('meanRRMSEspec:', meanRRMSEspec)
        print('meanCC:', meanCC)
        print('meanSNR:', meanSNR)
        print('*' * 10)
        meanRRMSEs.append(meanRRMSE)
        meanCCs.append(meanCC)
        meanSNRs.append(meanSNR)
        meanRRMSEspecs.append(meanRRMSEspec)

    os.makedirs(out_dir, exist_ok=True)
    io.savemat(os.path.join(out_dir, 'result_perSNR.mat'), {'meanRRMSEs': meanRRMSEs,
                                                            'meanCCs': meanCCs,
                                                            'meanSNRs': meanSNRs,
                                                            'meanRRMSEspec': meanRRMSEspecs})


import time, random
import pygame

screen_w = 800
screen_h = 600

# Create the window
screen = pygame.display.set_mode((screen_w,screen_h))

black = (0,0, 0)
red = (255,0,0)
green = (0, 200, 0)

class Game(object):

	def __init__(self):
		self.screen = pygame.display.set_mode((800,600))
		self.score = 0
		self.oldScore = 0
		self.speed = 5
		self.speedMultiplier = 1

	def gameInit(self):
		pygame.init()

	def gameName(self, name):
		pygame.display.set_caption(name)

	def setFPS(self, fps):
		pygame.time.Clock().tick(fps)

	def screenColor(self):
		self.screen.fill(green)

	def incSpeed(self, newSpeed):
		self.speed = newSpeed

	def resetGame(self):
		self.oldScore = self.score
		self.score = 0
		self.speed = 5
		self.speedMultiplier = 1

# Player Class
class Player(object):
	init_x = 150
	init_y = 405

	def __init__(self):
		self.x = Player.init_x
		self.y = Player.init_y
		self.width = 50
		self.height = 50
		self.vel = 10
		self.isJump = False
		self.jumpCount = 10

	# Function to draw the players geometry, in this case just a red square
	def draw(self):
		# Red square as player
		pygame.draw.rect(screen, red, [self.x, self.y, self.width, self.height])
		# Black contour defined by hitbox
		pygame.draw.rect(screen, black, (self.x, self.y, self.width, self.height), 3)

	def jump(self):
		# Using a mathematical equation with square calculations for jumping.
		# When the player reaches a certain height, jumpCount will get a negative value
		# so that the player starts falling down.
		if self.jumpCount >= -10:
			neg = 1
			if self.jumpCount < 0:
				neg = -1
			# Math equation
			self.y -= (self.jumpCount ** 2) * 0.8 * neg
			self.jumpCount -= 1
		else:
			self.isJump = False
			self.jumpCount = 10

	def setVel(self, vel):
		self.vel = vel

	# Reset position after death
	def reset(self):
		self.x = 150
		self.y = 405
		self.width = 50
		self.height = 50

# Enemy Class
class Enemy(object):
	def __init__(self, x, y, width, height):
		self.x = x
		self.y = y
		self. width = width
		self.height = height
		self.end = screen_w
		self.vel = 10

	# Function to draw the geometry of the enemies, in this case just a polygon in the form of a triangle
	def draw(self):
		# Calling function move() to actually draw the movement on the screen
		self.move()

		# If the enemy moves out of the screen it will appear on the other side again, looping
		if self.x < 0:
			self.x = self.end + random.randint(0,350)

		# Pointlist is a set of coordinates for the polygon [(x1,y1),(x2,y2) and so on]
		pointlist = [(self.x - self.width/2, self.y - self.width), (self.x - self.height/2, self.y - self.height), (self.x, self.y), (self.x - self.width, self.y)]
		# Draw the red contour
		pygame.draw.polygon(screen, red, pointlist, 3)
		# Draw the black triangle defined by pointlist
		pygame.draw.polygon(screen, black, pointlist, 0)

	# Function to move the enemy to the left towards the player at a certain velocity
	def move(self):
		self.x -= self.vel

	def setVel(self, newVel):
		self.vel = newVel

	# Resets position after death
	def reset(self, offset):
		self.x = random.randint(750,850) + offset
		self.y = 454
		self.width = 30
		self.height = 30
		self.vel = 10

# Function to call all class related functions to upload the drawings to the screen
def redrawGameWindow(player, enemyList):
	player.draw()
	for enemy in enemyList:
		enemy.draw()

	# Draws the red base in the game
	pygame.draw.rect(screen, red, [0,456, screen_w, 200])

	# Updates the screen with the drawings applied
	pygame.display.update()

# Whenever the player is touches the enemy this function is called and displayes the message DEAD on screen
def printMSG(msg, x, y, size):
	# Define font and size
	font = pygame.font.Font(None, size)
	# Define what message to display and it's color
	text_surface = font.render(msg, True, (0, 0, 0))
	# Print the message to screen using coordinates
	screen.blit(text_surface, (x,y))

# Collision calculation with enemies, when the square touches the triangles it will display message "DEAD"
def checkCollision(game, player, enemies):
	for enemy in enemies:
		if (player.x + player.width) >= (enemy.x - enemy.width) and player.x <= enemy.x:
			if (player.y + player.height) >= (enemy.y - enemy.height) and player.y <= enemy.y:
				printMSG("DEAD", 355, 250, 50)
				redrawGameWindow(player, enemies)
				time.sleep(1)
				# When collision occurs the game resets
				player.reset()
				enemies[0].reset(100)
				enemies[1].reset(450)
				game.resetGame()

# Increases and prints score as well as the old score
def scoreUpdate(game):
	game.score += game.speed
	printMSG(("Score: " + str(game.score)), 50, 50, 40)
	printMSG(("Old Score: " + str(game.oldScore)), 500, 50, 40)

# Function that increases the speed every 1000 score
def speedUpdate(game, enemylist):
	if game.score >= 2000 * game.speedMultiplier:
		game.speedMultiplier += 1
		for enemy in enemylist:
			enemy.setVel(enemy.vel + 1)

def main():

	# Game instance
	game = Game()
	game.gameInit()
	game.gameName("Running Game 2")

	# Player 1
	sq = Player()

	# Enemies 1 and 2
	ey = Enemy(random.randint(750,850),454,30,30)
	ey2 = Enemy(random.randint(1200,1400), 454, 30, 30)

	# Enemy list, if several add here
	enemyList = [ey, ey2]

	# Game condition
	running = True

	# Game loop
	while running:

		# Set screen color in RGB
		game.screenColor()

		# Continously check all events that are happening in the game
		for event in pygame.event.get():

			# Check if window is closed when the cross is pressed
			if event.type == pygame.QUIT:
				running = False

		# Variable for checking if any key is pressed
		keys = pygame.key.get_pressed()

		# Arrow key movements of player
		if keys[pygame.K_LEFT] and sq.x > 0:
			# Move player to the left with the given velocity when left key is pressed
			sq.x -= sq.vel
		if keys[pygame.K_RIGHT] and sq.x < screen_w - sq.width:
			sq.x += sq.vel

		# Jump function
		if not(sq.isJump):
			if keys[pygame.K_SPACE]:
				sq.isJump = True
		else:
			sq.jump()

		# Updates score every loop
		scoreUpdate(game)

		# Increases speed every 1000 score
		speedUpdate(game, enemyList)

		# Collision detection between player and enemies
		checkCollision(game, sq, enemyList)

		# Calling this function every loop to update the drawings to screen
		redrawGameWindow(sq, enemyList)

		# Frames per second
		game.setFPS(30)

if __name__ == "__main__":
	main()

pygame.quit()


# Копирование найденных надежных и ненадежных аудиозаписей по Социуму за 2017-2018 на архивный диск
import openpyxl, traceback
import os, string, sys, shutil
from collections import Counter
from lib import l, fine_snils_, read_config

FIND_CATALOG = '/media/da3/asteriskBeagleAl/'
#CHANGE_ON_WINDOWS = 'Z:/'
#OUTPUT_CATALOG = 'O:/Документы/Записи/'
OUTPUT_CATALOG = '/media/da3/backup/'
TRUSTREESTR = 'Надежные.xlsx'
PROBLEMREESTR = 'Остальные.xlsx'
REESTRS = '/home/da3/Beagle/потеряшкиАудиозаписи/реестры/'

def isSNILS(snils):
    if snils != None:
        t = str(snils).replace('\n',' ').replace('  ', ' ').replace('  ', ' ').replace('  ', ' ').strip()
        if len(t) > 11:
            if t[3] == '-' and t[7] == '-' and (t[11] == ' ' or t[11] == '_'):
                return True
            else:
                return False
        else:
            return False
    return False

def isAudio(audio):
    if audio != None:
        t = str(audio).replace('\n',' ').replace('  ', ' ').replace('  ', ' ').replace('  ', ' ').strip()
        t1 = t.split('/')[len(t.split(('/'))) - 1]
        if t1.endswith('.'):
            t1 = t1[:-1]
        if t1.endswith('.mp3') or t1.endswith('.wav'):
            t1 = t1[:-4]
        if len(t1) > 26:
            if t1[2] == '.' and t1[5] == '.' and t1[10] == '_' and (t1[13] == '-' or t1[13] == '_') and \
                    (t1[16] == '-' or t1[16] == '_'):
                return ['длинный', t1]
            elif len(''.join([char for i, char in enumerate(t1) if char in string.digits and i < 26])) == 25 \
                    and t1[14] == '_':
                return ['короткий', t1]
            else:
                return ['', audio]
        else:
            return ['', audio]
    return ['', audio]

def isSocium(audio):
    if audio != None:
        t = str(audio).replace('\n',' ').replace('  ', ' ').replace('  ', ' ').replace('  ', ' ').strip()
        t1 = t.split('/')[len(t.split(('/'))) - 1]
        if len(t1) > 26:
            if t1[2] == '.' and t1[5] == '.' and t1[10] == '_' and (t1[13] == '-' or t1[13] == '_') and \
                    (t1[16] == '-' or t1[16] == '_') and (t1[6:10] == '2017' or t1[6:10] == '2018'):
                return True
            elif len(''.join([char for i, char in enumerate(t1) if char in string.digits and i < 26])) == 25 \
                    and t1[14] == '_' and (t1[:4] == '2017' or t1[:4] == '2018'):
                return True
            else:
                return False
        else:
            return False
    return False

# расшифровка любой ошибки
def full_tb_write(*args):
    if not args:
        exc_type, exc_val, exc_tb = sys.exc_info()
        traceback.print_tb(exc_tb, file=sys.stdout)
    elif len(args) == 3:
        exc_type, exc_val, exc_tb = args
        traceback.print_tb(exc_tb, file=sys.stdout)
    elif len(args) == 1:
        exc_type, exc_val, exc_tb = args[0].__class__, args[0], args[0].__traceback__
        traceback.print_tb(exc_tb, file=sys.stdout)

snilsesTrust = {}
snilsesTrustShort = {}
wb = openpyxl.load_workbook(filename=TRUSTREESTR, read_only=True)
for sheetname in wb.sheetnames:
    sheet = wb[sheetname]
    if not sheet.max_row:
        print('Файл', TRUSTREESTR, 'Excel некорректно сохранен OpenPyxl. Откройте и пересохраните его')
        continue
    for j, row in enumerate(sheet.rows):
        snils = l(row[0].value)
        snilsTrustAudios = []
        for k, cell in enumerate(row):
            if k and cell.value:
                snilsTrustAudio = isAudio(cell.value)
                if snilsTrustAudio[1] not in snilsTrustAudios:
                    snilsTrustAudios.append(snilsTrustAudio[1])
                if snilsesTrust.get(snils, None):
                    if cell.value not in snilsesTrust[snils]:
                        snilsesTrust[snils].append(cell.value)
                else:
                    snilsesTrust[snils] = [cell.value]
        snilsesTrustShort[snils] = snilsTrustAudios

for i, snils in enumerate(snilsesTrust):
    sucess = False
    while not sucess:
        try:
            if not os.path.exists(OUTPUT_CATALOG + 'Выгрузки/' + fine_snils_(snils)):
                os.mkdir(OUTPUT_CATALOG + 'Выгрузки/' + fine_snils_(snils))
            audiofilesShort = []
            for audiofile in snilsesTrust[snils]:
                audiofileShort = isAudio(audiofile)[1]
                if os.path.exists(audiofile): #.replace(FIND_CATALOG, CHANGE_ON_WINDOWS)):
                    if audiofileShort in audiofilesShort:
                        if not os.path.exists(OUTPUT_CATALOG + 'Выгрузки/' + fine_snils_(snils) + '/' + audiofileShort +
                                              '-' + str(Counter(audiofilesShort)[audiofileShort]) + audiofile[-4:]):
                            shutil.copy(audiofile #.replace(FIND_CATALOG, CHANGE_ON_WINDOWS)
                                        , OUTPUT_CATALOG + 'Выгрузки/' + fine_snils_(snils) + '/' + audiofileShort +
                                        '-' + str(Counter(audiofilesShort)[audiofileShort]) + audiofile[-4:])
                            audiofilesShort.append(audiofileShort)
                    else:
                        if not os.path.exists(OUTPUT_CATALOG + 'Выгрузки/' + fine_snils_(snils) + '/' +
                                              audiofileShort + audiofile[-4:]):
                            shutil.copy(audiofile #.replace(FIND_CATALOG, CHANGE_ON_WINDOWS)
                                        , OUTPUT_CATALOG + 'Выгрузки/' + fine_snils_(snils) + '/' + audiofileShort +
                                        audiofile[-4:])
                            audiofilesShort.append(audiofileShort)
                else:
                    print('!!! Нет исходного файла', audiofile)
            sucess = True
        except Exception as e:
            full_tb_write(e)
            print('Ошибка - пробуем ещё раз')
    print('Скопировано', i, 'из', len(snilsesTrust))

print('\nТеперь Остальные\n')

snilsesProblem = {}
snilsesProblemShort = {}
wb = openpyxl.load_workbook(filename=PROBLEMREESTR, read_only=True)
for sheetname in wb.sheetnames:
    sheet = wb[sheetname]
    if not sheet.max_row:
        print('Файл', PROBLEMREESTR, 'Excel некорректно сохранен OpenPyxl. Откройте и пересохраните его')
        continue
    for j, row in enumerate(sheet.rows):
        snils = l(row[0].value)
        snilsProblemAudios = []
        for k, cell in enumerate(row):
            if k and cell.value:
                snilsProblemAudio = isAudio(cell.value)
                if snilsProblemAudio[1] not in snilsProblemAudios:
                    snilsProblemAudios.append(snilsProblemAudio[1])
                if snilsesProblem.get(snils, None):
                    if cell.value not in snilsesProblem[snils]:
                        snilsesProblem[snils].append(cell.value)
                else:
                    snilsesProblem[snils] = [cell.value]
        snilsesProblemShort[snils] = snilsProblemAudios

for i, snils in enumerate(snilsesProblem):
    sucess = False
    while not sucess:
        try:
            if not os.path.exists(OUTPUT_CATALOG + 'Остальные/' + fine_snils_(snils)):
                os.mkdir(OUTPUT_CATALOG + 'Остальные/' + fine_snils_(snils))
            audiofilesShort = []
            for audiofile in snilsesProblem[snils]:
                audiofileShort = isAudio(audiofile)[1]
                if os.path.exists(audiofile): #.replace(FIND_CATALOG, CHANGE_ON_WINDOWS)):
                    if audiofileShort in audiofilesShort:
                        if not os.path.exists(OUTPUT_CATALOG + 'Остальные/' + fine_snils_(snils) + '/' + audiofileShort +
                                              '-' + str(Counter(audiofilesShort)[audiofileShort]) + audiofile[-4:]):
                            shutil.copy(audiofile #.replace(FIND_CATALOG, CHANGE_ON_WINDOWS)
                                        , OUTPUT_CATALOG + 'Остальные/' + fine_snils_(snils) + '/' + audiofileShort +
                                        '-' + str(Counter(audiofilesShort)[audiofileShort]) + audiofile[-4:])
                            audiofilesShort.append(audiofileShort)
                    else:
                        if not os.path.exists(OUTPUT_CATALOG + 'Остальные/' + fine_snils_(snils) + '/' +
                                              audiofileShort + audiofile[-4:]):
                            shutil.copy(audiofile #.replace(FIND_CATALOG, CHANGE_ON_WINDOWS)
                                        , OUTPUT_CATALOG + 'Остальные/' + fine_snils_(snils) + '/' + audiofileShort +
                                        audiofile[-4:])
                            audiofilesShort.append(audiofileShort)
                else:
                    print('!!! Нет исходного файла', audiofile)
            sucess = True
        except Exception as e:
            full_tb_write(e)
            print('Ошибка - пробуем ещё раз')
    print('Скопировано', i, 'из', len(snilsesProblem))


from django.contrib.auth.models import AbstractBaseUser, UserManager, \
    PermissionsMixin
from django.core import validators
from django.db import models
from django.utils import timezone

from accounts import constants

class User(AbstractBaseUser, PermissionsMixin):
    #: The Permission level for this user
    permission = models.CharField(max_length=40, blank=True, null=True,
                                  choices=constants.PERMISSION_CHOICES)
    """
    An abstract base class implementing a fully featured User model with
    admin-compliant permissions.

    Username, password and email are required. Other fields are optional.
    """
    username = models.CharField(
        'username', max_length=30, unique=True, help_text=_(
            'Required. 30 characters or fewer. Letters, digits and '
            '@/./+/-/_ only.'),
        validators=[
            validators.RegexValidator(r'^[\w.@+-]+$', _(
                'Enter a valid username.'), 'invalid')
        ])
    first_name = models.CharField('first name', max_length=30, blank=True)
    last_name = models.CharField('last name', max_length=30, blank=True)
    email = models.EmailField('email address', blank=True)
    is_staff = models.BooleanField(
        'staff status', default=False,
        help_text='Designates whether the user can log into this admin site.')
    is_active = models.BooleanField('active', default=True)
    date_joined = models.DateTimeField('date joined', default=timezone.now)
    company = models.ForeignKey('company.Company', on_delete=models.SET_NULL)

    objects = UserManager()

    USERNAME_FIELD = 'username'
    REQUIRED_FIELDS = ['email']

    def get_full_name(self):
        """
        Returns the first_name plus the last_name, with a space in between.
        """
        full_name = '%s %s' % (self.first_name, self.last_name)
        return full_name.strip()

    def get_short_name(self):
        "Returns the short name for the user."
        return self.first_name

    def get_content_data(self):
        content = {
            'permission': self.permission,
            'username': self.username,
            'first_name': self.first_name,
            'last_name': self.last_name,
            'email': self.email,
            'is_staff': self.is_staff,
            'is_active': self.is_active,
            'date_joined': self.date_joined,
        }

        return content

    def __unicode__(self):
        if self.first_name:
            if self.last_name:
                return "{0} {1}'s Profile".format(
                    self.first_name, self.last_name)
            else:
                return "{0}'s Profile".format(self.first_name)
        else:
            return "{0}'s Profile".format(self.username)

    def save(self, *args, **kwargs):
        super(User, self).save(*args, **kwargs)


import pandas as pd
from sklearn.impute import KNNImputer
from imblearn.over_sampling import SMOTE
from matplotlib import pyplot as plt

pd.set_option('display.max_columns', None)
pd.set_option('display.width', 1000)

df = pd.read_csv('./alzheimer.csv', skiprows=1, names=(
	'Group', 'M/F', 'Age', 'EDUC', 'SES', 'MMSE', 'CDR', 'eTIV', 'nWBV', 'ASF'
))

print(df.describe())

X = df.copy()
print(X.columns)
y = df[['Group']]
del X['Group']

X = pd.get_dummies(X)  # convert m/f to dummy columns

imputer = KNNImputer()
X = pd.DataFrame(imputer.fit_transform(X), columns=X.columns)

# scaler = StandardScaler()
# X = scaler.fit_transform(X)

smote = SMOTE()
X_smote, y_smote = smote.fit_resample(X, y)

demented = df[df['Group'] == 'Demented']
not_demented = df[df['Group'] == 'Nondemented']
converted = df[df['Group'] == 'Converted']

plt.figure(figsize=(25, 12))

plt.subplot(331)
plt.hist([demented['M/F'], not_demented['M/F'], converted['M/F']], rwidth=0.8)
plt.legend(['Demented', 'Not demented', 'Converted'])
plt.title("Frequency of demented, non-demented, and converted patients by gender")

plt.subplot(332)
plt.hist([demented['Age'], not_demented['Age'], converted['Age']], rwidth=0.8)
plt.legend(['Demented', 'Not demented', 'Converted'])
plt.title("Frequency of demented, non-demented, and converted patients by age")

plt.subplot(333)
plt.hist([demented['EDUC'], not_demented['EDUC'], converted['EDUC']], rwidth=0.8)
plt.legend(['Demented', 'Not demented', 'Converted'])
plt.title("Frequency of demented, non-demented, and converted patients by education")

plt.subplot(334)
plt.hist([demented['SES'], not_demented['SES'], converted['SES']], rwidth=0.8)
plt.legend(['Demented', 'Not demented', 'Converted'])
plt.title("Frequency of demented, non-demented, and converted patients by SES")

plt.subplot(335)
plt.hist([demented['MMSE'], not_demented['MMSE'], converted['MMSE']], rwidth=0.8)
plt.legend(['Demented', 'Not demented', 'Converted'])
plt.title("Frequency of demented, non-demented, and converted patients by MMSE")

plt.subplot(336)
plt.hist([demented['CDR'], not_demented['CDR'], converted['CDR']], rwidth=0.8)
plt.legend(['Demented', 'Not demented', 'Converted'])
plt.title("Frequency of demented, non-demented, and converted patients by CDR")

plt.subplot(337)
plt.hist([demented['eTIV'], not_demented['eTIV'], converted['eTIV']], rwidth=0.8)
plt.legend(['Demented', 'Not demented', 'Converted'])
plt.title("Frequency of demented, non-demented, and converted patients by eTIV")

plt.subplot(338)
plt.hist([demented['nWBV'], not_demented['nWBV'], converted['nWBV']], rwidth=0.8)
plt.legend(['Demented', 'Not demented', 'Converted'])
plt.title("Frequency of demented, non-demented, and converted patients by nWBV")

plt.subplot(339)
plt.hist([demented['ASF'], not_demented['ASF'], converted['ASF']], rwidth=0.8)
plt.legend(['Demented', 'Not demented', 'Converted'])
plt.title("Frequency of demented, non-demented, and converted patients by ASF")

plt.show()

"""
Notes

methods used:
- imputing using KNNImputer for missing values in SES and MMSE
- dummy variables for the categorical M/F column
- SMOTE due to the 'Converted' target value being a significant minority and consistently miscategorized without SMOTE
- StandardScaler was tested for logistic regression with little to no improvement in results
- TODO: try binning age

Feature selection:
- RFE (for basic logistic regression)
- FFS (for basic logistic regression)
- Feature importance (in random forest module)
- Based on the results from these methods, removed from logistic and bagging model:
	- EDUC
	- MMSE
- For mlxtend and random forest, removed:
	- M/F
	- EDUC
	- nWBV
- For stacked model, TBD
- RFE results:
	SES
	CDR
	nWBV
	ASF
	M/F_F
- FFS results:
	feature         ffs
	7     ASF    1.430156
	5    eTIV    2.684062
	0     Age   14.149083
	8   M/F_F   18.656900
	9   M/F_M   18.656900
	6    nWBV   28.090159
	1    EDUC   28.760364
	2     SES   41.344708
	3    MMSE  170.239290
	4     CDR  496.623041
- feature importance results:
	importance    feature
	3    0.482053     CDR
	2    0.236816    MMSE
	1    0.091915     SES
	4    0.067490    eTIV
	5    0.066786     ASF
	0    0.054942     Age

Data correlations w/ positive dementia:
- Males had a higher proportion
- Age, no obvious relationship
- Loose correlation with mid-level education
- Higher SES loosely correlated (socioeconomic status)
- Lower MMSE strongly correlated (mini mental state examination)
- Higher CDR strongly correlated (clinical dementia rating)
- eTIV, no obvious correlation (estimated intracranial volume)
- Lower nWBV strongly correlated (normalized whole brain volume)
- ASF, no obvious correlation (atlas scaling factor)

"""


import frappe
import re
import jwt
from frappe import _
from frappe.utils.data import cstr, cint, flt
from frappe.utils import getdate
from erpnext.regional.india.e_invoice.utils import (GSPConnector,raise_document_name_too_long_error,read_json,get_transaction_details,\
	validate_mandatory_fields,get_doc_details,get_overseas_address_details,get_return_doc_reference,\
	get_eway_bill_details,validate_totals,show_link_to_error_log,santize_einvoice_fields,safe_json_load,get_payment_details,\
	validate_eligibility,update_item_taxes,get_invoice_value_details,get_party_details,update_other_charges)

from erpnext.regional.india.utils import get_gst_accounts,get_place_of_supply
import json

GST_INVOICE_NUMBER_FORMAT = re.compile(r"^[a-zA-Z0-9\-/]+$")   #alphanumeric and - /

def validate_einvoice_fields(doc):
	invoice_eligible = validate_eligibility(doc)

	if not invoice_eligible:
		return

	# Finbyz Changes Start: dont change posting date after irn generated
	if doc.irn and doc.docstatus == 1 and doc._action == 'submit':
		if str(doc.posting_date) != str(frappe.db.get_value("Sales Invoice",doc.name,"posting_date")):
			frappe.throw(_('You cannot edit the invoice after generating IRN'), title=_('Edit Not Allowed'))

	# Finbyz Changes End
	if doc.docstatus == 0 and doc._action == 'save':
		if doc.irn and not doc.eway_bill_cancelled and doc.grand_total != frappe.db.get_value("Sales Invoice",doc.name,"grand_total"):# Finbyz Changes:
			frappe.throw(_('You cannot edit the invoice after generating IRN'), title=_('Edit Not Allowed'))
		if len(doc.name) > 16 and doc.authority == 'Authorized':# Finbyz Changes
			raise_document_name_too_long_error()

	elif doc.docstatus == 1 and doc._action == 'submit' and not doc.irn and doc.irn_cancelled == 0: # finbyz
		frappe.throw(_('You must generate IRN before submitting the document.'), title=_('Missing IRN'))

	elif doc.irn and doc.docstatus == 2 and doc._action == 'cancel' and not doc.irn_cancelled:
		frappe.throw(_('You must cancel IRN before cancelling the document.'), title=_('Cancel Not Allowed'))

def make_einvoice(invoice):
	validate_mandatory_fields(invoice)

	schema = read_json('einv_template')

	transaction_details = get_transaction_details(invoice)
	item_list = get_item_list(invoice)
	doc_details = get_doc_details(invoice)
	invoice_value_details = get_invoice_value_details(invoice)
	seller_details = get_party_details(invoice.company_address)

	if invoice.gst_category == 'Overseas':
		buyer_details = get_overseas_address_details(invoice.customer_address)
	else:
		buyer_details = get_party_details(invoice.customer_address)
		place_of_supply = get_place_of_supply(invoice, invoice.doctype)
		if place_of_supply:
			place_of_supply = place_of_supply.split('-')[0]
		else:
			place_of_supply = invoice.billing_address_gstin[:2]
		buyer_details.update(dict(place_of_supply=place_of_supply))

	seller_details.update(dict(legal_name=invoice.company))
	buyer_details.update(dict(legal_name=invoice.billing_address_title or invoice.customer_name or invoice.customer)) # finbyz change add billing address title

	shipping_details = payment_details = prev_doc_details = eway_bill_details = frappe._dict({})
	if invoice.shipping_address_name and invoice.customer_address != invoice.shipping_address_name:
		if invoice.gst_category == 'Overseas':
			shipping_details = get_overseas_address_details(invoice.shipping_address_name)
		else:
			shipping_details = get_party_details(invoice.shipping_address_name, is_shipping_address=True)

	if invoice.is_pos and invoice.base_paid_amount:
		payment_details = get_payment_details(invoice)

	if invoice.is_return and invoice.return_against:
		prev_doc_details = get_return_doc_reference(invoice)

	if invoice.transporter and flt(invoice.distance) and not invoice.is_return:
		eway_bill_details = get_eway_bill_details(invoice)

	# not yet implemented
	dispatch_details = period_details = export_details = frappe._dict({})

	einvoice = schema.format(
		transaction_details=transaction_details, doc_details=doc_details, dispatch_details=dispatch_details,
		seller_details=seller_details, buyer_details=buyer_details, shipping_details=shipping_details,
		item_list=item_list, invoice_value_details=invoice_value_details, payment_details=payment_details,
		period_details=period_details, prev_doc_details=prev_doc_details,
		export_details=export_details, eway_bill_details=eway_bill_details
	)

	try:
		einvoice = safe_json_load(einvoice)
		einvoice = santize_einvoice_fields(einvoice)
	except Exception:
		show_link_to_error_log(invoice, einvoice)

	validate_totals(einvoice)

	return einvoice

def get_item_list(invoice):
	item_list = []

	for d in invoice.items:
		einvoice_item_schema = read_json('einv_item_template')
		item = frappe._dict({})
		item.update(d.as_dict())

		item.sr_no = d.idx
		item.description = json.dumps(d.item_group or d.item_name)[1:-1] # finbyz change add item group

		item.qty = abs(item.qty)

		if invoice.apply_discount_on == 'Net Total' and invoice.discount_amount:
			item.discount_amount = abs(item.base_amount - item.base_net_amount)
		else:
			item.discount_amount = 0

		item.unit_rate = abs((abs(item.taxable_value) - item.discount_amount)/ item.qty)
		item.gross_amount = abs(item.taxable_value) + item.discount_amount
		item.taxable_value = abs(item.taxable_value)

		item.batch_expiry_date = frappe.db.get_value('Batch', d.batch_no, 'expiry_date') if d.batch_no else None
		item.batch_expiry_date = format_date(item.batch_expiry_date, 'dd/mm/yyyy') if item.batch_expiry_date else None
		#finbyz Changes
		if frappe.db.get_value('Item', d.item_code, 'is_stock_item') or frappe.db.get_value('Item', d.item_code, 'is_not_service_item'):
			item.is_service_item = 'N'
		else:
			item.is_service_item = 'Y'
		#finbyz changes enditem.serial_no = ""

		item = update_item_taxes(invoice, item)

		item.total_value = abs(
			item.taxable_value + item.igst_amount + item.sgst_amount +
			item.cgst_amount + item.cess_amount + item.cess_nadv_amount + item.other_charges
		)
		einv_item = einvoice_item_schema.format(item=item)
		item_list.append(einv_item)

	return ', '.join(item_list)

# india utils.py

def validate_document_name(doc, method=None):
	"""Validate GST invoice number requirements."""
	country = frappe.get_cached_value("Company", doc.company, "country")

	# Date was chosen as start of next FY to avoid irritating current users.
	if country != "India" or getdate(doc.posting_date) < getdate("2021-04-01"):
		return

	if len(doc.name) > 16 and doc.authority == 'Authorized': #finbyz
		frappe.throw(_("Maximum length of document number should be 16 characters as per GST rules. Please change the naming series."))

	if not GST_INVOICE_NUMBER_FORMAT.match(doc.name):
		frappe.throw(_("Document name should only contain alphanumeric values, dash(-) and slash(/) characters as per GST rules. Please change the naming series."))


import operator
from itertools import tee, starmap, groupby
from typing import Literal

def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = tee(iterable)
    next(b, None)
    return zip(a, b)

def has_six_digits(i: int):
    return 100_000 <= i <= 999_999

def is_nondecreasing_sequence(i: int):
    pairs = pairwise(str(i))
    goes_up = starmap(operator.le, pairs)
    nondecreasing = all(goes_up)
    return nondecreasing

def one_pair_is_not_triplet(i: int):
    return 2 in {sum(1 for _ in g) for _, g in groupby(str(i))}

def has_pair(i: int):
    pairs = pairwise(str(i))
    is_equal = starmap(operator.eq, pairs)
    return any(is_equal)

def moar_numbers(start_inclusive, stop_inclusive, part=Literal['a', 'b']):
    current = start_inclusive
    check_extra = part == 'b'
    while current <= stop_inclusive:
        current_fulfils_spec = (
                has_six_digits(current) and
                is_nondecreasing_sequence(current) and
                has_pair(current) and
                (not check_extra or one_pair_is_not_triplet(current))
        )
        if current_fulfils_spec:
            yield current
        current += 1


from flask import Flask, render_template, request, redirect, session
from datetime import datetime
import random

app = Flask(__name__)
app.secret_key = 'JSaafE54!@#$%$#%^&*()_+'

@app.route('/')
def index():
        #crear una variable session
        if 'num_azar' not in session:
            session['num_azar'] = random.randint(1,100)
            session['resultado'] = ""
            session['count'] = 0
            print(session['num_azar'])
        elif 'num_azar' in session:
            if session['count'] > 0 and session['count']<= 4:
                print("la session ya existe")
            elif session['count'] > 4 and session['resultado'] != 'igual':
                session['resultado'] = "perdiste"
        print(session['num_azar'])
        return render_template("index.html")

@app.route('/guess', methods=['POST'])
def guess():
        numero_ingresado = int(request.form['numero'])
        print(numero_ingresado)
        if session['num_azar'] == numero_ingresado:
                session['resultado'] = "igual"
                print('paso por igual')
        elif session['num_azar'] > numero_ingresado:
                session['resultado'] = "mayor"
                print('paso por mayor')
        elif session['num_azar'] < numero_ingresado:
                session['resultado'] = "menor"
                print('paso por menor')
        session['count'] += 1
        return redirect('/')

@app.route('/play_again', methods=['POST'])
def play_again():
    session.pop('num_azar')
    session.pop('resultado')
    session.pop('count')

    return redirect('/')

@app.route('/ranking', methods=['POST'])
def ranking():
    print(session['ranking'])
    if 'ranking' not in session:
        session['ranking'] = []

    session['ranking'] = [
        dict(nombre=request.form['nombre'], intentos=session['count'])]
    print(request.form['nombre'])
    print(session['ranking'])
    return render_template("ranking.html", ranking=session['ranking'])

if __name__=="__main__":
    app.run(debug=True)


from openpyxl import load_workbook
wb = load_workbook("sample.xlsx")
ws = wb.active

# 번호 영어 수학
# 번호 (국어) 영어 수학

ws.move_range("B1:C11", rows=0, cols=1) # 0줄 밑으로, 1줄 오른쪽으로 이동
ws["B1"].value = "국어" # B1 셀에 '국어' 입력

# ws.move_range("C1:C11", rows=5, cols=-1) # 데이터 옮기면서 덮어씀

wb.save("sample_korean.xlsx")


import os
import v2_draw_dynamic as main_app
import logging
import sys

def cmd(cmdstr):
    print(cmdstr)
    os.system(cmdstr)

def main():
    '''
    if len(sys.argv) < 2:
        logging.error("please input msg log file")
        return
    '''
    while True:
        try:
            main_app.draw_dynamic(r"D:\Programs\bin3(EC2)_R\log\010011112222\msg.log")
        except Exception as e:
            print("catch exception",e)
            logging.error(str(e))

main()