import numpy as np
import pandas as pd
import pickle, gc, shap, math, random, time

import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
from sklearn import metrics
from sklearn import model_selection
import lightgbm as lgb
import xgboost as xgb
from catboost import CatBoostClassifier
from sklearn.linear_model import LinearRegression
from sklearn.svm import NuSVR, SVR

# This script can also be used for stacking purpose by inputing training data set with oof results as features additional to earlier engineered features from different data sets.

def train_model_classification(X, y, params, groups, folds, model_type='lgb', eval_metric='auc', columns=None, plot_feature_importance=False, model=None,
                               verbose=10000, early_stopping_rounds=200, n_estimators=50000, weight = None, seed='no'):
    """
    A function to train a variety of regression models.
    Returns dictionary with oof predictions, test predictions, scores and, if necessary, feature importances.

    :params: X - training data, can be pd.DataFrame
    :params: X_test - test data, can be pd.DataFrame
    :params: y - target
    :params: folds - folds to split data
    :params: model_type - type of model to use
    :params: eval_metric - metric to use
    :params: columns - columns to use. If None - use all columns
    :params: plot_feature_importance - whether to plot feature importance of LGB
    :params: model - sklearn model, works only for "sklearn" model type

    """
    columns = X.columns if columns is None else columns
    models = []

    metrics_dict = {'auc': {'lgb_metric_name': 'auc',
                        'catboost_metric_name': 'AUC',
                        'sklearn_scoring_function': metrics.roc_auc_score},
                    }

    result_dict = {}

    # out-of-fold predictions on train data
    oof = np.zeros(len(X))

    scores = []
    train_loss = []
    feature_importance = pd.DataFrame()

    if groups is None:
        splits = folds.split(X)

    elif groups == 'stra':
        splits = folds.split(X,y)

    else:
        splits = folds.split(X, groups = groups)
        print('no')

    for fold_n, (train_index, valid_index) in enumerate(splits):

        print(f'Fold {fold_n + 1} started at {time.ctime()}')

        if type(X) == np.ndarray:
            X_train, X_valid = X[columns][train_index], X[columns][valid_index]
            y_train, y_valid = y[train_index], y[valid_index]
            weight_train = weight[train_index]
        else:
            X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
            y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
            weight_train = weight[train_index]

        if model_type == 'lgb':

            model = lgb.LGBMClassifier(**params, n_estimators = n_estimators, n_jobs = -1)
            model.fit(X_train, y_train, sample_weight = weight_train,
                    eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric=metrics_dict[eval_metric]['lgb_metric_name'],
                    verbose=verbose, early_stopping_rounds=early_stopping_rounds)

            y_pred_valid = model.predict_proba(X_valid)[:, 1]
            y_pred_train = model.predict_proba(X_train)[:, 1]
            models.append(model)

        if model_type == 'xgb':
            train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
            valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)

            watchlist = [(train_data, 'train'), (valid_data, 'valid_data')]
            model = xgb.train(dtrain=train_data, num_boost_round=n_estimators, evals=watchlist, early_stopping_rounds=early_stopping_rounds, verbose_eval=verbose, params=params)
            y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
            y_pred_train = model.predict(xgb.DMatrix(X_train, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
            models.append(model)

        if model_type == 'sklearn':
            model = model
            model.fit(X_train, y_train)

            y_pred_valid = model.predict(X_valid).reshape(-1,)
            score = metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid)
            print(f'Fold {fold_n}. {eval_metric}: {score:.4f}.')
            print('')
            models.append(model)

        if model_type == 'cat':
            model = CatBoostClassifier(iterations=n_estimators,  eval_metric=metrics_dict[eval_metric]['catboost_metric_name'], **params,
                                      )
            model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=verbose,early_stopping_rounds=early_stopping_rounds)
            y_pred_valid = model.predict_proba(X_valid)[:,1]
            y_pred_train = model.predict_proba(X_train)[:,1]
            models.append(model)

        oof[valid_index] = y_pred_valid.reshape(-1,)
        if eval_metric != 'group_mae':
            scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
            train_loss.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_train, y_pred_train))
        else:
            scores.append(metrics_dict[eval_metric]['scoring_function'](y_valid, y_pred_valid, X_valid['type']))

        with open(f'./models/models_{model_type}_{seed}.pickle', 'wb') as handle:
            pickle.dump(models, handle, protocol = pickle.HIGHEST_PROTOCOL)

        gc.collect()

        if model_type == 'lgb' and plot_feature_importance:
            # feature importance
            fold_importance = pd.DataFrame()
            fold_importance["feature"] = columns
            fold_importance["importance"] = model.feature_importances_
            fold_importance["shap_values"] = abs(shap.TreeExplainer(model).shap_values(X_valid)[:,:len(columns)]).mean(axis=0).T
            fold_importance["fold"] = fold_n + 1
            feature_importance = pd.concat([feature_importance, fold_importance], axis=0)

    print('Train loss mean: {0:.6f}, std: {1:.6f}.'.format(np.mean(train_loss), np.std(train_loss)))
    print('CV mean score: {0:.6f}, std: {1:.6f}.'.format(np.mean(scores), np.std(scores)))

    result_dict['oof'] = oof
    result_dict['scores'] = scores
    result_dict['models'] = models

    if model_type == 'lgb':
        if plot_feature_importance:
            feature_importance["importance"] /= folds.n_splits
            cols = feature_importance[["feature", "importance"]].groupby("feature").mean().sort_values(
                by="importance", ascending=False)[:50].index

            best_features = feature_importance.loc[feature_importance.feature.isin(cols)]

            plt.figure(figsize=(16, 12));
            sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False));
            plt.title('LGB Features (avg over folds)');

            result_dict['feature_importance'] = feature_importance

    return result_dict

def predict(df, models, model_type = 'lgb'):
    prediction = np.zeros(len(df))
    for model in models:

        if model_type == 'lgb':
            pred = model.predict_proba(df)[:, 1]
        if model_type == 'xgb':
            pred = model.predict(xgb.DMatrix(df, feature_names=df.columns), ntree_limit=model.best_ntree_limit)
        if model_type == 'cat':
            pred = model.predict_proba(df)[:,1]
        prediction += pred/len(models)

    return prediction

class train_config:

    def __init__(self, n_splits, features, model_type, model_params, eval_metric, early_stopping_rounds, n_estimators,
                 save_oof, seed):

        self.n_splits = n_splits
        self.features = features
        self.model_type = model_type
        self.model_params = model_params
        self.eval_metric = eval_metric
        self.early_stopping_rounds = early_stopping_rounds
        self.n_estimators = n_estimators
        self.save_oof = save_oof
        self.seed = seed

def model_train(df_train, train_config):

    n_splits = train_config.n_splits
    seed = train_config.seed
    folds = KFold(n_splits, shuffle = True, random_state = seed)

    X_train = df_train[train_config.features]
    y_train = df_train['TARGET']
    weight = df_train['weight']

    result_dict = train_model_classification(
                             X=X_train,
                             y=y_train,
                             params=train_config.model_params,
                             groups = None,
                             folds=folds, model_type=train_config.model_type, eval_metric=train_config.eval_metric,
                             plot_feature_importance=True,verbose=100, early_stopping_rounds=train_config.early_stopping_rounds,
                             n_estimators=train_config.n_estimators,weight = weight, seed = seed)

    return result_dict

if __name__ == "__main__":

    with open('./processed/train_processed.pickle', 'rb') as handle:
        train = pickle.load(handle)

    with open('./processed/test_processed.pickle', 'rb') as handle:
        test = pickle.load(handle)

    config_path = sys.argv[1]
    with open(config_path) as json_file:
        config = json.load(json_file)

    t_config = train_config(
        n_splits = config['n_splits'],
        features = config['features'],
        model_type = config['model_type'],
        model_params = config['model_params'],
        eval_metric = config['eval_metric'],
        early_stopping_rounds = config['early_stopping_rounds'],
        n_estimators = config['n_estimators'],
        save_oof = config['save_oof'],
        seed = config['seed'],
    )

    result_dict = model_train(train, t_config)

    models = result_dict['models']
    prediction = predict(test[t_config.features], models)

    if t_config.save_oof == True:

        train[f'oof_{t_config.model_type}_{t_config.seed}'] = result_dict['oof']
        test[f'oof_{t_config.model_type}_{t_config.seed}'] = prediction

        with open('./processed/train_processed.pickle', 'wb') as handle:
            pickle.dump(train, handle, protocol = pickle.HIGHEST_PROTOCOL)
        with open('./processed/test_processed.pickle', 'wb') as handle:
            pickle.dump(test, handle, protocol = pickle.HIGHEST_PROTOCOL)

    sample_submission = pd.read_csv('./input/sample_submission.csv')
    sample_submission['TARGET'] = prediction
    sample_submission.to_csv(f'./submissions/submission_{t_config.model_type}_{t_config.seed}.csv', index=False)


# Problem: The function name, its docstring, and parameter list of 'fn' are hidden by
# the wrapper function.

# Solution: The functools.wraps decorator, opies the lost metadata from the undecorated function
# to the decorated closure.

from functools import wraps # wraps(fn) -> dadurch gehen Metadaten wie Docu nicht verloren
import time

# Debugger
print("----------DEBUGGER----------")

def debug(fn):
    @wraps(fn)
    def debugger(*args, **kwargs):
        args_values_types = [(a, type(a)) for a in args]
        kwargs_values_types = [(k, v, type(v)) for k, v in kwargs.items()]
        print("Args: {}".format(args_values_types))
        print("Kwargs: {}".format(kwargs_values_types))
        print("Function {} called".format(fn.__name__))
        fn_result = fn(*args, **kwargs)
        print("Function {} returns: {}".format(fn.__name__, fn_result))
        print("---------------------------------------------")
        return fn_result
    return debugger

@debug
def do_something(a, b, c=None):
    """Do something.
    """
    return a + b if c else 0

@debug
def do_something2(a, b, c=None):
    return a - b if c else 0

@debug
def do_something3(a, b, c=None):
    return a * b if c else 0

@debug
def do_something4(a, b, c=None):
    return a / b if c else 0

do_something(10, 20, c=1)
do_something2(10, 20, c=1)
do_something3(10, 20, c=1)
do_something4(10, 20, c=1)

# TIMER
print("----------TIMER----------")

def timing(fn):
    @wraps(fn)
    def timer(*args, **kwargs):
        start_time = time.perf_counter()
        fn_result = fn(*args, **kwargs)
        end_time = time.perf_counter()
        time_duration = end_time - start_time
        print("Function {} took: {} s".format(fn.__name__, time_duration))
        return fn_result
    return timer

@timing
def do_something5(a, b, c=None):
    """Do something.
    """
    return a + b if c else 0

do_something5(a=10, b=20, c=True)

@timing
def iterate(n):
    val = 0
    for i in range(n):
        val += i
    return val

iterate(1_000_000)


import numpy as np
import matplotlib.pyplot as plt
import math
import scipy.linalg as linalg
import scipy.signal as signal
import json
from collections import deque

def peakHold(sig, holdTime):
    """
    sig: 入力信号。
    holdTime: ホールド時間。単位はサンプル数。
    """
    out = np.empty(len(sig))
    buffer = deque([0 for _ in range(holdTime)])
    hold = deque([])
    for i in range(len(sig)):
        x0 = sig[i]

        if len(hold) > 0:
            idx = len(hold) - 1
            while idx >= 0:
                if hold[idx] < x0:
                    hold.pop()
                else:
                    break
                idx -= 1

        hold.append(x0)

        buffer.append(x0)
        delayOut = buffer.popleft()

        if len(hold) > 0 and delayOut == hold[0]:
            hold.popleft()

        out[i] = hold[0] if len(hold) > 0 else 0

    return out

def applyCharacteristicCurve(amp, threshold):
    with np.errstate(divide='ignore', invalid='ignore'):
        return np.where(amp > threshold, threshold / amp, 1)

def getTriangleWindow(length):
    win = signal.get_window("bartlett", length)[::-1]
    win /= np.sum(win)
    return win

def applyLimiter(sig, absed, holdTime):
    hold = peakHold(absed, holdTime)
    gain = applyCharacteristicCurve(hold, 1.0)
    fir = getTriangleWindow(holdTime)
    smoothed = np.convolve(gain, fir)[holdTime - 1:]
    return sig * smoothed

def generateSignal(upRate, length, seed=None):
    rng = np.random.default_rng(seed)
    sig = 2.0 * rng.binomial(1, 0.5, length) - 1
    up = signal.resample(sig, upRate * len(sig))
    downAbs = np.max(np.abs(up).reshape((len(sig), upRate)), axis=1)
    return (sig, up, downAbs)

def testGenerateSignal():
    length = 16
    seed = 6871867313
    sig, up, downAbs = generateSignal(16, length, seed)

    plt.plot(np.abs(up))
    plt.plot(np.repeat(downAbs, 16))
    plt.grid()
    plt.show()

if __name__ == "__main__":
    length = 2048
    holdTime = 128
    upRate = 16
    seed = 6415456
    fix = 10**(-0.2 / 20)  # Safe guard gain. Not used for demonstration.

    sig, up, downAbs = generateSignal(upRate, length, seed)

    # This implementation skips pre-lowpass which applies to `sig` to reduce overshoots.

    limited = applyLimiter(sig, downAbs, holdTime)

    upLimited = signal.resample(limited, upRate * len(limited))
    peak = np.max(np.abs(upLimited))
    print(f"Peak [dB]: {20 * math.log10(peak)}")
    # upLimited *= fix  # Safe guard gain is disabled for demonstration.

    peakIndex = np.where(np.abs(upLimited) > 1)[0]
    plt.plot(up, lw=1, color="red", alpha=0.5, label="16x Source")
    plt.plot(upLimited, lw=1, color="black", label="16x Limited")
    plt.scatter(peakIndex, upLimited[peakIndex], marker="s", zorder=10, label="Overshoot")
    plt.legend(loc="lower right")
    plt.grid(color="#f0f0f0")
    plt.tight_layout()
    plt.show()


# https://mp.weixin.qq.com/s?__biz=MzI5MzQzMjU4Mw==&mid=2247487529&idx=1&sn=1de78b8e7141a118dd9c5c4eed15c8b2&chksm=ec736c41db04e557a639cd09722a3f3bec4643ca8f17bba17e92a2bd2b5802c6604355c5f001&scene=0&xtrack=1&key=dd5051400a9fb58f039a8acf24b9536d9ceca5303e36d6f09878cc021639e70a07a2aabff04519fe73d498d6c2477435848bde455e9d69c01dcd8e875574ba88afd10f012f06dabb3611ff790e6cd4c9&ascene=14&uin=MjA4NjMyMjkzOA%3D%3D&devicetype=Windows+10&version=62060833&lang=zh_CN&pass_ticket=I5KQYeQ4jpPG9uBdFjRVvm16WXRq6ukHz1Wq%2FLH61ndvGmkcVF5dkzqao5OLMBR3

import folium
import time
import requests
from urllib.request import quote
import numpy as np
import pandas as pd
import seaborn as sns
import webbrowser
from folium.plugins import HeatMap
address = ['北京','天津','石家庄','太原','呼和浩特','沈阳','大连','长春',
           '哈尔滨','上海','南京','杭州','宁波','合肥','福州','厦门','南昌',
           '济南','青岛','郑州','武汉','长沙','广州','深圳','南宁','海口',
           '重庆','成都','贵阳','昆明','拉萨','西安','兰州','西宁','银川','乌鲁木齐']

def getid(dizhi):
    url = "http://api.map.baidu.com/geocoder/v2/"
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36'}
    payload = {
        'output': 'json',
        'ak': 'X8zlxPUdSe2weshrZ1WqnWxb43cfBI2N'
    }
    addinfo = []
    for i in dizhi:
        payload['address'] = i
        try:
            content = requests.get(url, params=payload, headers=header).json()
            addinfo.append(content['result']['location'])
            print("正在获取{}的地址！".format(i))
        except:
            print("地址{}获取失败，请稍后重试！".format(i))
            pass
        time.sleep(.5)
    print("所有地址均已获取完毕！！！")
    return (addinfo)

if __name__ == "__main__":
    # 计时开始：
    t0 = time.time()
    myaddress = getid(address)
    t1 = time.time()
    total = t1 - t0
    print("消耗时间：{}".format(total))
    lon = np.array([i["lng"] for i in myaddress], dtype=float)
    lat = np.array([i["lat"] for i in myaddress], dtype=float)
    scale = np.random.randint(100, 500, len(address))
    data1 = [[lat[i], lon[i], scale[i]] for i in range(len(address))]

    map_osm = folium.Map(location=[35, 110], zoom_start=5)
    HeatMap(data1).add_to(map_osm)
    file_path = "E:/研一/cities.html"
    map_osm.save(file_path)
    webbrowser.open(file_path)


from flask import jsonify, make_response, request
from flask_pymongo import PyMongo, ObjectId
from ..app import app

mongo = PyMongo(app)
valoraciones = mongo.db.valoraciones

# ---- ENDPOINTS -------------------------------------------------------------------------

@app.route("/api/v1/valoraciones", methods=["POST"])
def createValoracion():
    if request.is_json:
        if valoraciones.find_one({'idviaje' : ObjectId(request.json['idviaje']), 'usuarioQueValora' : ObjectId(request.json["usuarioQueValora"]), 'usuarioValorado' : ObjectId(request.json['usuarioValorado'])}) is None :

            try:
                res = valoraciones.insert_one({
                    "usuarioValorado": ObjectId(request.json["usuarioValorado"]),
                    "nota": str(request.json["nota"]),
                    "descripcion": str(request.json["descripcion"]),
                    "usuarioQueValora": ObjectId(request.json["usuarioQueValora"]),
                    "idviaje": ObjectId(request.json["idviaje"]),
                    "nombre": str(request.json["nombre"]),
                    "apellidos": str(request.json["apellidos"]),
                    "url_foto_perfil": str(request.json["url_foto_perfil"]),
                })
                return jsonify(msg="Valoración creada", new_id=str(res.inserted_id))

            except:
                respuesta = jsonify(msg="Petición no válida, faltan campos o no son del tipo correcto")
                return make_response(respuesta, 400)
        else :
            print(valoraciones.find_one({'idviaje' : ObjectId(request.json['idviaje']), 'usuarioQueValora' : ObjectId(request.json["usuarioQueValora"]), 'usuarioValorado' : ObjectId(request.json['usuarioValorado'])}))
            return jsonify(msg="El usuario ya ha valorado a esa persona en el mismo trayecto")
    else:
        respuesta = jsonify(msg="Petición no válida, se requiere JSON")
        return make_response(respuesta, 400)

@app.route("/api/v1/valoraciones/<id>", methods=["GET"])
def getValoracionesPorUsuario(id):
    media = 0
    res = []
    for doc in valoraciones.find({'usuarioValorado' : ObjectId(id)}):
        media = media + int(doc['nota'])
        res.append({
            '_id' : str(doc['_id']),
            'usuarioValorado' : str(doc['usuarioValorado']),
            'nota' : doc['nota'],
            'descripcion' : doc['descripcion'],
            'usuarioQueValora' : str(doc['usuarioQueValora']),
            'idviaje' : str(doc['idviaje']),
            "nombre": doc["nombre"],
            "apellidos": doc["apellidos"],
            "url_foto_perfil": doc["url_foto_perfil"]
        })

    if media > 0:
        media = int(media/len(res))
    return jsonify({ 'media' : media, 'valoraciones' : res})

@app.route("/api/v1/valoraciones/<uValorado>/<uValora>/<idViaje>", methods=["GET"])
def getValoracion(uValorado,uValora,idViaje):

    res = valoraciones.find_one({
        'idviaje' : ObjectId(idViaje),
        'usuarioQueValora' : ObjectId(uValora),
        'usuarioValorado' : ObjectId(uValorado)})
    print(res)
    if (res is not None):
        return jsonify({ "msg": str(res["_id"])})
    else:
        return jsonify("null")


from TOOLS.request_fetch import FETCH
from TOOLS.redissave import Redisclient

# 爬虫逻辑
import requests
import re
from lxml import etree
from .address import find_address
from .tessert_atb import get_phone
from TOOLS.md5encode import phone_encrypte
from TOOLS.mongosave import MongoDB
from .ippool import proxies, get_ip
from TOOLS.request_fetch import FETCH

class Atb_spider():
    def __init__(self):
        self.starturl = 'http://bj.atobo.com/'
        # 数据类型模板
        self.data_demo = {'_id': '', 'category_name': '', 'company_name': '', 'company_phone': '',
                          'company_address': ''}
        self.f = FETCH()
        self.m = MongoDB('mongodb://10.2.1.121:17017,10.2.1.122:17017,10.2.1.123:17017', 'clues_resources',
                         'ZZ_20201214_rollback')

    def get_html(self, url):
        html = self.f.fetch(url)
        return html

    def parse_category_html(self, html):
        # 解析种类页面获取种类名称和种类url
        response = etree.HTML(html)
        category_list = response.xpath('//div[@class="sidebar-category"]/ul//li/p[@class="pcategory_son"]/a')
        for category in category_list:
            category_name = category.xpath('./text()').get()
            category_url = category.xpath('./@href').get()
            category_url = 'http:' + category_url

            # yield category_url
            yield self.parse_more_html(self.get_html(category_url), category_name=category_name)

    def parse_more_html(self, html, category_name):
        # 获取更多页选项的url
        response = etree.HTML(html)
        more_company_url = response.xpath('//div[@class="product-list-more"]/a/@href').get()

        yield self.parse_allpage(self.get_html(more_company_url), category_name=category_name)

    def parse_allpage(self, html, category_name):
        # 获取所有翻页url
        # 调用parse_one_url解析每个企业url yield出去，一页结束进入下方代码解析下一页url，再解析每个企业url
        self.parse_one_url(html, category_name=category_name)

        response = etree.HTML(html)
        one_page_url = response.xpath('//a[contains(text(),"下一页")]/@href').extract()
        if one_page_url:
            one_page_url = 'http://www.atobo.com' + one_page_url[0]
            if one_page_url != 'http://www.atobo.com#':
                yield self.parse_allpage(self.get_html(one_page_url), category_name=category_name)

    def parse_one_url(self, html, category_name):
        # 获取一页当中每个企业的url list
        response = etree.HTML(html)
        one_url_list = response.xpath(
            '//li[@class="product_box"]//li[@class="pp_name"]//a[@class="CompanyName"]/@href').extract()
        for one_url in one_url_list:
            one_url = "http://www.atobo.com/" + one_url

            yield self.parse_data(self.get_html(one_url), category_name=category_name)

    def parse_data(self, html, category_name):
        # 获取最终数据
        response = etree.HTML(html)
        try:
            # 图片规则
            company_name = response.xpath('//div[@class="company-intro"]//tr[1]/td/text()').get()
            company_address = response.xpath('//div[@class="company-intro"]//table//tr[2]').get()
            company_address = find_address(company_address)
            if not company_name:
                # 非图片规则
                company_name = response.xpath('//div[@class="company-banner"]//p[@class="t-companyname"]/text()').get()
                company_address = response.xpath('//div[@class="card-context"]//ul[2]//li[2]').get()
                company_address = find_address(company_address)
        except Exception:
            company_name = None
            company_address = None

        try:
            # 有手机号的情况
            company_phone = response.xpath('//div[@class="company-intro"]//tr/td/img/@src').extract()[1]
            company_phone = 'http:' + company_phone
            phone2 = get_phone(picurl=company_phone)
            phone = ''.join(phone2)

            if not company_phone:
                phone1 = response.xpath('//ul[4]//li[2]').get()
                pattern = re.compile("\d+", re.S)
                phone = pattern.findall(phone1)
                if not phone:
                    phone3 = response.xpath('//div[@class="company-intro"]//tr/td/img/@src').extract()[0]
                    try:
                        phone4 = 'http:' + phone3
                        phone5 = get_phone(picurl=phone4)
                        phone = ''.join(phone5)
                    except:
                        phone4 = 'http:' + phone3
                        phone5 = get_phone(picurl=phone4)
                        phone = phone5[0]

        except Exception:
            phone = None

        if company_name and phone:
            # mongo储存数据
            id = phone_encrypte(phone)
            data = self.data_demo
            data["_id"] = id
            data["categoryname"] = category_name
            data["companyname"] = company_name
            data["companyaddress"] = company_address
            data["companyphone"] = phone
            self.m.save_data(datadict=data)
            yield

    def runspider(self):
        html = self.get_html(self.starturl)
        self.parse_category_html(html)


from unittest import TestCase
from unittest.mock import patch, call, MagicMock

from django.test import RequestFactory

from learn.infrastructure.database import Database
from learn.views.wrong_answer import exercise_wrong_answer

class ExerciseWrongAnswerTests(TestCase):
    @patch("learn.views.wrong_answer.render")
    def test_shouldRenderWrongAnswer_WithDictionaryPK_AndTranslationPK_AndTranslation(self, render_mock):
        # Given
        database = MagicMock(Database)
        factory = RequestFactory()
        request = factory.get('fake-url')

        # When
        exercise_wrong_answer(request, 23, 42, database=database)

        # Then
        expected_args = call(request, 'learn/exercise_wrong_answer.html', context={
            'dictionary_pk': 23,
            'translation_pk': 42,
            'translation': database.get_translation()
        })
        self.assertEqual(render_mock.call_args_list[0], expected_args)


import pygame
import random
import time

pygame.init()

displayW = 1920
displayH = 1080

Display = pygame.display.set_mode((displayW,displayH), pygame.FULLSCREEN)

pygame.display.set_caption('game')

clock = pygame.time.Clock()

black = (0, 0, 0)
white = (255,255,255)
red = (255, 1, 1)
blue = (1, 1, 255)

x = (displayW/10)
y = (displayH - 155)
width = 125
height = 125
speed = 10
jump = False
jumpCount = 10

OBx = displayW - 70
OBy = displayH - 130
OBw = 80
OBh = 80

OB2x = displayW - 70
OB2y = displayH - 230
OB2w = 80
OB2h = 80

OB3x = displayW - 70
OB3y = displayH - 330
OB3w = 80
OB3h = 80

on = True

points = 0
lives = 3
on = True
testv = random.randrange(1, 9)

def player():
	pygame.draw.rect(Display, red, (x, y, width, height))

def Object():
	pygame.draw.rect(Display, white, (OBx, OBy, OBw, OBh))

def Object2():
	pygame.draw.rect(Display, white, (OB2x, OB2y, OB2w, OB2h))

def Object3():
	pygame.draw.rect(Display, white, (OB3x, OB3y, OB3w, OB3h))

def movement():
	global x, y, speed, width, jump, jumpCount
	movement = pygame.key.get_pressed()
	if movement[pygame.K_a] or movement[pygame.K_LEFT] and x > speed + 25:
		x -= speed
	elif movement[pygame.K_d]or movement[pygame.K_RIGHT] and x < displayW - 25 - speed - width:
		x += speed
	if not(jump):
		if movement[pygame.K_w]or movement[pygame.K_UP]:
			jump = True;
	else:
		if jumpCount >= -10:
			y -= (jumpCount * abs(jumpCount)) * .4
			jumpCount -= .5
		else:
			jumpCount = 10
			jump = False
def test():
	global testv, OBx, OB2x, OB3x
	if testv >= 1 and testv <= 3:
		OBx -= 20
		Object()
		ObjectCollision()
	if testv >= 4 and testv <= 6:
		OB2x -= 20
		Object2()
		Object2Collision()
	if testv >= 7 and testv <= 9:
		OB3x -= 20
		Object3()
		Object3Collision()

def ObjectCollision():
	global OBx, OBy, DisplayH, DisplayW, x, y, testv
	if OBx < 30:
		OBx = displayW - 70
		OBy = displayH - 130
		testv = random.randrange(1, 9)
		print('1')
	if x + 95 > OBx and x < OBx + 80 and y + 95 > OBy and y < OBy + 80:
		hit()

def Object2Collision():
	global OB2x, OB2y, DisplayH, DisplayW, x, y, testv
	if OB2x < 30:
		OB2x = displayW - 70
		OB2y = displayH - 230
		testv = random.randrange(1, 9)
		print('2')
	if x + 95 > OB2x and x < OB2x + 80 and y + 95 > OB2y and y < OB2y + 80:
		hit()

def Object3Collision():
	global OB3x, OB3y, DisplayH, DisplayW, x, y, testv
	if OB3x < 30:
		OB3x = displayW - 70
		OB3y = displayH - 330
		testv = random.randrange(1, 9)
		print('3')
	if x + 95 > OB3x and x < OB3x + 80 and y < OB3y + 80 and y + 120 > OB3y:
		hit()

def Boundarybox():
	pygame.draw.rect(Display, blue, (0, 0, 30, displayH)) #left
	pygame.draw.rect(Display, blue, (displayW, 0, -30, displayH)) #right
	pygame.draw.rect(Display, blue, (displayW, displayH, -displayW, -30)) #bottom
	pygame.draw.rect(Display, blue, (0, 0, displayW, 30)) #top

def fonttype(point, font):
	fontOBor = font.render(point, True, white)
	return fontOBor, fontOBor.get_rect()

def hit():
	global on, displayW, displayH
	Display.fill(black)
	largeText = pygame.font.SysFont('arial',115)
	fontOBor, TextRect = fonttype("Lost", largeText)
	TextRect.center = ((displayW/2),(displayH/2))
	Display.blit(fontOBor, TextRect)
	pygame.display.update()
	clock.tick(15)
	time.sleep(2)
	on = False
	print('lost')

def game():
	global on, points, OBx, OB2x, OB3x
	on = True
	while on:
		for event in pygame.event.get():
			if event.type == pygame.QUIT:
				on = False

		movement()
		Display.fill(black)
		Boundarybox()
		test()
		player()
		pygame.display.update()
		clock.tick(75)
		QorE = pygame.key.get_pressed()
		if QorE[pygame.K_ESCAPE]:
			on = False
game()


from flask import Flask
from gevent.pywsgi import WSGIServer
import yaml
import requests
from datetime import datetime
import socket

class LocationException(Exception):
  def __init__(self, message):
    self.message = message

def at_time():
    now = datetime.now()
    return now.strftime("%Y-%m-%d %H:%M:%S")

app=Flask(__name__)

def read_config(file_name):
    with open(file_name, 'r') as file:
        load_config = yaml.safe_load(file)
        url=load_config['url']
        loc=load_config["location"]
    return url,loc

def check_rainfall(url,loc):
    resp=requests.get(url)
    status=""
    if resp.status_code == 200:
        timestamp=resp.json()["items"][0]["timestamp"]
        time=timestamp.split('T')[1][0:5]
        station_readings=resp.json()["items"][0]["readings"]
        reading_unit=resp.json()["metadata"]["reading_unit"]
        for station in resp.json()["metadata"]["stations"]:
            if station['name'] == loc:
                for reading in station_readings:
                    if reading["station_id"]==station['id']:
                        status="Raining" if reading["value"] >0 else "Not Raining"
                        output=station['name']+', '+time+', '+str(reading["value"])+reading_unit+', '+status
                        break
                status="found"
                break
        if status=="":
           raise LocationException("Location "+loc+" doesn't exist")
    else:
        print(at_time()+" - URL=> "+url+" is not correct")
        output="There is some internal error. Please contact the admin"
    return output

@app.route('/',methods=['GET'])
def query_rainfall():
    try:
        url,loc=read_config("conf/config.yaml")
        return check_rainfall(url,loc)
    except Exception as e:
        print(at_time()+" - "+str(e))
        return "There is some internal error. Please contact the admin"

if __name__=='__main__':
    http_server = WSGIServer(('',8080),app)
    print(at_time()+" - Rainfall Service started on "+ socket.gethostbyname(socket.gethostname())+':8080')
    http_server.serve_forever()


# -*- coding: utf-8 -*-
# @author: Vitor Vilas Boas
import itertools
import numpy as np
from scipy.fftpack import fft
from scipy.linalg import eigh
from sklearn.pipeline import Pipeline
from scipy.signal import lfilter, butter
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import cross_val_score, StratifiedShuffleSplit, StratifiedKFold

class CSP():
    def __init__(self, n_components):
        self.n_components = n_components
        # self.filters_ = None
    def fit(self, X, y):
        e, c, s = X.shape
        classes = np.unique(y)
        Xa = X[classes[0] == y,:,:]
        Xb = X[classes[1] == y,:,:]
        S0 = np.zeros((c, c))
        S1 = np.zeros((c, c))
        for epoca in range(int(e/2)):
            S0 += np.dot(Xa[epoca,:,:], Xa[epoca,:,:].T) #covA Xa[epoca]
            S1 += np.dot(Xb[epoca,:,:], Xb[epoca,:,:].T) #covB Xb[epoca]
        [D, W] = eigh(S0, S0 + S1)
        ind = np.empty(c, dtype=int)
        ind[0::2] = np.arange(c - 1, c // 2 - 1, -1)
        ind[1::2] = np.arange(0, c // 2)
        W = W[:, ind]
        self.filters_ = W.T[:self.n_components]
        return self # instruction add because cross-validation pipeline
    def transform(self, X):
        XT = np.asarray([np.dot(self.filters_, epoch) for epoch in X])
        XVAR = np.log(np.mean(XT ** 2, axis=2)) # Xcsp
        return XVAR

if __name__ == "__main__":

    subjects = ['aa','al','av','aw','ay']
    classes = [1, 3] # only RH, FT
    n_channels = 118
    csp_ncomp = np.arange(2, n_channels+1, 2)

    suj = subjects[1] # 0 a 4
    n_comp = 18 # 2 a n_channels

    fs = 100
    fl = 8
    fh = 30
    nf = fs/2.

    ordem = 5
    b, a = butter(ordem, [fl/nf, fh/nf], btype='bandpass')

    bin0 = int(fl * (fs/nf)) # para fl = 8 bin0 = 15
    binN = int(fh * (fs/nf)) # para fl = 8 bin0 = 15

    t_start,t_end = 0,4
    W_Start = int(t_start * fs)
    W_End = int(t_end * fs)

    X = list(np.load('/mnt/dados/datasets/eeg_epochs/BCI3_4a/' + suj + '.npy'))

    X = np.concatenate([X[0],X[1]]) # X[0]=RH, X[1]=FT
    y = np.concatenate([np.zeros(int(len(X)/2)), np.ones(int(len(X)/2))]) # vetor gabarito

    X = X[:,:,W_Start:W_End]

    # Spectral filtering
    XFF = fft(X)
    REAL = np.transpose(np.real(XFF)[:,:,bin0:binN], (2, 0, 1))
    IMAG = np.transpose(np.imag(XFF)[:,:,bin0:binN], (2, 0, 1))
    XF0 = list(itertools.chain.from_iterable(zip(IMAG, REAL)))
    XF = np.transpose(XF0, (1, 2, 0))

    # Temporal filtering
    # XF = lfilter(b, a, X)

    ## Cross-validation:
    process = Pipeline([('CSP', CSP(n_comp)), ('classifier', LinearDiscriminantAnalysis())])
    #kf = StratifiedShuffleSplit(10, test_size=0.1, random_state=0)
    kf = StratifiedKFold(10, False, 0)
    scores = cross_val_score(process, XF, y, cv=kf)

    acc = np.mean(scores)
    print(suj, round(acc*100,1))


GIVEN_FILE_NAME = "given_12.txt"
INPUT_FILE_NAME = "input_12.txt"

from typing import List, Tuple

class Grid:
    def __init__(self):
        self.rows: List[list] = []
        self.starting_point: Tuple(int, int) = None
        self.ending_point: Tuple(int, int) = None

    def add_row(self, row: list):
        self.rows.append(row)

    def get_rows(self) -> List[list]:
        return self.rows

    def get_columns(self) -> List[list]:
        return list(map(list, zip(*self.rows)))

    def get_item(self, column, row):
        if (
            row < 0
            or column < 0
            or row >= len(self.rows)
            or column >= len(self.rows[0])
        ):
            return None
        return self.rows[row][column]

    def set_starting_point(self, column: int, row: int):
        self.starting_point = (column, row)

    def set_ending_point(self, column: int, row: int):
        self.ending_point = (column, row)

    def __repr__(self):
        return "\n".join("".join(str(item) for item in row) for row in self.rows)

class Solver:
    STARTING_POINT = "S"
    ENDING_POINT = "E"

    def parse_file(self, input_file_name: str) -> Grid:
        grid = Grid()
        current_row = 0
        with open(input_file_name) as f:
            current_column = 0
            for line in f:
                stripped_line: str = line.strip()
                list_of_elevations: list[str] = []
                for elevation_raw in stripped_line:
                    if elevation_raw == self.STARTING_POINT:
                        list_of_elevations.append("a")
                        grid.set_starting_point(current_column, current_row)
                    elif elevation_raw == self.ENDING_POINT:
                        list_of_elevations.append("z")
                        grid.set_ending_point(current_column, current_row)
                    else:
                        list_of_elevations.append(elevation_raw)
                    current_column += 1
                grid.add_row(list_of_elevations)
                current_column = 0
                current_row += 1
        return grid

    def get_shortest_scenic_path(self, grid: Grid) -> int:
        min_length = None
        current_row = 0
        possible_starting_points = []
        for row in grid.rows:
            current_column = 0
            for height in row:
                if height == "a" or height == "S":
                    possible_starting_points.append((current_column, current_row))
                current_column += 1
            current_row += 1
        for starting_point in possible_starting_points:
            column, row = starting_point
            grid.set_starting_point(column, row)
            length = self.get_shortest_path_length(grid)
            if length > 0 and (min_length is None or length < min_length):
                min_length = length
        return min_length

    def get_shortest_path_length(self, grid: Grid) -> int:
        starting_column, starting_row = grid.starting_point
        ending_column, ending_row = grid.ending_point
        points_to_visit = [(starting_column, starting_row, 0)]
        min_length = None
        visited_points = set()
        while points_to_visit:
            current_column, current_row, length_so_far = points_to_visit.pop(0)
            if current_column == ending_column and current_row == ending_row:
                if min_length is None or length_so_far < min_length:
                    min_length = length_so_far
            current_elevation = grid.get_item(current_column, current_row)
            # Top
            top_elevation = grid.get_item(current_column, current_row - 1)
            if (
                top_elevation
                and ord(top_elevation) <= (ord(current_elevation) + 1)
                and (current_column, current_row - 1) not in visited_points
            ):
                points_to_visit.append(
                    (current_column, current_row - 1, length_so_far + 1)
                )
                visited_points.add((current_column, current_row - 1))

            # Left
            left_elevation = grid.get_item(current_column - 1, current_row)
            if (
                left_elevation
                and ord(left_elevation) <= (ord(current_elevation) + 1)
                and (current_column - 1, current_row) not in visited_points
            ):
                points_to_visit.append(
                    (current_column - 1, current_row, length_so_far + 1)
                )
                visited_points.add((current_column - 1, current_row))

            # Bottom
            bottom_elevation = grid.get_item(current_column, current_row + 1)
            if (
                bottom_elevation
                and ord(bottom_elevation) <= (ord(current_elevation) + 1)
                and (current_column, current_row + 1) not in visited_points
            ):
                points_to_visit.append(
                    (current_column, current_row + 1, length_so_far + 1)
                )
                visited_points.add((current_column, current_row + 1))

            # Right
            right_elevation = grid.get_item(current_column + 1, current_row)
            if (
                right_elevation
                and ord(right_elevation) <= (ord(current_elevation) + 1)
                and (current_column + 1, current_row) not in visited_points
            ):
                points_to_visit.append(
                    (current_column + 1, current_row, length_so_far + 1)
                )
                visited_points.add((current_column + 1, current_row))
        return min_length or -1

import unittest

class SolverTest(unittest.TestCase):
    def solve_1(self, input_file_name: str) -> int:
        solver = Solver()
        grid = solver.parse_file(input_file_name)
        shortest_path_length: int = solver.get_shortest_path_length(grid)
        return shortest_path_length

    def test_given_1(self):
        shortest_path_length = self.solve_1(GIVEN_FILE_NAME)
        self.assertEqual(shortest_path_length, 31)

    def test_input_1(self):
        shortest_path_length = self.solve_1(INPUT_FILE_NAME)
        self.assertEqual(shortest_path_length, 468)

    def solve_2(self, input_file_name: str) -> int:
        solver = Solver()
        grid = solver.parse_file(input_file_name)
        shortest_scenic_path: int = solver.get_shortest_scenic_path(grid)
        return shortest_scenic_path

    def test_given_2(self):
        shortest_scenic_path = self.solve_2(GIVEN_FILE_NAME)
        self.assertEqual(shortest_scenic_path, 29)

    def test_input_2(self):
        shortest_scenic_path = self.solve_2(INPUT_FILE_NAME)
        self.assertEqual(shortest_scenic_path, 459)

unittest.main(exit=False)


# -*- coding: utf-8 -*-
import json
import urllib.request as request
import os

def handleEsim():
    urlad = "http://172.16.139.92:8201/_cat/indices?h=index"
    req = request.Request(urlad)
    #req.add_header('Content-Type', 'application/json')
    response = request.urlopen(req)
    print(response)

from elasticsearch import Elasticsearch
from datetime import datetime

def handleEsim2():
    es = Elasticsearch(hosts="http://10.16.238.103:8200")
    for line in open("C:\\Users\\ay05\\Desktop\\new_e11.txt"):
        #print(es.search(index=line, body={"query": {"match_all": { }}}))
        #test(line)
        line = line.strip('\n')
        creation_date = test(line)
        print(creation_date)
        body ={"index": line, "creation_date": creation_date, "location": "E11"}
        es.index(index='index_create_date_e11', doc_type='_doc', body=body, id=None)

def test(indexname):
    req = request.Request("http://172.16.139.99:8201/"+indexname+"/_settings/index.creation_date")
    req.add_header('Content-Type', 'application/json')
    response = request.urlopen(req)
    jsonBody = json.loads(response.read())
    creation_date = jsonBody[indexname]["settings"]["index"]["creation_date"]
    return creation_date

if __name__ == '__main__':
    #test()
    handleEsim2()


import os
from arancino.utils.ArancinoUtils import ArancinoLogger
from gunicorn.glogging import Logger

class GunicornLogger(Logger):
    def setup(self, cfg):
        super().setup(cfg)
        self.access_log = ArancinoLogger.Instance().getLogger()
        self.error_log = ArancinoLogger.Instance().getLogger()

bind = '0.0.0.0:1475'
workers = 1
worker_class = 'sync'
timeout = 300
keepalive = 2

logger_class = GunicornLogger
# errorlog = os.path.join(os.getenv('ARANCINOLOG'), 'arancino.error.log')
# loglevel = 'info'
# accesslog = os.path.join(os.getenv('ARANCINOLOG'), 'arancino.log')
daemon = False


import operator
from typing import Any, Callable, List, Optional, overload, Set, Tuple, Union

from optcom.utils.utilities_helpers.terminal_display_helpers \
    import warning_terminal

def check_attr_type(attr: Any, attr_name: str, *types: Any) -> None:
    """Check if the provided attribute is of the provided type."""
    res = False
    for type in types:
        if (type is None):    # Allow attr to be None
            if (attr is None):
                res = True
        else:
            res = res or isinstance(attr, type)   # Keep the True
    if (not res):
        types_name = locals()['types']
        raise TypeError("{} must be one of the following type: {}"
                        .format(attr_name, types_name))

def check_attr_range(attr: Any, attr_name: str, min_attr: Union[int, float],
                     max_attr: Union[int, float], strict_left: bool = False,
                     strict_right: bool = False) -> None:
    """Check if the provided attribute is in the range provided."""
    if (strict_left):
        op_left = operator.lt
    else:
        op_left = operator.le
    if (strict_right):
        op_right = operator.gt
    else:
        op_right = operator.ge

    if (not(op_left(min_attr, attr) and op_right(max_attr, attr))):
        raise IndexError("Attribute {} out of range, must be in interval "
                         "[{},{}]"
                         .format(attr_name, min_attr, max_attr))

def check_attr_value(attr: Any, attr_values: List[Any], attr_default: Any
                     ) -> Any:
    """Check if the provided attribute is one of the allowed values
    provided. If not, return a provided default value.
    """
    if (attr not in attr_values):
        warning_terminal("The attribute specified '{}' is not "
            "supported, default attribute '{}' set instead."
            .format(attr, attr_default))

        return attr_default
    else:

        return attr

Quantum Cryptography/ML Kalman Filter/ml_kalman/Lib/site-packages/optcom/utils/utilities_helpers/attr_helpers.py


# Importing all necessary libraries
import cv2
import os

# Read the video from specified path
cam = cv2.VideoCapture("E:\\电影\\搭错车 1983.avi")

try:

    # creating a folder named data
    if not os.path.exists('data'):
        os.makedirs('data')

# if not created then raise error
except OSError:
    print('Error: Creating directory of data')

# frame
currentframe = 0

# reading from frame

Correctdata = [];
with open("timeSeries", "r", encoding='UTF-8') as f:  # 打开文件
    data = f.readlines()  # 读取文件
    for x in data:
        temp = x.replace('\n','');
        tempList = temp.split(":")
        if len(tempList) == 2:
            Correctdata.append(int(tempList[0]) * 60 + int(tempList[1]))
        if len(tempList) == 3:
            Correctdata.append(int(tempList[0]) * 3600 + int(tempList[1]) * 60 + int(tempList[2]))
print(Correctdata)

for x in Correctdata:
    cam.set(cv2.CAP_PROP_POS_MSEC, x * 1000)
    ret, frame = cam.read()
    if ret:
        name = './data/frame' + str(currentframe) + '.jpg'
        print('Creating...' + name)

        # writing the extracted images
        cv2.imwrite(name, frame)
        currentframe += 1

cam.release()
cv2.destroyAllWindows()


"""Test the open functionality."""
from aiofiles.threadpool import open as aioopen, wrap
import pytest

@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["r", "rb"])
async def test_file_not_found(mode):
    filename = "non_existent"

    try:
        open(filename, mode=mode)
    except Exception as e:
        expected = e

    assert expected

    try:
        await aioopen(filename, mode=mode)
    except Exception as e:
        actual = e

    assert actual

    assert actual.errno == expected.errno
    assert str(actual) == str(expected)

def test_unsupported_wrap():
    """A type error should be raised when wrapping something unsupported."""
    with pytest.raises(TypeError):
        wrap(int)


import os
import requests
import logging

def get_weather(city):
    """ Функция для получения ответа от сервиса openweathermap.org. """
    params = {
        'q': city,
        'units': 'metric',
        'APPID': os.environ['WEATHER_API_KEY'],
    }
    url = 'http://api.openweathermap.org/data/2.5/weather'
    try:
        resp = requests.get(url, params=params)
        return resp
    except (requests.exceptions.ConnectionError,
            requests.exceptions.ConnectTimeout) as e:
        logging.error(e)
        return None

def process_owm_response(response):
    """ Функция для разбора результата функции get_weather. """
    if response.status_code in [200, 404]:
        resp_json_dict = response.json()

    if response:
        answ_text = 'Сейчас {0} °C'.format(resp_json_dict['main']['temp'])

    elif response.status_code == 404 and resp_json_dict['message'] == 'city not found':
        answ_text = 'Я не знаю такого города.'

    else:
        answ_text = ('Ошибка. Сервер вернул код {0}'.format(response.status_code))

    return answ_text


"""It Hangs in the Balance"""
from typing import Iterator

def get_smallest_partitions(
    packages: list[int], goal: int, start: int = 0, smallest: int = None
) -> Iterator[list[int]]:
    smallest = len(packages) / 4 if smallest is None else smallest
    for index, package in enumerate(packages[start:], start):
        if package > goal:
            break
        if package == goal:
            yield [index]
            smallest = 1
            continue
        if smallest <= 1:
            break
        for sub_group in get_smallest_partitions(packages, goal - package, index + 1, smallest - 1):
            if smallest >= 1 + len(sub_group):
                smallest = 1 + len(sub_group)
                yield [index, *sub_group]

def product(factors: list[int]):
    p = 1
    for factor in factors:
        p *= factor
    return p

def get_smallest_partition_lowest_entanglement(packages, remaining, partitions: int) -> int:
    assert remaining[0] % partitions == 0
    partition_size = remaining[0] // partitions
    parts = {}
    for part in get_smallest_partitions(packages, partition_size):
        if len(part) not in parts:
            parts[len(part)] = []
        parts[len(part)].append(part)
    smallest_partitions = parts[min(parts)]
    entanglements = [product([packages[p] for p in part]) for part in smallest_partitions]
    return min(entanglements)

def solve(text: str) -> Iterator:
    packages = sorted([int(p) for p in text.strip().splitlines()])
    remaining = [sum(packages[i:]) for i in range(len(packages))]
    yield get_smallest_partition_lowest_entanglement(packages, remaining, 3)
    yield get_smallest_partition_lowest_entanglement(packages, remaining, 4)


import requests
import json

class Resource_group:
        #def __init__(self, token, id, name, region='us-south', url='https://resource-controller.ng.bluemix.net/v1/resource_instances'):
        #    self.id = id
        #    self.name = name
        #    self.token = token
        #    self.url = url
        #    self.region = region

        def __init__(self, config, id, name):
            if config['token'] is None:
                config['token'] = Token(config)
            self.token = config['token']
            self.url = config['url']['resource_controller_url']
            self.region = config['account']['region']
            self.id = id
            self.name = name

        def __getResources(self):
            bearer_token = 'Bearer '+ self.token.getAccessToken()
            headers = {'content-type': 'application/x-www-form-urlencoded', 'Accept': 'application/vnd.ibm.collection+json', 'authorization': bearer_token}
            PARAMS = {'region_id': self.region, 'resource_group_id': self.id}
            r = requests.get(url = self.url, params=PARAMS, headers=headers)
            #print json.dumps(r.json(), indent=4, sort_keys=True)
            resources = r.json()['resources']
            return resources

        def getResources(self,type=None, name=None):
            resource_list = [];
            resources = self.__getResources();
            for resource in resources:
                _resource_group_id = resource['resource_group_id']
                _name = resource['name']
                _crn = resource['crn']
                _type = _crn.split(':')[4]
                if type is None:
                    resource_list.append(Resource(id=resource['guid'], name=_name, crn=_crn, resource_group_id=_resource_group_id, type=_type))
                elif type ==_type:
                    if name is None:
                         resource_list.append(Resource(id=resource['guid'], name=_name, crn=_crn, resource_group_id=_resource_group_id, type=_type))
                    elif name == _name:
                         resource_list.append(Resource(id=resource['guid'], name=_name, crn=_crn, resource_group_id=_resource_group_id, type=_type))

            return resource_list

class Account:
        def __init__(self, token, url="https://resource-manager.bluemix.net/v1/resource_groups"):
            self.token = token
            self.URL = url

        def __init__(self, config):
            if config['token'] is None:
                config['token'] = Token(config)
            self.config = config
            self.token = config['token']
            self.URL = config['url']['resource_manager_url']

        def __getResourceGroups(self):
            bearer_token = 'Bearer '+ self.token.getAccessToken()
            headers = {'content-type': 'application/x-www-form-urlencoded', 'Accept': 'application/vnd.ibm.collection+json', 'authorization': bearer_token}
            PARAMS = {}
            r = requests.get(url = self.URL, params=PARAMS, headers=headers)
            # check for error
            resource_groups = r.json()['resources']
            return resource_groups

        def getResourceGroups(self):
            resource_group_list = [];
            resource_groups = self.__getResourceGroups();
            for resource_group in resource_groups:
                resource_group_list.append(Resource_group(self.config, resource_group['id'].encode('utf8'), resource_group['name'].encode('utf8')))

            return resource_group_list

        def getResourceGroup(self, resource_group_name='default'):
            resource_groups = self.__getResourceGroups();
            for resource_group in resource_groups:
                if resource_group_name == resource_group['name'].encode('utf8'):
                    return(Resource_group(self.config, resource_group['id'].encode('utf8'), resource_group['name'].encode('utf8')))
            return None

class Resource:
        def __init__(self, name, id, resource_group_id, type, crn):
            self.name = name
            self.id = id
            self.resource_group_id = resource_group_id
            self.resource_type = type
            self.crn = crn


import os
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.patheffects as pe

data = pd.read_excel("processed_data_reordered.xlsx")

# Функция переноса длинных словосочетаний
def split_long_words(keyword):
    words = keyword.split()
    new_words = []
    for word in words:
        if len(word) > 10:
            midpoint = len(word) // 2
            # Пытаемся найти ближайший пробел для переноса
            for idx in range(midpoint - 1, -1, -1):
                if word[idx] == ' ':
                    midpoint = idx
                    break
            new_words.append(word[:midpoint] + '\n' + word[midpoint:])
        else:
            new_words.append(word)
    return ' '.join(new_words)

data['keyword'] = data['keyword'].apply(split_long_words)

# Исключение строк с недопустимыми значениями в столбце 'count'
data = data[data['count'].apply(lambda x: str(x).isdigit())]

# Преобразование count в размер точек (масштабирование для лучшей визуализации) и увеличения базового размера и диапазона масштабирования
data['count'] = data['count'].astype(int)
data['point_size'] = (data['count'] - data['count'].min()) / (data['count'].max() - data['count'].min()) * 250 + 50

# Настройка стилей
sns.set_style("whitegrid")
plt.rcParams["figure.figsize"] = (15, 15)

# Папка для сохранения изображений
output_folder = "images"

# Создание папки для сохранения изображений, если она не существует
if not os.path.exists(output_folder):
    os.makedirs(output_folder)

# Визуализация для каждой области
areas = data['area'].unique()
for area in areas:
    area_data = data[data['area'] == area]

    plt.figure()

    # Создание диаграммы рассеяния и добавление легенды
    labels_added = []
    for cluster in area_data['cluster'].unique():
        cluster_data = area_data[area_data['cluster'] == cluster]
        label = f"Кластер {cluster}"
        if label not in labels_added:
            plt.scatter(cluster_data['x'], cluster_data['y'],
                        s=cluster_data['point_size'],
                        c=cluster_data['color'],
                        label=label,
                        edgecolors='black',
                        linewidths=0.5,
                        marker='o')
            labels_added.append(label)
        else:
            plt.scatter(cluster_data['x'], cluster_data['y'],
                        s=cluster_data['point_size'],
                        c=cluster_data['color'],
                        edgecolors='black',
                        linewidths=0.5,
                        marker='o')

    # Добавление подписей к точкам с белым контуром для лучшей читаемости
    for _, row in area_data.iterrows():
        plt.text(row['x'], row['y'], row['keyword'], fontsize=9, ha='center', va='center',
                 color='black', path_effects=[plt.matplotlib.patheffects.withStroke(linewidth=1.5, foreground='white')])

    # Настройка графика
    plt.title(f"Диаграмма рассеяния для области: {area}")
    plt.axis('off')  # Убрать оси
    plt.legend(loc="upper right")

    # Добавление Footer-подписи
    plt.annotate("Это тестовое задание. Выполнил Седнев А.В.", (0.5, 0.01),
                 xycoords='axes fraction', va='center', ha='center', fontsize=10, color='gray')

    # Сохранение изображения с заменой слэшей в имени файла
    filename = f"{output_folder}/{area.replace('/', '_')}_scatter_plot.png"
    plt.savefig(filename, dpi=300, bbox_inches='tight')

    # Закрытие графика
    plt.close()


from numbers import Integral
from pathlib import Path
from typing import Any, Callable, Dict, Optional, Union

import numpy as np
from PIL import Image
from torchvision.datasets.utils import download_and_extract_archive
from typing_extensions import Literal

from ._cifar10_base import _CIFAR10Base
from ._utils import md5_check

__all__ = ["CIFAR10C", "CIFAR100C"]

PathLike = Union[str, Path]
Corruptions = Literal[
    "brightness",
    "contrast",
    "defocus_blur",
    "elastic_transform",
    "fog",
    "frost",
    "gaussian_blur",
    "gaussian_noise",
    "glass_blur",
    "impulse_noise",
    "jpeg_compression",
    "motion_blur",
    "pixelate",
    "saturate",
    "shot_noise",
    "snow",
    "spatter",
    "speckle_noise",
    "zoom_blur",
]

Severity = Literal[1, 2, 3, 4, 5]

class CIFAR10C(_CIFAR10Base):
    """
    A CIFAR-10-based test dataset for benchmarking neural network robustness to common
    corruptions [1]_. For each corruption and for each severity (`1-5`), the
    dataset consists of the 10,000 test images from CIFAR-10 with that corruption
    applied to those images.

    References
    ----------
    .. [1] https://zenodo.org/record/2535967#.YnAe_9rMKbg
    """

    url = "https://zenodo.org/record/2535967/files/CIFAR-10-C.tar"
    filename = "CIFAR-10-C.tar"
    tar_md5: str = "56bf5dcef84df0e2308c6dcbcbbd8499"
    base_folder: str = "CIFAR-10-C"

    _file_to_md5: Dict[str, str] = {
        "brightness": "0a81ef75e0b523c3383219c330a85d48",
        "contrast": "3c8262171c51307f916c30a3308235a8",
        "defocus_blur": "7d1322666342a0702b1957e92f6254bc",
        "elastic_transform": "9421657c6cd452429cf6ce96cc412b5f",
        "fog": "7b397314b5670f825465fbcd1f6e9ccd",
        "frost": "31f6ab3bce1d9934abfb0cc13656f141",
        "gaussian_blur": "c33370155bc9b055fb4a89113d3c559d",
        "gaussian_noise": "ecaf8b9a2399ffeda7680934c33405fd",
        "glass_blur": "7361fb4019269e02dbf6925f083e8629",
        "impulse_noise": "2090e01c83519ec51427e65116af6b1a",
        "jpeg_compression": "2b9cc4c864e0193bb64db8d7728f8187",
        "labels": "c439b113295ed5254878798ffe28fd54",
        "motion_blur": "fffa5f852ff7ad299cfe8a7643f090f4",
        "pixelate": "0f14f7e2db14288304e1de10df16832f",
        "saturate": "1cfae0964219c5102abbb883e538cc56",
        "shot_noise": "3a7239bb118894f013d9bf1984be7f11",
        "snow": "bb238de8555123da9c282dea23bd6e55",
        "spatter": "8a5a3903a7f8f65b59501a6093b4311e",
        "speckle_noise": "ef00b87611792b00df09c0b0237a1e30",
        "zoom_blur": "6ea8e63f1c5cdee1517533840641641b",
    }

    all_corruptions = (
        "brightness",
        "contrast",
        "defocus_blur",
        "elastic_transform",
        "fog",
        "frost",
        "gaussian_blur",
        "gaussian_noise",
        "glass_blur",
        "impulse_noise",
        "jpeg_compression",
        "motion_blur",
        "pixelate",
        "saturate",
        "shot_noise",
        "snow",
        "spatter",
        "speckle_noise",
        "zoom_blur",
    )
    noise_corruptions = (
        "gaussian_noise",
        "impulse_noise",
        "shot_noise",
    )
    blur_corruptions = (
        "defocus_blur",
        "glass_blur",
        "motion_blur",
        "zoom_blur",
    )
    weather_corruptions = (
        "brightness",
        "fog",
        "frost",
        "snow",
    )
    digital_corruptions = (
        "contrast",
        "elastic_transform",
        "jpeg_compression",
        "pixelate",
    )

    # not included in augmix results
    extra_corruptions = (
        "gaussian_blur",
        "saturate",
        "spatter",
        "speckle_noise",
    )

    def __init__(
        self,
        root: PathLike,
        corruption: Corruptions,
        severity: Severity,
        transform: Optional[Callable[[Image.Image], Any]] = None,
        target_transform: Optional[Callable[[int], Any]] = None,
        download: bool = False,
    ) -> None:
        """
        Parameters
        ----------
        root : PathLike
            Root directory of dataset where directory
            ``CIFAR-10-C`` exists or will be saved to if download is set to True.

        corruption : str
            The type of corruption, e.g., "fog". See `CIFAR10C.all_corruptions()` for a full list of corruptions.

        severity : Literal[1, 2, 3, 4, 5]
            The severity-level of the corruption, ranging from 1 to 5.

        transform :  Optional[Callable[[Image], Any]]
            A function/transform that takes in a PIL image
            and returns a transformed version. E.g., ``transforms.RandomCrop``

        target_transform : Optional[Callable]
            A function/transform that takes in a target
            and returns a transformed version.

        download : bool, optional (default=False)
            If true, downloads the dataset from the internet and puts it in root directory.
            If dataset is already downloaded, it is not downloaded again."""

        super().__init__(
            root=str(root), transform=transform, target_transform=target_transform
        )

        self._root = (Path(self.root) / self.base_folder).resolve()
        assert corruption in self._file_to_md5
        self.corruption = corruption

        if download:
            self.download()

        if not self._check_integrity():
            raise RuntimeError(
                "Dataset not found or corrupted."
                + " You can use download=True to download it"
            )

        if not isinstance(severity, (int, Integral)) and 1 <= severity <= 5:
            raise ValueError(f"`severity` must be an integer in [1, 5]. Got {severity}")

        severity -= 1  # type: ignore

        mmap = np.load(self._root / (corruption + ".npy"), mmap_mode="r")
        size = len(mmap) // 5

        # shape-(N, H, W, C)
        self.data = np.array(mmap[severity * size : (severity + 1) * size])
        self.targets = np.load(self._root / "labels.npy")

    def _check_integrity(self) -> bool:
        img_path = self._root / (self.corruption + ".npy")
        labels_path = self._root / "labels.npy"

        if not img_path.is_file() and not labels_path.is_file():
            return False

        img_md5 = md5_check(img_path)
        label_md5 = md5_check(labels_path)

        return (
            img_md5 == self._file_to_md5[self.corruption]
            and label_md5 == self._file_to_md5["labels"]
        )

    def download(self) -> None:
        if self._check_integrity():
            print("Files already downloaded and verified")
            return
        download_and_extract_archive(
            self.url, self.root, filename=self.filename, md5=self.tar_md5
        )

    def extra_repr(self) -> str:
        return "Split: Test"

class CIFAR100C(CIFAR10C):
    """
    A CIFAR-100-based test dataset for benchmarking neural network robustness to common
    corruptions [1]_. For each corruption and for each severity (`1-5`), the
    dataset consists of the 10,000 test images from CIFAR-100 with that corruption
    applied to those images.

    Parameters
    ----------
    root : PathLike
        Root directory of dataset where directory
        ``CIFAR-100-C`` exists or will be saved to if download is set to True.

    corruption : str
        The type of corruption, e.g., "fog". See `CIFAR100C.all_corruptions()` for a full
        list of corruptions.

    severity : Literal[1, 2, 3, 4, 5]
        The severity-level of the corruption, ranging from 1 to 5.

    transform :  Optional[Callable[[Image], Any]]
        A function/transform that takes in an PIL image
        and returns a transformed version. E.g., ``transforms.RandomCrop``

    target_transform : Optional[Callable]
        A function/transform that takes in a target
        and returns a transformed version.

    download: bool, optional (default=False)
        If true, downloads the dataset from the internet and puts it in root directory.
        If dataset is already downloaded, it is not downloaded again.

    References
    ----------
    .. [1] https://zenodo.org/record/2535967#.YnAe_9rMKbg
    """

    url = "https://zenodo.org/record/3555552/files/CIFAR-100-C.tar"
    filename = "CIFAR-100-C.tar"
    tar_md5: str = "11f0ed0f1191edbf9fa23466ae6021d3"
    base_folder: str = "CIFAR-100-C"

    _file_to_md5: Dict[str, str] = {
        "brightness": "f22d7195aecd6abb541e27fca230c171",
        "contrast": "322bb385f1d05154ee197ca16535f71e",
        "defocus_blur": "d923e3d9c585a27f0956e2f2ad832564",
        "elastic_transform": "a0792bd6581f6810878be71acedfc65a",
        "fog": "4efc7ebd5e82b028bdbe13048e3ea564",
        "frost": "3a39c6823bdfaa0bf8b12fe7004b8117",
        "gaussian_blur": "5204ba0d557839772ef5a4196a052c3e",
        "gaussian_noise": "ecc4d366eac432bdf25c024086f5e97d",
        "glass_blur": "0bf384f38e5ccbf8dd479d9059b913e1",
        "impulse_noise": "3b3c210ddfa0b5cb918ff4537a429fef",
        "jpeg_compression": "c851b7f1324e1d2ffddeb76920576d11",
        "labels": "bb4026e9ce52996b95f439544568cdb2",
        "motion_blur": "732a7e2e54152ff97c742d4c388c5516",
        "pixelate": "96c00c60f144539e14cffb02ddbd0640",
        "saturate": "c0697e9fdd646916a61e9c312c77bf6b",
        "shot_noise": "b0a1fa6e1e465a747c1b204b1914048a",
        "snow": "0237be164583af146b7b144e73b43465",
        "spatter": "12ccf41d62564d36e1f6a6ada5022728",
        "speckle_noise": "e3f215b1a0f9fd9fd6f0d1cf94a7ce99",
        "zoom_blur": "0204613400c034a81c4830d5df81cb82",
    }

    classes = (
        "apple",
        "aquarium_fish",
        "baby",
        "bear",
        "beaver",
        "bed",
        "bee",
        "beetle",
        "bicycle",
        "bottle",
        "bowl",
        "boy",
        "bridge",
        "bus",
        "butterfly",
        "camel",
        "can",
        "castle",
        "caterpillar",
        "cattle",
        "chair",
        "chimpanzee",
        "clock",
        "cloud",
        "cockroach",
        "couch",
        "crab",
        "crocodile",
        "cup",
        "dinosaur",
        "dolphin",
        "elephant",
        "flatfish",
        "forest",
        "fox",
        "girl",
        "hamster",
        "house",
        "kangaroo",
        "keyboard",
        "lamp",
        "lawn_mower",
        "leopard",
        "lion",
        "lizard",
        "lobster",
        "man",
        "maple_tree",
        "motorcycle",
        "mountain",
        "mouse",
        "mushroom",
        "oak_tree",
        "orange",
        "orchid",
        "otter",
        "palm_tree",
        "pear",
        "pickup_truck",
        "pine_tree",
        "plain",
        "plate",
        "poppy",
        "porcupine",
        "possum",
        "rabbit",
        "raccoon",
        "ray",
        "road",
        "rocket",
        "rose",
        "sea",
        "seal",
        "shark",
        "shrew",
        "skunk",
        "skyscraper",
        "snail",
        "snake",
        "spider",
        "squirrel",
        "streetcar",
        "sunflower",
        "sweet_pepper",
        "table",
        "tank",
        "telephone",
        "television",
        "tiger",
        "tractor",
        "train",
        "trout",
        "tulip",
        "turtle",
        "wardrobe",
        "whale",
        "willow_tree",
        "wolf",
        "woman",
        "worm",
    )


import pandas as pd
import numpy as np

#load wine data set, reading only Alcohol and Malic acid information
df = pd.io.parsers.read_csv(
    'Data/wine.data',
     header=None,
     usecols=[0,1,2]
    )

df.columns=['Class label', 'Alcohol', 'Malic acid']

#Alcohol (percent/volumne) and Malic acid (g/l) are measured on different scales
#print(df.head()) #DataFrame

from sklearn import preprocessing

#set Mean and Std Deviation to Standardization Scaler instance
std_scale = preprocessing.StandardScaler().fit(df[['Alcohol', 'Malic acid']])
#scale data frame according to set Mean and Std Deviation parameters, using Standardization Scaler instance
df_std = std_scale.transform(df[['Alcohol', 'Malic acid']])

#set min and max values from Data Frame to MinMaxScaler instance
minmax_scale = preprocessing.MinMaxScaler().fit(df[['Alcohol', 'Malic acid']])
#scale data frame according to set min and max values, using Min Max Scaler instance
df_minmax = minmax_scale.transform(df[['Alcohol', 'Malic acid']])

print(df_std) #converted to numpy array
#Mean of data set lies at 0 now
print('mean of Alcohol, using Standardization Scaler = {:.2f}'.format(df_std[:,0].mean()))
print('mean of Malic Acid, using Standardization Scaler = {:.2f}'.format(df_std[:,1].mean()))

print('mean of Alcohol, using MinMax Scaler = {:.2f}'.format(df_minmax[:,0].mean())) #means are not set to 0 by transformation
print('mean of Malic Acid, using MinMax Scaler = {:.2f}'.format(df_minmax[:,1].mean())) #means are not set to 0 by transformation

#Min-Max standardizes all values to fall between 1 and 0

print('Min-value after min-max scaling:\nAlcohol={:.2f}, Malic acid={:.2f}'
      .format(df_minmax[:,0].min(), df_minmax[:,1].min()))
print('\nMax-value after min-max scaling:\nAlcohol={:.2f}, Malic acid={:.2f}'
      .format(df_minmax[:,0].max(), df_minmax[:,1].max()))

#plotting graphs

import matplotlib.pyplot as plt

def plot():
    plt.figure(figsize=(10,10)) #size of the graph

    plt.scatter(df['Alcohol'], df['Malic acid'], color='green', label='input scale', alpha=0.5) #input

    plt.scatter(df_std[:,0], df_std[:,1], color='red', label='Standardized [$N  (\mu=0, \; \sigma=1)$]', alpha=0.3) #standardization

    plt.scatter(df_minmax[:,0], df_minmax[:,1], color='blue', label='min-max scaled [min=0, max=1]', alpha=0.3) #normalization

    plt.title('Alcohol and Malic Acid content of the wine dataset')
    plt.xlabel('Alcohol')
    plt.ylabel('Malic Acid')
    plt.legend(loc='upper left')
    plt.grid()
    #plt.tight_layout()

plot()
plt.show()

#plots using input, Normalized input and Standardized input to differentiate the three classes

fig, ax = plt.subplots(3, figsize=(6,14))

for a,d,l in zip(range(len(ax)),
               (df[['Alcohol', 'Malic acid']].values, df_std, df_minmax), #values for d, array of graph data
               ('Input scale',
                'Standardized [$N  (\mu=0, \; \sigma=1)$]',
                'min-max scaled [min=0, max=1]')            #values for l, label for the graph
                ):
    for i,c in zip(range(1,4), ('red', 'blue', 'green')):  #looping over the three classes, for individual graph type
        ax[a].scatter(d[df['Class label'].values == i, 0], #setting value of alcohol amount corresponding to class of wine on x axis
                  d[df['Class label'].values == i, 1], #ssetting value of malic acid amount corresponding to class of wine to y axis
                  alpha=0.5,
                  color=c,
                  label='Class %s' %i
                  )
    ax[a].set_title(l)
    ax[a].set_xlabel('Alcohol')
    ax[a].set_ylabel('Malic Acid')
    ax[a].legend(loc='upper left')
    ax[a].grid()

plt.tight_layout()

plt.show()


from django.shortcuts import render,redirect

from .models import ReaderModel
from .forms import AddReaderForm
from django.http.response import HttpResponse

# Create your views here.

def SubscribeView(request):
    if request.method == 'POST':
        form = AddReaderForm(request.POST)
        if form.is_valid():
            f_name = form.cleaned_data['RName']
            f_number = form.cleaned_data['RNumber']
            f_gender = form.cleaned_data['Gender']
            f_city = form.cleaned_data['City']
            f_contactnumber = form.cleaned_data['ContactNumber']
            f_subscription = form.cleaned_data['Subscription']
            if f_subscription == "Month":
                f_charges = 100
            else:
                f_charges = 1000
            model_obj = ReaderModel(
                RName=f_name,
                RNumber=f_number,
                Gender=f_gender,
                City=f_city,
                ContactNumber=f_contactnumber,
                Subscription=f_subscription,
                Charges=f_charges
            )
            model_obj.save()
            # return HttpResponse('
This is Subscribe View
')
            return redirect('display_readers')
    else:
        form = AddReaderForm
    return render(request, 'subscribe.html', {'form':form})

def DisplayReadersView(request):
    post = ReaderModel.objects.all()[:10]
    cont = {
        'data': post,
    }
    return render(request, 'displayReaders.html', context=cont)
    # return HttpResponse('
This is DisplayReadersView View
')


import yaml

import logging.config
import os
import sys

project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
sys.path.insert(0, project_root)

from common import data_processing as dp
from cli_version import cli_argument_parser

CLI_VERSION = "cli_version"

with open(os.path.join(CLI_VERSION, "logging_config.yaml"), "r") as config:
    log_config = yaml.safe_load(config.read())

logging.config.dictConfig(log_config)
logger = logging.getLogger("cli_app")

parser = cli_argument_parser.CliArgumentParser()
parser.configure_parser()
args = parser.parse()
personal_data_path = args.pdata
financial_data_path = args.fdata
countries = args.countries

try:
    personal_data, financial_data = dp.load_data(personal_data_path, financial_data_path)
    logger.info(f"Loaded personal data with schema {personal_data.columns}.")
    logger.info(f"Loaded financial data with schema {financial_data.columns}.")

    if countries:
        personal_data = dp.filter_data(personal_data, countries)

    personal_data_filtered = personal_data.drop(columns=["first_name", "last_name", "country"])
    financial_data_filtered = financial_data.drop(columns=["cc_n"])

    datasets_merged = dp.create_client_data(
        personal_data_filtered,
        financial_data_filtered,
        {"id": "client_identifier", "btc_a": "bitcoin_address", "cc_t": "credit_card_type"}
    )
    logger.info(f"Merged two datasets. Final schema: {datasets_merged.columns}.")

    output_path = os.path.join(CLI_VERSION, "client_data")
    os.makedirs(output_path, exist_ok=True)

    dp.save_data(datasets_merged, os.path.join(output_path, "client_data.csv"))
except Exception as exception:
    logger.error(exception)


# Monte Carlo with Exploring Starts

import invertedPendulumGame as ipg
import numpy as np
#import matplotlib.pyplot as plt
import pygame
import sys, os, argparse
import pickle
from pygame.locals import *
from collections import defaultdict

WINDOWDIMS = (1200, 600)
CARTDIMS = (50, 10)
PENDULUMDIMS = (6, 200)
GRAVITY = 0.13
REFRESHFREQ = 100
A_CART = 0.15

class MCESagent(object):
    def __init__(self, pendulum, max_episode_length, soft_policy_prob, alpha):
        """accepts bin divisions to create discretized states
        x_bins and theta_bins given in range 0 to 1
        v_bins and omega_bins given in actual values
        """
        assert isinstance(pendulum, ipg.InvertedPendulum)
        self.pendulum = pendulum
        self.max_episode_length = max_episode_length
        self.soft_policy_prob = soft_policy_prob
        self.alpha = alpha
        self.epsilon = 10 ** (-10)
        self.x_bins = [self.pendulum.CARTWIDTH / 2,
                       self.pendulum.WINDOWWIDTH - self.pendulum.CARTWIDTH / 2 - self.epsilon,
                       np.inf]
        self.coarse_xs = [-1, 0, 1]
        self.theta_bins = [-np.pi / 2, -0.2, -0.1, -0.05, 0, 0.05, 0.1, 0.2,
                           np.pi / 2 - self.epsilon, np.inf]
        self.coarse_thetas = [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]
        self.v_bins = [-10 * self.pendulum.A_CART,
                       -1 * self.pendulum.A_CART,
                       self.pendulum.A_CART - self.epsilon,
                       10 * self.pendulum.A_CART, np.inf]
        self.coarse_vs = [-2, -1, 0, 1, 2]
        self.omega_bins = [-0.01, -0.001, 0.001, 0.01, np.inf]
        self.coarse_omegas = [-2, -1, 0, 1, 2]
        self.Q = defaultdict(float)

    @staticmethod
    def fine_to_coarse(value, bins, labels):
        for i, b in enumerate(bins):
            if value <= b:
                return labels[i]

    def coarse_state(self):
        is_dead, t, x, v, theta, omega = self.pendulum.get_state()
        x_coarse = self.fine_to_coarse(x, self.x_bins, self.coarse_xs)
        v_coarse = self.fine_to_coarse(v, self.v_bins, self.coarse_vs)
        theta_coarse = self.fine_to_coarse(theta, self.theta_bins,
                                           self.coarse_thetas)
        omega_coarse = self.fine_to_coarse(omega, self.omega_bins,
                                           self.coarse_omegas)
        return is_dead, (x_coarse, v_coarse, theta_coarse, omega_coarse)

    def actions(self, previous_action, coarse_state):
        """defines available actions given state and previous action-
        must stop moving in one direction before moving in the other
        """
        x, _, _, _ = coarse_state
        result = ['None']
        if previous_action != 'Right' and x != self.coarse_xs[0]:
            result.append('Left')
        if previous_action != 'Left' and x != self.coarse_xs[-1]:
            result.append('Right')
        return result

    def random_init(self):
        x = np.random.uniform(self.pendulum.CARTWIDTH / 2,
                              self.pendulum.WINDOWWIDTH - self.pendulum.CARTWIDTH / 2)
        theta = np.random.uniform(-np.pi / 2, np.pi / 2)
        v = np.random.normal(0, 1.0)
        omega = np.random.normal(0, 0.1)
        is_dead = False
        return is_dead, x, v, theta, omega

    def nice_init(self):
        is_dead = False
        x = np.random.choice([self.pendulum.CARTWIDTH / 2,
                              self.pendulum.WINDOWWIDTH / 2,
                              self.pendulum.WINDOWWIDTH - self.pendulum.CARTWIDTH / 2])
        x = self.pendulum.WINDOWWIDTH / 2
        v = 0
        theta = np.random.uniform(-0.01, 0.01)
        omega = 0
        return is_dead, x, v, theta, omega

    def optimal_policy(self, previous_action, state):
        return max(self.actions(previous_action, state),
                   key=lambda a:self.Q[(previous_action,state,a)])

    def epsilon_policy(self, previous_action, state):
        if np.random.uniform(0,1) > self.soft_policy_prob:
            return self.optimal_policy(previous_action, state)
        else:
            return np.random.choice(self.actions(previous_action, state))

    def episode(self, init, policy = None):
        if policy is None:
            policy = self.epsilon_policy
        is_dead, x, v, theta, omega = init
        self.pendulum.set_state((is_dead, 0, x, v, theta, omega))
        previous_action = "None"
        is_dead, state = self.coarse_state()
        action = policy(previous_action, state)
        state_actions = []
        i = 0
        while not is_dead and i < self.max_episode_length:
            # Check the state you will go into
            self.pendulum.update_state(action)
            is_dead, new_state = self.coarse_state()
            old_Q = self.Q[(previous_action, state, action)]
            new_Q = self.Q[(action, new_state, self.optimal_policy(action, new_state))]
            # Update the value estimate
            self.Q[(previous_action, state, action)] += self.alpha * (1 + new_Q - old_Q)
            state_actions.append((previous_action, state, action))
            # Change your state
            previous_action, state = action, new_state
            action = policy(previous_action, state)
            i += 1
        return state_actions

    @staticmethod
    def plot_episode(state_action):
        theta_coarse = [theta for (_, (_, _, theta, _), _) in state_action]
        plt.scatter(range(len(theta_coarse)), theta_coarse)
        plt.show()

    def run(self, num_runs, train=False, policy=None):
        run_durations = []
        for i in range(num_runs):
            init = self.random_init() if train else self.nice_init()
            state_action = self.episode(init, policy=policy)
            run_durations.append(len(state_action))
        return run_durations

    def test(self):
        #assert self.fine_to_coarse(0, self.theta_bins, self.coarse_thetas) == 0
        assert self.fine_to_coarse(-3 * A_CART, self.v_bins, self.coarse_vs) == -1
        assert self.actions("Left", (0, 0, 0, 0)) == ["None", "Left"]
        print("Tests passed")

class MCESgame(ipg.InvertedPendulumGame):
    def __init__(self, windowdims, cartdims, penddims,
                 gravity, a_cart, refreshfreq, agent):
        assert isinstance(agent, MCESagent)
        self.agent = agent
        super(MCESgame, self).__init__(windowdims, cartdims, penddims,
                                       gravity, a_cart, refreshfreq,
                                       agent.pendulum)

    def game_round(self):
        self.agent.pendulum.reset_state()
        previous_action = "None"
        while not self.agent.pendulum.is_dead:
            for event in pygame.event.get():
                if event.type == QUIT:
                    pygame.quit()
                    sys.exit()
                if event.type == KEYUP:
                    if event.key == K_ESCAPE:
                        pygame.quit()
                        sys.exit()
            is_dead, state = self.agent.coarse_state()
            action = self.agent.optimal_policy(previous_action, state)
            t, x, _, theta, _ = self.agent.pendulum.update_state(action)
            previous_action = action
            self.time = t
            self.surface.fill(self.WHITE)
            self.draw_cart(x, theta)

            time_text = "t = {}".format(self.time_seconds())
            self.render_text(time_text, (0.1 * self.WINDOWWIDTH, 0.1 * self.WINDOWHEIGHT),
                             position = "topleft", fontsize = 40)
            if action == 'Left':
                action_text = '<-       '
            elif action == 'None':
                action_text = '    |    '
            elif action == 'Right':
                action_text = '       ->'
            self.render_text(action_text, (0.1 * self.WINDOWWIDTH, 0.2 * self.WINDOWHEIGHT),
                             position = "topleft", fontsize = 40)
            state_text = str(state)
            self.render_text(state_text, (0.1 * self.WINDOWWIDTH, 0.3 * self.WINDOWHEIGHT),
                             position = "topleft", fontsize = 40)

            pygame.display.update()
            self.clock.tick(self.REFRESHFREQ)
        if (self.time_seconds()) > self.high_score:
            self.high_score = self.time_seconds()

    def game(self):
        self.starting_page()
        while True:
            for event in pygame.event.get():
                if event.type == QUIT:
                    pygame.quit()
                    sys.exit()
                if event.type == KEYDOWN:
                    if event.key == K_ESCAPE:
                        pygame.quit()
                        sys.exit()
            self.game_round()
            self.end_of_round()
            pygame.time.wait(500)

def main():
    parser = argparse.ArgumentParser(
        description="Pass name of pickle to use or write to")
    parser.add_argument('filename', type=str)
    args = parser.parse_args()
    FILEPATH = os.getcwd() + "/" + args.filename + ".pickle"
    if os.path.exists(FILEPATH):
        with open(FILEPATH, 'rb') as f:
            agent = pickle.load(f)
    else:
        pend = ipg.InvertedPendulum(WINDOWDIMS, CARTDIMS, PENDULUMDIMS,
                                    GRAVITY, A_CART)
        agent = MCESagent(pend, 20000, 0.1, 0.05)
        agent.test()
        performance = []
        for i in range(10):
            agent.run(100)
            durations = agent.run(50, train=False, policy=agent.optimal_policy)
            avg_duration = np.mean(durations)
            performance.append(avg_duration)
            print("Step: {}, Average Duration {}".format(i+1, avg_duration))
    ##        if i % 50 == 0:
    ##            e = agent.episode(agent.nice_init())
    ##            agent.plot_episode(e)
        with open(FILEPATH, 'wb') as output:
            pickle.dump(agent, output)
    game = MCESgame(WINDOWDIMS, CARTDIMS, PENDULUMDIMS, GRAVITY, A_CART,
                    REFRESHFREQ, agent)
    game.game()

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import numpy as np
import torch
from torch.utils.data import Dataset
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from torch.utils.data import DataLoader
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import pdb

# In[2]:

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# In[3]:

print(device)

# In[4]:

train_in = pd.read_csv('train_input.csv')
train_out = pd.read_csv('train_output.csv')

# In[5]:

train_in.head()

# In[6]:

train_out.head()

# In[35]:

class NumpyDataset(Dataset):

    def __init__(self, input_file_name, output_file_name, is_train=True):
        X_in = pd.read_csv(input_file_name)[:10000]
        X_in.drop(columns=['ID'], inplace=True)
        y_in = pd.read_csv(output_file_name)[:10000]
        y_in.drop(columns=['ID'], inplace=True)
        X_in['distance'] = y_in['distance']
        self.len = len(X_in)

        X_train, X_test = train_test_split(X_in, test_size=.01, random_state=1)

        if is_train:
            X = X_train
        else:
            X = X_test

        self.class_holder = dict()
        for i in range(1, 15):
            subsetX = X[X['distance'] == i].copy()
            subsetX.drop(columns=['distance'], inplace=True)
            self.class_holder[i] = subsetX.to_numpy()

    def __getitem__(self, index):

        choice_idx = np.random.choice(range(1, 15))

        x = self.class_holder[choice_idx]
        while len(x) == 0:
            choice_idx = np.random.choice(range(1, 15))
            x = self.class_holder[choice_idx]

        x_choice = x[np.random.choice(range(len(x)))]

        return x_choice, choice_idx

    def __len__(self):
        return self.len

# In[ ]:

# In[ ]:

# In[36]:

train_dataset = NumpyDataset('train_input.csv', 'train_output.csv')
trainloader = DataLoader(train_dataset, batch_size=64, shuffle=False)
valid_dataset = NumpyDataset('train_input.csv', 'train_output.csv', is_train=False)
testloader = DataLoader(valid_dataset, batch_size=64)

# In[22]:

class ResnetModel(nn.Module):
    def __init__(self, state_dim: int, one_hot_depth: int, h1_dim: int, resnet_dim: int, num_resnet_blocks: int,
                 out_dim: int, batch_norm: bool):
        super().__init__()
        self.one_hot_depth: int = one_hot_depth
        self.state_dim: int = state_dim
        self.blocks = nn.ModuleList()
        self.num_resnet_blocks: int = num_resnet_blocks
        self.batch_norm = batch_norm

        # first two hidden layers
        if one_hot_depth > 0:
            self.fc1 = nn.Linear(self.state_dim * self.one_hot_depth, h1_dim)
        else:
            self.fc1 = nn.Linear(self.state_dim, h1_dim)

        if self.batch_norm:
            self.bn1 = nn.BatchNorm1d(h1_dim)

        self.fc2 = nn.Linear(h1_dim, resnet_dim)

        if self.batch_norm:
            self.bn2 = nn.BatchNorm1d(resnet_dim)

        # resnet blocks
        for block_num in range(self.num_resnet_blocks):
            if self.batch_norm:
                res_fc1 = nn.Linear(resnet_dim, resnet_dim)
                res_bn1 = nn.BatchNorm1d(resnet_dim)
                res_fc2 = nn.Linear(resnet_dim, resnet_dim)
                res_bn2 = nn.BatchNorm1d(resnet_dim)
                self.blocks.append(nn.ModuleList([res_fc1, res_bn1, res_fc2, res_bn2]))
            else:
                res_fc1 = nn.Linear(resnet_dim, resnet_dim)
                res_fc2 = nn.Linear(resnet_dim, resnet_dim)
                self.blocks.append(nn.ModuleList([res_fc1, res_fc2]))

        # output
        self.fc_out = nn.Linear(resnet_dim, out_dim)

        self.dummy = nn.Linear(self.state_dim * self.one_hot_depth, out_dim, bias=True)

    def forward(self, states_nnet):
        x = states_nnet

        # preprocess input
        if self.one_hot_depth > 0:
            x = F.one_hot(x.long(), self.one_hot_depth)
            x = x.float()
            x = x.view(-1, self.state_dim * self.one_hot_depth)
        else:
            x = x.float()

        # first two hidden layers
        x = self.fc1(x)
        if self.batch_norm:
            x = self.bn1(x)

        x = F.relu(x)
        x = self.fc2(x)
        if self.batch_norm:
            x = self.bn2(x)

        x = F.relu(x)

        # resnet blocks
        for block_num in range(self.num_resnet_blocks):
            res_inp = x
            if self.batch_norm:
                x = self.blocks[block_num][0](x)
                x = self.blocks[block_num][1](x)
                x = F.relu(x)
                x = self.blocks[block_num][2](x)
                x = self.blocks[block_num][3](x)
            else:
                x = self.blocks[block_num][0](x)
                x = F.relu(x)
                x = self.blocks[block_num][1](x)

            x = F.relu(x + res_inp)

        # output
        x = self.fc_out(x)
#         x = F.one_hot(x.long(), self.one_hot_depth)
#         x = x.view(-1, self.state_dim * self.one_hot_depth)
#         x = x.float()
#         x= self.dummy(x)+11
        return x

# In[23]:

net = ResnetModel(24, 6, 1000, 100, 4, 1, True)
net = net.to(device)

# In[24]:

# loss
criterion = nn.MSELoss()
#critetion= nn.L1Loss()
# optimizer
optimizer = optim.Adam(net.parameters(), lr=0.0001)

# In[25]:

# class LogisticRegression(torch.nn.Module):
#     def __init__(self, input_dim, output_dim):
#         super(LogisticRegression, self).__init__()
#         self.linear = torch.nn.Linear(input_dim, output_dim)

#     def forward(self, x):
#         outputs = self.linear(x.float())
#         return outputs.float()

# In[26]:

# net = LogisticRegression(24, 1)
# criterion = torch.nn.MSELoss()
# optimizer = torch.optim.SGD(net.parameters(), lr=0.0001)
# net = net.to(device)

# In[27]:

# for x, y in trainloader:
#     print(x-1, x.shape)
#     print(y, y.shape)
#     break
# outputs = net(x-1)
# print(outputs, outputs.shape)

# In[28]:

def train(net, trainloader):
    i = 0
    for epoch in range(10):  # no. of epochs
        running_loss = 0
        for i, data in enumerate(trainloader):
            # data and labels to GPU if available

            inputs, labels = data[0].to(device, non_blocking=True) - 1, data[1].to(device, non_blocking=True)
#             print(inputs.shape, labels.shape)
            # pdb.set_trace()

            # set the parameter gradients to zero
            optimizer.zero_grad()
            outputs = net(inputs).float()
#             print(outputs.shape)
            loss = criterion(outputs.view(-1).float(), labels.view(-1).float())

            # propagate the loss backward
            loss.backward()
            # update the gradients
            optimizer.step()

            running_loss += loss.item()

            if not i % 500 and i > 0:
                print(loss.item())
#                 print(running_loss / i)

            i += 1
        print('[Epoch %d] loss: %.3f' %
              (epoch + 1, running_loss / len(trainloader)))
#               (epoch + 1, running_loss / len(trainloader)))

    print('Done Training')

# In[29]:

def test(net, testloader):
    correct = 0
    total = 0
    with torch.no_grad():
        for data in testloader:
            inputs, labels = data[0].to(device, non_blocking=True) - 1, data[1].to(device, non_blocking=True)
            outputs = net(inputs)
            # pdb.set_trace()
#             _, predicted = torch.max(outputs.data, 1)
#             print(outputs.data.shape)
#             print(predicted.shape)
            total += labels.size(0)
            correct += (torch.round(outputs.data.reshape(-1)) == labels).sum().item()
    print('Accuracy of the network on test set: %0.3f %%' % (100 * correct / total))

# In[39]:

train(net, trainloader)

# In[40]:

test(net, trainloader)

# In[41]:

test(net, testloader)

# In[ ]:

# In[ ]:


import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

def func(m, x, r, t):

    if x > r:
        return t

    return max(0, m - 1)

def MHI(filepath, ext, t ,r = 30.0, b = False):

    Ip = None
    FD = None
    i = 1
    while True:

        In = None

        if i < 10 and b == True:
            In = cv2.imread(filepath + "0" + str(i) + "." + ext)
        else:
            In = cv2.imread(filepath + str(i) + "." + ext)

        i += 1

        if In is None:
            FD = (FD / t) * 255.0
            return FD.astype(np.uint8)

        if len(In.shape) == 3:
            In = cv2.cvtColor(In, cv2.COLOR_BGR2GRAY)

        if i == 2:
            FD = np.zeros(In.shape, dtype= np.double)
        else:
            diff = abs(In.astype(np.double) - Ip.astype(np.double))
            FD = np.vectorize(func)(FD, diff, r, t)

        Ip = In

##Q2##

pirate = MHI("../pirate/frame", "png", 12)
course = MHI("../course/frame", "png", 40)
femme = MHI("../femme/frame", "png", 22)
action = MHI("../action/seq2_0", "png", 25, b = True)

cv2.imwrite("../.results/images/pirate_MHI.png", pirate)
cv2.imwrite("../.results/images/course_MHI.png", course)
cv2.imwrite("../.results/images/femme_MHI.png", femme)
cv2.imwrite("../.results/images/action_MHI.png", action)

fig = plt.figure(figsize = (12,7))
columns = 2
rows = 2

fig.add_subplot(rows, columns, 1)
plt.imshow(pirate, cmap='Greys_r', interpolation='nearest')
plt.axis('off')
plt.title('pirate_MHI')

fig.add_subplot(rows, columns, 2)
plt.imshow(course, cmap='Greys_r', interpolation='nearest')
plt.axis('off')
plt.title('course_MHI')

fig.add_subplot(rows, columns, 3)
plt.imshow(femme, cmap='Greys_r', interpolation='nearest')
plt.axis('off')
plt.title('femme_MHI')

fig.add_subplot(rows, columns, 4)
plt.imshow(action, cmap='Greys_r', interpolation='nearest')
plt.axis('off')
plt.title('action_MHI')

plt.show()


from flask import jsonify

def remove_friends_by_friend_id(user_id, friend_id, connection):
    DELETE_FRIEND_BY_FRIEND_ID = """
    DELETE FROM friendships WHERE user_id = %s AND friend_id = %s
    RETURNING *;
    """
    with connection:
        with connection.cursor() as cursor:
            cursor.execute(DELETE_FRIEND_BY_FRIEND_ID, (user_id, friend_id,))
            deleted_friend = cursor.fetchone()

            if deleted_friend:
                return jsonify({'message': 'Friendship deleted successfully'})
            else:
                return jsonify({'message': 'Friendship not found'})