"""
main.py

train the deep iamge prior model and get the denoised figure, calculate PSNR when required.
"""
import hydra
from pytorch_lightning import Trainer, seed_everything

from src.conf import Config
from src.data.datamodule import DeepImagePriorDataModule
from src.model.model import DeepImagePriorModel

import logging
logging.getLogger("lightning").setLevel(logging.ERROR)

@hydra.main(config_path=".", config_name="base_config", version_base="1.2")
def train_app(conf: Config) -> None:
    """
    The main train loop.
    """
    # * seed
    if conf.train.random_seed:
        seed_everything(conf.train.random_seed)

    for idx, fig_dir in enumerate(conf.data.root_dir.iterdir()):
        if fig_dir.is_dir():
            light_data = DeepImagePriorDataModule(conf.data, img_dir=fig_dir)
            light_data.setup(stage="fit")
            print(f"train fig{idx}, key: {light_data.dataset[0]['key']}")
            light_model = DeepImagePriorModel(conf)

            train_conf = conf.train
            trainer = Trainer(
                accelerator=train_conf.accelerator,
                devices=(
                    train_conf.distributed_devices if train_conf.accelerator == "gpu" else None),
                max_epochs=train_conf.epochs,
                num_sanity_val_steps=0,
                check_val_every_n_epoch=train_conf.check_val_every_n_epoch,
                enable_progress_bar=True
            )
            trainer.fit(light_model, light_data)
        if idx == conf.data.total_run_figs_number-1:
            break

if __name__ == "__main__":
    train_app()  # pylint: disable=no-value-for-parameter


read_me = """Python 3 script that logs data from the Meraki dashboard into a MongoDB database.
You will need to have MongoDB installed and supply a configuration file for this script to run.
You can get the MongoDB Community Server here: https://www.mongodb.com/try/download/community
You can find a sample configuration file here:
  https://github.com/meraki/automation-scripts/blob/master/offline_logging/config.yaml

Script syntax:
    python offline_logging.py -c <config_file>

Required Python 3 modules:
    requests
    pyyaml
    pymongo

To install these Python 3 modules via pip you can use the following commands:
    pip install requests
    pip install pyyaml
    pip install pymongo

Depending on your operating system and Python environment, you may need to use commands
 "python3" and "pip3" instead of "python" and "pip".

View the created database with a MongoDB viewing tool such as MongoDB Compass:
 https://www.mongodb.com/products/compass
 A version of MongoDB Compass can be installed with the MongoDB Community Server.
"""

import sys, getopt, yaml, time, datetime, pymongo

from urllib.parse import urlencode
from requests import Session, utils

class NoRebuildAuthSession(Session):
    def rebuild_auth(self, prepared_request, response):
        """
        This method is intentionally empty. Needed to prevent auth header stripping on redirect. More info:
        https://stackoverflow.com/questions/60358216/python-requests-post-request-dropping-authorization-header
        """

API_MAX_RETRIES         = 3
API_CONNECT_TIMEOUT     = 60
API_TRANSMIT_TIMEOUT    = 60
API_STATUS_RATE_LIMIT   = 429

#Set to True or False to enable/disable console logging of sent API requests
FLAG_REQUEST_VERBOSE    = True

API_BASE_URL            = "https://api.meraki.com/api/v1"

def merakiRequest(p_apiKey, p_httpVerb, p_endpoint, p_additionalHeaders=None, p_queryItems=None,
        p_requestBody=None, p_verbose=False, p_retry=0):
    #returns success, errors, responseHeaders, responseBody

    if p_retry > API_MAX_RETRIES:
        if(p_verbose):
            print("ERROR: Reached max retries")
        return False, None, None, None

    bearerString = "Bearer " + p_apiKey
    headers = {"Authorization": bearerString}
    if not p_additionalHeaders is None:
        headers.update(p_additionalHeaders)

    query = ""
    if not p_queryItems is None:
        query = "?" + urlencode(p_queryItems, True)
    url = API_BASE_URL + p_endpoint + query

    verb = p_httpVerb.upper()

    session = NoRebuildAuthSession()

    try:
        if(p_verbose):
            print(verb, url)
        if verb == "GET":
            r = session.get(
                url,
                headers =   headers,
                timeout =   (API_CONNECT_TIMEOUT, API_TRANSMIT_TIMEOUT)
            )
        elif verb == "PUT":
            if not p_requestBody is None:
                if (p_verbose):
                    print("body", p_requestBody)
                r = session.put(
                    url,
                    headers =   headers,
                    json    =   p_requestBody,
                    timeout =   (API_CONNECT_TIMEOUT, API_TRANSMIT_TIMEOUT)
                )
        elif verb == "POST":
            if not p_requestBody is None:
                if (p_verbose):
                    print("body", p_requestBody)
                r = session.post(
                    url,
                    headers =   headers,
                    json    =   p_requestBody,
                    timeout =   (API_CONNECT_TIMEOUT, API_TRANSMIT_TIMEOUT)
                )
        elif verb == "DELETE":
            r = session.delete(
                url,
                headers =   headers,
                timeout =   (API_CONNECT_TIMEOUT, API_TRANSMIT_TIMEOUT)
            )
        else:
            return False, None, None, None
    except:
        return False, None, None, None

    if(p_verbose):
        print(r.status_code)

    success         = r.status_code in range (200, 299)
    errors          = None
    responseHeaders = None
    responseBody    = None

    if r.status_code == API_STATUS_RATE_LIMIT:
        if(p_verbose):
            print("INFO: Hit max request rate. Retrying %s after %s seconds" % (p_retry+1, r.headers["Retry-After"]))
        time.sleep(int(r.headers["Retry-After"]))
        success, errors, responseHeaders, responseBody = merakiRequest(p_apiKey, p_httpVerb, p_endpoint, p_additionalHeaders,
            p_queryItems, p_requestBody, p_verbose, p_retry+1)
        return success, errors, responseHeaders, responseBody

    try:
        rjson = r.json()
    except:
        rjson = None

    if not rjson is None:
        if "errors" in rjson:
            errors = rjson["errors"]
            if(p_verbose):
                print(errors)
        else:
            responseBody = rjson

    if "Link" in r.headers:
        parsedLinks = utils.parse_header_links(r.headers["Link"])
        for link in parsedLinks:
            if link["rel"] == "next":
                if(p_verbose):
                    print("Next page:", link["url"])
                splitLink = link["url"].split("/api/v1")
                success, errors, responseHeaders, nextBody = merakiRequest(p_apiKey, p_httpVerb, splitLink[1],
                    p_additionalHeaders=p_additionalHeaders,
                    p_requestBody=p_requestBody,
                    p_verbose=p_verbose)
                if success:
                    if not responseBody is None:
                        responseBody = responseBody + nextBody
                else:
                    responseBody = None

    return success, errors, responseHeaders, responseBody

def getNetworks(p_apiKey, p_organizationId):
    endpoint = "/organizations/%s/networks" % p_organizationId
    success, errors, headers, response = merakiRequest(p_apiKey, "GET", endpoint, p_verbose=FLAG_REQUEST_VERBOSE)
    return success, errors, headers, response

def getClients(p_apiKey, p_networkId, p_timespan):
    endpoint = "/networks/%s/clients" % p_networkId
    query = {"timespan": p_timespan}
    success, errors, headers, response = merakiRequest(p_apiKey, "GET", endpoint, p_queryItems=query, p_verbose=FLAG_REQUEST_VERBOSE)
    return success, errors, headers, response

def getApplicationUsage(p_apiKey, p_networkId, p_clientsStr, p_timespan):
    endpoint = "/networks/%s/clients/applicationUsage" % p_networkId
    query = {"clients": p_clientsStr, "timespan": p_timespan}
    success, errors, headers, response = merakiRequest(p_apiKey, "GET", endpoint, p_queryItems=query, p_verbose=FLAG_REQUEST_VERBOSE)
    return success, errors, headers, response

def getClientTrafficHistory(p_apiKey, p_networkId, p_clientId):
    endpoint = "/networks/%s/clients/%s/trafficHistory" % (p_networkId, p_clientId)
    success, errors, headers, response = merakiRequest(p_apiKey, "GET", endpoint, p_verbose=FLAG_REQUEST_VERBOSE)
    return success, errors, headers, response

def getNetworkMerakiAuthUsers(p_apiKey, p_networkId):
    endpoint = "/networks/%s/merakiAuthUsers" % p_networkId
    success, errors, headers, response = merakiRequest(p_apiKey, "GET", endpoint, p_verbose=FLAG_REQUEST_VERBOSE)
    return success, errors, headers, response

def getNetworkSmDevices(p_apiKey, p_networkId):
    endpoint = "/networks/%s/sm/devices" % p_networkId
    query = {"fields[]": ['ip', 'systemType', 'lastConnected', 'location', 'lastUser',
        'ownerEmail', 'ownerUsername', 'imei', 'simCarrierNetwork']}

    success, errors, headers, response = merakiRequest(p_apiKey, "GET", endpoint, p_queryItems=query,
        p_verbose=FLAG_REQUEST_VERBOSE)

    return success, errors, headers, response

def getOrganizationAdmins(p_apiKey, p_organizationId):
    endpoint = "/organizations/%s/admins" % p_organizationId
    success, errors, headers, response = merakiRequest(p_apiKey, "GET", endpoint, p_verbose=FLAG_REQUEST_VERBOSE)
    return success, errors, headers, response

def kill_script():
    print(read_me)
    sys.exit(2)

def load_config(p_file):

    config = None

    with open(p_file) as file:
        config = yaml.load(file, Loader=yaml.FullLoader)

    return config

def filter_networks(config_sources, networks):
    result = []

    if config_sources['include_all_networks']:
        return networks

    for net in networks:
        found_match = False
        if not config_sources['network_names'] is None:
            if net['name'] in config_sources['network_names']:
                result.append(net)
                found_match = True
        if not found_match:
            if not config_sources['network_ids'] is None:
                if net['id'] in config_sources['network_ids']:
                    result.append(net)
                    found_match = True
                    print ('match id ' + net['id'])
            if not found_match:
                if not config_sources['network_tags'] is None:
                    for tag in config_sources['network_tags']:
                        if tag in net['tags']:
                            result.append(net)
                            break
    return result

def filter_admins(p_admins, p_networks, p_tags):
    # Return admin if they have org access, or net/tag access to an item matching filters

    result = []
    for admin in p_admins:
        include_admin = False
        if admin['orgAccess'] != 'none':
            include_admin = True
        else:
            for anet in admin['networks']:
                for onet in p_networks:
                    if anet['id'] == onet['id']:
                        include_admin = True
                        break
                if include_admin:
                    break
            if not include_admin:
                for atag in admin['tags']:
                    if atag['tag'] in p_tags:
                        include_admin = True
                        break
        if include_admin:
            result.append(admin)
    return result

def log_to_database(db, document, collection, mode='append', keyValuePair=None):
    dbc = db[collection]

    if mode == 'append':
        try:
            dbc.insert_one(document)
        except Exception as e:
            print(e)
            print("ERROR: Could not create document in database")
            return False
    elif mode == 'update':
        try:
            dbc.update_one(keyValuePair, {"$set": document}, upsert=True)
        except Exception as e:
            print(e)
            print("ERROR: Could not update document in database")
            return False

    return True

def database_delete_all_matches(db, collection, filter):
    dbc = db[collection]
    try:
        dbc.delete_many(filter)
    except Exception as e:
        print(e)
        print("ERROR: Could not delete document in database")
        return False

    return True

def split_history_array(history, max_records):
    result = []
    line = []
    for record in history:
        line.append(record)
        if len(line) >= max_records:
            result.append(line)
            line = []

    if len(line) > 0:
        result.append(line)

    return result

def perform_scan(config):
    print(str(datetime.datetime.now()) + " -- Starting scan")

    api_key         = config['meraki_dashboard_api']['api_key']
    org_id          = config['meraki_dashboard_api']['organization_id']
    scan_interval   = config['scan_interval_minutes']*60

    success, errors, headers, all_networks = getNetworks(api_key, org_id)

    if not success:
        print("ERROR: Unable to get networks' list")
    else:
        filtered_networks = filter_networks(config['sources'], all_networks)

        mongo_client = pymongo.MongoClient("mongodb://" + config['mongodb']['host'] + ":" + str(config['mongodb']['port']) + "/")
        db = mongo_client[config['mongodb']['database_name']]

        if 'getOrganizationAdmins' in config['endpoints'] and config['endpoints']['getOrganizationAdmins']['enabled']:
            success, errors, headers, all_admins = getOrganizationAdmins(api_key, org_id)
            if not all_admins is None:
                admins = filter_admins(all_admins, filtered_networks, config['sources']['network_tags'])
                for admin in admins:
                    log_to_database(db, admin, config['endpoints']['getOrganizationAdmins']['collection'],
                            config['endpoints']['getOrganizationAdmins']['mode'],
                            keyValuePair={'id': admin['id']})

        for network in filtered_networks:
            # value used as a flag if "getNetworkClients" is disabled
            clients = None

            if 'getNetworkClients' in config['endpoints'] and config['endpoints']['getNetworkClients']['enabled']:
                success, errors, headers, raw_clients = getClients(api_key, network['id'], scan_interval)
                if raw_clients is None:
                    print("ERROR: Cloud not fetch clients for net %s" % network['id'])
                else:
                    scan_time = datetime.datetime.now()

                    if config['endpoints']['getNetworkClients']['ignore_manufacturer_meraki']:
                        clients = []
                        for client in raw_clients:
                            if not client['manufacturer'] in ["Cisco Meraki", "Meraki"]:
                                clients.append(client)
                    else:
                        clients = raw_clients

                    for client in clients:
                        document = client
                        document['scanTime'] = scan_time
                        document['scanIntervalMinutes'] = config['scan_interval_minutes']
                        document['networkId'] = network['id']
                        document['networkName'] = network['name']
                        log_to_database(db, document, config['endpoints']['getNetworkClients']['collection'],
                            config['endpoints']['getNetworkClients']['mode'])
            if 'getNetworkClientsApplicationUsage' in config['endpoints'] and config['endpoints']['getNetworkClientsApplicationUsage']['enabled']:
                if clients is None:
                    print("ERROR: Client list must be fetched for getNetworkClientsApplicationUsage")
                else:
                    client_list = ""
                    for client in clients:
                        if client_list != "":
                            client_list += ","
                        client_list += client['id']

                    success, errors, headers, usage = getApplicationUsage(api_key, network['id'], client_list, scan_interval)

                    if usage is None:
                        print("ERROR: Cloud not fetch clients' usage for net %s" % network['id'])
                    else:
                        scan_time = datetime.datetime.now()
                        for item in usage:
                            document = item
                            document['scanTime'] = scan_time
                            document['scanIntervalMinutes'] = config['scan_interval_minutes']
                            document['networkId'] = network['id']
                            document['networkName'] = network['name']
                            log_to_database(db, document, config['endpoints']['getNetworkClientsApplicationUsage']['collection'],
                                config['endpoints']['getNetworkClientsApplicationUsage']['mode'])
            if 'getNetworkClientTrafficHistory' in config['endpoints'] and config['endpoints']['getNetworkClientTrafficHistory']['enabled']:
                if clients is None:
                    print("ERROR: Client list must be fetched for getNetworkClientTrafficHistory")
                else:
                    for client in clients:
                        success, errors, headers, traffic_history = getClientTrafficHistory(api_key, network['id'], client['id'])

                        if not traffic_history is None:
                            history_pages = split_history_array(traffic_history,
                                config['endpoints']['getNetworkClientTrafficHistory']['max_history_records_per_document'])

                            total_pages = len(history_pages)

                            if total_pages > 0:
                                base_info = {
                                    'clientId'              : client['id'],
                                    'clientMac'             : client['mac'],
                                    'clientIp'              : client['ip'],
                                    'clientDescription'     : client['description'],
                                    'networkId'             : network['id'],
                                    'networkName'           : network['name'],
                                    'scanTime'              : scan_time,
                                    'scanIntervalMinutes'   : config['scan_interval_minutes'],
                                    'totalPages'            : total_pages
                                }

                                filter = {
                                    'clientId'  : base_info['clientId'],
                                    'networkId' : base_info['networkId']
                                }

                                if config['endpoints']['getNetworkClientTrafficHistory']['mode'] == 'update':
                                    success = database_delete_all_matches(db,
                                        config['endpoints']['getNetworkClientTrafficHistory']['collection'], filter)

                                page_number = 0
                                for page in history_pages:
                                    page_number += 1
                                    document = {}
                                    for key in base_info:
                                        document[key] = base_info[key]
                                    document['pageNumber'] = page_number
                                    document['trafficHistory'] = page
                                    success = log_to_database(db, document, config['endpoints']['getNetworkClientTrafficHistory']['collection'], mode="append")
                                    if not success:
                                        print("clientId                    : %s" % document['clientId'])
                                        print("clientMac                   : %s" % document['clientMac'])
                                        print("clientIp                    : %s" % document['clientIp'])
                                        print("clientDescription           : %s" % document['clientDescription'])
                                        print("networkId                   : %s" % document['networkId'])
                                        print("networkName                 : %s" % document['networkName'])
                                        print("pageNumber                   : %s" % document['pageNumber'])
                                        print("trafficHistory record count : %s" % len(document['trafficHistory']))
            if 'getNetworkMerakiAuthUsers' in config['endpoints'] and config['endpoints']['getNetworkMerakiAuthUsers']['enabled']:
                success, errors, headers, auth_users = getNetworkMerakiAuthUsers(api_key, network['id'])
                if 'configTemplateId' in network and config['endpoints']['getNetworkMerakiAuthUsers']['include_template_users']:
                    success, errors, headers, template_users = getNetworkMerakiAuthUsers(api_key, network['configTemplateId'])
                    if not template_users is None:
                        if not auth_users is None:
                            auth_users += template_users
                        else:
                            auth_users = template_users
                if not auth_users is None:
                    for user in auth_users:
                        document = user
                        document['networkId'] = network['id']
                        log_to_database(db, document, config['endpoints']['getNetworkMerakiAuthUsers']['collection'],
                            config['endpoints']['getNetworkMerakiAuthUsers']['mode'],
                            keyValuePair={'id': user['id'], 'networkId': network['id']})
            if 'getNetworkSmDevices' in config['endpoints'] and config['endpoints']['getNetworkSmDevices']['enabled']:
                if 'systemsManager' in network['productTypes']:
                    success, errors, headers, sm_devices = getNetworkSmDevices(api_key, network['id'])
                    if not sm_devices is None:
                        tag_disabled = not config['endpoints']['getNetworkSmDevices']['filter_by_device_tag_enabled']
                        tag_filter = config['endpoints']['getNetworkSmDevices']['target_device_tag']
                        scan_time = datetime.datetime.now()
                        for device in sm_devices:
                            if tag_disabled or tag_filter in device['tags']:
                                document = {
                                    'scanTime': scan_time,
                                    'scanIntervalMinutes': config['scan_interval_minutes'],
                                    'networkId': network['id'],
                                    'networkName': network['name']
                                }
                                for key in device:
                                    document[key] = device[key]

                                log_to_database(db, document,
                                    config['endpoints']['getNetworkSmDevices']['collection'],
                                    config['endpoints']['getNetworkSmDevices']['mode'],
                                    keyValuePair={'id': device['id']})

    print(str(datetime.datetime.now()) + " -- Scan complete")

def main(argv):
    arg_config_file = None

    try:
        opts, args = getopt.getopt(argv, 'c:')
    except getopt.GetoptError:
        sys.exit(2)

    for opt, arg in opts:
        if opt == '-c':
            arg_config_file = arg

    if arg_config_file is None:
        kill_script()

    try:
        config = load_config(arg_config_file)
        print(str(datetime.datetime.now()) + " -- Initializing script")
    except:
        kill_script()

    while(True):
        perform_scan(config)
        print(str(datetime.datetime.now()) + " -- Next scan in " + str(config['scan_interval_minutes']) + " minutes")
        time.sleep(config['scan_interval_minutes']*60)

if __name__ == '__main__':
    main(sys.argv[1:])


import numpy as np
import galois
from nacl.public import PrivateKey, Box
from io import BytesIO

import cProfile
import re

# A library for Shamir sharing arrays of field elements
# - An ArrayShare object is one share of the whole array
#   - ArrayShare objects have encrypt and decrypt methods
# - share_array secret shares an array, returning a dict mapping x coordinates
#   to their corresponding ArrayShare objects
# - reconstruct_array reconstructs the original array, given a list of
#   ArrayShare objects
# - sum_share_array performs a column-wise sum of a list of ArrayShare objects
#   that all agree on the x coordinate

# convert a numpy array to bytes
def array_to_bytes(x: np.ndarray) -> bytes:
    np_bytes = BytesIO()
    np.save(np_bytes, x, allow_pickle=True)
    return np_bytes.getvalue()

# de-serialize a numpy array from bytes
def bytes_to_array(b: bytes) -> np.ndarray:
    np_bytes = BytesIO(b)
    return np.load(np_bytes, allow_pickle=True)

class ArrayShare:
    """One Shamir share of an array. Stores its x coordinate, and an array of y coordinates,
       one y coordinate per element of the original array. All of the x coordinates must match."""
    def __init__(self, x, ys, T, GF, K=1, encrypted=False):
        self.x = x
        self.ys = ys
        self.GF = GF
        self.T = T
        self.K = K
        self.encrypted = encrypted

    def encrypt(self, sk, pk):
        assert not self.encrypted
        b = array_to_bytes(self.ys)
        enc_b = Box(sk, pk).encrypt(b)
        return ArrayShare(self.x, enc_b, self.T, self.GF, K=self.K, encrypted=True)

    def decrypt(self, sk, pk):
        assert self.encrypted
        m = Box(sk, pk).decrypt(self.ys)
        array = bytes_to_array(m)
        return ArrayShare(self.x, array, self.T, self.GF, K=self.K, encrypted=False)

    def __str__(self):
        return f'ArrayShare(x={self.x}, len={len(self.ys)}, T={self.T}, K={self.K}, enc={self.encrypted})'

    __repr__ = __str__

def reshape(secrets, K, GF):
    if len(secrets) %K == 0:
        return secrets.reshape((len(secrets)//K, K))

    true_len = (len(secrets)//K + 1) * K
    flat_pad = GF.Zeros((true_len))
    flat_pad[:len(secrets)] = secrets
    return flat_pad.reshape((len(secrets)//K + 1), K)

def share_packed(secrets, range_shares, T, K, GF):
    """
    secrets: flat array
    """
    secrets = reshape(secrets, K, GF)
    secrets = np.atleast_2d(secrets)
    p_size = (secrets.shape[0], K + T - 1)

    poly_points = GF.Random((p_size))
    poly_points[:, :K] = secrets
    xs = GF.Range(0, p_size[1])
    polys = [galois.lagrange_poly(xs, pps) for pps in poly_points]
    shares = {x: ArrayShare(x+K, GF(np.array([poly(x+K) for poly in polys])), T, GF, K=K) \
              for x in range_shares}
    return shares

def share_array(secrets, range_shares, T, GF, K=1):
    """Secret shares an array of secrets. Returns a dict mapping the x coordinate of each share
       to an ArrayShare object with that x coordinate."""
    return share_packed(secrets, range_shares, T, K, GF)

def reconstruct_array(array_shares):
    """Given a list of ArrayShare objects, reconstructs the original array"""
    assert len(array_shares) > 0
    array_len = len(array_shares[0].ys)
    GF = array_shares[0].GF
    T = array_shares[0].T
    K = array_shares[0].K

    assert len(array_shares) >= T + K, f'we have {len(array_shares)} shares, and we need {T + K}'

    # Error checking
    for s in array_shares:
        assert len(s.ys) == array_len
        assert s.GF == GF
        assert s.T == T
        assert s.K == K

    # Reconstruction
    arr = []
    xs = GF([s.x for s in array_shares])
    for i in range(array_len):
        # TODO: check T
        ys = GF([s.ys[i] for s in array_shares])
        poly = galois.lagrange_poly(xs, ys)
        arr.extend([poly(i) for i in range(0, K)])

    return GF(arr)

def sum_share_array(shares):
    """Given a list of ArrayShare objects with matching x coordinates, returns a new
       ArrayShare object representing the column-wise sum of the input shares"""
    assert len(shares) > 0
    x = shares[0].x
    GF = shares[0].GF
    T = shares[0].T
    K = shares[0].K

    for s in shares:
        assert not s.encrypted
        assert s.x == x
        assert s.GF == GF
        assert s.T == T
        assert s.K == K

    share_matrix = GF([s.ys for s in shares])
    sum_ys = share_matrix.sum(axis=0)

    return ArrayShare(x, sum_ys, T, GF, K=K, encrypted=False)

def prof():
    for _ in range(1):
        GF = galois.GF(2**31-1)
        vals = GF(np.random.randint(5, 6, 500))
        shares = share_array(vals, range(1,65), 4, GF, K=50)
        #print(shares)
        r = reconstruct_array(list(shares.values()))
        print(r)

if __name__ == '__main__':
    prof()
    #cProfile.run('prof()', sort='cumtime')


from typing import Any, Callable, Dict, Optional

import torch
import torch.nn as nn

from .gaussian_diffusion import GaussianDiffusion
from .k_diffusion import karras_sample

DEFAULT_KARRAS_STEPS = 64
DEFAULT_KARRAS_SIGMA_MIN = 1e-3
DEFAULT_KARRAS_SIGMA_MAX = 160
DEFAULT_KARRAS_S_CHURN = 0.0

def uncond_guide_model(
    model: Callable[..., torch.Tensor], scale: float
) -> Callable[..., torch.Tensor]:
    def model_fn(x_t, ts, **kwargs):
        half = x_t[: len(x_t) // 2]
        combined = torch.cat([half, half], dim=0)
        model_out = model(combined, ts, **kwargs)
        eps, rest = model_out[:, :3], model_out[:, 3:]
        cond_eps, uncond_eps = torch.chunk(eps, 2, dim=0)
        half_eps = uncond_eps + scale * (cond_eps - uncond_eps)
        eps = torch.cat([half_eps, half_eps], dim=0)
        return torch.cat([eps, rest], dim=1)

    return model_fn

def sample_latents(
    *,
    batch_size: int,
    model: nn.Module,
    diffusion: GaussianDiffusion,
    model_kwargs: Dict[str, Any],
    guidance_scale: float,
    clip_denoised: bool,
    use_fp16: bool,
    use_karras: bool,
    karras_steps: int,
    sigma_min: float,
    sigma_max: float,
    s_churn: float,
    device: Optional[torch.device] = None,
    progress: bool = False,
) -> torch.Tensor:
    sample_shape = (batch_size, model.d_latent)

    if device is None:
        device = next(model.parameters()).device

    if hasattr(model, "cached_model_kwargs"):
        model_kwargs = model.cached_model_kwargs(batch_size, model_kwargs)
    if guidance_scale != 1.0 and guidance_scale != 0.0:
        for k, v in model_kwargs.copy().items():
            model_kwargs[k] = torch.cat([v, torch.zeros_like(v)], dim=0)

    sample_shape = (batch_size, model.d_latent)
    with torch.autocast(device_type=device.type, enabled=use_fp16):
        if use_karras:
            samples = karras_sample(
                diffusion=diffusion,
                model=model,
                shape=sample_shape,
                steps=karras_steps,
                clip_denoised=clip_denoised,
                model_kwargs=model_kwargs,
                device=device,
                sigma_min=sigma_min,
                sigma_max=sigma_max,
                s_churn=s_churn,
                guidance_scale=guidance_scale,
                progress=progress,
            )
        else:
            internal_batch_size = batch_size
            if guidance_scale != 1.0:
                model = uncond_guide_model(model, guidance_scale)
                internal_batch_size *= 2
            samples = diffusion.p_sample_loop(
                model,
                shape=(internal_batch_size, *sample_shape[1:]),
                model_kwargs=model_kwargs,
                device=device,
                clip_denoised=clip_denoised,
                progress=progress,
            )

    return samples


from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from builtins import open
from builtins import super
from builtins import str
from future import standard_library
from future.utils import with_metaclass
standard_library.install_aliases()

import abc
import http.client
import http.server
import io
import logging
import os
import socketserver

from . import io as avro_io
from . import protocol
from . import schema

logger = logging.getLogger(__name__)

# ------------------------------------------------------------------------------
# Constants

def LoadResource(name):
  dir_path = os.path.dirname(__file__)
  rsrc_path = os.path.join(dir_path, name)
  with open(rsrc_path, 'r') as f:
    return f.read()

# Handshake schema is pulled in during build
HANDSHAKE_REQUEST_SCHEMA_JSON = LoadResource('HandshakeRequest.avsc')
HANDSHAKE_RESPONSE_SCHEMA_JSON = LoadResource('HandshakeResponse.avsc')

HANDSHAKE_REQUEST_SCHEMA = schema.Parse(HANDSHAKE_REQUEST_SCHEMA_JSON)
HANDSHAKE_RESPONSE_SCHEMA = schema.Parse(HANDSHAKE_RESPONSE_SCHEMA_JSON)

HANDSHAKE_REQUESTOR_WRITER = avro_io.DatumWriter(HANDSHAKE_REQUEST_SCHEMA)
HANDSHAKE_REQUESTOR_READER = avro_io.DatumReader(HANDSHAKE_RESPONSE_SCHEMA)
HANDSHAKE_RESPONDER_WRITER = avro_io.DatumWriter(HANDSHAKE_RESPONSE_SCHEMA)
HANDSHAKE_RESPONDER_READER = avro_io.DatumReader(HANDSHAKE_REQUEST_SCHEMA)

META_SCHEMA = schema.Parse('{"type": "map", "values": "bytes"}')
META_WRITER = avro_io.DatumWriter(META_SCHEMA)
META_READER = avro_io.DatumReader(META_SCHEMA)

SYSTEM_ERROR_SCHEMA = schema.Parse('["string"]')

AVRO_RPC_MIME = 'avro/binary'

# protocol cache

# Map: remote name -> remote MD5 hash
_REMOTE_HASHES = {}

# Decoder/encoder for a 32 bits big-endian integer.
UINT32_BE = avro_io.STRUCT_INT

# Default size of the buffers use to frame messages:
BUFFER_SIZE = 8192

# ------------------------------------------------------------------------------
# Exceptions

class AvroRemoteException(schema.AvroException):
  """
  Raised when an error message is sent by an Avro requestor or responder.
  """
  def __init__(self, fail_msg=None):
    schema.AvroException.__init__(self, fail_msg)

class ConnectionClosedException(schema.AvroException):
  pass

# ------------------------------------------------------------------------------
# Base IPC Classes (Requestor/Responder)

class BaseRequestor( with_metaclass( abc.ABCMeta, object ) ):
  """Base class for the client side of a protocol interaction."""

  def __init__(self, local_protocol, transceiver):
    """Initializes a new requestor object.

    Args:
      local_protocol: Avro Protocol describing the messages sent and received.
      transceiver: Transceiver instance to channel messages through.
    """
    self._local_protocol = local_protocol
    self._transceiver = transceiver
    self._remote_protocol = None
    self._remote_hash = None
    self._send_protocol = None

  @property
  def local_protocol(self):
    """Returns: the Avro Protocol describing the messages sent and received."""
    return self._local_protocol

  @property
  def transceiver(self):
    """Returns: the underlying channel used by this requestor."""
    return self._transceiver

  @abc.abstractmethod
  def _IssueRequest(self, call_request, message_name, request_datum):
    """TODO: Document this method.

    Args:
      call_request: ???
      message_name: Name of the message.
      request_datum: ???
    Returns:
      ???
    """
    raise Error('Abstract method')

  def Request(self, message_name, request_datum):
    """Writes a request message and reads a response or error message.

    Args:
      message_name: Name of the IPC method.
      request_datum: IPC request.
    Returns:
      The IPC response.
    """
    # build handshake and call request
    buffer_writer = io.BytesIO()
    buffer_encoder = avro_io.BinaryEncoder(buffer_writer)
    self._WriteHandshakeRequest(buffer_encoder)
    self._WriteCallRequest(message_name, request_datum, buffer_encoder)

    # send the handshake and call request; block until call response
    call_request = buffer_writer.getvalue()
    return self._IssueRequest(call_request, message_name, request_datum)

  def _WriteHandshakeRequest(self, encoder):
    """Emits the handshake request.

    Args:
      encoder: Encoder to write the handshake request into.
    """
    local_hash = self._local_protocol.md5

    # if self._remote_hash is None:
    #   remote_name = self.transceiver.remote_name
    #   self._remote_hash = _REMOTE_HASHES.get(remote_name)

    if self._remote_hash is None:
      self._remote_hash = local_hash
      self._remote_protocol = self._local_protocol

    request_datum = {
      'clientHash': local_hash,
      'serverHash': self._remote_hash,
    }
    if self._send_protocol:
      request_datum['clientProtocol'] = str(self._local_protocol)

    logger.info('Sending handshake request: %s', request_datum)
    HANDSHAKE_REQUESTOR_WRITER.write(request_datum, encoder)

  def _WriteCallRequest(self, message_name, request_datum, encoder):
    """
    The format of a call request is:
      * request metadata, a map with values of type bytes
      * the message name, an Avro string, followed by
      * the message parameters. Parameters are serialized according to
        the message's request declaration.
    """
    # request metadata (not yet implemented)
    request_metadata = {}
    META_WRITER.write(request_metadata, encoder)

    # Identify message to send:
    message = self.local_protocol.message_map.get(message_name)
    if message is None:
      raise schema.AvroException('Unknown message: %s' % message_name)
    encoder.write_utf8(message.name)

    # message parameters
    self._WriteRequest(message.request, request_datum, encoder)

  def _WriteRequest(self, request_schema, request_datum, encoder):
    logger.info('writing request: %s', request_datum)
    datum_writer = avro_io.DatumWriter(request_schema)
    datum_writer.write(request_datum, encoder)

  def _ReadHandshakeResponse(self, decoder):
    """Reads and processes the handshake response message.

    Args:
      decoder: Decoder to read messages from.
    Returns:
      call-response exists (boolean) ???
    Raises:
      schema.AvroException on ???
    """
    handshake_response = HANDSHAKE_REQUESTOR_READER.read(decoder)
    logger.info('Processing handshake response: %s', handshake_response)
    match = handshake_response['match']
    if match == 'BOTH':
      # Both client and server protocol hashes match:
      self._send_protocol = False
      return True

    elif match == 'CLIENT':
      # Client's side hash mismatch:
      self._remote_protocol = \
          protocol.Parse(handshake_response['serverProtocol'])
      self._remote_hash = handshake_response['serverHash']
      self._send_protocol = False
      return True

    elif match == 'NONE':
      # Neither client nor server match:
      self._remote_protocol = \
          protocol.Parse(handshake_response['serverProtocol'])
      self._remote_hash = handshake_response['serverHash']
      self._send_protocol = True
      return False
    else:
      raise schema.AvroException('handshake_response.match=%r' % match)

  def _ReadCallResponse(self, message_name, decoder):
    """Reads and processes a method call response.

    The format of a call response is:
      - response metadata, a map with values of type bytes
      - a one-byte error flag boolean, followed by either:
        - if the error flag is false,
          the message response, serialized per the message's response schema.
        - if the error flag is true,
          the error, serialized per the message's error union schema.

    Args:
      message_name:
      decoder:
    Returns:
      ???
    Raises:
      schema.AvroException on ???
    """
    # response metadata
    response_metadata = META_READER.read(decoder)

    # remote response schema
    remote_message_schema = self._remote_protocol.message_map.get(message_name)
    if remote_message_schema is None:
      raise schema.AvroException('Unknown remote message: %s' % message_name)

    # local response schema
    local_message_schema = self._local_protocol.message_map.get(message_name)
    if local_message_schema is None:
      raise schema.AvroException('Unknown local message: %s' % message_name)

    # error flag
    if not decoder.read_boolean():
      writer_schema = remote_message_schema.response
      reader_schema = local_message_schema.response
      return self._ReadResponse(writer_schema, reader_schema, decoder)
    else:
      writer_schema = remote_message_schema.errors
      reader_schema = local_message_schema.errors
      raise self._ReadError(writer_schema, reader_schema, decoder)

  def _ReadResponse(self, writer_schema, reader_schema, decoder):
    datum_reader = avro_io.DatumReader(writer_schema, reader_schema)
    result = datum_reader.read(decoder)
    return result

  def _ReadError(self, writer_schema, reader_schema, decoder):
    datum_reader = avro_io.DatumReader(writer_schema, reader_schema)
    return AvroRemoteException(datum_reader.read(decoder))

class Requestor(BaseRequestor):
  """Concrete requestor implementation."""

  def _IssueRequest(self, call_request, message_name, request_datum):
    call_response = self.transceiver.Transceive(call_request)

    # process the handshake and call response
    buffer_decoder = avro_io.BinaryDecoder(io.BytesIO(call_response))
    call_response_exists = self._ReadHandshakeResponse(buffer_decoder)
    if call_response_exists:
      return self._ReadCallResponse(message_name, buffer_decoder)
    else:
      return self.Request(message_name, request_datum)

# ------------------------------------------------------------------------------

class Responder( with_metaclass( abc.ABCMeta, object ) ):
  """Base class for the server side of a protocol interaction."""

  def __init__(self, local_protocol):
    self._local_protocol = local_protocol
    self._local_hash = self._local_protocol.md5
    self._protocol_cache = {}

    self.set_protocol_cache(self._local_hash, self._local_protocol)

  @property
  def local_protocol(self):
    return self._local_protocol

  # utility functions to manipulate protocol cache
  def get_protocol_cache(self, hash):
    return self._protocol_cache.get(hash)

  def set_protocol_cache(self, hash, protocol):
    self._protocol_cache[hash] = protocol

  def Respond(self, call_request):
    """Entry point to process one procedure call.

    Args:
      call_request: Serialized procedure call request.
    Returns:
      Serialized procedure call response.
    Raises:
      ???
    """
    buffer_reader = io.BytesIO(call_request)
    buffer_decoder = avro_io.BinaryDecoder(buffer_reader)
    buffer_writer = io.BytesIO()
    buffer_encoder = avro_io.BinaryEncoder(buffer_writer)
    error = None
    response_metadata = {}

    try:
      remote_protocol = self._ProcessHandshake(buffer_decoder, buffer_encoder)
      # handshake failure
      if remote_protocol is None:
        return buffer_writer.getvalue()

      # read request using remote protocol
      request_metadata = META_READER.read(buffer_decoder)
      remote_message_name = buffer_decoder.read_utf8()

      # get remote and local request schemas so we can do
      # schema resolution (one fine day)
      remote_message = remote_protocol.message_map.get(remote_message_name)
      if remote_message is None:
        fail_msg = 'Unknown remote message: %s' % remote_message_name
        raise schema.AvroException(fail_msg)
      local_message = self.local_protocol.message_map.get(remote_message_name)
      if local_message is None:
        fail_msg = 'Unknown local message: %s' % remote_message_name
        raise schema.AvroException(fail_msg)
      writer_schema = remote_message.request
      reader_schema = local_message.request
      request = self._ReadRequest(writer_schema, reader_schema, buffer_decoder)
      logger.info('Processing request: %r', request)

      # perform server logic
      try:
        response = self.Invoke(local_message, request)
      except AvroRemoteException as exn:
        error = exn
      except Exception as exn:
        error = AvroRemoteException(str(exn))

      # write response using local protocol
      META_WRITER.write(response_metadata, buffer_encoder)
      buffer_encoder.write_boolean(error is not None)
      if error is None:
        writer_schema = local_message.response
        self._WriteResponse(writer_schema, response, buffer_encoder)
      else:
        writer_schema = local_message.errors
        self._WriteError(writer_schema, error, buffer_encoder)
    except schema.AvroException as exn:
      error = AvroRemoteException(str(exn))
      buffer_encoder = avro_io.BinaryEncoder(io.StringIO())
      META_WRITER.write(response_metadata, buffer_encoder)
      buffer_encoder.write_boolean(True)
      self._WriteError(SYSTEM_ERROR_SCHEMA, error, buffer_encoder)
    return buffer_writer.getvalue()

  def _ProcessHandshake(self, decoder, encoder):
    """Processes an RPC handshake.

    Args:
      decoder: Where to read from.
      encoder: Where to write to.
    Returns:
      The requested Protocol.
    """
    handshake_request = HANDSHAKE_RESPONDER_READER.read(decoder)
    logger.info('Processing handshake request: %s', handshake_request)

    # determine the remote protocol
    client_hash = handshake_request.get('clientHash')
    client_protocol = handshake_request.get('clientProtocol')
    remote_protocol = self.get_protocol_cache(client_hash)
    if remote_protocol is None and client_protocol is not None:
      remote_protocol = protocol.Parse(client_protocol)
      self.set_protocol_cache(client_hash, remote_protocol)

    # evaluate remote's guess of the local protocol
    server_hash = handshake_request.get('serverHash')

    handshake_response = {}
    if self._local_hash == server_hash:
      if remote_protocol is None:
        handshake_response['match'] = 'NONE'
      else:
        handshake_response['match'] = 'BOTH'
    else:
      if remote_protocol is None:
        handshake_response['match'] = 'NONE'
      else:
        handshake_response['match'] = 'CLIENT'

    if handshake_response['match'] != 'BOTH':
      handshake_response['serverProtocol'] = str(self.local_protocol)
      handshake_response['serverHash'] = self._local_hash

    logger.info('Handshake response: %s', handshake_response)
    HANDSHAKE_RESPONDER_WRITER.write(handshake_response, encoder)
    return remote_protocol

  @abc.abstractmethod
  def Invoke(self, local_message, request):
    """Processes one procedure call.

    Args:
      local_message: Avro message specification.
      request: Call request.
    Returns:
      Call response.
    Raises:
      ???
    """
    raise Error('abtract method')

  def _ReadRequest(self, writer_schema, reader_schema, decoder):
    datum_reader = avro_io.DatumReader(writer_schema, reader_schema)
    return datum_reader.read(decoder)

  def _WriteResponse(self, writer_schema, response_datum, encoder):
    datum_writer = avro_io.DatumWriter(writer_schema)
    datum_writer.write(response_datum, encoder)

  def _WriteError(self, writer_schema, error_exception, encoder):
    datum_writer = avro_io.DatumWriter(writer_schema)
    datum_writer.write(str(error_exception), encoder)

# ------------------------------------------------------------------------------
# Framed message

class FramedReader(object):
  """Wrapper around a file-like object to read framed data."""

  def __init__(self, reader):
    self._reader = reader

  def Read(self):
    """Reads one message from the configured reader.

    Returns:
      The message, as bytes.
    """
    message = io.BytesIO()
    # Read and append frames until we encounter a 0-size frame:
    while self._ReadFrame(message) > 0: pass
    return message.getvalue()

  def _ReadFrame(self, message):
    """Reads and appends one frame into the given message bytes.

    Args:
      message: Message to append the frame to.
    Returns:
      Size of the frame that was read.
      The empty frame (size 0) indicates the end of a message.
    """
    frame_size = self._ReadInt32()
    remaining = frame_size
    while remaining > 0:
      data_bytes = self._reader.read(remaining)
      if len(data_bytes) == 0:
        raise ConnectionClosedException(
            'FramedReader: expecting %d more bytes in frame of size %d, got 0.'
            % (remaining, frame_size))
      message.write(data_bytes)
      remaining -= len(data_bytes)
    return frame_size

  def _ReadInt32(self):
    encoded = self._reader.read(UINT32_BE.size)
    if len(encoded) != UINT32_BE.size:
      raise ConnectionClosedException('Invalid header: %r' % encoded)
    return UINT32_BE.unpack(encoded)[0]

class FramedWriter(object):
  """Wrapper around a file-like object to write framed data."""

  def __init__(self, writer):
    self._writer = writer

  def Write(self, message):
    """Writes a message.

    Message is chunked into sequences of frames terminated by an empty frame.

    Args:
      message: Message to write, as bytes.
    """
    while len(message) > 0:
      chunk_size = max(BUFFER_SIZE, len(message))
      chunk = message[:chunk_size]
      self._WriteBuffer(chunk)
      message = message[chunk_size:]

    # A message is always terminated by a zero-length buffer.
    self._WriteUnsignedInt32(0)

  def _WriteBuffer(self, chunk):
    self._WriteUnsignedInt32(len(chunk))
    self._writer.write(chunk)

  def _WriteUnsignedInt32(self, uint32):
    self._writer.write(UINT32_BE.pack(uint32))

# ------------------------------------------------------------------------------
# Transceiver (send/receive channel)

class Transceiver( with_metaclass( abc.ABCMeta, object ) ):
  @abc.abstractproperty
  def remote_name(self):
    pass

  @abc.abstractmethod
  def ReadMessage(self):
    """Reads a single message from the channel.

    Blocks until a message can be read.

    Returns:
      The message read from the channel.
    """
    pass

  @abc.abstractmethod
  def WriteMessage(self, message):
    """Writes a message into the channel.

    Blocks until the message has been written.

    Args:
      message: Message to write.
    """
    pass

  def Transceive(self, request):
    """Processes a single request-reply interaction.

    Synchronous request-reply interaction.

    Args:
      request: Request message.
    Returns:
      The reply message.
    """
    self.WriteMessage(request)
    result = self.ReadMessage()
    return result

  def Close(self):
    """Closes this transceiver."""
    pass

class HTTPTransceiver(Transceiver):
  """HTTP-based transceiver implementation."""

  def __init__(self, host, port, req_resource='/'):
    """Initializes a new HTTP transceiver.

    Args:
      host: Name or IP address of the remote host to interact with.
      port: Port the remote server is listening on.
      req_resource: Optional HTTP resource path to use, '/' by default.
    """
    self._req_resource = req_resource
    self._conn = http.client.HTTPConnection(host, port)
    self._conn.connect()
    self._remote_name = self._conn.sock.getsockname()

  @property
  def remote_name(self):
    return self._remote_name

  def ReadMessage(self):
    response = self._conn.getresponse()
    response_reader = FramedReader(response)
    framed_message = response_reader.Read()
    response.read()    # ensure we're ready for subsequent requests
    return framed_message

  def WriteMessage(self, message):
    req_method = 'POST'
    req_headers = {'Content-Type': AVRO_RPC_MIME}

    bio = io.BytesIO()
    req_body_buffer = FramedWriter(bio)
    req_body_buffer.Write(message)
    req_body = bio.getvalue()

    self._conn.request(req_method, self._req_resource, req_body, req_headers)

  def Close(self):
    self._conn.close()
    self._conn = None

# ------------------------------------------------------------------------------
# Server Implementations

def _MakeHandlerClass(responder):
  class AvroHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
    def do_POST(self):
      reader = FramedReader(self.rfile)
      call_request = reader.Read()
      logger.info('Serialized request: %r', call_request)
      call_response = responder.Respond(call_request)
      logger.info('Serialized response: %r', call_response)

      self.send_response(200)
      self.send_header('Content-type', AVRO_RPC_MIME)
      self.end_headers()

      framed_writer = FramedWriter(self.wfile)
      framed_writer.Write(call_response)
      self.wfile.flush()
      logger.info('Response sent')

  return AvroHTTPRequestHandler

class MultiThreadedHTTPServer(
    socketserver.ThreadingMixIn,
    http.server.HTTPServer,
):
  """Multi-threaded HTTP server."""
  pass

class AvroIpcHttpServer(MultiThreadedHTTPServer):
  """Avro IPC server implemented on top of an HTTP server."""

  def __init__(self, interface, port, responder):
    """Initializes a new Avro IPC server.

    Args:
      interface: Interface the server listens on, eg. 'localhost' or '0.0.0.0'.
      port: TCP port the server listens on, eg. 8000.
      responder: Responder implementation to handle RPCs.
    """
    super(AvroIpcHttpServer, self).__init__(
        server_address=(interface, port),
        RequestHandlerClass=_MakeHandlerClass(responder),
    )

if __name__ == '__main__':
  raise Exception('Not a standalone module')


import asyncio

import aiohttp

from warnings import warn

with open("vk_access_token.txt", mode="r") as file:
    vk_access_token = file.read()
vk_api_version = "5.154"
owner_id = "-160464793"
url = f"https://api.vk.ru/method/wall.get?v={vk_api_version}&owner_id={owner_id}&count=1&access_token={vk_access_token}"

async def get_last_text_message():
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            a = await response.json()
            try:
                a = a["response"]["items"][0]["text"]
            except:
                a = None
            return a

async def ticker(delay):
    while True:
        yield await get_last_text_message()
        await asyncio.sleep(delay)

async def set_morning_trigger(trigger_function, delay=30):
    if delay < 28:
        warn("The delay is too small. Limit of wall.get usage per day may be exceeded.")
    generator = ticker(delay)
    async def get_last_text():
        return await generator.asend(None)
    text = ""
    while True:
        new_text = await get_last_text()
        if new_text != None and text != new_text:
            text = new_text
            asyncio.gather(trigger_function(text))

if __name__ == "__main__":
    async def function(string):
        print(string)
    asyncio.run(set_morning_trigger(function))


import os
from fastapi import status, HTTPException
from pydantic import BaseModel, validator
from typing import Union

ROOT_DIR = os.path.abspath('.')
ROOT_DIR = os.path.join(ROOT_DIR, 'assets')
PATH_REGEX = r'^(?![0-9._/])(?!.*[._]$)(?!.*\d_)(?!.*_\d)[a-zA-Z0-9_/]+$'

class Predicted(BaseModel):
    recognized: Union[list, None] = []
    unknowns: Union[int, None] = 0
    peoples: Union[int, None] = 0
    id_img: str
    spend_time: float

class SelectModel(BaseModel):
    model_file: str = 'example_model.pk'

    @validator('model_file', pre=True)
    def validate_trained_model_folder(cls, value):
        directory_path = os.path.join(ROOT_DIR, 'trained_models')
        file_path = os.path.join(directory_path, value)
        split_current = os.path.split(file_path)[1]
        if len(split_current) > 20:
            raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST,
                                detail='Name cannot be longer than 20 characters')
        if not value.endswith('.pk'):
            join_pk = f'{file_path}.pk'
            if not os.path.isfile(join_pk):
                raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST,
                                    detail=f'Not found path file {join_pk}')
            return join_pk
        if not os.path.isfile(file_path):
            raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST,
                                detail=f'Not found path file {file_path}')
        return file_path

    class Config:
        schema_extra = {
            'example': {
                'model_file': 'example_model.pk'
            }
        }


import os, sys
import numpy as np
import pickle
from UserCode.HGCalMaskResolutionAna import Argparser
from array import array as Carray
from collections import OrderedDict

from ROOT import TCanvas, TLatex, TFile, TMath, TH1F
from ROOT import TLegend, TH2F, TLorentzVector, TProfile, TH1D, TGraphErrors
from ROOT import Double, gStyle, gROOT, kTemperatureMap
from UserCode.HGCalMaskVisualProd.SystemUtils import averageContiguousVals as Av
from UserCode.HGCalMaskVisualProd.SystemUtils import EtaStr as ES
from UserCode.HGCalMaskResolutionAna.SoftwareCorrection import IncompleteShowersCorrection
from UserCode.HGCalMaskResolutionAna.SoftwareCorrection import DifferentiateShowersByEnergy
from UserCode.HGCalMaskResolutionAna.Calibration import Calibration
from UserCode.HGCalMaskResolutionAna.PartialWafersStudies import PartialWafersStudies
from UserCode.HGCalMaskVisualProd.RootPlotting import RootPlotting
from UserCode.HGCalMaskVisualProd.RootObjects import RootHistograms

def plotHistograms(histos, cdims, pcoords, cname):
    """
    Plots histograms from a list.
    Arguments:
    -> histos: list of histograms
    -> cdims & pccords: as described in RooUtils
    -> cname: name of the canvas to be created
    """
    if not isinstance(histos, list):
        raise TypeError('The histograms have to be passed in a list.,')

    npads = len(pcoords[0])
    hdiv, stack = ([] for _ in range(2))
    with RootPlotting(ncanvas=1, npads=npads, cdims=cdims, pcoords=pcoords) as plot:
        titles = ['Resolution', 'Resolution vs Eta',
                  'RMS vs Eta', 'Bias vs Eta', 'RMS/(1+Bias) vs Eta',
                  'Resolution', 'Nevents', 'Resolution / Nevents']
        if FLAGS.mode == 1:
            legends1 = [TLegend(0.12, 0.76, 0.44, 0.89) for  _ in range(3)]
            it = -1
            for ih in range(len(histos[:-3])):
                if ih%3==0: it += 1
                h = histos[ih]
                if ih<6:
                    plot.plotHistogram(cpos=0, ppos=ih, h=h,
                                       lw=3,mc=1,msize=.5,lc=1,
                                       title=titles[it], draw_options='colz')
                    if ih<3:
                        tex = plot.setLatex(ts=0.04)
                        plot.fitHistogram(h=h, fname='crystalball',
                                          frange=(-1.,1.), tex=tex)
                else:
                    plot.plotGraph(cpos=0, ppos=ih, g=h,
                                   lw=3,mc=4,msize=.5,lc=4,
                                   yranges=(-0.5,3.5),
                                   title=titles[it],
                                   draw_options='AP')

        elif FLAGS.mode == 2:
            if FLAGS.apply_weights:
                legends1 = [TLegend(0.12, 0.76, 0.44, 0.89) for  _ in range(3)]
            else:
                legends1 = [TLegend(0.56, 0.66, 0.86, 0.89) for  _ in range(3)]
                legends2 = [TLegend(0.68, 0.75, 0.86, 0.89) for _ in range(3)]

            for ih in range(NREG):
                if FLAGS.apply_weights:
                    for ixx in range(int(npads/NREG)):
                        idx = ih+NREG*ixx
                        if ixx == 0:
                            plot.plotHistogram(cpos=0, ppos=idx, h=histos[idx],
                                               lw=3,mc=4,msize=.5,lc=4,
                                               draw_options='colz')
                        elif ixx == 1:
                            plot.plotHistogram(cpos=0, ppos=idx, h=histos[idx],
                                               lw=3,mc=4,msize=.5,lc=4,
                                               title=titles[ixx],
                                               draw_options='colz')
                        else:
                            plot.plotGraph(cpos=0, ppos=idx, g=histos[idx],
                                           lw=3,mc=4,msize=.5,lc=4,
                                           yranges=(-0.5,3.5),
                                           title=titles[ixx],
                                           name = str(ixx*ih),
                                           draw_options='AP')
                        tex = plot.setLatex()
                        if ixx == 0:
                            plot.fitHistogram(h=histos[idx], fname='crystalball',
                                              frange=(-1.,1.), tex=tex)
                            if FLAGS.samples == 'inner':
                                tex.DrawLatex(0.58,0.92,'Inner radius;  SR{}'
                                              .format((ih%3)+1))
                            elif FLAGS.samples == 'outer':
                                tex.DrawLatex(0.58,0.92,'Outer radius;  SR{}'
                                              .format((ih%3)+1))

                            tex.DrawLatex(0.11,0.92,'#bf{CMS} #it{simulation preliminary}')
                            tex.SetTextAlign(31)
                elif not FLAGS.apply_weights:
                    linec = [4, 2, 3, 7]
                    plot.plotHistogram(cpos=0, ppos=ih, h=histos[ih],
                                       lw=3,mc=linec[0],msize=.5,lc=linec[0],
                                       draw_options='E')
                    plot.plotHistogram(cpos=0, ppos=ih, h=histos[ih+3],
                                       lw=3, mc=linec[1], msize=.5, lc=linec[1],
                                       draw_options='same E')
                    plot.plotHistogram(cpos=0, ppos=ih, h=histos[ih+6],
                                       lw=3, mc=linec[2], msize=.5, lc=linec[2],
                                       draw_options='same E')
                    plot.plotHistogram(cpos=0, ppos=ih, h=histos[ih+9],
                                       lw=3, mc=linec[3], msize=.5, lc=linec[3],
                                       draw_options='same E')
                    tex = plot.setLatex()
                    bckgcuts_str = [str(i) for i in bckgcuts]
                    if FLAGS.samples == 'inner':
                        tex.DrawLatex(0.58,0.92,'Inner radius;  SR{}'.format((ih%3)+1))
                    elif FLAGS.samples == 'outer':
                        tex.DrawLatex(0.58,0.92,'Outer radius;  SR{}'.format((ih%3)+1))

                    legends1[ih].AddEntry(histos[ih],
                                          'Cumdiff < '+bckgcuts_str[0], 'L')
                    for it in range(len(bckgcuts_str)-1):
                        legends1[ih].AddEntry(histos[ih+3*(it+1)],
                                    bckgcuts_str[it]+'< Cumdiff < '+bckgcuts_str[it+1], 'L')
                    legends1[ih].AddEntry(histos[ih+3*(it+2)],
                                    'Cumdiff > '+bckgcuts_str[it+1], 'L')

                    tex.DrawLatex(0.11,0.92,'#bf{CMS} #it{simulation preliminary}')
                    tex.SetTextAlign(31)
                    legends1[ih].Draw()

                    hdiv.append(histos[ih+3].Clone('weight1_sr{}'.format(ih+1)))
                    hdiv.append(histos[ih+6].Clone('weight2_sr{}'.format(ih+1)))
                    hdiv.append(histos[ih+9].Clone('weight3_sr{}'.format(ih+1)))
                    for idiv in range(len(bckgcuts)):
                        hdiv[-3+idiv].Divide(histos[ih])
                        extrastr = '' if idiv==0 else 'same'
                        hdiv[-3+idiv].GetYaxis().SetRangeUser(0., 6.)
                        plot.plotHistogram(cpos=0, ppos=ih+3, h=hdiv[-3+idiv],
                                           yaxis_title='Weight',
                                           lw=3, mc=linec[idiv+1], msize=.5,
                                           lc=linec[idiv+1],
                                           draw_options='HIST'+extrastr, copy=True)
                    tex = plot.setLatex()
                    if FLAGS.samples == 'inner':
                        tex.DrawLatex(0.58,0.92,'Inner radius;  SR{}'.format((ih%3)+1))
                    elif FLAGS.samples == 'outer':
                        tex.DrawLatex(0.58,0.92,'Outer radius;  SR{}'.format((ih%3)+1))
                    for iv in range(len(bckgcuts)):
                        legends2[ih].AddEntry(hdiv[iv], 'weight'+str(iv+1), 'L')
                        legends2[ih].Draw()
                    tex.DrawLatex(0.11,0.92,'#bf{CMS} #it{simulation preliminary}')
                    tex.SetTextAlign(31)

        plot.save(cpos=0, name=cname)

    if not FLAGS.apply_weights:
        save_str = base.paths.weights
        RootHistograms(histos).save(save_str)
        RootHistograms(hdiv).save(save_str, mode='UPDATE')

def main():
    #gStyle.SetOptStat(0)
    gROOT.SetBatch(True)
    gStyle.SetPalette(kTemperatureMap)

    fIn=TFile.Open(FLAGS.noPUFile)
    data=fIn.Get('data')
    calib_str = base.paths.calibrations_nopu
    """
    calibration = Calibration(FLAGS)
    calibration.nopu_calibration()
    calibration.save(calib_str)
    """
    with open(calib_str, 'r') as cachefile:
        calib = pickle.load(cachefile)

    if FLAGS.apply_weights:
        calibshowers_str = base.paths.weights
        bckgcuts_extended = np.append(bckgcuts, 0.9)
        showercorr = IncompleteShowersCorrection(calibshowers_str,
                                                 discrvals=Av(bckgcuts_extended))
        weights = showercorr.CorrectionWeights()
        if FLAGS.samples == 'inner':
            boundaries = [5, 5, 5]
            corr_mode = 'left'
        elif FLAGS.samples == 'outer':
            boundaries = [23, 23, 23]
            corr_mode = 'right'
        lowstats_factors = showercorr.calculateLowStatisticsFactor(boundaries, corr_mode)
        weights_graphs = [showercorr.buildCorrectionWeightsGraphs(region=i+1)
                          for i in range(NREG)]

    histos=OrderedDict()
    enbins, eninf, ensup = 200, -2.01, 1.99
    if FLAGS.samples == 'inner':
        phibins, etabins, etainf, etasup = 12, 10, 2.69, 3.04
    elif FLAGS.samples == 'outer':
        phibins, etabins, etainf, etasup = 12, 10, 1.44, 1.66

    if FLAGS.mode == 1:
        hn = ['den{}', 'den_eta{}', 'den{}_2D_res', 'den{}_2D_events']
        for ireg in range(1,NREG+1):
            bins = Carray('d', np.arange(-1.05, .8, 0.01))
            strings = ';#Delta E/E_{gen};PDF'
            histos[hn[0].format(ireg)] = TH1F(hn[0].format(ireg), strings,
                                              len(bins)-1, bins)
            histos[hn[1].format(ireg)] = TH2F(hn[1].format(ireg), ';|#eta|;#Delta E/E',
                                              etabins, etainf, etasup,
                                              enbins, eninf, ensup)
            histos[hn[2].format(ireg)] = TH2F(hn[2].format(ireg), ';|#eta|;#phi',
                                              50, etainf, etasup,
                                              phibins, -TMath.Pi(), TMath.Pi())
            histos[hn[3].format(ireg)] = TH2F(hn[3].format(ireg), ';|#eta|;#phi',
                                              50, etainf, etasup,
                                              phibins, -TMath.Pi(), TMath.Pi())
    elif FLAGS.mode == 2:
        fracEn = np.zeros((NREG,NLAYERS), dtype=float)
        countfracEn = np.zeros((NREG,NLAYERS), dtype=int)

        for i in range(0, data.GetEntriesFast()):
            data.GetEntry(i)
            genen    = getattr(data,'genen')
            geneta   = abs(getattr(data,'geneta'))
            genphi   = getattr(data,'genphi')

            for ireg in range(1,NREG+1):
                recen = getattr(data,'en_sr{}_ROI'.format(ireg))
                avgnoise = getattr(data,'noise_sr3_ROI')*A[ireg-1]/A[2]

                #Calibration factors. f2 is used for PU.
                f1, f2 = 1., 0.
                etaregions_shifted = np.roll(etaregions, shift=-1)[:-1]
                for ieta1,ieta2 in zip(etaregions[:-1], etaregions_shifted):
                    #in case it lies outside the limits of the calibration
                    #the event is calibrated with the full calibration region
                    if geneta < etaregions[0] or geneta > etaregions[-1]:
                        idstr = 'sr{}_from{}to{}'.format(ireg,
                                                         ES(etaregions[0]),
                                                         ES(etaregions[-1]))
                    elif (geneta < ieta1 or geneta >= ieta2):
                        continue
                    else:
                        idstr = 'sr{}_from{}to{}'.format(ireg, ES(ieta1), ES(ieta2))
                    if 'L0' in calib:
                        f1 /= calib['L0'][idstr].Eval(geneta)+1.0
                        if 'L1' in calib:
                            f1 /= calib['L1'][idstr].Eval(f1*recen)+1.0
                            if 'L2' in calib and ireg in calib['L2']:
                                f2 = calib['L2'][idstr].Eval(avgnoise)
                    recen = f1*recen - f2
                for il in range(1,NLAYERS+1):
                    v = f1*getattr(data,'en_sr{}_layer{}'.format(ireg,il)) - f2
                    if ( (FLAGS.samples == "inner" and geneta < etaregions[0]+0.05 or
                          FLAGS.samples == "outer" and geneta > etaregions[-1]-0.05) and
                         recen != 0 ):
                        fracEn[ireg-1,il-1] += v / recen
                        countfracEn[ireg-1,il-1] += 1
        fracEn /= countfracEn

        hn = ['res_complete_before{}',          'res_complete_after{}',
              'res_incomplete_before{}',        'res_incomplete_after{}',
              'res_total_before{}',             'res_total_after{}',
              'res_vs_eta_before{}',            'res_vs_eta_after{}',
              'en{}_per_layer_signal',          'en{}_per_layer_bckg1',
              'en{}_per_layer_bckg2',           'en{}_per_layer_bckg3',
              'noise{}_per_layer_signal',       'noise{}_per_layer_bckg1',
              'noise{}_per_layer_bckg2',        'noise{}_per_layer_bckg3']
        for ireg in range(1,NREG+1):
            bins = Carray('d', np.arange(-1.05, .8, 0.01))
            strings = ';#Delta E/E_{gen};PDF'
            for ih in range(6):
                histos[hn[ih].format(ireg)] = TH1F(hn[ih].format(ireg), strings,
                                               len(bins)-1, bins)

            histos[hn[6].format(ireg)] = TH2F(hn[6].format(ireg), ';|#eta|;#Delta E/E',
                                              etabins, etainf, etasup,
                                              enbins, eninf, ensup)
            histos[hn[7].format(ireg)] = TH2F(hn[7].format(ireg), ';|#eta|;#Delta E/E',
                                              etabins, etainf, etasup,
                                              enbins, eninf, ensup)

            bins = Carray('d', np.arange(0.5,29,1.))
            strings = ';Layer;E_{reco} / E_{gen}'
            for ih in range(8,16):
                histos[hn[ih].format(ireg)] = TProfile(hn[ih].format(ireg), strings,
                                                       len(bins)-1, bins)
    for h in histos:
        histos[h].Sumw2()
        histos[h].SetMarkerStyle(20)
        histos[h].SetDirectory(0)

    for i in range(0, data.GetEntriesFast()):
        data.GetEntry(i)
        genen    = getattr(data,'genen')
        geneta   = abs(getattr(data,'geneta'))
        genphi   = getattr(data,'genphi')

        for ireg in range(1,NREG+1):
            recen = getattr(data,'en_sr{}_ROI'.format(ireg))
            avgnoise = getattr(data,'noise_sr3_ROI')*A[ireg-1]/A[2]

            #Calibration factors. f2 is used for PU.
            f1, f2 = 1., 0.
            etaregions_shifted = np.roll(etaregions, shift=-1)[:-1]
            for ieta1,ieta2 in zip(etaregions[:-1], etaregions_shifted):
                if geneta < etaregions[0] or geneta > etaregions[-1]:
                    idstr = 'sr{}_from{}to{}'.format(ireg,
                                                     ES(etaregions[0]), ES(etaregions[-1]))
                elif (geneta < ieta1 or geneta > ieta2):
                    continue
                else:
                    idstr = 'sr{}_from{}to{}'.format(ireg, ES(ieta1), ES(ieta2))
                if 'L0' in calib:
                    f1 /= calib['L0'][idstr].Eval(geneta)+1.0
                    if 'L1' in calib:
                        f1 /= calib['L1'][idstr].Eval(f1*recen)+1.0
                        if 'L2' in calib and ireg in calib['L2']:
                            f2 = calib['L2'][idstr].Eval(avgnoise)
            assert f1 != 1.
            recen = f1*recen - f2
            deltaE = recen/genen-1.

            ###Store the energy resolution###
            if FLAGS.mode == 1:
                if deltaE > -1:
                    histos[hn[0].format(ireg)].Fill(deltaE)
                    histos[hn[1].format(ireg)].Fill(geneta, deltaE)
                histos[hn[2].format(ireg)].Fill(geneta, genphi, deltaE)
                histos[hn[3].format(ireg)].Fill(geneta, genphi)

            elif FLAGS.mode == 2:
                #differentiate complete from incomplete showers
                ROI_en = np.zeros((NLAYERS), dtype=float)
                for il in range(1,NLAYERS+1):
                    v = f1*getattr(data,'en_sr{}_layer{}'.format(ireg,il)) - f2
                    try:
                        ROI_en[il-1] = v/recen
                    except ZeroDivisionError:
                        ROI_en[il-1] = 0.

                lshift = [1., 1., 1.] if FLAGS.samples == 'outer' else [.65, .59, .48] #layer shift
                assert len(bckgcuts) == len(lshift)
                showerid = DifferentiateShowersByEnergy(ROI_en, fracEn[ireg-1,:],
                                                        thresholds=bckgcuts, min_val=0.05)

                ###Calculate andc calibrate the energy per layer###
                recen_corr = 0
                for il in range(1,NLAYERS+1):
                    if FLAGS.apply_weights:
                        if FLAGS.samples == 'inner':
                            weight_limit = il > boundaries[ireg-1]
                        else:
                            weight_limit = il < boundaries[ireg-1]

                    b = histos[hn[8].format(ireg)].FindBin(il)
                    if showerid==0: #complete shower
                        v = f1*getattr(data,'en_sr{}_layer{}'.format(ireg,il)) - f2
                        try:
                            histos[hn[8].format(ireg)].Fill(b,v/recen)
                        except ZeroDivisionError:
                            histos[hn[8].format(ireg)].Fill(b,0.)
                        if FLAGS.apply_weights:
                            recen_corr += v
                        v = (f1*getattr(data,'noise_sr3_layer{}'.format(il))
                             *A[ireg-1]/A[2] - f2)
                        try:
                            histos[hn[12].format(ireg)].Fill(b,v/recen)
                        except ZeroDivisionError:
                            histos[hn[12].format(ireg)].Fill(b,0.)
                    else:
                        w = showerid-1
                        v = f1*getattr(data,'en_sr{}_layer{}'.format(ireg,il)) - f2
                        try:
                            histos[hn[9+w].format(ireg)].Fill(b*lshift[w],v/recen)
                        except ZeroDivisionError:
                            histos[hn[9+w].format(ireg)].Fill(b*lshift[w],0.)
                        if ( FLAGS.apply_weights and
                             weights[ireg-1][w][il-1]!=0 and
                             weight_limit):
                            recen_corr += v/weights[ireg-1][w][int(round((il-1)*lshift[w],0))]
                            #weight_graphs[ireg][il].SetBit(weight_graphs[ireg][il].klsSortedX)
                            #weight_graphs[ireg][il].Eval(geneta, spline=0, 'S')
                        v = (f1*getattr(data,'noise_sr3_layer{}'.format(il))
                             *A[ireg-1]/A[2] - f2)
                        try:
                            histos[hn[13+w].format(ireg)].Fill(b,v/recen)
                        except ZeroDivisionError:
                            histos[hn[13+w].format(ireg)].Fill(b,0.)

                if FLAGS.apply_weights and FLAGS.method == 'ed':
                    if showerid==0: #complete shower
                        deltaE_corr = recen_corr/genen-1.
                        histos[hn[0].format(ireg)].Fill(deltaE)
                        histos[hn[1].format(ireg)].Fill(deltaE_corr)
                    else:
                        recen_corr *= (1 / (1-lowstats_factors[ireg-1]) )
                        distshift = 0.09 if FLAGS.samples == 'inner' else 0.08
                        recen_corr *= 1/distshift
                        deltaE_corr = recen_corr/genen-1.
                        if deltaE>-.95 and deltaE<-0.1:
                            histos[hn[2].format(ireg)].Fill(deltaE)
                            histos[hn[3].format(ireg)].Fill(deltaE_corr)
                    histos[hn[6].format(ireg)].Fill(geneta, deltaE)
                    histos[hn[7].format(ireg)].Fill(geneta, deltaE_corr)
    #end of tree loop
    fIn.Close()

    if FLAGS.mode == 1:
        pcoords = [[[0.01,0.755,0.33,0.995],
                    [0.34,0.755,0.66,0.995],
                    [0.67,0.755,0.99,0.995],
                    [0.01,0.505,0.33,0.745],
                    [0.34,0.505,0.66,0.745],
                    [0.67,0.505,0.99,0.745],
                    [0.01,0.255,0.33,0.495],
                    [0.34,0.255,0.66,0.495],
                    [0.67,0.255,0.99,0.495],
                    [0.01,0.005,0.33,0.245],
                    [0.34,0.005,0.66,0.245],
                    [0.67,0.005,0.99,0.245]]]
        cdims = [[1600,2000]]
        picname = '1comp_'+FLAGS.samples+'_'+FLAGS.method
        if FLAGS.apply_weights:
            picname += '_corrected'
    else:
        pcoords = [[[0.01,0.51,0.33,0.99],
                    [0.34,0.51,0.66,0.99],
                    [0.67,0.51,0.99,0.99],
                    [0.01,0.01,0.33,0.49],
                    [0.34,0.01,0.66,0.49],
                    [0.67,0.01,0.99,0.49]]]
        cdims = [[1000,600]]
        picname = '2comp_'+FLAGS.samples+'_'+FLAGS.method
        if FLAGS.apply_weights:
            picname += '_corrected'

    correct_order = []
    for i in range(len(hn)):
        for ireg in range(1,NREG+1):
            correct_order.append(hn[i].format(ireg))
    assert len(correct_order) == len(histos.keys())
    histos = [histos[correct_order[i]] for i in range(len(correct_order))]

    if FLAGS.mode == 1:
        histos.append(histos[6].Clone())
        histos.append(histos[7].Clone())
        histos.append(histos[8].Clone())
        histos[-3].Divide(histos[9])
        histos[-2].Divide(histos[10])
        histos[-1].Divide(histos[11])

        htmp = []
        for ireg in range(NREG):
            h = histos[3+ireg]
            xbins, exbins, rms, erms, bias, ebias = ([] for _ in range(6))
            for xbin in xrange(1,h.GetNbinsX()+1):
                tmp = h.ProjectionY('tmp', xbin, xbin)
                xbins.append(h.GetXaxis().GetBinCenter(xbin))
                horizerror = ( h.GetXaxis().GetBinCenter(1) -
                               h.GetXaxis().GetBinLowEdge(1) )
                exbins.append( horizerror )
                rms.append(tmp.GetRMS())
                erms.append(tmp.GetRMSError())
                """
                xq = Carray('d', [0.16,0.5,0.84])
                yq = Carray('d', [0.0,0.0,0.0 ])
                tmp.GetQuantiles(3,yq,xq)
                bias.append(yq[1])
                """
                bias.append(tmp.GetMean())
                #ebias.append((yq[0]+yq[2])/2)
                ebias.append(tmp.GetMeanError())
                tmp.Delete()
            xbins, exbins = np.array(xbins), np.array(exbins)
            rms, erms = np.array(rms), np.array(erms)
            bias, ebias = np.array(bias), np.array(ebias)
            indep = rms/(1.+bias)
            eindep = indep * np.sqrt( erms**2/rms**2 + ebias**2/(1+bias**2)  )
            htmp.append( TGraphErrors(etabins, xbins, rms, exbins, erms) )
            htmp.append( TGraphErrors(etabins, xbins, bias, exbins, ebias) )
            htmp.append( TGraphErrors(etabins, xbins, indep, exbins, eindep) )

        ht_tmp = [htmp[0],htmp[3],htmp[6],htmp[1],htmp[4],htmp[7],htmp[2],htmp[5],htmp[8]]
        histos = histos[:6] + ht_tmp
        if FLAGS.method == 'fineeta':
            htitles = ['rmsVSeta1',  'rmsVSeta2',  'rmsVSeta3',
                       'biasVSeta1', 'biasVSeta2', 'biasVSeta3',
                       'indepVSeta1','indepVSeta2','indepVSeta3']
            indices = [1, 2, 3] * 3
            indices.sort()
            fOut = TFile( base.paths.plots, 'RECREATE' )
            fOut.cd()
            for ih,h in enumerate(ht_tmp):
                h.SetName(htitles[ih])
                h.Write(htitles[ih])
            fOut.Write()
            fOut.Close()

        plotHistograms(histos, cdims, pcoords,
                       os.path.join(FLAGS.outpath,picname+'.png'))

    elif FLAGS.mode == 2:
        if FLAGS.apply_weights:
            fOut = TFile( base.paths.plots, 'RECREATE' )
            fOut.cd()
            for ireg in range(NREG):
                str1 = hn[4].format(ireg+1)
                str2 = hn[5].format(ireg+1)
                histos[12+ireg] = histos[ireg].Clone(str1)
                histos[15+ireg] = histos[3+ireg].Clone(str2)
                histos[12+ireg].Add(histos[6+ireg])
                histos[15+ireg].Add(histos[9+ireg])
            for h in histos:
                h.Write()
            histos_complete = histos[:6]
            histos_incomplete = histos[6:12]
            histos_total = histos[12:18]
            histos_res2D_before = histos[21:24]
            histos_res2D_after = histos[21:24]
            plotHistograms(histos_complete, cdims, pcoords,
                           os.path.join(FLAGS.outpath,picname+'_complete.png'))
            plotHistograms(histos_incomplete, cdims, pcoords,
                           os.path.join(FLAGS.outpath,picname+'_incomplete.png'))

            ht = histos_total[3:] + histos_res2D_after #res 1D + res 2D
            ht_tmp = []
            for ireg in range(NREG):
                h = ht[3+ireg]
                xbins, exbins, rms, erms, bias, ebias = ([] for _ in range(6))
                for xbin in xrange(1,h.GetNbinsX()+1):
                    tmp = h.ProjectionY('tmp', xbin, xbin)
                    xbins.append(h.GetXaxis().GetBinCenter(xbin))
                    horizerror = ( h.GetXaxis().GetBinCenter(1) -
                                   h.GetXaxis().GetBinLowEdge(1) )
                    exbins.append( horizerror )
                    rms.append(tmp.GetRMS())
                    erms.append(tmp.GetRMSError())
                    """
                    xq = Carray('d', [0.16,0.5,0.84])
                    yq = Carray('d', [0.0,0.0,0.0 ])
                    tmp.GetQuantiles(3,yq,xq)
                    bias.append(yq[1])
                    """
                    bias.append(tmp.GetMean())
                    #ebias.append((yq[0]+yq[2])/2)
                    ebias.append(tmp.GetMeanError())
                    tmp.Delete()
                xbins, exbins = np.array(xbins), np.array(exbins)
                rms, erms = np.array(rms), np.array(erms)
                bias, ebias = np.array(bias), np.array(ebias)
                indep = rms/(1.+bias)
                eindep = indep * np.sqrt( erms**2/rms**2 + ebias**2/(1+bias**2)  )
                ht_tmp.append( TGraphErrors(etabins, xbins, rms, exbins, erms) )
                ht_tmp.append( TGraphErrors(etabins, xbins, bias, exbins, ebias) )
                ht_tmp.append( TGraphErrors(etabins, xbins, indep, exbins, eindep) )

            fOut.cd()
            ht_tmp = [ht_tmp[-9],ht_tmp[-6],ht_tmp[-3],
                      ht_tmp[-8],ht_tmp[-5],ht_tmp[-2],
                      ht_tmp[-7],ht_tmp[-4],ht_tmp[-1]]
            ht_titles = ['rmsVSeta1',  'rmsVSeta2',  'rmsVSeta3',
                         'biasVSeta1', 'biasVSeta2', 'biasVSeta3',
                         'indepVSeta1','indepVSeta2','indepVSeta3']
            indices = [1, 2, 3] * 3
            indices.sort()
            for ih,h in enumerate(ht_tmp):
                h.SetName(ht_titles[ih])
                h.Write(ht_titles[ih])
            pcoords = [[[0.01,0.805,0.33,0.995],
                        [0.34,0.805,0.66,0.995],
                        [0.67,0.805,0.99,0.995],
                        [0.01,0.605,0.33,0.795],
                        [0.34,0.605,0.66,0.795],
                        [0.67,0.605,0.99,0.795],
                        [0.01,0.405,0.33,0.595],
                        [0.34,0.405,0.66,0.595],
                        [0.67,0.406,0.99,0.595],
                        [0.01,0.205,0.33,0.395],
                        [0.34,0.205,0.66,0.395],
                        [0.67,0.205,0.99,0.395],
                        [0.01,0.005,0.33,0.195],
                        [0.34,0.005,0.66,0.195],
                        [0.67,0.005,0.99,0.195]]]
            cdims = [[1600,2000]]
            ht += ht_tmp
            plotHistograms(ht, cdims, pcoords,
                           os.path.join(FLAGS.outpath,picname+'_total.png'))
            fOut.Write()
            fOut.Close()

        else:
            histos = histos[24:]
            plotHistograms(histos, cdims, pcoords,
                           os.path.join(FLAGS.outpath,picname+'.png'))

if __name__ == "__main__":
    parser = Argparser.Argparser()
    FLAGS = parser.get_flags()
    parser.print_args()
    if FLAGS.apply_weights and FLAGS.mode != 2:
        raise ValueError('The weights can only be used when mode==2.')
    base = PartialWafersStudies(FLAGS)
    NREG, NLAYERS, A = base.nsr, base.nlayers, base.sr_area
    etaregions = base.etaregions
    if FLAGS.method == 'ed':
        bckgcuts = np.array(FLAGS.bckgcuts)
    main()


import requests
from bs4 import BeautifulSoup
import re
from googlesearch import search

def remove_tags(text):
    TAG_RE = re.compile(r'<[^>]+>')
    return TAG_RE.sub('', text)

def spamcalls(num):

    lists = []

    r = requests.get("https://spamcalls.net/en/search?q={}".format(num))
    if r.status_code == 200:
        try:
            parse = BeautifulSoup(r.content.decode('utf-8'), 'html.parser')
            name = parse.findAll('strong')
            for names in name:
                lists.append(str(names))
            lists.pop(0)
            return{"spamcalls" :  remove_tags(", ".join(lists))}
        except Exception as e:
            return{'err' : e}

def scamcallfighters(num):

    r = requests.get("http://www.scamcallfighters.com/search-phone-{}.html".format(num.replace("+", "")))
    if r.status_code == 200:
        try:
            parse = BeautifulSoup(r.content.decode('utf-8'), 'html.parser')
            for g in parse.find_all('div', class_='nrp_headmat1'):
		            records = g.find_all('p')
            return{"scamcallfighters" : remove_tags(str(records))}
        except Exception as e:
            return{'err' : e}

def urls(num, countrycode, localnumber):

    return{"fouroneone": "https://www.411.com/phone/{}".format(num.replace('+', '').replace(' ', '-')), "truecaller": "https://www.truecaller.com/{}/{}".format(countrycode, localnumber), 'truepeoplesearch': "https://www.truepeoplesearch.com/results?phoneno={}".format(num.replace(' ', '')), 'syncme': "https://sync.me/search/?number={}".format(num.replace("+", ""))}
    try:
        for r in search(num):
            return{"URL": r}
    except:
        return{"err" : "error occured"}


from django.urls import path

from . import views

app_name = 'assure'
urlpatterns = [
    path('', views.IndexView.as_view(), name='index'),
    path('<int:pk>/detail/', views.DetailView.as_view(), name='detail'),
    path('<int:pk>/results/', views.ResultsView.as_view(), name='results'),
    path('<int:site_id>/comment/', views.comment, name='comment'),
]