import json
import os
import sys
import argparse
import glob
from collections import defaultdict
import shutil
import pickle
import pandas as pd
import matplotlib.pyplot as plt
import utils
import numpy as np
import matplotlib.cm as cm
import traceback
import seaborn as sns
import scipy as sp
import math
import multiprocessing as mp   # multithread processing
from functools import partial  # partial evaluation of a function
from matplotlib.cm import ScalarMappable
# plt.rcParams['text.usetex'] = True
import matplotlib as mpl
import pdb

class ForkedPdb(pdb.Pdb):
    """A Pdb subclass that may be used
    from a forked multiprocessing child

    """
    def interaction(self, *args, **kwargs):
        _stdin = sys.stdin
        try:
            sys.stdin = open('/dev/stdin')
            pdb.Pdb.interaction(self, *args, **kwargs)
        finally:
            sys.stdin = _stdin

# mpl.use("pgf")
mpl.rcParams.update(
    {
        # Adjust to your LaTex-Engine
        "pgf.texsystem": "pdflatex",
        "font.family": "serif",
        "text.usetex": True,
        "pgf.rcfonts": False,
        "axes.unicode_minus": False,
        "lines.linewidth":   5,
        "lines.markersize": 14,
        "savefig.bbox":"tight",
    }
)
# optional font setting
font = {'family' : 'serif',
        'weight' : 'normal',
        'size'   : 24}
mpl.rc('font', **font)
mpl.rcParams['figure.figsize'] = (6, 6)  # Use the values from \printsizes

np.seterr(divide="ignore")  # ignore the log of -1

def set_default(obj):
    if isinstance(obj, set):
        return list(obj)
    raise TypeError

def main(args):
    if args.diff:
        diff(args)
    elif args.compute:
        compute2(args)
    elif args.compare is not None or args.plot or args.contour:
        plot(args)
    else:
        export(args)

def diff(args):

    lst_file = []
    exclude = ["name", "dataset_fname", "config", "time"]
    for dr in args.dirs:

        # lst_file.extend(glob.glob(os.path.join(dr, "**", "config.json"), recursive=True))  # all config files)
        lst_file = glob.glob(os.path.join(dr, "**", "config.json"), recursive=True)  # all config files)

        if len(lst_file) < 2:  # do not process if less than 2 different configurations
            continue

        root = os.path.commonpath(lst_file)
        diff_dict = cmpt_dict(lst_file, exclude)# for k,v in merge_dct.items():
            # if len(v) == 1:
                # merge_dct.pop(k)

        with open(os.path.join(root, "filter.json"), 'w') as fp:
            json.dump(diff_dict, fp, indent=0, sort_keys=True, default=set_default)

    return

def cmpt_dict(lst_file, exclude=["name", "dataset_fname", "config"]):

    merge_dct = defaultdict(set)
    for f in lst_file:

        with open(f) as fconf:
            data = json.load(fconf)
            if not merge_dct:  # first time
                for k,v in data.items():
                    if isinstance(v, (list, dict)):
                        continue
                    merge_dct[k] |= set([v])   # convert the values to set
            else:
                for k,v in data.items():
                    if isinstance(v, (list, dict)):
                        continue
                    if k not in merge_dct.keys():
                        print("Warning! ", k, " not in the keys")
                    merge_dct[k] |= set([v])

    diff_dct = {k :v for k,v in merge_dct.items() if len(v) > 1 and not k in exclude}
    return diff_dct

def export(args):

    with open(args.filter) as fconf:
        conf = json.load(fconf)
    dname, name = os.path.split(os.path.splitext(args.filter)[0])

    lst_file = (glob.glob(os.path.join(dname, "**", "config.json"), recursive=True))  # all config files)

    outroot = os.path.join(os.path.dirname(args.filter), "merge", "export")
    pnames = ["loss", "ranks"]
    outdir = os.path.join(outroot, name)
    if args.reset:
        shutil.rmtree(outdir, ignore_errors=True)
    for f in lst_file:

        with open(f) as fconf:
            data = json.load(fconf)
        if not all([data[k] in conf[k] for k in conf.keys()]):
            continue
        root = os.path.dirname(f)
        namestr = to_string(sorted(conf.keys()), data)
        for pname in pnames:
            try:
                src = os.path.join(root, "plot", f"{pname}.pdf")
                # outdir = os.path.join(outroot, name, pname)
                dst = os.path.join(outdir, pname, f"{namestr}.pdf")
                os.makedirs(os.path.dirname(dst), exist_ok=True)
                shutil.copyfile(src, dst)
            except FileNotFoundError:
                traceback.print_exc()

def compute(args):

    EPS = 4
    dname, name = os.path.split(os.path.splitext(args.filter)[0])
    lst_file = (glob.glob(os.path.join(dname, "**", "config.json"), recursive=True))  # all config files)

    with open(args.filter) as fconf:
        conf = json.load(fconf)

    outdir = os.path.join(dname, "export", "compute")
    os.makedirs(outdir, exist_ok=True)

    for fname in lst_file:

        dfname = os.path.join(os.path.dirname(fname), "dfs.pkl")
        try:
            with open(dfname, "rb") as _f:
                dfs = pickle.load(_f)
            with open(fname) as fconf:
                fargs = json.load(fconf)
        except:
            print("can't open the dataframe at ", fname)
            continue

        key = tuple([fargs[k] for k in conf.keys()])
        outname = df_to_string(key, conf.keys())
        # outfold = os.path.join(outdir, outname)
        os.makedirs(outdir, exist_ok=True)
        df = dfs["quant"].dropna(axis=1, how='all')
        lr = fargs["learning_rate"]
        if not 'time' in df.columns:
            df["time"] = df.index * lr   # assume a constant learning rate

        diff = df["diff loss to OPT"]
        logdiff = np.log10(diff)
        wass = df["wass. dist to OPT"]  # criterion for convergence
        if not (wass < EPS).any():
            print("theshold not crossed in ", fname)
            continue  # leave the slope at NaN, not converged
        # df.loc[:, "slope"] = float('nan')
        logwass = np.log10(wass)  # the log10 of wass
        s = int(0.01 / lr + 0.5)  # sample in one per centisecond
        logwassewm = logwass.ewm(span=s).mean()[wass.index[::s]]  # subsample the signal after smoothing it
        dlogwass = (logwassewm.shift(-1) - logwassewm.shift(1)) /2

        dabs = dlogwass.abs()

        d2abs = (dabs.shift(-1) - dabs.shift(1)) / 2
        d2abs0 = d2abs.copy()
        d2abs0[d2abs.abs() < d2abs.abs().max()/10] = 0

        posabs = d2abs0 > 0
        negabs = d2abs0 < 0
        nullabs = d2abs0 == 0

        begidx = np.where(nullabs & posabs.shift(-1))[0]
        endidx = np.where(negabs & nullabs.shift(-1))[0]

        begpts = d2abs.index[begidx]
        endpts = d2abs.index[endidx]

        th = dabs.max() * 2/3
        mask = dabs > th
        components = np.where(mask)[0]
        brkpts = np.where(np.diff(components) != 1)[0]  # the breakpoints at which the connected components end
        aas = [components[0]] + components[brkpts + 1].tolist()
        bbs = components[brkpts].tolist() + [components[-1]]  # add the last point
        # idx = components[0]  # first index where the threshold is reached
        # intervals = []  # list of possible intervals for the linear regression

        # ptidx = components[brkpts]  # the breakpoints in the mask frame
        # frameidx = dabs.index[ptidx]  # the breakpoints in the original frame
        aaframe = dabs.index[aas]
        bbframe = dabs.index[bbs]
        aaarr = np.array(aaframe)  # numpy conversion
        bbarr = np.array(bbframe)
        begarr = np.array(begpts)
        endarr = np.array(endpts)
        # test all the different lengths
        testbeg = (begarr.reshape(1, -1, 1) - aaarr.reshape(-1, 1, 1)) >= 0
        testend = (bbarr.reshape(-1, 1, 1) - endarr.reshape(1, 1, -1)) >= 0
        testBE = ((endarr.reshape(1, 1, -1) - begarr.reshape(1, -1, 1) > 0)  |
                  (bbarr.reshape(-1, 1, 1) - begarr.reshape(1, -1, 1) > 0) |
                  (endarr.reshape(1, 1, -1) - aaarr.reshape(-1, 1, 1) > 0))

        # test

        allowed = np.where((testbeg  | testend ) & (testBE))  # requires the correct interval orientation

        if len(allowed[0]) == 0:  # we did not find
            print("can't find a slope in ", fname)
            continue

        maxidx_pts = allowed[0].max()  # the index of the last correct interval in allowed frame
        maxidx_beg = allowed[1].max()
        maxidx_end = allowed[2].max()

        # inteval_idx = (allowed[1][maxidx], allowed[2][maxidx])

        interval = max(aaarr[maxidx_pts], begarr[maxidx_beg]), bbarr[maxidx_pts]

        if interval[0] >= interval[1]:
            print("Wrong interval value in ", fname)
            continue

        x = logdiff.index[interval[0]:interval[1]]
        y = logdiff[interval[0]:interval[1]]
        mask = np.isfinite(y)  # only consider finite data
        regress = sp.stats.linregress(x[mask]*lr, y[mask]) # slope in s^-1

        plt.figure()
        ax = logdiff.plot()
        ax.plot(x, regress.intercept + regress.slope * x, 'r')
        outstat = os.path.join(outdir, "regress")
        os.makedirs(outstat, exist_ok=True)
        plt.savefig(os.path.join(outstat, f"{outname}.pdf"))
        plt.close("all")

        computed = {'regress': regress, 'interval': interval}
        savefname = os.path.join(os.path.dirname(fname), "regress.pkl")
        with open(savefname, 'wb') as fp:
            pickle.dump(computed, fp)

def check_params(conf, fname):
    """Checks if a json file has allowed parameter compared to a config file"""

    with open(fname, 'r') as fp:

        fconf = json.load(fp)

    return all([fconf[k] in conf[k] for k in conf.keys()])

def compute2(args):

    EPS = 1e-4
    if args.filter is not None:
        dname, name = os.path.split(os.path.splitext(args.filter)[0])
        lst_file = (glob.glob(os.path.join(dname, "**", "config.json"), recursive=True))  # all config files)
    # lst_regress = glob.glob(os.path.join(dname, "**", "regress.pkl"), recursive=True)

        with open(args.filter) as fconf:
            conf = json.load(fconf)

        lst_file = [l for l in lst_file if check_params(conf, l)]

    else:
        conf = None
        lst_file = []
        lst_lst = [glob.glob(os.path.join(dir_, "**", "config.json"), recursive=True) for dir_ in args.dirs]
        for sublst in lst_lst:
            lst_file.extend(sublst)

    if args.filter is not None:
        dname, name = os.path.split(os.path.splitext(args.filter)[0])
        outdir = os.path.join(dname, "merge", name, "compute") if name != "filter" else os.path.join(dname, "merge", "compute")
    else:
        if len(lst_file) > 1:  # more than one file to process
            dname = os.path.commonpath(lst_file)
            outdir = os.path.join(dname, "merge", "compute")
            conf = cmpt_dict(lst_file)
        else:
            dname = os.path.dirname(lst_file[0])
            outdir = os.path.join(dname, "compute")

    os.makedirs(outdir, exist_ok=True)
    failed_lst = []  # list for all the failed processings
    failed_fname = os.path.join(outdir, "failed.txt")
    partial_cpt = partial(compute_fname, conf=conf, outdir=outdir)
    # if os.path.isfile(f):
        # os.remove(FAILED)
    with mp.Pool(processes=args.nprocs) as pool:
        pool.map(partial_cpt, lst_file)

    # processes = [Process(target=compute_fname, args=(fname,conf,outdir)) for fname in lst_file]

    # for prc in processes:
        # prc.start()

    # for prc in processes:
        # prc.join()

# failed_lst.sort()
# with open(failed_fname, 'w') as _f:
    # _f.writelines("\n".join(failed_lst))

def compute_fname(fname, conf, outdir):

    dfname = os.path.join(os.path.dirname(fname), "dfs.pkl")
    try:
        with open(dfname, "rb") as _f:
            dfs = pickle.load(_f)
        with open(fname) as fconf:
            fargs = json.load(fconf)
    except:
        print("can't open the dataframe at ", fname)
        # failed_lst.append(fname)
        # continue
        return

    savefname = os.path.join(os.path.dirname(fname), "regress.pkl")
    if os.path.isfile(savefname) and not args.force:
        # continue
        return
    if conf is not None:
        key = tuple([fargs[k] for k in conf.keys()])
        outname = df_to_string(key, conf.keys())
    else:
        outname = "debug"
    # outfold = os.path.join(outdir, outname)
    # os.makedirs(outdir, exist_ok=True)
    df = dfs["quant"].dropna(axis=1, how='all')
    lr = fargs["learning_rate"]
    if not 'time' in df.columns:
        df["time"] = df.index * lr   # assume a constant learning rate

    WIN_SIZE = int(0.1  / lr + 0.5)  # window size over which we compute the linear regression in samples
    diff = df["diff loss to OPT"]
    logdiff = np.log10(diff)
    wass = df["wass. dist to OPT"]  # criterion for convergence
    # if not (wass < EPS).any():
        # print("theshold not crossed in ", fname)
        # continue  # leave the slope at NaN, not converged
    # df.loc[:, "slope"] = float('nan')
    logwass = np.log10(wass)  # the log10 of wass
    logwassfin = logwass[np.isfinite(logwass)]
    logdifffin = logdiff[np.isfinite(logdiff)]
    s = int(1e-4 / lr + 0.5)  # sample in 1 per millisecond
    logwassewm = logwassfin.ewm(span=s).mean()[logwassfin.index[::s]]  # subsample the signal after smoothing it
    logdiffewm = logdifffin.ewm(span=s).mean()[logdifffin.index[::s]]  # subsample the signal after smoothing it
    diffewm = diff.ewm(span=s).mean()[diff.index[::s]]

    mask = (logdiffewm < -2) & (logdiffewm > -3.5)  #
    # mask = (diffewm < 1e-2) & (diffewm > 1e-4 )  #
    # mask = (logdiffewm > -3.5) #

    # brkpts = np.where(np.diff(np.where(mask)[0]) != 1)[0]
    windows_idx = np.where(np.diff(mask))[0]
    if mask.iloc[0]:
        windows_idx = np.concatenate(([0], windows_idx), axis=0)
    if mask.iloc[-1]:
        windows_idx = np.concatenate((windows_idx, [len(mask)-1]), axis=0)
    window_idx = None

    if len(windows_idx) == 0 or len(windows_idx)==1:
        print("Signal not found at ", fname)
        # failed_lst.append(fname)
        # with open(FAILED, 'a') as _f:
            # _f.write(fname, '\n')
        # continue
        return

    windows_size = np.diff(windows_idx)[::2]  # every two are the lengths of the True regions
    win_max = 2*np.argmax(windows_size)
    window_idx = ( windows_idx[win_max],  windows_idx[win_max+1] )

    # interval = mask.index[window_idx[0]], mask.index[min(len(mask)-1, window_idx[1]+1)]
    interval = max(0, mask.index[window_idx[1]] - WIN_SIZE), mask.index[window_idx[1]]
    # assert all(mask.loc[interval[0]:interval[1]])

    x = logdiff.index[interval[0]:interval[1]]
    y = logdiff[interval[0]:interval[1]]
    # x = diff.index[interval[0]:interval[1]]
    # y = diff[interval[0]:interval[1]]
    mask = np.isfinite(y)
    regress = sp.stats.linregress(x[mask]*lr, y[mask]) # slope in s^-1
    # regress = sp.stats.linregress(x*lr, y) # slope in s^-1
    lograte = np.log(diff[interval[1]-1] / diff[interval[0]]) / (lr*(diff.index[interval[1]] - diff.index[interval[0]]))

    plt.figure()
    ax = logdiff.plot()
    logdiffewm.plot(ax=ax)
    ax.plot(x, regress.intercept + regress.slope*lr * x, 'r')
    outstat = os.path.join(outdir, "regress")
    os.makedirs(outstat, exist_ok=True)
    plt.savefig(os.path.join(outstat, f"{outname}.pdf"))
    plt.close("all")

    computed = {'regress': regress, 'interval': interval, 'lograte': lograte}
    with open(savefname, 'wb') as fp:
        pickle.dump(computed, fp)

def append_df(df_lst, keyval, args, xaxis, yaxis, orderkey):
    # f = fnames[key]
    key, f = keyval
    dfname = os.path.join(os.path.dirname(f), "dfs.pkl")
    try:
        with open(dfname, "rb") as _f:
            dfs = pickle.load(_f)
        with open(f, 'r') as _f:  # the configuration of the run
            fargs = json.load(_f)

    except:
        return

    # ForkedPdb().set_trace()
    if yaxis in ("slope", "lograte") or args.contour in ("slope", "r", "lograte"):
        regressfname = os.path.join(os.path.dirname(f), "regress.pkl")
        try:
            with open(regressfname, "rb") as _f:
                data = pickle.load(_f)
                regress = data["regress"]
                # lograte = data["lograte"]
        except:
            return

    df = dfs["quant"].dropna(axis=1, how='all')
    # if we want to compute additional quantities
    # compute the slope if it as converged
    if xaxis != df.index.name:  #means the xaxis is a hyperparameter
        df = df.tail(1)  # only keep the last one,
    if yaxis in "slope" or args.contour in ("slope", "r"):
        df["slope"] = regress.slope
        df["r"] = r2 = regress.rvalue**2
        df["Pred_C"] = dfs["vals"]["Prec_C"]
    # elif yaxis == "lograte" or args.contour in ("lograte"):  # lograte not
    # working
        # df["lograte"] = lograte
    if yaxis == "smin":
        try:
            vals = dfs["vals"]
            if all([k in vals.keys() for k in ["smin", "smin*", "smin**"]]):
                smin, sminstar, smin2star = vals["smin"], vals["smin*"], vals["smin**"]
            else:  # compute the value of smin based on the trainset
                ds =fargs["dataset_fname"]
                sets, _ = utils.load_dataset(fname=ds)  #
                if sets:
                    trainset = sets["train"]
                    # args.load_dataset = fname_ds
                    # STD = trainset.std
                    # Sigma0 = STD.mm(STD.t())  # covariance of the target distribution
                    dmin = min(fargs["xdim"], fargs["zdim"], fargs["width"])
                    r = min(fargs["xdim"], fargs["zdim"])
                    lambdas = trainset.Lambda
                    smin2star = math.sqrt(lambdas[dmin-1])
                    sminstar = math.sqrt(lambdas[r-1])
                    smin = math.sqrt(lambdas[-1])
                else:
                    raise IOError("Can't open the trainset for read")
            df["smin**"] = smin2star
            df["smin*"] = sminstar
            df["smin"] = smin
        except:
            traceback.print_exc()

    # index = index.union(df.index)
    # index.name = "niter"
    if args.contour and yaxis == "slope":
        tpls = [(*key, regress.slope, s) for s in df.columns]
    # elif args.contour == "slope" and yaxis == "smin":
        # tpls = [(*key, s) for s in df.columns]
    else:
        tpls = [(*key, s) for s in df.columns]
    columns = pd.MultiIndex.from_tuples(tpls,names=orderkey + ["stat"])
    df.columns = columns
    # columns = columns.union(df.columns)
    # df_merge = df_merge.reindex(index)
    df_lst.append(df)
    # if df_merge.empty:  # first time in the loop
        # df_merge = df
    # else:
        # df_merge = pd.concat([df_merge, df], axis=1)# .loc[:, key] = df

def plot(args):  # assume the compare key is not None

    with open(args.filter) as fconf:
        conf = json.load(fconf)
    dname, name = os.path.split(os.path.splitext(args.filter)[0])

    yaxis = args.y  # it's a list, make it a loop?

    if args.y in ("slope", "lograte") or args.contour in ("slope", "r"):
        # regressfname = os.path.join(os.path.dirname(f), "regress.pkl")
        lst_file_regress = (glob.glob(os.path.join(dname, "**", "regress.pkl"), recursive=True))  # all config files)
        lst_file = [f.replace("regress.pkl", "config.json") for f in lst_file_regress]
    else:
        lst_file = (glob.glob(os.path.join(dname, "**", "config.json"), recursive=True))  # all config files)

    outroot = os.path.join(os.path.dirname(args.filter), "merge")
    pnames = ["loss", "ranks"]
    if name == "filter":
        name = ''  # remove the "filter" part if not changed
    outdir = os.path.join(outroot, name)

    compkey = args.compare
    xaxis = args.x
    keys = []
    noncomp = [k for k in sorted(conf.keys()) if k!=compkey and k!=xaxis and  k !=yaxis and len(conf[k]) > 1]
    if compkey:
        orderkey =[ compkey] +  noncomp
    else:
        orderkey = noncomp

    if args.contour:
        # yaxis = args.y[0]
        # if yaxis in conf.keys():
        orderkey = orderkey + [xaxis, yaxis]  # keep the contour the most enclosed
        # else:
            # orderkey = orderkey + [xaxis]  # will use the computed value as axis
    # if args.contour or xaxis != "niter":
    elif xaxis != "niter":
        orderkey = [xaxis] + orderkey

    fnames = dict()
    shared_conf = None
    for  f in lst_file:  # all possible experiemnets, will filter them based on conf

        with open(f) as fconf:
            data = json.load(fconf)
        try:
            if not all([data[k] in conf[k] for k in conf.keys()]):
                # if we don't have all the same parameters as the conf filter
                continue
        except:
            # print("missing key {}".format(k))
            continue

        key = tuple([data[k] for k in orderkey if k in data.keys() and (not k in conf.keys() or len(conf[k]) > 1)])
        if key in keys:
            print(f"Key {key} already in keys!")
        keys.append(key)
        fnames[key] = f
        shared_conf = {k:v for k,v in shared_conf.items() if k in data.keys() and v == data[k]} if shared_conf is not None else data

    # columns = pd.MultiIndex.from_tuples(keys, names = orderkey)
    columns = pd.Index([])  # empyt at first
    # index
    df_merge = pd.DataFrame(index=pd.Index([], name=xaxis), columns=columns)
    index = df_merge.index
    columns = df_merge.columns
    if args.contour:
        plotdir = os.path.join(outdir, "contour", f"{xaxis}-{yaxis}")
    elif compkey is not None:
        plotdir = os.path.join(outdir, "plot", xaxis, compkey)
    # elif args.heatmap:
        # plotdir = os.path.join(outdir, "heatmap", f"{xaxis}-{yaxis}")
    else:
        plotdir = os.path.join(outdir, "plot", f"{xaxis}")
    os.makedirs(plotdir, exist_ok=True)

    with open(os.path.join(plotdir, "shared_conf.json"), 'w') as fp:
        json.dump(shared_conf, fp, indent=0, sort_keys=True, default=set_default)

    if args.reset:
        shutil.rmtree(plotdir, ignore_errors=True)

    # for stat in args.y:  # each quantity after the other
    df_lst = []

    # for key in fnames.keys():  # for all the configurations, need to merge the dataframe

    partial_fn = partial(append_df, args=args, xaxis=xaxis, yaxis=yaxis, orderkey=orderkey)

    pool = mp.Pool(processes=args.nprocs)
    manager = mp.Manager()
    L = manager.list()
    [pool.apply_async(partial_fn, args=[L, (key,f)]) for (key,f) in fnames.items()]
    pool.close()
    pool.join()

    # ForkedPdb().set_trace()
    df_merge = pd.concat(L, axis=1)
    if noncomp:  # if there are other keys
        dfu = df_merge.unstack().unstack(level=noncomp)  # group the different results with
    else:
        # we don't need to compare a key
        dfu = df_merge.unstack().dropna(how='all').droplevel("niter").unstack()  # problem if there are no other keys! todo
        # remove the niter level, dangerous? not test on many cases
        # but otherwise if multiple niter for different experiments leads to
        # many NaNs,
        # Before, the niter are the columns? and along the x is the compare key

    dfu = dfu.dropna(how='all', axis=0)  # test ?
    totstats = df_merge.columns.get_level_values("stat").unique()
    userdef = args.contour if args.contour else args.y
    stats =  totstats if userdef is None else  [userdef]
    # if args.contour:

    for stat in stats:  # for all the stats we want to plot
        # dfu = df_merge.unstack()
        if "stat" in dfu.index.names:
            df_filt = dfu.loc[dfu.index.get_level_values("stat") == stat,: ].droplevel("stat")
        else:  # in the collumns, only the case when noncomp is False?
            df_filt = dfu.loc[:, dfu.columns.get_level_values("stat") == stat]
            df_filt = df_filt.stack().droplevel("stat")

        def process_contour(df_filt, dfu, stat, args=args):

            iterkeys = noncomp or df_filt.isna().any().any()  #  iterate over the keys
            if iterkeys:  # if we have some missing values
                keys = df_filt.keys()  # we take all remaining keys
            else:
                keys = ['']

            fill = stat == "r"
            for key in keys:  # each of the configurations
                keyname = ', '.join(dfu.keys().names)
                string = df_to_string(key, dfu.keys().names) if iterkeys else str(key) # the name of the file
                df_plot = df_filt[key] if iterkeys else df_filt
                df_plot = df_plot.dropna()
                if "niter" in df_plot.index.names:  # if we have several niter per configuration
                    all_xy = df_plot.index.droplevel("niter")
                    if len(all_xy) == len(all_xy.unique()):
                        df_plot = df_plot.droplevel('niter')
                    else:
                        pass # todo: take the last niter for each configuration

                if df_plot.empty:
                    continue

                if iterkeys:
                    df_plot = df_plot.unstack(level=xaxis)
                plotfolder = os.path.join(plotdir, string)
                # niterstr = key if df_filt.columns.name == 'niter' else ''
                if isinstance(df_plot, pd.DataFrame) and df_plot.columns.name == 'niter':
                    niter = df_plot.columns[-1]  # take the last iteration for plotting
                    df_plot = df_plot[niter]
                    df_plot = df_plot.unstack(0)  # put it in a 2d array
                elif isinstance(df_plot, pd.Series):
                    df_plot = df_plot.unstack(level=xaxis)  # put it in a 2darray
                    # niterstr = f"iter-{niter}"
                # plt.contourf(df_plot.reindex(df_plot.inde
                # ax = sns.contour(df_plot.reindex(df_plot.index.sort_values(ascending=False)))  # flip the y values
                xx = df_plot.columns
                yy = df_plot.index
                X, Y = np.meshgrid(xx, yy)
                # vmin = int(df_plot.min().min() + 0.5)
                vmin = df_plot.min().min()
                # vmax = max(vmin+1, int(df_plot.max().max() + 0.5))
                vmax = df_plot.max().max()
                if vmin == vmax:
                    continue
                levels = 40
                level_boundaries = np.linspace(vmin, vmax, levels+1)
                fig, ax = plt.subplots(figsize=(7, 6))
                if fill:
                    ctr = ax.contourf(X, Y, df_plot, level_boundaries, vmin=vmin, vmax=vmax)  # flip the y values
                else:
                    ctr = ax.contour(X, Y, df_plot, level_boundaries, vmin=vmin, vmax=vmax)  # flip the y values
                os.makedirs(plotfolder, exist_ok=True)
                plotnamepgf = f"{stat}.pgf"
                plotname = f"{stat}.pdf"
                plotnametex = f"{stat}.tex"
                # plt.title(stat + niterstr )

                keyval = str(key) if not isinstance(key, tuple) else ', '.join([str(k) for k in key])
                ctr.axes.set_xlabel(xx.name)
                ctr.axes.set_ylabel("$\lambda_{\\mathrm{min}}(\Sigma_0^{1/2})$")
                ctr.axes.figure.colorbar(ScalarMappable(norm=ctr.norm, cmap=ctr.cmap),
                                         ticks = np.linspace(vmin, vmax, num=5),
                                         boundaries=level_boundaries,
                                         values= (level_boundaries[:-1] + level_boundaries[1:])/2,
                                         ax=ctr.axes
                                         )
                # ctr.axes.figure.suptitle(f"{stat} for {keyname} = {keyval}")
                plt.savefig(os.path.join(plotfolder,  plotname), bbox_inches="tight")
                plt.savefig(os.path.join(plotfolder,  plotnamepgf))
                plt.close("all")

        def process_keys(noncomp, df_filt, dfu, stat):
            if noncomp or not noncomp and compkey is None:  # we have keys
                keys = df_filt.keys()
            elif compkey is not None:  # only compkey to vary on
                keys = [compkey]

            for key in keys:
                string = df_to_string(key, dfu.keys().names) if noncomp else str(key) # the name of the file
                df_plot = df_filt[key].dropna(how='all', axis=0) if noncomp else df_filt

                niter = None
                several_niter = False
                if xaxis != "niter":
                    if "niter" in df_plot.index.names:
                        all_keys =  df_plot.index.droplevel("niter")
                        if len(all_keys) == len(all_keys.unique()):  # one iter per configuration

                            niter = df_plot.index.get_level_values("niter")  # keep for reference?
                            df_plot = df_plot.droplevel("niter")
                        # else:
                            # several_niter = True

                        else:
                            # niter = niter[-1]
                            print("Several niter found for ", key)
                            # if compkey is not None and compkey != "niter":
                            if compkey is not None and compkey != "niter":
                                niter = niter[-1]  # take the last niter available
                                df_plot = df_plot.loc[df_plot.index.get_level_values("niter") == niter, :]
                            else:
                                several_niter = True
                            # else:
                                # compkey = "niter"
                                # df_plot = df_plot.unstack(level="niter")

                        # niterstr = ", niter = " + str(niter)
                    else:
                        niter = key
                    if not isinstance(niter, pd.Index):
                        niterstr = ", niter = " + str(niter)
                    else:
                        niterstr = ""

                else:
                    niterstr = ""
                if compkey is not None:
                    df_plot = df_plot.unstack(level=compkey)
                if compkey is None and several_niter:  # we comp on the niter
                    df_plot = df_plot.unstack(level="niter")
                # df_piv = df_filt.unstack().reset_index()#.dropna(axis=0, how="any")
                # dfu = df_piv.pivot(index=xaxis, columns=compkey, values=0)
            # for key in dfu.keys():  # for each configuration
                # df_plot = dfu.loc[:, key].to_frame(name=stat)
                if df_plot.empty:
                    continue
                kwargs = {}
                if stat == "rank W":
                    kwargs["alpha"] = 0.3
                if stat in ("slope", "lograte"):
                    compute_regress = True
                    kwargs["style"] = '+'
                    kwargs["zorder"] = 2  # in front of the regression line

                # if stat in ("rank W", "norm grad"):
                    # kwargs["style"] = "."
                # ax = df_plot.unstack(level=compkey).plot(y=stat, **kwargs)
                if not args.contour:
                    ax = df_plot.plot(**kwargs)
                    if stat == args.y and stat in ("slope", "lograte"):  # plot the slope
                        # ax.set_xlabel(None)
                        # leg = ax.get_legend()
                        # leg_labels = leg.get_texts()
                        handles, labels = ax.get_legend_handles_labels()
                        if isinstance(df_plot, pd.DataFrame) and len(df_plot.columns) > 1:
                            for idx, ckey in enumerate(df_plot.columns):  # the total compare keys available
                                reg = sp.stats.linregress(df_plot[ckey].index, df_plot[ckey])
                                linplot = ax.plot(df_plot[ckey].index, reg.slope * df_plot[ckey].index + reg.intercept, color=ax.lines[idx].get_color(), linestyle='dashed')
                                r2 = reg.rvalue**2
                                labels.append(f"$a={reg.slope:.2f}$, $r^2$={r2:.3f}")
                                handles.append(linplot[0])

                        else:  # only one linear regression to perform
                            if isinstance(df_plot, pd.DataFrame):
                                reg = sp.stats.linregress(df_plot.index, df_plot.values[:, 0])
                            else:
                                reg = sp.stats.linregress(df_plot.index, df_plot.values)
                            linplot = ax.plot(df_plot.index, reg.slope * df_plot.index + reg.intercept, color='grey', linestyle='dashed', alpha=0.9, zorder=0)
                            r2 = reg.rvalue**2
                            labels.append(f"$a={reg.slope:.2f}$, $r^2$={r2:.3f}")
                            handles.append(linplot[0])

                        p = len(handles) // 2
                        ax.legend(handles[p:], labels[p:], ncol=1)  # only plot the regression legend

                    # if stat == "slope":
                        # ax.set_ylabel("Rate of convergence")
                    # else:
                    ax.set_ylabel(stat)

                else:  # should never be the case?
                    ax = sns.contour(df_plot, **kwargs)

                if xaxis == "niter" and args.with_elevel and stat.find("loss A") != -1:
                    # we have to compute the target singularvalues for the
                    # betroffene files
                    # take a tuple of the current key
                    tkey = (key if isinstance(key, tuple) else (key, )) if noncomp else ()
                    namekeys = [tuple([k, *tkey]) for k in df_plot.dropna(axis=1,how="all").columns]
                    fs = [fnames[k] for k in namekeys]
                    # key  is missing the compkey
                    fargss = []
                    for f in fs:
                        with open(f) as fconf:
                            fargss.append(json.load(fconf))
                    dss = set( frgs["dataset_fname"] for frgs in fargss )
                    xdims = set( frgs["xdim"] for frgs in fargss )
                    if len(dss) == len(xdims) == 1:  # same dataset loaded
                        ds = dss.pop()
                        xdim = xdims.pop()
                        sets, _ = utils.load_dataset(fname=ds)  #
                        if sets:
                            trainset = sets["train"]
                            # args.load_dataset = fname_ds
                            STD = trainset.std
                            Sigma0 = STD.mm(STD.t())  # covariance of the target distribution
                            lambdas, U = np.linalg.eigh(Sigma0)
                            Es  = lambdas.cumsum()[::-1]  # the different energy levels, lambdas are <
                            x = np.linspace(0, 1, len(Es))
                            grbow = cm.get_cmap(plt.get_cmap("gist_rainbow"))(x)
                            [ax.axhline(E, color=grbow[i, :], alpha=0.5, linestyle='--', zorder=0) for i, E in enumerate(Es)]
                            xmax = ax.get_xlim()[1]
                            [ax.figure.text(xmax, E, i, transform=ax.transData, fontsize='x-small') for i, E in enumerate(Es,1) if i == 1 or i%5 == 0]
                    if stat == "loss A":
                        stat += " elevel"

                    # df_plot.plot(x="niter", y=stat)
                keyname = ', '.join(dfu.keys().names)
                keyval = str(key) if not isinstance(key, tuple) else ', '.join([str(k) for k in key])
                plotfolder = os.path.join(plotdir, stat)
                os.makedirs(plotfolder, exist_ok=True)
                # plt.suptitle(f"{keyname} = {keyval}")
                # plt.title(stat + niterstr)
                plt.savefig(os.path.join(plotfolder,  f"{string}.pgf"), bbox_inches="tight")
                plt.savefig(os.path.join(plotfolder,  f"{string}.pdf"))
                plt.close("all")
        # end of process keys
        # df_filt = df_merge.loc[:, df_merge.columns.get_level_values("stat") == stat]
        # the keys are either the different configurations or the niter
        # df_filt = dfu
        # df_filt.droplevel("stat")
        if args.contour:
            process_contour(df_filt, dfu, stat)
        else:
            process_keys(noncomp, df_filt, dfu, stat)

        # else:  # if there are no other keys as the comparison key
            # # dfu should have the iterations as columns and the compkey as
            # # xaxis
            # process_keys([compkey], df_filt, dfu, stat)

    return

def df_to_string(key, names):

    lst = []
    if not isinstance(key, tuple):  # if there is ony one key
        key = (key,)

    for k, name in zip(key, names):
        n = name[0]
        if isinstance(k, bool):
            lst.append(f'{n}' if k else f'no-{n}')
        else:
            val = str(k)
            if isinstance(k, (int, float)):
                lst.append(f'{n}{val}')
            # elif isinstance(arg, dict):
                # dirname.append('-'.join([f'{k}-{v}' for k,v in arg.items()]))
            else:
                lst.append(f'{n}-{val}')
    # string = os.path.join(name, '-'.join(name))
    string = '-'.join(lst)
    return string

def to_string(keys, dct):

    name = []
    for  key in keys:
        arg = dct[key]
        k = key[0]
        if isinstance(arg, bool):
            name.append( f'{k}' if arg else f'no-{k}')
        else:
            val = str(arg)
            if isinstance(arg, (int, float)):
                name.append(f'{k}{val}')
            # elif isinstance(arg, dict):
                # dirname.append('-'.join([f'{k}-{v}' for k,v in arg.items()]))
            else:
                name.append(f'{k}-{val}')
    # string = os.path.join(name, '-'.join(name))
    string = '-'.join(name)
    return string

if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    parser.add_argument("-o", "--output", type=str, help="output the config diff between the different experiments")

    parser.add_argument("--nprocs","-n", type=int, default=1, help="the number of threads to span")
    parser.add_argument("--filter", help="json file use to filter")
    parser.add_argument("--diff", action="store_true", help="diff mode, write to file")
    parser.add_argument("--compare", type=str, help="the key for the pot")
    parser.add_argument("--plot", action="store_true", help="simply plots the data again")
    parser.add_argument("--reset", action="store_true", help="set to reset the output")
    parser.add_argument("--with-elevel", action="store_true", help="energy level plotting")
    parser.add_argument("-c", "--compute", action="store_true", help="compute additional properties on the data")
    parser.add_argument("--force", action="store_true", help="force new computations")
    parser.add_argument("-x", default="niter",  help="the x axis")
    parser.add_argument("-y", help="the quantities to plot as a function of x")
    parser.add_argument("-ctr", "--contour", choices=("slope", "r", "lograte"), help="plot a heat map of the value with x/y coordinates")
    parser.add_argument("dirs", nargs="*", help="the different directories to look for")

    args = parser.parse_args()
    main(args)


#! /usr/bin/env python3
# -*- coding: utf-8 -*-

import unittest
import uuid
import time
import redis
import inspect
r = redis.Redis(host='localhost', port=46379, db=0)

def random_key():
    caller = inspect.stack()[1][3]
    return f'{caller}_{uuid.uuid4().hex[0:5]}'

class UnitTest(unittest.TestCase):

    def test_sadd(self):
        k = random_key()
        r.sadd(k, 1, 1, 2, 2, 3, 3)
        self.assertEqual(3, len(r.smembers(k)))

    # Get the number of members in a set
    def test_scard(self):
        k = random_key()
        r.sadd(k, 1, 1, 2, 2, 3, 3)
        self.assertEqual(3, r.scard(k))

    def test_srem(self):
        k = random_key()
        r.sadd(k, 1, 1, 2, 2, 3, 3)
        r.srem(k, 3)
        self.assertEqual(2, r.scard(k))

    def test_srandmember(self):
        k = random_key()
        r.sadd(k, 1, 2, 3, 4, 5, 6)
        self.assertEqual(2, len(r.srandmember(k, 2)))
        while True:
            if b'3' in r.srandmember(k, 2):
                return

    # SPOP key [count]
    # Remove and return one or multiple random members from a set
    def test_spop(self):
        k = random_key()
        r.sadd(k, 1, 2, 3, 4, 5, 6)
        while True:
            if b'3' in r.spop(k, 2):
                return

    # SMOVE source destination member
    # Move a member from one set to another
    def test_smove(self):
        k1 = random_key()
        r.sadd(k1, 1, 2, 3, 4, 5, 6)
        k2 = random_key()
        r.smove(k1, k2, 3)
        self.assertTrue(b'3' not in r.smembers(k1))
        self.assertTrue(b'3' in r.smembers(k2))

    def test_sdiff(self):
        k1 = random_key()
        r.sadd(k1, 1, 2, 3)
        k2 = random_key()
        r.sadd(k2, 3, 4, 5)
        self.assertEqual(set([b'1', b'2']), r.sdiff(k1, k2))
        self.assertEqual(set([b'4', b'5']), r.sdiff(k2, k1))

    def test_sinter(self):
        k1 = random_key()
        r.sadd(k1, 1, 2, 3)
        k2 = random_key()
        r.sadd(k2, 3, 4, 5)
        self.assertEqual(set([b'3']), r.sinter(k1, k2))
        self.assertEqual(set([b'3']), r.sinter(k2, k1))

    def test_sunion(self):
        k1 = random_key()
        r.sadd(k1, 1, 2, 3)
        k2 = random_key()
        r.sadd(k2, 3, 4, 5)
        self.assertEqual(set([b'1', b'2', b'3', b'4', b'5']), r.sunion(k1, k2))
        self.assertEqual(set([b'1', b'2', b'3', b'4', b'5']), r.sunion(k2, k1))

if __name__ == '__main__':
    unittest.main(verbosity=2)


from db_creation import *
from itertools import combinations, chain, product
from lib import logging, report, query

def get_cube_name(cube_id):
    for c in Cube.raw('select name from cube where id = %s' % cube_id):
        return c.name

def get_cube_measures_dimensions(cube_id):
    md = {}
    for m in Cube_Value.raw('select value_id from cube_value where cube_id = %s' % cube_id):
        for v in Value.raw('select nvalue, fvalue from value where id = %s' % m.value_id):
            md[m.value_id] = (v.nvalue, v.fvalue)

    dd = {}
    for cd in Cube_Dimension.raw('select dimension_id from cube_dimension where cube_id = %s' % cube_id):
        for d in Dimension.raw('select label from dimension where id = %s' % cd.dimension_id):
            dd[cd.dimension_id] = d.label

    return md, dd

def get_all_dimension_combinations(dd, cube_id, clever=True):
    """если clever==True, то комбинации составляются на основе БД (см. таблицу combinations)"""

    dimension_sets = []  # все возможные сочетания измерений
    if not clever:
        for i in range(1, len(dd) + 1):
            l = list(combinations(dd.keys(), i))
            dimension_sets.append(l)

        dimension_sets = list(chain(*dimension_sets))  # уплощение листа
    else:
        # TODO: 3NF в БД?
        for comb in Combination.raw('select combination from combination where cube_id = %s' % cube_id):
            dimension_sets.append(tuple(map(int, comb.combination.split())))

    return dimension_sets

def get_all_possible_combinations(md, dd, d_sets):
    log_1st_step = ''
    full_sets = []

    for m in md:
        for idx, d_set in enumerate(d_sets):
            d = ''
            for elem in d_set:
                d += dd[elem] + ' '
            log_1st_step += '{}. M:{} D:{}\n'.format(idx, md[m][1], d)
            full_sets.append([m, d_set])

    logging('1st step', log_1st_step)

    return full_sets

def docs_needed(md, dd, d_sets):
    dim_count = {}
    for d in dd:
        count = Dimension_Value.raw('select count(*) from dimension_value where dimension_id = %s' % d).scalar()

        if count == 0:
            count = 1

        dim_count[d] = count

    res = 0
    for item in d_sets:
        r = 1
        for elem in item:
            r *= dim_count[elem]
        res += r

    res *= len(md)

    return dim_count, res

def generate_documents(mdict, ddict, full_sets, cube_name):
    """Генерация документов"""

    # выбор всех измерений со значениями
    dim_vals = []
    for idx, d in enumerate(ddict):
        dim_vals.append([])
        dim_vals[idx].append(d)
        dim_vals[idx].append([])
        for dv in Dimension_Value.raw('select value_id from dimension_value where dimension_id = %s' % d):
            for v in Value.raw('select fvalue, nvalue from value where id = %s' % dv.value_id):
                dim_vals[idx][1].append((v.nvalue, v.fvalue))

    docs = []
    mdx_template = 'SELECT {} ON COLUMNS FROM [{}.DB] WHERE ({})'

    # TODO: рефакторить код
    for idx, set in enumerate(full_sets):
        # если мы используем только одно измерение
        if len(set[1]) == 1:
            for dim_val in dim_vals:
                if dim_val[0] == set[1][0]:
                    for dim_v in dim_val[1]:
                        fr = mdx_template.format('{[MEASURES].[' + mdict[set[0]][1] + ']}', cube_name,
                                                 '[{}].[{}]'.format(ddict[set[1][0]], dim_v[1]))
                        nr = mdict[set[0]][0] + ' ' + dim_v[0]
                        docs.append([fr, nr])
        else:
            # массив со всеми (за некоторыми исключениями) значениями набора измерений
            l = []

            # проверка наличия измерения территория в наборе
            territory_filter = False
            for d_id in set[1]:
                if ddict[d_id] == 'TERRITORIES':
                    territory_filter = True
                    break

            for d_id in set[1]:
                for dim_val in dim_vals:
                    if dim_val[0] == d_id:
                        # если в наборе есть территория и сейчас идет итерация по измерению BGLEVELS
                        if ddict[d_id] == 'BGLEVELS' and territory_filter is True:
                            l.append([('бюджет субъекта', '09-3')])
                        else:
                            l.append(dim_val[1])
                        break

            combs = list(product(*l))
            q = 1
            f = open("C:\\Users\\The Cat Trex\\Desktop\\solr-6.3.0\\example\\exampledocs\\newdata.txt", 'w')

            for item in combs:
                s, k = '', ''

                for elem, d_id in zip(item, set[1]):
                    s += '[{}].[{}],'.format(ddict[d_id], elem[1])
                    k += elem[0] + ' '

                s, k = s[:-1], k[:-1]

                fr = mdx_template.format('{[MEASURES].[' + mdict[set[0]][1] + ']}', cube_name, s)
                nr = mdict[set[0]][0] + ' ' + k
                docs.append([fr, nr])
                print(len(docs))

    print('Документов создалось: {}'.format(len(docs)))
    j = 0
    for q in docs:
        if j == 1:
            json_string = '[{"ID":"' + str(j) + '","MDX_Request":"' + q[0] + '","Normal_request":"' + q[1] + '"}'
            f.write(json_string)
        else:
            json_string = ',{"ID":"' + str(j) + '","MDX_Request":"' + q[0] + '","Normal_request":"' + q[1] + '"}'
            f.write(json_string)
        j += 1

    f.write(']')
    f.close()
    return docs

def learn_model(docs):
    """Отсеивание нерабочих документов"""

    new_docs = []
    for item in docs:
        if query(item[0]):
            new_docs.append(item)
    print('Документов осталось: {}'.format(len(new_docs)))
    return new_docs

id_cube = 1
name_cube = get_cube_name(id_cube)
mdict, ddict = get_cube_measures_dimensions(id_cube)
dim_sets = get_all_dimension_combinations(ddict, id_cube)
measure_dim_sets = get_all_possible_combinations(mdict, ddict, dim_sets)
dim_count, doc_count = docs_needed(mdict, ddict, dim_sets)

# генерация документов
documents = generate_documents(mdict, ddict, measure_dim_sets, name_cube)

# обучение путем проверки корректности документа, через реальный запрос к серверу
# !ОСТОРОЖНО!
# only_working_documents = learn_model(documents)
# !ОСТОРОЖНО!

report(id_cube, name_cube, mdict, ddict, dim_sets, measure_dim_sets, dim_count, doc_count)


import csv
import numpy as np
import matplotlib.pyplot as plt

class csvOperations:
    def __init__ (self, fileName):
        self.fileName = fileName
        # print("in csv, fileName=",self.fileName)

    def readFile(self):
        with open(self.fileName,'r') as csv_File:
            content = csv.reader(csv_File, delimiter = ',')
            xValue = []
            yValue = []
            for row in content:
                # print(row[0],row[1])
                xValue.append(row[1])
                yValue.append(row[0])
        return xValue, yValue

class fitting:
    def __init__ (self, x, y, degree):
        self.x = x
        self.y = y
        self.degree = degree

    def fit(self):
        fitResult = np.polyfit(self.x, self.y, self.degree)
        return fitResult
        # print(fitResult)

def main():
    # print("----")
    csvObject = csvOperations('/Users/katherinetian/Documents/Polynomial Fitting/sampleRHData.csv')
    xValue1 = []
    yValue1 = []
    xValFloat=[]
    yValFloat=[]
    xValue1, yValue1 = csvObject.readFile()

    xValFloat = [float(i) for i in xValue1]
    yValFloat = [float(i) for i in yValue1]

    # print(xValFloat)
    # print(yValFloat)

    fitObject = fitting(xValFloat,yValFloat, 4)
    fitObject1 = fitObject.fit()

    # plt.scatter(xValFloat, yValFloat)
    # plt.show()

    p = np.poly1d(fitObject1)
    print('y = ' + str(np.poly1d(p)))

    yValFit = p(xValFloat)
    print(yValFit)

    # plt.scatter(xValFloat, yValFit)
    # plt.show()

    fig, x1 = plt.subplots()

    plt.xlabel("Frequency (Hz)")
    plt.ylabel("Relative Humidity")

    x1.plot(xValFloat, yValFloat, color = 'red')
    x1.legend(['Original'])
    x2 = x1.twinx()
    x2.plot(xValFloat, yValFit, color = 'blue')
    x2.legend(['Polynomial Fitted'], loc = 'upper left')
    fig.tight_layout()

    plt.show()

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from homeassistant.const import __version__
import requests

def main():
    #local_version = '0.84.6'
    local_version = __version__
    online_version = requests.get('https://pypi.python.org/pypi/homeassistant/json').json()['info']['version']
    if local_version == online_version:
        print('已安装最新版本: %s'%local_version)
    else:
        print('有新版本(%s)可升级！'%online_version)

if __name__ == '__main__':
    main()


import boto3
import csv
import json
import io
import tempfile

nome_bucket = 'fiap-fase3-grupo-l-manual'
s3 = boto3.resource('s3')
s3_client = boto3.client('s3')
my_bucket = s3.Bucket(nome_bucket)
prefix_orig = 'json-raw/'
sqs = boto3.client('sqs')

urlSQS='https://sqs.us-west-2.amazonaws.com/676658052077/SQS-fiap-fase3-grupo-l-manual-json-to-firehose'

for objeto_s3 in my_bucket.objects.filter(Prefix=prefix_orig):
    if objeto_s3.key not in prefix_orig:
        response = sqs.send_message(
                                    QueueUrl=urlSQS,
                                    MessageBody='{"bucket": "' +nome_bucket+ '", "key": "'+ objeto_s3.key+'"}',
                                    DelaySeconds=12
                                    )

        print (response)


# -*- coding: utf-8 -*-
import html
import re
import requests
from datetime import datetime, timedelta

import functions as bf
import database as database
import config as config
from funclist import Funclist

@bf.register(Funclist.standartfunclist)
@bf.addtrigger(bf.standart_trigger)
class SetGroup:
    keywords = ['группа']
    commands = keywords
    attachment = []
    help_name = 'Группа'
    help_content = 'Бот запомнит вашу группу (по сайту с расписанием)'
    help_deep = 'Использование: "бот группа 9999", цифры брать из ссылки с расписания'

    def answer(self, message, vk, *rest):
        group = re.findall(r'[0-9]+', message.text)[0]

        if database.ForAllDataBase.getbyunique(int(message.peer_id), 'narfu_group'):
            database.ForAllDataBase.update(int(message.peer_id), 'narfu_group', group)
        else:
            database.ForAllDataBase.add(int(message.peer_id), 'narfu_group', group)

        bf.sendmessage(vk, peer_id=message.peer_id, answer="Сохранил, ещё раз проверьте что вы скинули мне номер"
                                                           " группы из https://ruz.narfu.ru/?timetable&group=ТУТНОМЕР,"
                                                           " а не номер группы шестизначный")

@bf.register(Funclist.standartfunclist)
@bf.addtrigger(bf.standart_trigger)
class Raspisanie:
    keywords = ['расписание', 'пары', 'test']
    commands = keywords
    attachment = []
    help_name = 'Расписание'
    help_content = 'Бот пришлет вам расписание сафу'
    help_deep = 'Использование: "бот расписание" или "бот пары" или "бот пары завтра" или "бот расписание на завтра"'

    def answer(self, message, vk, *rest):
        ans = ''
        error = ''
        group = database.ForAllDataBase.getbyunique(int(message.peer_id), 'narfu_group')

        if not group:
            ans = 'Я не знаю вашей группы. "бот группа 9999" - команда чтобы сообщить мне его. Номер брать с сайта' \
                  ' расписания: https://ruz.narfu.ru/?timetable&group=ТУТНОМЕР'

        if not ans:
            try:
                req = requests.get(f'https://ruz.narfu.ru/?timetable&group={int(group)}')
                req.encoding = 'utf-8'

                day = (datetime.today() + timedelta(days=(1 if 'завтра' in message.text.lower() else 0),
                                                    hours=(0 if 'сегодня' in message.text.lower() else 8))).strftime(
                    '%d.%m.%Y')

                days = re.findall(r'<div class="list(?: last)? col-md-2([^♥]*?)', req.text)
                classes = [x for x in days if re.findall(r'[0-9]{2}.[0-9]{2}.[0-9]{4}', x)[0] == day][0]
                classes = re.findall(r'<div class="timetable_sheet_xs visible-xs visible-sm([^♥]*?)
', classes)
                result = []

                for clas in classes:
                    result.append(dict(
                        num_para=re.findall(r'([^♥]*?)', clas)[0],
                        time_para=html.unescape(
                            re.findall(r'([^♥]*?)', clas)[0].strip()),
                        kind_of_work=re.findall(r'([^♥]*?)', clas)[0],
                        discipline=re.findall(r'([^♥]*?)', clas)[0],
                        lecturer=re.findall(r'([^♥]*?)', clas),
                        auditorium=html.unescape(re.findall(r'([^♥]*?)', clas)[0].strip()),
                        group=re.findall(r'([^♥]*?)', clas)
                    ))

                # 📗📘📙📕
                emoji = {'Практические занятия': '📙', 'Лабораторные занятия': '📘', 'Лекции': '📗', 'empty': '📕'}
                for x in result:
                    ans += f'{x.get("num_para")}) ' \
                           f'{emoji.get(x.get("kind_of_work")) if x.get("kind_of_work") in emoji else emoji["empty"]}' \
                           f'{x.get("discipline")}\n' \
                           f'{x.get("time_para")} {x.get("kind_of_work")} ' \
                           f'{x.get("lecturer")[0] if x.get("lecturer") else ""}\n' \
                           f'{x.get("auditorium")} {x.get("group")[0] if x.get("group") else ""}\n'

                if not ans:
                    ans = 'Я не нашел пар или их в указанный день нет'

                ans = f'{day}:\n{ans}'

            except Exception as e:
                bf.sendmessage(vk, peer_id=config.creator_id, answer=f'Ошибка в запросе расписания: {e}')

        bf.sendmessage(vk, peer_id=message.peer_id, answer=ans)


def solution(numbers):
    answer = []
    for number in numbers :
        rest = number % 4
        if rest  <= 2 :
            answer.append(number + 1)
        else :
            temp = '0' + bin(number)[2:]
            for i in range(len(temp)-1, -1, -1):
                if temp[i] == '0':
                    answer.append(int(temp[0:i] + "10" + temp[i+2:],2))
                    break
    return answer

if __name__ == '__main__' :
    numbers = [2,7]
    print(solution(numbers))

2개 이하로 다른 비트.py


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from . import __version__ as app_version

app_name = "smart_bot"
app_title = "Smart Bot"
app_publisher = "Yefri Tavarez"
app_description = "An application for helping the the user to do more with less in the Frappe framework"
app_icon = "octicon octicon-hubot"
app_color = "#194085"
app_email = "yefritavarez@hotmail.com"
app_license = "General Public License v3"

# Includes in <head>
# ------------------

# include js, css files in header of desk.html
# app_include_css = "/assets/smart_bot/css/smart_bot.css"
# app_include_js = "/assets/smart_bot/js/smart_bot.js"

# include js, css files in header of web template
# web_include_css = "/assets/smart_bot/css/smart_bot.css"
# web_include_js = "/assets/smart_bot/js/smart_bot.js"

# Home Pages
# ----------

# application home page (will override Website Settings)
# home_page = "login"

# website user home page (by Role)
# role_home_page = {
#	"Role": "home_page"
# }

# Website user home page (by function)
# get_website_user_home_page = "smart_bot.utils.get_home_page"

# Generators
# ----------

# automatically create page for each record of this doctype
# website_generators = ["Web Page"]

# Installation
# ------------

# before_install = "smart_bot.install.before_install"
after_install = "smart_bot.boot.after_install"

# Desk Notifications
# ------------------
# See frappe.core.notifications.get_notification_config

# notification_config = "smart_bot.notifications.get_notification_config"

# Permissions
# -----------
# Permissions evaluated in scripted ways

# permission_query_conditions = {
# 	"Event": "frappe.desk.doctype.event.event.get_permission_query_conditions",
# }
#
# has_permission = {
# 	"Event": "frappe.desk.doctype.event.event.has_permission",
# }

# Document Events
# ---------------
# Hook on document methods and events

# doc_events = {
# 	"*": {
# 		"on_update": "method",
# 		"on_cancel": "method",
# 		"on_trash": "method"
#	}
# }

# Scheduled Tasks
# ---------------

scheduler_events = {
	"all": [
		"smart_bot.bot.do_task"
	]
}

# scheduler_events = {
# 	"all": [
# 		"smart_bot.tasks.all"
# 	],
# 	"daily": [
# 		"smart_bot.tasks.daily"
# 	],
# 	"hourly": [
# 		"smart_bot.tasks.hourly"
# 	],
# 	"weekly": [
# 		"smart_bot.tasks.weekly"
# 	]
# 	"monthly": [
# 		"smart_bot.tasks.monthly"
# 	]
# }

# Testing
# -------

# before_tests = "smart_bot.install.before_tests"

# Overriding Whitelisted Methods
# ------------------------------
#
# override_whitelisted_methods = {
# 	"frappe.desk.doctype.event.event.get_events": "smart_bot.event.get_events"
# }


import argparse
import time

from picamera import PiCamera, Color
from aiy.vision import inference
from aiy.leds import Leds
from aiy.vision.models import utils
from gpiozero import Button
from aiy.pins import BUTTON_GPIO_PIN
from gpiozero import LED
from aiy.pins import PIN_A
from aiy.pins import PIN_B
from aiy.pins import PIN_C

# Initialize LED (in the button on the top of AIY Google Vision box)
leds = Leds()
leds.update(Leds.rgb_off())

# Initialize the GPIO pins A,B,C
pin_A = LED(PIN_A)
pin_B = LED(PIN_B)
pin_C = LED(PIN_C)

# Colors used for LED at the top of Google Vision AIY kit
RED = (0xFF, 0x00, 0x00)
GREEN = (0x00, 0xFF, 0x00)
BLUE = (0x00, 0x00, 0xFF)
PURPLE = (0xFF, 0x00, 0xFF)

# Set status of GPIO pin
def pinStatus(pin,status,gpio_logic):
    if gpio_logic=='INVERSE':
        if status=='HIGH':
            pin.off()
        if status=='LOW':
            pin.on()
    else:
        if status=='HIGH':
            pin.on()
        if status=='LOW':
            pin.off()

"""
index  label           function        pin_A pin_B pin_C
0      stop            stop            0     0     0
1      left            turn 90 degree  0     1     0
2      right           turn -90 degree 0     0     1
3      slow            slow speed      0     1     1
4      background      move forward    1     0     0
"""
# Convert the most likely result to 3 binary signal and sent it out
def send_signal_to_pins(result0,gpio_logic):
    if 'stop' in result0:
        pinStatus(pin_A,'LOW',gpio_logic)
        pinStatus(pin_B,'LOW',gpio_logic)
        pinStatus(pin_C,'LOW',gpio_logic)
        leds.update(Leds.rgb_on(RED))
    elif 'left' in result0:
        pinStatus(pin_A,'LOW',gpio_logic)
        pinStatus(pin_B,'LOW',gpio_logic)
        pinStatus(pin_C,'HIGH',gpio_logic)
        leds.update(Leds.rgb_on(BLUE))
    elif 'right' in result0:
        pinStatus(pin_A,'LOW',gpio_logic)
        pinStatus(pin_B,'HIGH',gpio_logic)
        pinStatus(pin_C,'LOW',gpio_logic)
        leds.update(Leds.rgb_on(PURPLE))
    elif 'slow' in result0:
        pinStatus(pin_A,'LOW',gpio_logic)
        pinStatus(pin_B,'HIGH',gpio_logic)
        pinStatus(pin_C,'HIGH',gpio_logic)
        leds.update(Leds.rgb_on(GREEN))
    else:
        pinStatus(pin_A,'HIGH',gpio_logic)
        pinStatus(pin_B,'LOW',gpio_logic)
        pinStatus(pin_C,'LOW',gpio_logic)
        leds.update(Leds.rgb_off())
    time.sleep(1)

# Get label names from retrained_labels.txt
def read_labels(label_path):
    with open(label_path) as label_file:
        return [label.strip() for label in label_file.readlines()]

def get_message(result, threshold, top_k):
    if result:
        return 'Detecting:\n %s' % '\n'.join(result)

    return 'Nothing detected when threshold=%.2f, top_k=%d' % (threshold, top_k)

def process(result, labels, tensor_name, threshold, top_k):
    """Processes inference result and returns labels sorted by confidence."""
    # MobileNet based classification model returns one result vector.
    assert len(result.tensors) == 1
    tensor = result.tensors[tensor_name]
    probs, shape = tensor.data, tensor.shape
    assert shape.depth == len(labels)
    pairs = [pair for pair in enumerate(probs) if pair[1] > threshold]
    pairs = sorted(pairs, key=lambda pair: pair[1], reverse=True)
    pairs = pairs[0:top_k]
    return [' %s (%.2f)' % (labels[index], prob) for index, prob in pairs]

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--model_path', required=True,
        help='Path to converted model file that can run on VisionKit.')
    parser.add_argument('--label_path', required=True,
        help='Path to label file that corresponds to the model.')
    parser.add_argument('--input_height', type=int, required=True, help='Input height.')
    parser.add_argument('--input_width', type=int, required=True, help='Input width.')
    parser.add_argument('--input_layer', required=True, help='Name of input layer.')
    parser.add_argument('--output_layer', required=True, help='Name of output layer.')
    parser.add_argument('--num_frames', type=int, default=None,
        help='Sets the number of frames to run for, otherwise runs forever.')
    parser.add_argument('--input_mean', type=float, default=128.0, help='Input mean.')
    parser.add_argument('--input_std', type=float, default=128.0, help='Input std.')
    parser.add_argument('--input_depth', type=int, default=3, help='Input depth.')
    parser.add_argument('--threshold', type=float, default=0.1,
        help='Threshold for classification score (from output tensor).')
    parser.add_argument('--top_k', type=int, default=3, help='Keep at most top_k labels.')
    parser.add_argument('--preview', action='store_true', default=False,
        help='Enables camera preview in addition to printing result to terminal.')
    parser.add_argument('--show_fps', action='store_true', default=False,
        help='Shows end to end FPS.')
    parser.add_argument('--gpio_logic', default='NORMAL',
        help='Indicates if NORMAL or INVERSE logic is used in GPIO pins.')
    args = parser.parse_args()

    model = inference.ModelDescriptor(
        name='mobilenet_based_classifier',
        input_shape=(1, args.input_height, args.input_width, args.input_depth),
        input_normalizer=(args.input_mean, args.input_std),
        compute_graph=utils.load_compute_graph(args.model_path))
    labels = read_labels(args.label_path)

    with PiCamera(sensor_mode=4, resolution=(1640, 1232), framerate=30) as camera:
        if args.preview:
            camera.start_preview()

        with inference.CameraInference(model) as camera_inference:
            for result in camera_inference.run(args.num_frames):
                processed_result = process(result, labels, args.output_layer,
                                           args.threshold, args.top_k)
                send_signal_to_pins(processed_result[0], args.gpio_logic)
                message = get_message(processed_result, args.threshold, args.top_k)
                if args.show_fps:
                    message += '\nWith %.1f FPS.' % camera_inference.rate
                print(message)

                if args.preview:
                    camera.annotate_foreground = Color('black')
                    camera.annotate_background = Color('white')
                    # PiCamera text annotation only supports ascii.
                    camera.annotate_text = '\n %s' % message.encode(
                        'ascii', 'backslashreplace').decode('ascii')

        if args.preview:
            camera.stop_preview()

if __name__ == '__main__':
    main()


from tensorflow.keras import backend as K
import tensorflow as tf

def jaccard_distance(y_true, y_pred, smooth=100):
    """Jaccard distance for semantic segmentation.

    Also known as the intersection-over-union loss.

    This loss is useful when you have unbalanced numbers of pixels within an image
    because it gives all classes equal weight. However, it is not the defacto
    standard for image segmentation.

    For example, assume you are trying to predict if
    each pixel is cat, dog, or background.
    You have 80% background pixels, 10% dog, and 10% cat.
    If the model predicts 100% background
    should it be be 80% right (as with categorical cross entropy)
    or 30% (with this loss)?

    The loss has been modified to have a smooth gradient as it converges on zero.
    This has been shifted so it converges on 0 and is smoothed to avoid exploding
    or disappearing gradient.

    Jaccard = (|X & Y|)/ (|X|+ |Y| - |X & Y|)
            = sum(|A*B|)/(sum(|A|)+sum(|B|)-sum(|A*B|))

    # Arguments
        y_true: The ground truth tensor.
        y_pred: The predicted tensor
        smooth: Smoothing factor. Default is 100.

    # Returns
        The Jaccard distance between the two tensors.

    # References
        - [What is a good evaluation measure for semantic segmentation?](
           http://www.bmva.org/bmvc/2013/Papers/paper0032/paper0032.pdf)

    """
    intersection = K.sum(K.abs(y_true * y_pred), axis=-1)
    sum_ = K.sum(K.abs(y_true) + K.abs(y_pred), axis=-1)
    jac = (intersection + smooth) / (sum_ - intersection + smooth)
    return (1 - jac) * smooth

# Class Dice coefficient averaged over batch
def dice_coef( y_true, y_pred, axis=1, eps=1e-6):
    intersection = tf.reduce_sum(input_tensor=y_pred * y_true, axis=axis)
    union = tf.reduce_sum(input_tensor=y_pred * y_pred + y_true * y_true, axis=axis)
    dice = (2. * intersection + eps) / (union + eps)
    return tf.reduce_mean(input_tensor=dice, axis=0)  # average over batch

def dice_loss(y_true, y_pred):
    n_classes = y_pred.shape[-1]

    flat_logits = tf.reshape(tf.cast(y_pred, tf.float32),
                             [tf.shape(input=y_pred)[0], -1, n_classes])
    flat_labels = tf.reshape(y_true,
                             [tf.shape(input=y_pred)[0], -1, n_classes])

    dice_loss = tf.reduce_mean(input_tensor = 1 - dice_coef( tf.keras.activations.softmax(flat_logits, axis=-1), flat_labels), name='dice_loss_ref')
    return dice_loss

def adaptive_loss(y_true, y_pred, switch_at_threshold=0.3):
    dice_l = dice_loss(y_true, y_pred)

    return tf.cond(
        dice_l < switch_at_threshold,
        true_fn=lambda: dice_l,
        false_fn=lambda: tf.keras.losses.categorical_crossentropy(y_true, y_pred, from_logits=False, label_smoothing=0)
        )


import numpy
import random as rand
import math
import matplotlib.pyplot as plt
from sklearn.datasets import load_breast_cancer

def distance(p1, p2):
	if len(p1) != len(p2):
		print("Dimension of points do not match...")
		return

	d, dim = 0.0, len(p1)
	for i in range(dim):
		d += (p1[i]-p2[i])**2

	return math.sqrt(d)

def mean(points):
	if not points:
		print("No points to compute on...")
		return

	# Initialize vars
	count      = len(points)
	dim 	   = len(points[0])
	mean_point = [0.0] * dim

	for d in range(dim):
		for p in points:
			mean_point[d] += p[d]
		mean_point[d] /= count

	return mean_point

def cost(centroid_info):
	J = 0.0
	for c in centroid_info:
		J += c[1]**2
	return J / len(centroid_info)

def is_change(old_points, new_points):
	if len(old_points) != len(new_points) or not old_points or not new_points:
		return True

	points_count = len(new_points)
	dim 		 = len(new_points[0])

	for p in range(points_count):
		for d in range(dim):
			if old_points[p][d] != new_points[p][d]:
				return True

	return False

def kmeans(data = [], k = 2):
	##################################
	###### INITIALIZE VARIABLES ######
	##################################
	data_size 			= len(data)  # size of dataset
	centroid_positions  = [] 		 # array of centroid positions
	is_centroid_change  = True		 # keep track of centroid positions change
	if data_size == 0:
		raise Exception("No data provided...")

	# 1. Randomly initialize k centroids (i.e. initial guesses for centroid positions)
	for _ in range(k):
		rand_i = rand.randrange(data_size)
		centroid_positions.append(data[rand_i])

	# loop while no more change in centroid positions
	while is_centroid_change:

		# 2. Compute distance b/w each datapoint and centroids -> dataIndex:[distance1, distance2, ...]
		distances_to_centroids = []
		for d in range(data_size):
			c_pos = []
			for c in centroid_positions:
				c_pos.append(distance(data[d], c))
			distances_to_centroids.append(c_pos)

		# 3a. Assign datapoint to centroid that is closest to it -> dataIndex:[centroidIndex, distance]
		for d in range(data_size):
			min_val = min(distances_to_centroids[d])
			distances_to_centroids[d] = [distances_to_centroids[d].index(min_val), min_val]

		# 3b. Assign data coordinates to centroids -> centroidIndex: [[data1], [data2], ...]
		cluster_data_points = {}
		for d in range(data_size):
			if distances_to_centroids[d][0] not in cluster_data_points:
				cluster_data_points[distances_to_centroids[d][0]] = [data[d]]
			else:
				cluster_data_points[distances_to_centroids[d][0]].append(data[d])

		# 4b. Update location of cluster centroids (i.e. move the cluster centroid to the mean position)
		new_positions = [[]] * (1 + max(cluster_data_points.keys()))
		for _k in cluster_data_points.keys():
			new_positions[_k] = mean(cluster_data_points[_k])

		if is_change(centroid_positions, new_positions):
			is_centroid_change = True
			centroid_positions = new_positions.copy()
		else:
			is_centroid_change = False

	return centroid_positions, cluster_data_points, cost(distances_to_centroids)

def kmeans_distortion_plot(x_label, x_data, y_label, y_data):
	plt.plot(x_data, y_data, 'ro')
	plt.xlabel(x_label)
	plt.ylabel(y_label)
	plt.show()

if __name__ == "__main__":
	# load in data as numpy array of n-dim datapoints
	dataset = load_breast_cancer()['data']

	cluster_centroid_locations, cluster_centroid_assignments, j_cost = kmeans(data=dataset, k=2)

	iters, costs = list(range(2, 8)), []

	for i in iters:
		cluster_centroid_locations, cluster_centroid_assignments, j_cost = kmeans(data=dataset, k=i)
		costs.append(j_cost)

	kmeans_distortion_plot(x_label='k-Values', x_data=iters, y_label='Distortion (J-Cost)', y_data=costs)


import csv
import os
import shutil

donnees = []
with open("numerotation.csv") as csvf:
	lecteur = csv.reader(csvf, delimiter='\t', quotechar="|")

	for l in lecteur:
		foliotation = l[0]
		fichier = l[1].split("/")[-1]
		if not l[1] and not l[0]:
			break
		if l[0] and l[1]:
			donnees.append([foliotation, fichier])

for d in donnees:
	shutil.copy(d[1], f"./sortie/{d[0]}.jpg")


'''
Created on Jul 6, 2010

@author: bdf
'''

'''
Save 1 occurs after all the unit orders are given, but before the data is sent to the host.
This is the save that each client would send for pbem.

Save 2 occurs after the host has calculated the results of the orders, but before they are carried out.
Thus in this save, the unit positions and health have not changed compared to save 1.  The only thing that is
different is that the results of orders are available.  This save would be the email that is sent from host
back to clients.

Save 3 occurs after the order resolution from host has been carried out.  This is the state of the game
at which each player can put in new orders.

'''

import Queue
from events import *
from evmanager import *

from copy import deepcopy
from internalconstants import *
import os
import boautils as boaut

def savehero(hero,profile):
    ppath = os.path.join(profilespath, profile)
    pwn = os.path.join(ppath,hero.name)
    boaut.saveclasstodisk(hero,pwn)

def loadhero(profdir, herofilename):
    h = os.path.join(profdir, herofilename)
    return boaut.loadclassfromdisk(h)

class GameSave():
    def __init__(self, game):

#        self.viewlevelChanges = {}
        self.tiles_within_party_normal_view = {}
        self.tiles_within_party_dim_view = {}
#        self.citydict = []
#        self.unitgroups = {}
        self.objectIdDict = {}
        self.nextid = 0
        self.myprofname = None
        self.Time = 0.
        self.tot_num_pulses = 0.

        self.stores = []
        self.chars = []
        self.charnamesid = {}
        self.encounters = {}
        self.map_data = {}
#        self.map = {}

#        self.chardata = {}
#        for char in game.chars:
#            self.chardata[char.name]['orders'] = []

#        self.mynation = None
#        self.myteam = None
#
#        self.nation = {}
#
#        for nn in game.nations:
#            self.nation[nn] = {}
#
#            self.nation[nn]['units'] = []
#            self.nation[nn]['movableObjects'] = []
#            self.nation[nn]['movementgroups'] = []
#            self.nation[nn]['citylist'] = []
#            self.nation[nn]['nationorderslist'] = []
#            self.nation[nn]['unitorderslist'] = []
#            self.nation[nn]['money'] = 0

class SaveLoad():
    def __init__(self, evManager, game):
        self.evManager = evManager
        listencats = []
        self.evManager.RegisterListener( self, listencats )

        self.game = game

        self.currentstate = None

    def doDeepcopySave(self, savename):     # creates a copy of game in a new memory location,
                                            # and assigns thissave to that location

        thissave = self.game.saves[savename]

#        thissave.viewlevelChanges = deepcopy(self.game.viewlevelChanges)
        thissave.tiles_within_party_normal_view = deepcopy(self.game.tiles_within_party_normal_view)
        thissave.tiles_within_party_dim_view = deepcopy(self.game.tiles_within_party_dim_view)
        thissave.objectIdDict = deepcopy(self.game.objectIdDict)
        thissave.nextid = self.game.nextid
        thissave.Time = self.game.Time
        thissave.myprofname = self.game.myprofname
        thissave.tot_num_pulses = self.game.tot_num_pulses

        thissave.stores = deepcopy(self.game.stores)
        thissave.chars = deepcopy(self.game.chars)
        thissave.charnamesid = deepcopy(self.game.charnamesid)
        thissave.encounters = deepcopy(self.game.encounters)
        thissave.map_data = deepcopy(self.game.map_data)
#        thissave.map = deepcopy(self.game.map)

    def doSave(self, savename):     # assigns thissave to the existing game memory location

        self.currentstate = savename

        thissave = self.game.saves[savename]

#        thissave.viewlevelChanges = self.game.viewlevelChanges
        thissave.tiles_within_party_normal_view = self.game.tiles_within_party_normal_view
        thissave.tiles_within_party_dim_view = self.game.tiles_within_party_dim_view
#        thissave.citydict = self.game.citydict
#        thissave.unitgroups = self.game.unitgroups
        thissave.objectIdDict = self.game.objectIdDict
        thissave.nextid = self.game.nextid
        thissave.Time = self.game.Time
        thissave.myprofname = self.game.myprofname
        thissave.tot_num_pulses = self.game.tot_num_pulses

        thissave.stores = self.game.stores
        thissave.chars = self.game.chars
        thissave.charnamesid = self.game.charnamesid
        thissave.encounters = self.game.encounters
        thissave.map_data = self.game.map_data
#        thissave.map = self.game.map

    def restoreDeepcopySave(self, savename):    # creates a copy of thissave in a new memory location
                                                # and assigns game to this new location

        thissave = self.game.saves[savename]

#        self.game.viewlevelChanges = deepcopy(thissave.viewlevelChanges)
        self.game.tiles_within_party_normal_view = deepcopy(thissave.tiles_within_party_normal_view)
        self.game.tiles_within_party_dim_view = deepcopy(thissave.tiles_within_party_dim_view)
        self.game.citydict = deepcopy(thissave.citydict)
#        self.game.unitgroups = deepcopy(thissave.unitgroups)
        self.game.objectIdDict = deepcopy(thissave.objectIdDict)
        self.game.nextid = thissave.nextid

        for natname in self.game.nations:
            nation = self.game.Nations[natname]

            nation.units = deepcopy(thissave.nation[natname]['units'])
            nation.movableObjects = deepcopy(thissave.nation[natname]['movableObjects'])
            nation.movementgroups = deepcopy(thissave.nation[natname]['movementgroups'])
            nation.citylist = deepcopy(thissave.nation[natname]['citylist'])
            nation.orderslist = deepcopy(thissave.nation[natname]['orderslist'])
            nation.unitorderslist = deepcopy(thissave.nation[natname]['unitorderslist'])
            nation.money = thissave.nation[natname]['money']

    def restoreSave(self, savename):        # assigns game's memory location to the memory location of thissave

        thissave = self.game.saves[savename]

#        self.game.viewlevelChanges = thissave.viewlevelChanges
        self.game.tiles_within_party_normal_view = thissave.tiles_within_party_normal_view
        self.game.tiles_within_party_dim_view = thissave.tiles_within_party_dim_view
#        self.game.citydict = thissave.citydict
#        self.game.unitgroups = thissave.unitgroups
        self.game.objectIdDict = thissave.objectIdDict

        tempid = self.game.nextid
        self.game.nextid = thissave.nextid
        self.game.saves[self.currentstate].nextid = tempid

        tempnum = self.game.Time
        self.game.Time = thissave.Time
        self.game.saves[self.currentstate].Time = tempnum

        self.game.myprofname = thissave.myprofname
        self.game.tot_num_pulses = thissave.tot_num_pulses

        self.game.stores = thissave.stores
        self.game.chars = thissave.chars
        self.game.charnamesid = thissave.charnamesid
        self.game.encounters = thissave.encounters
        self.game.map_data = thissave.map_data
#        self.game.map = thissave.map

        self.currentstate = savename

    def preMovieSave(self):

        for natname in self.game.nations:
            nation = self.game.Nations[natname]

            nation.premoviesave_units = deepcopy(nation.units)
            nation.premoviesave_movableObjects = deepcopy(nation.movableObjects)
            nation.premoviesave_citylist = deepcopy(nation.citylist)
            nation.premoviesave_orderslist = deepcopy( nation.orderslist )
            nation.premoviesave_unitorderslist = deepcopy( nation.unitorderslist )

            nation.premoviesave_money = nation.money

        self.game.premoviesave_viewlevelChanges = deepcopy(self.game.viewlevelChanges)
        self.game.premoviesave_tiles_within_party_normal_view = deepcopy(self.game.tiles_within_party_normal_view)
        self.game.premoviesave_citydict = deepcopy(self.game.citydict)
        self.game.premoviesave_unitgroups = deepcopy(self.game.unitgroups)
        self.game.premoviesave_objectIdDict = deepcopy(self.game.objectIdDict)
        #self.game.premoviesave.tempIdDict = deepcopy(self.game.tempIdDict)

        self.game.premoviesave_nextid = self.game.nextid

    def restoreToPreMovieSave(self):

#        for natname in self.game.nations:
#            nation = self.game.Nations[natname]
#
#            nation.units = nation.premoviesave_units
#            nation.movableObjects = nation.premoviesave_movableObjects
#            nation.citylist = nation.premoviesave_citylist
#            nation.orderslist = nation.premoviesave_orderslist
#            nation.unitorderslist = nation.premoviesave_unitorderslist
#
#            nation.money = nation.premoviesave_money
#
#
#        self.game.viewlevelChanges = self.game.premoviesave_viewlevelChanges
#        self.game.tilesWithinView = self.game.premoviesave_tilesWithinView
#        self.game.citydict = self.game.premoviesave_citydict
#        self.game.unitgroups = self.game.premoviesave_unitgroups
#        self.game.objectIdDict = self.game.premoviesave_objectIdDict
#        #self.game.premoviesave.tempIdDict = deepcopy(self.game.tempIdDict)
#
#        self.game.nextid = self.game.premoviesave_nextid

        for natname in self.game.nations:
            nation = self.game.Nations[natname]

            nation.units = deepcopy(nation.premoviesave_units)
            nation.movableObjects = deepcopy(nation.premoviesave_movableObjects)
            nation.citylist = deepcopy(nation.premoviesave_citylist)
            nation.orderslist = deepcopy(nation.premoviesave_orderslist)
            nation.unitorderslist = deepcopy(nation.premoviesave_unitorderslist)

            nation.money = nation.premoviesave_money

        self.game.viewlevelChanges = deepcopy(self.game.premoviesave_viewlevelChanges)
        self.game.tiles_within_party_normal_view = deepcopy(self.game.premoviesave_tilesWithinView)
        self.game.citydict = deepcopy(self.game.premoviesave_citydict)
        self.game.unitgroups = deepcopy(self.game.premoviesave_unitgroups)
        self.game.objectIdDict = deepcopy(self.game.premoviesave_objectIdDict)
        #self.game.premoviesave.tempIdDict = deepcopy(self.game.tempIdDict)

        self.game.nextid = self.game.premoviesave_nextid

    def postMovieSave(self):

        for natname in self.game.nations:
            nation = self.game.Nations[natname]

            nation.postmoviesave_units = deepcopy(nation.units)
            nation.postmoviesave_movableObjects = deepcopy(nation.movableObjects)
            nation.postmoviesave_citylist = deepcopy(nation.citylist)
            nation.postmoviesave_orderslist = deepcopy( nation.orderslist )
            nation.postmoviesave_unitorderslist = deepcopy( nation.unitorderslist )

            nation.postmoviesave_money = nation.money

        self.game.postmoviesave_viewlevelChanges = deepcopy(self.game.viewlevelChanges)
        self.game.postmoviesave_tilesWithinView = deepcopy(self.game.tiles_within_party_normal_view)
        self.game.postmoviesave_citydict = deepcopy(self.game.citydict)
        self.game.postmoviesave_unitgroups = deepcopy(self.game.unitgroups)
        self.game.postmoviesave_objectIdDict = deepcopy(self.game.objectIdDict)
        #self.game.postmoviesave.tempIdDict = deepcopy(self.game.tempIdDict)

        self.game.postmoviesave_nextid = self.game.nextid

    def restoreToPostMovieSave(self):

#        for natname in self.game.nations:
#            nation = self.game.Nations[natname]
#
#            nation.units = nation.postmoviesave_units
#            nation.movableObjects = nation.postmoviesave_movableObjects
#            nation.citylist = nation.postmoviesave_citylist
#            nation.orderslist = nation.postmoviesave_orderslist
#            nation.unitorderslist = nation.postmoviesave_unitorderslist
#
#            nation.money = nation.postmoviesave_money
#
#
#        self.game.viewlevelChanges = self.game.postmoviesave_viewlevelChanges
#        self.game.tilesWithinView = self.game.postmoviesave_tilesWithinView
#        self.game.citydict = self.game.postmoviesave_citydict
#        self.game.unitgroups = self.game.postmoviesave_unitgroups
#        self.game.objectIdDict = self.game.postmoviesave_objectIdDict
#        #self.game.postmoviesave.tempIdDict = deepcopy(self.game.tempIdDict)
#
#        self.game.nextid = self.game.postmoviesave_nextid

        for natname in self.game.nations:
            nation = self.game.Nations[natname]

            nation.units = deepcopy(nation.postmoviesave_units)
            nation.movableObjects = deepcopy(nation.postmoviesave_movableObjects)
            nation.citylist = deepcopy(nation.postmoviesave_citylist)
            nation.orderslist = deepcopy(nation.postmoviesave_orderslist)
            nation.unitorderslist = deepcopy(nation.postmoviesave_unitorderslist)

            nation.money = nation.postmoviesave_money

        self.game.viewlevelChanges = deepcopy(self.game.postmoviesave_viewlevelChanges)
        self.game.tiles_within_party_normal_view = deepcopy(self.game.postmoviesave_tilesWithinView)
        self.game.citydict = deepcopy(self.game.postmoviesave_citydict)
        self.game.unitgroups = deepcopy(self.game.postmoviesave_unitgroups)
        self.game.objectIdDict = deepcopy(self.game.postmoviesave_objectIdDict)
        #self.game.postmoviesave.tempIdDict = deepcopy(self.game.tempIdDict)

        self.game.nextid = self.game.postmoviesave_nextid

    def Notify(self, event):

        if isinstance( event, PreMovieSaveEvent ):
            self.preMovieSave()

        elif isinstance( event, PostMovieSaveEvent ):
            self.postMovieSave()

        elif isinstance( event, RestoreToPostMovieEvent ):
            self.restoreToPostMovieSave()


import numpy as np
import pandas as pd

import usfhn.fieldwork
import usfhn.datasets
import usfhn.constants
import usfhn.institutions

def get_gini_coefficients(
    by_year=False,
    by_new_hire=False,
    by_faculty_rank=False,
    by_seniority=False,
):
    from usfhn.stats import add_groupby_annotations_to_df

    groupby_cols = [
        'TaxonomyLevel',
        'TaxonomyValue',
    ]

    if by_year:
        groupby_cols.append('Year')

    df = usfhn.datasets.get_dataset_df('data', by_year=by_year)

    df = usfhn.fieldwork.linearize_df_taxonomy_hierarchy(df)

    df = df[
        groupby_cols + [
            'PersonId',
            'DegreeInstitutionId',
        ]
    ].drop_duplicates()

    df, extra_groupby_cols = add_groupby_annotations_to_df(
        df,
        by_faculty_rank=by_faculty_rank,
        by_new_hire=by_new_hire,
        by_seniority=by_seniority,
    )

    groupby_cols += extra_groupby_cols

    return get_gini_coefficients_for_df(df, groupby_cols)

def gini_coefficient(population):
    """
    args:
    - population: a list of integer counts

    returns:
    - the gini coefficient for the list

    calculated by way of the simplified version at:
    https://en.wikipedia.org/wiki/Gini_coefficient#Calculation
    """
    numerator = 0
    denominator = 0
    n = len(population)
    for i, y_i in enumerate(sorted(population)):
        numerator += ((2 * i + 1) - n - 1) * y_i
        denominator += n * y_i

    if denominator == 0:
        return 0

    return round(numerator / denominator, 3)

def get_gini_coefficients_for_df(df, groupby_cols):
    """
    expects df to have the following columns:
    - PersonId
    - DegreeInstitutionId
    - `groupby_cols`
    """
    df = df.copy()

    # add in zero values for academia
    institutions = set(usfhn.institutions.get_us_phd_granting_institutions())
    degree_institutions = set(df['DegreeInstitutionId'].unique())
    institutions_to_add_to_academia = institutions - degree_institutions

    df['TotalProduction'] = df.groupby(groupby_cols)['PersonId'].transform('nunique')

    df['InstitutionProduction'] = df.groupby(
        ['DegreeInstitutionId'] + groupby_cols
    )['PersonId'].transform('nunique')

    df['ProductionFraction'] = df['InstitutionProduction'] / df['TotalProduction']
    df = df[
        [
            'ProductionFraction',
            'DegreeInstitutionId',
        ]
        +
        groupby_cols
    ].drop_duplicates()

    academia_rows = []
    for institution in institutions_to_add_to_academia:
        academia_rows.append({
            'DegreeInstitutionId': institution,
            'TaxonomyLevel': 'Academia',
            'TaxonomyValue': 'Academia',
            'ProductionFraction': 0,
        })

    academia_rows = pd.DataFrame(academia_rows)

    for column in [c for c in groupby_cols if c not in ['TaxonomyLevel', 'TaxonomyValue']]:
        extra_academia_rows = []
        for column_value in list(df[column].unique()):
            new_academia_rows = academia_rows.copy()
            new_academia_rows[column] = column_value
            extra_academia_rows.append(new_academia_rows)

        academia_rows = pd.concat(extra_academia_rows)

    df = pd.concat([df, academia_rows])

    gini_coefficients = []
    for _, rows in df.groupby(groupby_cols):
        rows = rows.drop_duplicates(subset=['DegreeInstitutionId'])

        result = {c: rows.iloc[0][c] for c in groupby_cols}
        result['GiniCoefficient'] = gini_coefficient(rows['ProductionFraction'])

        gini_coefficients.append(result)
        # if result['GiniCoefficient']:
        #     gini_coefficients.append(result)

    return pd.DataFrame(gini_coefficients)


memberNum = int(input())
member = list(map(int,input().split()))
result = 0
i = 0
member.sort()
# 방법 1:

# 방법 2:
result = 0
count = 0
for i in member:
    count += 1
    if count == i:
        result +=1
        count=0

print("방법1 결과: ",result)


from model.user import User
from servers.tasks.service.services.app import db

@staticmethod
def registrationUser(email:str):
    _user = User(
          email=email
    )
    db.session.add(_user)
    db.session.commit()

@staticmethod
def getIdUser(email:str):
    id = User.query.filter_by(id=id).first()

@staticmethod
def deleteUser(id):
    _user = User.query.filter_by(id=id).first()
    db.session.delete(_user)
    db.session.commit()


#!/usr/bin/python3
import os
import regex
import sys
import io

# Matt Postiff, April 2022
# This script converts the USFM of Waali in accordance with rules
# given to us by Ron Webber. The results are a
# dictionary that is printed, followed by the conversions.
# This is a test bench.
# See waaliOrthographicConvert2.py for the version that uses the
# same functions but prints out actual USFM.

# Usage: python3 waaliOrthographicConvert.py waaliOrthoTest.SFM > waaliOrthoTest.out

script = sys.argv.pop(0)

if len(sys.argv) < 1:
    print(f"Usage: {script} file [file ...]\n")
    exit(1)

# The dictionary that we are building...the main goal of this code
# It is augmented in the learnWord function
wordCntDict = {}

DEBUG = 0
def debug(msg:str, lineEnd=''):
    if (DEBUG):
        print(msg) # end=lineEnd)

def error(msg:str):
    print(f"ERROR: {msg}")

def learnWord(wrd:str):
    #debug((f"Examining {wrd}, whose last char is {wrd[-1]}"))
    # Remove trailing , or .
    # Notice to match period, you use r".", not "\."
    if ((wrd[-1] == r".") or (wrd[-1] == r',')):
        #debug(f"Removing last char of {wrd}")
        wrd = wrd[:-1]

    # Remove USFM marker at the end of the word like \w or \k*
    # May have to do more than once for like  word\w*\f
    while (True):
        match = regex.search(r"\\[a-z]+(\*)?", wrd)
        if match is not None:
            wrd = wrd.replace(match.group(0), "")
            #debug(f"Removed marker {match.group(0)} from {wrd}")
        else:
            break;

    # Remove other offensive characters
    wrd = wrd.translate({ord(c): None for c in "()«»"})

    if (len(wrd) == 0):
        return

    if (wrd[0] == '\\'):
        # We have a USFM code. Ignore it.
        #debug(f"Ignoring USFM marker: {wrd}")
        return
    elif (regex.search("[0-9]+:[0-9]+", wrd)):
        #debug(f"Ignoring verse range: {wrd}")
        pass
    elif (regex.search("[0-9]+-[0-9]+", wrd)):
        #debug(f"Ignoring numerical range: {wrd}")
        pass
    elif (regex.search("^[0-9]+$", wrd)):
        #debug(f"Ignoring verse number or similar: {wrd}")
        pass
    else:
        if wrd in wordCntDict:
            wordCntDict[wrd] = wordCntDict[wrd]+1
            #debug(f"Seen again: {wrd}")
        else:
            wordCntDict[wrd] = 1
            #debug(f"Added: {wrd}")

def learnLine(line:str):
    #debug(f"LINE: {line}")

    words = line.split()
    for i in range(0, len(words)):
        learnWord(words[i])
    return

# Examples, see waaliOrthoTest.SFM for these and other tests
# 'teng-gbani  > teŋgbane
# (1) ng converted to ŋ  (2) stressed portion ignored  (3) final portion converts vowels
# 'be-fora > befɔra
# bung'vori > boŋ'vori
# 'bi-poga >  bipɔga
# 'Ree-chel > Reekyɛl
# kpe'ji-pegi'yiruu > kpɛgyi pɛgeyiruu (this is actually two words with a hyphen)
def convertConsonants(wrd:str):
    debug(f"Changing consonants in : {wrd}")
    wrd = regex.sub("j", "gy", wrd)
    wrd = regex.sub("J", "Gy", wrd)
    wrd = regex.sub("ch", "ky", wrd)
    wrd = regex.sub("Ch", "Ky", wrd)
    wrd = regex.sub("ng", "ŋ", wrd)  # character U+014B
    wrd = regex.sub("Ng", "Ŋ", wrd)  # character U+014A
    return wrd

def convertUnstressedVowels(wrdPart:str):
    wrdPart = regex.sub("e", "ɛ", wrdPart) # character U+025B
    wrdPart = regex.sub("E", "Ɛ", wrdPart) # character U+0190
    wrdPart = regex.sub("i", "e", wrdPart)
    wrdPart = regex.sub("I", "E", wrdPart)
    wrdPart = regex.sub("o", "ɔ", wrdPart) # character U+0254
    wrdPart = regex.sub("O", "Ɔ", wrdPart) # character U+0186
    wrdPart = regex.sub("u", "o", wrdPart)
    wrdPart = regex.sub("U", "O", wrdPart)
    return wrdPart

def convertVowels(wrd:str):
    # Vowel conversions only happen with close vowels, not "open" ones
    # In other words, we ignored stressed portions of words and convert
    # the vowels in unstressed portions
    debug(f"Changing vowels in : {wrd}")
    # Abstract examples
    # 'XXX-yyy paradigm wor (or 'XXXXXX) where yyy will undergo vowel conversions
    # yyy'XXX paradigm word, where yyy will undergo vowel conversions
    # Strange hyphenated words, like yy'XXX-yyy where yyy will undergo vowel conversions
    # Strange hyphenated words, like yyy'XX-yyy'XXXX where yyy will undergo vowel conversions
    # yyyyyy (no ticks, no apostrophes), which all letters are candidates for vowel conversion
    newWrd = ""
    if (wrd[0] == '\''): # This is a 'XXX-yyy paradigm wor (or 'XXXXXX) where yyy will undergo vowel conversions
        debug(f"Case 1 - 'XXX-yyy paradigm")
        wrdAsList = wrd.split("-", -1) # Split on the hyphen this case
        for wrdPart in wrdAsList:
            if (wrdPart[0] == '\''):
                # This part of the word is stressed, so its vowels remain unchanged (open)
                pass
            else:
                # This part of the word is unstressed, so its vowels need changed (close)
                wrdPart = convertUnstressedVowels(wrdPart)

            newWrd = newWrd + wrdPart # We eliminate the hyphen in the new version of the language
            #NOT FOR NOW newWrd = regex.sub("\'", "", newWrd) # discard apostrophes when work is finalized

    #elif (wrd.find('\'', 1) == 0): # This is a yyy'XXX paradigm word, where yyy will undergo vowel conversions
    #    # The way this was originally written, the comparison was != -1, meaning that at least one ' was found.
    #    # The problem with this was that several ticks could be found, and the assertion below would fail
    #    # I rewrote it as == 1, thinking find returns a count, but it actually returns in INDEX. This
    #    # was wrong because it would find only words with a tick in the second position (idx = 1).
    #    # I found that I could remove this section entirely and let the next elif cover it all.
    #    # The next elif can cover everything, in fact, but for speed it is good to have a couple of special cases (above and below).
    #    debug(f"Case 2 - yyy'XXX paradigm")
    #    wrdAsList = wrd.split("'", -1) # Split on the apostrophe this case
    #    assert (len(wrdAsList) <= 2), f"ERROR: {wrd} has more than two sections like yyy'XXX'???"
    #    wrdPart = convertUnstressedVowels(wrdAsList[0])
    #    newWrd = wrdPart + '\'' + wrdAsList[1] # must add back in the apostrophe, as the split "deletes" it

    elif (wrd.find('\'', 1) >= 1): # A 'tick' or apostrophe is found some index > 0
        debug(f"Case 3 - state machine")
        # This is most often a word like yyy'XXX where yyy undergoes vowel conversions.
        # This could be a strange word like kpe'ji-pegi'yiruu which is actually two words hyphenated together.
        # This overloads the hyphen, using it like English. Only the close vowels will be changed--those
        # in parts 1 and 3 of the word but not 2 and 4. Build a state machine to handle it.
        CLOSE = 1
        OPEN = 2
        state = CLOSE
        newWrd = ""  # I might be able to make this be the algorithm for the every case.
        for ch in wrd:
            if ch == '\'':
                state = OPEN
                newWrd = newWrd + ch
            elif ch == '-':
                state = CLOSE
                newWrd = newWrd + ch
            else:
                if (state == OPEN): # No vowel conversion
                    newWrd = newWrd + ch
                else: # State is close, needs vowel conversion
                    newWrd = newWrd + convertUnstressedVowels(ch)

    else: # This is just a regular old word, no initial apostrophe, no mid-apostrophe, so convert all vowels
        debug(f"Case 4: change all vowels")
        newWrd = convertUnstressedVowels(wrd)

    return newWrd

# Main routine
for file in sys.argv:
    #debug("Processing " + file)
    if os.path.isdir(file):
        error("Cannot process directory " + file + "\n")
        continue

    fi = open(file, mode="r", newline='')

    for cnt, line in enumerate(fi):
            learnLine(line)

    fi.close()

for o in wordCntDict:
    convertedWord = convertVowels(convertConsonants(o))
    print(f"{o} : {convertedWord} : {wordCntDict[o]}")

# The end