# TODO:
# - Allow windows to be read from a file.

import copy
import cProfile
import gc
import logging
import pickle
import pstats
import time
import warnings
from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor
from concurrent.futures import wait as cf_wait
from functools import partial
from itertools import islice
from multiprocessing import cpu_count
from multiprocessing.pool import ThreadPool as Pool

import dask
import healpy as hp
# from jax import jit
import jax
import jax.numpy as jnp
import numpy as np
import psutil
import sparse
import zarr
from dask import delayed
from dask.distributed import Client, Semaphore, get_client
from dask.threaded import get
from distributed import LocalCluster
from distributed.client import Future
from scipy.interpolate import interp1d

from skylens.binning import *
from skylens.utils import *
from skylens.wigner_transform import *

class window_utils:
    def __init__(
        self,
        window_l=None,
        window_lmax=None,
        l=None,
        l_bins=None,
        l_cl=None,
        corrs=None,
        s1_s2s=None,
        use_window=None,  # f_sky=None,
        corr_indxs=None,
        z_bins=None,
        WT=None,
        xi_bin_utils=None,
        do_xi=None,
        store_win=None,
        Win=None,
        wigner_files=None,
        step=None,
        keep_wig3j_in_memory=None,  # xi_win_approx=False,
        client=None,
        scheduler_info=None,
        wigner_step=None,
        skylens_b_xi=None,
        bin_theta_window=None,
        njobs_submit_per_worker=10,
        zarr_parallel_read=25,
        skylens_class0=None,
        skylens_class_b=None,
        bin_window=None,
        do_pseudo_cl=None,
    ):

        self.__dict__.update(
            locals()
        )  # assign all input args to the class as properties
        if self.l_cl is None:
            self.l_cl = self.l
        nl = len(self.l)
        nwl = len(self.window_l) * 1.0

        self.MF = (
            2 * self.l_cl[:, None] + 1
        )  # this is multiplied with coupling matrices, before binning.

        workers, self.nworkers, self.njobs_submit = get_workers_njobs(
            scheduler_info, njobs_submit_per_worker
        )

        self.step = wigner_step
        if self.step is None:
            #             self.step=jnp.int32(200.*((2000./nl)**2)*(100./nwl)) #small step is useful for lower memory load
            self.step = jnp.int32(nl / self.nworkers / 2)
            if nl / self.step < self.nworkers:
                self.step = jnp.int32(nl / self.nworkers)
            self.step = jnp.int32(min(self.step, nl + 1))
            self.step = jnp.int32(max(self.step, 1))

        print(
            "Win gen: step size",
            self.step,
            nl,
            nwl,
            self.nworkers,
            self.use_window,
            self.Win is None,
        )  # ,self.lms)
        self.lms = np.int32(np.arange(nl, step=self.step))
        self.workers_lm = {}
        worker_i = 0

        for lm in self.lms:
            if scheduler_info is not None:
                self.workers_lm[lm] = workers[worker_i % self.nworkers]
            else:
                self.workers_lm[lm] = None
            worker_i += 1

    def get_Win(
        self,
    ):
        if not self.use_window:
            return
        if self.Win is not None:
            print("window exists, will not compute. Size: ", get_size_pickle(self.Win))
            return
        print("Win not found, computing now")
        self.set_binning()

        self.cl_keys = self.set_cl_keys(corrs=None, corr_indxs=None)
        print("get_Win, total cl keys: ", len(self.cl_keys))

        wig_3j_2 = None

        WT_kwargs, xibu = self.get_xi_win_WT()

        self.do_cl = True
        Win_cl = self.set_window_cl(WT_kwargs=WT_kwargs, xi_bin_utils=xibu)
        if self.do_xi:
            self.Win = self.get_xi_win(Win_cl=Win_cl)

        if self.do_pseudo_cl:
            if self.do_xi:
                print("Warning: window for xi is different from cl.")
            wig_3j_2 = self.set_wig3j(self.keep_wig3j_in_memory)
            self.Win = self.get_cl_win(Win_cl=Win_cl, wig_3j_2=wig_3j_2)

        self.cleanup()

    def get_cl_win(self, Win_cl=None, wig_3j_2=None):
        if self.store_win:
            Win = self.set_store_window(
                corrs=self.corrs,
                corr_indxs=self.corr_indxs,
                client=None,
                cl_bin_utils=self.cl_bin_utils,
                Win_cl=Win_cl,
                wig_3j_2=wig_3j_2,
            )
            Win = Win.result()
        else:
            Win = self.set_window_graph(
                corrs=self.corrs,
                corr_indxs=self.corr_indxs,
                client=None,
                cl_bin_utils=self.cl_bin_utils,
                Win_cl=Win_cl,
                wig_3j_2=wig_3j_2,
            )
        return Win

    def get_xi_win_WT(self):
        if not self.do_xi:
            return None, None
        WT_kwargs = {"cl": None}
        client = client_get(self.scheduler_info)

        s1_s2 = (0, 0)
        lcl = client.scatter(self.window_l, broadcast=True)

        WT_kwargs = {
            "cl": {
                "l_cl": lcl,
                "s1_s2": s1_s2,
                "wig_d": self.WT.wig_d[(0, 0)],
                "wig_l": self.WT.l,
                "wig_norm": self.WT.wig_norm,
                "grad_l": self.WT.grad_l,
            }
        }
        WT_kwargs = scatter_dict(
            WT_kwargs, broadcast=True, scheduler_info=self.scheduler_info, depth=2
        )

        xibu = None
        if self.xi_bin_utils is not None:
            xibu = self.xi_bin_utils[(0, 0)]
        return WT_kwargs, xibu

    def get_xi_win(self, Win_cl=None):
        client = client_get(self.scheduler_info)
        Win = delayed(self.combine_coupling_xi)(Win_cl)
        print("Got xi win graph")
        if self.store_win:
            if client is None and self.scheduler_info is None:
                client = get_client()
            elif client is None and self.scheduler_info is not None:
                client = get_client(address=self.scheduler_info["address"])
            Win = client.compute(Win)  # .result()
        return Win

    def set_binning(
        self,
    ):
        """
        Set binning if the coupling matrices or correlation function windows need to be binned.
        """
        self.binning = binning()
        self.c_ell0 = None
        self.c_ell_b = None

        self.cl_bin_utils = None
        if self.bin_window:
            self.binnings = binning()
            self.cl_bin_utils = self.skylens_class0.cl_bin_utils

            self.c_ell0 = self.skylens_class0.cl_tomo()["cl"]
            if self.skylens_class_b is not None:
                self.c_ell_b = self.skylens_class_b.cl_tomo()["cl"]
            else:
                self.c_ell_b = self.skylens_class0.cl_tomo()["cl_b"]
        self.xi0 = None
        self.xi_b = None
        if self.bin_theta_window and self.do_xi:
            self.xi0 = self.skylens_class0.xi_tomo()["xi"]
            self.xi_b = self.skylens_b_xi.xi_tomo()["xi"]
        keys = ["skylens_class0", "skylens_class_b", "skylens_b_xi"]
        for k in keys:
            if hasattr(self, k):
                del self.__dict__[k]

    def wig3j_step_read(self, m=0, lm=None, sem_lock=None):
        """
        wigner matrices are large. so we read them step by step
        """
        step = self.step
        wig_3j = zarr.open(self.wigner_files[m], mode="r")
        if sem_lock is None:
            out = wig_3j.oindex[
                jnp.int32(self.window_l),
                jnp.int32(self.l[lm : lm + step]),
                jnp.int32(self.l_cl),
            ]
        else:
            with sem_lock:
                out = wig_3j.oindex[
                    jnp.int32(self.window_l),
                    jnp.int32(self.l[lm : lm + step]),
                    jnp.int32(self.l_cl),
                ]
        out = out.transpose(1, 2, 0)
        del wig_3j
        return out

    def set_wig3j_step_multiplied(self, lm=None, sem_lock=None):
        """
        product of two partial migner matrices
        """
        wig_3j_2 = {}
        wig_3j_1 = {
            m1: self.wig3j_step_read(m=m1, lm=lm, sem_lock=sem_lock) for m1 in self.m_s
        }
        mi = 0
        for m1 in self.m_s:
            for m2 in self.m_s[mi:]:
                wig_3j_2[str(m1) + str(m2)] = {
                    0: wig_3j_1[m1] * wig_3j_1[m2].astype("float64")
                }  # numpy dot appears to run faster with 64bit ... ????
                #                 if m1!=0 or m2!=0:
                mf = self.set_window_pm_step(lm=lm)
                wig_3j_2[str(m1) + str(m2)][2] = (
                    wig_3j_2[str(m1) + str(m2)][0] * mf["mf_p"]
                )
                wig_3j_2[str(m1) + str(m2)][-2] = wig_3j_2[str(m1) + str(m2)][0] * (
                    1 - mf["mf_p"]
                )
            mi += 1
        #         del wig_3j_1
        #         open_fd = len(psutil.Process().open_files())
        print("got wig3j", lm)  # ,get_size_pickle(wig_3j_2),thread_count(),open_fd)
        return wig_3j_2

    def set_wig3j_step_spin(self, wig2, mf_pm, W_pm):
        """
        wig2 is product of two wigner matrices. Here multply with the spin dependent factors
        """
        if W_pm == 2:  # W_+
            mf = mf_pm[
                "mf_p"
            ]  # .astype('float64') #https://stackoverflow.com/questions/45479363/numpy-multiplying-large-arrays-with-dtype-int8-is-slow
        if W_pm == -2:  # W_+
            mf = 1 - mf_pm["mf_p"]
        return wig2 * mf

    def set_window_pm_step(self, lm=None):
        """
        Here we set the spin dependent multiplicative factors (X+, X-).
        """
        li1 = jnp.int32(self.window_l).reshape(len(self.window_l), 1, 1)
        li3 = jnp.int32(self.l).reshape(1, 1, len(self.l))
        li2 = jnp.int32(self.l[lm : lm + self.step]).reshape(
            1, len(self.l[lm : lm + self.step]), 1
        )
        mf = (-1.0) ** (li1 + li2 + li3)
        mf = mf.transpose(1, 2, 0)
        out = {}
        out["mf_p"] = (1.0 + mf) / 2.0
        # out['mf_p']=jnp.int8((1.+mf)/2.)#.astype('bool')
        # bool doesn't help in itself, as it is also byte size in numpy.
        # we donot need to store mf_n, as it is simply a 0-1 flip or "not" when written as bool
        # using bool or int does cost somewhat in computation as numpy only computes with float 64 (or 32
        # in 32 bit systems). If memory is not an
        # issue, use float64 here and then use mf_n=1-mf_p.
        #         del mf
        return out

    def set_wig3j(self, set_wig_3j_2=False):
        """
        Set up a graph (dask delayed), where nodes to read in partial wigner matrices, get their products and
        also the spin depednent multiplicative factors.
        """
        self.wig_3j = {}
        if not self.use_window:
            return

        m_s = jnp.concatenate(
            [jnp.abs(jnp.array(i)).flatten() for i in self.s1_s2s.values()]
        )
        self.m_s = jnp.sort(jnp.unique(m_s))
        self.m_s = np.array(self.m_s)

        print("wigner_files:", self.wigner_files)

        if self.store_win:
            client = client_get(
                scheduler_info=self.scheduler_info
            )  # this seems to be the correct thing to do

        self.wig_3j_2 = {}
        self.wig_3j_1 = {}
        self.mf_pm = {}
        #         self.sem_lock = Semaphore(max_leases=self.zarr_parallel_read, name="database",client=client_get(self.scheduler_info))
        wig_3j_2 = {lm: None for lm in self.lms}
        mf_pm = {lm: None for lm in self.lms}
        if set_wig_3j_2:
            for lm in self.lms:  # set in get_coupling_lm_all_win
                wig_3j_2[lm] = delayed(self.set_wig3j_step_multiplied)(
                    lm=lm
                )  # ,sem_lock=self.sem_lock)
                wig_3j_2[lm] = client.compute(
                    wig_3j_2[lm], worker=self.workers_lm[lm], allow_other_workers=False
                )
        #             self.mf_pm[lm]=delayed(self.set_window_pm_step)(lm=lm)

        self.wig_s1s2s = {}
        for corr in self.corrs:
            mi = jnp.sort(jnp.absolute(jnp.array(self.s1_s2s[corr])).flatten())
            self.wig_s1s2s[corr] = str(mi[0]) + str(mi[1])
        print("wigner done", self.wig_3j.keys())
        return wig_3j_2

    def coupling_matrix_large(
        self, win, wig_3j_2, mf_pm, bin_wt, W_pm, lm, cl_bin_utils=None
    ):
        """
        get the large coupling matrices from windows power spectra, wigner functions and spin dependent
        multiplicative factors. Also do the binning if called for.
        This function supports on partial matrices.
        """
        wig = wig_3j_2[W_pm]

        M = {}
        for k in win.keys():
            mf = self.MF[lm : lm + self.step, :]

            M[k] = coupling_M(wig, win[k], self.window_l, mf)

            if self.bin_window:  # and bin_wt is not None:
                M[k] = self.binnings.bin_2d_coupling(
                    M=M[k],
                    bin_utils=cl_bin_utils,
                    partial_bin_side=2,
                    lm=lm,
                    lm_step=self.step,
                    wt0=bin_wt[k]["wt0"],
                    wt_b=bin_wt[k]["wt_b"],
                )
                # FIXME: Wrong binning for noise.

        return M

    def mask_comb(self, win1, win2):
        """
        combined the mask from two windows which maybe partially overlapping.
        Useful for some covariance calculations, specially SSC, where we assume a uniform window.
        """
        W = win1 * win2
        x = jnp.logical_or(win1 == hp.UNSEEN, win2 == hp.UNSEEN)
        W[x] = hp.UNSEEN
        W[~x] = 1.0  # mask = 0,1
        fsky = (~x).mean()
        return fsky, W  # .astype('int16')

    def get_cl_coupling_lm(
        self, corr_indxs, win, lm, wig_3j_2_lm, mf_pm, cl_bin_utils=None
    ):
        """
        This function gets the partial coupling matrix given window power spectra and wigner functions.
        Note that it get the matrices for both signal and noise as well as for E/B modes if applicable.
        """
        #         indx=self.cl_keys.index(corr_indxs)
        win0 = win  # [indx]
        win2 = {}
        i = 0
        k = win0["corr"] + win0["indxs"]

        corr = (k[0], k[1])

        wig_3j_2 = wig_3j_2_lm[self.wig_s1s2s[corr]]
        #         win=win0#[i]#[k]
        #             assert win['corr']==corr
        win2 = {
            "M": None,
            "M_noise": None,
            "M_B": None,
            "M_B_noise": None,
            "binning_util": win["binning_util"],
        }
        for kt in ["corr", "indxs", "s1s2"]:
            win2[kt] = win0[kt]
        if lm == 0:
            win2 = copy.deepcopy(win)

        win_M = self.coupling_matrix_large(
            win[12],
            wig_3j_2=wig_3j_2,
            mf_pm=mf_pm,
            bin_wt=win["bin_wt"],
            W_pm=win["W_pm"],
            lm=lm,
            cl_bin_utils=cl_bin_utils,
        )
        win2["M"] = win_M["cl"]
        if "N" in win_M.keys():
            win2["M_noise"] = win_M["N"]
        if (
            win["corr"] == ("shear", "shear") and win["indxs"][0] == win["indxs"][1]
        ):  # B mode.
            win_M_B = self.coupling_matrix_large(
                win[12],
                wig_3j_2=wig_3j_2,
                mf_pm=mf_pm,
                W_pm=-2,
                bin_wt=win["bin_wt"],
                lm=lm,
                cl_bin_utils=cl_bin_utils,
            )
            win2["M_B_noise"] = win_M_B["N"]
            win2["M_B"] = win_M_B["cl"]
        i += 1
        return win2

    def combine_coupling_cl(self, result):
        """
        This function combines the partial coupling matrices computed above. It loops over all combinations of tracers
        and returns a dictionary of coupling matrices for all C_ells.
        """
        dic = {}
        nl = len(self.l)

        if self.bin_window:
            nl = len(self.l_bins) - 1

        for ii_t in range(len(self.cl_keys)):  # list(result[0].keys()):
            ii = ii_t  # 0#because we are deleting below
            ckt = self.cl_keys[ii_t]

            result_ii = result[0][ii]
            corr = result_ii["corr"]
            indxs = result_ii["indxs"]

            result0 = {}
            for k in result_ii.keys():
                result0[k] = result_ii[k]

            result0["M"] = jnp.zeros((nl, nl))
            if result_ii["M_noise"] is not None:
                result0["M_noise"] = jnp.zeros((nl, nl))
            if corr == ("shear", "shear") and indxs[0] == indxs[1]:
                result0["M_B_noise"] = jnp.zeros((nl, nl))
                result0["M_B"] = jnp.zeros((nl, nl))

            for i_lm in range(len(self.lms)):
                # lm=jnp.asscalar(self.lms[i_lm])
                lm = self.lms[i_lm]
                start_i = 0 if self.bin_window else lm
                end_i = nl if self.bin_window else lm + self.step

                result0["M"] = (
                    result0["M"].at[start_i:end_i, :].set(result[lm][ii]["M"])
                )

                if result_ii["M_noise"] is not None:
                    result0["M_noise"] = (
                        result0["M_noise"]
                        .at[start_i:end_i, :]
                        .set(result[lm][ii]["M_noise"])
                    )
                if corr == ("shear", "shear") and indxs[0] == indxs[1]:
                    result0["M_B_noise"] = (
                        result0["M_B_noise"]
                        .at[start_i:end_i, :]
                        .set(result[lm][ii]["M_B_noise"])
                    )
                    result0["M_B"] = (
                        result0["M_B"].at[start_i:end_i, :].set(result[lm][ii]["M_B"])
                    )

            corr21 = corr[::-1]
            if dic.get(corr) is None:
                dic[corr] = {}
            if dic.get(corr21) is None:
                dic[corr21] = {}
            dic[corr][indxs] = result0
            dic[corr[::-1]][indxs[::-1]] = result0
        return dic

    def combine_coupling_xi(self, result):
        dic = {}
        i = 0
        for ii in self.cl_keys:  # list(result.keys()):
            result_ii = result[i]
            corr = result_ii["corr"]
            indxs = result_ii["indxs"]
            corr21 = corr[::-1]
            if dic.get(corr) is None:
                dic[corr] = {}
            if dic.get(corr21) is None:
                dic[corr21] = {}
            dic[corr][indxs] = result_ii
            dic[corr[::-1]][indxs[::-1]] = result_ii
            i += 1
        return dic

    def set_cl_keys(self, corrs=None, corr_indxs=None):
        if corrs is None:
            corrs = self.corrs
        if corr_indxs is None:
            corr_indxs = self.corr_indxs
        cl_keys = [corr + indx for corr in corrs for indx in corr_indxs[corr]]
        return cl_keys

    def set_window_cl(
        self,
        corrs=None,
        corr_indxs=None,
        npartitions=100,
        z_bins=None,
        WT_kwargs=None,
        xi_bin_utils=None,
    ):
        """
        This function sets the graph for computing power spectra of windows
        for both C_ell and covariance matrices.
        """
        if corrs is None:
            corrs = self.corrs
        if corr_indxs is None:
            corr_indxs = self.corr_indxs
        if z_bins is None:
            z_bins = self.z_bins
        if WT_kwargs is None:
            WT_kwargs = {"cl": None}
        t1 = time.time()
        client = client_get(scheduler_info=self.scheduler_info)
        client_func = client.compute  # used later

        if self.store_win and self.do_cl:
            client_func = client.compute  # used later

            if self.c_ell0 is not None:
                self.c_ell0 = client.compute(self.c_ell0).result()
                self.c_ell_b = client.compute(self.c_ell_b).result()
                #                 replicate_dict(self.c_ell0, branching_factor=1,scheduler_info=self.scheduler_info)#doesn't work because we need to change the depth of Future
                #                 replicate_dict(self.c_ell_b, branching_factor=1,scheduler_info=self.scheduler_info)
                self.c_ell0 = scatter_dict(
                    self.c_ell0,
                    scheduler_info=self.scheduler_info,
                    broadcast=True,
                    depth=2,
                )
                self.c_ell_b = scatter_dict(
                    self.c_ell_b,
                    scheduler_info=self.scheduler_info,
                    broadcast=True,
                    depth=2,
                )

            if self.xi0 is not None:
                self.xi0 = client.compute(self.xi0).result()
                self.xi_b = client.compute(self.xi_b).result()
                self.xi0 = scatter_dict(
                    self.xi0,
                    scheduler_info=self.scheduler_info,
                    broadcast=True,
                    depth=2,
                )
                self.xi_b = scatter_dict(
                    self.xi_b,
                    scheduler_info=self.scheduler_info,
                    broadcast=True,
                    depth=2,
                )

        print("set window_cl: cl0,cl_b done", time.time() - t1)
        WU = client.scatter(self, broadcast=True)
        Win_cl = None
        if self.do_cl:

            def Win_cli():
                i = 0
                while i < len(self.cl_keys):
                    ck = self.cl_keys[i]
                    corr = (ck[0], ck[1])
                    indx = (ck[2], ck[3])
                    c_ell0 = (
                        {corr: {indx: self.c_ell0[corr][indx]}}
                        if self.c_ell0 is not None
                        else None
                    )
                    c_ell_b = (
                        {corr: {indx: self.c_ell_b[corr][indx]}}
                        if self.c_ell_b is not None
                        else None
                    )
                    yield delayed(get_window_power_cl)(
                        ck,
                        WU,
                        c_ell0=c_ell0,
                        c_ell_b=c_ell_b,
                        z_bin1=z_bins[ck[0]][ck[2]],
                        z_bin2=z_bins[ck[1]][ck[3]],
                        WT_kwargs=WT_kwargs["cl"],
                        xi_bin_utils=xi_bin_utils,
                    )
                    i += 1

            if self.store_win:
                Win_cl = client_func(list(Win_cli()), priority=1)
            else:
                Win_cl = list(Win_cli())
            print(
                "set window_cl: cl done",
                time.time() - t1,
                get_size_pickle(self),
                get_size_pickle(WT_kwargs),
            )

        njobs = self.njobs_submit
        if self.store_win:
            if self.do_cl:
                wait_futures(Win_cl)
        print("set_window_cl done", time.time() - t1)
        return Win_cl

    # def combine_coupling_cl(self,win_lm):
    #     Win={}
    #     if self.do_cl:
    #         win_cl_lm={lmi:win_lm[lmi]['cl'] for lmi in self.lms}
    #         Win['cl']=self.combine_coupling_cl(win_cl_lm)
    #     return Win

    def combine_coupling_xi(self, win_cl):
        Win = {}
        if self.do_cl:
            Win["cl"] = self.combine_coupling_xi(win_cl)
        return Win

    def set_store_window(
        self,
        corrs=None,
        corr_indxs=None,
        client=None,
        cl_bin_utils=None,
        Win_cl=None,
        wig_3j_2={},
    ):
        """
        This function sets and computes the graph for the coupling matrices. It first calls the function to
        generate graph for window power spectra, which is then combined with the graphs for wigner functions
        to get the final graph.
        """
        if self.store_win:
            client = client_get(
                scheduler_info=self.scheduler_info
            )  # this seems to be the correct thing to do
        print("setting windows, coupling matrices ", client)
        # if self.do_cl:
        #     print('WU dict size: ',dict_size_pickle(self.__dict__,print_prefact='WU dict size: ',depth=2))
        #         self.set_window_cl(corrs=corrs,corr_indxs=corr_indxs,client=client)
        #         print('got window cls, now to coupling matrices.',len(self.cl_keys),len(self.cov_keys),self.Win_cl )

        Win = {}
        WU = client.scatter(self, broadcast=True)
        allow_other_workers = True
        if self.keep_wig3j_in_memory:
            allow_other_workers = False
        if self.do_pseudo_cl:  # this is probably redundant due to if statement in init.
            Win_cl_lm = {}
            Win_lm = {}
            workers = list(client.scheduler_info()["workers"].keys())
            nworkers = len(workers)
            njobs = self.njobs_submit

            worker_i = 0
            job_i = 0
            lm_submitted = []

            t1 = time.time()
            for lm in self.lms:
                Win_cl_lm[lm] = {}
                Win_lm[lm] = {}

                Win_lm[lm] = delayed(get_coupling_lm_all_win)(
                    WU, Win_cl, lm, wig_3j_2.get(lm), None, cl_bin_utils=cl_bin_utils
                )
                #### Donot delete
                if self.store_win:
                    client_func = client.compute
                    Win_lm[lm] = client_func(
                        Win_lm[lm],
                        priority=2,
                        workers=self.workers_lm[lm],
                        allow_other_workers=allow_other_workers,
                    )
                    worker_i += 1
                    job_i += 1
                    lm_submitted += [lm]
            Win = delayed(combine_coupling_cl)(WU, Win_lm)

            if self.store_win:
                Win = client.compute(
                    Win,
                    priority=2,
                    workers=self.workers_lm[0],
                    allow_other_workers=allow_other_workers,
                )  # .result()
                if len(lm_submitted) >= nworkers:
                    wait_futures(
                        Win_lm,
                        threshold=max(0.5, 1 - nworkers * 1.2 / len(lm_submitted)),
                    )
                self.cleanup()
        print("done combine lm", time.time() - t1)
        return Win

    def reduce_win_cl(self, win, win2):

        dic = win
        corr = win2["corr"]
        corr2 = corr[::-1]
        indxs = win2["indxs"]
        indxs2 = indxs[::-1]
        if dic.get(corr) is None:
            self.Win["cl"][corr] = {}
        if dic.get(corr2) is None:
            self.Win["cl"][corr2] = {}
        dic[corr2][indxs2] = win2
        dic[corr][indxs] = win2
        return dic

    def cleanup(
        self,
    ):  # need to free all references to wigner_3j, mf and wigner_3j_2... this doesnot help with peak memory usage
        pass

def get_cl_coupling_lm(WU, lm, wig_3j_2_lm, mf_pm, win, cl_bin_utils=None):
    #     print('get_cl_coupling_lm',win,WU,lm)
    return WU.get_cl_coupling_lm(
        None, win, lm, wig_3j_2_lm, mf_pm, cl_bin_utils=cl_bin_utils
    )

def combine_coupling_cl(WU, win_lm):  # win_cl_lm,win_cov_lm):
    t1 = time.time()
    self = WU
    Win = {}
    #         client=client_get(scheduler_info=self.scheduler_info)
    if self.do_cl:
        win_cl_lm = {lmi: win_lm[lmi]["cl"] for lmi in self.lms}
        # try:
        Win["cl"] = self.combine_coupling_cl(win_cl_lm)
        # except Exception as err:
        #     # print('combine_coupling_cl error: ',win_cl_lm[0][0])
        #     print('combine_coupling_cl error: ',err,Win)
        #     crash
        del win_cl_lm
    del win_lm, WU
    print("done combine lm", time.time() - t1)
    return Win

def get_coupling_lm_all_win(WU, Win_cl, lm, wig_3j_2_lm, mf_pm, cl_bin_utils=None):
    self = WU
    t1 = time.time()
    print("get_coupling_lm_all_win", lm)
    if wig_3j_2_lm is None:
        wig_3j_2_lm = self.set_wig3j_step_multiplied(lm=lm)  # ,sem_lock=self.sem_lock)
        mf_pm = None

    Win_lm = {}
    thread_workers = 5
    if self.do_cl:
        Win_lm["cl"] = [
            self.get_cl_coupling_lm(
                None, Wc, lm, wig_3j_2_lm, mf_pm, cl_bin_utils=cl_bin_utils
            )
            for Wc in Win_cl
        ]
    print("done lm", lm, time.time() - t1, flush=True)
    return Win_lm

def get_window_power_cl(
    corr_indxs,
    WU,
    c_ell0=None,
    c_ell_b=None,
    z_bin1=None,
    z_bin2=None,
    WT_kwargs=None,
    xi_bin_utils=None,
):  # corr={},indxs={}):
    """
    Get the cross power spectra of windows given two tracers.
    Note that noise and signal have different windows and power spectra for both
    cases.
    Spin factors and binning weights if needed are also set here.
    """
    #         print('getting window power cl',WT_kwargs)
    self = WU
    corr = (corr_indxs[0], corr_indxs[1])
    indxs = (corr_indxs[2], corr_indxs[3])
    win = {}
    win["corr"] = corr
    win["indxs"] = indxs

    s1s2 = jnp.absolute(jnp.array(self.s1_s2s[corr])).flatten()

    W_pm = 0
    if jnp.sum(s1s2) != 0:
        W_pm = 2  # we only deal with E mode\
        if corr == ("shearB", "shearB"):
            W_pm = -2
    if self.do_xi:
        W_pm = 0  # for xi estimators, there is no +/-. Note that this will result in wrong thing for pseudo-C_ell.
        # FIXME: hence pseudo-C_ell and xi together are not supported right now

    win[12] = {}  # to keep some naming uniformity with the covariance window
    win[12]["cl"] = jnp.asarray(
        hp.anafast(map1=z_bin1["window"], map2=z_bin2["window"], lmax=self.window_lmax)
    )[self.window_l]

    if corr[0] == corr[1] and indxs[0] == indxs[1]:
        map1 = z_bin1["window_N"]
        if map1 is None:
            map1 = jnp.sqrt(z_bin1["window"])
            mask = z_bin1["window"] == hp.UNSEEN
            map1[mask] = hp.UNSEEN
        win[12]["N"] = jnp.asarray(hp.anafast(map1=map1, lmax=self.window_lmax))[
            self.window_l
        ]

    win["binning_util"] = None
    win["bin_wt"] = None
    if self.bin_window and self.do_pseudo_cl:
        cl0 = c_ell0[corr][indxs]
        cl_b = c_ell_b[corr][indxs]
        win["bin_wt"] = {}
        win["bin_wt"]["cl"] = {"wt_b": 1.0 / cl_b, "wt0": cl0}
        win["bin_wt"]["N"] = {"wt_b": jnp.ones_like(cl_b), "wt0": jnp.ones_like(cl0)}
        if jnp.all(cl_b == 0):  # avoid nan
            win["bin_wt"]["cl"] = {"wt_b": cl_b * 0, "wt0": cl0 * 0}
            win["bin_wt"]["N"] = {"wt_b": cl_b * 0, "wt0": cl0 * 0}
    win["W_pm"] = W_pm
    win["s1s2"] = s1s2
    if self.do_xi:
        th, win["xi"] = self.WT.projected_correlation(
            cl=win[12]["cl"], **WT_kwargs
        )  # this is ~f_sky
        win["xi_b"] = self.binning.bin_1d(
            xi=win["xi"], bin_utils=xi_bin_utils
        )  # xi_bin_utils[(0,0)]

    win[
        "M"
    ] = (
        {}
    )  # self.coupling_matrix_large(win['cl'], s1s2,wig_3j_2=wig_3j_2,W_pm=W_pm)*(2*self.l[:,None]+1) #FIXME: check ordering
    win["M_noise"] = None
    return win

def multiply_window(win1, win2):
    """
    Take product of two windows which maybe partially overlapping and mask it properly.
    """
    W = win1 * win2
    x = jnp.logical_or(win1 == hp.UNSEEN, win2 == hp.UNSEEN)
    W[x] = hp.UNSEEN
    return W

# @jit
def coupling_M(wig, win, window_l, MF):
    # print('coupling_M:',wig.shape,win.shape,window_l.shape,MF.shape)
    # M=wig@(win*(2*window_l+1))
    M = jnp.dot(wig, win * (2 * window_l + 1))
    M /= 4.0 * jnp.pi
    M *= MF
    return M


import prep
import torchvision
import torch
import time

traindir = "/data/pytorch-imagenet-data/train/"
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                        std=[0.229, 0.224, 0.225])
transform = transforms.Compose([
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        normalize,
])

def pil_loader(path):
    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
    with open(path, 'rb') as f:
        img = Image.open(f)
        return img.convert('RGB')

train_dataset = torchvision.datasets.ImageFolder(traindir, prep.transform, loader=fakeloader)

train_loader = torch.utils.data.DataLoader(
        train_dataset,
        batch_size=prep.batch_size, shuffle=True,
        num_workers=prep.worker, pin_memory=True)

count = 0
end = time.time()
with open("loading/pytorch-realloader-"+str(prep.worker)+".txt","w") as f:
        for i, (input, target) in enumerate(train_loader):
                cost = time.time()-end
                end = time.time()
                if prep.sleep:
                        time.sleep(prep.sleep_time)
                print(cost, "\t", prep.batch_size/cost, sep="", file=f)
                count += 1
                if count == prep.limit:
                        break


import numpy as np
import networkx as nx
import torch
from scipy.linalg import fractional_matrix_power, inv
import scipy.sparse as sp
import random
import yaml
import logging

def load_best_configs(args, path):
    with open(path, "r") as f:
        configs = yaml.load(f, yaml.FullLoader)

    if args.dataset not in configs:
        print("Best args not found")
        return args

    # print("Using best configs")
    configs = configs[args.dataset]

    for k, v in configs.items():
        if "lr" in k or "weight_decay" in k:
            v = float(v)
        setattr(args, k, v)
    print("------ Use best configs ------")
    return args

import argparse
def build_args():
    parser = argparse.ArgumentParser(description='model', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--dataset', type=str, default='cora')
    parser.add_argument('--seed', type=int, default=999)
    parser.add_argument('--sample_size', type=int, default=2708)
    parser.add_argument('--batch_size', type=int, default=1)
    parser.add_argument('--gamma', type=float, default=1)
    parser.add_argument('--sigma', type=float, default=0.001)
    parser.add_argument('--tau_feat', type=float, default=1.0)
    parser.add_argument('--tau_node', type=float, default=10.0)
    parser.add_argument('--nb_epochs', type=int, default=500)
    parser.add_argument('--patience', type=int, default=30)
    parser.add_argument('--cnt_wait', type=int, default=0)
    parser.add_argument('--best_t', type=int, default=0)
    parser.add_argument('--lr', type=float, default=0.001)
    parser.add_argument('--l2_coef', type=float, default=0.0)
    parser.add_argument('--best', type=float, default=1e9)
    parser.add_argument('--hid_units', type=int, default=512)
    parser.add_argument('--self_sample', type=int, default=50)
    parser.add_argument('--max_epoch', type=int, default=300)
    parser.add_argument('--sparse', type=bool, default=False)
    parser.add_argument('--verbose', type=bool, default=False)
    args = parser.parse_args()
    return args

def get_A_r(adj_label, r):
    if r == 1:
        adj_label = adj_label
    elif r == 2:
        adj_label = adj_label @ adj_label
    elif r == 3:
        adj_label = adj_label @ adj_label @ adj_label
    elif r == 4:
        adj_label = adj_label @ adj_label @ adj_label @ adj_label
    return adj_label

def setup_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)
    torch.manual_seed(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

def compute_ppr(a, alpha=0.2, self_loop=True):
    if self_loop:
        a = a + np.eye(a.shape[0])  # A^ = A + I_n
    d = np.diag(np.sum(a, 1))  # D^ = Sigma A^_ii
    dinv = fractional_matrix_power(d, -0.5)  # D^(-1/2)
    at = np.matmul(np.matmul(dinv, a), dinv)  # A~ = D^(-1/2) x A^ x D^(-1/2)
    return alpha * inv((np.eye(a.shape[0]) - (1 - alpha) * at))  # a(I_n-(1-a)A~)^-1

def compute_heat(graph: nx.Graph, t=5, self_loop=True):
    a = nx.convert_matrix.to_numpy_array(graph)
    if self_loop:
        a = a + np.eye(a.shape[0])
    d = np.diag(np.sum(a, 1))
    return np.exp(t * (np.matmul(a, inv(d)) - 1))

def sparse_to_tuple(sparse_mx):
    """Convert sparse matrix to tuple representation."""

    def to_tuple(mx):
        if not sp.isspmatrix_coo(mx):
            mx = mx.tocoo()
        coords = np.vstack((mx.row, mx.col)).transpose()
        values = mx.data
        shape = mx.shape
        return coords, values, shape

    if isinstance(sparse_mx, list):
        for i in range(len(sparse_mx)):
            sparse_mx[i] = to_tuple(sparse_mx[i])
    else:
        sparse_mx = to_tuple(sparse_mx)

    return sparse_mx

def preprocess_features(features):
    """Row-normalize feature matrix and convert to tuple representation"""
    rowsum = np.array(features.sum(1))
    r_inv = np.power(rowsum, -1).flatten()
    r_inv[np.isinf(r_inv)] = 0.
    r_mat_inv = sp.diags(r_inv)
    features = r_mat_inv.dot(features)
    if isinstance(features, np.ndarray):
        return features
    else:
        return features.todense(), sparse_to_tuple(features)

def select_DatasSet(labels):
    label_set = {}
    for i, label in enumerate(labels, 0):
        if label not in label_set.keys():
            label_set[label] = [i]
        else:
            label_set[label].extend([i])

    train_id = []
    valid_id = []
    test_id = []

    for key, value in label_set.items():

        train_id.extend(value[:int(len(value) * 0.1)])
        valid_id.extend(value[int(len(value) * 0.1): int(len(value) * 0.2)])
        test_id.extend(value[int(len(value) * 0.2):])

    return np.array(train_id), np.array(valid_id), np.array(test_id)

def normalize_adj(adj, self_loop=True):
    """Symmetrically normalize adjacency matrix."""
    if self_loop:
        adj = adj + sp.eye(adj.shape[0])
    adj = sp.coo_matrix(adj)
    rowsum = np.array(adj.sum(1))
    d_inv_sqrt = np.power(rowsum, -0.5).flatten()
    d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0.
    d_mat_inv_sqrt = sp.diags(d_inv_sqrt)
    return adj.dot(d_mat_inv_sqrt).transpose().dot(d_mat_inv_sqrt).tocoo()

def sparse_mx_to_torch_sparse_tensor(sparse_mx):
    """Convert a scipy sparse matrix to a torch sparse tensor."""
    sparse_mx = sparse_mx.tocoo().astype(np.float32)
    indices = torch.from_numpy(
        np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64))
    values = torch.from_numpy(sparse_mx.data)
    shape = torch.Size(sparse_mx.shape)
    return torch.sparse.FloatTensor(indices, values, shape)


"""
Template for running simulations.
"""

import os
import sys
import time
import copy
import json

import numpy as np
import pandas as pd
from context import pyblip, blip_sims
from blip_sims import parser
from blip_sims import utilities
from blip_sims.gen_data import generate_regression_data

from sklearn import linear_model
import scipy.cluster.hierarchy as hierarchy

# Global seed for dgp
DGP_SEED = 111

# Specifies the type of simulation
DIR_TYPE = os.path.split(os.path.abspath(__file__))[1].split(".py")[0]

# columns of dataframe
COLUMNS = [
	'method',
	'bsize',
	'model_time',
	'blip_time',
	'power',
	'nfd',
	'fdr',
	'well_specified',
	'nsample',
	'chains',
	'y_dist',
	'covmethod',
	'kappa',
	'p',
	'sparsity',
	'k',
	'rank',
	'coeff_size',
	'seed',
]

def single_seed_sim(
	seed,
	y_dist,
	covmethod,
	kappa,
	p,
	sparsity,
	k,
	rank,
	coeff_size,
	args,
	tol=1e-3
):
	np.random.seed(seed)
	output = []
	dgp_args = [
		y_dist, covmethod, kappa, p, sparsity, k, rank, coeff_size, seed
	]

	# Create data
	n = int(kappa * p)
	X, y, beta = generate_regression_data(
		y_dist=y_dist,
		covmethod=covmethod,
		n=n,
		p=p,
		sparsity=sparsity,
		k=k,
		rank=rank,
		coeff_dist=args.get('coeff_dist', ['normal'])[0],
		coeff_size=coeff_size,
		min_coeff=args.get('min_coeff', [0.1 * coeff_size])[0],
		dgp_seed=DGP_SEED
	)

	q = args.get('q', [0.1])[0]
	for well_specified in args.get('well_specified', [True]):
		if well_specified:
			p0 = 1 - sparsity
			min_p0 = 0
			tau2 = coeff_size
			sigma2 = 1
			update = False
		else:
			p0 = 0.99
			min_p0 = 0.9
			tau2 = 1
			sigma2 = 1
			update = True

		# Method type 1: BLiP + SpikeSlab
		# Run Gaussian sampler if (1) not well-specified or (2) gaussian response
		if y_dist == 'gaussian' or not well_specified:
			lss_model = pyblip.linear.LinearSpikeSlab(
				X=X,
				y=y.astype(np.float64),
				p0=p0,
				min_p0=min_p0,
				update_p0=update,
				tau2=tau2,
				update_tau2=update,
				sigma2=sigma2,
				update_sigma2=update,
			)
			models = [lss_model]
			method_names = ['LSS + BLiP']
		else:
			models = []
			method_names = []
		if y_dist != 'gaussian':
			models.append(pyblip.probit.ProbitSpikeSlab(
				X=X,
				y=y,
				p0=p0,
				min_p0=min_p0,
				update_p0=update,
				tau2=tau2,
				update_tau2=update,
				sigma2=sigma2,
				update_sigma2=update,
			))
			method_names.append('PSS + BLiP')
		for nsample in args.get("nsample", [1000]):
			for model, mname in zip(models, method_names):
				for bsize in args.get("bsize", [1, 3, 5]):
					chains_vals = args.get("chains", [10])
					max_chains = max(chains_vals)
					t0 = time.time()
					model.sample(
						N=nsample,
						chains=max_chains,
						burn=int(0.1*nsample),
						bsize=bsize
					)
					mtime = time.time() - t0
					for chains in chains_vals:
						t0 = time.time()
						inclusions = model.betas[0:int(chains*nsample)] != 0
						max_pep = args.get('max_pep', [2*q])[0]
						max_size = args.get('max_size', [25])[0]
						prenarrow = args.get('prenarrow', [0])[0]
						cand_groups = pyblip.create_groups.all_cand_groups(
								samples=inclusions,
								X=X,
								max_pep=max_pep,
								max_size=max_size,
						)

						# Run BLiP
						detections = pyblip.blip.BLiP(
							cand_groups=cand_groups,
							q=q,
							error='fdr',
							max_pep=max_pep,
							perturb=True,
							deterministic=True
						)
						blip_time = time.time() - t0
						nfd, fdr, power = utilities.nodrej2power(detections, beta)
						output.append(
							[mname, bsize, chains * mtime / max_chains, blip_time, power, nfd, fdr, well_specified, nsample, chains] + dgp_args
						)

	# Method Type 2: susie-based methods
	if args.get('run_susie', [False])[0]:
		t0 = time.time()
		susie_alphas, susie_sets = blip_sims.susie.run_susie(
			X, y, L=np.ceil(p*sparsity), q=q
		)
		susie_time = time.time() - t0
		nfd, fdr, power = blip_sims.utilities.rejset_power(
			susie_sets, beta
		)
		output.append(
			['susie', susie_time, 0, power, nfd, fdr, True, 0] + dgp_args
		)
		# Now apply BLiP on top of susie
		t0 = time.time()
		cand_groups = pyblip.create_groups.susie_groups(
			alphas=susie_alphas,
			X=X,
			q=q,
			prenarrow=prenarrow,
			max_size=max_size
		)
		detections = pyblip.blip.BLiP(
			cand_groups=cand_groups,
			q=q,
			error='fdr',
			max_pep=max_pep,
			perturb=True,
			deterministic=True
		)
		blip_time = time.time() - t0
		nfd, fdr, power = utilities.nodrej2power(detections, beta)
		output.append(
			['susie + BLiP', 'all', susie_time, blip_time, power, nfd, fdr, True, 0, 0] + dgp_args
		)

	# # Frequentist methods
	# if kappa > 1:
	# 	t0 = time.time()
	# 	regtree = blip_sims.tree_methods.RegressionTree(
	# 		X=X, y=y, levels=args.get('levels', [10])[0], max_size=max_size
	# 	)
	# 	regtree.fit(family='gaussian') # this works better than using family = binomial
	# 	# even when y follows a binomial distribution
	# 	mtime = time.time() - t0
	# 	_, rej_yek = regtree.ptree.outer_nodes_yekutieli(q=q)
	# 	rej_fbh, _ = regtree.ptree.tree_fbh(q=q)
	# 	for mname, rej in zip(['FBH', 'Yekutieli'], [rej_fbh, rej_yek]):
	# 		nfd, fdr, power = utilities.nodrej2power(rej, beta)
	# 		output.append(
	# 			[mname, mtime, 0, power, nfd, fdr, True, 0] + dgp_args
	# 		)
	return output

def main(args):
	# Parse arguments
	args = parser.parse_args(args)
	print(args['description'])
	sys.stdout.flush()
	reps = args.get('reps', [1])[0]
	num_processes = args.get('num_processes', [1])[0]

	# Save args, create output dir
	output_dir, today, hour = utilities.create_output_directory(
		args, dir_type=DIR_TYPE, return_date=True
	)
	result_path = output_dir + "/results.csv"

	# Run outputs
	time0 = time.time()
	all_outputs = []
	seed_start = max(args.get('seed_start', [1])[0], 1)
	for covmethod in args.get('covmethod', ['ark']):
		for p in args.get('p', [500]):
			for kappa in args.get('kappa',[0.2]):
				for sparsity in args.get('sparsity', [0.05]):
					for k in args.get('k', [1]):
						for y_dist in args.get('y_dist', ['gaussian']):
							for coeff_size in args.get('coeff_size', [1]):
								for rank in args.get('rank', [1]):
									sample_kwargs=dict(
										y_dist=y_dist,
										covmethod=covmethod,
										kappa=kappa,
										p=p,
										sparsity=sparsity,
										k=k,
										rank=rank,
										coeff_size=coeff_size,
									)
									# Inputs to multiprocessing
									constant_inputs = sample_kwargs.copy()
									constant_inputs['args'] = args
									seeds = list(range(seed_start, reps+seed_start))
									outputs = utilities.apply_pool(
										func=single_seed_sim,
										seed=seeds,
										constant_inputs=constant_inputs,
										num_processes=num_processes,
									)
									for out in outputs:
										all_outputs.extend(out)
									msg = f"Finished with {sample_kwargs}"
									msg += f" at {np.around(time.time() - time0, 2)}"
									print(msg)

									# Save
									out_df = pd.DataFrame(all_outputs, columns=COLUMNS)
									out_df.to_csv(result_path, index=False)
									groupers = [
										'method', 'bsize', 'chains', 'y_dist', 'covmethod', 'kappa', 'rank',
										'p', 'sparsity', 'k', 'well_specified', 'nsample',
									]
									meas = ['model_time', 'power', 'fdr', 'blip_time']
									summary_df = out_df.groupby(groupers)[meas].mean()
									print(summary_df.reset_index())

if __name__ == '__main__':
	main(sys.argv)


from utils.args import args
from utils.formats import get_custom_format

from ast import literal_eval

import numpy as np

def convert_to_float(v, format):
    if not 'nan' in v:
        v = '0x' + v
        CustomData = get_custom_format(format + '32')
        data = CustomData(0.0)
        d_t = data.from_bits(literal_eval(v))
        t = float(d_t)
        return t
    return np.nan

def convert_hex_to_bin(v):
    return bin(literal_eval('0x' + v)).replace('0b', '')

def max_bit_changed(value):
    if '.' in str(value):
        value = str(str(value).split('.')[0])
    if not 'nan' in value:
        value_bin = convert_hex_to_bin(value)
        size = len(value_bin)
        missing = args.bits - size
        value_bin = '0' * missing + value_bin
        pos = args.bits - 1
        for bin_data in value_bin:
            if not bin_data == '1':
                pos -= 1
            else:
                return pos
    else:
        return np.nan

def bits_chaged(value):
    if '.' in str(value):
        value = str(str(value).split('.')[0])
    if not 'nan' in value:
        value_bin = convert_hex_to_bin(value)
        size = len(value_bin)
        missing = args.bits - size
        value_bin = '0' * missing + value_bin
        value_bin = value_bin[::-1]
        indexes = [
            index for index in range(len(value_bin))
            if value_bin.startswith('1', index)
        ]
        return indexes

def relative_error(real, fault):
    return np.abs(np.abs(real - fault) / real) * 100.0

def abs_error(real, fault):
    return np.abs(real - fault)

def num_bits_changed(bits_):
    return len(bits_)


from config import config
from main_profile import *
import psycopg2

def app():
    try:
        params = config()
        connection = psycopg2.connect(**params)
        cursor = connection.cursor()
        postgreSQL_select_Query = "select * from names;"

        cursor.execute(postgreSQL_select_Query)
        print("Selecting rows from names table using cursor.fetchall")
        profile_records = cursor.fetchall()

        print("Print each row and it's columns values")
        for row in profile_records:
            profile = scrape_infos(row[2])
            cursor.execute("INSERT INTO profiles (full_name, post, post2, email, location, phoneT, related_services, related_sectors) VALUES(%s, %s, %s, %s, %s, %s, %s, %s)", (profile["full_name"], profile["post"], profile["post2"], profile["email"], profile["location"], profile["phoneT"], profile["related_services"], profile["related_sectors"]))
            connection.commit()

    except (Exception, psycopg2.Error) as error:
        print("Error while fetching data from PostgreSQL", error)
        pass

    finally:
        # closing database connection.
        if connection:

            cursor.close()
            connection.close()
            print("PostgreSQL connection is closed")

if __name__ == "__main__":
    app()


def sum_two(a, b):
    return a + b

def min_(a, b):
    if a < b:
        return a
    return b

def min_multi(*args):
    if len(args) == 0:
        raise ValueError()
    if len(args) == 1:
        return args[0]
    rv = args[0]
    for elem in args[1:]:
        rv = min_(rv, elem)
    return rv

if __name__ == '__main__':
    print("hello")
    print("me")
    print("another feature")


import pathlib

import setuptools

README = (pathlib.Path(__file__).parent / 'README.md').read_text()

install_requires = [
    'pynvml==11.5.0',
]

setuptools.setup(
    name='gpu_affinity',
    version='0.1.0',
    author='NVIDIA Corporation',
    description='GPU Affinity',
    long_description=README,
    long_description_content_type='text/markdown',
    url='',
    packages=['gpu_affinity'],
    install_package_data=True,
    install_requires=install_requires,
    license='Apache2',
    license_file='./LICENSE',
    classifiers=[
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.6',
        'Operating System :: POSIX :: Linux',
    ],
    python_requires='>=3.6',
    extras_require={
        'dev': [
            'black==23.10.1',
            'isort==5.10.1',
            'pre-commit==3.5.0',
        ],
    },
)


import lavatar.lavatar as lavatar
from openface_configuration import *

CSV_PATH = "../openface/smile.csv"
WAV_PATH = ""
VIDEO_PATH = ""
JSON_PATH = "../godot/assets/json/smile.json"

indices = None
frames = []

def load_csv( path ):

	global indices

	raw_file = open( path )

	for line in raw_file:
		line = line.strip()
		if len(line) == 0:
			continue
		if indices == None:
			indices = extract_indices( line )
			if indices == None:
				return
		else:
			f = lavatar.parse_text_frame( line.split( FIELD_SEPARATOR ), indices, MAT_TRANSLATION, MAT_ROTATION )
			if not f['valid']:
				continue
			frames.append( f )

load_csv( CSV_PATH )

animation = lavatar.pack_animation( frames, indices )
lavatar.add_structure( animation, STRUCTURE )
lavatar.add_sound( animation, WAV_PATH )
#lavatar.add_video( animation, VIDEO_PATH )
lavatar.save_animation_json( animation, JSON_PATH )


# Import necessary libraries
from wordcloud import WordCloud
import matplotlib.pyplot as plt

# Text data
def choose_file():
    text = input("filename: ")
    with open(text,'r') as f:
        text = f.read()
        return text

# Generate word cloud
def customize_wordcloud():
    bg_color = input("background color: ")
    font = int(input("font size: "))
    wordcloud = WordCloud(background_color=bg_color, max_font_size=font).generate(choose_file())
    return wordcloud

# Save word cloud
def save_cloud():
    file_name = input("enter file name to save: ")
    return file_name

# Plot word cloud
plt.imshow(customize_wordcloud(), interpolation='bilinear')
plt.axis("off")
plt.savefig(save_cloud())
plt.show()


import dash_core_components as dcc
import dash_html_components as html

import dash_bootstrap_components as dbc

import pathlib
import page_builder as pb
from settings import *
from charts import empty_chart

chapter_num = '3.6'
bannerImgSrc = IMAGES_PATH+'OceanicSections/Inorganic Carbon_Tomas Szumski.jpg'
bannerImgCredit = 'Tomasz Szumski'

introText = """
        The ocean absorbs significant quantities of carbon through natural cycles
        driven by ocean circulation, biogeochemistry and biology.
        It is estimated that the ocean has absorbed between 20–30% of total anthropogenic CO\u2082
        emissions since the 1980s thereby reducing atmospheric accumulation and thus partially
        mitigating global warming.
        """
bulletPoint1 = """
        Estimates of future carbon dioxide levels indicate that by the end of this
        century the surface waters of the ocean could be nearly 150% more acidic,
        a condition not experienced for more than 20 million years.
        """
bulletPoint2 = """
        Measurements of surface waters in the Rockall Trough to the west of Ireland between
        1991 and 2013 indicate increasing acidity  comparable to the rate of change in other ocean time series.
        """
bulletPoints = [bulletPoint1, bulletPoint2]
trendChartTitle = ''
trendChart = empty_chart()

# trendCaption = """
# Deep-water dissolved inorganic carbon concentration (µmol/kg) section for the Rockall Trough in the winter of 2013 (left),
# during the annual survey on the Irish Shelf (right). The rows of vertical dots represent sampling points at different depths (left)
# and correspond to the blue point locations indicated in the red polygon (right). Measurements in the surface waters of the Rockall
# Trough between 1991 and 2013 indicated an increase in anthropogenic dissolved inorganic carbon, equivalent to a decrease of 0.050 pH units.
# This represents an increase in acidity that is comparable to the rate of change in other ocean time series.
#         """

infrastructureText = """
The Marine Institute and NUI Galway have undertaken measurements of the inorganic carbon system and associated biogeochemical
parameters in Irish coastal, shelf and offshore waters. Since 2009, Marine Institute annual repeat ship-based hydrography
surveys measuring ECVs in Irish Shelf waters and across the southern Rockall Trough have included water sampling for dissolved
inorganic carbon and total alkalinity.

        """
infrastructureMap = empty_chart()

infoLinks = [
    {'text': 'Inorganic Carbon ESSENTIAL CLIMATE VARIABLE (ECV). GCOS FACTSHEETS',
     'url': 'https://gcos.wmo.int/en/essential-climate-variables/inorganic-carbon/'},

    {'text': 'Ocean Acidification Observing in Coastal, Shelf and Ocean Waters around Ireland, (2019) E. McGovern, T. McGrath, M. Cronin,  G. O’Donnell, B. Ward, C. Cusack, R. Cave, Poster at Conference: 4th Global Ocean Acidification Observing Network (GOA-ON) International Workshop, Hangzhou',
     'url': 'https://www.researchgate.net/publication/332950133_Ocean_Acidification_Observing_in_Coastal_Shelf_and_Ocean_Waters_around_Ireland'},

    {'text': 'Information about ICES',
     'url': 'https://www.ices.dk/'},

    {'text': 'Surface Ocean CO\u2082 Atlas',
     'url': 'http://www.socat.info'},

    {'text': 'The GO-SHIP programme',
     'url': 'https://www.go-ship.org/About.html'},

    {'text': 'The Marine Institute operated Research Vessels',
     'url': 'https://www.marine.ie/Home/site-area/infrastructure-facilities/research-vessels/research-vessels'},
]

########################################################################################################################
chapter_dict = next(
    (item for item in CHAPTERS if item['chapter-num'] == chapter_num), None)

trendTitle1 = 'Deep-water Dissolved Inorganic Carbon Concentration'
figure_3_13 = IMAGES_PATH + \
    'OceanicSections/Figure3.13_DeepWater_DissolvedInorganicConcentration_RockallTrough.png'
trendCaption1 = """
Deep-water dissolved inorganic carbon concentration (µmol/kg) section for the Rockall Trough in the winter of 2013 (left),
during the annual survey on the Irish Shelf (right). The rows of vertical dots represent sampling points at different depths (left)
and correspond to the blue point locations indicated in the red polygon (right). Measurements in the surface waters of the Rockall
Trough between 1991 and 2013 indicated an increase in anthropogenic dissolved inorganic carbon, equivalent to a decrease of 0.050 pH units.
This represents an increase in acidity that is comparable to the rate of change in other ocean time series.
"""
trendTitle2 = 'Surface Water Dissolved Inorganic Carbon Concentration'
figure_3_14_a = IMAGES_PATH+'OceanicSections/HR_Final_ Winter2017_18 shelf pCO2_2.jpg'
figure_3_14_b = IMAGES_PATH+'OceanicSections/HR_Final_Summer 2018_final_2.jpg'
trendCaption2 = """
Surface water pCO\u2082 measurements in a) Winter 2017-18 and b) Summer 2018, as collected by the RV Celtic Explorer.
Temporal (seasonal) variations due to increasing primary production, e.g. photosynthesis from the spring phytoplankton
bloom and shell formation from growth of organisms, results in increased uptake of pCO\u2082 from water, giving lower surface
seawater pCO\u2082 in spring and summer periods. In addition, warmer water can hold less dissolved CO\u2082. Conversely,
reduced primary production and the seasonal die-off of phytoplankton in autumn, together with the gradual remineralisation of
organic matter, releases CO\u2082 back into the surface waters and therefore the highest concentrations are observed in winter periods.
"""

custom_trend = dbc.Container(
    className='sr-trends',
    style={'borderColor': chapter_dict['domain-color']},
    id='trends',
    children=[
        html.H3(
            className='sr-section-heading',
            style={'color': chapter_dict['domain-color']},
            children='Trends',
        ),
        dbc.Row(
            children=[
                dbc.Col(className="col-md-10 offset-md-1",
                        children=[
                            html.H4(
                                className='sr-chart-title',
                                children=trendTitle1),
                            html.Img(
                                className='w-100',
                                src=figure_3_13
                            )
                        ]
                        )
            ]
        ),
        dbc.Row(
            children=[
                dbc.Col(
                    className="col-md-10 offset-md-1",
                    children=[
                        html.P(
                            className='sr-chart-caption',
                            children=[html.I(className="fas fa-play _up",
                                             style={"color": chapter_dict['domain-color']}),
                                      trendCaption1]
                        )]
                )
            ]
        ),
        dbc.Row(
            children=[
                dbc.Col(className="col-12",
                        children=[
                            html.H4(
                                className='sr-chart-title',
                                children=trendTitle2)]
                        ),
                dbc.Col(className="col-12 col-md-6 text-center",
                        children=[
                            html.H4(
                                className='sr-chart-title',
                                children='Winter 2017-18 '),
                            html.Img(
                                style={"marginTop": "-20px",
                                       "position": "relative",
                                       "z-index": "-1"},
                                src=figure_3_14_a
                            )]
                        ),
                dbc.Col(className="col-12 col-md-6 text-center",
                        children=[
                            html.H4(
                                className='sr-chart-title',
                                children='Summer 2018'),
                            html.Img(
                                style={"marginTop": "-20px",
                                       "position": "relative",
                                       "z-index": "-1"},
                                src=figure_3_14_b
                            )
                        ]
                        )
            ]
        ),
        dbc.Row(
            children=[
                dbc.Col(
                    className="col-md-10 offset-md-1",
                    children=[
                        html.P(
                            className='sr-chart-caption',
                            style={"marginTop": "-20px",
                                   "position": "relative", },
                            children=[html.I(className="fas fa-play _up",
                                             style={"color": chapter_dict['domain-color']}),
                                      trendCaption2]
                        )]
                )
            ]
        ),

    ])

custom_infrastructure = dbc.Container(
    className='sr-infrastructure',
    style={'borderColor': chapter_dict['domain-color']},
    id='infrastructure',
    children=[
        html.H3(
            className='sr-section-heading',
            children='Infrastructure',
            style={'color': chapter_dict['domain-color']},
        ),
        dbc.Row(
            children=[
                dbc.Col(className="col-12 col-md-6 my-auto",
                        children=[
                            html.P(infrastructureText)]
                        ),
                dbc.Col(className="col-12 col-md-6 my-auto",
                        children=[html.Div(
                            style={"position": "relative"},
                            children=[html.Img(
                                style={"width": "100%"},
                                src=IMAGES_PATH+'OceanicSections/Celtic_Explorer_TomaszSzumski.jpg'
                            ),
                                html.Span(
                                className='sr-img-credit',
                                children='Credit: Tomasz Szumski'
                            ),
                            ])])
            ])
    ])

def create_layout(app):
    return html.Div(
        children=[
            pb.build_banner(bannerImgSrc,
                            bannerImgCredit,
                            chapter_dict
                            ),
            pb.build_breadcrumb(chapter_dict),
            pb.build_nav(chapter_dict),
            pb.build_intro(introText,
                           bulletPoints,
                           chapter_dict
                           ),
            custom_trend,

            custom_infrastructure,
            # pb.build_infrastructure(infrastructureText,
            #                         infrastructureMap,
            #                         chapter_dict
            #                         ),
            pb.build_info(infoLinks,
                          chapter_dict),

            pb.build_nav_carousel(chapter_dict)
        ])


Import('*')

srcs = Split('print.c make_rep.c env.c fp.c read.c')
objs = env.Object(srcs)

libname = 'gslieeeutils'
lib = env.StaticLibrary(libname, objs)
env['PRJ_LIBS'][libname] = lib
env['PRJ_OBJS'][libname] = objs

exename = 'ieeeutils_test'
exelibs = [
    lib,
    env['PRJ_LIBS']['gsltest'],
    env['PRJ_LIBS']['gslsys'],
    #env['PRJ_LIBS']['utils']
]
exe = env.Program(exename, 'test.c', LIBS=exelibs)
env['PRJ_EXES'][exename] = exe


class Class(object):
    def __init__(self,name):
        self.name =name
        self.methodCount =0
        self.depth =0
        self.children =0
        self.methods =[]

classes =dict()
lastClass =None

fileName =open('Enter file name\n')

file =open(fileName,'r')

#Weighted Average Method

for line in file:
    if 'class' in line:
        if lastClass != None:
            # classes.append(last_class)
            classes[lastClass.name] =lastClass
        line =line[line.index(' ')+1:]
        lastClass =Class(line[:line.index('(')])
        continue

    if 'def' in line:
        lastClass.methodCount+=1

classes[lastClass.name] = lastClass
file.close()
print(len(classes.keys()))

#print
for eachClass in classes.keys():
    classObject =classes[eachClass]
    print(classObject.name,classObject.methodCount)

#Count depth of inheritance

classes['object'] =Class('object')

file =open(fileName,'r')
for line in file:
    if 'class' in line:
        line =line[line.index(' ')+1:]
        className =line[:line.index('(')]
        inheritedClassName =line[line.index('(')+1:line.index(')')]
        print(className,inheritedClassName)
        classObject =classes[className]
        inheritedClassObject =classes[inheritedClassName]
        inheritedClassObject.children+=1
        classObject.depth=inheritedClassObject.depth+1
file.close()
maxDepth =-100
for eachClass in classes.keys():
    classObject =classes[eachClass]
    if maxDepth <= classObject.depth:
        maxDepth =classObject.depth

print('depth of Inheritance:',maxDepth)

#number of immediate sub-classes of a class

for eachClass in classes.keys():
    classObject =classes[eachClass]
    print(classObject.name,classObject.children)

#Total CK 4 ck metrics done. 2 left.
# Mood left.


# 给定一个二叉搜索树，编写一个函数 kthSmallest 来查找其中第 k 个最小的元素。
# 说明：
# 你可以假设 k 总是有效的，1 ≤ k ≤ 二叉搜索树元素个数。
# 示例 1:
# 输入: root = [3,1,4,null,2], k = 1
#    3
#   / \
#  1   4
#   \
#    2
# 输出: 1
# 示例 2:
# 输入: root = [5,3,6,2,4,null,null,1], k = 3
#        5
#       / \
#      3   6
#     / \
#    2   4
#   /
#  1
# 输出: 3
# 进阶：
# 如果二叉搜索树经常被修改（插入/删除操作）并且你需要频繁地查找第 k 小的值，你将如何优化 kthSmallest 函数？

# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

class Solution:
    def kthSmallest(self, root: TreeNode, k: int) -> int:
        stack = []
        cur = root
        while stack or cur:

            if cur:
                stack.append(cur)
                cur = cur.left
            else:
                k -= 1
                cur = stack.pop()
                if k == 0:
                    return cur.val
                if cur.right:
                    cur = cur.right
                else:
                    cur = None


import logging
import os
import azure.functions as func
from azure.identity import DefaultAzureCredential
import openai
import json
from opentelemetry.sdk.trace import TracerProvider

from azure.monitor.events.extension import track_event
from azure.monitor.opentelemetry import configure_azure_monitor
from opentelemetry import trace
from opentelemetry import metrics
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator

configure_azure_monitor(
    connection_string=os.environ.get("APPLICATIONINSIGHTS_CONNECTION_STRING"),
)

trace.set_tracer_provider(TracerProvider())
tracer = trace.get_tracer(__name__)

meter = metrics.get_meter_provider().get_meter("openaifunction")
token_counter = meter.create_counter("openaitokens")

# Request credential
default_credential = DefaultAzureCredential()
token = default_credential.get_token("https://cognitiveservices.azure.com/.default")

# Setup parameters
openai.api_type = "azure_ad"
openai.api_key = token.token
openai.api_base = os.environ.get("API_BASE")
openai.api_version = "2023-05-15"
engine = os.environ.get("ENGINE")

def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse:
    logging.info('Python HTTP trigger function processed a request.')
    logging.info(f"Invocation id: {context.invocation_id}")
    logging.info(f"function name: {context.function_name}")
    logging.info(f"context: {context}")
    logging.info(f"trace context: {context.trace_context}")
    logging.info(f"trace parent: {context.trace_context.Traceparent}")
    logging.info(f"trace state: {context.trace_context.Tracestate}")
    operation_id = context.trace_context.Traceparent.split("-")[1]
    w3c_trace_context = { "traceparent": context.trace_context.Traceparent }
    ctx = TraceContextTextMapPropagator().extract(carrier=w3c_trace_context)
    with tracer.start_as_current_span("openai-function", context=ctx):
        question = req.params.get('question')
        if not question:
            try:
                req_body = req.get_json()
            except ValueError:
                pass
            else:
                question = req_body.get('question')

        user_prompt = "What is blue milk?"
        if question:
            user_prompt = question

        response = openai.ChatCompletion.create(
            engine=engine,
            n=1,
            top_p=1,
            frequency_penalty=0,
            presence_penalty=0,
            messages=[
                {"role": "system", "content": 'You are an AI assistant that helps people find information about "Star Wars".\n\nInstructions\n- only answer questions related to Star Wars\n- If an answer is not related to Star Wars, respond with "This is not the AI you are looking for..."'},
                {"role": "user", "content": user_prompt},
            ],
        )

        logging.info(f"{list(req.headers.keys())}")
        for key in req.headers.keys():
            logging.info(f"{key}: {req.headers[key]}")
        logging.info(f"{req.headers}")
        logging.info(response)

        token_counter.add(response["usage"]["total_tokens"], { "function": os.environ.get("OTEL_SERVICE_NAME"), "operation_id": operation_id, "streaming": "false" })
        track_event("openai-tokens", {"function": os.environ.get("OTEL_SERVICE_NAME"), "total_tokens": response["usage"]["total_tokens"], "operation_id": operation_id , "streaming": "false"})
        return func.HttpResponse(response["choices"][0]["message"]["content"])


#very quick write.

from __future__ import division
from Bio import SeqIO
Ns = open('contentNs.txt', 'w')
for sequence in SeqIO.parse('/home/muliano/VGP-running/pri.asm.20180817.fasta-buscos0', 'fasta'):
        id = sequence.id
        count1 = len(sequence)
        countNs = sequence.seq.upper().count('N')
        out = id + ' ' + str(countNs) + '\n'
        Ns.write(out)
Ns.close()


import json
import numpy as np
import torch
import torch.nn as nn
import time
import math
import collections

use_frequency = True

def sigmoid(x):
  return 1 / (1 + math.exp(-x))

#with open('shuffled_class_dict.json', 'r') as f1:
with open('culled_class_dict_bitcost_compar.json', 'r') as f1:
    class_dict = json.load(f1)

for k in list(class_dict):
    num_pns = len(class_dict[k][1])
    break

exps = sorted(set(e for k in list(class_dict) for e in class_dict[k][1]))
print('e', exps)

torch.set_printoptions(sci_mode=False)

num_exps = len(exps)
print('There are', num_pns, 'pns')
print('There are', num_exps, 'exps')
exponent2idx = {exp: i for i, exp in enumerate(exps)}

stem_embedding_dim = 1
num_epochs = 400
verbose = False
lexeme_index_to_identifier = collections.defaultdict(lambda: None)
targets = np.zeros((len(class_dict), num_pns))
for i, k in enumerate(list(class_dict)):
    pattern_set = class_dict[k][1]
    lexeme_index_to_identifier[str(i)] = k
    for j in range(num_pns):
        targets[i, j] = exponent2idx[pattern_set[j]]

class LearnActivationsWeights(nn.Module):

    def __init__(self, num_exps, num_pns, class_dict, stem_embedding_dim):
        super(LearnActivationsWeights, self).__init__()
        self.class_dict = class_dict
        self.embeddings_stems = nn.Embedding(len(self.class_dict), num_exps)
        self.embeddings_pn = nn.Embedding(num_pns, num_exps)
        self.dep_bias = nn.Embedding(1, num_exps)
        self.sigmoid = nn.Sigmoid()
        self.dep = torch.nn.Parameter(torch.rand((1), requires_grad=True, dtype=torch.float))
        self.n_digits_stem = 3
        self.n_digits_pn = 3

    def forward(self, stem_index, round_for_testing=False):
        self.embs_stem = self.sigmoid(self.embeddings_stems(stem_index))
        self.embs_stem = torch.unsqueeze(self.embs_stem.repeat(num_pns),0)# / 2
        self.embs_pn_0  = self.sigmoid(self.embeddings_pn(torch.tensor(0))) #/ 2
        self.embs_pn_1  = self.sigmoid(self.embeddings_pn(torch.tensor(1))) #/ 2
        self.embs_pn_2  = self.sigmoid(self.embeddings_pn(torch.tensor(2))) #/ 2
        self.embs_pn_3  = self.sigmoid(self.embeddings_pn(torch.tensor(3))) #/ 2
        self.embs_pn_4  = self.sigmoid(self.embeddings_pn(torch.tensor(4))) #/ 2
        self.embs_pn = torch.unsqueeze(torch.cat((self.embs_pn_0, self.embs_pn_1, self.embs_pn_2, self.embs_pn_3, self.embs_pn_4), 0),0)
        coalesced = (self.embs_stem + self.embs_pn)
        self.bias = self.dep_bias(torch.tensor(0)).repeat(num_pns)

        max_reward = coalesced
        dep_penalty = (torch.ones(coalesced.size()) - coalesced)

        if not round_for_testing:
            return max_reward - dep_penalty, max_reward, dep_penalty, coalesced
        else:
            with torch.no_grad():
                self.embs_stem = torch.round(self.sigmoid(self.embeddings_stems(stem_index))*2**self.n_digits_stem)/2**self.n_digits_stem
                self.embs_stem = torch.unsqueeze(self.embs_stem.repeat(num_pns),0)# / 2
                self.embs_pn_0  = torch.round(self.sigmoid(self.embeddings_pn(torch.tensor(0)))*2**self.n_digits_pn)/2**self.n_digits_pn #/ 2
                self.embs_pn_1  = torch.round(self.sigmoid(self.embeddings_pn(torch.tensor(1)))*2**self.n_digits_pn)/2**self.n_digits_pn   #/ 2
                self.embs_pn_2  = torch.round(self.sigmoid(self.embeddings_pn(torch.tensor(2)))*2**self.n_digits_pn)/2**self.n_digits_pn   #/ 2
                self.embs_pn_3  = torch.round(self.sigmoid(self.embeddings_pn(torch.tensor(3)))*2**self.n_digits_pn)/2**self.n_digits_pn   #/ 2
                self.embs_pn_4  = torch.round(self.sigmoid(self.embeddings_pn(torch.tensor(4)))*2**self.n_digits_pn)/2**self.n_digits_pn   #/ 2
                self.embs_pn = torch.unsqueeze(torch.cat((self.embs_pn_0, self.embs_pn_1, self.embs_pn_2, self.embs_pn_3, self.embs_pn_4), 0),0)
                coalesced = (self.embs_stem + self.embs_pn)
                max_reward = coalesced
                dep_penalty = (torch.ones(coalesced.size()) - coalesced)
                return max_reward - dep_penalty, max_reward, dep_penalty, coalesced

def train(class_dict, targets, net, lr, num_epochs):
    criterion = nn.CrossEntropyLoss()
    optimiser = torch.optim.Adam(net.parameters(), lr=lr)
    num_examples = len(class_dict)
    for epoch in range(num_epochs):
        ep_loss = 0.
        start_time = time.time()
        num_tested = 0
        num_correct = 0
        margins = []
        for i in torch.randperm(len(class_dict)):
            #    continue
            key = list(class_dict)[i]
            preds, max_reward, dep_penalty, coalesced = net(torch.tensor(i).clone().detach())
            loss = 0
            for j in range(num_pns):
                if use_frequency:
                    num_tested += class_dict[key][0]
                else:
                    num_tested += 1
                target = torch.tensor(targets[i,j])
                target = target.contiguous().view(-1)
                target = target.long()
                target_name = exps[int(targets[i,j])]
                pred = preds[:,j*num_exps:j*num_exps+num_exps]
                pred_name = exps[int(torch.argmax(pred).item())]
                if pred_name == target_name:
                    if use_frequency:
                        num_correct += class_dict[key][0]
                    else:
                        num_correct += 1
                    preds_ordered = list(sorted(pred.detach().numpy().tolist()[0], reverse=True))
                    margin = round(preds_ordered[0] - preds_ordered[1], 4)
                    margins.append(margin)
                else:
                    if epoch == num_epochs - 1 and verbose == True:
                        print('error', 'target is', target, 'j=', j, lexeme_index_to_identifier[str(i.detach().numpy())],  i, end=' ')
                        for val in pred.contiguous().view(-1).detach().numpy().tolist():
                            print(round(val, 3), end=' ')
                        print()
                if epoch == num_epochs - 1 and verbose == True:
                    print('Harmonies for lexeme', lexeme_index_to_identifier[str(i.detach().numpy())], end=' ')
                    for val in pred.contiguous().view(-1).detach().numpy().tolist():
                        print(round(val, 3), end=' ')
                    print()
                if use_frequency:
                    loss += criterion(pred, target) * class_dict[key][0] / 100
                else:
                    loss += criterion(pred, target)

            loss.backward()
            optimiser.step()
            optimiser.zero_grad()
            ep_loss += loss.detach()

        print(epoch, ep_loss, round(time.time()-start_time, 3))
        print('Accuracy', round(num_correct / num_tested, 4), num_tested, num_correct)
        print('Margins', list(sorted(margins))[0])
        if epoch %10 ==0:
            print()
            print()
            print('Embeddings')
            for i in range(len(class_dict)):
                print(lexeme_index_to_identifier[str(i)], end=' ')
                row = net.sigmoid(net.embeddings_stems.weight[i,:])
                row_sum = torch.sum(row).detach().numpy()
                for element in row.detach().numpy().tolist():
                    if round(element, 3) == 0.0:
                        print('_', end=' ')
                    else:
                        print(round(element, 3), end=' ')
                print()
            print()

            print('Embeddings pn')
            for i in range(num_pns):
                row = net.sigmoid(net.embeddings_pn.weight[i,:])
                row_sum = torch.sum(row).detach().numpy()
                for element in row.detach().numpy():
                    if round(element, 3) == 0.0:
                        print('_', end=' ')
                    else:
                        print(round(element, 3), end=' ')
                print()
            print()

    print('Now testing')
    num_examples = len(class_dict)
    num_tested = 0
    num_correct = 0
    margins = []
    lexemes_to_exclude = []
    with torch.no_grad():
        for i in torch.randperm(len(class_dict)):
            key = list(class_dict)[i]
            preds, max_reward, dep_penalty, coalesced = net(torch.tensor(i).clone().detach(), True)
            for j in range(num_pns):
                if use_frequency:
                    num_tested += class_dict[key][0]
                else:
                    num_tested += 1
                target = torch.tensor(targets[i,j])
                target = target.contiguous().view(-1)
                target = target.long()
                target_name = exps[int(targets[i,j])]
                pred = preds[:,j*num_exps:j*num_exps+num_exps]
                pred_name = exps[int(torch.argmax(pred).item())]
                if pred_name == target_name:
                    if use_frequency:
                        num_correct += class_dict[key][0]
                    else:
                        num_correct += 1
                    preds_ordered = list(sorted(pred.detach().numpy().tolist()[0], reverse=True))
                    margin = round(preds_ordered[0] - preds_ordered[1], 4)
                    margins.append(margin)
                else:
                    #print('On testing, missed lexeme', i, 'for pn', j, 'with frequency', class_dict[key][0])
                    if class_dict[key][0] in ['1', 1] and i not in lexemes_to_exclude:
                        lexemes_to_exclude.append(i.item())
                    if epoch == num_epochs - 1 and verbose == True:
                        print('error', 'target is', target, 'j=', j, lexeme_index_to_identifier[str(i.detach().numpy())],  i, end=' ')
                        for val in pred.contiguous().view(-1).detach().numpy().tolist():
                            print(round(val, 3), end=' ')
                        print()
                if epoch == num_epochs - 1 and verbose == True:
                    print('Harmonies for lexeme', lexeme_index_to_identifier[str(i.detach().numpy())], end=' ')
                    for val in pred.contiguous().view(-1).detach().numpy().tolist():
                        print(round(val, 3), end=' ')
                    print()
        print('Accuracy', round(num_correct / num_tested, 4), num_tested, num_correct)
        print()
        print()
        print('Embeddings')
        for i in range(len(class_dict)):
            print(lexeme_index_to_identifier[str(i)], end=' ')
            row = torch.round(net.sigmoid(net.embeddings_stems(torch.tensor(i)))*2**net.n_digits_stem)/2**net.n_digits_stem
            for element in row.detach().numpy().tolist():
                if element == 0.0:
                    print('_', end=' ')
                else:
                    print(round(element, net.n_digits_stem), end=' ')
            print()
        print()

        print('Embeddings pn')
        for i in range(num_pns):
            row = torch.round(net.sigmoid(net.embeddings_pn(torch.tensor(i)))*2**net.n_digits_pn)/2**net.n_digits_pn
            for element in row.detach().numpy():
                if round(element, 3) == 0.0:
                    print('_', end=' ')
                else:
                    print(round(element, net.n_digits_pn), end=' ')
            print()

net = LearnActivationsWeights(num_exps, num_pns, class_dict, stem_embedding_dim)
lr = 0.003
train(class_dict, targets, net, lr, num_epochs)


import torch.nn as nn
import torch.nn.functional as F

class FCNet(nn.Module):
    def __init__(self):
        super(FCNet, self).__init__()
        self.fc1 = nn.Linear(784, 500)
        self.fc2 = nn.Linear(500, 120)
        self.fc3 = nn.Linear(120, 10)

    def forward(self, x):
        out = self.fc1(x)
        out = F.relu(out)
        out = self.fc2(out)
        out = F.relu(out)
        out = self.fc3(out)
        return out


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    The Alkemy app model is designed to help keep track of the game data.

    Default language
    ----------------

    As Alkemy is a french edited game with most of its audience in France, the
    default language is set to french and so are the labels.

    updatedAt
    ---------

    Most if not all of the models does have a updatedAt field. That field is
    used to keep track of the last update of the model.

"""

from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible

class GameDataCommon(models.Model):
    """
    Common attributes for data classes.

    Statuses should be:

      - published: Something
      - draft: Something else
      - removed: The last thing

    :param lang: Language of this object, ISO format
    :param updatedAt: Timestamp of the last update, intended to be used for sync features.
    :param status: Status of the stat, the expected behavior is described above.
    """
    FRENCH = 'fr'
    ENGLISH = 'en'
    LANG_CHOICES = (
        (FRENCH, u'Français'),
        (ENGLISH, u'Anglais')
    )

    STATUS_PUBLISHED = 'pub'
    STATUS_DRAFT = 'dft'
    STATUS_REMOVED = 'del'
    STATUS_CHOICES = (
        (STATUS_DRAFT, u'Brouillon'),
        (STATUS_PUBLISHED, u'Publié'),
        (STATUS_REMOVED, u'Supprimé')
    )

    lang = models.CharField(max_length=2,
                            choices=LANG_CHOICES,
                            default=FRENCH)

    status = models.CharField(max_length=3,
                              choices=STATUS_CHOICES,
                              default=STATUS_DRAFT)

    updatedAt = models.DateTimeField(auto_now=True)

    class Meta:
        abstract = True

@python_2_unicode_compatible
class Faction(GameDataCommon):
    """
    Each profile belongs to one or more faction(s).

    :param name: Name of the faction
    """
    name = models.CharField(max_length=50)

    def __str__(self):
        return self.name

    def __eq__(self, other):
        return self.name == other.name

@python_2_unicode_compatible
class Ability(GameDataCommon):
    """
    Standard description of abilities. This will be displayed in the list of
    abilities and in the details of a profile.

    :param name: The name of the ability
    :param description: A description of this ability
    """
    name = models.CharField(max_length=50)
    description = models.TextField()

    class Meta:
        verbose_name_plural = "Abilities"

    def __str__(self):
        return self.name

@python_2_unicode_compatible
class Profile(GameDataCommon):
    """
    Profile object describes a single Alkemy profile.
    """
    name = models.CharField('Nom', max_length=30)
    title = models.CharField('Titre', max_length=50, blank=True)
    life_points = models.CommaSeparatedIntegerField('Points de vie',
                                                    max_length=8)
    movement = models.CommaSeparatedIntegerField('Mouvement', max_length=8)
    damage_cc = models.CommaSeparatedIntegerField('Corps à corps',
                                                  max_length=16)
    damage_ra = models.CommaSeparatedIntegerField('Combat à distance',
                                                  max_length=16, blank=True)
    action_points = models.IntegerField('Points d\'Action')
    cost = models.IntegerField('Cout')
    combat = models.IntegerField('Combat')
    defense = models.IntegerField('Défense')
    mind = models.IntegerField('Esprit')
    reflexes = models.IntegerField('Réflexes')
    size = models.IntegerField('Taille', default=2)

    factions = models.ManyToManyField(Faction)

    abilities = models.ManyToManyField(Ability, through='AbilityValue')

    def __str__(self):
        return self.name

@python_2_unicode_compatible
class Alchemist(Profile):

    element = models.CharField(max_length=10)
    rank = models.CharField(max_length=30)
    stones = models.IntegerField()

    def __str__(self):
        return self.name

    class Meta:
        ordering = ['name']

@python_2_unicode_compatible
class Formula(GameDataCommon):
    THRESHOLD_TARGET = 'Cible'
    THRESHOLD_AUTO = 'Auto'
    THRESHOLD_CHOICES = (
        (THRESHOLD_TARGET, 'Cible'),
        (THRESHOLD_AUTO, 'Auto')
    )

    name = models.CharField(max_length=50)
    effect = models.TextField()
    focus_level = models.IntegerField()
    components = models.CharField(max_length=20)
    threshold = models.CharField(max_length=5, choices=THRESHOLD_CHOICES,
                                 default=THRESHOLD_TARGET)
    range = models.IntegerField()

    alchemist = models.ForeignKey(Alchemist, on_delete=models.CASCADE)

    def __str__(self):
        return self.name

@python_2_unicode_compatible
class Enhancement(GameDataCommon):
    """
    Enhancements are formula related objects.
    """
    description = models.TextField()

    formula = models.ManyToManyField(Formula,
                                     null=True)

    def __str__(self):
        return self.description

@python_2_unicode_compatible
class AbilityValue(models.Model):
    """
    This class associate an ability to a profile as an ability may have specific
    value for a profile. The value is optional.
    """
    ability = models.ForeignKey(Ability, on_delete=models.CASCADE)
    profile = models.ForeignKey(Profile, on_delete=models.CASCADE)

    value = models.CharField(max_length=10, blank=True)

    def __str__(self):
        """
        String description is formated depending on the value.

        :return:
         The string representation of this object which is either the ability
         name or the ability name and the value associated.
        """
        if self.value:
            return '%s (%s)' % (self.ability.name, self.value)
        else:
            return '%s' % self.ability.name


from flask import Flask, render_template, session, redirect, request, flash

from flask_app import app

from flask_app.models.user import User
from flask_app.models.task import Task
from flask_app.models.project import Project

# variablized route files

dashboard = 'projects.html'
project_detail_page = 'group_projects_dashboard.html'
new_project_page = 'projects_form_new.html'
edit_project = 'projects_form_edit.html'

@app.route('/dashboard')
def projects_home():
    if 'user_id' not in session:
        flash('You must be logged in to view this page!')
        return redirect('/')
    if 'project_id' in session:
        session.pop('project_id')
    print(session)
    data = {
        'id': session['user_id']
    }
    user = User.get_user_by_id(data)
    projects = Project.get_all_projects()
    all_user_tasks = Task.get_active_user_tasks(data)

    return render_template(dashboard, user=user, projects = projects, tasks=all_user_tasks)

@app.route("/projects/<int:project_id>")
def project_detail(project_id):
    session['project_id'] = project_id
    user_data = {'id': session['user_id']}
    data = {
        'project_id': project_id,
        'user_id': session['user_id']
    }
    user = User.get_user_by_id(user_data)
    project = Project.get_one_project_by_id(data)
    own_tasks = Task.get_user_tasks_by_project(data)
    other_tasks = Task.get_other_tasks_by_project(data)
    completed_tasks = Task.get_completed_tasks_by_project(data)
    return render_template(project_detail_page, user=user, project=project, own_tasks=own_tasks, other_tasks=other_tasks, completed_tasks = completed_tasks)

@app.route("/new_project")
def new_project():
    data = {
        id: session['user_id']
    }
    user = User.get_user_by_id(data)
    return render_template(new_project_page, user=user)

@app.route("/create_project", methods=['POST'])
def create_new_project():
    print(session['user_id'])
    if not Project.validate_project(request.form):
        return redirect('/new_project')
    data = {
        'title': request.form['title'],
        'description': request.form['description'],
        'user_id': request.form['assigner']
    }
    new_project =Project.create_project(data)
    return redirect(f'/projects/{new_project}')

@app.route("/delete_project/<int:project_id>")
def delete_project(project_id):
    if "user_id" not in session:
        return redirect("/")
    Project.delete_project_by_id(project_id)
    return redirect('/dashboard')

@app.route("/edit_project/<int:project_id>")
def edit_project_page(project_id):
    user_data = {'id': session['user_id']}

    data = {
        'project_id': project_id,
        'user_id': session['user_id']
    }
    project = Project.get_one_project_by_id(data)
    if session['user_id'] != project.assigner.id:
        return redirect('/dashboard')
    user = User.get_user_by_id(user_data)
    return render_template(edit_project, user=user, project=project)

@app.route("/update_project/<int:project_id>", methods=['POST'])
def update_project(project_id):
    if not Project.validate_project(request.form):
        return redirect(f'/edit_project/{project_id}')
    data = {
        'id': project_id,
        'title': request.form['title'],
        'description': request.form['description']
    }
    Project.update_project(data)
    return redirect(f'/projects/{project_id}')


import matplotlib.pyplot as plt
import matplotlib
import numpy as np
matplotlib.rcParams.update({'font.size':14, 'font.family':'sans-serif'})
fig,(ax2,ax) = plt.subplots(2,1,sharex=True)
colors = ['C0','C1']

with open('./data/result.data','rb') as fp:
    for cn,state_size in enumerate([2,4]):
        Ep = np.load(fp)
        Ave = np.load(fp)
        Std = np.load(fp)
        ax.plot(Ep,Ave[:,0],color=colors[cn],label=r'$N_T=%s$'%state_size)
        ax.fill_between(Ep,Ave[:,0]-Std[:,0],Ave[:,0]+Std[:,0], color=colors[cn],alpha=0.2)
        ax2.plot(Ep,Ave[:,0],color=colors[cn],label=r'$N_T=%s$'%state_size)
        ax2.fill_between(Ep,Ave[:,0]-Std[:,0],Ave[:,0]+Std[:,0], color=colors[cn],alpha=0.2)
y0 = 162.10051016264254
ax.plot([Ep[0],Ep[-1]],[y0,y0],'k--')
ax.text(50,y0+1,r'$\Delta A_\mathrm{II}$')
ax2.plot([Ep[0],Ep[-1]],[y0,y0],'k--')
ax2.text(50,y0+1,r'$\Delta A_\mathrm{II}$')

y0 = 17.22748822645095
ax.plot([Ep[0],Ep[-1]],[y0,y0],'k--')
ax.text(50,y0-5,r'$\Delta A_\mathrm{IV}$')
ax2.plot([Ep[0],Ep[-1]],[y0,y0],'k--')
ax2.text(50,y0-5,r'$\Delta A_\mathrm{IV}$')
ax.set_ylim((-5,40))
ax2.set_ylim((150,180))
ax.set_xlabel('Epoch')
ax.set_ylabel('Error (degree)')
ax2.legend()

ax2.spines['bottom'].set_visible(False)
ax.spines['top'].set_visible(False)
ax2.xaxis.tick_top()
ax2.tick_params(labeltop=False)  # don't put tick labels at the top
ax.xaxis.tick_bottom()

d = .015  # how big to make the diagonal lines in axes coordinates
# arguments to pass to plot, just so we don't keep repeating them
kwargs = dict(transform=ax2.transAxes, color='k', clip_on=False)
ax2.plot((-d, +d), (-d, +d), **kwargs)        # top-left diagonal
ax2.plot((1 - d, 1 + d), (-d, +d), **kwargs)  # top-right diagonal

kwargs.update(transform=ax.transAxes)  # switch to the bottom axes
ax.plot((-d, +d), (1 - d, 1 + d), **kwargs)  # bottom-left diagonal
ax.plot((1 - d, 1 + d), (1 - d, 1 + d), **kwargs)  # bottom-right diagonal
ax.yaxis.set_label_coords(-0.1,1.1)

plt.show()


# 2020-03-04 Lyon
# MNIST数据集:http://yann.lecun.com/exdb/mnist/
from PIL import Image,ImageOps
import struct
import tensorflow as tf
from tensorflow import keras
import matplotlib.pyplot as plt
import os
import random
import numpy as np
from network import LeNet

for gpu in tf.config.experimental.list_physical_devices('GPU'):
    tf.config.experimental.set_memory_growth(gpu, True)

def read_image(filename, out_dir):
    f = open(filename,'rb')
    index = 0
    buf = f.read()
    f.close()
    magic, images, rows, columns = struct.unpack_from('>IIII' , buf , index)
    index += struct.calcsize('>IIII')

    for i in range(images):
        image = Image.new('L', (columns, rows)) # 'L'为单通道黑白图；'RGB'为三通道彩图，其他模式:https://pillow.readthedocs.io/en/3.1.x/handbook/concepts.html#concept-modes
        for x in range(rows):
            for y in range(columns):
                image.putpixel((y, x), int(struct.unpack_from('>B', buf, index)[0]))
                index += struct.calcsize('>B')
        path = out_dir + '/' + str(i) + '.png'
        image.save(path)
        print('save image:%d >>> %s' % (i, path))

def read_label(filename, out_path):
    f = open(filename, 'rb')
    index = 0
    buf = f.read()
    f.close()
    magic, labels = struct.unpack_from('>II' , buf , index)
    index += struct.calcsize('>II')
    labelArr = [0] * labels
    for x in range(labels):
        labelArr[x] = int(struct.unpack_from('>B', buf, index)[0])
        index += struct.calcsize('>B')
        save = open(out_path, 'w')
        save.write(','.join(map(lambda x: str(x), labelArr)))
        save.write('\n')
        save.close()
        print('save label:%d success' % x)

def transform_data():
    """MNIST数据预处理，将图片从byte转为png,将labels转为txt"""
    read_image('./dataset/MNIST/data/train-images-idx3-ubyte', './dataset/MNIST/train_images')
    read_label('./dataset/MNIST/data/train-labels-idx1-ubyte', './dataset/MNIST/train_label.txt')
    read_image('./dataset/MNIST/data/t10k-images-idx3-ubyte', './dataset/MNIST/test_images')
    read_label('./dataset/MNIST/data/t10k-labels-idx1-ubyte', './dataset/MNIST/test_label.txt')

def load_and_preprocess_image(path):
    """图像预处理，将输入图片转换成28×28的单通道、归一化的图片矩阵"""
    image = tf.io.read_file(path)
    image = tf.image.decode_png(image, channels=1)
    image = tf.image.resize(image, [28, 28])
    image /= 255.0
    return image

def data_loader(image_dir, label_path):
    """数据加载"""
    all_image_paths = list(os.listdir(image_dir))
    all_image_paths.sort(key= lambda x:int(x[:-4]))
    all_image_paths = [(image_dir + os.sep + str(path))for path in all_image_paths]

    label_names = ['0','1','2','3','4','5','6','7','8','9']
    label_index = dict((name, index) for index, name in enumerate(label_names))  # {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9}

    with open(label_path, 'r', encoding='utf-8') as f:
        all_image_labels = f.read().strip('\n').split(',')
    all_image_labels = [label_index[label] for label in all_image_labels]

    print("\nFirst 10 images path: ", all_image_paths[:10])
    print("\nFirst 10 labels: ", all_image_labels[:10])

    path_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)
    image_ds = path_ds.map(load_and_preprocess_image, num_parallel_calls=tf.data.experimental.AUTOTUNE)
    label_ds = tf.data.Dataset.from_tensor_slices(tf.cast(all_image_labels, tf.int64))
    image_label_ds = tf.data.Dataset.zip((image_ds, label_ds))

    return image_label_ds

def train_mnist(dir_path, label_path, epochs, save_path):
    """模型训练"""
    train_dataset = data_loader(dir_path, label_path)
    train_dataset = train_dataset.batch(60000)
    train_dataset = train_dataset.shuffle(60000)
    train_dataset = train_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
    train_images, train_labels = next(iter(train_dataset))
    # 定义模型——三种方式：１．sequential；2.functional API；3.自定义子类Model
    model = LeNet.build_lenet('sequential')
    # 2. functional API  model = LeNet.build_lenet('functional')
    # 3. 自定义子类Model   model = LeNet.build_lenet('subclass')

    # 定义优化器
    optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
    # 模型编译
    model.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy'])
    # 训练
    model.fit(train_images, train_labels, epochs=epochs)
    # 模型保存——两种方式：1.只存权重 2、存包含优化器和损失函数在内的整个模型(仅支持functional和sequential类型的model)
    model.save(save_path)
    # 只存权重 model.save_weights(save_path)

def test_mnist(dir_path, label_path, model_path):
    # 方式１：直接加载已经训练好的模型
    # model= keras.models.load_model(model_path)
    # 方式２：重新构造模型，并加载训练好的权重
    model = LeNet.build_lenet('subclass')
    model.compile(loss='sparse_categorical_crossentropy', metrics=['accuracy'])
    model.build((1,28,28,1))
    model.load_weights(model_path)

    # 验证准确率
    test_dataset = data_loader(dir_path, label_path)
    test_dataset = test_dataset.batch(10000)
    test_images, test_labels = next(iter(test_dataset))
    loss, acc = model.evaluate(test_images,  test_labels, verbose=2)
    print("Restored model, accuracy: {:5.2f}%".format(100*acc))

    # 显示模型网络结构
    # print(model.summary())

def test_one_image(image_path, model_path):
    # 加载模型
    model= keras.models.load_model(model_path)
    # 加载需要预测的图片
    image = load_and_preprocess_image(image_path)
    # 把数据转换成float32
    image = tf.image.convert_image_dtype(image, tf.float32)
    tensor = tf.reshape(image, [1, 28, 28, 1])
    predictions = model.predict(tensor)
    # 输出分类矩阵和预测结构
    print("predictions:", predictions[0])
    print ('the predict number is : ', np.argmax(predictions[0]))

if __name__ == '__main__':
    # MNIST数据转换(预处理)
    # transform_data()

    # 模型训练
    path = './weight/10_epoch_lenet_weight.h5'
    # train_mnist('./dataset/MNIST/train_images', './dataset/MNIST/train_label.txt', 10, path)

    # 模型测试
    test_mnist('./dataset/MNIST/test_images', './dataset/MNIST/test_label.txt', path)  # acc 99.18

    # 测试单张图片
    test_one_image('./dataset/MNIST/8.png', path)