#! /usr/bin/env python

import numpy as np
import pickle
import os
from flask import Flask, render_template, url_for, send_from_directory
from flask_frozen import Freezer

# Global table data variable:
t = None

app = Flask(__name__)
app.config['DOC_FOLDER'] = os.path.join(os.path.dirname(__file__), 'doc/build/html/')
freezer = Freezer(app)

@app.route('/')
def home():
    return render_template('home.html')

@app.route('/about/')
def about():
    return render_template('about.html')

@freezer.register_generator
def source():
    for source_id in t.keys():
        yield {'source_id': source_id}

@app.route('/source/<int:source_id>.html')
def source(source_id):
    if source_id not in t.keys():
        return render_template('source_not_found.html', source_id=source_id)
    else:
        data = t[source_id]
        keys = data.keys()
        keys.sort()
        #data['sdss_thumbnail'] = url_for('static', filename='img/' + os.path.basename(data['sdss_thumbnail']))
        data['sdss_thumbnail'] = os.path.basename(data['sdss_thumbnail'])
        data['shortsum_png'] = os.path.basename(data['shortsum_png'])

        current = np.where(np.array(t.keys()) == source_id)[0][0]
        if current == t.keys()[0]:
            prev = None
        else:
            prev = str(t.keys()[current - 1])

        if current == t.keys()[len(t) - 1]:
            next = None
        else:
            next = str(t.keys()[current + 1])
        nav = {'keys':keys, 'prev':prev, 'next':next, 'first':t.keys()[0], 'last':t.keys()[-1]}

        return render_template('source.html', source_id=source_id, data=data, nav=nav)

def load_data(pickle_file):
    global t
    with open(pickle_file, 'r') as f:
        t = pickle.load(f)
    t = t.to_dict(orient='index')

@freezer.register_generator
def doc():
    all_files = []
    for root, dirs, files in os.walk(app.config['DOC_FOLDER']):
        root = root[len(app.config['DOC_FOLDER']):]
        for name in files:
            if name[0] != '.':
                all_files.append({'filename': os.path.join(root, name)})
    return all_files

@app.route('/doc/')
@app.route('/doc/<path:filename>')
def doc(filename='index.html'):
    return send_from_directory(app.config['DOC_FOLDER'], filename)

def host_cigma(pickle_file='./cigma_data.pkl', static=False):
    '''
    Read in pickle file and host the CIGMA website.
    '''
    # Read in data:
    load_data(pickle_file)

    if static:
        # Generate static html files:
        freezer.freeze()
        print('To host static files:')
        print('  cd {:s} ; open http://0.0.0.0:8000 ; python3 -m http.server\n'.format( \
            os.path.relpath(os.path.join(os.path.dirname(__file__), app.config['FREEZER_DESTINATION']))))
    else:
        # Launch flask server:
        app.run(debug=True)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description='Host the CIGMA website.')
    parser.add_argument('-p', dest='pickle_file', type=str,
        help='Pickle file containing data.',
        default='./cigma_data.pkl')
    parser.add_argument('-s', '--static', dest='static', default=False, action='store_true',
        help='Build static pages rather than hosting a dynamic webserver. [default=False]')
    args = parser.parse_args()

    host_cigma(args.pickle_file, static=args.static)


# Library imports
from bs4 import BeautifulSoup
import requests

# Generic web parser providing basic functionality to load a webpage
# Use this as a base class for more specific webpage parsers
class WebpageParser:
    # WebpageParser version
    version = '0.1'

    # Used to request web page
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36'}

    def __init__(self, url = None):
        # Store current URL
        self.current_url = url
        # Store full page content
        self.pageContent = None
        self.pageLoaded = False
        # Load page if URL provided
        if not (url == None):
            # setUrl also calls loadCurrentPage
            self.setUrl(url)

    def getVersion(self):
        return self.version

    # Set URL and attempt to load page
    # Return the return value of 'loadCurrentPage()
    def setUrl(self, url):
        self.current_url = url
        self.pageContent = None
        self.pageLoaded = False
        return self.loadCurrentPage()

    def getUrl(self):
        # Check if URL has not been specified yet
        if self.current_url == None:
            print("Note: A valid URL has not been specified!")
            return ""
        # Return URL
        else:
            return self.current_url

    # Attempt to load current page; Return True if successful
    def loadCurrentPage(self):
        # Check if a URL has been specified
        if self.current_url == None:
            print("Error: Cannot retrieve current page! A URL has not been specified!")
            self.pageLoaded = False
            return self.pageLoaded
        # Check if page for current URL has already been loaded
        elif self.pageLoaded:
            print("Note: Page has already been loaded for current URL.")
        # Assume that page for current URL has not been loaded
        else:
            # Try to load page
            try:
                req = requests.get(self.current_url, headers=self.headers)
                self.pageContent = BeautifulSoup(req.content, "lxml")
                self.pageLoaded = True
            except:
                print("Warning: Could not load page: %s" % self.current_url)
                self.pageContent = None
                self.pageLoaded = False
        # Return Boolean indicating success or failuer
        return self.pageLoaded

    # Get text from the html section specified by tag and attributes
    # tag is a string and attributes is a dictionary of attribute/value pairs
    def getTextFromSection(self, tag, attributes):
        text = ""
        # Return if URL has not been specified
        if self.current_url == None:
            print("Warning: Cannot get text from specified html, because URL has not been specified!")
            return text
        # If this point is reached, URL has been specified
        html = self.pageContent.find(tag, attributes)
        if not (html == None):
            text = html.text
        else:
            print("Warning: Cannot get text from specified html! Returning empty string.")
            text = ""
        return text


#PCA (principal component analysis) algorithm used for feature
# extraction and for dimensionality reduction.
#also can be used for explaining data behaviour
#Used in analysis of genome data and gene expression

#helps identify patterns based on correlation between features
#PCA aims to find directions of maximum variance in high-dimesnional
#data and project it into a new subspace with equal or fewer dimensions

# PCA for dimensionality reduction can be used for
# reducing a [DxK] vector to [n x D] vector where D>n
# Thus leads to vertical collapse of the table

# the first principal component will have largest possible
# variance and all consequent principal components are uncorrleated
# E(x).E(y)=E(x.y) or orthogonal in nature

# 1. standardizing the d - dimensional matrix
# 2. construct a covariance matrix
# 3. Decompose the covariance matrix into eignvalues and eignmatrix
# 4. Sort the eignvalues by decreasing order to rank the corresponding eigenvectors
# 5. Sort k eigenvectors which correspond to the k largest eigenvalues ,
#    where k is the dimesionality of the new feature subspace
# 6. Cosntruct a projection matrix W from the top k eigen values
# 7. Transform the d-dimensional input dataset using projection matrix
#    W to obtain the new k-dimesnional feature subspace

import pandas as pd

df_wine = pd.read_csv('https://archive.ics.uci.edu/ml/'
                      'machine-learning-databases/wine/wine.data',
                      header=None)

print(df_wine.head())

#dividing wine data into training and test set
# 70 : 30 split

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

#split into training and test sets

X , y = df_wine.iloc[:, 1:].values , df_wine.iloc[:,0].values
X_train , X_test , y_train , y_test = train_test_split(
  X , y , test_size = 0.3, stratify = y,
    random_state = 0
)

# standadizing the features

sc = StandardScaler()
X_train_std = sc.fit_transform(X_train)

# constructing co variance matrix
# the symetrix [d x d] matrix stores pairwise
# covariances between the different features

# covaraince between two features Xj and Xk given as
# 1/n * summision(X(i)j - Uj)*(X(i)k)-Uk)
# where U is the sample mean

# Sample means are zero if we standardized  the dataset
# A postive covariance shows that sample incerease and decrease together
# A negative covariance shows features vary in opposite manner

# eignvectors of covariance matrix represent the principal components
# (the direction of maximum covariance ) whereas the corresponding
# eign values define their magnitude
# we obtain 13 eigne vectors and eignvalues from 13*13 covariance matrix

# we need to obatian eigen pairs of covariance matrix
# eigen vector v satisfies  sum of v = lamda v
# lamda is a scale or the eigen value

import numpy as np
cov_mat = np.cov(X_train_std.T)
eigen_vals , eigen_vecs = np.linalg.eig(cov_mat)

# to reduce dimesionality of our dataset by comporessing it ,
# we only select the subset of eigenvectors (principal components)
# that have the most information (variance)

# plotting varince explained ratios

import matplotlib.pyplot as plt

# calculating cumulative sum of explained variances
tot = sum(eigen_vals)
var_exp = [(i/tot) for i in sorted(eigen_vals,reverse=True)]
cum_var_exp = np.cumsum(var_exp) # cumsum returns cummalitve sum along a axis

#plot explained variances
plt.bar(range(1,14),var_exp,alpha=0.5,align='center',label='indivisual explained variance')
plt.step(range(1,14),cum_var_exp,where='mid',label="cummalative explained varaince")
plt.xlabel("Principal component index")
plt.ylabel("Explained variance ratio")
plt.legend(loc="best")
plt.show()

# the plot shows that  the first principal component alone accounts for 40% of the variance
# the plot also shows tha the first and second principal component combined account for 60%
# variance

# sort the eigen pairs in descending manner , construct a porjection matrix
#from slected eigenvectors and use projection matrix to transform the data to lower dimension

#making list of eigenvalue , eigenvector tuples
eigen_pairs = [(np.abs(eigen_vals[i]),eigen_vecs[:,i]) for i in range(len(eigen_vals))]

#sort the (eigenvalue , eigenvector ) tuple from high to low
eigen_pairs.sort(key=lambda k:k[0], reverse=True)

#now we capture the top two eigen pairs to capture 60% of the variance
# selected two eigen values since ploting two dimensional values
# number of principal components has to be trade off between computational
# effiecny and performance

w = np.hstack((eigen_pairs[0][1][:,np.newaxis],eigen_pairs[1][1][:,np.newaxis]))
print('Matrix W: \n',w)

# hence W = projection matrix is [13*2] matix formed using top eigen vectors
# using w we now transform sample x represtend as [1*13] onto the PCA subsapce
# obtaining x' , x' = x . W

X_train_std[0].dot(w)

# similarly we can transform the entire [124* 13] training set onto
# two principal components by calculating the dot product
# X' = X.W

X_train_pca = X_train_std.dot(w)

# hence visualising wine data set as 124 * 2 dimesnional scatter plot

colors = ['r', 'b', 'g']
markers = ['s', 'x', 'o']
for l, c, m in zip(np.unique(y_train), colors, markers):
    plt.scatter(X_train_pca[y_train==l, 0],
                X_train_pca[y_train==l, 1],
                c=c, label=l, marker=m)
plt.xlabel('PC 1')
plt.ylabel('PC 2')
plt.legend(loc='lower left')
plt.show()


import json

from flask import Flask, render_template, request, Response
from flask_restful import Resource, Api
from flask_cors import CORS

import requests

app = Flask(__name__)
api = Api(app)
CORS(app)

@app.route('/')
def video():
    return render_template("video.html")

def setToken():
    f = open(f'conf/token.txt', 'r')
    token = f.read()
    Token = token
    f.close()
    return Token

def push_video(num):
    Token = setToken
    with open(f"rawdata/video.json" , 'r') as reader:
        items = json.loads(reader.read())
    try:
        requests.post(
        url='https://notify-api.line.me/api/notify',
        headers={
            'Authorization': f'Bearer {Token}'
        },
        data={'message': f"\n銀魂第{num}集~\n{items[num]}",
                })
    except Exception as e:
        print(e)

def get_video_list():
    with open(f"rawdata/video.json", 'r') as reader:
        data = json.loads(reader.read())
    return data

class Videos(Resource):
    def get(self):
        videos_list = get_video_list()
        return {"Data": {"Message": "Get an item", "Output": videos_list}}, 200

class GetVideo(Resource):
    def post(self, gid):
        try:
            push_video(gid)
            return {"Data": {"Message": "ok"}}, 200
        except Exception as e:
            return {"Data": {"Message": e}}, 401

api.add_resource(Videos, '/videos')
api.add_resource(GetVideo, '/getvideo/<string:gid>')

if __name__ == "__main__":
    app.run(host="0.0.0.0",port=5000)


from nornir import InitNornir
from nornir_scrapli.tasks import send_configs_from_file
from nornir_utils.plugins.functions import print_result

nr = InitNornir(config_file = "config.yaml")

def random_configs(task):
    task.run(task = send_configs_from_file, file = "002_bad_config.txt")

results = nr.run(task = random_configs)
print_result(results)

"""
Since NORNIR wraps all the exit codes in an aggregated object/results which completes execution with OK, regardless of any errors during the config push to devices.
If there is an error during the config push, NORNIR will save the error in data.failed_hosts.
Following code block will raise Error explicitly if the config push detects a failure, which Jenkins will detect, and mark the build as FAILED.
"""

failures = nr.data.failed_hosts
if failures:
    # print("FAILURE")
    raise NornirExecutionError("Nornir Failure Detected")


import numpy as np
import time

from ase import Atoms
from ase.calculators.calculator import Calculator, all_changes

from qml.kernels import (get_atomic_local_gradient_kernel,
                         get_atomic_local_kernel)
from qml.representations import generate_fchl_acsf

from rdkit import Chem
from rdkit.Chem import AllChem, ChemicalForceFields
from rdkit.Chem import rdmolfiles

def get_forcefield(molobj):

    ffprop = ChemicalForceFields.MMFFGetMoleculeProperties(molobj)
    forcefield = ChemicalForceFields.MMFFGetMoleculeForceField(molobj, ffprop) # 0.01 overhead

    return ffprop, forcefield

def run_forcefield(ff, steps, energy=1e-2, force=1e-3):
    """
    """

    try:
        status = ff.Minimize(maxIts=steps, energyTol=energy, forceTol=force)
    except RuntimeError:
        return 1

    return status

class QMLCalculator(Calculator):
    name = 'QMLCalculator'
    implemented_properties = ['energy', 'forces']

    def __init__(self, parameters, representations, charges, alphas, **kwargs):
        super().__init__(**kwargs)
        # unpack parameters
        offset = parameters["offset"]
        sigma = parameters["sigma"]

        self.set_model(alphas, representations, charges, offset, sigma)

        self.energy = 0.0

    def calculate(self, atoms: Atoms = None, properties=('energy', 'forces'),
                  system_changes=all_changes):

        if atoms is None:
            atoms = self.atoms
        if atoms is None:
            raise ValueError(
                'No ASE atoms supplied to calculation, and no ASE atoms supplied with initialisation.')

        self.query(atoms)

        if 'energy' in properties:
            self.results['energy'] = self.energy

        if 'forces' in properties:
            self.results['forces'] = self.forces

        return

    def set_model(self, alphas, representations, charges, offset, sigma):

        self.alphas = alphas
        self.repr = representations
        self.charges = charges

        # Offset from training
        self.offset = offset

        # Hyper-parameters
        self.sigma = sigma
        self.max_atoms = self.repr.shape[1]

        self.n_atoms = len(charges[0])

        return

    def query(self, atoms=None, print_time=True):

        if print_time:
            start = time.time()

        # kcal/mol til ev
        # kcal/mol/aangstrom til ev / aangstorm
        conv_energy = 0.0433635093659
        conv_force = 0.0433635093659

        coordinates = atoms.get_positions()
        nuclear_charges = atoms.get_atomic_numbers()
        n_atoms = coordinates.shape[0]

        new_cut = 4.0

        cut_parameters = {
                "rcut": new_cut,
                "acut": new_cut,
                # "nRs2": int(24 * new_cut / 8.0),
                # "nRs3": int(20 * new_cut / 8.0),
        }

        rep, drep = generate_fchl_acsf(
            nuclear_charges,
            coordinates,
            gradients=True,
            elements=[1,6,8],
            pad=self.max_atoms,
            **cut_parameters)

        # Put data into arrays
        Qs = [nuclear_charges]
        Xs = np.array([rep], order="F")
        dXs = np.array([drep], order="F")

        # Get kernels
        Kse = get_atomic_local_kernel(self.repr, Xs, self.charges, Qs, self.sigma)
        Ks = get_atomic_local_gradient_kernel(self.repr, Xs, dXs, self.charges, Qs, self.sigma)

        # Energy prediction
        energy_predicted = np.dot(Kse, self.alphas)[0] + self.offset
        self.energy = energy_predicted * conv_energy

        # Force prediction
        forces_predicted = np.dot(Ks, self.alphas).reshape((n_atoms, 3))
        self.forces = forces_predicted * conv_force

        if print_time:
            end = time.time()
            print("qml query {:7.3f}s {:10.3f} ".format(end-start, energy_predicted))

        return

    def get_potential_energy(self, atoms=None, force_consistent=False):

        energy = self.energy

        return energy

    def get_forces(self, atoms=None):

        self.query(atoms=atoms)
        forces = self.forces

        return forces

def get_calculator(dataname):

    datadir = "data/"
    ext = ".npy"

    filename_sigma = datadir + dataname + "_sigma" + ext
    filename_offset = datadir + dataname + "_offset" + ext
    filename_alphas = datadir + dataname + "_alphas" + ext
    filename_representations = datadir + dataname + "_X" + ext
    filename_charges = datadir + dataname + "_Q" + ext

    offset = np.load(filename_offset)
    sigma = np.load(filename_sigma)

    # LOAD AND SET MODEL
    parameters = {}
    parameters["offset"] = offset
    parameters["sigma"] = sigma

    alphas = np.load(filename_alphas)
    X = np.load(filename_representations)
    Q = np.load(filename_charges, allow_pickle=True)

    alphas = np.array(alphas, order="F")
    X = np.array(X, order="F")

    # SET CALCULATE CLASS
    calculator = QMLCalculator(parameters, X, Q, alphas)

    return calculator

class RdkitCalculator(Calculator):
    name = 'RdkitCalculator'
    implemented_properties = ['energy', 'forces']

    def __init__(self, molobj, **kwargs):
        super().__init__(**kwargs)

        self.set_model(molobj)
        self.energy = 0.0

    def calculate(self, atoms: Atoms = None, properties=('energy', 'forces'),
                  system_changes=all_changes):

        if atoms is None:
            atoms = self.atoms
        if atoms is None:
            raise ValueError(
                'No ASE atoms supplied to calculation, and no ASE atoms supplied with initialisation.')

        self.query(atoms)

        if 'energy' in properties:
            self.results['energy'] = self.energy

        if 'forces' in properties:
            self.results['forces'] = self.forces

        return

    def set_model(self, molobj):

        ffprop, ff = get_forcefield(molobj)
        conformer = molobj.GetConformer()

        self.conformer = conformer
        self.forcefield = ff

        coordinates = conformer.GetPositions()
        coordinates = np.asarray(coordinates)

        atoms = molobj.GetAtoms()
        atoms = [atom.GetAtomicNum() for atom in atoms]
        atoms = np.array(atoms)

        # start
        self.coordinates = coordinates
        self.nuclear_charges = atoms

        return

    def set_coordinates(self, coordinates):

        for i, pos in enumerate(coordinates):
            self.conformer.SetAtomPosition(i, pos)

        return

    def get_energy(self):

        energy = self.forcefield.CalcEnergy()

        return energy

    def query(self, atoms=None, print_time=True):

        if print_time:
            start = time.time()

        # kcal/mol til ev
        # kcal/mol/aangstrom til ev / aangstorm
        conv_energy = 0.0433635093659
        conv_force = 0.0433635093659

        coordinates = atoms.get_positions()
        nuclear_charges = atoms.get_atomic_numbers()
        n_atoms = coordinates.shape[0]

        self.set_coordinates(coordinates)

        # Energy prediction
        energy_predicted = self.get_energy()
        self.energy = energy_predicted * conv_energy

        # Force prediction
        forces_predicted = self.get_forces()
        self.forces = forces_predicted * conv_force

        if print_time:
            end = time.time()
            print("rdkit query {:7.4f}".format(end-start))

        return

    def get_potential_energy(self, atoms=None, force_consistent=False):

        energy = self.get_energy()

        return energy

    def get_forces(self, atoms=None):

        print(atoms)

        forces = self.forcefield.CalcGrad()
        forces = np.array(forces)
        # forces = forces.reshape(())

        return -forces

def read_sdffile(filename, remove_hs=False, sanitize=True):
    """
    """

    ext = filename.split(".")[-1]

    if ext == "sdf":

        suppl = Chem.SDMolSupplier(filename,
            removeHs=remove_hs,
            sanitize=sanitize)

    elif ext == "gz":

        fobj = gzip.open(filename)
        suppl = Chem.ForwardSDMolSupplier(fobj,
            removeHs=remove_hs,
            sanitize=sanitize)

    else:
        print("could not read file")
        quit()

    return suppl

def get_calculator_rdkit(filename):

    molobjs = read_sdffile(filename)
    molobjs = [molobj for molobj in molobjs]
    molobj = molobjs[0]

    calculator = RdkitCalculator(molobj)
    coordinates = calculator.coordinates
    nuclear_charges = calculator.nuclear_charges

    return nuclear_charges, coordinates, calculator


import numpy as np
from scipy.linalg import khatri_rao

def cp_decomposition(
    X : np.ndarray,
    rank : int = 5,
    max_iteration : int = 50
) -> tuple[np.ndarray, list[np.ndarray, np.ndarray, np.ndarray]]:
    '''
    '''
    I, J, K = X.shape
    A1 = np.random.randn(I, rank)
    A2 = np.random.randn(J, rank)
    A3 = np.random.randn(K, rank)

    A1 = A1 / np.linalg.norm(A1, axis = 0)
    A2 = A2 / np.linalg.norm(A2, axis = 0)
    A3 = A3 / np.linalg.norm(A3, axis = 0)

    factors = [A1, A2, A3]

    for iteration in range(max_iteration):
        for k in range(3):
            before = k - 1
            after = (k + 1) % 3
            V = np.multiply(
                factors[before].T @ factors[before],
                factors[after].T @ factors[after]
            )

            # Get kth mode of the tensor
            X_k = np.moveaxis(X, k, 0)
            X_k = X_k.reshape((X.shape[k], -1))

            # Get updates of factor matrices
            khatri_rao_product = khatri_rao(factors[before], factors[after])
            inverse = np.linalg.pinv(V.T @ V)
            factors[k] = X_k @ khatri_rao_product @ inverse @ V.T

    A1, A2, A3 = factors

    # Get the 1st mode unfolding and refold
    core_1st_mode = A1 @ khatri_rao(A2, A3).T
    core = core_1st_mode.reshape(X.shape)

    return core, factors


import torch as T
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np
import random

class DeepQNetwork(nn.Module):
    def __init__(self,
                 lr,
                 ):
        super().__init__()
        # FIXME: optimize for general case

        self.conv1 = nn.Conv2d(1, 32, 8, stride=4, padding=1)
        self.conv2 = nn.Conv2d(32, 64, 4, stride=2)
        self.conv3 = nn.Conv2d(64, 128, 3)
        self.fc1 = nn.Linear(128*19*8, 512)
        self.fc2 = nn.Linear(512, 6) # 6 actions

        self.optimizer = optim.RMSprop(self.parameters(), lr=lr)
        self.loss = nn.MSELoss()
        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')
        self.to(self.device)

    def forward(self, observations):
        observations = T.Tensor(observations).to(self.device)
        observations = observations.view(-1, 1, 185, 95)
        observations = F.relu(self.conv1(observations))
        observations = F.relu(self.conv2(observations))
        observations = F.relu(self.conv3(observations))
        observations = observations.view(-1, 128*19*8)
        observations = F.relu(self.fc1(observations))

        actions = self.fc2(observations)

        return actions

class Agent(object):
    def __init__(self, gamma, epsilon, lr,
                 maxMemorySize, epsEnd=0.05,
                 replace=10000, actionSpace=[0, 1, 2, 3, 4, 5]):
        self.GAMMA = gamma
        self.EPSILON = epsilon
        self.EPS_END = epsEnd
        self.actionSpace = actionSpace
        # self.replace = replace
        self.memSize = maxMemorySize
        self.steps = 0
        self.learn_step_counter = 0

        self.memory = []
        self.memCntr = 0
        self.replace_target_cnt = replace
        self.Q_eval = DeepQNetwork(lr)
        self.Q_next = DeepQNetwork(lr)

    def storeTransition(self, state, action, reward, state_):
        if self.memCntr < self.memSize:
            self.memory.append([state, action, reward, state_])
        else:
            self.memory[self.memCntr % self.memSize] = [state,
                                                        action,
                                                        reward,
                                                        state_]
        self.memCntr += 1

    def chooseAction(self, observation):
        rand = np.random.random()
        actions = self.Q_eval.forward(observation)
        if rand < 1 - self.EPSILON:
            action = T.argmax(actions[1]).item()
        else:
            action = np.random.choice(self.actionSpace)
        self.steps += 1

        return action

    def learn(self, batch_size):
        self.Q_eval.optimizer.zero_grad()
        if (self.replace_target_cnt is not None and \
            self.learn_step_counter % self.replace_target_cnt == 0):
            self.Q_next.load_state_dict(self.Q_eval.state_dict())

        if self.memCntr + batch_size < self.memSize:
            memStart = int(np.random.choice(range(self.memCntr)))
        else:
            memStart = int(np.random.choice(range(self.memCntr - batch_size - 1)))
        miniBatch = self.memory[memStart:memStart+batch_size]
        memory = np.array(miniBatch)

        Qpred = self.Q_eval.forward(list(memory[:, 0][:])).to(self.Q_eval.device) # 0 - state
        Qnext = self.Q_next.forward(list(memory[:, 3][:])).to(self.Q_next.device) # 3 - state_

        maxA = T.argmax(Qnext, dim=1).to(self.Q_next.device)
        rewards = T.Tensor(list(memory[:, 2])).to(self.Q_next.device)
        Qtarget = Qpred
        Qtarget[: maxA] = rewards + self.GAMMA*T.max(Qnext[1])

        if self.steps > 500:
            if self.EPSILON - 1e-4 > self.EPS_END:
                self.EPSILON -= 1e-4
            else:
                self.EPSILON = self.EPS_END

        loss = self.Q_eval.loss(Qtarget, Qpred).to(self.Q_eval.device)
        loss.backward()
        self.Q_eval.optimizer.step()
        self.learn_step_counter += 1


# -*- coding: utf-8 -*-
"""
@File    : analysis_list.py
@Time    : 2023/5/18 1:02 下午
@Author  : xxlaila
@Software: PyCharm
"""
from django.db import models
import logging
import uuid
from domains.models.domain_list import CLOUD_CHOICES

__all__ = ['AnalysisList']
logger = logging.getLogger(__name__)

MonitorStatus__CHOICES = (
    ("OK", "正常"),
    ("WARN", "告警"),
    ("DOWN", "宕机")
)

ENABLE = 1
DISABLE = 0
STATUS_CHOICES = (
    (ENABLE, "Enable"),
    (DISABLE, "Disable"),
)

class AnalysisList(models.Model):

    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
    defaultns = models.CharField(null=True, blank=True, max_length=32, verbose_name="默认记录")
    domainid = models.CharField(null=True, blank=True, max_length=256, verbose_name="域名标识")
    line = models.CharField(null=True, blank=True, max_length=32, verbose_name="线路")
    lineid = models.CharField(null=True, blank=True, max_length=32, verbose_name="线路Id")
    mx = models.CharField(null=True, blank=True, max_length=32, verbose_name="MX值")
    monitorstatus = models.CharField(choices=MonitorStatus__CHOICES, max_length=12, verbose_name="记录监控状态")
    subdomain = models.CharField(null=True, blank=True, max_length=64, verbose_name="主机名")
    secordid = models.BigIntegerField(null=True, blank=True, verbose_name="记录Id")
    remark = models.CharField(null=True, blank=True, max_length=256, verbose_name="备注")
    # status = models.CharField(null=True, blank=True, max_length=32)
    status = models.IntegerField(choices=STATUS_CHOICES, verbose_name="记录状态")
    ttl = models.CharField(null=True, blank=True, max_length=12, verbose_name="缓存时间")
    type = models.CharField(null=True, blank=True, max_length=32, verbose_name="记录类型")
    updatedon = models.DateTimeField(auto_now=True, null=True, verbose_name="更新时间")
    value = models.CharField(null=True, blank=True, max_length=128, verbose_name="记录值")
    weight = models.CharField(null=True, blank=True, max_length=32,  verbose_name="记录权重")
    cloud = models.CharField(choices=CLOUD_CHOICES, db_index=True, max_length=32, verbose_name='云')
    domain_name = models.CharField(null=True, max_length=128, blank=True, verbose_name="域名")
    created_by = models.CharField(max_length=128, blank=True, null=True, verbose_name='创建人')
    editd_by = models.CharField(max_length=128, blank=True, null=True, verbose_name="修改人")
    demand_by = models.CharField(max_length=128, blank=True, null=True, verbose_name='需求人')
    created_time = models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间')
    updated_time = models.DateTimeField(auto_now=True, verbose_name="更新时间")

    def __str__(self):
        return self.subdomain

    class Meta:
        ordering = ['-updated_time']
        verbose_name = "AnalysisList"
        verbose_name_plural = "AnalysisList"


import os
import numpy as np
import pandas as pd

import plotly

import plotly.express as px

import plotly.io as pio

from utilities.plotting_template import local_theme

pio.templates.default = local_theme

# Define population statistics

population_mean = 20

population_std = 3

# Number of samples to take for round of events

observation_count_range = np.arange(1, 31)

# Number of observations to take for each sampling event

sampling_events = 100

# Perform the sampling events

records = []

for sample_count in observation_count_range:
    for event in np.arange(sampling_events):
        # the sample consists of random variables drawn normally distributed data

        # (as defined with the population statistics above)

        sample = np.random.normal(
            loc=population_mean, scale=population_std, size=sample_count
        )

        sample_mean = np.mean(sample)

        records.append({"count": sample_count, "event": event, "mean": sample_mean})

# Create the dataframe (table) for the calculated values

df = pd.DataFrame(records)

# Plot the data to show how variation in sample mean changes with the sample count

fig = px.scatter(df, x="count", y="mean").update_layout(
    height=500,
    width=800,
    xaxis=dict(title="sample count (N)", rangemode="tozero"),
    yaxis=dict(title="sample mean"),
    margin=dict(t=20, b=20, l=20, r=20),
)

# Add the horizontal lines and labels

fig.add_hline(
    y=population_mean + population_std,
).add_hline(
    y=population_mean - population_std,
).add_annotation(
    text="Horizontal lines represent mean +/- one standard deviation",
    xref="paper",
    x=0.5,
    yref="paper",
    y=0.8,
    showarrow=False,
)

fig.show()

fig.write_image("small-numbers.svg")


"""
The Purpose of this script is to download skins into the project image directory.
Further processing will be necessary to determine format of the skin. Ex. v1.8 Standard vs v1.8 Slim vs Legacy version
"""
import concurrent
from concurrent.futures.thread import ThreadPoolExecutor

import requests
from progress.bar import Bar

def download_skin(skin_url, skin_id):
    response = requests.get(skin_url.format(skin_id), stream=True)
    # Check if the image was retrieved successfully
    if response.status_code == 200:
        # Open a local file with wb ( write binary ) permission.
        with open('images/skins/{}.png'.format(skin_id), 'wb') as f:
            f.write(response.content)
            return True
    return False

def main():
    skin_url = "https://www.minecraftskins.com/skin/download/{}"  # [15000000:15206628]
    start = 15148205
    stop = 15206629
    with Bar('Processing', max=stop - start) as bar:
        with ThreadPoolExecutor(max_workers=20) as executor:
            for skin_start in range(start, stop, 100):
                future_skins = {executor.submit(download_skin, skin_url, skin_id): skin_id for skin_id in
                                range(skin_start, skin_start + 100)}
                for future in concurrent.futures.as_completed(future_skins):
                    if future.result():
                        bar.next()

            # for skin_id in range(start, stop):
            #     download_skin(skin_url, skin_id)
            #     bar.next()

if __name__ == "__main__":
    main()


from .abstract_handler import AbstractHandler
from database.dbconn import DBConn
from database.dbquery import DBQuery
from database.dbaccess_manager import access_manager
from server.key_data_checker import valid_key_checker
import json
import uuid

class DeleteUserHandler(AbstractHandler):
	def request(self, req, res):
		res.content_type = "application/json"
		try:
			json_data = json.loads(req.data)
		except json.decoder.JSONDecodeError:
			res.status_code=403
			res.data = json.dumps({"error" : "Empty json"})
			return
		key = str(req.headers.get("Authorization"))
		# Authorization: Explicit Key
		explicit_key="Explicit: "
		idx = key.find(explicit_key)
		if idx == -1:
			res.status_code=401
			return

		key = key[len(explicit_key):]
		staff_id = valid_key_checker.get_owner(key)
		if staff_id == -1:
			res.status_code=401
			return

		if not json_data.__contains__('acc_id'):
			res.status_code=403
			res.data = json.dumps({"error" : "Expected values was not received"})
			return

		status = self.__queryDb(json_data['acc_id'])
		if not status:
			res.status_code=500
			return

		res.status_code=204

	def __queryDb(self, acc_id : str):
		conn_name = str(uuid.uuid4())
		conn = access_manager.connect(conn_name)
		str_query = '''DELETE FROM access WHERE acc_id = {id}
		'''.format(id=acc_id)
		query = DBQuery(conn)
		if not query.exec_query(str_query):
			print(query.get_error())
			return False

		access_manager.disconnect(conn_name)
		return True


import torch
import torchvision
from torch import nn
from torchvision import models
from torch.autograd import Variable
from torch.utils.data import DataLoader
import torch.nn.functional as F
from torchvision import transforms
from torchvision.utils import save_image
from skimage.measure import compare_ssim as ssim
from skimage.measure import compare_psnr as psnr
import os
from PIL import Image
from dataset.dataset import NYUUWDataset
from tqdm import tqdm
import random
from torchvision import models
import numpy as np
from models.networks import Classifier, UNetEncoder, UNetDecoder
import click
import datetime

def to_img(x):
	"""
		Convert the tanh (-1 to 1) ranged tensor to image (0 to 1) tensor
	"""

	x = 0.5 * (x + 1)
	x = x.clamp(0, 1)
	x = x.view(x.size(0), 3, 256, 256)

	return x

def set_requires_grad(nets, requires_grad=False):
	"""
		Make parameters of the given network not trainable
	"""

	if not isinstance(nets, list):
		nets = [nets]
	for net in nets:
		if net is not None:
			for param in net.parameters():
				param.requires_grad = requires_grad

	return requires_grad

def compute_val_metrics(fE, fI, fN, dataloader, no_adv_loss):
	"""
		Compute SSIM, PSNR scores for the validation set
	"""

	fE.eval()
	fI.eval()
	fN.eval()

	mse_scores = []
	ssim_scores = []
	psnr_scores = []
	corr = 0

	criterion_MSE = nn.MSELoss().cuda()

	for idx, data in tqdm(enumerate(dataloader)):
		uw_img, cl_img, water_type, _ = data
		uw_img = Variable(uw_img).cuda()
		cl_img = Variable(cl_img, requires_grad=False).cuda()

		fE_out, enc_outs = fE(uw_img)
		fI_out = to_img(fI(fE_out, enc_outs))
		fN_out = F.softmax(fN(fE_out), dim=1)

		if int(fN_out.max(1)[1].item()) == int(water_type.item()):
			corr += 1

		mse_scores.append(criterion_MSE(fI_out, cl_img).item())

		fI_out = (fI_out * 255).squeeze(0).cpu().data.numpy().transpose(1, 2, 0).astype(np.uint8)
		cl_img = (cl_img * 255).squeeze(0).cpu().data.numpy().transpose(1, 2, 0).astype(np.uint8)

		ssim_scores.append(ssim(fI_out, cl_img, multichannel=True))
		psnr_scores.append(psnr(cl_img, fI_out))

	fE.train()
	fI.train()
	if not no_adv_loss:
		fN.train()

	return sum(ssim_scores)/len(dataloader), sum(psnr_scores)/len(dataloader), sum(mse_scores)/len(dataloader), corr/len(dataloader)

def backward_I_loss(fI, fE_out, enc_outs, cl_img, criterion_MSE, optimizer_fI, lambda_I_loss, retain_graph):
	"""
		Backpropagate the reconstruction loss
	"""

	fI_out = to_img(fI(fE_out, enc_outs))
	I_loss = criterion_MSE(fI_out, cl_img) * lambda_I_loss
	optimizer_fI.zero_grad()
	I_loss.backward(retain_graph=retain_graph)
	optimizer_fI.step()

	return fI_out, I_loss

def backward_N_loss(fN, fE_out, actual_target, criterion_CE, optimizer_fN, lambda_N_loss):
	"""
		Backpropagate the nuisance loss
	"""

	fN_out = fN(fE_out.detach())
	N_loss = criterion_CE(fN_out, actual_target) * lambda_N_loss
	optimizer_fN.zero_grad()
	N_loss.backward()
	optimizer_fN.step()

	return N_loss

def backward_adv_loss(fN, fE_out, lambda_adv_loss, num_classes, neg_entropy):
	"""
		Backpropagate the adversarial loss
	"""

	fN_out = fN(fE_out)
	adv_loss = calc_adv_loss(fN_out, num_classes, neg_entropy) * lambda_adv_loss
	adv_loss.backward()

	return adv_loss

def write_to_log(log_file_path, status):
	"""
		Write to the log file
	"""

	with open(log_file_path, "a") as log_file:
		log_file.write(status+'\n')

def calc_adv_loss(fN_out, num_classes, neg_entropy):
	"""
		Calculate the adversarial loss (negative entropy or cross entropy with uniform distribution)
	"""

	if neg_entropy:
		fN_out_softmax = F.softmax(fN_out, dim=1)
		return torch.mean(torch.sum(fN_out_softmax * torch.log(torch.clamp(fN_out_softmax, min=1e-10, max=1.0)), 1))
	else:
		fN_out_log_softmax = F.log_softmax(fN_out, dim=1)
		return -torch.mean(torch.div(torch.sum(fN_out_log_softmax, 1), num_classes))

@click.command()
@click.argument('name')
@click.option('--data_path', default=None, help='Path of training input data')
@click.option('--label_path', default=None, help='Path of training label data')
@click.option('--learning_rate', default=1e-3, help='Learning rate')
@click.option('--batch_size', default=4, help='Batch size')
@click.option('--save_interval', default=5, help='Save models after this many epochs')
@click.option('--start_epoch', default=1, help='Start training from this epoch')
@click.option('--end_epoch', default=200, help='Train till this epoch')
@click.option('--num_classes', default=6, help='Number of water types')
@click.option('--num_channels', default=3, help='Number of input image channels')
@click.option('--train_size', default=30000, help='Size of the training dataset')
@click.option('--test_size', default=3000, help='Size of the testing dataset')
@click.option('--val_size', default=3000, help='Size of the validation dataset')
@click.option('--fe_load_path', default=None, help='Load path for pretrained fN')
@click.option('--fi_load_path', default=None, help='Load path for pretrained fE')
@click.option('--fn_load_path', default=None, help='Load path for pretrained fN')
@click.option('--lambda_i_loss', default=100.0, help='Lambda for I loss')
@click.option('--lambda_n_loss', default=1.0, help='Lambda for N loss')
@click.option('--lambda_adv_loss', default=1.0, help='Lambda for adv loss')
@click.option('--fi_threshold', default=0.9, help='Train fI till this threshold')
@click.option('--fn_threshold', default=0.85, help='Train fN till this threshold')
@click.option('--continue_train', is_flag=True, help='Continue training from start_epoch')
@click.option('--neg_entropy', default=True, help='Use KL divergence instead of cross entropy with uniform distribution')
@click.option('--no_adv_loss', is_flag=True, help='Use adversarial loss during training or not')
def main(name, data_path, label_path, learning_rate, batch_size, save_interval, start_epoch, end_epoch, num_classes, num_channels,
 train_size, test_size, val_size, fe_load_path, fi_load_path, fn_load_path, lambda_i_loss, lambda_n_loss, lambda_adv_loss,
 fi_threshold, fn_threshold, continue_train, neg_entropy, no_adv_loss):

	fE_load_path = fe_load_path
	fI_load_path = fi_load_path
	fN_load_path = fn_load_path

	lambda_I_loss = lambda_i_loss
	lambda_N_loss = lambda_n_loss

	fI_threshold = fi_threshold
	fN_threshold = fn_threshold

	# Define datasets and dataloaders
	train_dataset = NYUUWDataset(data_path,
		label_path,
		size=train_size,
		train_start=0,
		mode='train')

	val_dataset = NYUUWDataset(data_path,
		label_path,
		size=val_size,
		val_start=42000,
		mode='val')

	test_dataset = NYUUWDataset(data_path,
		label_path,
		size=test_size,
		test_start=46000,
		mode='test')

	train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
	test_dataloader = DataLoader(test_dataset, batch_size=1, shuffle=True)
	val_dataloader = DataLoader(val_dataset, batch_size=1, shuffle=True)

	if not no_adv_loss:
		"""
			Define the nuisance classifier to include the adversarial loss in the model
		"""

		fN = Classifier(num_classes).cuda()
		fN_req_grad = True
		fN.train()
		criterion_CE = nn.CrossEntropyLoss().cuda()
		optimizer_fN = torch.optim.Adam(fN.parameters(), lr=learning_rate,
								 weight_decay=1e-5)

	# Define models, criterion and optimizers
	fE = UNetEncoder(num_channels).cuda()
	fI = UNetDecoder(num_channels).cuda()

	criterion_MSE = nn.MSELoss().cuda()

	optimizer_fE = torch.optim.Adam(fE.parameters(), lr=learning_rate,
								 weight_decay=1e-5)
	optimizer_fI = torch.optim.Adam(fI.parameters(), lr=learning_rate,
								 weight_decay=1e-5)

	fE.train()
	fI.train()

	if continue_train:
		"""
			Load pretrained models to continue training
		"""

		if fE_load_path:
			fE.load_state_dict(torch.load(fE_load_path))
			print ('Loaded fE from {}'.format(fE_load_path))
		if fI_load_path:
			fI.load_state_dict(torch.load(fI_load_path))
			print ('Loaded fI from {}'.format(fI_load_path))
		if not no_adv_loss and fN_load_path:
			fN.load_state_dict(torch.load(fN_load_path))
			print ('Loaded fN from {}'.format(fN_load_path))

	if not os.path.exists('./checkpoints/{}'.format(name)):
		os.mkdir('./checkpoints/{}'.format(name))

	log_file_path = './checkpoints/{}/log_file.txt'.format(name)

	now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")

	status = '\nTRAINING SESSION STARTED ON {}\n'.format(now)
	write_to_log(log_file_path, status)

	# Compute the initial cross validation scores
	if continue_train and not no_adv_loss:
		fI_val_ssim, _, _, fN_val_acc = compute_val_metrics(fE, fI, fN, val_dataloader, no_adv_loss)
	else:
		fI_val_ssim = -1
		fN_val_acc = -1

	# Train only the encoder-decoder upto a certain threshold
	while fI_val_ssim < fI_threshold and not continue_train:
		epoch = start_epoch

		status = 'Avg fI val SSIM: {}, Avg fN val acc: {}\nCurrent fI threshold: {}, Current fN threshold: {}'.format(fI_val_ssim, fN_val_acc, fI_threshold, fN_threshold)
		print (status)
		write_to_log(log_file_path, status)

		for idx, data in tqdm(enumerate(train_dataloader)):
			uw_img, cl_img, water_type, _ = data
			uw_img = Variable(uw_img).cuda()
			cl_img = Variable(cl_img, requires_grad=False).cuda()

			fE_out, enc_outs = fE(uw_img)
			optimizer_fE.zero_grad()
			fI_out, I_loss = backward_I_loss(fI, fE_out, enc_outs, cl_img, criterion_MSE, optimizer_fI, lambda_I_loss, retain_graph=not no_adv_loss)

			progress = "\tEpoch: {}\tIter: {}\tI_loss: {}".format(epoch, idx, I_loss.item())

			optimizer_fE.step()

			if idx % 50 == 0:
				save_image(uw_img.cpu().data, './results/uw_img.png')
				save_image(fI_out.cpu().data, './results/fI_out.png')

				print (progress)
				write_to_log(log_file_path, progress)

		torch.save(fE.state_dict(), './checkpoints/{}/fE_latest.pth'.format(name))
		torch.save(fI.state_dict(), './checkpoints/{}/fI_latest.pth'.format(name))
		if epoch % save_interval == 0:
			torch.save(fE.state_dict(), './checkpoints/{}/fE_{}.pth'.format(name, epoch))
			torch.save(fI.state_dict(), './checkpoints/{}/fI_{}.pth'.format(name, epoch))

		status = 'End of epoch. Models saved.'
		print (status)
		write_to_log(log_file_path, status)

		fI_val_ssim, _, _, fN_val_acc = compute_val_metrics(fE, fI, fN, val_dataloader, no_adv_loss)

		start_epoch += 1

	for epoch in range(start_epoch, end_epoch):
		"""
			Main training loop
		"""

		if not no_adv_loss:
			"""
				Print the current cross-validation scores
			"""

			status = 'Avg fI val SSIM: {}, Avg fN val acc: {}\nCurrent fI threshold: {}, Current fN threshold: {}'.format(fI_val_ssim, fN_val_acc, fI_threshold, fN_threshold)
			print (status)
			write_to_log(log_file_path, status)

		for idx, data in tqdm(enumerate(train_dataloader)):
			uw_img, cl_img, water_type, _ = data
			uw_img = Variable(uw_img).cuda()
			cl_img = Variable(cl_img, requires_grad=False).cuda()
			actual_target = Variable(water_type, requires_grad=False).cuda()

			fE_out, enc_outs = fE(uw_img)

			if fI_val_ssim < fI_threshold:
				"""
					Train the encoder-decoder only
				"""

				optimizer_fE.zero_grad()
				fI_out, I_loss = backward_I_loss(fI, fE_out, enc_outs, cl_img, criterion_MSE, optimizer_fI, lambda_I_loss, retain_graph=not no_adv_loss)

				if not no_adv_loss:
					if fN_req_grad:
						fN_req_grad = set_requires_grad(fN, requires_grad=False)
					adv_loss = backward_adv_loss(fN, fE_out, lambda_adv_loss, num_classes, neg_entropy)
					progress = "\tEpoch: {}\tIter: {}\tI_loss: {}\tadv_loss: {}".format(epoch, idx, I_loss.item(), adv_loss.item())
				else:
					progress = "\tEpoch: {}\tIter: {}\tI_loss: {}".format(epoch, idx, I_loss.item())

				optimizer_fE.step()

				if idx % 50 == 0:
					save_image(uw_img.cpu().data, './results/uw_img.png')
					save_image(fI_out.cpu().data, './results/fI_out.png')

			elif fN_val_acc < fN_threshold:
				"""
					Train the nusiance classifier only
				"""

				if not fN_req_grad:
					fN_req_grad = set_requires_grad(fN, requires_grad=True)

				N_loss = backward_N_loss(fN, fE_out, actual_target, criterion_CE, optimizer_fN, lambda_N_loss)
				progress = "\tEpoch: {}\tIter: {}\tN_loss: {}".format(epoch, idx, N_loss.item())

			else:
				"""
					Train the encoder-decoder only
				"""

				optimizer_fE.zero_grad()
				fI_out, I_loss = backward_I_loss(fI, fE_out, enc_outs, cl_img, criterion_MSE, optimizer_fI, lambda_I_loss, retain_graph=not no_adv_loss)

				if not no_adv_loss:
					if fN_req_grad:
						fN_req_grad = set_requires_grad(fN, requires_grad=False)
					adv_loss = backward_adv_loss(fN, fE_out, lambda_adv_loss, num_classes, neg_entropy)

					progress = "\tEpoch: {}\tIter: {}\tI_loss: {}\tadv_loss: {}".format(epoch, idx, I_loss.item(), adv_loss.item())

				else:
					progress = "\tEpoch: {}\tIter: {}\tI_loss: {}".format(epoch, idx, I_loss.item())

				optimizer_fE.step()

				if idx % 50 == 0:
					save_image(uw_img.cpu().data, './results/uw_img.png')
					save_image(fI_out.cpu().data, './results/fI_out.png')

			if idx % 50 == 0:
				print (progress)
				write_to_log(log_file_path, progress)

		# Save models
		torch.save(fE.state_dict(), './checkpoints/{}/fE_latest.pth'.format(name))
		torch.save(fI.state_dict(), './checkpoints/{}/fI_latest.pth'.format(name))
		if not no_adv_loss:
			torch.save(fN.state_dict(), './checkpoints/{}/fN_latest.pth'.format(name))

		if epoch % save_interval == 0:
			torch.save(fE.state_dict(), './checkpoints/{}/fE_{}.pth'.format(name, epoch))
			torch.save(fI.state_dict(), './checkpoints/{}/fI_{}.pth'.format(name, epoch))
			if not no_adv_loss:
				torch.save(fN.state_dict(), './checkpoints/{}/fN_{}.pth'.format(name, epoch))

		status = 'End of epoch. Models saved.'
		print (status)
		write_to_log(log_file_path, status)

		if not no_adv_loss:
			"""
				Compute the cross validation scores after the epoch
			"""
			fI_val_ssim, _, _, fN_val_acc = compute_val_metrics(fE, fI, fN, val_dataloader, no_adv_loss)

if __name__== "__main__":
	if not os.path.exists('./results'):
		os.mkdir('./results')
	if not os.path.exists('./checkpoints'):
		os.mkdir('./checkpoints')

	main()


from django.urls import path

from .views import cartitems, checkout, contact, detail, home, index, shop, updateItem

app_name = "shop"

urlpatterns = [
    path("", home, name="home"),
    path("contact/", contact, name="contact"),
    path("cart/", cartitems, name="cart"),
    path("checkout/", checkout, name="checkout"),
    path("index/", index, name="index"),
    path("shop/", shop, name="shop"),
    path("detail/<str:id>/", detail, name="detail"),
    path("update-item/", updateItem, name="updateItem"),
]


# this is sample python code
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in
import os
import pandas as pd
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import seaborn as sns
import matplotlib.pyplot as plt
pd.set_option('display.float_format', lambda x: '%.2f' % x)
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory
#set plot size
plt.rcParams['figure.figsize']=(20,10)
print(os.listdir("../input/"))
dt = pd.read_csv("e:/dev/Kaggle/input/commodity_trade_statistics_data.csv", na_values=["No Quantity",0.0,''],sep=',')
# Any results you write to the current directory are saved as output.
#first things first, find total number of rows and columns
print("Shape................ :\n" + str(dt.shape))
#List all columns after data is laoded
#temp.columns.values # so we have 8,225,871 rows × 10 columns present in the data
#see names and data types
print("Data type Information.............. :\n" +  str(dt.dtypes))
#sort the data by year to tidy things before moving on
dt = dt.sort_values("year")
#delete temp dataset
#del(temp)
# Preview the data
dt.head(10)
# see missing values
dt.count()
#Preview preliminary statistics as is from the dat awithout cleanup
dt.describe(include="all")
dt = dt.dropna(how='any').reset_index(drop=True)
dt.describe(include="all")
#all missing values 'NaN' are messing up with prelim statisitics, lets treat missing cases by each variable
#dt.describe(include="all")
#First lets filter the data where there is no missing values
#pick first variable
#find unique values present other than nan
print('Country or Area')
dt['country_or_area'].nunique()
dt['country_or_area'].unique()
#there are 209 overall country or areas present in the data,
#Now lets see frequencies of each of these catergories.. only top 15, since there are a lot of categories
dt['country_or_area'].value_counts()[:15]

print('Commodity')
dt['commodity'].nunique()
dt['commodity'].unique()
dt['commodity'].value_counts()[:20]
dt.commodity.unique()

print('flow variable')
dt['flow'].nunique()
dt['flow'].unique()
dt['flow'].value_counts()
print('category')
dt['category'].nunique()
dt['category'].unique()
#there are 209 overall country or areas present in the data, Now lets see frequencies of each of these catergories, pick first 20 most occured
dt['category'].value_counts()[:20]

dfSheeps = dt[dt['commodity']=='Sheep, live'].reset_index(drop=True)
dfGoats = dt[dt['commodity']=='Goats, live'].reset_index(drop=True)
dfSheeps.head()
dfSheepsGrouped = pd.DataFrame({'weight_kg' : dfSheeps.groupby( ["year","flow","commodity"] )["weight_kg"].sum()}).reset_index()
dfGoatsGrouped = pd.DataFrame({'weight_kg' : dfGoats.groupby( ["year","flow","commodity"] )["weight_kg"].sum()}).reset_index()
dfSheepsGrouped.head()

f, ax = plt.subplots(1, 1)
dfgr = pd.concat([dfSheepsGrouped,dfGoatsGrouped])
ax = sns.pointplot(ax=ax,x="year",y="weight_kg",data=dfgr[dfgr['flow']=='Import'],hue='commodity')
_ = ax.set_title('Global imports of kgs by animal')

dfSheepsGrouped = pd.DataFrame({'weight_kg' : dfSheeps.groupby( ["country_or_area","flow","commodity"] )["weight_kg"].sum()}).reset_index()
dfSheepsGrouped.head()

sheepsImportsCountry = dfSheepsGrouped[dfSheepsGrouped['flow']=='Import']
sheepsExportsCountry = dfSheepsGrouped[dfSheepsGrouped['flow']=='Export']
sheepsImportsCountry.head()

ax = sns.barplot(x="weight_kg", y="country_or_area", data=sheepsImportsCountry.sort_values('weight_kg',ascending=False)[:15])
_ = ax.set(xlabel='Kgs', ylabel='Country or area',title = "Countries or areas that imported more kgs of Sheeps")

ax = sns.barplot(x="weight_kg", y="country_or_area", data=sheepsExportsCountry.sort_values('weight_kg',ascending=False)[:15])
_ = ax.set(xlabel='Kgs', ylabel='Country or area',title = "Countries or areas that exported more kgs of Sheeps")

#List of top 10 countries having highest trade in terms of dollars
dt.groupby(['country_or_area'])['trade_usd'].aggregate(np.sum).nlargest(10)

#List of top 10 countries having smallest trade in terms of dollars
dt.groupby(['country_or_area'])['trade_usd'].aggregate(np.sum).nsmallest(10)
#List of top 10 countries having highest trade in terms of weight
dt.groupby(['country_or_area'])['weight_kg'].aggregate(np.sum).nlargest(10)
#List of top 10 countries having smallest trade in terms of weight
dt.groupby(['country_or_area'])['weight_kg'].aggregate(np.sum).nsmallest(10)


import requests
import json
import log
import argparse
logger = log.get_logger(__name__)

def top_question(N, LABEL):
    URL = 'https://api.stackexchange.com//2.2/questions?pagesize={}&order=desc&sort=votes&tagged={}&site=stackoverflow'  # NOQA
    ses = requests.Session()
    resp = ses.get(URL.format(N, LABEL))
    if resp.status_code == 200:
        contents = json.loads(resp.text)['items']
        count = 0
        if not contents:
            print('Need to input valid Number and Label - '
                  'format int and str-str')
        else:
            for idx, data in enumerate(contents):
                count += 1
                print('Question {}: '.format(count) + contents[idx]['title']
                      + (' - Asker: ') + contents[idx]['owner']['display_name']
                      + '\n' + '--> Link: ' + contents[idx]['link'])

def main():
    parse = argparse.ArgumentParser(
        description="Get top question in stackoverflow.con")
    parse.add_argument('N', type=int, help='Number of Question')
    parse.add_argument('LABEL', type=str,
                       help='Input your keyword with sympol \- as whitespace')
    input_data = parse.parse_args()
    print(top_question(input_data.N, input_data.LABEL))

if __name__ == "__main__":
    main()


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 30 14:02:04 2017

@author: michael
"""

from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

import numpy as np
import tensorflow as tf
from deepchem.nn import activations
from deepchem.nn import initializations
from deepchem.nn import model_ops

from deepchem.models.tensorgraph.layers import Layer
from deepchem.models.tensorgraph.layers import convert_to_layers

class Combine_AP(Layer):

  def __init__(self, **kwargs):
    super(Combine_AP, self).__init__(**kwargs)

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)
    A = in_layers[0].out_tensor
    P = in_layers[1].out_tensor
    out_tensor = [A, P]
    if set_tensors:
      self.out_tensor = out_tensor
    return out_tensor

class Separate_AP(Layer):

  def __init__(self, **kwargs):
    super(Separate_AP, self).__init__(**kwargs)

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)
    out_tensor = in_layers[0].out_tensor[0]
    if set_tensors:
      self.out_tensor = out_tensor
    return out_tensor

class WeaveLayer(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.WeaveLayer

  """

  def __init__(self,
               n_atom_input_feat=75,
               n_pair_input_feat=14,
               n_atom_output_feat=50,
               n_pair_output_feat=50,
               n_hidden_AA=50,
               n_hidden_PA=50,
               n_hidden_AP=50,
               n_hidden_PP=50,
               update_pair=True,
               init='glorot_uniform',
               activation='relu',
               dropout=None,
               **kwargs):
    """
    Parameters
    ----------
    n_atom_input_feat: int
      Number of features for each atom in input.
    n_pair_input_feat: int
      Number of features for each pair of atoms in input.
    n_atom_output_feat: int
      Number of features for each atom in output.
    n_pair_output_feat: int
      Number of features for each pair of atoms in output.
    n_hidden_XX: int
      Number of units(convolution depths) in corresponding hidden layer
    update_pair: bool, optional
      Whether to calculate for pair features,
      could be turned off for last layer
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied
    dropout: float, optional
      Dropout probability, not supported here

    """
    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations
    self.update_pair = update_pair  # last weave layer does not need to update
    self.n_hidden_AA = n_hidden_AA
    self.n_hidden_PA = n_hidden_PA
    self.n_hidden_AP = n_hidden_AP
    self.n_hidden_PP = n_hidden_PP
    self.n_hidden_A = n_hidden_AA + n_hidden_PA
    self.n_hidden_P = n_hidden_AP + n_hidden_PP

    self.n_atom_input_feat = n_atom_input_feat
    self.n_pair_input_feat = n_pair_input_feat
    self.n_atom_output_feat = n_atom_output_feat
    self.n_pair_output_feat = n_pair_output_feat
    super(WeaveLayer, self).__init__(**kwargs)

  def build(self):
    """ Construct internal trainable weights.

    TODO(rbharath): Need to make this not set instance variables to
    follow style in other layers.
    """

    self.W_AA = self.init([self.n_atom_input_feat, self.n_hidden_AA])
    self.b_AA = model_ops.zeros(shape=[
        self.n_hidden_AA,
    ])

    self.W_PA = self.init([self.n_pair_input_feat, self.n_hidden_PA])
    self.b_PA = model_ops.zeros(shape=[
        self.n_hidden_PA,
    ])

    self.W_A = self.init([self.n_hidden_A, self.n_atom_output_feat])
    self.b_A = model_ops.zeros(shape=[
        self.n_atom_output_feat,
    ])

    self.trainable_weights = [
        self.W_AA, self.b_AA, self.W_PA, self.b_PA, self.W_A, self.b_A
    ]
    if self.update_pair:
      self.W_AP = self.init([self.n_atom_input_feat * 2, self.n_hidden_AP])
      self.b_AP = model_ops.zeros(shape=[
          self.n_hidden_AP,
      ])

      self.W_PP = self.init([self.n_pair_input_feat, self.n_hidden_PP])
      self.b_PP = model_ops.zeros(shape=[
          self.n_hidden_PP,
      ])

      self.W_P = self.init([self.n_hidden_P, self.n_pair_output_feat])
      self.b_P = model_ops.zeros(shape=[
          self.n_pair_output_feat,
      ])

      self.trainable_weights.extend(
          [self.W_AP, self.b_AP, self.W_PP, self.b_PP, self.W_P, self.b_P])

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """ description and explanation refer to deepchem.nn.WeaveLayer
    parent layers: [atom_features, pair_features], pair_split, atom_to_pair
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    self.build()

    atom_features = in_layers[0].out_tensor[0]
    pair_features = in_layers[0].out_tensor[1]

    pair_split = in_layers[1].out_tensor
    atom_to_pair = in_layers[2].out_tensor

    AA = tf.matmul(atom_features, self.W_AA) + self.b_AA
    AA = self.activation(AA)
    PA = tf.matmul(pair_features, self.W_PA) + self.b_PA
    PA = self.activation(PA)
    PA = tf.segment_sum(PA, pair_split)

    A = tf.matmul(tf.concat([AA, PA], 1), self.W_A) + self.b_A
    A = self.activation(A)

    if self.update_pair:
      AP_ij = tf.matmul(
          tf.reshape(
              tf.gather(atom_features, atom_to_pair),
              [-1, 2 * self.n_atom_input_feat]), self.W_AP) + self.b_AP
      AP_ij = self.activation(AP_ij)
      AP_ji = tf.matmul(
          tf.reshape(
              tf.gather(atom_features, tf.reverse(atom_to_pair, [1])),
              [-1, 2 * self.n_atom_input_feat]), self.W_AP) + self.b_AP
      AP_ji = self.activation(AP_ji)

      PP = tf.matmul(pair_features, self.W_PP) + self.b_PP
      PP = self.activation(PP)
      P = tf.matmul(tf.concat([AP_ij + AP_ji, PP], 1), self.W_P) + self.b_P
      P = self.activation(P)
    else:
      P = pair_features
    out_tensor = [A, P]
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

class WeaveGather(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.WeaveGather
  """

  def __init__(self,
               batch_size,
               n_input=128,
               gaussian_expand=False,
               init='glorot_uniform',
               activation='tanh',
               epsilon=1e-3,
               momentum=0.99,
               **kwargs):
    """
    Parameters
    ----------
    batch_size: int
      number of molecules in a batch
    n_input: int, optional
      number of features for each input molecule
    gaussian_expand: boolean. optional
      Whether to expand each dimension of atomic features by gaussian histogram
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied

    """
    self.n_input = n_input
    self.batch_size = batch_size
    self.gaussian_expand = gaussian_expand
    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations
    self.epsilon = epsilon
    self.momentum = momentum
    super(WeaveGather, self).__init__(**kwargs)

  def build(self):
    if self.gaussian_expand:
      self.W = self.init([self.n_input * 11, self.n_input])
      self.b = model_ops.zeros(shape=[
          self.n_input,
      ])
      self.trainable_weights = self.W + self.b
    else:
      self.trainable_weights = None

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """ description and explanation refer to deepchem.nn.WeaveGather
    parent layers: atom_features, atom_split
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    self.build()
    outputs = in_layers[0].out_tensor
    atom_split = in_layers[1].out_tensor

    if self.gaussian_expand:
      outputs = self.gaussian_histogram(outputs)

    output_molecules = tf.segment_sum(outputs, atom_split)

    if self.gaussian_expand:
      output_molecules = tf.matmul(output_molecules, self.W) + self.b
      output_molecules = self.activation(output_molecules)
    out_tensor = output_molecules
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

  def gaussian_histogram(self, x):
    gaussian_memberships = [(-1.645, 0.283), (-1.080, 0.170), (-0.739, 0.134),
                            (-0.468, 0.118), (-0.228, 0.114), (0., 0.114),
                            (0.228, 0.114), (0.468, 0.118), (0.739, 0.134),
                            (1.080, 0.170), (1.645, 0.283)]
    dist = [
        tf.contrib.distributions.Normal(mu=p[0], sigma=p[1])
        for p in gaussian_memberships
    ]
    dist_max = [dist[i].pdf(gaussian_memberships[i][0]) for i in range(11)]
    outputs = [dist[i].pdf(x) / dist_max[i] for i in range(11)]
    outputs = tf.stack(outputs, axis=2)
    outputs = outputs / tf.reduce_sum(outputs, axis=2, keep_dims=True)
    outputs = tf.reshape(outputs, [-1, self.n_input * 11])
    return outputs

class DTNNEmbedding(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.DTNNEmbedding
  """

  def __init__(self,
               n_embedding=30,
               periodic_table_length=83,
               init='glorot_uniform',
               **kwargs):
    """
    Parameters
    ----------
    n_embedding: int, optional
      Number of features for each atom
    periodic_table_length: int, optional
      Length of embedding, 83=Bi
    init: str, optional
      Weight initialization for filters.
    """
    self.n_embedding = n_embedding
    self.periodic_table_length = periodic_table_length
    self.init = initializations.get(init)  # Set weight initialization

    super(DTNNEmbedding, self).__init__(**kwargs)

  def build(self):

    self.embedding_list = self.init(
        [self.periodic_table_length, self.n_embedding])
    self.trainable_weights = [self.embedding_list]

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """description and explanation refer to deepchem.nn.DTNNEmbedding
    parent layers: atom_number
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    self.build()
    atom_number = in_layers[0].out_tensor
    atom_features = tf.nn.embedding_lookup(self.embedding_list, atom_number)
    self.out_tensor = atom_features

class DTNNStep(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.DTNNStep
  """

  def __init__(self,
               n_embedding=30,
               n_distance=100,
               n_hidden=60,
               init='glorot_uniform',
               activation='tanh',
               **kwargs):
    """
    Parameters
    ----------
    n_embedding: int, optional
      Number of features for each atom
    n_distance: int, optional
      granularity of distance matrix
    n_hidden: int, optional
      Number of nodes in hidden layer
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied
    """
    self.n_embedding = n_embedding
    self.n_distance = n_distance
    self.n_hidden = n_hidden
    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations

    super(DTNNStep, self).__init__(**kwargs)

  def build(self):
    self.W_cf = self.init([self.n_embedding, self.n_hidden])
    self.W_df = self.init([self.n_distance, self.n_hidden])
    self.W_fc = self.init([self.n_hidden, self.n_embedding])
    self.b_cf = model_ops.zeros(shape=[
        self.n_hidden,
    ])
    self.b_df = model_ops.zeros(shape=[
        self.n_hidden,
    ])

    self.trainable_weights = [
        self.W_cf, self.W_df, self.W_fc, self.b_cf, self.b_df
    ]

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """description and explanation refer to deepchem.nn.DTNNStep
    parent layers: atom_features, distance, distance_membership_i, distance_membership_j
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    self.build()
    atom_features = in_layers[0].out_tensor
    distance = in_layers[1].out_tensor
    distance_membership_i = in_layers[2].out_tensor
    distance_membership_j = in_layers[3].out_tensor
    distance_hidden = tf.matmul(distance, self.W_df) + self.b_df
    atom_features_hidden = tf.matmul(atom_features, self.W_cf) + self.b_cf
    outputs = tf.multiply(distance_hidden,
                          tf.gather(atom_features_hidden,
                                    distance_membership_j))

    # for atom i in a molecule m, this step multiplies together distance info of atom pair(i,j)
    # and embeddings of atom j(both gone through a hidden layer)
    outputs = tf.matmul(outputs, self.W_fc)
    outputs = self.activation(outputs)

    output_ii = tf.multiply(self.b_df, atom_features_hidden)
    output_ii = tf.matmul(output_ii, self.W_fc)
    output_ii = self.activation(output_ii)

    # for atom i, sum the influence from all other atom j in the molecule
    outputs = tf.segment_sum(outputs,
                             distance_membership_i) - output_ii + atom_features
    out_tensor = outputs
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

class DTNNGather(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.DTNNGather
  """

  def __init__(self,
               n_embedding=30,
               n_outputs=100,
               layer_sizes=[100],
               init='glorot_uniform',
               activation='tanh',
               **kwargs):
    """
    Parameters
    ----------
    n_embedding: int, optional
      Number of features for each atom
    n_outputs: int, optional
      Number of features for each molecule(output)
    layer_sizes: list of int, optional(default=[1000])
      Structure of hidden layer(s)
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied
    """
    self.n_embedding = n_embedding
    self.n_outputs = n_outputs
    self.layer_sizes = layer_sizes
    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations

    super(DTNNGather, self).__init__(**kwargs)

  def build(self):
    self.W_list = []
    self.b_list = []
    prev_layer_size = self.n_embedding
    for i, layer_size in enumerate(self.layer_sizes):
      self.W_list.append(self.init([prev_layer_size, layer_size]))
      self.b_list.append(model_ops.zeros(shape=[
          layer_size,
      ]))
      prev_layer_size = layer_size
    self.W_list.append(self.init([prev_layer_size, self.n_outputs]))
    self.b_list.append(model_ops.zeros(shape=[
        self.n_outputs,
    ]))
    prev_layer_size = self.n_outputs

    self.trainable_weights = self.W_list + self.b_list

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """description and explanation refer to deepchem.nn.DTNNGather
    parent layers: atom_features, atom_membership
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    self.build()
    output = in_layers[0].out_tensor
    atom_membership = in_layers[1].out_tensor
    for i, W in enumerate(self.W_list):
      output = tf.matmul(output, W) + self.b_list[i]
      output = self.activation(output)
    output = tf.segment_sum(output, atom_membership)
    out_tensor = output
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

class DAGLayer(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.DAGLayer
  """

  def __init__(self,
               n_graph_feat=30,
               n_atom_feat=75,
               max_atoms=50,
               layer_sizes=[100],
               init='glorot_uniform',
               activation='relu',
               dropout=None,
               batch_size=64,
               **kwargs):
    """
    Parameters
    ----------
    n_graph_feat: int, optional
      Number of features for each node(and the whole grah).
    n_atom_feat: int, optional
      Number of features listed per atom.
    max_atoms: int, optional
      Maximum number of atoms in molecules.
    layer_sizes: list of int, optional(default=[1000])
      Structure of hidden layer(s)
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied
    dropout: float, optional
      Dropout probability, not supported here
    batch_size: int, optional
      number of molecules in a batch
    """
    super(DAGLayer, self).__init__(**kwargs)

    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations
    self.layer_sizes = layer_sizes
    self.dropout = dropout
    self.max_atoms = max_atoms
    self.batch_size = batch_size
    self.n_inputs = n_atom_feat + (self.max_atoms - 1) * n_graph_feat
    # number of inputs each step
    self.n_graph_feat = n_graph_feat
    self.n_outputs = n_graph_feat
    self.n_atom_feat = n_atom_feat

  def build(self):
    """"Construct internal trainable weights.
    """

    self.W_list = []
    self.b_list = []
    prev_layer_size = self.n_inputs
    for layer_size in self.layer_sizes:
      self.W_list.append(self.init([prev_layer_size, layer_size]))
      self.b_list.append(model_ops.zeros(shape=[
          layer_size,
      ]))
      prev_layer_size = layer_size
    self.W_list.append(self.init([prev_layer_size, self.n_outputs]))
    self.b_list.append(model_ops.zeros(shape=[
        self.n_outputs,
    ]))

    self.trainable_weights = self.W_list + self.b_list

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """description and explanation refer to deepchem.nn.DAGLayer
    parent layers: atom_features, parents, calculation_orders, calculation_masks, n_atoms
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    # Add trainable weights
    self.build()

    atom_features = in_layers[0].out_tensor
    # each atom corresponds to a graph, which is represented by the `max_atoms*max_atoms` int32 matrix of index
    # each gragh include `max_atoms` of steps(corresponding to rows) of calculating graph features
    parents = in_layers[1].out_tensor
    # target atoms for each step: (batch_size*max_atoms) * max_atoms
    calculation_orders = in_layers[2].out_tensor
    calculation_masks = in_layers[3].out_tensor

    n_atoms = in_layers[4].out_tensor
    # initialize graph features for each graph
    graph_features_initial = tf.zeros((self.max_atoms * self.batch_size,
                                       self.max_atoms + 1, self.n_graph_feat))
    # initialize graph features for each graph
    # another row of zeros is generated for padded dummy atoms
    graph_features = tf.Variable(graph_features_initial, trainable=False)

    for count in range(self.max_atoms):
      # `count`-th step
      # extracting atom features of target atoms: (batch_size*max_atoms) * n_atom_features
      mask = calculation_masks[:, count]
      current_round = tf.boolean_mask(calculation_orders[:, count], mask)
      batch_atom_features = tf.gather(atom_features, current_round)

      # generating index for graph features used in the inputs
      index = tf.stack(
          [
              tf.reshape(
                  tf.stack(
                      [tf.boolean_mask(tf.range(n_atoms), mask)] *
                      (self.max_atoms - 1),
                      axis=1), [-1]),
              tf.reshape(tf.boolean_mask(parents[:, count, 1:], mask), [-1])
          ],
          axis=1)
      # extracting graph features for parents of the target atoms, then flatten
      # shape: (batch_size*max_atoms) * [(max_atoms-1)*n_graph_features]
      batch_graph_features = tf.reshape(
          tf.gather_nd(graph_features, index),
          [-1, (self.max_atoms - 1) * self.n_graph_feat])

      # concat into the input tensor: (batch_size*max_atoms) * n_inputs
      batch_inputs = tf.concat(
          axis=1, values=[batch_atom_features, batch_graph_features])
      # DAGgraph_step maps from batch_inputs to a batch of graph_features
      # of shape: (batch_size*max_atoms) * n_graph_features
      # representing the graph features of target atoms in each graph
      batch_outputs = self.DAGgraph_step(batch_inputs, self.W_list, self.b_list)

      # index for targe atoms
      target_index = tf.stack([tf.range(n_atoms), parents[:, count, 0]], axis=1)
      target_index = tf.boolean_mask(target_index, mask)
      # update the graph features for target atoms
      graph_features = tf.scatter_nd_update(graph_features, target_index,
                                            batch_outputs)

    out_tensor = batch_outputs
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

  def DAGgraph_step(self, batch_inputs, W_list, b_list):
    outputs = batch_inputs
    for idw, W in enumerate(W_list):
      outputs = tf.nn.xw_plus_b(outputs, W, b_list[idw])
      outputs = self.activation(outputs)
    return outputs

class DAGGather(Layer):
  """ TensorGraph style implementation
  The same as deepchem.nn.DAGGather
  """

  def __init__(self,
               n_graph_feat=30,
               n_outputs=30,
               max_atoms=50,
               layer_sizes=[100],
               init='glorot_uniform',
               activation='relu',
               dropout=None,
               **kwargs):
    """
    Parameters
    ----------
    n_graph_feat: int, optional
      Number of features for each atom
    n_outputs: int, optional
      Number of features for each molecule.
    max_atoms: int, optional
      Maximum number of atoms in molecules.
    layer_sizes: list of int, optional
      Structure of hidden layer(s)
    init: str, optional
      Weight initialization for filters.
    activation: str, optional
      Activation function applied
    dropout: float, optional
      Dropout probability, not supported
    """
    super(DAGGather, self).__init__(**kwargs)

    self.init = initializations.get(init)  # Set weight initialization
    self.activation = activations.get(activation)  # Get activations
    self.layer_sizes = layer_sizes
    self.dropout = dropout
    self.max_atoms = max_atoms
    self.n_graph_feat = n_graph_feat
    self.n_outputs = n_outputs

  def build(self):
    """"Construct internal trainable weights.
    """

    self.W_list = []
    self.b_list = []
    prev_layer_size = self.n_graph_feat
    for layer_size in self.layer_sizes:
      self.W_list.append(self.init([prev_layer_size, layer_size]))
      self.b_list.append(model_ops.zeros(shape=[
          layer_size,
      ]))
      prev_layer_size = layer_size
    self.W_list.append(self.init([prev_layer_size, self.n_outputs]))
    self.b_list.append(model_ops.zeros(shape=[
        self.n_outputs,
    ]))

    self.trainable_weights = self.W_list + self.b_list

  def create_tensor(self, in_layers=None, set_tensors=True, **kwargs):
    """description and explanation refer to deepchem.nn.DAGGather
    parent layers: atom_features, membership
    """
    if in_layers is None:
      in_layers = self.in_layers
    in_layers = convert_to_layers(in_layers)

    # Add trainable weights
    self.build()

    # Extract atom_features
    atom_features = in_layers[0].out_tensor
    membership = in_layers[1].out_tensor
    # Extract atom_features
    graph_features = tf.segment_sum(atom_features, membership)
    # sum all graph outputs
    outputs = self.DAGgraph_step(graph_features, self.W_list, self.b_list)
    out_tensor = outputs
    if set_tensors:
      self.variables = self.trainable_weights
      self.out_tensor = out_tensor
    return out_tensor

  def DAGgraph_step(self, batch_inputs, W_list, b_list):
    outputs = batch_inputs
    for idw, W in enumerate(W_list):
      outputs = tf.nn.xw_plus_b(outputs, W, b_list[idw])
      outputs = self.activation(outputs)
    return outputs