import os
import json
import numpy as np
from enum import IntEnum
import random
from mopgrid.simerrors import SimulationError, SimulationErrorCode

class CellType(IntEnum):
    EMPTY = 0
    WALL = 1
    DIRTY = 2

class Coords:
    def __init__(self, row, col):
        self.row = row
        self.col = col

    def __str__(self):
        return "[" + str(self.row) + ", " + str(self.col) + "]"

class AgentState:
    def __init__(self, agent_id, loc):
        self.agent_id = agent_id
        self.loc = loc

    def move_to(self, to_loc):
        self.loc = to_loc

class Command:
    def __init__(self, agent_id, command, location=None):
        self.agent_id = agent_id
        self.command = command
        self.location = location
        self.status = False
        self.failure_reason = None

def clean_command(agent_id, loc):
    return Command(agent_id=agent_id, command="clean", location=loc)

def moveto_command(agent_id, loc):
    return Command(agent_id=agent_id, command="move_to", location=loc)

class Space:
    def __init__(self, size=Coords(10, 10), dirty_prob=0.3, wall_prob=0.2):
        self.size = size
        self.dirty_prob = dirty_prob
        self.wall_prob = wall_prob
        self.space = np.zeros([size.row, size.col], dtype=int)
        self.agent_space = np.zeros([size.row, size.col], dtype=int)
        self.random_wall()
        self.random_dirty()
        self.agents = {}

    def random_wall(self):
        for y in range(self.size.row):
            for x in range(self.size.col):
                r = random.random()
                if r <= self.wall_prob:
                    self.space[y, x] = CellType.WALL

    def random_dirty(self):
        for y in range(self.size.row):
            for x in range(self.size.col):
                if self.space[y, x] != CellType.WALL:
                    r = random.random()
                    if r <= self.dirty_prob:
                        self.space[y, x] = CellType.DIRTY

    def query(self, loc):
        return self.space[loc.row, loc.col]

    def is_dirty(self, loc):
        return self.query(loc) == CellType.DIRTY

    def is_wall(self, loc):
        return self.query(loc) == CellType.WALL

    def is_clean(self, loc):
        return not self.is_dirty(loc)

    def has_agent(self, loc):
        val = self.query(loc)
        if val > 2 and val % 2 == 1:
            return True
        else:
            return False

    def can_place_agent(self, loc):
        if self.query(loc) == CellType.EMPTY or self.is_dirty(loc):
            return True
        return False

    def where_am_i(self, agent_id):
        a = self.agents[agent_id]
        if a is not None:
            return a.loc
        else:
            return None

    def init_agent(self):
        count_tries = 0
        while count_tries < 10:
            row = random.randint(0, self.size.row - 1)
            col = random.randint(0, self.size.col - 1)
            loc = Coords(row, col)

            if self.can_place_agent(loc):
                agent_id = -1
                if len(self.agents.keys()) == 0:
                    agent_id = 3
                else:
                    agent_id = max(self.agents.keys()) + 2
                print("agent added = " + str(agent_id))
                self.agents[agent_id] = AgentState(agent_id, loc)
                self.agent_space[loc.row, loc.col] = agent_id
                return agent_id
            count_tries += 1

    def check_range(self, loc):
        if loc.row < 0 or loc.row >= self.size.row or loc.col < 0 or loc.col >= self.size.col:
            raise SimulationError("Row/col is out of range: " + str(loc),
                                  SimulationErrorCode.SIM_ERR_OUT_OF_RANGE, loc)

    def move_to(self, agent_id, to_loc):
        self.check_range(to_loc)
        if agent_id in self.agents:
            a = self.agents[agent_id]
            if abs(to_loc.row - a.loc.row) < 2 and abs(to_loc.col - a.loc.col) < 2:
                if not self.is_wall(to_loc):
                    if self.is_clean(to_loc) or self.is_dirty(to_loc):
                        self.agent_space[a.loc.row, a.loc.col] = CellType.EMPTY
                        self.agent_space[to_loc.row, to_loc.col] = agent_id
                        a.move_to(to_loc)
                    else:
                        raise SimulationError("There is already an agent there!",
                                              SimulationErrorCode.SIM_ERR_AGENT_COLLISION, to_loc)
                else:
                    raise SimulationError("Cannot move into a wall!",
                                          SimulationErrorCode.SIM_ERR_MOVE_TO_WALL, to_loc)
            else:
                raise SimulationError("New location is not adjacent!",
                                      SimulationErrorCode.SIM_ERR_LOCATION_NOT_ADJACENT, to_loc)
        else:
            raise SimulationError("Agent does not exist!",
                                  SimulationErrorCode.SIM_ERR_NO_SUCH_AGENT, to_loc)

    def clean(self, agent_id, to_loc):
        self.check_range(to_loc)
        if agent_id in self.agents:
            a = self.agents[agent_id]
            if a.loc.row == to_loc.row and a.loc.col == to_loc.col:
                if self.is_dirty(to_loc):
                    self.space[to_loc.row, to_loc.col] = CellType.EMPTY
                else:
                    raise SimulationError("Location is not dirty!", SimulationErrorCode.SIM_ERR_LOCATION_NOT_DIRTY,
                                          to_loc)
            else:
                raise SimulationError("Agent is not at the location to clean!",
                                      SimulationErrorCode.SIM_ERR_AGENT_NOT_AT_LOCATION, to_loc)
        else:
            raise SimulationError("Agent does not exist!",
                                  SimulationErrorCode.SIM_ERR_NO_SUCH_AGENT, to_loc)

    def total_dirty(self):
        count = 0
        for i in range(self.size.row):
            for j in range(self.size.col):
                if self.is_dirty(Coords(i, j)):
                    count += 1
        return count

    def has_dirty(self):
        # can improve by returning after finding first dirty cell
        return self.total_dirty() > 0

def sample_grid():
    gpath = os.path.join(os.getcwd(), "..", "test", "data", "sample0.json")
    with open(gpath, "r") as f:
        gstr = f.read()
        g = json.loads(gstr)
    return g

if __name__ == "__main__":
    s = Space()
    print(s.space)


#!/usr/bin/env python3

import json
from nba_db_record import PlayerGameStatsRecord, PlayerRecord, TeamRecord, GameRecord, StatAggregateRecord
from flask import Flask, request, jsonify
import requests
import pyodbc
from json import JSONEncoder
from sql_connection import SqlConnection

app = Flask(__name__)

# ------------------------
# Player Api's
# ------------------------

@app.route('/api/nba/player', methods=['GET'])
def get_player_by_name():
    first_name = request.args.get('first_name')
    last_name = request.args.get('last_name')

    player_rec = PlayerRecord()
    player_rec.FirstName = first_name
    player_rec.LastName = last_name

    cn = SqlConnection()

    player_rec.get_player_rec_by_name(cn)
    json_rec = player_rec.__dict__
    response = jsonify(json_rec)
    response.headers.add("Access-Control-Allow-Origin", "*")

    return response

@app.route('/api/nba/player/list', methods=['GET'])
def get_player_list():

    player_rec = PlayerRecord()

    cn = SqlConnection()

    player_list = player_rec.get_player_list(cn)

    # iterate over each player rec and convert into dictionary
    player_json_list = []
    for rec in player_list:
        player_json_list.append(rec.__dict__)

    response = jsonify(player_json_list)
    response.headers.add("Access-Control-Allow-Origin", "*")

    return response

# ------------------------
# Team Api's
# ------------------------

@app.route('/api/nba/team', methods=['GET'])
def get_team_by_abbreviation():
    # get query string parameter from the Flask request object
    abbreviation = request.args.get('abbreviation')

    # create team record instance
    team_rec = TeamRecord()

    # assign the abbreviation field with the value of the query string parameter
    team_rec.Abbreviation = abbreviation

    # get sql connection
    cn = SqlConnection()

    # populate record instance with database values
    team_rec.get_team_rec_by_abbreviation(cn)

    # convert record instance to a dictionary
    json_rec = team_rec.__dict__

    # create a Flask response object and set the mime-type to application/json
    response = jsonify(json_rec)

    # allow CORS (Cross Origin Resource Sharing)
    response.headers.add("Access-Control-Allow-Origin", "*")

    return response

@app.route('/api/nba/team/list', methods=['GET'])
def get_team_list():

    team_rec = TeamRecord()

    cn = SqlConnection()

    # get list of TeamRecord instances
    team_list = team_rec.get_team_list(cn)

    # list to hold TeamRecord dictionaries
    team_json_list = []

    # iterate over each TeamRecord instance, convert it to a dictionary and
    # add to dictionary list
    for rec in team_list:
        team_json_list.append(rec.__dict__)

    response = jsonify(team_json_list)

    response.headers.add("Access-Control-Allow-Origin", "*")

    return response

# ------------------------
# Game Api's
# ------------------------

@app.route('/api/nba/game/list', methods=['GET'])
def get_game_list():
    game_rec = GameRecord()

    cn = SqlConnection()

    # get list of TeamRecord instances
    game_list = game_rec.get_game_list(cn)

    # list to hold TeamRecord dictionaries
    game_json_list = []

    # iterate over each TeamRecord instance, convert it to a dictionary and
    # add to dictionary list
    for rec in game_list:
        game_json_list.append(rec.__dict__)

    response = jsonify(game_json_list)

    response.headers.add("Access-Control-Allow-Origin", "*")

    return response

# ------------------------
# Game Stats Api
# ------------------------

@app.route('/api/nba/game/stats/list', methods=['GET'])
def get_game_stats_list():
    response = {}

    # get query string parameter from the Flask request object
    season = request.args.get('season')
    if season is None:
        season = 2020

    stat = request.args.get('stat')
    if stat is None:
        return response

    game_stats_rec = PlayerGameStatsRecord()

    cn = SqlConnection()

    # get list of TeamRecord instances
    game_stats_list = game_stats_rec.get_game_stats_list(cn, season, stat)

    # list to hold TeamRecord dictionaries
    game_stats_json_list = []

    # iterate over each TeamRecord instance, convert it to a dictionary and
    # add to dictionary list
    for rec in game_stats_list:
        new_dict = {'GameDate': rec.GameDate, 'PostSeason': rec.PostSeason}
        if stat == 'Min':
            new_dict['Min'] = rec.Min
            game_stats_json_list.append(new_dict)
        elif stat == 'Ast':
            new_dict['Ast'] = rec.Ast
            game_stats_json_list.append(new_dict)
        elif stat == 'Blk':
            new_dict['Blk'] = rec.Blk
            game_stats_json_list.append(new_dict)
        elif stat == 'Dreb':
            new_dict['Dreb'] = rec.Dreb
            game_stats_json_list.append(new_dict)
        elif stat == 'Fg3a':
            new_dict['Fg3a'] = rec.Fg3a
            game_stats_json_list.append(new_dict)
        elif stat == 'Fg3m':
            new_dict['Fg3m'] = rec.Fg3m
            game_stats_json_list.append(new_dict)
        elif stat == 'Fg3Pct':
            new_dict['Fg3Pct'] = rec.Fg3Pct
            game_stats_json_list.append(new_dict)
        elif stat == 'Fga':
            new_dict['Fga'] = rec.Fga
            game_stats_json_list.append(new_dict)
        elif stat == 'Fgm':
            new_dict['Fgm'] = rec.Fgm
            game_stats_json_list.append(new_dict)
        elif stat == 'FgPct':
            new_dict['FgPct'] = rec.FgPct
            game_stats_json_list.append(new_dict)
        elif stat == 'Fta':
            new_dict['Fta'] = rec.Fta
            game_stats_json_list.append(new_dict)
        elif stat == 'Ftm':
            new_dict['Ftm'] = rec.Ftm
            game_stats_json_list.append(new_dict)
        elif stat == 'FtPct':
            new_dict['FtPct'] = rec.FtPct
            game_stats_json_list.append(new_dict)
        elif stat == 'Oreb':
            new_dict['Oreb'] = rec.Oreb
            game_stats_json_list.append(new_dict)
        elif stat == 'Pf':
            new_dict['Pf'] = rec.Pf
            game_stats_json_list.append(new_dict)
        elif stat == 'Pts':
            new_dict['Pts'] = rec.Pts
            game_stats_json_list.append(new_dict)
        elif stat == 'Reb':
            new_dict['Reb'] = rec.Reb
            game_stats_json_list.append(new_dict)
        elif stat == 'Stl':
            new_dict['Stl'] = rec.Stl
            game_stats_json_list.append(new_dict)
        elif stat == 'Turnover':
            new_dict['Turnover'] = rec.Turnover
            game_stats_json_list.append(new_dict)
        # game_stats_json_list.append(rec.__dict__)

    response = jsonify(game_stats_json_list)

    response.headers.add("Access-Control-Allow-Origin", "*")

    return response

@app.route('/api/nba/game/stats/aggregate', methods=['GET'])
def get_stat_aggregate_list():
    response = {}

    # get query string parameter from the Flask request object
    season = request.args.get('season')
    if season is None:
        season = 2020

    stat = request.args.get('stat')
    if stat is None:
        return response

    stat_aggregate_rec = StatAggregateRecord()

    cn = SqlConnection()

    stat_aggregate_list = stat_aggregate_rec.get_stat_aggregate(cn, season, stat)

    stat_aggregate_json_list = []

    for rec in stat_aggregate_list:
        new_dict = {'StatName': rec.StatName, 'StatDate': rec.StatDate, 'StatAvg': rec.StatAvg, 'StatMax': rec.StatAvg}
        stat_aggregate_json_list.append(rec.__dict__)

    response = jsonify(stat_aggregate_json_list)

    response.headers.add("Access-Control-Allow-Origin", "*")

    return response

if __name__ == '__main__':
    from waitress import serve
    serve(app, host="localhost", port=5001)


"""
A module for cropping data from the VGG Face 2 dataset,
available at http://www.robots.ox.ac.uk/~vgg/data/vgg_face2/

The website contains not only images but also face bounding boxes
that we use to crop each image. We then resize each image
to a default of 160 x 160 x 3, which is the input
for the FaceNet model available from
https://github.com/nyoki-mtl/keras-facenet
"""
import os
import tensorflow as tf
import numpy as np
import pandas as pd
import h5py
from tqdm import tqdm
from skimage.util import img_as_ubyte
from utils import set_up_environment, maximum_center_crop, prewhiten, l2_normalize, read_sampled_identities

from PIL import Image

from absl import app, flags

VGG_BASE = '/projects/leelab3/image_datasets/vgg_face/'
#VGG_BASE = '/data/vggface'

FLAGS = flags.FLAGS
flags.DEFINE_string('image_directory',
                    os.path.join(VGG_BASE, 'test'),
                    'Top level directory for images')
flags.DEFINE_string('output_directory',
                    os.path.join(VGG_BASE, 'test_preprocessed_sampled'),
                    'Top level directory to output adversarially-modified images')
flags.DEFINE_string('bbox_file',
                    os.path.join(VGG_BASE, 'bb_landmark/loose_bb_test.csv'),
                    'CSV file containing bounding boxes for each desired image')
flags.DEFINE_string('sampled_identities',
                    'sampled_identities.txt',
                    'txt file generated by sample_identities.py to preprocess')
flags.DEFINE_string('op',
                   'preprocess',
                    'One of `preprocess` or `write_embeddings` - operation mode for script')
flags.DEFINE_integer('resize_dimension',
                     160,
                     'Dimension to resize all images to')
flags.DEFINE_string('visible_devices', '0', 'CUDA parameter')
flags.DEFINE_integer('batch_size',
                     64,
                     'Batch size to use for creating embeddings')
flags.DEFINE_string('model_path',
                    'keras-facenet/model/facenet_keras.h5',
                    'Path to keras model')

flags.DEFINE_boolean('process_train', False, 'Turn on this flag to process the training set')

def preprocess():
    """
    Pre-processes the VGG Face 2 dataset by cropping, resizing and
    writing as h5 files, which should be faster to read than
    numpy arrays or jpg files.
    """
    # Bounding box data is from
    # http://www.robots.ox.ac.uk/~vgg/data/vgg_face2/meta_infor.html
    # CSV file is of the form: NAME_ID, X, Y, W, H
    bbox_df = pd.read_csv(FLAGS.bbox_file)

    if FLAGS.sampled_identities != "":
        sampled = read_sampled_identities(FLAGS.sampled_identities)
        sampled = pd.DataFrame(np.array([["{}/{}".format(k, v.split(".")[0]) for v in sampled[k]] for k in sampled]).flatten())
        # do a join to filter selected identities and frames
        bbox_df = pd.merge(left=bbox_df, right=sampled, left_on='NAME_ID', right_on=0)

    image_batch = []
    for index, row in tqdm(bbox_df.iterrows()):
#         name_id, x, y, w, h, _ = row
        name_id, x, y, w, h = row
        current_identity = name_id.split('/')[0]
        x = max(x, 0)
        y = max(y, 0)
        # Read in the image
        image_path = os.path.join(FLAGS.image_directory, name_id + '.jpg')
        image = Image.open(image_path).crop((x, y, x + w, y + h)).resize(size=(FLAGS.resize_dimension,
                                                                              FLAGS.resize_dimension))
        resized_image = np.array(image)
        ubyte_image    = img_as_ubyte(resized_image)
        image_batch.append(ubyte_image)
        # When we are done with a batch of images (one identity)...
        if index == len(bbox_df) - 1 or \
           current_identity != bbox_df.loc[index + 1, 'NAME_ID'].split('/')[0]:
            image_batch = np.stack(image_batch, axis=0)
            os.makedirs(os.path.join(FLAGS.output_directory,
                                     current_identity), exist_ok=True)
            # ...we save the batch of images belonging to that identity to an
            # h5 file.
            file_path = os.path.join(FLAGS.output_directory, current_identity, 'images.h5')
            with h5py.File(file_path, 'w') as dataset_file:
                dataset_file.create_dataset('images', data=image_batch)

            image_batch = []

def write_embeddings():
    """
    Runs the embedding model over the pre-processed dataset and saves the embeddings into h5py datasets.
    """
    set_up_environment(visible_devices=FLAGS.visible_devices)
    model = tf.keras.models.load_model(FLAGS.model_path)

    if FLAGS.sampled_identities != "":
        sampled = read_sampled_identities(FLAGS.sampled_identities)

    identities = os.listdir(FLAGS.image_directory)
    for identity in tqdm(identities):
        # bypass identities not in sample, if a sampled file is provided
        if FLAGS.sampled_identities != "" and not (identity in sampled):
            continue

        image_file = os.path.join(FLAGS.image_directory,
                                  identity,
                                  'images.h5')

        with h5py.File(image_file, 'r') as f:
            images = f['images'][:]

        images_whitened = prewhiten(images).astype(np.float32)
        embeddings = model.predict(images_whitened, batch_size=FLAGS.batch_size)
        normalized_embeddings = l2_normalize(embeddings)

        os.makedirs(os.path.join(FLAGS.output_directory,
                                 identity),
                    exist_ok=True)

        embedding_path = os.path.join(FLAGS.output_directory,
                                      identity,
                                      'embeddings.h5')
        with h5py.File(embedding_path, 'w') as dataset_file:
            dataset_file.create_dataset('embeddings', data=normalized_embeddings)

def main(argv=None):
    '''
    Call either preprocess or write_embeddings and print error if not one of these
    '''
    try:
        func = globals()[FLAGS.op]
    except KeyError:
        raise Exception("FLAGS.op must be one of `preprocess` or `write_embeddings`, currently is {}".format(FLAGS.op))

    func()

if __name__ == '__main__':
    app.run(main)


import types

import numpy as np
import scipy.integrate as integrate

from pgmpy.factors.base import BaseFactor
from pgmpy.factors.distributions import GaussianDistribution, CustomDistribution

class ContinuousFactor(BaseFactor):
    """
    Base class for factors representing various multivariate
    representations.
    """

    def __init__(self, variables, pdf, *args, **kwargs):
        """
        Parameters
        ----------
        variables: list or array-like
            The variables for which the distribution is defined.

        pdf: function
            The probability density function of the distribution.

        Examples
        --------
        >>> import numpy as np
        >>> from scipy.special import beta
        >>> from pgmpy.factors.continuous import ContinuousFactor
        # Two variable dirichlet distribution with alpha = (1,2)
        >>> def dirichlet_pdf(x, y):
        ...     return (np.power(x, 1) * np.power(y, 2)) / beta(x, y)
        >>> dirichlet_factor = ContinuousFactor(['x', 'y'], dirichlet_pdf)
        >>> dirichlet_factor.scope()
        ['x', 'y']
        >>> dirichlet_factor.assignment(5,6)
        226800.0
        """
        if not isinstance(variables, (list, tuple, np.ndarray)):
            raise TypeError(
                f"variables: Expected type list or array-like, got type {type(variables)}"
            )

        if len(set(variables)) != len(variables):
            raise ValueError("Variable names cannot be same.")

        variables = list(variables)

        if isinstance(pdf, str):
            if pdf == "gaussian":
                self.distribution = GaussianDistribution(
                    variables=variables,
                    mean=kwargs["mean"],
                    covariance=kwargs["covariance"],
                )
            else:
                raise NotImplementedError(
                    f"{pdf} distribution not supported. Please use CustomDistribution"
                )

        elif isinstance(pdf, CustomDistribution):
            self.distribution = pdf

        elif callable(pdf):
            self.distribution = CustomDistribution(
                variables=variables, distribution=pdf
            )

        else:
            raise ValueError(
                f"pdf: Expected type: str or function, Got: {type(variables)}"
            )

    @property
    def pdf(self):
        """
        Returns the pdf of the ContinuousFactor.
        """
        return self.distribution.pdf

    @property
    def variable(self):
        return self.scope()[0]

    def scope(self):
        """
        Returns the scope of the factor.

        Returns
        -------
        list: List of variable names in the scope of the factor.

        Examples
        --------
        >>> from pgmpy.factors.continuous import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> normal_pdf = lambda x: multivariate_normal(x, [0, 0], [[1, 0], [0, 1]])
        >>> phi = ContinuousFactor(['x1', 'x2'], normal_pdf)
        >>> phi.scope()
        ['x1', 'x2']
        """
        return self.distribution.variables

    def get_evidence(self):
        return self.scope()[1:]

    def assignment(self, *args):
        """
        Returns a list of pdf assignments for the corresponding values.

        Parameters
        ----------
        *args: values
            Values whose assignment is to be computed.

        Examples
        --------
        >>> from pgmpy.factors.continuous import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> normal_pdf = lambda x1, x2: multivariate_normal.pdf((x1, x2), [0, 0], [[1, 0], [0, 1]])
        >>> phi = ContinuousFactor(['x1', 'x2'], normal_pdf)
        >>> phi.assignment(1, 2)
        0.013064233284684921
        """
        return self.distribution.assignment(*args)

    def copy(self):
        """
        Return a copy of the distribution.

        Returns
        -------
        ContinuousFactor object: copy of the distribution

        Examples
        --------
        >>> import numpy as np
        >>> from scipy.special import beta
        >>> from pgmpy.factors.continuous import ContinuousFactor
        # Two variable dirichlet distribution with alpha = (1,2)
        >>> def dirichlet_pdf(x, y):
        ...     return (np.power(x, 1) * np.power(y, 2)) / beta(x, y)
        >>> dirichlet_factor = ContinuousFactor(['x', 'y'], dirichlet_pdf)
        >>> dirichlet_factor.variables
        ['x', 'y']
        >>> copy_factor = dirichlet_factor.copy()
        >>> copy_factor.variables
        ['x', 'y']
        """
        return ContinuousFactor(self.scope(), self.distribution.copy())

    def discretize(self, method, *args, **kwargs):
        """
        Discretizes the continuous distribution into discrete
        probability masses using various methods.

        Parameters
        ----------
        method : A Discretizer Class from pgmpy.discretize

        *args, **kwargs:
            The parameters to be given to the Discretizer Class.

        Returns
        -------
        An n-D array or a DiscreteFactor object according to the discretiztion
        method used.

        Examples
        --------
        >>> import numpy as np
        >>> from scipy.special import beta
        >>> from pgmpy.factors.continuous import ContinuousFactor
        >>> from pgmpy.factors.continuous import RoundingDiscretizer
        >>> def dirichlet_pdf(x, y):
        ...     return (np.power(x, 1) * np.power(y, 2)) / beta(x, y)
        >>> dirichlet_factor = ContinuousFactor(['x', 'y'], dirichlet_pdf)
        >>> dirichlet_factor.discretize(RoundingDiscretizer, low=1, high=2, cardinality=5)
        # TODO: finish this
        """
        return method(self, *args, **kwargs).get_discrete_values()

    def reduce(self, values, inplace=True):
        """
        Reduces the factor to the context of the given variable values.

        Parameters
        ----------
        values: list, array-like
            A list of tuples of the form (variable_name, variable_value).

        inplace: boolean
            If inplace=True it will modify the factor itself, else would return
            a new ContinuousFactor object.

        Returns
        -------
        ContinuousFactor or None: if inplace=True (default) returns None
                                  if inplace=False returns a new ContinuousFactor instance.

        Examples
        --------
        >>> import numpy as np
        >>> from scipy.special import beta
        >>> from pgmpy.factors.continuous import ContinuousFactor
        >>> def custom_pdf(x, y, z):
        ...     return z*(np.power(x, 1) * np.power(y, 2)) / beta(x, y)
        >>> custom_factor = ContinuousFactor(['x', 'y', 'z'], custom_pdf)
        >>> custom_factor.variables
        ['x', 'y', 'z']
        >>> custom_factor.assignment(1, 2, 3)
        24.0

        >>> custom_factor.reduce([('y', 2)])
        >>> custom_factor.variables
        ['x', 'z']
        >>> custom_factor.assignment(1, 3)
        24.0
        """
        phi = self if inplace else self.copy()

        phi.distribution = phi.distribution.reduce(values, inplace=False)
        if not inplace:
            return phi

    def marginalize(self, variables, inplace=True):
        """
        Marginalize the factor with respect to the given variables.

        Parameters
        ----------
        variables: list, array-like
            List of variables with respect to which factor is to be maximized.

        inplace: boolean
            If inplace=True it will modify the factor itself, else would return
            a new ContinuousFactor instance.

        Returns
        -------
        DiscreteFactor or None: if inplace=True (default) returns None
                        if inplace=False returns a new ContinuousFactor instance.

        Examples
        --------
        >>> from pgmpy.factors.continuous import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> std_normal_pdf = lambda *x: multivariate_normal.pdf(x, [0, 0], [[1, 0], [0, 1]])
        >>> std_normal = ContinuousFactor(['x1', 'x2'], std_normal_pdf)
        >>> std_normal.scope()
        ['x1', 'x2']
        >>> std_normal.assignment([1, 1])
        0.058549831524319168
        >>> std_normal.marginalize(['x2'])
        >>> std_normal.scope()
        ['x1']
        >>> std_normal.assignment(1)

        """
        phi = self if inplace else self.copy()
        phi.distribution = phi.distribution.marginalize(variables, inplace=False)

        if not inplace:
            return phi

    def normalize(self, inplace=True):
        """
        Normalizes the pdf of the continuous factor so that it integrates to
        1 over all the variables.

        Parameters
        ----------
        inplace: boolean
            If inplace=True it will modify the factor itself, else would return
            a new factor.

        Returns
        -------
        ContinuousFactor or None:
             if inplace=True (default) returns None
             if inplace=False returns a new ContinuousFactor instance.

        Examples
        --------
        >>> from pgmpy.factors.continuous import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> std_normal_pdf = lambda x: 2 * multivariate_normal.pdf(x, [0, 0], [[1, 0], [0, 1]])
        >>> std_normal = ContinuousFactor(['x1', 'x2'], std_normal_pdf)
        >>> std_normal.assignment(1, 1)
        0.117099663049
        >>> std_normal.normalize()
        >>> std_normal.assignment(1, 1)
        0.0585498315243

        """
        phi = self if inplace else self.copy()
        phi.distribution.normalize(inplace=True)

        if not inplace:
            return phi

    def is_valid_cpd(self):
        return self.distribution.is_valid_cpd()

    def _operate(self, other, operation, inplace=True):
        """
        Gives the ContinuousFactor operation (product or divide) with
        the other factor.

        Parameters
        ----------
        other: ContinuousFactor
            The ContinuousFactor to be multiplied.

        operation: String
            'product' for multiplication operation and 'divide' for
            division operation.

        inplace: boolean
            If inplace=True it will modify the factor itself, else would return
            a new factor.

        Returns
        -------
        ContinuousFactor or None:
                        if inplace=True (default) returns None
                        if inplace=False returns a new `DiscreteFactor` instance.

        """
        if not isinstance(other, ContinuousFactor):
            raise TypeError(
                f"ContinuousFactor objects can only be multiplied ",
                f"or divided with another ContinuousFactor object. ",
                f"Got {type(other)}, expected: ContinuousFactor.",
            )

        phi = self if inplace else self.copy()
        phi.distribution = phi.distribution._operate(
            other=other.distribution, operation=operation, inplace=False
        )

        if not inplace:
            return phi

    def product(self, other, inplace=True):
        """
        Gives the ContinuousFactor product with the other factor.

        Parameters
        ----------
        other: ContinuousFactor
            The ContinuousFactor to be multiplied.

        Returns
        -------
        ContinuousFactor or None:
                        if inplace=True (default) returns None
                        if inplace=False returns a new `ContinuousFactor` instance.

        Examples
        --------
        >>> from pgmpy.factors.continuous import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> sn_pdf1 = lambda x: multivariate_normal.pdf([x], [0], [[1]])
        >>> sn_pdf2 = lambda x1,x2: multivariate_normal.pdf([x1, x2], [0, 0], [[1, 0], [0, 1]])
        >>> sn1 = ContinuousFactor(['x2'], sn_pdf1)
        >>> sn2 = ContinuousFactor(['x1', 'x2'], sn_pdf2)

        >>> sn3 = sn1.product(sn2, inplace=False)
        >>> sn3.assignment(0, 0)
        0.063493635934240983

        >>> sn3 = sn1 * sn2
        >>> sn3.assignment(0, 0)
        0.063493635934240983
        """
        return self._operate(other, "product", inplace)

    def divide(self, other, inplace=True):
        """
        Gives the ContinuousFactor divide with the other factor.

        Parameters
        ----------
        other: ContinuousFactor
            The ContinuousFactor to be divided.

        Returns
        -------
        ContinuousFactor or None:
                        if inplace=True (default) returns None
                        if inplace=False returns a new `ContinuousFactor` instance.

        Examples
        --------
        >>> from pgmpy.factors.continuous import ContinuousFactor
        >>> from scipy.stats import multivariate_normal
        >>> sn_pdf1 = lambda x: multivariate_normal.pdf([x], [0], [[1]])
        >>> sn_pdf2 = lambda x1,x2: multivariate_normal.pdf([x1, x2], [0, 0], [[1, 0], [0, 1]])
        >>> sn1 = ContinuousFactor(['x2'], sn_pdf1)
        >>> sn2 = ContinuousFactor(['x1', 'x2'], sn_pdf2)

        >>> sn4 = sn2.divide(sn1, inplace=False)
        >>> sn4.assignment(0, 0)
        0.3989422804014327

        >>> sn4 = sn2 / sn1
        >>> sn4.assignment(0, 0)
        0.3989422804014327
        """
        if set(other.scope()) - set(self.scope()):
            raise ValueError("Scope of divisor should be a subset of dividend")

        return self._operate(other, "divide", inplace)

    def __mul__(self, other):
        return self.product(other, inplace=False)

    def __rmul__(self, other):
        return self.__mul__(other)

    def __truediv__(self, other):
        return self.divide(other, inplace=False)

    __div__ = __truediv__


import psycopg2
from datetime import datetime
from nebula_feed import config

conn = psycopg2.connect(config.db_url, sslmode="require")

# Open a cursor to perform database operations
cur = conn.cursor()

# Execute a command: this creates a new table
#cur.execute("create table ClaimedPlanets (TokenId int not null, PlanetName varchar(50) not null, ClaimedDate timestamp not null);")

#cur.execute("insert into ClaimedPlanets(TokenId, PlanetName, ClaimedDate) values (%s, %s, %s);",
#    (100, "test", datetime.now()))

#cur.execute("truncate table ClaimedPlanets;")

cur.execute("select * from ClaimedPlanets;")
test = cur.fetchall()
print(test)

# Make the changes to the database persistent
conn.commit()

# Close communication with the database
cur.close()
conn.close()


from airflow.models import DAG
from airflow.operators.empty import EmptyOperator
from airflow.operators.python import BranchPythonOperator, PythonOperator
import pandas as pd
import os
import pendulum

from great_expectations_provider.operators.great_expectations import GreatExpectationsOperator
from great_expectations.core.batch import BatchRequest
from great_expectations.data_context.types.base import (
    DataContextConfig,
    CheckpointConfig
)

ge_root_dir='/opt/x5/great_expectations'
data_path = '/data'
archive_name = 'retailhero-uplift.zip'

dag = DAG(
    dag_id="x5-data-preparation",
    schedule_interval="@daily",
    start_date=pendulum.datetime(2022, 1, 1, tz="UTC"),
)

def download_zip_archive():
    if not archive_name in os.listdir():
        os.system('wget -P ./ https://storage.yandexcloud.net/datasouls-ods/materials/9c6913e5/retailhero-uplift.zip')

def unzip_data_archive():
    os.system(f'unzip ./{archive_name}')

def join_data():
    df_clients = pd.read_csv(f'{data_path}/clients.csv', index_col='client_id')
    df_train = pd.read_csv(f'{data_path}/uplift_train.csv', index_col='client_id')
    df_test = pd.read_csv(f'{data_path}/uplift_test.csv', index_col='client_id')

    df_clients['first_issue_unixtime'] = pd.to_datetime(df_clients['first_issue_date']).astype(int)/10**9
    df_clients['first_redeem_unixtime'] = pd.to_datetime(df_clients['first_redeem_date']).astype(int)/10**9
    df_features = pd.DataFrame({
        'gender_M': (df_clients['gender'] == 'M').astype(int),
        'gender_F': (df_clients['gender'] == 'F').astype(int),
        'gender_U': (df_clients['gender'] == 'U').astype(int),
        'age': df_clients['age'],
        'first_issue_time': df_clients['first_issue_unixtime'],
        'first_redeem_time': df_clients['first_redeem_unixtime'],
        'issue_redeem_delay': df_clients['first_redeem_unixtime'] - df_clients['first_issue_unixtime'],
    }).fillna(0)
    df_features.to_csv(f'{data_path}/df_features.csv')

download_zip = PythonOperator(
    task_id='download_zip',
    python_callable = download_zip_archive,
    op_kwargs = {},
    dag=dag,
)

unzip_archive = PythonOperator(
    task_id='unzip_archive',
    python_callable = unzip_data_archive,
    op_kwargs = {},
    dag=dag,
)

data_peprocessing = PythonOperator(
    task_id='data_peprocessing',
    python_callable = join_data,
    op_kwargs = {},
    dag=dag,
)

ge_client_data = GreatExpectationsOperator(
    task_id="ge_client_data",
    data_context_root_dir=ge_root_dir,
    checkpoint_name="getting_started",
)

#test

download_zip >> unzip_archive >> ge_client_data  >> data_peprocessing


from typing import Any
from .file import FileExerciseRun, SelfFileExercise
from ..design import design, DS
from ..utils.string import get_scopes_var_assign
from ..utils.input import input_type

def input_prompts(text: str) -> tuple[list[Any], dict[str, Any]]:
  scopes = get_scopes_var_assign(text)
  args = []
  locals = {}
  for var, type, prompt in scopes:
    try: type = eval(type)
    except: type = str

    prompt = design(prompt, DS.Strip("\n"))
    value = input_type(prompt, type)
    if var: locals[var] = value
    else: args.append(value)

  return (args, locals)

def process(
  __source: str,
  __globals: dict[str, Any],
  __locals: dict[str, Any]
) -> dict[str, Any]:
  exec(__source, __globals, __locals)
  return dict(locals())["__locals"]

class ExerciseRun(FileExerciseRun):
  _file: SelfFileExercise

  @property
  def statement(self) -> bool:
    return self._file.section.statement is not None

  @property
  def short_statement(self) -> bool:
    return self._file.section.short_statement is not None

  @property
  def input_prompts(self) -> bool:
    text = self._file.section.input_prompts
    if text is None: return False
    try: argv, locals = input_prompts(text)
    except: return False
    self._locals.update(locals)
    self._globals["argv"] = argv
    return True

  @property
  def process(self) -> bool:
    source = self._file.section.process
    success = True
    try: self._locals = process(source, self._globals, self._locals)
    except: success = False
    return success

  @property
  def output_menssage(self) -> bool:
    output = self._file.section.output_menssage
    success = True
    try: self._output = output.format(*self._globals["argv"], **self._locals)
    except: success = False
    return success

  def exercise(self, force: bool = False) -> str | None:
    if not self._output or force:
      self.input_prompts,
      properties = (
        self.process,
        self.output_menssage
      )
      if not all(properties): return
    return self._output


import os
import pyarrow.parquet as pq
import pandas as pd
import matplotlib.pyplot as plt

from module_package import load_data
from module_package import transformations

import logging
logging.basicConfig(filename="log.txt", format='%(asctime)s %(message)s', \
                    datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO)
logging.info("New execution started")

source_dir = 'dataset'
target_dir = 'output'

# Call json_parquet function to read JSON and to create parquet files
load_data.json_parquet(source_dir, target_dir)
logging.info("Data set loading is completed")

# Get absolute path of parquet directory
dirname = os.path.dirname(__file__)
parquet_directory = os.path.join(dirname, source_dir, target_dir)

# Read parquet dataset
try:
    dataset = pq.ParquetDataset(parquet_directory)
    table = dataset.read()
    source_df = table.to_pandas()
    source_df['ORDERDATE'] = pd.to_datetime(source_df['ORDERDATE'])
except:
    logging.info('Unable to load parquet data')

logging.info('Parquet data is loaded')
# Set status values and calls function to get total value data
values = ['Cancelled','On Hold']
data_df = transformations.total_value(source_df, values)
sales_loss = data_df.plot(kind = 'bar', title = 'Total (Cancelled, On Hold) Sales Per Year',
                          xlabel = 'Year', ylabel = 'Sales', rot = 0, subplots = True)
load_data.plot_diagram(plt)
logging.info("Sales data plot is completed")

# Call function to get products per line data
product_df = transformations.product_per_line(source_df)
product_count = product_df.plot(kind = 'bar', title = 'Products per line',
                                xlabel = 'Product Line', ylabel = 'Number of Products', rot = 0)
load_data.plot_diagram(plt)
logging.info("Product line plot is completed")

# Get the classic car trend data
product_lines = ['Classic Cars']
trend_df = transformations.classic_cars_trend(source_df, product_lines)
trend = trend_df.plot(kind = 'line', title = 'Classic Car Trend', x = 'YEAR_MONTH',
                      xlabel = 'Year_Month', ylabel = 'Quantity Ordered')
load_data.plot_diagram(plt)
logging.info("Classic car plot is completed")

# Call discount function to get the related trend for given product lines
values = ['Classic Cars', 'Vintage Cars', 'Motorcycles', 'Trucks and Buses']
discount_df = transformations.discount(source_df, values)
discount = discount_df.plot(kind = 'bar', title = 'Discount on Products', x = 'PRODUCTLINE',
                            y = 'DISCOUNT', xlabel = 'Product Line', ylabel = 'Discount', rot = 0)
load_data.plot_diagram(plt)
logging.info("Discount plot is completed")

logging.info("Execution completed")


#
# @lc app=leetcode id=347 lang=python
#
# [347] Top K Frequent Elements
#
# https://leetcode.com/problems/top-k-frequent-elements/description/
#
# algorithms
# Medium (56.06%)
# Total Accepted:    230.4K
# Total Submissions: 411K
# Testcase Example:  '[1,1,1,2,2,3]\n2'
#
# Given a non-empty array of integers, return the k most frequent elements.
#
# Example 1:
#
#
# Input: nums = [1,1,1,2,2,3], k = 2
# Output: [1,2]
#
#
#
# Example 2:
#
#
# Input: nums = [1], k = 1
# Output: [1]
#
#
# Note:
#
#
# You may assume k is always valid, 1 ≤ k ≤ number of unique elements.
# Your algorithm's time complexity must be better than O(n log n), where n is
# the array's size.
#
#
#
from collections import OrderedDict,Counter
class Solution(object):
    def topKFrequent(self, nums, k):
        """
        :type nums: List[int]
        :type k: int
        :rtype: List[int]
        """

        count = Counter(nums)
        dd = OrderedDict(sorted(count.items(), key=lambda x: -x[1]))
        ans = []
        for item in dd.items():
            ans.append(item[0])
            k -= 1
            if k == 0:
                break
        return ans


'''
Ficheros en Python
'''
from Compras import mostrarMenuCompra
from RRHH import *
from Ventas import *
from Compras import *
from Pagos import *
from Gastos import *
from colorama import init, Fore
init(autoreset=True)

from Color import *

def leer_todas_las_lineas2_ComprobarExisteUsuario(archivo, usuario):
    leerArchivo = open(archivo, "r")

    for linea in leerArchivo.readlines():
        comprobarUsuario = linea.split(" , ")
        if comprobarUsuario[0].lower() == usuario.lower():
            print(color.RED+"Error !!... el usuario "+ usuario.upper()+ " ya existe !!"+color.END)
            return True

    leerArchivo.close()

def leer_todas_las_lineas2_IniciarSesion(archivo, usuario, contrasenya):
    leerArchivo = open(archivo, "r")

    for linea in leerArchivo.readlines():
        comprobarUsuario = linea.split(" , ")
        if comprobarUsuario[0].lower() == usuario.lower() and comprobarUsuario[1] == contrasenya:
            return True
    leerArchivo.close()

def elegirOpcionDepartamento(archivo, usuario, contrasenya):
    leerArchivo = open(archivo, "r")
    contador =0

    for linea in leerArchivo.readlines():
        comprobarUsuario = linea.split(" , ")

        if comprobarUsuario[0].lower() == usuario.lower() and comprobarUsuario[1] == contrasenya:

            print()

            print(color.BLUE+"Buenos días "+ usuario.upper()+ ", elige un departamento: \n"+color.END)
            for i in range (2, len(comprobarUsuario)):
                contador +=1
                print("\t"+color.BLUE+ str(contador)+color.END+" "+str(comprobarUsuario[i]))

            respuesta = input(color.BLUE+"Escribe el nombre del Departamento: "+color.END)

            print()

            while respuesta.lower() != "ventas" and  respuesta != "compras" and  respuesta != "pagos" and  respuesta != "gastos" and  respuesta != "rrhh":
                print(color.RED+"Error !!... introduce el nombre del departamento !!"+ color.END)
                respuesta = input(color.BLUE+"Escribe el nombre del Departamento: "+color.END)

            if respuesta.lower() == "ventas":
                print(color.BLUE+"Bienvenido al departamento de Ventas"+color.END)
                mostrarMenuVenta()

            if respuesta.lower() == "compras":
                print(color.BLUE+"Bienvenido al departamento de Compras"+color.END)
                mostrarMenuCompra()

            if respuesta.lower()== "pagos":
                print(color.BLUE+"Bienvenido al departamento de Pagos: "+color.END)
                pagarFacturas(usuario)

            if respuesta.lower()== "gastos":
                print(color.BLUE+"Bienvenido al departamento de Gastos: "+color.END)
                eligeGastos(usuario)

            if respuesta.lower()== "rrhh":
                print(color.BLUE+"Bienvenido al departamento de RRHH: "+color.END)
                MenuRRHH()

    leerArchivo.close()

def escribir_varias_lineas(archivo,nombre,contrasenya, departamento):

    archivo = open(archivo, "a+")

    lista = [nombre," , " ,contrasenya," , " ,departamento,"\n"]

    archivo.writelines(lista)

def tipoActividad():
    resultado = ""

    print("Elige Departamento:\n"
          "\t1 Compras\n"
          "\t2 Ventas\n"
          "\t3 Salir"
          )

    opcion = int(input("Opcion: "))
    while opcion != 3:
        if (opcion == 1):
            resultado = "Compras"
            print("Has añadido el departamento COMPRAS")
        if (opcion == 2):
            resultado = "Ventas"
            print("Has añadido el departamento VENTAS")

        opcion = int(input("Quieres añadir mas departamentos? Opcion: "))

    return resultado

# Estas funciones al final no las he utilizado, para crear un usuario le mando directamente a la funcion de 'crear un trabajador' en el departamento de RRHH
def crearUsuarioConDepartamentos(archivo,nombre,apellido,contrasenya):

    archivo = open(archivo, "a+")
    lista = [nombre," , ",contrasenya,]

    print()

    print("Estos son los departamentos que existen:\n\n"+
          color.BLUE+"\tCompras\n"
          "\tVentas\n"+color.END)

    opcion1 = False
    opcion2 = False

    while not opcion1 and not opcion2:

        compras = input(color.BLUE + "Quieres añadir al departamento COMPRAS ? (S/N): " + color.END)
        while (compras!= "s" and compras != "n"):
            print(color.RED + "Error !!, introduce 's' o 'n'" + color.END)
            compras = input(color.BLUE + "Quieres añadir al departamento COMPRAS ? (S/N): " + color.END)

        ventas = input(color.BLUE + "Quieres añadir al departamento VENTAS ? (S/N): " + color.END)
        while (ventas!= "s" and ventas != "n"):
            print(color.RED + "Error !!, introduce 's' o 'n'" + color.END)
            compras = input(color.BLUE + "Quieres añadir al departamento VENTAS ? (S/N): " + color.END)

        print()

        if (compras.lower() =="s" and ventas.lower() == "s"):
            resultado = " , Compras , Ventas\n"
            lista.append(resultado)
            print(Fore.GREEN+"Se ha creado correctamente el comercial: "+nombre.upper()+ " "+apellido.upper()+ " con acceso a los departamentos: COMPRAS y VENTAS ")
            opcion1 = True
            opcion2 = True

        if (compras.lower() =="s" and ventas.lower() == "n"):
            resultado = " , Compras\n"
            lista.append(resultado)
            print(Fore.GREEN+"Se ha creado correctamente el comercial: "+nombre.upper()+ " "+apellido.upper()+ " con acceso al departamento: COMPRAS")
            opcion1 = True
            opcion2 = False

        if (compras.lower() =="n" and ventas.lower() == "s"):
            resultado = " , Ventas\n"
            lista.append(resultado)
            print(Fore.GREEN+"Se ha creado correctamente el comercial: "+nombre.upper()+ " "+apellido.upper()+ " con acceso al departamento: VENTAS")
            opcion1 = False
            opcion2 = True

        if not opcion1 and not opcion2:
            print(color.RED + "Error !... debes elegir como mínimo una opción !!" + color.END)
            print()

    archivo.writelines(lista)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Sep 22 17:27:21 2019

@author: ubuntu
"""

import torch
from torch.autograd import Function

"""如何在pytorch中手写一个Function类，来利用pytorch的自动求导功能。
从而可以用来实现前向计算和反向传播，也就可以用来自定义层。
参考自：叠加态的猫 https://www.cnblogs.com/hellcat/p/8453615.html
"""
class Sigmoid(Function):

    @staticmethod
    def forward(ctx, x):  # ctx类似于Function类的self, 用来在forward函数好backward函数之间共享参数，只需要把要共享的参数通过命令ctx.save_for_backward保存，就可以在backward中通过ctx.saved_tensors()获得。
        output = 1 / (1 + torch.exp(-x))  #
        ctx.save_for_backward(output)
        return output

    @staticmethod
    def backward(ctx, grad_output): # 这里grad_output就相当于accum_grad，是前一级的输出梯度
        output,  = ctx.saved_tensors  # 新版pytorch用saved_tensors替代了saved_variables
        grad_x = output * (1 - output) * grad_output
        return grad_x

# 采用数值逼近方式检验计算梯度的公式对不对
#test_input = V(torch.randn(3,4), requires_grad=True)  # 早期版本需要用Varaiable
test_input = torch.randn(3, 4, requires_grad=True)
torch.autograd.gradcheck(Sigmoid.apply, (test_input,), eps=1e-3)

def f_sigmoid(x):
    y = Sigmoid.apply(x)
    y.backward(torch.ones(x.size()))

def f_naive(x):
    y =  1/(1 + torch.exp(-x))
    y.backward(torch.ones(x.size()))

def f_th(x):
    y = torch.sigmoid(x)
    y.backward(torch.ones(x.size()))

x=torch.randn(100, 100, requires_grad=True)
#%timeit -n 100 f_sigmoid(x)
#%timeit -n 100 f_naive(x)
#%timeit -n 100 f_th(x)


from roct.upper_bound import maximum_adversarial_accuracy

import groot.datasets as datasets_module

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

# Avoid type 3 fonts
import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42

import seaborn as sns
sns.set_theme(style="whitegrid", context="paper", palette="colorblind", font_scale=1.2)

datasets = ["banknote-authentication", "blood-transfusion", "breast-cancer", "cylinder-bands", "diabetes", "haberman", "ionosphere", "wine"]
data_dir = "data/"

percentages = [0.75, 0.5, 0.25]

results = []
for dataset in datasets:
    # From the groot.datasets module, find the function with name load_<selected_dataset> then execute it
    X, y = getattr(datasets_module, f"load_{dataset.replace('-', '_')}")()[1:3]
    X = MinMaxScaler().fit_transform(X)

    for epsilon in np.linspace(0, 0.5, 51):
        Delta_l = Delta_r = np.full(X.shape[1], epsilon)
        maximum_score = maximum_adversarial_accuracy(X, y, Delta_l, Delta_r)
        results.append((dataset, epsilon, maximum_score))

results_df = pd.DataFrame(results, columns=["Dataset", "Epsilon", "Adversarial accuracy bound"])

_, ax = plt.subplots(figsize=(7.5, 5))
sns.lineplot(data=results_df, x="Epsilon", y="Adversarial accuracy bound", hue="Dataset", style="Dataset", markers=True, dashes=False, ax=ax)
plt.xlabel("$\epsilon$")
plt.legend(bbox_to_anchor=(0.5, 1.05), loc=8, borderaxespad=0., ncol=4)
plt.tight_layout()
plt.savefig("out/datasets_epsilon.png")
plt.savefig("out/datasets_epsilon.pdf")
plt.close()

epsilon_table = []
for dataset in datasets:
    dataset_df = results_df[results_df["Dataset"] == dataset]

    min_robustness = dataset_df["Adversarial accuracy bound"].min()
    max_robustness = dataset_df["Adversarial accuracy bound"].max()

    epsilons = []
    for percentage in percentages:
        find_bound = (max_robustness - min_robustness) * percentage + min_robustness
        index = (dataset_df["Adversarial accuracy bound"] - find_bound).abs().idxmin()
        epsilon = dataset_df.loc[index]["Epsilon"]
        epsilons.append(epsilon)
    epsilon_table.append((dataset, *epsilons))

epsilon_df = pd.DataFrame(epsilon_table, columns=["Dataset", "75%", "50%", "25%"])
epsilon_df.to_csv("out/epsilon_choices.csv")
epsilon_df.to_latex("out/epsilon_choices.tex")


import torch
from torch.nn.utils import parameters_to_vector
from utils.model_utils import LeNet
from scripts.classification_datasets.mnist.mnist_torch_dataset import get_dataloaders
from laplace_package_extensions.laplace_extension import Laplace

'''
This script tests the outputs of the Laplace approximation for a very high value of \lambda
("frozen" essentially a deterministic network) and the actual deterministic network on which the Laplace approximation is fitted.
We see that there are some differences in the outputs between the deterministic network and the "frozen" Laplace
approximation model. We located the source of the difference in the laplace_package_extensions.baselapalce_extension file
in the function _nn_posterior_predictive_samples line 691. Whenever new samples are loaded in the model for evaluation
there are some small deviations from the actual deterministic values. These deviations, we believe, accumulate and make
the predictions between the deterministic and LA "frozen" model also different.
'''

#Load data and model
dir_MNIST= '/Users/Kostas/PycharmProjects/cold-warm-posteriors/scripts/classification_datasets/mnist/dataset'

test_dataloader,train_dataloader,train_suffix_dataloader,validation_dataloader,true_dataloader,true_suffix_dataloader = \
    get_dataloaders(dir=dir_MNIST,mode='classification',suffix_size = 5000,validation_size=5000,seed_split=42)

#Define model
model = LeNet()
model.load_state_dict(torch.load('MNIST/results/MNIST_lambda_0_to_1000/model_0/prior_mean.pt',map_location=torch.device('cpu')))

#Load datasamples
X,y = next(iter(test_dataloader))

#Make predictions (the LA baseclass applies a softmax internally, the deterministic model doesn't)
preds_model = model(X)
preds_model = torch.softmax(preds_model,dim=-1)

a  = parameters_to_vector(model.parameters()).detach()

#Fit Laplace approximation
la = Laplace(model, likelihood='classification', prior_precision=1,
                  subset_of_weights='all',
                  hessian_structure='isotropic', prior_mean=0)
la.fit(train_suffix_dataloader)

#Make \lambda very large
la.temperature = 10000000000000
preds_la = la(X, pred_type='nn', mode='posterior', model_averaging=True,
                     n_samples=100)

#Estimate difference between deterministic and LA "frozen" predictions
diff = preds_model.detach()-preds_la.detach()

end = 1


import os
import torch
import  numpy as np
import random
import matplotlib.pyplot as plt

# Setting seed
def set_seed(seed):
    # os.environ['PYTHONASHSEED'] = 0 무작위화 비활성화
    os.environ['PYTHONHASHSEED'] = str(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)

# Save plt as png
def save_figure(figure_name, figure_base_path = './assets/', figure_extension='.png', resolution=300):
    # make directory
    try:
        if not os.path.exists(figure_base_path):
            os.makedirs(figure_base_path)
    except:
      print('already exists')

    figure_path = figure_base_path + figure_name + figure_extension
    print('save figure: ', figure_name)

    plt.savefig(figure_path, bbox_inches='tight', format=figure_extension[1:], dpi=resolution)

# Get cpu or gpu device for training
def check_device():
    if torch.cuda.is_available():
        DEVICE = torch.device('cuda')
    else:
        DEVICE = torch.deivce('cpu')

    print('Using Pytorch version : ',  torch.__version__, 'DEVICE : ', DEVICE)

    return DEVICE

# show image & save image
def imshow(img, label):
  img = img / 2 + 0.5  # unnormalize
  npimg = img.numpy()
  plt.figure(figsize=(5, 5))
  plt.imshow(np.transpose(npimg, (1, 2, 0)))
  plt.axis('off')
  plt.show()


###############################################################################
# Tests for RxnData and XmlParser classes.

# Class methods are used on different XML files located in xml-files directory.
# Each version of the reactions XML file tests different features / cases:
#   rxns_ideal: Contains data in correct manner (the "ideal" example file).
#   rxns_neg_A: Same as rxns_ideal except for negative A rateCoeff in first
#       reaction.
###############################################################################

from pytest import approx

from chemkin import pckg_xml_path
from chemkin.chemkin_errors import ChemKinError
from chemkin.preprocessing.parse_xml import RxnType, XmlParser

def test_parse_basic_functionality ():
    """Ensures number of reactions returned is correct and that attributes
    of the <reaction> element in the XML file are parsed correctly.
    """
    xml = XmlParser(pckg_xml_path('rxns_ideal.xml'))
    species, rxns = xml.load()

    # Correct number of reactions returned.
    err_msg = 'Expected 2 reactions but received {}.'.format(len(rxns))
    assert len(rxns) == 2, err_msg

    # Attributes of <reaction> element parsed properly.
    err_msg = 'reversible attribute not parsed properly.'
    assert rxns[0].reversible == False, err_msg
    assert rxns[1].reversible == False, err_msg

    err_msg = 'rxn_id attribute not parsed properly.'
    assert rxns[0].rxn_id == 'reaction01', err_msg
    assert rxns[1].rxn_id == 'reaction02', err_msg

    err_msg = 'type attribute not parsed properly.'
    assert rxns[0].type == RxnType.Elementary, err_msg
    assert rxns[1].type == RxnType.Elementary, err_msg

def test_parse_reactants_products ():
    """Ensures reactants and products parsed correctly."""
    xml = XmlParser(pckg_xml_path('rxns_ideal.xml'))
    species, rxns = xml.load()

    err_msg = 'reactants not parsed correctly.'
    assert rxns[0].reactants == {'H':1, 'O2':1}, err_msg
    assert rxns[1].reactants == {'H2':1, 'O':1}, err_msg

    err_msg = 'products not parsed correctly.'
    assert rxns[0].products == {'OH':1, 'O':1}, err_msg
    assert rxns[1].products == {'OH':1, 'H':1}, err_msg

def test_parse_rxn_coeff ():
    """Ensures reaction coefficients are what we expect them to be."""
    xml = XmlParser(pckg_xml_path('rxns.xml'))
    species, rxns = xml.load()

    rxn1_coeff = rxns[0].rate_coeff
    rxn2_coeff = rxns[1].rate_coeff
    rxn3_coeff = rxns[2].rate_coeff

    # reaction01
    assert rxn1_coeff[0] == approx(3.52e+10)
    assert rxn1_coeff[1] == approx(7.14e+04)

    # reaction02
    assert rxn2_coeff[0] == approx(5.06e-2)
    assert rxn2_coeff[1] == approx(2.7)
    assert rxn2_coeff[2] == approx(2.63e+04)

    # reaction03
    assert rxn3_coeff == approx(1.0e+03)

def test_rxndata_equation ():
    """Ensures equation representation of RxnData is correct."""
    xml = XmlParser(pckg_xml_path('rxns_ideal.xml'))
    species, rxns = xml.load()

    err_msg = 'equation() method result different than expected.'
    expected_0 = 'H + O2 [=] O + OH'
    print(rxns[0].equation())
    assert rxns[0].equation() == expected_0, err_msg

    err_msg = 'equation() method result different than expected.'
    expected_1 = 'H2 + O [=] H + OH'
    assert rxns[1].equation() == expected_1, err_msg

def test_badparse_negative_A_arr ():
    """Ensures ChemKinError raised when A coefficient for one of the
    reactions is negative.
    """
    xml = XmlParser(pckg_xml_path('rxns_neg_A_2'))
    try:
        rxns = xml.load()
    except ChemKinError as err:
        assert type(err) == ChemKinError
        assert str(err).find(
              'A coeff < 0 in reaction with id = reaction02') != -1

def test_badparse_negative_A_modarr ():
    """Ensures ChemKinError raised when A coefficient for one of the
    reactions is negative.
    """
    xml = XmlParser(pckg_xml_path('rxns_neg_A'))
    try:
        rxns = xml.load()
    except ChemKinError as err:
        assert type(err) == ChemKinError
        assert str(err).find(
              'A coeff < 0 in reaction with id = reaction01') != -1


import unittest

import paddle.v2.fluid.core as core

from paddle.v2.fluid.framework import Program, default_startup_program

main_program = default_startup_program()

class TestOperator(unittest.TestCase):
    def test_error_type(self):
        block = main_program.create_block()
        try:
            block.append_op()
            self.assertFail()
        except ValueError as v_err:
            self.assertEqual(
                v_err.message,
                "`type` to initilized an Operator can not be None.")
        try:
            block.append_op(type="no_such_op")
            self.assertFail()
        except ValueError as a_err:
            self.assertEqual(a_err.message,
                             "Operator \"no_such_op\" has not been registered.")

    def test_op_desc_creation(self):
        program = Program()
        block = program.current_block()
        mul_x = block.create_var(
            dtype="float32", shape=[5, 10], lod_level=0, name="mul.x")
        mul_y = block.create_var(
            dtype="float32", shape=[10, 8], lod_level=0, name="mul.y")
        mul_out = block.create_var(
            dtype="float32", shape=[5, 8], lod_level=0, name="mul.out")
        mul_op = block.append_op(
            type="mul",
            inputs={"X": [mul_x],
                    "Y": mul_y},
            outputs={"Out": [mul_out]},
            attrs={"x_num_col_dims": 1})

        self.assertNotEqual(str(mul_op), "")
        self.assertEqual(mul_op.type, "mul")
        self.assertEqual(mul_op.input_names, ["X", "Y"])
        self.assertEqual(mul_op.input("X"), ["mul.x"])
        self.assertEqual(mul_op.input("Y"), ["mul.y"])
        self.assertEqual(mul_op.output_names, ["Out"])
        self.assertEqual(mul_op.output("Out"), ["mul.out"])
        self.assertEqual(
            set(mul_op.attr_names), set(["x_num_col_dims", "y_num_col_dims"]))
        self.assertEqual(mul_op.has_attr("x_num_col_dims"), True)
        self.assertEqual(mul_op.attr_type("x_num_col_dims"), core.AttrType.INT)
        self.assertEqual(mul_op.attr("x_num_col_dims"), 1)
        self.assertEqual(mul_op.has_attr("y_num_col_dims"), True)
        self.assertEqual(mul_op.attr_type("y_num_col_dims"), core.AttrType.INT)
        self.assertEqual(mul_op.attr("y_num_col_dims"), 1)
        self.assertEqual(mul_op.idx, 0)
        self.assertEqual(mul_out.op, mul_op)

    def test_mult_input(self):
        program = Program()
        block = program.current_block()
        sum_x1 = block.create_var(
            dtype="int", shape=[3, 4], lod_level=0, name="sum.x1")
        sum_x2 = block.create_var(
            dtype="int", shape=[3, 4], lod_level=0, name="sum.x2")
        sum_x3 = block.create_var(
            dtype="int", shape=[3, 4], lod_level=0, name="sum.x3")
        sum_out = block.create_var(
            dtype="int", shape=[3, 4], lod_level=0, name="sum.out")
        sum_op = block.append_op(
            type="sum",
            inputs={"X": [sum_x1, sum_x2, sum_x3]},
            outputs={"Out": sum_out})
        self.assertEqual(sum_op.type, "sum")
        self.assertEqual(sum_op.input_names, ["X"])
        self.assertEqual(sum_op.input("X"), ["sum.x1", "sum.x2", "sum.x3"])
        self.assertEqual(sum_op.output_names, ["Out"])
        self.assertEqual(sum_op.output("Out"), ["sum.out"])
        self.assertEqual(sum_op.idx, 0)
        self.assertEqual(sum_out.op, sum_op)

if __name__ == '__main__':
    unittest.main()


"""Solve various problems concerning the formula
"""
from __future__ import absolute_import
import copy

try:
    from networkx import DiGraph
except ImportError:
    from .minnx import DiGraph

from .tstruct import stategen

def forallexists_pre(tsys, A):
    tmp = set()
    for s in A:
        for pre_s in tsys.G.predecessors(s):
            for envpost in tsys.envtrans[pre_s]:
                canreach = False
                for post_pre_s in tsys.G.successors(pre_s):
                    if tuple([post_pre_s[i] for i in tsys.ind_uncontrolled]) != envpost:
                        continue
                    if post_pre_s in A:
                        canreach = True
                        break
                if not canreach:
                    break
            if canreach:
                tmp.add(pre_s)
    return tmp

def get_winning_set(tsys, return_intermediates=False):
    S = set(tsys.G.nodes())
    Z = [S.copy() for i in range(tsys.num_sgoals)]
    change_among_Z = True
    while change_among_Z:
        change_among_Z = False
        Z_prev = [this_Z.copy() for this_Z in Z]
        if return_intermediates:
            Y_list = [[] for i in range(len(Z))]
            X_list = [[] for i in range(len(Z))]
        for i in range(len(Z)):
            this_Z_prev = Z[i].copy()
            Y = set()
            if return_intermediates:
                num_sublevels = 0
            goal_progress = forallexists_pre(tsys, Z_prev[(i+1) % tsys.num_sgoals])
            goal_progress &= set([s for s in S if 'SYSGOAL'+str(i) in tsys.G.nodes[s]['sat']])
            while True:
                Y_prev = Y.copy()
                if return_intermediates:
                    Y_list[i].append(Y_prev)
                    num_sublevels += 1
                Y = set()
                Y_exmodal = forallexists_pre(tsys, Y_prev)
                reach_goal_progress = goal_progress.union(Y_exmodal)
                if return_intermediates:
                    X_list[i].append([])
                for j in range(tsys.num_egoals):
                    X = S.copy()
                    while True:
                        X_prev = X.copy()
                        X = forallexists_pre(tsys, X_prev)
                        X &= set([s for s in S if 'ENVGOAL'+str(j) not in tsys.G.nodes[s]['sat']])
                        X |= reach_goal_progress
                        if X == X_prev:
                            break
                        X &= X_prev
                    if return_intermediates:
                        X_list[i][num_sublevels-1].append(X)
                    Y |= X

                if Y == Y_prev:
                    if return_intermediates:
                        num_sublevels -= 1
                        X_list[i].pop()
                    break
                Y |= Y_prev
            Z[i] = Y
            if Z[i] != this_Z_prev:
                change_among_Z = True
                Z[i] &= this_Z_prev
    if return_intermediates:
        return Z[0], Y_list, X_list
    else:
        return Z[0]

def get_initial_states(W, tsys, exprtab, init_flags='ALL_ENV_EXIST_SYS_INIT'):
    """

    If initial conditions are not satisfied on the winning set, return None.
    """
    assert init_flags.upper() == 'ALL_ENV_EXIST_SYS_INIT', 'Only the initial condition interpretation ALL_ENV_EXIST_SYS_INIT is supported.'

    evalglobals = {'__builtins__': None, 'True': True, 'False': False}
    identifiers = [v['name'] for v in tsys.symtab]

    initial_states = list()

    for state in stategen([v for v in tsys.symtab if v['uncontrolled']]):
        stated = dict(zip(identifiers, state))
        if not eval(exprtab['ENVINIT'], evalglobals, stated):
            continue
        found_match = False
        for s in W:
            if (tuple([s[i] for i in tsys.ind_uncontrolled]) == state
                and 'SYSINIT' in tsys.G.nodes[s]['sat']):
                initial_states.append(copy.deepcopy(s))
                found_match = True
                break
        if not found_match:
            return None

    return initial_states

def check_realizable(tsys, exprtab, init_flags='ALL_ENV_EXIST_SYS_INIT'):
    W = get_winning_set(tsys)
    if get_initial_states(W, tsys, exprtab, init_flags) is None:
        return False
    else:
        return True

def synthesize(tsys, exprtab, init_flags='ALL_ENV_EXIST_SYS_INIT'):
    assert init_flags.upper() == 'ALL_ENV_EXIST_SYS_INIT', 'Only the initial condition interpretation ALL_ENV_EXIST_SYS_INIT is supported.'

    W, Y_list, X_list = get_winning_set(tsys, return_intermediates=True)
    initial_states = get_initial_states(W, tsys, exprtab, init_flags)
    if initial_states is None:
        return None

    goalnames = ['SYSGOAL'+str(i) for i in range(tsys.num_sgoals)]

    for goalmode in range(tsys.num_sgoals):
        Y_list[goalmode][0] = set([s for s in W if goalnames[goalmode] in tsys.G.nodes[s]['sat']])

    strategy = DiGraph()
    next_id = len(initial_states)
    workset = list(range(next_id))
    strategy.add_nodes_from([(i, {'state': s, 'mode': 0, 'initial': True})
                             for (i,s) in enumerate(initial_states)])
    while len(workset) > 0:
        nd = workset.pop()

        j = 0
        while j < len(Y_list[strategy.nodes[nd]['mode']]):
            if strategy.nodes[nd]['state'] in Y_list[strategy.nodes[nd]['mode']][j]:
                break
            j += 1
        if j == 0:
            assert goalnames[strategy.nodes[nd]['mode']] in tsys.G.nodes[strategy.nodes[nd]['state']]['sat']
            original_mode = strategy.nodes[nd]['mode']
            while goalnames[strategy.nodes[nd]['mode']] in tsys.G.nodes[strategy.nodes[nd]['state']]['sat']:
                strategy.nodes[nd]['mode'] = (strategy.nodes[nd]['mode'] + 1) % tsys.num_sgoals
                if strategy.nodes[nd]['mode'] == original_mode:
                    break
            if strategy.nodes[nd]['mode'] != original_mode:
                repeat_found = False
                for possible_repeat, attr in list(strategy.nodes(data=True)):
                    if (possible_repeat != nd
                        and attr['mode'] == strategy.nodes[nd]['mode']
                        and attr['state'] == strategy.nodes[nd]['state']):
                        repeat_found = True
                        for (u,v) in strategy.in_edges(nd):
                            strategy.add_edge(u, possible_repeat)
                        strategy.remove_edges_from(
                            list(strategy.in_edges(nd)))
                        strategy.remove_node(nd)
                        break
                if repeat_found:
                    continue

            j = 0
            while j < len(Y_list[strategy.nodes[nd]['mode']]):
                if strategy.nodes[nd]['state'] in Y_list[strategy.nodes[nd]['mode']][j]:
                    break
                j += 1
            if j == 0:
                assert goalnames[strategy.nodes[nd]['mode']] in tsys.G.nodes[strategy.nodes[nd]['state']]['sat']

        for envpost in tsys.envtrans[strategy.nodes[nd]['state']]:
            next_state = None
            for succ_nd in tsys.G.successors(strategy.nodes[nd]['state']):
                if (tuple([succ_nd[i] for i in tsys.ind_uncontrolled]) == envpost
                    and ((j > 0 and succ_nd in Y_list[strategy.nodes[nd]['mode']][j-1])
                         or (j == 0 and succ_nd in W))):
                    next_state = succ_nd
                    break

            if next_state is None:
                assert j > 0
                if j == 0:
                    import pdb; pdb.set_trace()
                blocking_index = None
                blocking_sets = X_list[strategy.nodes[nd]['mode']][j-1]
                for k in range(len(blocking_sets)):
                    if strategy.nodes[nd]['state'] in blocking_sets[k]:
                        blocking_index = k
                        break
                assert blocking_index is not None
                for succ_nd in tsys.G.successors(strategy.nodes[nd]['state']):
                    if (tuple([succ_nd[i] for i in tsys.ind_uncontrolled]) == envpost
                        and succ_nd in blocking_sets[blocking_index]):
                        next_state = succ_nd
                        break
                assert next_state is not None

            foundmatch = False
            for candidate, cattr in strategy.nodes(data=True):
                if cattr['state'] == next_state and cattr['mode'] == strategy.nodes[nd]['mode']:
                    strategy.add_edge(nd, candidate)
                    foundmatch = True
                    break
            if not foundmatch:
                if j == 0:
                    new_mode = (strategy.nodes[nd]['mode'] + 1) % tsys.num_sgoals
                else:
                    new_mode = strategy.nodes[nd]['mode']

                workset.append(next_id)
                strategy.add_node(
                    next_id,
                    state=next_state,
                    mode=new_mode,
                    initial=False)
                strategy.add_edge(nd, next_id)
                next_id += 1

    return strategy


import numpy as np
import torch

from metagrad.loss import BCELoss
from metagrad.tensor import Tensor

def test_bce():
    x = np.random.randn(3)
    t = np.random.random(3)
    mx = Tensor(x, requires_grad=True)
    mt = Tensor(t)

    tx = torch.tensor(x, dtype=torch.float32, requires_grad=True)
    tt = torch.tensor(t, dtype=torch.float32)

    my_loss = BCELoss()
    torch_loss = torch.nn.BCEWithLogitsLoss()

    ml = my_loss(mx, mt)
    tl = torch_loss(tx, tt)

    assert np.allclose(ml.item(), tl.item())

    ml.backward()
    tl.backward()

    assert np.allclose(mx.grad, tx.grad)


import numpy as np
import cv2
import math
import time
from scipy.interpolate import UnivariateSpline, interp1d
from scipy import interpolate
from scipy import optimize
from scipy import integrate
from scipy import signal
from scipy import stats as st

def integrateImage(image_list, spline_x, spline_y, t_vals, arc_points):
        """
        Compute numerical integral of pixel intensity in an image along the given spline curve.
        Input:
        image_list (list): python list containing np.arrays of each image channel to be integrated
        spline_x (scipy.interpolate.UnivariateSpline): spline curve of x pixel positions (parameterized by t)
        spline_y (scipy.interpolate.UnivariateSpline): spline curve of y pixel positions (parameterized by t)
        t_vals (np.array): values of t that define points to take samples from
        arc_points (np.array): arc-length values of curve at points defined from spline(t_vals)

        Output:
        [total_val, ROI_length, width]
        total_val (np.array): array representing integrated values corresponding to  all points in arc_points
        ROI_length (np.array): image representing the straightened region of interest that we integrate. Useful for alignment.
        width (np.array): width distribution of food (deprecated)
        """
        x_deriv = spline_x.derivative()
        y_deriv = spline_y.derivative()

        average_val = []
        t_max = max(t_vals)
        ##Create sub-pixel sample points along skeleton
        radii = np.linspace(-25,25, 300)

        total_val = np.zeros((len(arc_points),len(image_list)))
        total_green_val = np.zeros(len(arc_points))
        global ROI, z, w
        ROI = np.zeros((300,len(arc_points)))
        ROI_green = np.zeros((300,len(arc_points)))
        i = 0
        theta_prev = 0
        for point_num in range(len(arc_points)): #for every sample point along arc
            t_current = t_vals[point_num]
            x = spline_x(t_current)
            y = spline_y(t_current)
            dy_dx = y_deriv(t_current)/x_deriv(t_current) #xy derivative gained from paramaterization... could result in error if denom is zero.
            #test.append(dy_dx)

            theta = np.arctan(dy_dx)
            theta_normal = theta + np.pi/2

            if point_num > 0:
                if theta_normal - theta_prev >= np.pi/2:
                    theta_normal = theta_normal - np.pi
                elif theta_prev - theta_normal >= np.pi/2:
                    theta_normal = theta_normal + np.pi
            theta_prev = theta_normal
            #want 150 sample points along normal line
            #15 px in both directions
            xs = radii * np.cos(theta_normal) + x
            ys = radii * np.sin(theta_normal) + y
            #plt.plot(xs,ys,"blue")
            #return
            floor_x = np.floor(xs-0.5)
            ceil_x = np.ceil(xs-0.5)
            floor_y = np.floor(ys-0.5)
            ceil_y = np.ceil(ys-0.5)

            #create block diagonal
            length = len(xs) * 2
            diag_mat = np.zeros((length, length,len(image_list)))
            #diag_mat_green = np.zeros((length, length))

            #global evens, lin
            evens = np.linspace(0,length-2, int(length/2))
            odds = evens + 1

            evens = evens.astype(int)
            odds = odds.astype(int)

            floor_x = floor_x.astype(int)
            floor_y = floor_y.astype(int)
            ceil_x = ceil_x.astype(int)
            ceil_y = ceil_y.astype(int)
            for image_idx in range(len(image_list)):
                diag_mat[evens, evens,image_idx] = image_list[image_idx][floor_y,floor_x]
                diag_mat[evens, odds,image_idx] = image_list[image_idx][ceil_y,floor_x]
                diag_mat[odds, evens,image_idx] = image_list[image_idx][floor_y,ceil_x]
                diag_mat[odds, odds,image_idx] = image_list[image_idx][ceil_y,ceil_x]
            #create flattened array
            xs = xs-0.5-floor_x
            ys = ys -0.5-floor_y
            one_minus_xs = 1-xs
            one_minus_ys = 1-ys
            interp_xs = np.array((one_minus_xs, xs)).T.flatten()#is two stacked on top of eo
            interp_ys = np.array((one_minus_ys, ys)).T.flatten()
            for image_idx in range(len(image_list)):
                total_val[point_num,image_idx] = (np.dot(interp_xs @ diag_mat[:,:,image_idx], interp_ys))
            #total_green_val[point_num] = (np.dot(interp_xs @ diag_mat_green, interp_ys))

            ##For visualizing spread of intensity
            diag_y_mat = np.zeros((length, int(length/2)))
            lin = np.linspace(0,int(length/2)-1, int(length/2))
            lin = lin.astype(int)
            diag_y_mat[evens,lin] = interp_ys[evens]
            diag_y_mat[odds,lin] = interp_ys[odds]
            #smoothed_signal = signal.savgol_filter(interp_xs @ diag_mat[:,:,1] @ diag_y_mat, 50, 3)

            ROI[:,i] = interp_xs @ diag_mat[:,:,0] @ diag_y_mat
            #ROI_green[:,i] = interp_xs @ diag_mat_green @ diag_y_mat

            i+=1

        SROI = ROI.copy()
        SROIG = ROI_green.copy()
        flatten = ROI.flatten()
        mode = st.mode(np.rint(flatten[flatten > 20]))[0][0]
        w = np.count_nonzero(ROI, axis = 0)
        #ROI[ROI == 0] = -mode
        width = countWidth(ROI)
        ROI_length = np.zeros((SROI.shape[0], math.floor(arc_points.max())+1))
        #ROI_green_length = np.zeros((SROIG.shape[0], math.floor(arc_points.max())+1))
        #plt.imshow(SROI)
        #plt.show()
        for row in range(SROI.shape[0]):
            #plt.plot(arc_points, ROI[row,:])
            #plt.show()
            func = interp1d(arc_points, SROI[row,:])
            ROI_length[row,:] = func(np.linspace(0,math.floor(arc_points.max()), math.floor(arc_points.max())+1))

            func_green = interp1d(arc_points, SROIG[row,:])

        return [total_val, ROI_length, width]

def arcLengthSpline(spline_x, spline_y, x_vals, y_vals):
    """
    Given splines of x and y return np array containing (x,y) of equally spaced points along curve
    """
    arc_points = np.zeros(len(x_vals)) #arc_points is list of cumulative arc lengts
    radii = np.linspace(-25,25, 300)
    i = 0
    theta_prev = 0
    for point_num in range(1,len(x_vals)): #for every sample point along arc
        x = x_vals[point_num] # current x coordinate
        y = y_vals[point_num] # current y coordinate

        arc_points[point_num] = arc_points[point_num-1] + math.sqrt((x-x_vals[point_num-1])**2 + (y-y_vals[point_num-1])**2) # populate cumulative arc length for each point
    return arc_points

def pointsOnSpline(spline_x, spline_y, t_max, resolution):
    """
    Given scipy spline objects of the t-parameterized curve of both x and y, and query t values, return numpy array representing points along body
    Input:
        spline_x (scipy.interpolate.UnivariateSpline) spline fit of x
        spline_y (scipy.interpolate.UnivariateSpline) spline fit of y
        resolution (int) number of points between pixels
        t_max (int) t-value to interpolate to
    Output:
        pts_on_curve (np.array)
    """
    t_on_curve = np.linspace(0,int(t_max), int(t_max*resolution))
    pts_on_curve = np.array([spline_x(t_on_curve), spline_y(t_on_curve)]).T
    return t_on_curve, pts_on_curve

def fitSpline(skeleton_points):
    '''
    Given a list of points representing skeleton ordered from head to tail or tail to head, return scipy spline fit
    Input:
        skeleton_points (np.array): Ordered array containing (x,y) of points along skeleton.

    Output:
        list: [spline_x, spline_y, t_vals, x_vals, y_vals]
        spline_x (scipy.interpolate.UnivariateSpline) spline fit of x
        spline_y (scipy.interpolate.UnivariateSpline) spline fit of y
    '''
    x_vals = []
    y_vals = []
    t_vals = []

##Fit skeleton to spline curve
    for point_num in range(len(skeleton_points)-1): #Arrange x, y values in increasing order
        x_vals.append(skeleton_points[point_num][1]) #look here for flipped x,y
        y_vals.append(skeleton_points[point_num][0])
        t_vals.append(point_num)
    t_vals.append(len(skeleton_points)) # assign parameter t to each point
    x_vals.append(skeleton_points[-1][1])
    y_vals.append(skeleton_points[-1][0])
    spline_x = UnivariateSpline(t_vals, x_vals) # fit x values to spline
    #global x_deriv, y_deriv
    x_deriv = spline_x.derivative() # x derivative
    spline_y = UnivariateSpline(t_vals,y_vals) #fit y values to spline
    y_deriv = spline_y.derivative() #y derivative

    #ax2.plot(spline_x(t_vals), spline_y(t_vals))

    return [spline_x, spline_y, t_vals, x_vals, y_vals]

#intestine_start_pos = 470
#intestine_max_pos = 570

def correlation_lags(in1_len, in2_len, mode='full'):
    # calculate lag ranges in different modes of operation
    if mode == "full":
        # the output is the full discrete linear convolution
        # of the inputs. (Default)
        lags = np.arange(-in2_len + 1, in1_len)
    elif mode == "same":
        # the output is the same size as `in1`, centeresshs
        # with respect to the 'full' output.
        # calculate the full output
        lags = np.arange(-in2_len + 1, in1_len)
        # determine the midpoint in the full output
        mid = lags.size // 2
        # determine lag_bound to be used with respect
        # to the midpoint
        lag_bound = in1_len // 2
        # calculate lag ranges for even and odd scenarios
        if in1_len % 2 == 0:
            lags = lags[(mid-lag_bound):(mid+lag_bound)]
        else:
            lags = lags[(mid-lag_bound):(mid+lag_bound)+1]
    elif mode == "valid":
        # the output consists only of those elements that do not
        # rely on the zero-padding. In 'valid' mode, either `in1` or `in2`
        # must be at least as large as the other in every dimension.

        # the lag_bound will be either negative or positive
        # this let's us infer how to present the lag range
        lag_bound = in1_len - in2_len
        if lag_bound >= 0:
            lags = np.arange(lag_bound + 1)
        else:
            lags = np.arange(lag_bound, 1)
    return lags

def bilinear_interpolation(x, y, points):
    '''Interpolate (x,y) from values associated with four points.

    The four points are a list of four triplets:  (x, y, value).
    The four points can be in any order.  They should form a rectangle.

        >>> bilinear_interpolation(12, 5.5,
        ...                        [(10, 4, 100),
        ...                         (20, 4, 200),
        ...                         (10, 6, 150),
        ...                         (20, 6, 300)])
        165.0

    '''
    # See formula at:  http://en.wikipedia.org/wiki/Bilinear_interpolation

    points = sorted(points)               # order points by x, then by y
    (x1, y1, q11), (_x1, y2, q12), (x2, _y1, q21), (_x2, _y2, q22) = points

    if x1 != _x1 or x2 != _x2 or y1 != _y1 or y2 != _y2:
        raise ValueError('points do not form a rectangle')
    if not x1 <= x <= x2 or not y1 <= y <= y2:
        raise ValueError('(x, y) not within the rectangle')

    return (q11 * (x2 - x) * (y2 - y) +
            q21 * (x - x1) * (y2 - y) +
            q12 * (x2 - x) * (y - y1) +
            q22 * (x - x1) * (y - y1)
           ) / ((x2 - x1) * (y2 - y1) + 0.0)

def interpolateIntensity(values, xpoints, max_arc_length,spacing): #only for first frame now
    """
    Interpolate the array "values" at integer arc length inputs.
    Input:
    values (np.array): array representing the food distribution function evaluated at non equispaced intervals (non equispaced due to how we step along the spline curve of worm)
    xpoints (np.array): array representing the arc length values of each corresponding value
    max_arc_length (int): integer representing the rounded max domain of sequence. We create an array of length "max_arc_length" for the output interpolated sequence.
    spacing (int): integer representing how many inter-integer points we want to output.

    Output:
    interpolated_vals (np.array): array representing the food distribution at standardized domain values. Useful for kymograph representation.
    """

    row_vec = np.zeros((max_arc_length*spacing+400))
    if xpoints[0] > 0:
        initial_l = xpoints[0]
        xpoints = np.pad(xpoints, [(math.ceil(initial_l),0)],mode = 'linear_ramp')
        for val in range(len(values)):
            values[val] = np.pad(values[val], [(math.ceil(initial_l),0)],mode='constant')

    diff = math.ceil(max_arc_length - xpoints.max()) #max arc length for first frame is always gonna be lower by 10%. Diff is positive
    if diff > 0: #fill with zeros
        xpoints = np.pad(xpoints, [(0,diff)], mode='linear_ramp',end_values = ((0,xpoints.max() + diff)))
        for val in range(len(values)):
            values[val] = np.pad(values[val], [(0,int(diff))],mode = 'constant')

        #print("change?")
    #plt.plot(xpoints,total_val)
    #plt.show()
    dict_func = {}
    for val in range(len(values)):
        dict_func[val] = interp1d(xpoints, values[val])

    '''Change based on expected length'''
    xnew = np.linspace(1,max_arc_length,max_arc_length*spacing)
    interpolated_vals = []
    for val in range(len(values)):
        row_vec[200:max_arc_length*spacing+200] = dict_func[val](xnew)
        interpolated_vals.append(row_vec.copy())
    return interpolated_vals.copy()

def countWidth(FI):
    #split down middle,
    FI = FI.copy()
    width = np.zeros(FI.shape[1])
    half1 = FI.shape[0] // 2
    half2 = FI.shape[0] - half1
    for col_num in range(FI.shape[1]):
        col = FI[:,col_num]
        col[col < 0] = -1
        col[col > 0] = 0
        col_first_half = col[:half1]
        num_zeros_first_half = len(col_first_half) - len(np.trim_zeros(col_first_half,'b'))
        col_second_half = col[half2:]
        num_zeros_second_half = len(col_second_half) - len(np.trim_zeros(col_second_half,'f'))
        width[col_num] = num_zeros_first_half + num_zeros_second_half
    return width

def displayThreshold(image, spline_x, spline_y, t_vals, x_vals,y_vals):
        x_deriv = spline_x.derivative()
        y_deriv = spline_y.derivative()

        average_val = []
        t_max = max(t_vals)
        x1_vals = x_vals[:]
        y1_vals = y_vals[:]
        t1_vals = t_vals[:]
        x_vals = [] #x coordinate list of points to be sampled along arc
        y_vals = [] #y coordinate list of points to be sampled along arc
        t_vals = [] #t paramaterization
        ##Create sub-pixel sample points along skeleton
        for sample_pt in range(t_max*4): #currently we have one sample point per pixel gained from the skeleton, but since we have the spline fit we can sample subpixel points along arc.
            x_vals.append(float(spline_x(sample_pt*0.25)))
            y_vals.append(float(spline_y(sample_pt*0.25)))
            t_vals.append(sample_pt*0.25)
        #self.t_vals = t_vals
        #self.x_vals = x_vals
        #self.y_vals = y_vals
        #times = []
        #test = []
        arc_points = np.zeros(len(x_vals)) #arc_points is list of cumulative arc length
        arc_points[0] = 0
        #splx = UnivariateSpline(t_vals,x_vals)
        #sply = UnivariateSpline(t_vals,y_vals)
        #dx = splx.derivative()
        #dy = sply.derivative()
        radii = np.linspace(-25,25, 300)

        total_val = np.zeros(len(x_vals))
        global ROI, z, w
        ROI = np.zeros((300,len(x_vals)))
        i = 0
        switch = True
        for point_num in range(len(x_vals)): #for every sample point along arc
            #print(len(x_vals))
            x = x_vals[point_num] # current x coordinate
            y = y_vals[point_num] # current y coordinate
            #if point_num == 1720:
                #print(x,y)
            if point_num ==0:
                arc_points[point_num] = 0
            else:
                arc_points[point_num] = arc_points[point_num-1] + math.sqrt((x-x_vals[point_num-1])**2 + (y-y_vals[point_num-1])**2) # populate cumulative arc length for each point
            t_current = t_vals[point_num]
            dy_dx = y_deriv(t_current)/x_deriv(t_current) #xy derivative gained from paramaterization... could result in error if denom is zero.
            #test.append(dy_dx)

            theta = np.arctan(dy_dx)
            if switch == True:
                print(theta)
                switch = False
            theta_normal = theta + np.pi/2

            if point_num > 0:
                if theta_normal - theta_prev >= np.pi/2:
                    theta_normal = theta_normal - np.pi
                elif theta_prev - theta_normal >= np.pi/2:
                    theta_normal = theta_normal + np.pi
            theta_prev = theta_normal
            #want 150 sample points along normal line
            #15 px in both directions
            xs = radii * np.cos(theta_normal) + x
            ys = radii * np.sin(theta_normal) + y
            #plt.plot(xs,ys,"blue")
            #return
            floor_x = np.floor(xs-0.5)
            ceil_x = np.ceil(xs-0.5)
            floor_y = np.floor(ys-0.5)
            ceil_y = np.ceil(ys-0.5)

            #create block diagonal
            length = len(xs) * 2
            diag_mat = np.zeros((length, length))

            #global evens, lin
            evens = np.linspace(0,length-2, int(length/2))
            odds = evens + 1

            evens = evens.astype(int)
            odds = odds.astype(int)

            floor_x = floor_x.astype(int)
            floor_y = floor_y.astype(int)
            ceil_x = ceil_x.astype(int)
            ceil_y = ceil_y.astype(int)
            #for i in range(len(xs)):
                #print(i)
                #diag_mat[i*2, i*2] = img[int(floor_y[i]),int(floor_x[i])]
                #diag_mat[i*2, i*2+1] = img[int(ceil_y[i]),int(floor_x[i])]
                #diag_mat[i*2+1, i*2] = img[int(floor_y[i]),int(ceil_x[i])]
                #diag_mat[i*2+1, i*2+1] = img[int(ceil_y[i]),int(ceil_x[i])]
            diag_mat[evens, evens] = image[floor_y,floor_x]
            diag_mat[evens, odds] = image[ceil_y,floor_x]
            diag_mat[odds, evens] = image[floor_y,ceil_x]
            diag_mat[odds, odds] = image[ceil_y,ceil_x]

            #create flattened array
            xs = xs-0.5-floor_x
            ys = ys -0.5-floor_y
            one_minus_xs = 1-xs
            one_minus_ys = 1-ys
            interp_xs = np.array((one_minus_xs, xs)).T.flatten()#is two stacked on top of eo
            interp_ys = np.array((one_minus_ys, ys)).T.flatten()
            total_val[point_num] = (np.dot(interp_xs @ diag_mat, interp_ys))

            ##For visualizing spread of intensity
            diag_y_mat = np.zeros((length, int(length/2)))
            lin = np.linspace(0,int(length/2)-1, int(length/2))
            lin = lin.astype(int)
            diag_y_mat[evens,lin] = interp_ys[evens]
            diag_y_mat[odds,lin] = interp_ys[odds]
            smoothed_signal = signal.savgol_filter(interp_xs @ diag_mat @ diag_y_mat, 50, 3)

            ROI[:,i] = interp_xs @ diag_mat @ diag_y_mat

            i+=1

        flatten = ROI.flatten()
        #mode = st.mode(np.rint(flatten[flatten > 20]))[0][0] + threshold
        #ROI[ROI < mode] = 0 #try no threshold
        return ROI

def correlate2d(mat1,mat2):
    """
    Compute 2d cross correlation between 2 arrays, only along columns. mat2 is sliding on mat1

    """
    #region 1, mat1 is sliding onto mat2
    mat1_length = mat1.shape[1]
    mat2_length = mat2.shape[1]
    mat2 = mat2-mat2.mean()

    if mat1_length < mat2_length:
        z = np.zeros((mat1.shape[0],(mat2_length-mat1_length)*2))
        print(z.shape)
        mat1 = np.concatenate((z,mat1,z),axis = 1)
        #print(mat1.shape,mat2.shape)
        mat1_length = mat1.shape[1]
    #print(mat1_length, mat2_length)

    corr = np.zeros((mat1_length-mat2_length))
    #for i in range(1,mat2_length):
    #    res = np.sum(np.multiply(mat1[:,:i],mat2[:,-i:]))
    #    #print(res)
    #    corr[i-1] = res
    for i in range(mat1_length-mat2_length):
        res = np.sum(np.multiply(mat1[:,i:i+mat2_length]-mat1[:,i:i+mat2_length].mean(), mat2))
        res = res/math.sqrt(np.sum(np.multiply(np.power(mat1[:,i:i+mat2_length]-mat1[:,i:i+mat2_length].mean(),2), np.power(mat2,2))))

        corr[i] = res
    #for i in range(1,mat2_length):
    #    res = np.sum(np.multiply(mat1[:,-i:],mat2[:,:i]))
    #    corr[i+mat1_length-1] = res
    return corr


#!/usr/bin/env python
from __future__ import absolute_import, division, print_function

import imagehash
import six
from PIL import Image
import datetime
import configparser
from multiprocessing import Pool, Semaphore, cpu_count

from dbman import Db
from hashtools import runhash, is_image, cropres

# It takes an image, hashes it, and adds it to the database
class ImageData:
    def __init__(self, img):
        db_location, _ = fetch_config(version="Default")
        self.db = Db(db_location)
        self.mode = "init"
        self.exists = self.db.check_exist(img)
        self.img = img
        self.hashed = False

    def hash_img(self):
        """
        It takes an image, runs it through a series of image hashing functions, and returns the results
        """
        self.ahash = runhash(self.img, imagehash.average_hash)
        self.phash = runhash(self.img, imagehash.phash)
        self.dhash = runhash(self.img, imagehash.dhash)
        self.haar = runhash(self.img, imagehash.whash)
        self.db4 = runhash(self.img, lambda img: imagehash.whash(img, mode="db4"))
        self.color = runhash(self.img, imagehash.colorhash)
        self.crop = cropres(self.img)
        if str(self.ahash) != "":
            self.hashed = True

    def add_to_db(self):
        """
        It takes the image, and then it hashes it using the following methods: ahash, phash, dhash, haar,
        db4, color, and crop
        """

        self.db.add_hash(
            filename=self.img,
            ahash=self.ahash,
            phash=self.phash,
            dhash=self.dhash,
            haar=self.haar,
            db4=self.db4,
            color=self.color,
            crop=self.crop,
        )

def process_image(img):
    """
    It takes an image, checks if it exists in the database, hashes it if it doesn't, and adds it to the
    database if it's been hashed

    :param img: The image to be processed
    """
    image = ImageData(img=img)
    if image.exists == False:
        image.hash_img()
        if image.hashed:
            image.add_to_db()

def worker(img):
    """
    It takes an image, checks if it exists in the database, hashes it if it doesn't, and adds it to the
    database if it's been hashed

    :param img: The image to be processed
    """

    image = ImageData(img=img)
    if image.exists == False:
        image.hash_img()
        if image.hashed:
            image.add_to_db()

def fetch_config(version, file="config.ini"):
    """
    It reads the config.ini file and returns the values of the db and source keys in the section that
    matches the version parameter

    :param version: The version of the config file to use
    :param file: the name of the config file, defaults to config.ini (optional)
    :return: the values of the variables db_location and source.
    """
    config = configparser.ConfigParser()
    config.read(file)
    config_version = version

    db_location = config[config_version]["db"]
    source = config[config_version]["source"]
    return db_location, source

if __name__ == "__main__":
    import os
    import sys

    global db

    db_location, source = fetch_config(version="Default")

    # connect to database
    db = Db(db_location)

    paths = os.listdir(source)
    start = datetime.datetime.now()

    fullfiles = []

    for p in paths:
        image_name = os.path.join(source, p)
        if is_image(image_name):
            fullfiles.append(image_name)

    poolsize = Semaphore(cpu_count())
    print(poolsize)
    p = Pool(cpu_count() - 1)
    p.map(worker, fullfiles)

    db.writedupes()
    db.cropresist_csv()
    print(datetime.datetime.now() - start)