from __init__ import logging, os
from time import perf_counter
from collections import defaultdict

from __init__ import np

class Genetic:
    """ Class of Genetic Algorithm. Holds all the variables and functions
        needed for performing said algorithm.\n

        Attributes:\n
        P                       -- Size of initial population.\n
        n                       -- Multiplier that describes the children
        population, calculated as n*P.\n
        p                       -- Probability of mutation.\n
        T                       -- Number of generations.\n
        d                       -- Distance between nodes.\n
        size                    -- Number of nodes.\n
        best                    -- Best combination so far.\n
        parents                 -- List of current parents.\n
        mating_pool             -- Number of children, results of n*P with
        guard in case the result is odd.\n
        children                -- List of current children.\n

        Methods:\n
        run                     -- Runs the prepaired algorithm with the
        parameters set in __init__ and with the distance matrix and
        information about number of nodes passed.\n
        parents_and_best        -- Creates placeholders for parents and
        best solution.\n
        select                  -- Selects parents for mating.\n
        mate                    -- Fills children table with newly created
        ones.\n
        mutate_evaluate_cull    -- Mutates the children, then evaluates their
        fitness and lastly takes old parents and children together and chooses
        P best ones to go to next generation.\n
        finish                  -- Puts final information to the logfile.
    """

    def __init__(self, params: dict) -> None:
        """ Initialization of Genetic class, takes dict as parameter and
            assigns values from it into corresponding variables.
        """

        self.P, self.n, self.p, self.T = params.values()

    def run(self, d: np.ndarray, size: int) -> None:
        """ Runs the algorithm. Takes distance matrix and number of nodes as
            parameters and using the parameters set in __init__ performs a
            simulation of survival of the fittest.
        """

        start = perf_counter()
        logging.info(
            "Staring algorithm",
            extra={'runtime': perf_counter() - start}
        )
        self.d = d
        self.size = size
        self.parents_and_best()
        # Simulate all the generations
        self.mating_pool = int(self.n*self.P)
        if self.mating_pool & 0x1:
            self.mating_pool -= 1
        for gen in range(self.T):
            logging.info(
                'Generation %s',
                str(gen),
                extra={'runtime': perf_counter() - start}
            )
            selected = self.select()
            self.mate(selected)
            self.mutate_evaluate_cull()

        self.finish(start)

    def parents_and_best(self) -> None:
        """ Creates placeholders for the fittest genome and parents.
        """

        # Prepaire container for best and parents
        self.best = np.arange(self.size + 1, dtype='object')
        self.best[-1] = np.Inf
        self.parents = np.zeros((self.P, self.size + 1), dtype='object')
        for parent in self.parents:
            tmp = np.random.permutation(self.size)
            parent[:-1] = tmp.copy()
            parent[-1] = self.d[tmp, np.roll(tmp, -1)].sum()

    def select(self) -> np.ndarray:
        """ Based on the fitness of parents, they are chosen to mate, returns
            them in the form of np.ndarray.
        """

        # Selecting parents for mating and pair them together
        fitness = self.parents[:, -1].copy()
        probs = (np.max(fitness) + 1 - fitness).astype('float64')
        probs /= np.sum(probs)
        mating = np.random.choice(
            np.arange(self.P),
            self.mating_pool,
            False,
            probs.flatten()
        )
        return np.random.choice(mating, (int(self.mating_pool/2), 2), False)

    def mate(self, mating: np.ndarray) -> None:
        """ Performs mating using chosen parents received in form of
            np.ndarray as parameter.
        """

        self.children = np.zeros((self.mating_pool, self.size + 1))
        for pair, (p1, p2) in enumerate(mating):
            par1 = self.parents[p1, :-1]
            par2 = self.parents[p2, :-1]
            cut1 = np.random.randint(self.size)
            cut2 = np.random.randint(cut1, self.size + 1)
            child1 = np.full_like(par1, self.size + 1)
            child2 = np.full_like(par2, self.size + 1)

            child1[cut1:cut2] = par2[cut1:cut2].copy()
            child2[cut1:cut2] = par1[cut1:cut2].copy()

            for i in range(self.size):
                if i < cut1 or i >= cut2:
                    v1 = par1[i]
                    v2 = par2[i]
                    while (v1 in child1):
                        v1 = par1[np.where(child1 == v1)][0]
                    while (v2 in child2):
                        v2 = par2[np.where(child2 == v2)][0]
                    child1[i] = v1
                    child2[i] = v2

            self.children[2*pair, :-1] = child1.copy()
            self.children[2*pair + 1, :-1] = child2.copy()

    def mutate_evaluate_cull(self) -> None:
        """ Performs mutation on children, calculates their fitness and then
            assigns P of the fittest genomes into new parents.
        """

        # Mutation
        mutation = np.random.rand(self.mating_pool)
        mutation = np.where(mutation >= self.p)[0]
        genes = np.random.choice(np.arange(self.size), (mutation.shape[0], 2))
        for m, (g1, g2) in zip(mutation, genes):
            child = self.children[m, :]
            child[g1], child[g2] = child[g2], child[g1]

        # Evaluate children
        for child in self.children:
            tmp = child[:-1].astype('int32')
            child[-1] = self.d[tmp, np.roll(tmp, -1)].sum()

        # Create new parents
        cross_gen = np.concatenate((self.parents, self.children), axis=0)
        cross_gen = cross_gen[cross_gen[:, -1].argsort()]
        self.parents = cross_gen[:self.P, :].copy()
        self.best = self.parents[0, :].copy() if self.best[
            -1] > self.parents[0, -1] else self.best

    def finish(self, start: float) -> None:
        """ Finish up function logging information about path and it's lenght.
            Receives time when program was started as a float.
        """
        end = perf_counter() - start
        logging.info(
            'Finished.',
            extra={'runtime': end}
        )
        logging.info(
            'Best Distance: %s',
            str(self.best[-1]),
            extra={'runtime': 0}
        )
        logging.info(
            'Best Path:\n%s',
            '->'.join([str(int(v)) for v in self.best[:-1]]),
            extra={'runtime': 0}
        )
        print(end)
        print(str(self.best[-1]))
        print('->'.join([str(int(v)) for v in self.best[:-1]]))

class Ant:
    """ Class of Ant Colony Algorithm. Holds all the variables and functions
        needed for performing said algorithm.\n

        Attributes:\n
        tours           -- Number of tours simulated by the program.\n
        alpha           -- Impact of the pheromone on the ant decision making.
        \n
        beta            -- Impact of the distance between the nodes on the ant
        decision making.\n
        rho             -- Evaporation rate of the pheromone.\n
        d               -- Distance between nodes.\n
        size            -- Number of nodes.\n
        tau             -- Table of pheromone for each edge between nodes.\n
        n               -- Inverse of distance between nodes.\n
        ants            -- Table with currently simulated ants.\n
        a               -- Table with decision making variables for each edge
        between nodes.\n
        start           -- Starting cities of simulated ants.\n

        Methods:\n
        run             -- Runs the prepaired algorithm with the parameters
        parameters set in __init__ and with the distance matrix and
        information about number of nodes passed.\n
        ants_tables     -- Creates tau, n and best tables that are used in the
        algorithm.\n
        ants_and_a      -- Creates ants, a and start for each tour.\n
        ants_traveling  -- Simulates the traveling process of ants along with
        decision making.\n
        pheromones      -- Simulates the evaporation of pheromone in tau.\n
        finish          -- Puts final information to the logfile.
    """

    def __init__(self, params: dict) -> None:
        """ Initialization of Ant class, takes dictionary as parameter and
            assigns values from it into corresponding variables.
        """

        self.tours, self.alpha, self.beta, self.rho = params.values()

    def run(self, d: np.ndarray, size: int) -> None:
        """ Runs the algorithm. Takes distance matrix and number of nodes as
            parameters and using the parameters set in __init__ performs a
            simulation of Ant Colony finding shortest path.
        """

        start = perf_counter()
        logging.info(
            "Staring algorithm",
            extra={'runtime': perf_counter() - start}
        )
        self.d = d
        self.size = size
        best = self.ants_tables()
        for trip in range(self.tours):
            logging.info(
                'Trip %s',
                str(trip),
                extra={'runtime': perf_counter() - start}
            )
            self.ants_and_a()
            self.ants_traveling()
            index = np.argmin(self.ants[:, -1])
            best = self.ants[index] if best[-1] > self.ants[index][
                -1] else best
            self.pheromones()
        self.finish(start, best)

    def ants_tables(self) -> np.ndarray:
        """ Creates necessary tables for the algorithm to run. These tables
            need to be created just once. Returnes np.ndarray
        """

        self.tau = np.full((self.size, self.size), 1/self.d.max())
        self.n = 1/self.d
        np.fill_diagonal(self.n, 0)
        best = np.arange(self.size + 1, dtype='object')
        best[-1] = np.Inf
        return best

    def ants_and_a(self) -> None:
        """ Creates necessary tables for the algorithm to run. These tables
            are created anew for each tour.
        """

        self.ants = np.full(
            (self.size, self.size + 1),
            self.size + 1,
            dtype="object"
        )
        self.ants[:, -1] = float(0)
        self.a = self.tau ** self.alpha * self.n ** self.beta
        self.start = np.random.randint(0, self.size, self.size, dtype='int32')
        self.a /= self.a.sum(axis=1, keepdims=True)

    def ants_traveling(self) -> None:
        """ Simulates the behaviour and the traveling of ants in search of the
            optimal path.
        """

        self.ants[:, 0] = self.start
        for ant in self.ants:
            for j in range(1, self.size):
                p = [
                    self.a[ant[j-1], q] * (q not in ant[0:-1])
                    for q in range(self.size)
                ]
                p /= np.sum(p)
                q = np.argmax(np.random.multinomial(1, p, size=1))
                ant[j] = int(q)
            ant[-1] = self.d[
                ant[:-1].astype('int32'),
                np.roll(ant[:-1], -1).astype('int32')
            ].sum()

    def pheromones(self) -> None:
        """ Updates the ammount of pheromone on each edge represented by tau.
        """

        self.tau *= (1-self.rho)
        for ant in self.ants:
            a = ant[0: -1].astype('int32')
            self.tau[a, np.roll(a, -1)] += 1/ant[-1]
            self.tau[a, np.roll(a, 1)] += 1/ant[-1]

    def finish(self, start: float, best: np.ndarray) -> None:
        """ Finish up function logging information about path and it's lenght.
            Receives time when program was started as a float and a best
            solution as np.ndarray.
        """
        end = perf_counter() - start
        logging.info(
            'Finished.',
            extra={'runtime': end}
        )
        logging.info(
            'Best Distance: %s',
            str(best[-1]),
            extra={'runtime': 0}
        )
        logging.info(
            'Best path:\n%s',
            '->'.join([str(v) for v in best[:-1]]),
            extra={'runtime': 0}
        )
        print(end)
        print(str(best[-1]))
        print('->'.join([str(v) for v in best[:-1]]))

class Smallest_Edge_Algorithm:
    """ Class of Smallest Edge Algorithm. Because the method itself is so
        simple, it consists of single function that performs the algorithm.\n

        Attributes:\n
        init_d      -- Distance between nodes.\n
        size        -- Number of nodes.\n
        d           -- List of edges in descending order of length.\n
        added       -- Holds in edges in order in which they were added.\n
        count       -- Counts number of times each vertex is present in current
        solution.\n
        free        -- Ensures that vertex cannot be used twice on the same
        position in edges in current solution.\n
        edge        -- Number of edges in current solution.\n

        Methods:\n
        run         -- Runs the algorithm with the distance matrix and
        information about number of nodes passed.\n
        setup       -- Prepaires variables for the algorithm.\n
        add         -- Checks if edge is elligible to be added to the solution.
        \n
        check_cycle -- Checks if in the given path a cycle is present.\n
        finish      -- Puts final information to the logfile.
    """

    def __init__(self, params: dict) -> None:
        """ Dummy, made only to make sure there are no errors when empty
            params are passed.
        """

        pass

    def run(self, d: np.ndarray, size: int) -> None:
        """ Runs the algorithm. Takes distance matrix and number of nodes and
            performs the algorithm.
        """

        self.init_d = d
        self.size = size

        self.setup(d)

        start = perf_counter()
        logging.info(
            "Staring algorithm",
            extra={'runtime': perf_counter() - start}
        )
        self.d = self.d[self.d[:, -1].argsort(), :]
        for i, row in enumerate(self.d):
            logging.info(
                'Row %s / %s',
                str(i),
                str(self.d.shape[0]),
                extra={'runtime': perf_counter() - start}
            )
            self.add(row)

        self.finish(start)

    def setup(self, d: np.ndarray) -> None:
        """ Prepaires the variables that are needed for the program execution.
            Takes np.ndarray as parameter.
        """

        ind = np.triu_indices(self.size, 1)
        self.d = np.array([i for i in zip(ind[0], ind[1], d[ind])] +
                          [i for i in zip(ind[1], ind[0], d[ind])],
                          dtype='object')
        self.added = []
        self.count = defaultdict(lambda: 0)
        self.free = defaultdict(lambda: True)
        self.edge = 0

    def add(self, row: np.ndarray) -> None:
        """ Checks if the edge passed as np.ndarray is eligible to be added to
            the solution, and if yes adds it.
        """

        c1, c2, _ = row
        if (self.count[c1] < 2 and
                self.count[c2] < 2 and
                self.free[(1, c1)] and
                self.free[(2, c2)]):
            new_path = self.added.copy()
            new_path.append((c1, c2))
            if not self.check_cycle(new_path) or self.edge == (self.size - 1):
                self.added.append((c1, c2))
                self.edge += 1
                self.count[c1] += 1
                self.count[c2] += 1
                self.free[(1, c1)] = False
                self.free[(2, c2)] = False

    def check_cycle(self, path: list) -> bool:
        """ Checks if the list od edges passed as parameters contains any
            cycles and returns corresponding boolean value.
        """

        cycle = False
        while path:
            visited = []
            p0 = path[0]
            while p0 not in visited:
                if len(visited) == len(path):
                    break
                visited.append(p0)
                for p1 in path:
                    if p0[1] == p1[0]:
                        p0 = p1
                        break
            if visited[0][0] == visited[-1][1]:
                cycle = True
            for v in visited:
                path.remove(v)
        return cycle

    def finish(self, start: float) -> None:
        """ Obtains path from the list of edges that constitute the solution,
            calculates the lenghth of the path, and logs all the information.
            Takes time at which program started as parameter.
        """

        path = np.zeros((self.size, 2))
        path[0] = self.added[0]
        for i, p in enumerate(path[:-1]):
            for a in self.added:
                if p[1] == a[0]:
                    path[i + 1] = a
                    break
        path = np.array(path[:, 0]).astype('int32')
        sol = self.init_d[path, np.roll(path, 1)].sum()

        end = perf_counter() - start
        logging.info(
            'Finished.',
            extra={'runtime': end}
        )
        logging.info(
            'Best Distance: %s',
            str(sol),
            extra={'runtime': 0}
        )
        logging.info(
            'Best Path:\n%s',
            '->'.join([str(v) for v in path]),
            extra={'runtime': 0}
        )
        print(end)
        print(str(sol))
        print('->'.join([str(v) for v in path]))

class Particle_Swarm_Optimisation:
    """ Class of Particle Swarm Optimisation. Holds all the variables and
        functions needed for performing said algorithm.\n

        Attributes:\n
        i                           -- Number of iterations.\n
        n                           -- Number of particles.\n
        c1_init                     -- Initial weight of current solution.\n
        c2                          -- Weight of the particle's best solution.
        \n
        c3                          -- Weight of the globaly best solution.\n
        d                           -- Distance between nodes.\n
        size                        -- Number of nodes.\n
        particles                   -- List of current locations of particles.
        \n
        pbest                       -- Best path for each prticle.\n
        gbest                       -- Best path so far.\n
        velocities                  -- Velocieties for each particle.\n
        c1                          -- Weight of the current solution for given
        itteration.\n

        Methods:\n
        run                         -- Runs the algorithm with the distance
        matrix and information about number of nodes passed.\n
        setup                       -- Prepaires variables for the algorithm.\n
        calc_velocity_and_position  -- Calculates new voloticty and position
        for each particle.\n
        finish                      -- Puts final information to the logfile.
    """

    def __init__(self, params: dict) -> None:
        """ Initialization of PSO class, takes dict as parameter and assigns
            values from it into corresponding variables.
        """

        coefs, self.i, self.n = params.values()
        self.c1_init, self.c2, self.c3 = coefs

    def run(self, d: np.ndarray, size: int) -> None:
        """ Runs the algorithm. Takes distance matrix and number of nodes as
            parameters and using the parameters set in __init__ performs a
            simulation of Particle Swarm finding the shortest path.
        """

        self.d = d
        self.size = size
        start = perf_counter()
        logging.info(
            "Staring algorithm",
            extra={'runtime': perf_counter() - start}
        )
        self.setup()
        for i in range(self.i):
            logging.info(
                'Iteration %s',
                str(i),
                extra={'runtime': perf_counter() - start}
            )
            self.c1 = self.c1_init - i / (2 * self.i)
            for particle, pbest, V in zip(self.particles, self.pbest, self.velocities):
                self.calc_velocity_and_position(particle, pbest, V)
                if particle[-1] < pbest[-1]:
                    pbest = particle.copy()
            candidate = self.particles[np.argmin(self.particles[:, -1]), :]
            if candidate[-1] < self.gbest[-1]:
                self.gbest = candidate.copy()
        self.finish(start)

    def setup(self) -> None:
        """ Sets up all the necessary containers such as particles, pbest,
            gbest and velocities.
        """

        self.particles = np.zeros((self.n, self.size + 1), dtype='object')
        for particle in self.particles:
            tmp = np.random.permutation(self.size)
            particle[:-1] = tmp.copy()
            particle[-1] = self.d[tmp, np.roll(tmp, -1)].sum()
        self.pbest = self.particles.copy()
        self.gbest = self.particles[np.argmin(self.particles[:, -1]), :].copy()
        self.velocities = np.zeros((self.n, self.size))

    def calc_velocity_and_position(
            self,
            particle: np.ndarray,
            pbest: np.ndarray,
            V: np.ndarray
        ) -> None:

        """ Calculates the velocity of the particle, and using this velocity
            updates the positiion of the particle. Takes particle, it's best
            position so far and Velocity as parameters.
        """

        p = particle[:-1].astype('int32').copy()
        y = np.array([1 - 2 * (np.random.rand() < 0.5) if x == pb and x == gb
            else int(x == gb) - int(x == pb) for (x, pb, gb) in zip(p, pbest, self.gbest)])
        r1, r2 = np.random.rand(2)
        V1 = self.c1 * V + r1 * self.c2 * (- 1 - y) + r2 * self.c3 * (1 - y)
        lam = V1 + y
        alpha = 0.3
        y = [int(v > alpha) - int(v < -alpha) for v in lam]
        new_p = np.full_like(p, self.size + 1, dtype='int32')
        for j, v in enumerate(y):
            flag = True
            if v != 0:
                r = pbest[j] if v == -1 else self.gbest[j]
                if (r not in new_p):
                    new_p[j] = r
                    flag = False
            if flag:
                r = np.random.randint(self.size)
                while (r in new_p):
                    r = np.random.randint(self.size)
                new_p[j] = r
        particle[:-1] = new_p.copy()
        particle[-1] = self.d[new_p, np.roll(new_p, -1)].sum()

    def finish(self, start: float) -> None:
        """ Finish up function logging information about path and it's lenght.
            Takes time when p[rogram started in float format as an input.]
        """
        end = perf_counter() - start
        logging.info(
            'Finished.',
            extra={'runtime': end}
        )
        logging.info(
            'Best Distance: %s',
            str(self.gbest[-1]),
            extra={'runtime': 0}
        )
        logging.info(
            'Best Path:\n%s',
            '->'.join([str(v) for v in self.gbest[:-1]]),
            extra={'runtime': 0}
        )
        print(end)
        print(str(self.gbest[-1]))
        print('->'.join([str(v) for v in self.gbest[:-1]]))

class Opt2:
    """ Class of 2-Opt Algorithm. Holds all the variables and functions
        needed for performing said algorithm.\n

        Methods:\n
        run     -- Runs the algorithm with the distance matrix and information
        about number of nodes passed.\n
        finish  -- Puts final information to the logfile.
    """

    def __init__(self, params: dict) -> None:
        """ Dummy, made only to make sure there are no errors when empty
            params are passed.
        """

        pass

    def run(self, d: np.ndarray, size: int) -> None:
        """ Runs the algorithm. Takes distance matrix and number of nodes and
            performs the algorithm.
        """

        self.d = d
        self.size = size
        start = perf_counter()
        logging.info(
            "Staring algorithm",
            extra={'runtime': perf_counter() - start}
        )
        path = np.random.permutation(size)
        cur_len = d[path, np.roll(path, -1)].sum()
        improved = True
        count = 0
        while (improved):
            logging.info(
                'Improvement attempt: %s',
                str(count),
                extra={'runtime': perf_counter() - start}
            )
            improved = False
            for i in range(size - 2):
                for j in range(i + 1, size - 1):
                    i1 = i + 1 % size
                    j1 = j + 1 % size
                    diff = -d[path[[i, j]], path[[i1, j1]]].sum() +\
                        d[path[[i, i1]], path[[j, j1]]].sum()
                    if diff < 0:
                        path[i+1:j+1] = path[i+1:j+1][::-1]
                        cur_len += diff
                        improved = True
            count += 1
        self.finish(start, cur_len, path)

    def finish(self, start: float, cur_len: float, path: np.ndarray) -> None:
        """ Finish up function logging information about path and it's lenght.
            Receives time when program was started as a float, lenght of the
            shortest path as a float and the shortest path in the form of
            np.ndarray.
        """
        end = perf_counter() - start
        logging.info(
            'Finished.',
            extra={'runtime': end}
        )
        logging.info(
            'Best Distance: %s',
            str(cur_len),
            extra={'runtime': 0}
        )
        logging.info(
            'Best Path:\n%s',
            '->'.join([str(v) for v in path]),
            extra={'runtime': 0}
        )
        print(end)
        print(str(cur_len))
        print('->'.join([str(v) for v in path]))

class Concorde:
    """ Class of Concorde. Holds all the variables and functions needed for
        running the Concorde program.\n

        Methods:\n
        run         -- Runs the algorithm with the distance matrix and information
        about number of nodes passed.\n
        setup       -- Prepaires file that is used by concorde executable to
        caluclate shortest path.\n
        get_output  --
    """

    def __init__(self, params: dict) -> None:
        """ Dummy, made only to make sure there are no errors when empty
            params are passed.
        """

        pass

    def run(self, d: np.ndarray, size: int) -> None:
        """ Runs the Concorde program. Takes distance matrix and number of
            nodes.
        """
        start = perf_counter()
        logging.info(
            "Staring algorithm",
            extra={'runtime': perf_counter() - start}
        )
        self.setup(d, size)

        # call the concorde based on the OS
        if os.name == 'nt':
            os.system('cmd /c "concorde.exe tmp.tsp > out.tmp"')
        else:
            os.system('./concorde tmp.tsp > out.tmp')

        # get results out of the created files and log them
        self.get_output(size)

        self.cleanup()

        logging.info(
            'Finished.',
            extra={'runtime': perf_counter() - start}
        )

    def setup(self, d, size) -> None:
        """ Setups the .tsp file that will be used by Concorde to calculate
            shortest path.
        """
        # prepaire the file using d
        with open('tmp.tsp', 'w') as f:
            f.write('NAME : tmp\n')
            f.write('TYPE : TSP\n')
            f.write(f'DIMENSION : {size}\n' )
            f.write('EDGE_WEIGHT_TYPE : EXPLICIT\n')
            f.write('EDGE_WEIGHT_FORMAT : FULL_MATRIX\n')
            f.write('EDGE_WEIGHT_SECTION\n')
            for row in d:
                f.write(' ' + ' '.join([str(int(v)) for v in row]) + '\n')
            f.write('EOF')

    def get_output(self, size) -> None:
        """ Reads the desired output out of the file created by command line /
            terminal and logs it.
        """

        with open('out.tmp', 'r') as f:
            line = ''
            while 'Optimal Solution' not in line:
                line = f.readline()
            sol = line.split()[-1]
            while 'Total Running Time' not in line:
                line = f.readline()
            prog_time = line.split()[-2]

        with open('tmp.sol', 'r') as f:
            f.readline()
            order = f.readlines()
        path = np.zeros(size, dtype='int32')
        counter = 0
        for o in order:
            for el in o.strip().split():
                path[counter] = int(el)
                counter += 1

        logging.info(
            'Best Distance: %s',
            str(sol),
            extra={'runtime': 0}
        )
        logging.info(
            'Best Path:\n%s',
            '->'.join([str(v) for v in path]),
            extra={'runtime': 0}
        )
        logging.info(
            'Program Execution time: %s',
            str(prog_time),
            extra={'runtime': 0}
        )
        print(str(sol))
        print('->'.join([str(v) for v in path]))

    def cleanup(self) -> None:
        """ Remove unnecesary files created by Concorde and this program.
        """

        # clean up the mess with files.
        files = ['Otmp.mas', 'Otmp.pul', 'Otmp.sav', 'out.tmp', 'tmp.mas',
                 'tmp.pul', 'tmp.sav', 'tmp.sol', 'O.sav', 'O.pul', 'tmp.tsp',
                 '.mas', '.pul', '.sav', '.sol', 'O.mas', 'Otmp.res', 'tmp.res']
        for file in files:
            if os.path.isfile(file):
                os.remove(file)


"""
This script scrapes the City GIS Map Gallery page for URLs and URL status and produces a csv of the results.

The 3rd party module selenium is used to open URLs and scrape details using Google Chrome and the WebDriver for Chrome.
The WebDriver for Chrome executable that matches the installed Chrome version needs to be separately downloaded
from the Chromium.org website and pointed to by the webdriver module object, but does not require installation.

Python version: 3.7
Author: Gavin Leavitt

"""

# 3rd party imports
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException, InvalidArgumentException
# Native imports
from datetime import date
import time
import csv

# start time of script
start_time = time.time()

# Not sure why this is needed, think this needs to exist but can be empty
opts = Options()
# opens browser in headless mode, no UI running in background
opts.headless = True
# Driver path, make sure driver matches version of browser, this driver is for chrome 83
driver_path = r'\chrome83\chromedriver_win32\chromedriver.exe'
# driver_path = r'B:\Python\Web_scraping\Chrome_Driver\chrome83\chromedriver_win32\chromedriver.exe'
# how long to wait until page load times out
waittime = 5
# Build driver, any major browser can be used, Chrome version 83 is used here, driver version must match browser
# install version
driver = webdriver.Chrome(options=opts, executable_path=driver_path)
# fields for which links will be tested, these values are used to add to the result dictionary as links are checked
link_values = ['REST_Link', 'AGOL_Link', 'Metadata_Link', 'WFS_Link', 'WMS_Link', 'Iframe_Webmap_Link']
# Common error messages that occur when links are wrong/broken, selenium can't check for HTTP status responses,
# so the web page contents need to be checked against using this values
error_msg = ["Please contact your system administrator.", "Unauthorized", "Not Found",
             "Item does not exist or is inaccessible.", "Error occurred while processing request", "Error occurred",
             "Please contact your system administrator."]
# Fields to be generated and populated in the output csv
csv_fields = ["gallery_entry_name", 'Gallery_Link', "REST_Link", "AGOL_Link", "Metadata_Link", "WFS_Link", "WMS_Link",
              "Iframe_Webmap_Link", "Iframe_Webmap_ID", "PDF_Link_Text", "PDF_Download_LINK", "PDF_File_Size",
              "REST_Link_Status", "AGOL_Link_Status", "Metadata_Link_Status", "WFS_Link_Status", "WMS_Link_Status",
              "Iframe_Webmap_Link_Status", "Summary Status"]
# current day for naming output file
currentday = date.today().strftime("%Y%m%d")
# output csv location
# output_csv = r'B:\Python\Web_scraping\scraping_results' + f"_{currentday}.csv"
output_csv = r'\scraping_results' + f"_{currentday}.csv"

# functions
def load_map_page(URL, waittime=waittime):
    """
    Uses selenium to load the map gallery entry page. Selenium will wait up to waittime seconds for the page to load.
    After page is loaded then the driver object is returned, this method is used because some pages may require more
    time for AJAX data to load, after the main page is loaded.

    :param URL: HTTPS URL of the map gallery entry to be loaded
    :return: Chrome Driver Object.
    """
    # Get the website
    driver.get(URL)
    # Wait up to waittime seconds for page to load, page is considered loaded when the link to the services directory
    # exists. This was chosen because this is the primary link of interest on the page
    try:
        WebDriverWait(driver, waittime).until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, "Services Directory")))
        return driver
    except TimeoutException:
        # If element doesn't load by waittime, catch the TimeoutException and return the normal driver.get result,
        # even though the page and elements aren't properly loaded, try/except blocks further in the code will catch
        # the errors and handle them in the CSV output
        print("Timed out on page load")
        return driver

# Get total number of map gallery pages to review
def totalpages(URL):
    # get the web page from URL
    driver.get(URL)
    # Use the "last page" link at the bottom of the page to determine how many pages the map gallery contains
    # spaces in class name must be replaced by "." since this DOM element has two classes, "pager-last" and "last"
    last_page = str(
        driver.find_element_by_class_name('pager-last.last').find_element_by_tag_name("a").get_attribute('href'))
    # Grab the end of the link text that states the last page
    max_pg = last_page.split("=")[1]
    return max_pg

def gallery_links_visit(URL):
    """
    Builds a list of links to visit. Grabs all map gallery entries inside the input URL.
    :param URL: Map Gallery page to visit formatted as a HTTPS URL
    :return: List of formatted HTTPS URLs for each map gallery entry inside the provided URL
    """
    # get the web page from URL
    driver.get(URL)
    # All map gallery entries are contained within divs with the class name "brgt mcfw"
    # spaces in class name must be replaced by "." since this DOM element has two classes, "brgt" and "mcfw"
    linkcontainers = driver.find_elements_by_class_name('brgt.mcfw')
    links_visited = []
    # Loop over result object list(divs containing the map gallery entries)
    for item in linkcontainers:
        # Dig into the list objects to get the href link: get the link element with the link tag name
        # <a> then the href contents and extend the links visited list
        links_visited.extend([item.find_element_by_tag_name("a").get_attribute('href')])
    return links_visited

def galleryentries(links_to_visit):
    """
    Go through each map gallery entry URL gathering the GIS relevant URLs and add to dictionary. For each link selenium
    attempts to find the relevant link, if found it adds the link to the associated second level dictionary key, if
    a selenium exception is thrown because the element can't be found, then the key value is assigned "Error".

    This function doesn't check if the links are valid, only that they exist.

    :param links_to_visit: List of HTTPS URLs to visit
    :return: Dictionary: Two level dictionary with first level being service name and second level being URLs to GIS
        relevant locations
    """
    res = {}
    for webpage in links_to_visit:
        print(f"Working on webpage: {webpage}")
        # Split the service name from the URL
        shortname = webpage.split("/map/")[1]
        # use service name as key to first level dict
        res[shortname] = {}
        # get the web page from URL, using function to allow page load time
        driver = load_map_page(webpage)
        # Map Gallery entry URL
        res[shortname]['Gallery_Link'] = webpage
        # Populate dictionary with try/except blocks
        try:
            # rest = driver.find_element_by_partial_link_text("View ArcGIS REST Services Directory").get_attribute('href')
            # Allows for typos or dynamic link texts when searching for elements
            rest = driver.find_element_by_partial_link_text("Services Directory").get_attribute('href')
            res[shortname]['REST_Link'] = rest
        except NoSuchElementException:
            res[shortname]['REST_Link'] = "Error"
        try:
            agol = driver.find_element_by_partial_link_text("ArcGIS Online").get_attribute('href')
            res[shortname]['AGOL_Link'] = agol
        except NoSuchElementException:
            res[shortname]['AGOL_Link'] = "Error"
        try:
            metadata = driver.find_element_by_class_name("pane-node-field-map-meta-data").find_element_by_tag_name(
                "a").get_attribute('href')
            res[shortname]['Metadata_Link'] = metadata
        except NoSuchElementException:
            res[shortname]['Metadata_Link'] = "Error"
        try:
            WFS = driver.find_element_by_link_text("WFS").get_attribute('href')
            res[shortname]['WFS_Link'] = WFS
        except NoSuchElementException:
            res[shortname]['WFS_Link'] = "Error"
        try:
            WMS = driver.find_element_by_link_text("WMS").get_attribute('href')
            res[shortname]['WMS_Link'] = WMS
        except NoSuchElementException:
            res[shortname]['WMS_Link'] = "Error"
        # Get iframe details
        # Find esri map iframe, have to grab it separately from google iframe on the page. This can be done by getting a
        # div that contains the desired iframe, then selected from that element based on the iframe tag
        try:
            iframeElement = driver.find_element_by_class_name("field-name-field-map-iframe").find_element_by_tag_name(
                'iframe')
            # Pull the web map link the iframe is pointing to, split it at "&extent" to just get the link
            # without URL parameters
            webmaplink = iframeElement.get_attribute('src').split("&extent")[0]
            res[shortname]['Iframe_Webmap_Link'] = webmaplink
            # Pull web map ID
            res[shortname]['Iframe_Webmap_ID'] = webmaplink.split("webmap=")[1]
        except NoSuchElementException:
            res[shortname]['Iframe_Webmap_Link'] = "Error"
            res[shortname]['Iframe_Webmap_ID'] = "Error"
        # Get PDF details
        try:
            PDFDiv = driver.find_element_by_class_name("pane-node-field-map-pdf")
            res[shortname]['PDF_Link_Text'] = PDFDiv.find_element_by_tag_name("a").text
            res[shortname]['PDF_Download_Link'] = PDFDiv.find_element_by_tag_name("a").get_attribute('href')
            res[shortname]['PDF_File_Size'] = PDFDiv.find_element_by_class_name("file-size").text
        except NoSuchElementException:
            res[shortname]['PDF_Link_Text'] = "Error"
            res[shortname]['PDF_Download_Link'] = "Error"
            res[shortname]['PDF_File_Size'] = "Error"

        # This code lets you enter a iframe and grab its details then return to the normal page, don't need this for now
        # Enter iframe to scrape details of iframe DOM
        # driver.switch_to.frame(iframeElement)
        # Leave iframe
        # driver.switch_to.default_content()
    return res

def checklinks(gallerylinks):
    """

    :param gallerylinks: Existing dictionary populated with GIS relevant details, will be mutated
    :return: Nothing, mutates existing galleryLink dictionary with new second level keys and values
    """
    # Loop over all map keys, maps on the gallery
    for map_key in gallerylinks.keys():
        print(f"Checking links for {map_key}")
        # loop over all map gallery values in nested dictionary
        # for id_key in gallerylinks[map_key].keys():
        for id_key in link_values:
            # Check to see if id key value should contain a link to check
            # If previous attempt to find link returned an error, mark the status as the same
            if gallerylinks[map_key][id_key] == "Error":
                gallerylinks[map_key][id_key + "_Status"] = "Error"
            else:
                # Get URL to check from dictionary value
                URL = gallerylinks[map_key][id_key]
                # get web page
                driver.get(URL)
                # Get web page source text
                page_text = driver.page_source
                # Check to see if page source text contains any common error messages
                # Selenium can't directly test HTTP error messages(401,404,etc)
                valid_link = any(i in page_text for i in error_msg)
                if valid_link:
                    gallerylinks[map_key][id_key + "_Status"] = "Error"
                else:
                    gallerylinks[map_key][id_key + "_Status"] = "Active"
            # Check if any entries with WFS errors are Raster layers as they should only have WMS services
            # Only checks the first layer, at "/0" in the service, we don't currently have any raster services
            # with multiple layers
        if gallerylinks[map_key]['WFS_Link'] == "Error":
            try:
                url = str(gallerylinks[map_key]['REST_Link']) + "/0"
                driver.get(url)
                if "Raster Layer" in driver.page_source:
                    gallerylinks[map_key]['WFS_Link'] = "NA-Raster"
                    gallerylinks[map_key]['WFS_Link_Status'] = "NA-Raster"
            except InvalidArgumentException:
                print(f"Error checking if the following is a raster layer: {gallerylinks[map_key]}")
                gallerylinks[map_key]['WFS_Link_Status'] = "Error"
    return gallerylinks

def dictsumvalue(gallerylinks):
    """
    Checks all nested second level entries for error values and populates a new second level key with the results.
    This allows for a single column to be checked for an error statement in the output CSV.

    :param gallerylinks:
    :return: Nothing, mutates galleryLinks dictionary
    """
    for map_keys in gallerylinks.keys():
        if "Error" in gallerylinks[map_keys].values():
            gallerylinks[map_keys]["Summary Status"] = "Error"
        else:
            gallerylinks[map_keys]["Summary Status"] = "No problem"

# Kick off script:
# Starting URL, gallery page 1
URL = "https://www.cityofsalinas.org/our-government/information-center/map-gallery"
# Total map gallery pages
pages = int(totalpages(URL))
# Empty list of web gallery web pages that need to be visited
links_to_visit = []
# Visit page 1 and extend list with results
links_to_visit.extend(gallery_links_visit(URL))
# Loop through remaining map gallery pages extending the list of links of map gallery entries
# First page of gallery doesn't contain "page=", this starts on the second page with "page=1", loop accounts for this
for page in range(1, (pages + 1)):
    URL = (f"https://www.cityofsalinas.org/our-government/information-center/map-gallery?page={page}")
    print(f"visting URL {URL}")
    links_to_visit.extend(gallery_links_visit(URL))

# Build out the galleryLinks dictionary
gallerylinks = galleryentries(links_to_visit)
gallerylinks = checklinks(gallerylinks)
print(gallerylinks)
dictsumvalue(gallerylinks)

# Write dictionary to CSV
# This solution used as reference to write the dict to csv:
# https://stackoverflow.com/a/29400823
print("Writing csv")
with open(output_csv, "w", newline='') as csvfile:
    w = csv.DictWriter(csvfile, csv_fields)
    w.writeheader()
    for k in gallerylinks:
        w.writerow({field: gallerylinks[k].get(field) or k for field in csv_fields})
print(f"CSV written to: {output_csv}")
print(f"All done! This script took {round(time.time() - start_time, 2)} seconds to run!")


# -*- coding: utf-8 -*-
"""
Created on Thu Dec 17 08:27:07 2020

@author: Marysia
Puzzle 17 part 2 from https://adventofcode.com/2020/day/17

Starting with your given initial configuration, simulate six cycles in
a 4-dimensional space. How many cubes are left in the active state after the sixth cycle?
"""

import itertools
from copy import copy
import numpy as np

def split(word):
    '''
    Splits string into list of chars.
    '''
    return [char for char in word]

def check_neighbours(cube_in, _i, _j, _k, _l):
    '''
    Checks the sum of neighbour's values.
    '''
    suma = 0
    for case in POSITIONS_TO_CHECK:
        suma += cube_in[_i+case[0], _j+case[1], _k+case[2], _l +case[3]]
    return suma

with open('input.txt', 'r') as file:
    INPUT = file.readlines()

INPUT = np.array([split(x.strip()) for x in INPUT])
# convert '#' to 1 and '.' to 0
INPUT = np.where(INPUT == '#', 1, INPUT)
INPUT = np.where(INPUT == '.', 0, INPUT)
DIM_INPUT = len(INPUT[0, :])

MAX_CYCLE = 6

POSITIONS_TO_CHECK = [p for p in itertools.product([1, 0, -1], repeat=4)]
POSITIONS_TO_CHECK.remove(POSITIONS_TO_CHECK[POSITIONS_TO_CHECK.index((0, 0, 0, 0))])

N = DIM_INPUT+2+MAX_CYCLE*2 + DIM_INPUT % 2 + 1 # 27 minimum dimension of final cube
cube = [[[[0 for k in range(N)] for j in range(N)] for i in range(N)] for l in range(N)]
cube = np.array(cube)
# the middle of a grid has the coordinates [5][5][5]=[n_mid][n_mid][n_mid] n_mid=(n-1)/2
N_MID = int((N-1)/2)
DIM1 = N_MID - int(len(INPUT[0, :])/2)
DIM2 = N_MID + int(len(INPUT[0, :])/2) + DIM_INPUT % 2

cube[DIM1:DIM2, DIM1:DIM2, N_MID, N_MID] = INPUT
cube_tmp = copy(cube)

cycle = 0
while cycle < MAX_CYCLE:
    # screw the beauty
    for i in range(DIM1-1-cycle, DIM2+cycle+1):
        for j in range(DIM1-1-cycle, DIM2+cycle+1):
            for k in range(N_MID-1-cycle, N_MID+2+cycle):
                for l in range(N_MID-1-cycle, N_MID+2+cycle):
                    number_of_active = check_neighbours(cube, i, j, k, l)
                    if cube[i, j, k, l] == 1 and number_of_active not in (2, 3):
                        cube_tmp[i, j, k, l] = 0
                    elif cube[i, j, k, l] == 0 and number_of_active == 3:
                        cube_tmp[i, j, k, l] = 1
    cube = copy(cube_tmp)
    cycle += 1

SUM = 0
for i in range(0, N):
    for j in range(0, N):
        for k in range(0, N):
            for l in range(0, N):
                SUM += cube[i, j, k, l]

print(SUM)


from wtypes_h import (
    ENUM,
    UINT,
    POINTER,
    HRESULT,
    BOOL,
    DWORD_PTR,
    BSTR,
    DWORD,
    HANDLE,
    INT,
    HANDLE_PTR,
    VARIANT_BOOL,
    REFCLSID,
    REFIID,
)
from guiddef_h import (
    IID,
)
import ctypes
import ntdef_h
import comtypes
from activdbg_h import * # NOQA
from activscp_h import * # NOQA

from winapifamily_h import * # NOQA

__REQUIRED_RPCNDR_H_VERSION__ = 0x000001F4
__REQUIRED_RPCSAL_H_VERSION__ = 0x00000064
# from rpc_h import * # NOQA
# from rpcndr_h import * # NOQA
# from windows_h import * # NOQA
# from ole2_h import * # NOQA
IID_IDebugApplicationNode100 = IID(
    '{90a7734e-841b-4f77-9384-a2891e76e7e2}'
)

class IDebugApplicationNode100(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IDebugApplicationNode100
    _idlflags_ = []

IID_IWebAppDiagnosticsSetup = IID(
    '{379BFBE1-C6C9-432A-93E1-6D17656C538C}'
)

class IWebAppDiagnosticsSetup(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IWebAppDiagnosticsSetup
    _idlflags_ = []

IID_IRemoteDebugApplication110 = IID(
    '{D5FE005B-2836-485e-B1F9-89D91AA24FD4}'
)

class IRemoteDebugApplication110(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IRemoteDebugApplication110
    _idlflags_ = []

IID_IDebugApplication11032 = IID(
    '{BDB3B5DE-89F2-4E11-84A5-97445F941C7D}'
)

class IDebugApplication11032(IRemoteDebugApplication110):
    _case_insensitive_ = True
    _iid_ = IID_IDebugApplication11032
    _idlflags_ = []

IID_IDebugApplication11064 = IID(
    '{2039D958-4EEB-496A-87BB-2E5201EADEEF}'
)

class IDebugApplication11064(IRemoteDebugApplication110):
    _case_insensitive_ = True
    _iid_ = IID_IDebugApplication11064
    _idlflags_ = []

IID_IWebAppDiagnosticsObjectInitialization = IID(
    '{16FF3A42-A5F5-432B-B625-8E8E16F57E15}'
)

class IWebAppDiagnosticsObjectInitialization(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IWebAppDiagnosticsObjectInitialization
    _idlflags_ = []

IID_IActiveScriptWinRTErrorDebug = IID(
    '{73A3F82A-0FE9-4B33-BA3B-FE095F697E0A}'
)

class IActiveScriptWinRTErrorDebug(IActiveScriptError):
    _case_insensitive_ = True
    _iid_ = IID_IActiveScriptWinRTErrorDebug
    _idlflags_ = []

IID_IActiveScriptErrorDebug110 = IID(
    '{516E42B6-89A8-4530-937B-5F0708431442}'
)

class IActiveScriptErrorDebug110(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IActiveScriptErrorDebug110
    _idlflags_ = []

IID_IDebugApplicationThreadEvents110 = IID(
    '{84E5E468-D5DA-48A8-83F4-40366429007B}'
)

class IDebugApplicationThreadEvents110(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IDebugApplicationThreadEvents110
    _idlflags_ = []

IID_IDebugApplicationThread11032 = IID(
    '{2194AC5C-6561-404A-A2E9-F57D72DE3702}'
)

class IDebugApplicationThread11032(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IDebugApplicationThread11032
    _idlflags_ = []

IID_IDebugApplicationThread11064 = IID(
    '{420AA4CC-EFD8-4DAC-983B-47127826917D}'
)

class IDebugApplicationThread11064(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IDebugApplicationThread11064
    _idlflags_ = []

IID_IRemoteDebugCriticalErrorEvent110 = IID(
    '{2f69c611-6b14-47e8-9260-4bb7c52f504b}'
)

class IRemoteDebugCriticalErrorEvent110(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IRemoteDebugCriticalErrorEvent110
    _idlflags_ = []

IID_IScriptInvocationContext = IID(
    '{5D7741B7-AF7E-4A2A-85E5-C77F4D0659FB}'
)

class IScriptInvocationContext(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IScriptInvocationContext
    _idlflags_ = []

IID_IDebugStackFrame110 = IID(
    '{4B509611-B6EA-4B24-ADCB-D0CCFD1A7E33}'
)

class IDebugStackFrame110(IDebugStackFrame):
    _case_insensitive_ = True
    _iid_ = IID_IDebugStackFrame110
    _idlflags_ = []

IID_IRemoteDebugInfoEvent110 = IID(
    '{9FF56BB6-EB89-4C0F-8823-CC2A4C0B7F26}'
)

class IRemoteDebugInfoEvent110(comtypes.IUnknown):
    _case_insensitive_ = True
    _iid_ = IID_IRemoteDebugInfoEvent110
    _idlflags_ = []

class tagAPPLICATION_NODE_EVENT_FILTER(ENUM):
    FILTER_EXCLUDE_NOTHING = 0
    FILTER_EXCLUDE_ANONYMOUS_CODE = 0x1
    FILTER_EXCLUDE_EVAL_CODE = 0x2

APPLICATION_NODE_EVENT_FILTER = tagAPPLICATION_NODE_EVENT_FILTER

class tagTEXT_DOCUMENT_ARRAY(ctypes.Structure):
    _fields_ = [
        ('dwCount', DWORD),
        ('Members', POINTER(POINTER(IDebugDocumentText))),
    ]

TEXT_DOCUMENT_ARRAY = tagTEXT_DOCUMENT_ARRAY

COMMETHOD = comtypes.COMMETHOD

IDebugApplicationNode100._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'SetFilterForEventSink',
        ([], DWORD, 'dwCookie'),
        ([], APPLICATION_NODE_EVENT_FILTER, 'filter'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'GetExcludedDocuments',
        ([], APPLICATION_NODE_EVENT_FILTER, 'filter'),
        ([], POINTER(TEXT_DOCUMENT_ARRAY), 'pDocuments'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'QueryIsChildNode',
        ([], POINTER(IDebugDocument), 'pSearchKey'),
    ),
]

IWebAppDiagnosticsSetup._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'DiagnosticsSupported',
        ([], POINTER(VARIANT_BOOL), 'pRetVal'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'CreateObjectWithSiteAtWebApp',
        ([], REFCLSID, 'rclsid'),
        ([], DWORD, 'dwClsContext'),
        ([], REFIID, 'riid'),
        ([], DWORD_PTR, 'hPassToObject'),
    ),
]

class SCRIPT_DEBUGGER_OPTIONS(ENUM):
    SDO_NONE = 0
    SDO_ENABLE_FIRST_CHANCE_EXCEPTIONS = 0x1
    SDO_ENABLE_WEB_WORKER_SUPPORT = 0x2
    SDO_ENABLE_NONUSER_CODE_SUPPORT = 0x4
    SDO_ENABLE_LIBRARY_STACK_FRAME = 0x8

IRemoteDebugApplication110._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'SetDebuggerOptions',
        ([], SCRIPT_DEBUGGER_OPTIONS, 'mask'),
        ([], SCRIPT_DEBUGGER_OPTIONS, 'value'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'GetCurrentDebuggerOptions',
        ([], POINTER(SCRIPT_DEBUGGER_OPTIONS), 'pCurrentOptions'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'GetMainThread',
        ([], POINTER(POINTER(IRemoteDebugApplicationThread)), 'ppThread'),
    ),
]

IDebugApplication110 = IDebugApplication11064
IID_IDebugApplication110 = IID_IDebugApplication11064

IDebugApplication11032._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'SynchronousCallInMainThread',
        ([], POINTER(IDebugThreadCall32), 'pptc'),
        ([], DWORD_PTR, 'dwParam1'),
        ([], DWORD_PTR, 'dwParam2'),
        ([], DWORD_PTR, 'dwParam3'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'AsynchronousCallInMainThread',
        ([], POINTER(IDebugThreadCall32), 'pptc'),
        ([], DWORD_PTR, 'dwParam1'),
        ([], DWORD_PTR, 'dwParam2'),
        ([], DWORD_PTR, 'dwParam3'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'CallableWaitForHandles',
        ([], DWORD, 'handleCount'),
        ([], POINTER(HANDLE), 'pHandles'),
        ([], POINTER(DWORD), 'pIndex'),
    ),
]

IDebugApplication11064._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'SynchronousCallInMainThread',
        ([], POINTER(IDebugThreadCall64), 'pptc'),
        ([], DWORD_PTR, 'dwParam1'),
        ([], DWORD_PTR, 'dwParam2'),
        ([], DWORD_PTR, 'dwParam3'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'AsynchronousCallInMainThread',
        ([], POINTER(IDebugThreadCall64), 'pptc'),
        ([], DWORD_PTR, 'dwParam1'),
        ([], DWORD_PTR, 'dwParam2'),
        ([], DWORD_PTR, 'dwParam3'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'CallableWaitForHandles',
        ([], DWORD, 'handleCount'),
        ([], POINTER(HANDLE), 'pHandles'),
        ([], POINTER(DWORD), 'pIndex'),
    ),
]

IWebAppDiagnosticsObjectInitialization._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'Initialize',
        ([], HANDLE_PTR, 'hPassedHandle'),
        ([], POINTER(comtypes.IUnknown), 'pDebugApplication'),
    ),
]

IActiveScriptWinRTErrorDebug._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'GetRestrictedErrorString',
        ([], POINTER(BSTR), 'errorString'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'GetRestrictedErrorReference',
        ([], POINTER(BSTR), 'referenceString'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'GetCapabilitySid',
        ([], POINTER(BSTR), 'capabilitySid'),
    ),
]

class tagSCRIPT_ERROR_DEBUG_EXCEPTION_THROWN_KIND(ENUM):
    ETK_FIRST_CHANCE = 0
    ETK_USER_UNHANDLED = 0x1
    ETK_UNHANDLED = 0x2

SCRIPT_ERROR_DEBUG_EXCEPTION_THROWN_KIND = tagSCRIPT_ERROR_DEBUG_EXCEPTION_THROWN_KIND

IActiveScriptErrorDebug110._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'GetExceptionThrownKind',
        ([], POINTER(SCRIPT_ERROR_DEBUG_EXCEPTION_THROWN_KIND), 'pExceptionKind'),
    ),
]

IDebugApplicationThreadEvents110._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'OnBeginThreadRequest'
    ),
]

IDebugApplicationThread110 = IDebugApplicationThread11064
IID_IDebugApplicationThread110 = IID_IDebugApplicationThread11064

IDebugApplicationThread11032._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'GetActiveThreadRequestCount',
        ([], POINTER(UINT), 'puiThreadRequests'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'IsSuspendedForBreakPoINT',
        ([], POINTER(BOOL), 'pfIsSuspended'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'IsThreadCallable',
        ([], POINTER(BOOL), 'pfIsCallable'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'AsynchronousCallIntoThread',
        ([], POINTER(IDebugThreadCall32), 'pptc'),
        ([], DWORD_PTR, 'dwParam1'),
        ([], DWORD_PTR, 'dwParam2'),
        ([], DWORD_PTR, 'dwParam3'),
    ),
]

IDebugApplicationThread11064._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'GetActiveThreadRequestCount',
        ([], POINTER(UINT), 'puiThreadRequests'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'IsSuspendedForBreakPoINT',
        ([], POINTER(BOOL), 'pfIsSuspended'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'IsThreadCallable',
        ([], POINTER(BOOL), 'pfIsCallable'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'AsynchronousCallIntoThread',
        ([], POINTER(IDebugThreadCall64), 'pptc'),
        ([], DWORD_PTR, 'dwParam1'),
        ([], DWORD_PTR, 'dwParam2'),
        ([], DWORD_PTR, 'dwParam3'),
    ),
]

IRemoteDebugCriticalErrorEvent110._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'GetErrorInfo',
        ([], POINTER(BSTR), 'pbstrSource'),
        ([], POINTER(INT), 'pMessageId'),
        ([], POINTER(BSTR), 'pbstrMessage'),
        ([], POINTER(POINTER(IDebugDocumentContext)), 'ppLocation'),
    ),
]

class tagSCRIPT_INVOCATION_CONTEXT_TYPE(ENUM):
    SICT_Event = 0
    SICT_SetTimeout = SICT_Event + 1
    SICT_SetInterval = SICT_SetTimeout + 1
    SICT_SetImmediate = SICT_SetInterval + 1
    SICT_RequestAnimationFrame = SICT_SetImmediate + 1
    SICT_ToString = SICT_RequestAnimationFrame + 1
    SICT_MutationObserverCheckpoINT = SICT_ToString + 1
    SICT_WWAExecUnsafeLocalFunction = SICT_MutationObserverCheckpoINT + 1
    SICT_WWAExecAtPriority = SICT_WWAExecUnsafeLocalFunction + 1

SCRIPT_INVOCATION_CONTEXT_TYPE = tagSCRIPT_INVOCATION_CONTEXT_TYPE

IScriptInvocationContext._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'GetContextType',
        ([], POINTER(SCRIPT_INVOCATION_CONTEXT_TYPE), 'pInvocationContextType'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'GetContextDescription',
        ([], POINTER(BSTR), 'pDescription'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'GetContextObject',
        ([], POINTER(POINTER(comtypes.IUnknown)), 'ppContextObject'),
    ),
]

class tagDEBUG_STACKFRAME_TYPE(ENUM):
    DST_SCRIPT_FRAME = 0
    DST_INTERNAL_FRAME = DST_SCRIPT_FRAME + 1
    DST_INVOCATION_FRAME = DST_INTERNAL_FRAME + 1

DEBUG_STACKFRAME_TYPE = tagDEBUG_STACKFRAME_TYPE

IDebugStackFrame110._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'GetStackFrameType',
        ([], POINTER(DEBUG_STACKFRAME_TYPE), 'pStackFrameKind'),
    ),
    COMMETHOD(
        [],
        HRESULT,
        'GetScriptInvocationContext',
        ([], POINTER(POINTER(IScriptInvocationContext)), 'ppInvocationContext'),
    ),
]

class tagDEBUG_EVENT_INFO_TYPE(ENUM):
    DEIT_GENERAL = 0
    DEIT_ASMJS_IN_DEBUGGING = DEIT_GENERAL + 1
    DEIT_ASMJS_SUCCEEDED = DEIT_ASMJS_IN_DEBUGGING + 1
    DEIT_ASMJS_FAILED = DEIT_ASMJS_SUCCEEDED + 1

DEBUG_EVENT_INFO_TYPE = tagDEBUG_EVENT_INFO_TYPE

IRemoteDebugInfoEvent110._methods_ = [
    COMMETHOD(
        [],
        HRESULT,
        'GetEventInfo',
        ([], POINTER(DEBUG_EVENT_INFO_TYPE), 'pMessageType'),
        ([], POINTER(BSTR), 'pbstrMessage'),
        ([], POINTER(BSTR), 'pbstrUrl'),
        ([], POINTER(POINTER(IDebugDocumentContext)), 'ppLocation'),
    ),
]

__all__ = (
    'IDebugApplicationThreadEvents110', 'IRemoteDebugCriticalErrorEvent110',
    'IDebugStackFrame110', 'IScriptInvocationContext', 'IDebugApplication110',
    'IActiveScriptErrorDebug110', 'IWebAppDiagnosticsObjectInitialization',
    'IWebAppDiagnosticsSetup', 'IDebugApplicationNode100', '',
    'IRemoteDebugApplication110', 'IDebugApplicationThread11064',
    'IRemoteDebugInfoEvent110', 'IDebugApplication11032',
    'IDebugApplicationThread11032', 'IDebugApplication11064',
    'IActiveScriptWinRTErrorDebug', '__REQUIRED_RPCNDR_H_VERSION__',
    'IDebugApplicationThread110', 'IID_IDebugApplicationThread110',
    'IID_IDebugApplication110', '__REQUIRED_RPCSAL_H_VERSION__',
    'tagSCRIPT_INVOCATION_CONTEXT_TYPE', 'DEBUG_EVENT_INFO_TYPE',
    'tagDEBUG_STACKFRAME_TYPE', 'tagSCRIPT_ERROR_DEBUG_EXCEPTION_THROWN_KIND',
    'SCRIPT_INVOCATION_CONTEXT_TYPE', 'APPLICATION_NODE_EVENT_FILTER',
    'SCRIPT_DEBUGGER_OPTIONS', 'DEBUG_STACKFRAME_TYPE', 'TEXT_DOCUMENT_ARRAY',
    'tagAPPLICATION_NODE_EVENT_FILTER', 'tagDEBUG_EVENT_INFO_TYPE',
    'SCRIPT_ERROR_DEBUG_EXCEPTION_THROWN_KIND', 'tagTEXT_DOCUMENT_ARRAY',
)


from pymongo.collection import Collection

def query2(collection: Collection, size: int, part_type: str, region: str):
    return collection.aggregate([
        {"$match": {
            "$and": [
                {"partsupp.part.size": {"$eq": size}},
                {"partsupp.part.type": {"$eq": part_type}},
                {"partsupp.supp.nation.region.name": {"$eq": region}},
                {"partsupp.supplycost": {"$eq": get_min(query2_aux(collection=collection,
                                                                   region=region))}}
            ]
        }},
        {"$project": {
            "s_acctbal": "$partsupp.supp.acctbal",
            "s_name": "$partsupp.supp.name",
            "n_name": "$partsupp.supp.nation.name",
            "p_partkey": "$partsupp.part.key",
            "p_mfgr": "$partsupp.part.mfgr",
            "s_address": "$partsupp.supp.address",
            "s_phone": "$partsupp.supp.phone",
            "s_comment": "$partsupp.supp.comment"
        }},
        {"$sort": {
            "s_acctbal": -1,
            "n_name": 1,
            "s_name": 1,
            "p_partkey": 1
        }}
    ])

def query2_aux(collection: Collection, region: str):
    return collection.aggregate([
        {"$match": {
            "partsupp.supp.nation.region.name": {"$eq": region}
        }},
        {"$project": {
            "ps_supplycost": "$partsupp.supplycost"
        }},
        {"$group": {
            "_id": {},
            "min_supplycost": {"$min": "$ps_supplycost"}
        }}
    ])

def get_min(res):
    for item in res:
        return item.get('min_supplycost')


import math
import json
IN1=open(r"pos_text_train.txt")
pos2pos={}
#calculate corresponding frequency
for sentence in IN1:
    word_pos_list=sentence.strip().split(" ")
    pos_list = []
    for wordpos in word_pos_list:
        word_pos = wordpos.split("/")
        pos_list.append(word_pos[-1])
    for i in range(len(pos_list)-1):
        if not pos2pos.get(pos_list[i]):
            pos2pos[pos_list[i]]={}
            pos2pos[pos_list[i]]['total'] = 0#total一开始放错位置效果比较差
        if not pos2pos.get(pos_list[i]).get(pos_list[i+1]):
            pos2pos[pos_list[i]][pos_list[i+1]]=0
        pos2pos[pos_list[i]][pos_list[i+1]] += 1
        pos2pos[pos_list[i]]['total'] += 1
# #calculate transfer pro
for head_pos in pos2pos.keys():#fromkyes()是用来新建词典的
    for follow_pos in pos2pos[head_pos].keys():
        if follow_pos=='total':continue
        pos2pos[head_pos][follow_pos]=math.log(pos2pos[head_pos][follow_pos]/pos2pos[head_pos]['total'])
out1=open("pos2pos.json","w+")
json.dump(pos2pos,out1)


# Process:
# Import necessary libraries
# Get spreadsheet location and name
# Store values in variables
# Make the input text of the name of the stock uppercase
# Present a small menu which will allow to either edit, add, or view
# Create functions for each with proper logic

import openpyxl
import os

from openpyxl import workbook
from openpyxl import load_workbook

def setup():
    directory = "C:\\Users\\shrey\\Desktop"
    directory = directory.lower()
    os.chdir(directory)
    spreadname = "Stocks.xlsx"
    workbook = openpyxl.load_workbook(spreadname)
    """ sheet_Arr = workbook.sheetnames
    answer = ''
    sheetname = ''
    while answer != 'Y':
        for x in sheet_Arr:
            answer = input(x + ": Is this the sheetname which you are trying to access? (Y/N)  ")
            answer = answer.upper()
            if (answer == 'Y'):
                sheetname = sheet_Arr[0]
                break
    print(sheetname)
     choose(sheet, workbook) """
    sheet = workbook["Sheet1"]
    return sheet

def showStock(stockName):
    sheet = setup()
    stockName = stockName.upper()
    stockRow = 0
    toReturn = ''
    for i in range(1, sheet.max_row + 1):
        if sheet.cell(row=i, column = 2).value == stockName:
            stockRow = i
    toReturn += '\n'
    toReturn += "Here is the info for " + str(stockName) + ":" + '\n'
    for i in range(1, sheet.max_column + 1):
        if i == 8 or i == 3 or i == 2 or i == 1:
            toReturn += str(i) + ": " + sheet.cell(row = 1, column = i).value + ": " + str(sheet.cell(row = stockRow, column = i).value)
        else:
             toReturn += str(i) + ": " + sheet.cell(row = 1, column = i).value + ": " + str(float((sheet.cell(row = stockRow, column = i).value)))
        toReturn += '\n'
    return toReturn

def showCurrentStocks() -> str:
    sheet = setup()
    toReturn = ''
    toReturn += "Here is a current list of your stocks: " + '\n'
    for i in range(2, sheet.max_row + 1):
        toReturn += "Investment Name: " + str(sheet.cell(row = i, column = 1).value) + '\n' + "Stock Symbol: " + str(sheet.cell(row = i, column = 2).value)
        toReturn += '\n'
    return toReturn

def getStockRow(stock):
    sheet = setup()
    stockRow = 1
    for i in range(1, sheet.max_row + 1):
        if sheet.cell(row=i, column = 2).value == stock:
            stockRow = i
    return stockRow

def editStock(choice, stockSymbol, edit):
    directory = "C:\\Users\\shrey\\Desktop"
    directory = directory.lower()
    os.chdir(directory)
    spreadname = "Stocks.xlsx"
    workbook = openpyxl.load_workbook(spreadname)
    sheet = workbook["Sheet1"]
    stockRow = getStockRow(stockSymbol)

    if choice == 7 or choice == 2 or choice == 3 or choice == 1:
        sheet.cell(row = stockRow, column = choice).value = edit
    else:
        sheet.cell(row = stockRow, column = choice).value = float(edit)
    workbook.save("Stocks.xlsx")

def addStock(sheet, workbook):
    line = sheet.max_row + 1
    for i in range(1, sheet.max_column + 1):
        if i == 7 or i == 2 or i == 1:
            add = input(sheet.cell(row = 1, column = i).value + ": ")
            sheet.cell(row = line, column = i).value = add
        else:
            add = input(sheet.cell(row = 1, column = i).value + ": ")
            sheet.cell(row = line, column = i).value = float(add)
    directory = "C:\\Users\\shrey\\Desktop"
    directory = directory.lower()
    os.chdir(directory)
    workbook.save("Stocks.xlsx")

def main():
    answer = 'Y'
    while answer == 'Y':
        setup()
        answer = input("Continue? (Y/N)  ")

if __name__ == "__main__":
    main()


# -*- coding:utf-8  -*-
# Time  : 2021/9/24 下午2:42
# Author: Yahui Cui
import copy
import sys
import numpy as np

from env.simulators.game import Game
from utils.discrete_sc2 import Discrete_SC2
from pysc2.env import sc2_env
from absl import flags

FLAGS = flags.FLAGS
FLAGS(sys.argv)

class SC2(Game):
    def __init__(self, conf):
        super(SC2, self).__init__(conf['n_player'], conf['is_obs_continuous'], conf['is_act_continuous'],
                                         conf['game_name'], conf['agent_nums'], conf['obs_type'])

        self.players = [sc2_env.Agent(sc2_env.Race[agent_type]) for agent_type in conf["agent_type"]]

        self.env_core = sc2_env.SC2Env(map_name=conf["map_name"], players=self.players,
                             agent_interface_format=sc2_env.AgentInterfaceFormat(
                                 feature_dimensions=sc2_env.Dimensions(screen=84, minimap=64)), step_mul=16,
                             game_steps_per_episode=200 * 16)

        self.max_step = int(conf["max_step"])
        self.dones = False
        self.done = False
        timesteps = self.env_core.reset()
        self.current_state = timesteps
        self.all_observes = self.get_all_observevs()
        self.joint_action_space = self.set_action_space(timesteps)
        self.action_dim = self.joint_action_space
        self.input_dimension = None

        self.init_info = None
        self.step_cnt = 0
        self.won = {}
        self.n_return = [0] * self.n_player

    def reset(self):
        self.dones = False
        self.done = False
        self.init_info = None
        timesteps = self.env_core.reset()
        self.current_state = timesteps
        self.all_observes = self.get_all_observevs()
        self.joint_action_space = self.set_action_space(timesteps)
        self.action_dim = self.joint_action_space
        self.step_cnt = 0
        self.won = {}
        self.n_return = [0] * self.n_player

    def step(self, joint_action):
        info_before = ''
        joint_action_decode = self.decode(joint_action)
        timesteps = self.env_core.step(joint_action_decode)
        self.current_state = timesteps
        self.all_observes = self.get_all_observevs()
        reward = self.set_n_return()
        done = self.is_terminal()
        self.joint_action_space = self.set_action_space(timesteps)
        self.step_cnt += 1
        info_after = ''
        return self.all_observes, reward, done, info_before, info_after

    def set_action_space(self, timesteps):
        new_joint_action_space = []
        for timestep, agent_spec in zip(timesteps, self.env_core.action_spec()):
            new_joint_action_space.append([Discrete_SC2(timestep.observation.available_actions, agent_spec)])
        return new_joint_action_space

    def get_single_action_space(self, player_id):
        return self.joint_action_space[player_id]

    def decode(self, joint_action):
        joint_action_decode = []
        for act in joint_action:
            joint_action_decode.append(act[0])
        return joint_action_decode

    def is_valid_action(self, joint_action):

        if len(joint_action) != self.n_player:
            raise Exception("Input joint action dimension should be {}, not {}".format(
                self.n_player, len(joint_action)))

        for i in range(self.n_player):
            if joint_action[i][0].function not in self.joint_action_space[i][0].available_actions:
                raise Exception("The input action dimension for player {} should be {}, does not have {}".format(
                    i, self.joint_action_space[i][0].available_actions, joint_action[i][0].function))

    def get_all_observevs(self):
        all_observes = []
        for i in range(self.n_player):
            each = copy.deepcopy(self.current_state[i])
            each = {"obs": each, "controlled_player_index": i}
            all_observes.append(each)
        return all_observes

    def set_n_return(self):
        reward = []
        for idx, obs in enumerate(self.current_state):
            self.n_return[idx] += obs.reward
            reward.append(obs.reward)
        return reward

    def is_terminal(self):
        if self.step_cnt >= self.max_step:
            self.done = True

        for obs in self.current_state:
            if obs.last():
                self.done = True

        return self.done

    def check_win(self):
        if len(self.n_return) == 1:
            return ''
        else:
            all_equal = True
            for i in range(1, len(self.n_return)):
                if self.n_return[i-1] != self.n_return[i]:
                    all_equal = False
                    break
            if all_equal:
                return -1
            return np.argmax(self.n_return)


import pandas as pd
import numpy as np
from sqlalchemy import create_engine, text
from datetime import datetime, timedelta
from math import ceil

def equipement(abonne, cible):
    try:
        return (abonne / cible) * 100
    except (ValueError, ZeroDivisionError):
        return 0
    # return 0 if cible <= 0 else (abonne / cible)*100

# engine = create_engine(
#     "postgresql+psycopg2://postgres:password@localhost:5432/produit_cash_m",
#     isolation_level="SERIALIZABLE",
# )

engine = create_engine(
    "postgresql+psycopg2://postgres:password@localhost:5432/bnk",
    # isolation_level="SERIALIZABLE",
)
connection = engine.connect()

DBD = 'DBPPT'

class Donnees():
    def __init__(self, date):
        self.current_date = datetime.strptime(date, "%Y-%m-%d")
        self.current_month = self.current_date.strftime('%m')
        self.current_year = self.current_date.strftime('%Y')

        self.next_date = self.current_date + timedelta(days=31)
        self.next_month = self.next_date.strftime('%m')
        self.next_year = self.next_date.strftime('%Y')

        self.data = None

    @property
    def getPeriode(self):
        return f"periode BETWEEN '{self.current_year}-{self.current_month}-01' AND '{self.next_year}-{self.next_month}-01'"

    def getData(self):
        request = f"SELECT * FROM abonne_produit WHERE {self.getPeriode}"
        data = pd.read_sql(sql=text(request), con=connection)
        data = data.drop(columns=['id'])
        data['periode'] = data['periode'].astype(str)
        data['directions'] = data['directions'].apply(lambda x: x if x is not None else DBD)
        data['services'] = data['services'].apply(lambda x: x if x is not None else DBD)
        self.data = data

    def getPortefeuille(self):
        self.getData()
        df_portefeuille = self.data['directions'].value_counts()
        dict_portefeuille = []
        for i in range(df_portefeuille.shape[0]):
            _ = {'directions': str(df_portefeuille.index[i]), 'portefeuille': int(df_portefeuille.values[i])}
            dict_portefeuille.append(_)
        for dic in dict_portefeuille:
            if dic["directions"] == DBD:
                dic["directions"] = None
                dic["portefeuille"] = 0
        return dict_portefeuille

    def evoProduit(self, produit):
        request = f"""
                SELECT *
                FROM
                    ((SELECT periode, COUNT(racine) as abonne_corpo
                    FROM abonne_produit
                    WHERE (periode BETWEEN '{self.current_year}-01-01' AND '{self.current_year}-12-31')
                    AND directions IS NOT NULL AND {produit}='Abonné'
                    GROUP BY periode
                    ORDER BY periode ASC) as x

                LEFT JOIN
                    (SELECT periode as periode2, COUNT(racine) as cible
                    FROM abonne_produit
                    WHERE cible_{produit}='oui' AND (periode BETWEEN '{self.current_year}-01-01' AND '{self.current_year}-12-31')
                    GROUP BY periode) as z
                ON x.periode = z.periode2) as n

                LEFT JOIN
                    (SELECT periode as periode3, COUNT(racine) as abonne
                    FROM abonne_produit
                    WHERE {produit}='Abonné' AND (periode BETWEEN '{self.current_year}-01-01' AND '{self.current_year}-12-31')
                    GROUP BY periode3) as y
                ON n.periode = y.periode3
            """

        df = pd.read_sql(sql=text(request), con=connection)
        df = df.drop(columns=['periode2', 'periode3'])
        # df['croissance'] = 0
        df['equipement'] = np.vectorize(equipement)(df['abonne_corpo'], df['cible'])

        df['periode'] = df['periode'].astype(str)
        df = df.rename(columns={'periode': 'date'})
        df = df.rename(str.capitalize, axis='columns')

        # print(df)
        dict_df = []
        for j in range(df.shape[0]):
            _ = {
                df.columns[0]: df[df.columns[0]][j],
                df.columns[1]: int(df[df.columns[1]][j]),
                df.columns[2]: int(df[df.columns[2]][j]),
                df.columns[3]: int(df[df.columns[3]][j]),
                # df.columns[4]: int(df[df.columns[4]][j]),
                df.columns[4]: float(round(df[df.columns[4]][j], 2), )
            }
            dict_df.append(_)
        return dict_df

    def abonneCorpo(self, prod):
        x = self.data.groupby(by=["directions"]).count()[[f'{prod}']]
        x = x.drop(DBD)
        x = x.rename_axis('directions').reset_index()
        return x[f'{prod}'].sum()

    def getInfoProduct(self, produit):
        abonne = self.data[produit].value_counts()[0]
        corpo_abonne = self.data[self.data[f'{produit}'] == 'Abonné']
        corpo_abonne = corpo_abonne[corpo_abonne['directions'] != DBD][produit].value_counts()[0]
        cible = self.data[f'cible_{produit}'].value_counts()[0]
        non_abonne = cible - abonne
        try:
            equipement = (corpo_abonne / cible) * 100
        except ValueError:
            equipement = 0

        return [{
            'Abonne': int(abonne),
            'corpo_abonne': int(corpo_abonne),
            'Cible': int(cible),
            'Non Abonné': int(non_abonne),
            'Equipement': round(equipement, 2)
        }]

    def getResumeGraph(self, produit):
        request = f"""
                SELECT *
                FROM
                    ((SELECT COALESCE(directions, '{DBD}') as directions, COUNT(racine) as portefeuille
                    FROM abonne_produit
                    WHERE {self.getPeriode}
                    GROUP BY directions
                    ORDER BY directions ASC) as x

                LEFT JOIN
                    (SELECT COALESCE(directions, '{DBD}') as directions2, COUNT(racine) as cible
                    FROM abonne_produit
                    WHERE cible_{produit}='oui' AND {self.getPeriode}
                    GROUP BY directions2) as z
                ON x.directions = z.directions2) as n

                LEFT JOIN
                    (SELECT COALESCE(directions, '{DBD}') as directions3, COUNT(racine) as abonne
                    FROM abonne_produit
                    WHERE {produit}='Abonné' AND {self.getPeriode}
                    GROUP BY directions3) as y
                ON n.directions = y.directions3
            """

        df = pd.read_sql(sql=text(request), con=connection)
        df = df.drop(columns=['directions2', 'directions3'])
        df['non_abonne'] = df['cible'] - df['abonne']
        df['equipement'] = np.vectorize(equipement)(df['abonne'], df['cible'])
        df = df.fillna(0)

        dict_df = []
        for j in range(df.shape[0]):
            # _ = {col: df[col][j] for col in df.columns}
            # print(_)
            _ = {
                df.columns[0]: df[df.columns[0]][j],
                df.columns[1]: int(df[df.columns[1]][j]),
                df.columns[2]: int(df[df.columns[2]][j]),
                df.columns[3]: int(df[df.columns[3]][j]),
                df.columns[4]: int(df[df.columns[4]][j]),
                df.columns[5]: float(round(df[df.columns[5]][j], 2), )
            }
            dict_df.append(_)

        for dic in dict_df:
            if dic["directions"] == DBD:
                dic["equipement"] = None
                dic["non_abonne"] = None
                dic["cible"] = None
                dic["portefeuille"] = None
        return dict_df

    def directionResumed(self, produit):
        request = f"""
                SELECT *
                FROM
                    ((SELECT COALESCE(directions, '{DBD}') as directions, COALESCE(services, 'DBPPT') as services,
                    COUNT(racine) as portefeuille
                    FROM abonne_produit
                    WHERE {self.getPeriode}
                    GROUP BY directions, services
                    ORDER BY directions ASC) as x

                LEFT JOIN
                    (SELECT COUNT(racine) as cible, COALESCE(services, '{DBD}') as services2
                    FROM abonne_produit
                    WHERE cible_{produit}='oui' AND {self.getPeriode}
                    GROUP BY services2) as z
                ON x.services = z.services2) as n

                LEFT JOIN
                    (SELECT COALESCE(services, '{DBD}') as services3,
                    COUNT(racine) as abonnés
                    FROM abonne_produit
                    WHERE {produit}='Abonné' AND {self.getPeriode}
                    GROUP BY services3) as y
                ON n.services = y.services3
            """

        df = pd.read_sql(sql=text(request), con=connection)
        df = df.rename(str.capitalize, axis='columns')
        df = df.drop(columns=['Services2', 'Services3'])

        df['Non Abonnés'] = df['Cible'] - df['Abonnés']
        df['Equipement'] = np.vectorize(equipement)(df['Abonnés'], df['Cible'])
        # df = df.rename_axis(['Directions', 'Services']).reset_index()
        df = df.fillna(0)

        dict_df = []
        for j in range(df.shape[0]):
            _ = {
                df.columns[0]: df[df.columns[0]][j],
                df.columns[1]: df[df.columns[1]][j],
                df.columns[2]: int(df[df.columns[2]][j]),
                df.columns[3]: int(df[df.columns[3]][j]),
                df.columns[4]: int(df[df.columns[4]][j]),
                df.columns[5]: int(df[df.columns[5]][j]),
                df.columns[6]: float(round(df[df.columns[6]][j], 2)),
            }
            dict_df.append(_)

        for dic in dict_df:
            if dic["Equipement"] > 200:
                dic["Equipement"] = None
                dic["Non Abonnés"] = None
                dic["Cible"] = None
                dic["Portefeuille"] = None
                dic["Services"] = None
        return dict_df

    def getNonAbonne(self, produit):
        request = f"""
            SELECT racine, nom_client, contact_date, Secteur, Nom_gestionnaire,
           COALESCE(directions, '{DBD}') as directions, COALESCE(services, 'DBPPT') as services,
           cible_{produit}, agence
           FROM abonne_produit
           WHERE cible_{produit}='oui' and {produit} IS NULL AND {self.getPeriode}
        """
        df = pd.read_sql(sql=text(request), con=connection)
        df = df.rename(columns={f'cible_{produit}': 'Ciblé'})
        df['contact_date'] = pd.to_datetime(df['contact_date'], format='%Y%m%d', errors='ignore')
        df['contact_date'] = df['contact_date'].apply(lambda date: date.strftime('%Y-%m-%d'))
        df = df.rename(str.capitalize, axis='columns')
        df = df.fillna('')

        dict_df = []
        for j in range(df.shape[0]):
            _ = {
                df.columns[0]: df[df.columns[0]][j],
                df.columns[1]: df[df.columns[1]][j],
                df.columns[2]: df[df.columns[2]][j],
                df.columns[3]: df[df.columns[3]][j],
                df.columns[4]: df[df.columns[4]][j],
                df.columns[5]: df[df.columns[5]][j],
                df.columns[6]: df[df.columns[6]][j],
                df.columns[7]: df[df.columns[7]][j],
                df.columns[8]: df[df.columns[8]][j],
            }
            dict_df.append(_)
        return dict_df

    def getInfoGestionnaire(self, produit):
        request = f"""
                SELECT *
                FROM
                    ((SELECT nom_gestionnaire as gestionnaires, COUNT(racine) as portefeuille
                    FROM abonne_produit
                    WHERE {self.getPeriode}
                    GROUP BY gestionnaires
                    ORDER BY gestionnaires ASC) as x

                LEFT JOIN
                    (SELECT nom_gestionnaire as gestionnaires2, COUNT(racine) as cible
                    FROM abonne_produit
                    WHERE cible_{produit}='oui' AND {self.getPeriode}
                    GROUP BY gestionnaires2) as z
                ON x.gestionnaires = z.gestionnaires2) as n

                LEFT JOIN
                    (SELECT nom_gestionnaire as gestionnaires3, COUNT(racine) as abonnés
                    FROM abonne_produit
                    WHERE {produit}='Abonné' AND {self.getPeriode}
                    GROUP BY gestionnaires3) as y
                ON n.gestionnaires = y.gestionnaires3
            """
        df = pd.read_sql(sql=text(request), con=connection)
        df = df.rename(str.capitalize, axis='columns')
        df = df.drop(columns=['Gestionnaires2', 'Gestionnaires3'])

        df = df.dropna(subset=['Gestionnaires'])
        df = df.fillna(0)
        df['Non Abonnés'] = df['Cible'] - df['Abonnés']
        df['Equipement'] = np.vectorize(equipement)(df['Abonnés'], df['Cible'])
        df['Non Abonnés'] = df['Non Abonnés'].apply(lambda x: max(int(x), 0))
        df['Equipement'] = df['Equipement'].fillna(0)

        dict_df = []
        for j in range(df.shape[0]):
            _ = {
                df.columns[0]: df[df.columns[0]][j],
                df.columns[1]: int(df[df.columns[1]][j]),
                df.columns[2]: int(df[df.columns[2]][j]),
                df.columns[3]: int(df[df.columns[3]][j]),
                df.columns[4]: int(df[df.columns[4]][j]),
                df.columns[5]: float(round(df[df.columns[5]][j], 2)),
            }
            dict_df.append(_)
        return dict_df

class TableProduit():
    def __init__(self, date, day=0):
        self.date = date
        self.day = day

        self.start_date = None
        self.end_date = None
        self.data = None

    def abonneCorpo(self, prod):
        x = self.data.groupby(by=["directions"]).count()[[f'{prod}']]
        x = x.drop(DBD)
        x = x.rename_axis('directions').reset_index()
        return x[f'{prod}'].sum()

    def getDate(self):
        if self.date is not None:
            current_periode = datetime.strptime(self.date, "%Y-%m-%d")
            current_periode = current_periode - timedelta(days=self.day)
        else:
            self.date = datetime.now()

        current_month = current_periode.strftime('%m')
        current_year = current_periode.strftime('%Y')

        next_periode = current_periode + timedelta(days=31)
        next_month = next_periode.strftime('%m')
        next_year = next_periode.strftime('%Y')

        start_date = f'{current_year}-{current_month}-01'
        end_date = f'{next_year}-{next_month}-01'

        return start_date, end_date

    def getData(self):
        start_date, end_date = self.getDate()
        request = f"""
            SELECT periode, semaine, COALESCE(directions, '{DBD}') as directions, cible_anet,
            cible_tpe, cible_acs, cible_ecnps, cible_ramassage, cible_efacture, anet, tpe, acs,
            ecnps, ramassage, efacture
            FROM abonne_produit
            WHERE (periode BETWEEN '{start_date}' AND '{end_date}')
        """
        self.data = pd.read_sql(sql=text(request), con=connection)

    def compaProd(self):
        self.getData()
        if len(self.data.index) > 0:
            produits = list(self.data.columns[-6:])
            df = pd.DataFrame({'produits': produits})
            df['cible'] = df['produits'].apply(lambda prod: self.data[f'cible_{prod}'].value_counts())
            df['abonné'] = df['produits'].apply(lambda prod: self.data[prod].value_counts())
            df['abonné corpo'] = df['produits'].apply(self.abonneCorpo)
        else:
            produits = ['anet', 'tpe', 'acs', 'ecnps', 'ramassage', 'efacture']
            df = pd.DataFrame({'produits': produits})
            df['cible'] = 0
            df['abonné'] = 0
            df['abonné corpo'] = 0
        df['produits'] = df['produits'].apply(lambda prod: prod.upper())
        return df

    def getEquipement(self):
        if self.data is None:
            self.getData()

        produits = list(self.data.columns[-6:])
        df = None

        if len(self.data.index) > 0:
            for prod in produits:
                x = pd.DataFrame()
                x['abonné'] = self.data.groupby(by=["directions"]).count()[[f'{prod}']].drop(DBD)
                x['cible'] = self.data.groupby(by=["directions"]).count()[[f'cible_{prod}']]
                x['equipement_encours'] = np.vectorize(equipement)(x['abonné'], x['cible'])
                x = x.drop(columns=['cible', 'abonné']).T
                x['produit'] = f'{prod}'

                df = x if df is None else pd.concat([df, x])
                df = df.reset_index(drop=True)
                df['produit'] = df['produit'].apply(lambda x: str(x).upper())

        else:
            df = pd.DataFrame({'produit': produits})
            df['DGEI'] = 0
            df['DMS'] = 0
            df['DPME'] = 0
            df = df[['DGEI', 'DMS', 'DPME', 'produit']]
        return df

def tc(current_ab, last_ab):
    return 0 if last_ab <= 0 else ((current_ab - last_ab) / last_ab) * 100

def prodData(date=None):
    current_date = '2022-09-30'
    if date is None:
        date = '2022-09-30'

    current = TableProduit(date=current_date)
    current_df = current.compaProd()
    current_df['equipement'] = np.vectorize(equipement)(current_df['abonné corpo'], current_df['cible'])

    encours = TableProduit(date=date)
    encours_df = encours.compaProd()
    encours_df['equipement_encours'] = np.vectorize(equipement)(encours_df['abonné corpo'], encours_df['cible'])

    m_1 = TableProduit(date=date, day=31)
    m_1df = m_1.compaProd()
    m_1df = m_1df.drop(columns=['cible', 'abonné corpo'])

    n_1 = TableProduit(date=date, day=365)
    n_1df = n_1.compaProd()
    n_1df = n_1df.drop(columns=['cible', 'abonné corpo'])

    df = pd.merge(current_df, encours_df, how='inner', on='produits', suffixes=('', '_encours'))
    df = pd.merge(df, m_1df, how='inner', on='produits', suffixes=('', '_m1'))
    df = pd.merge(df, n_1df, how='inner', on='produits', suffixes=('', '_n1'))
    df['tc_m1'] = np.vectorize(tc)(df['abonné_encours'], df['abonné_m1'])
    df['tc_n1'] = np.vectorize(tc)(df['abonné_encours'], df['abonné_n1'])

    dict_df = []
    for j in range(df.shape[0]):
        _ = {
            df.columns[0]: df[df.columns[0]][j],
            df.columns[1]: int(df[df.columns[1]][j]),
            df.columns[2]: int(df[df.columns[2]][j]),
            df.columns[3]: int(df[df.columns[3]][j]),
            df.columns[4]: int(df[df.columns[4]][j]),
            df.columns[5]: int(df[df.columns[5]][j]),
            df.columns[6]: int(df[df.columns[6]][j]),
            df.columns[7]: int(df[df.columns[7]][j]),
            df.columns[8]: int(df[df.columns[8]][j]),
            df.columns[9]: int(df[df.columns[9]][j]),
            df.columns[10]: int(df[df.columns[10]][j]),
            df.columns[11]: float(round(df[df.columns[11]][j], 2)),
            df.columns[12]: float(round(df[df.columns[12]][j], 2)),
        }
        dict_df.append(_)

    return dict_df

def getProdEquipement(date=None):
    if date is None:
        date = '2022-09-22'
    df = TableProduit(date=date)
    df = df.getEquipement()
    dict_df = []
    for j in range(df.shape[0]):
        _ = {
            df.columns[0]: float(round(df[df.columns[0]][j], 2)),
            df.columns[1]: float(round(df[df.columns[1]][j], 2)),
            df.columns[2]: float(round(df[df.columns[2]][j], 2)),
            df.columns[3]: df[df.columns[3]][j],
        }
        dict_df.append(_)
    return dict_df


#!/usr/bin/python3
""" extend your Python script to export data in the JSON format. """
import json
import requests
from sys import argv

if __name__ == "__main__":
    url = "https://jsonplaceholder.typicode.com"

    TOTAL_NUMBER_OF_TASKS = requests.get(
        "{}/todos?userId={}".format(url, argv[1])).json()
    USERNAME = requests.get(
        "{}/users/{}".format(url, argv[1])).json()["username"]
    task_list = []
    for task in TOTAL_NUMBER_OF_TASKS:
        task_list.append({"task": task["title"],
                          "completed": task["completed"],
                          "username": USERNAME})
    task_list = {argv[1]: task_list}
    with open("{}.json".format(argv[1]), "w") as file:
        json.dump(task_list, file)


from typing import Iterable
from contextvars import ContextVar
from sanic.request import Request
from sanic.response import BaseHTTPResponse, json
from http import HTTPStatus

class SanicEndpoint:

    def __init__(self, uri: str, methods: Iterable, context: ContextVar, *args, **kwargs):
        self.uri = uri
        self.methods = methods
        self.database_context = context
        self.__name__ = self.__class__.__name__

    async def __call__(self, request: Request, *args, **kwargs):
        return await self.handler(request)

    async def handler(self, request: Request, *args, **kwargs):

        body = {}
        body.update(await self.import_body_json(request))
        body.update(await self.import_headers_json(request))

        return await self._method(request, body, *args, **kwargs)

    async def _method(self, request: Request, body: dict, *args, **kwargs) -> BaseHTTPResponse:
        method = request.method.lower()
        func_name = f'method_{method}'

        if hasattr(self, func_name):
            func = getattr(self, func_name)
            return await func(request, body, *args, **kwargs)
        return await self.method_not_allowed()

    @staticmethod
    async def make_response_json(
            body: dict = None,
            message: str = None,
            status: int = 200) -> BaseHTTPResponse:
        if body is None:
            body = {
                'message': message or HTTPStatus(status).phrase,
            }

        return json(body, status)

    @staticmethod
    async def import_body_json(request: Request) -> dict:
        if 'application/json' in request.content_type and request.json is not None:
            return dict(request.json)
        return {}

    @staticmethod
    async def import_headers_json(request: Request) -> dict:
        return {
            header : value
            for header, value in request.headers.items()
            if header.lower().startswith('-x') ##
        }

    async def method_not_allowed(self) -> BaseHTTPResponse:
        return await self.make_response_json(status=501)

    async def method_not_implemented(self) -> BaseHTTPResponse:
        return await self.make_response_json(status=501)

    async def method_get(self, request: Request, body: dict, *args, **kwargs):
        return await self.method_not_implemented()

    async def method_post(self, request: Request, body: dict, *args, **kwargs):
        return await self.method_not_implemented()

    async def method_patch(self, request: Request, body: dict, *args, **kwargs):
        return await self.method_not_implemented()

    async def method_delete(self, request: Request, body: dict, *args, **kwargs):
        return await self.method_not_implemented()