import numpy as np
import math
import random
import sys
import itertools
from pathlib import Path
home = str(Path.home())
import sys
rankability_path = "%s/rankability_toolbox_dev"%home
if rankability_path not in sys.path:
    sys.path.insert(0,rankability_path)
import pyrankability
from pyrankability.rank import solve
from pyrankability.search import solve_pair_min_tau
import json
from tqdm import tqdm
from scipy import stats
from utilities import *

######## STATISTICS OF P ########
# Functions of P which package certain similar statistics
# together for sake of efficiency. These functions will probably
# replace the current "RankabilityMetric" classes since running
# cross-validated linear models seems to be the better alternative
# to hand-crafting metrics.

# This function found at: https://stackoverflow.com/a/48916127
# TODO: Check to make sure you dont need np.argsort
def kendall_w(expt_ratings):
    expt_ratings = np.array(expt_ratings)
    if expt_ratings.ndim!=2:
        raise 'ratings matrix must be 2-dimensional'
    m = expt_ratings.shape[0] # number of raters
    n = expt_ratings.shape[1] # number of items
    denom = m**2*(n**3-n)
    rating_sums = np.sum(expt_ratings, axis=0)
    S = n*np.var(rating_sums)
    return "kendall_w", 12*S/denom

# "Lowerbound" because many random restarts may not find full P
# TODO: Add SCIP compatibility to grab full P set rather than estimate
def p_len(P):
    return "p_lowerbound", len(P)

def l2_dist_stats(P):
    p = len(P)
    max_dist = 0.0
    mean_dist = 0.0
    for r1 in range(p):
        for r2 in range(r1+1, p):
            dist = np.linalg.norm(np.array(P[r1]) - np.array(P[r2]))
            mean_dist += dist
            if dist > max_dist:
                max_dist = dist

    if p > 1:
        mean_dist /= (p*(p-1)/2.0)

    return ["max_L2_dist", "mean_L2_dist"], [max_dist, mean_dist]

def tau_stats(P):
    p = len(P)
    min_tau = 1.0
    mean_tau = 0.0
    for r1 in range(p):
        for r2 in range(r1+1, p):
            tau = kendall_tau(P[r1], P[r2])
            mean_tau += tau
            if tau < min_tau:
                min_tau = tau

    if p > 1:
        mean_tau /= (p*(p-1)/2.0)
    else:
        mean_tau = 1.0

    return ["min_tau", "mean_tau"], [min_tau, mean_tau]

def get_P_stats(P):
    # Ensure P is a set, then make addressable as a list
    P = list(set(P))
    results = {}
    for func in [kendall_w, p_len, l2_dist_stats, tau_stats]:
        stat_names, stat_values = func(P)
        if type(stat_names) == list:
            for i in range(len(stat_names)):
                results[stat_names[i]] = stat_values[i]
        else:
            results[stat_names] = stat_values
    return results

######## RANKABILITY METRICS ########
# Not currently employed by the core testbed

class RankabilityMetric:

    def compute(self, k, details):
        P = list(set(details["P"]))
        return self._compute(k, P)

    def _compute(self, k, P):
        # Child classes should compute their rankability metric from k and P
        raise NotImplemented("Don't use the generic RankabilityMetric class")

class RatioToMaxMetric(RankabilityMetric):
    def _compute(self, k, P):
        n = len(P[0])
        return 1.0 - (k*len(P) / ((n**2 - n)/2 * math.factorial(n)))

class KendallWMetric(RankabilityMetric):

    def _compute(self, k, P):
        n = len(P[0])
        _, w = kendall_w(P)
        return 1 - ((k / (n**3 - n**2)) * (1 - w))

class L2DifferenceMetric(RankabilityMetric):

    def __init__(self, strategy="max"):
        strategy=strategy.lower()
        self.strategy = strategy
        if strategy == "max":
            self.get_dist_stat = self.get_max_dist
        elif strategy == "mean":
            self.get_dist_stat = self.get_mean_dist
        else:
            raise ValueError("Unrecognized L2DifferenceMetric strategy: %s" % strategy)

    def get_mean_dist(self, P):
        p = len(P)
        mean_dist = 0.0
        for r1 in range(p):
            for r2 in range(r1+1, p):
                mean_dist += np.linalg.norm(np.array(P[r1]) - np.array(P[r2]))

        if p == 1:
            return mean_dist
        else:
            return mean_dist / (p*(p-1)/2.0)

    def get_max_dist(self, P):
        p = len(P)
        max_dist = 0.0
        for r1 in range(p):
            for r2 in range(r1+1, p):
                dist = np.linalg.norm(np.array(P[r1]) - np.array(P[r2]))
                if dist > max_dist:
                    max_dist = dist
        return max_dist

    def _compute(self, k, P):
        p = len(P)
        n = len(P[0])
        dist = self.get_dist_stat(P)
        return 1.0 - (k / (n**3 - n**2) * (dist / np.sqrt((n / 3) * (n**2 - 1))))

class MeanTauMetric(RankabilityMetric):
    # Two similar statistics exist for the use of mean tau
    #      "hays"  -->  W_a defined by Hays (1960)
    # "ehrenberg"  -->  W_t defined by Ehrenberg (1952)
    #
    # The two strategies are the same when m (# of rankings) is even
    # Hays' W_a is computed with m+1 instead of m when m is odd
    # This ensures that 0 is the minimum value at all values of m
    #
    # In keeping with the statistics literature I've read:
    #    m  <-- number of rankings (equivalent to p)
    #    n  <-- number of items to rank / length of rankings
    #    u  <-- mean kendall tau over all pairs of rankings

    def __init__(self, strategy="hays"):
        strategy = strategy.lower()
        self.strategy = strategy
        if strategy == "hays":
            self.get_W = self.get_W_a
        elif strategy == "ehrenberg":
            self.get_W = self.get_W_t
        else:
            raise ValueError("Unrecognized MeanTauMetric strategy: %s" % strategy)

    # Defined by Ehrenberg (1952) as a possible measure of concordance
    def get_W_t(self, m, u):
        return ((m - 1.0) * u + 1.0) / m

    # Defined by Hays (1960) so that a minimum of 0 is always possible for all values of m
    def get_W_a(self, m, u):
        if m & 1 == 1:
            m += 1
        return ((m - 1.0) * u + 1.0) / m

    def _compute(self, k, P):
        p = len(P)
        if p == 1:
            return 1.0
        if p == 2:
            u = kendall_tau(P[0], P[1])
        else:
            u = 0.0
            for r1 in range(p):
                for r2 in range(r1+1, p):
                    tau = kendall_tau(P[r1], P[r2])
                    u += tau
            u /= p*(p-1)/2
        n = len(P[0])
        w_stat = self.get_W(p, u)
        return 1 - (k / (n**3 - n**2) * (1.0 - w_stat))

######## NOISE GENERATORS ########

class NoiseGenerator:

    def apply_noise(self, D):
        # Child classes should return numpy array of D_tilde
        raise NotImplemented("Don't use the generic NoiseGenerator class")

class IdentityNoise(NoiseGenerator):
    def apply_noise(self, D):
        return D

    def __str__(self):
        return "IdentityNoise()"

class BinaryFlipNoise(NoiseGenerator):
    def __init__(self, noisePercentage):
        self.noisePercentage = noisePercentage

    def apply_noise(self, D):
        D_noisy = np.copy(D)
        n = len(D_noisy)
        num_flips = (np.square(n) - n) * self.noisePercentage
        unique_elems = set()
        for flip in range(int(num_flips)):
            i, j = random.sample(range(n), 2) #Ensures that i and j are distinct
            while ((i, j) in unique_elems): i, j = random.sample(range(n), 2)
            unique_elems.add((i, j))
            D_noisy[i][j] = 1 - D_noisy[i][j]
        return D_noisy

    def __str__(self):
        return "BinaryFlipNoise({})".format(self.noisePercentage)

class SwapNoise(NoiseGenerator):
    # Flips (swaps edge weights for (i,j) and (j,i)) for given percentage
    # of relationships. 100% noise corresponds to returning D.T

    def __init__(self, noisePercentage):
        self.noisePercentage = noisePercentage

    def apply_noise(self, D):
        D_noisy = np.copy(D)
        n = len(D_noisy)
        num_swaps = int((n*n - n)/2 * self.noisePercentage)
        if self.noisePercentage > 0.5:
            D_noisy = D_noisy.T
            num_swaps = int((n*n - n)/2 - num_swaps)
        # Get indices of upper-triangular elements
        i_arr, j_arr = np.triu_indices(n,1)
        num_offdiag = len(i_arr)
        indices = random.sample(range(num_offdiag), num_swaps)
        for idx in indices:
            i, j = i_arr[idx], j_arr[idx]
            # Reverse this relationship
            temp = D_noisy[i,j]
            D_noisy[i][j] = D_noisy[j][i]
            D_noisy[j][i] = temp
        return D_noisy

    def __str__(self):
        return "SwapNoise({})".format(self.noisePercentage)

class BootstrapResamplingNoise(NoiseGenerator):

    def __init__(self, noisePercentage, byRow=True):
        self.noisePercentage = noisePercentage
        self.byRow = byRow

    def apply_noise(self, D):
        D_noisy = np.copy(D)
        n = len(D_noisy)
        num_resampled = int((n*n - n) * self.noisePercentage)
        # Get indices of all off-diagonal elements
        i_arr, j_arr = np.where(~np.eye(D.shape[0],dtype=bool))

        # Select the elements to be resampled and the bootstrap samples to use
        num_offdiag = len(i_arr)
        indices = random.sample(range(num_offdiag), num_resampled) # Without replacement
        if self.byRow:
            # Replaces elements from ones in the same row (resampling only within the row)
            bootstrap = np.random.randint(n-1, size=num_resampled) # With replacement
            for idx, rel_idx in enumerate(indices):
                # rel_idx is the index of the off-diagonal relationship to be replaced
                # idx is the index of the bootstrap sample to replace it with from the same row
                bs_idx = bootstrap[idx]
                # Replace the chosen off-diag element of D_noisy using the
                # off-diag element chosen from the same row
                row = i_arr[rel_idx]
                old_col = j_arr[rel_idx]
                new_col = bs_idx if bs_idx < row else bs_idx + 1
                D_noisy[row, old_col] = D[row, new_col]
        else:
            # Replaces elements with random off-diagonal element from entire D
            bootstrap = np.random.randint(num_offdiag, size=num_resampled) # With replacement
            for idx, rel_idx in enumerate(indices):
                # rel_idx is the index of the off-diagonal relationship to be replaced
                # idx is the index of the bootstrap sample to replace it with
                bs_rel_idx = bootstrap[idx]
                # Replace the chosen off-diag element of D_noisy using the
                # off-diag element chosen from D
                D_noisy[i_arr[rel_idx], j_arr[rel_idx]] = D[i_arr[bs_rel_idx], j_arr[bs_rel_idx]]

        return D_noisy

    def __str__(self):
        return "BootstrapResamplingNoise({},{})".format(self.noisePercentage, self.byRow)

class NormalResamplingNoise(NoiseGenerator):

    def __init__(self, noisePercentage, clip_range=False):
        self.noisePercentage = noisePercentage
        self.clip_range = clip_range

    # TODO: Change from replacement to additive noise
    def apply_noise(self, D):
        D_noisy = np.copy(D)
        n = len(D_noisy)
        num_resampled = int((n*n - n) * self.noisePercentage)

        # Get indices of all off-diagonal elements
        i_arr, j_arr = np.where(~np.eye(D.shape[0],dtype=bool))
        offdiags = D[i_arr, j_arr]

        # Get the range of the off-diagonal elements if clipping
        if self.clip_range:
            D_min = np.amin(offdiags)
            D_max = np.amax(offdiags)

        # Fit a normal distribution to all off-diagonal elements
        mu, sigma = stats.norm.fit(offdiags)

        # Select the elements to be resampled
        indices = random.sample(range(len(i_arr)), num_resampled) # Without replacement

        for rel_idx in indices:
            # rel_idx is the index of the off-diagonal relationship to be replaced
            # Replace the chosen off-diag element of D_noisy using a
            # sample from the normal distribution fit to the data
            sample = stats.norm.rvs(loc=mu, scale=sigma)
            if self.clip_range:
                sample = np.clip(sample, D_min, D_max)
            D_noisy[i_arr[rel_idx], j_arr[rel_idx]] = sample

        return D_noisy

    def __str__(self):
        return "NormalResamplingNoise({})".format(self.noisePercentage)

class NormalAdditiveNoise(NoiseGenerator):

    def __init__(self, noisePercentage, varPercentage=1.0, explicitVar=None):
        self.noisePercentage = noisePercentage
        self.varPercentage = varPercentage
        self.explicitVar = explicitVar

    def apply_noise(self, D):
        D_noisy = np.copy(D)
        n = len(D_noisy)
        num_noised = int((n*n - n) * self.noisePercentage)

        # Get indices of all off-diagonal elements
        i_arr, j_arr = np.where(~np.eye(D.shape[0],dtype=bool))
        offdiags = D[i_arr, j_arr]

        # Calculate variance of additive noise
        if self.explicitVar is not None:
            sigma = self.explicitVar * self.varPercentage
        else:
            _, sigma = stats.norm.fit(offdiags)
            sigma = sigma * self.varPercentage

        # Select the elements to be noised
        indices = random.sample(range(len(i_arr)), num_noised) # Without replacement

        for rel_idx in indices:
            D_noisy[i_arr[rel_idx], j_arr[rel_idx]] += stats.norm.rvs(loc=0, scale=sigma)

        return D_noisy

    def __str__(self):
        return "NormalAdditiveNoise({},{})".format(self.noisePercentage, self.varPercentage)

######## DATA SOURCES ########

class DataSource:

    def init_D(self):
        # Child classes should return numpy array
        raise NotImplemented("Don't use the generic DataSource class")

class LOLib(DataSource):
    def __init__(self, file_name, full_path=False):
        self.n = 0
        self.file_name = file_name
        self.full_path = full_path

    def get_filepath(self):
        if self.full_path:
            return self.file_name
        else:
            return "./lolib_data/" + self.file_name

    def get_n(self):
        if self.n is None or self.n < 1:
            with open(self.get_filepath(), 'r') as f:
                self.n = int(f.readline())
        return self.n

    def init_D(self):
        with open(self.get_filepath(), 'r') as f:
            unparsed = f.read().splitlines()
        self.n = int(unparsed[0]) #0th element is dim of D
        elements = [row.split(' ') for row in unparsed[1:]]
        D = []
        for row in elements:
            D.append([int(e) for e in row if e != ''])
        return np.array(D)

    def __str__(self):
        return "LOLib({})".format(self.file_name)

class PerfectBinarySource(DataSource):

    def __init__(self, n):
        self.n = n

    def init_D(self):
        D = np.zeros((self.n,self.n), dtype=int)
        D[np.triu_indices(self.n,1)] = 1
        return D

    def __str__(self):
        return "PerfectBinarySource({})".format(self.n)

class PerfectWeightedSource(DataSource):

    def __init__(self, n, scale=1):
        self.n = n
        self.scale = scale

    def init_D(self):
        D = np.zeros((self.n,self.n), dtype=float)
        i_arr, j_arr = np.triu_indices(self.n,1)
        for idx in range(len(i_arr)):
            i, j = i_arr[idx], j_arr[idx]
            D[i, j] = self.scale * (-i+j)
        return D

    def __str__(self):
        return "PerfectWeightedSource({},{})".format(self.n, self.scale)

class SynthELOTournamentSource(DataSource):
    # Simulates a tournament played by competitors with
    # normally distributed ELO scores
    # This ELO system uses a logistic curve to predict win probability

    def __init__(self, n, n_games=5, comp_var=200, elo_scale=400):
        # n --> the number of competitors
        # n_games --> the number of games played between every pair of competitors
        # comp_var --> the variance of the normal distribution competitor ELOs are drawn from
        #     a smaller comp_var/elo_scale ratio will produce a very rankable matrix
        #     a larger comp_var/elo_scale ratio will produce a very unrankable matrix
        # elo_scale --> the scale parameter for the ELO system
        self.n = n
        self.n_games = n_games
        self.comp_var = comp_var
        self.elo_scale = elo_scale

    def sigmoid(self, x):
        return 1 / (1 + math.exp(-x))

    def init_D(self):
        ELOs = sorted(stats.norm.rvs(scale=self.comp_var, size=self.n), reverse=True)
        D = np.zeros((self.n,self.n), dtype=float)
        i_arr, j_arr = np.triu_indices(self.n,1)
        for idx in range(len(i_arr)):
            i, j = i_arr[idx], j_arr[idx]
            scaled_elo_diff = (ELOs[i] - ELOs[j])/self.elo_scale
            prob_i_win = self.sigmoid(scaled_elo_diff)
            i_wins = np.random.binomial(n=self.n_games, p=prob_i_win)
            j_wins = self.n_games - i_wins
            D[i, j] = i_wins
            D[j, i] = j_wins
        return D

    def __str__(self):
        return "SynthELOTournamentSource({},{},{},{})".format(self.n, self.n_games, self.comp_var, self.elo_scale)

class UniformRandomSource(DataSource):
    # Every off-diagonal element of the matrix is drawn independently from a uniform
    # distribution over the range of integers [0, max_val].

    def __init__(self, n, max_val=1):
        # n --> the number of items
        # max_val --> the maximum value that can appear as an element in the matrix
        self.n = n
        self.max_val = max_val

    def init_D(self):
        D = np.random.randint(low=0, high=self.max_val+1, size=(self.n,self.n), dtype=int)
        for i in range(self.n):
            D[i,i] = 0
        return D

    def __str__(self):
        return "UniformRandomSource({},{})".format(self.n, self.max_val)

######## RANKING ALGORITHMS ########

class RankingAlgorithm:

    def rank(D):
        # Child classes should return numpy array of ranking vector
        raise NotImplemented("Don't use the generic RankingAlgorithm class")

class LOPRankingAlgorithm(RankingAlgorithm):

    def rank(self, D):
        k, details = solve(D)
        # This could return the full P set or randomly sample from it rather
        # than reporting the first it finds.
        return details["P"][0]

    def __str__(self):
        return "LOPRankingAlgorithm"

class ColleyRankingAlgorithm(RankingAlgorithm):
    def rank(self, D):
        wins = [sum(D[i]) for i in range(0,D.shape[0])]
        losses = [sum(np.transpose(D)[i]) for i in range(0,D.shape[0])]
        totalevents = [wins[i] + losses[i] for i in range(0,D.shape[0])]
        b = [1 + (wins[i] - losses[i])/2 for i in range(0,D.shape[0])]
        C = np.zeros(D.shape)
        for i in range(0,D.shape[0]):
            C[i][i] = 2 + totalevents[i]
        for x in range(D.shape[0]):
            for y in range(D.shape[1]):
                if x != y:
                    C[x][y] = (D[x][y] + D[y][x]) * -1
        r = np.linalg.solve(C, b)
        r = sorted([(r[i - 1], i) for i in range(1, D.shape[0] + 1)])
        retvec = [r[i][1]-1 for i in range(len(r)-1, -1, -1)]
        return retvec

    def __str__(self):
        return "ColleyRankingAlgorithm"

class MasseyRankingAlgorithm(RankingAlgorithm):

    def rank(self, D):
        wins = [sum(D[i]) for i in range(0,D.shape[0])]
        losses = [sum(np.transpose(D)[i]) for i in range(0,D.shape[0])]
        totalevents = [wins[i] + losses[i] for i in range(0,D.shape[0])]
        b = [1 + (wins[i] - losses[i])/2 for i in range(0,D.shape[0])]
        C = np.zeros(D.shape)
        for i in range(0,D.shape[0]):
            C[i][i] = totalevents[i]
        for x in range(D.shape[0]):
            for y in range(D.shape[1]):
                if x != y:
                    C[x][y] = (D[x][y] + D[y][x]) * -1
        C[D.shape[0] - 1] = np.ones(D.shape[0])
        b[D.shape[0] - 1] = 0
        r = np.linalg.solve(C, b)
        r = sorted([(r[i - 1], i) for i in range(1, D.shape[0] + 1)])
        retvec = [r[i][1]-1 for i in range(len(r)-1, -1, -1)]
        return retvec

    def __str__(self):
        return "MasseyRankingAlgorithm"

class MarkovChainRankingAlgorithm(RankingAlgorithm):
    def rank(self, D):
        #f = np.vectorize(lambda x: 1 if x > 0 else 0)
        '''for i in range(D.shape[0]):
            for j in range(i, D.shape[1]):
                if D[i][j] > D[j][i] and i != j:
                    D[i][j] = 1
                    D[j][i] = 0
                elif D[i][j] < D[j][i] and i != j:
                    D[j][i] = 1
                    D[i][j] = 0
        '''
        V = np.transpose(D.astype(float))
        n = D.shape[0]
        wins = [sum(D[i]) for i in range(0,D.shape[0])]
        losses = [sum(np.transpose(D)[i]) for i in range(0,D.shape[0])]
        totalevents = [wins[i] + losses[i] for i in range(0,D.shape[0])]
        #print("totalevents: ", totalevents)
        maxevents = max(totalevents)
        #print(V)
        for i in range(V.shape[0]):
            if sum(V[i]) != 0:
                V[i] = np.divide(V[i], sum(V[i]))
            else:
                V[i] = np.zeros(n)
                V[i][i] = 1
        print(V)
        eigenvals, eigenvecs = np.linalg.eig(V)
        #print("eigenvals:\n", eigenvals)
        #for i in range(len(eigenvecs)):
        #    print(eigenvecs[i])
        #print("eigenvecs:\n", eigenvecs)
        print("Max eigenvalue/vector:", max(eigenvals), eigenvecs[np.argmax(eigenvals)]/np.linalg.norm(eigenvecs[np.argmax(eigenvals)]))
        if True in np.iscomplex(eigenvecs[np.argmax(eigenvals)]):
            print("Complex rating vector")
            return
        return np.argsort(eigenvecs[np.argmax(eigenvals)])

    def __str__(self):
        return "MarkovChainRankingAlgorithm"

class MarkovModifiedRankingAlgorithm(RankingAlgorithm):
    def rank(self, D):
        V = D.astype(float)
        n = D.shape[0]
        wins = [sum(D[i]) for i in range(0,D.shape[0])]
        losses = [sum(np.transpose(D)[i]) for i in range(0,D.shape[0])]
        totalevents = [wins[i] + losses[i] for i in range(0,D.shape[0])]
        maxevents = max(totalevents)
        #print(V)
        for i in range(V.shape[0]):
            if sum(V[i]) != 0:
                V[i] = np.divide(V[i], sum(V[i]))
            else:
                V[i] = np.zeros(n)
                V[i][i] = 1
        #print(V)
        return np.argsort(V.sum(axis=0))

    def __str__(self):
        return "MarkovModifiedRankingAlgorithm"

####### SOME HELPFUL GLOBALS #######
# These are the defaults used by the testbed

ALL_RANKING_ALGS = [LOPRankingAlgorithm(), MasseyRankingAlgorithm(), ColleyRankingAlgorithm()]
LOW_INTENSITY_NOISE_GENS = [SwapNoise(0.05), BinaryFlipNoise(0.05)]
CORRELATION_METRICS = [kendall_tau, stats.spearmanr]

######## PROBLEM INSTANCE ########

class ProblemInstance:

    def __init__(self, dataSource):
        self.dataSource = dataSource
        self._D = None

    def get_D(self, refresh=False):
        if self._D is None or refresh:
            self._D = self.dataSource.init_D()
        return self._D

    def get_optimal_rankings(self,
                             model="lop",
                             num_random_restarts=0):
        if model in ["lop", "hillside"]:
             return solve(self.get_D(), method=model, num_random_restarts=num_random_restarts, lazy=True)
        else:
            raise ValueError("Unrecognized model name '{}' should be one of 'lop' or 'hillside'".format(model))

    def get_most_distant_optimal_rankings(self,
                                          model="lop"):
        if model in ["lop","hillside"]:
            return solve_pair_min_tau(self.get_D(), method=model, verbose=False)
        else:
            raise ValueError("Unrecognized model name '{}' should be one of 'lop' or 'hillside'".format(model))

    def collect_data(self,
                     ranking_algorithms=ALL_RANKING_ALGS,
                     noise_generators=LOW_INTENSITY_NOISE_GENS,
                     correlation_metrics=CORRELATION_METRICS,
                     model="lop",
                     num_random_restarts=200,
                     n_sensitivity_trials=50):
        D = self.get_D(refresh=True)
        total_weight = np.sum(D)
        k, details = self.get_optimal_rankings(model=model,
                                               num_random_restarts=num_random_restarts)
        P = details["P"]
        k_furthest, details_furthest = self.get_most_distant_optimal_rankings(model=model)
        if details_furthest is not None:
            P.append(details_furthest["perm_x"])
            P.append(details_furthest["perm_y"])
        data = get_P_stats(P)
        data["k"] = k
        if model == "lop":
            data["degree_of_linearity"] = k / total_weight
        else:
            k_lop, _ = self.get_optimal_rankings(model="lop", num_random_restarts=0)
            data["degree_of_linearity"] = k_lop / total_weight
        data["model"] = model
        data["D"] = json.dumps(D.tolist())
        data["Source"] = str(self.dataSource)
        data["n_items"] = D.shape[0]
        data["P"] = str(list(set(P)))
        data["P_repeats"] = str(P)

        for rankingAlg in ranking_algorithms:
            perfect_ranking = rankingAlg.rank(D)
            data["{}_DOL".format(str(rankingAlg))] = np.triu(D[:,perfect_ranking][perfect_ranking,:],1).sum() / total_weight
            for noiseGenerator in noise_generators:
                # Measure the similarity between original ranking and post-noise ranking
                # for many samples of noise.
                taus = []
                rhos = []
                for trial_index in range(n_sensitivity_trials):
                    D_noisy = noiseGenerator.apply_noise(D)
                    noisy_ranking = rankingAlg.rank(D_noisy)
                    tau = kendall_tau(perfect_ranking, noisy_ranking)
                    taus.append(tau)
                    rho = spearman_r(perfect_ranking, noisy_ranking)
                    rhos.append(rho)

                # Transforms taus into "sensitivity scores" such that higher values mean more sensitive to noise
                sensitivities = (1.0 - np.array(taus)) / 2.0
                mean_tau_name = "mean_tau_sensitivity({},{})".format(str(rankingAlg), str(noiseGenerator))
                data[mean_tau_name] = np.mean(sensitivities)
                std_tau_name = "std_tau_sensitivity({},{})".format(str(rankingAlg), str(noiseGenerator))
                data[std_tau_name] = np.std(sensitivities)

                sensitivitiesrho = (1.0 - np.array(rhos)) / 2.0
                mean_rho_name = "mean_rho_sensitivity({},{})".format(str(rankingAlg), str(noiseGenerator))
                data[mean_rho_name] = np.mean(sensitivitiesrho)
                std_rho_name = "std_rho_sensitivity({},{})".format(str(rankingAlg), str(noiseGenerator))
                data[std_rho_name] = np.std(sensitivitiesrho)

        return data

######## MAIN, FOR DEBUG ONLY ########

def main():
    mcra = MarkovChainRankingAlgorithm()
    mmra = MarkovModifiedRankingAlgorithm()

    testmatrix = PerfectBinarySource(10)
    perf = testmatrix.init_D()

    eloTournament = SynthELOTournamentSource(10, 5, 260, 800)
    eloMatrix = eloTournament.init_D()
    eloTournament5 = SynthELOTournamentSource(5, 5, 260, 800)
    eloMatrix5 = eloTournament5.init_D()
    eloTournament2 = SynthELOTournamentSource(3, 5, 260, 800)
    eloMatrix2 = eloTournament2.init_D()
    eloTournament4 = SynthELOTournamentSource(4, 5, 260, 800)
    eloMatrix4 = eloTournament4.init_D()

    fivegood = np.array([[0,1,1,1,0],
                        [0,0,1,1,1],
                        [0,1,0,1,1],
                        [0,0,0,0,1],
                        [3,3,3,3,0]])

    np.set_printoptions(formatter={'float': lambda x: str(x)})

    print(eloMatrix5)
    print(mmra.rank(eloMatrix5))
    print(eloMatrix)
    print(mmra.rank(eloMatrix))
    print(eloMatrix2)
    print(mmra.rank(eloMatrix2))
    print(eloMatrix4)
    print(mmra.rank(eloMatrix4))

#if __name__ == "__main__":
#    main()


# -*- coding:utf-8 -*-
import logging.config
import re

from bs4 import BeautifulSoup

from utils import spider_utils

logging.config.fileConfig('log.ini')
file_logger = logging.getLogger(name="fileLogger")

def get_dianping_url(url):
    """
    获取大众点评数据 url
    :return:
    """
    # url = "http://www.dianping.com/search/keyword/1/0_美容医院"

    # url = "http://www.dianping.com/shanghai/ch50/g183"
    # response = spider_utils.requests_dianping(url)
    # # print(response.text)
    #
    # soup = BeautifulSoup(response.text, features="lxml")
    #
    # # 存储获取到的url
    # url_set = set([])
    # for link in soup.find_all("a"):
    #     ll =  link.get("href")
    #
    #     # 获取满足条件的url
    #     if isinstance(ll, str):
    #         # print(ll)
    #         # print(ll.find("http://www.dianping.com/shop"))
    #         if ll.find("http://www.dianping.com/shop") == 0 and ll.find("#")==-1:
    #             # print(ll)
    #             if ll.find("review") == -1:
    #                 url_set.add(ll)
    #
    # return url_set

    # url = "http://www.dianping.com/shanghai/ch50/g183"

    response = spider_utils.requests_dianping2(url)
    # print(response.text)

    soup = BeautifulSoup(response.text, features="lxml")

    # 存储获取到的url
    url_set = set([])
    for link in soup.find_all("a"):
        ll =  link.get("href")

        # 获取满足条件的url
        if isinstance(ll, str):
            # print(ll)
            # print(ll.find("http://www.dianping.com/shop"))
            if ll.find("http://www.dianping.com/shop") == 0 and ll.find("#")==-1:
                # print(ll)
                if ll.find("review") == -1:
                    url_set.add(ll)

    return url_set

def parse_dianping_url():
    """
    解析大众点评 url
    :return:
    """
    # url_set = get_dazhong_url()

    url_set = {'http://www.dianping.com/shop/98440846', 'http://www.dianping.com/shop/68960534',
               'http://www.dianping.com/shop/20816759', 'http://www.dianping.com/shop/92726325',
               'http://www.dianping.com/shop/77220510', 'http://www.dianping.com/shop/72448927',
               'http://www.dianping.com/shop/58387941', 'http://www.dianping.com/shop/23510196',
               'http://www.dianping.com/shop/98632891', 'http://www.dianping.com/shop/23123189',
               'http://www.dianping.com/shop/90058481', 'http://www.dianping.com/shop/14894422',
               'http://www.dianping.com/shop/92197169', 'http://www.dianping.com/shop/22799131',
               'http://www.dianping.com/shop/8984185'}

    all_page_links = get_all_page_links()

    for url in all_page_links:
        every_page_links_set = get_dianping_url(url)

        for url_detail in every_page_links_set:

            # response = spider_utils.requests_dianping("http://www.dianping.com/shop/98440846")

            response = spider_utils.requests_dianping(url_detail)
            soup = BeautifulSoup(response.text, features="lxml")

            # print(response.text)
            # 医院名
            hospital_name, b = soup.find("h1").stripped_strings
            print(hospital_name)

            # 地址
            address = ""
            for i in soup.find("div", attrs={"class":"expand-info address"}).find_all("span"):
                address += (str(i.get_text()).strip())
            print(address)

            # 电话
            tel = str(soup.find("p", attrs={"class":"expand-info tel"}).find("span", "item").get_text()).strip()
            print(tel)

            # 星级
            stars_str = soup.find("div", attrs={"class":"brief-info"}).find_all("span")[0]

            print(stars_str)
            score = re.findall(r"\d+\.?\d*",str(stars_str).strip())[0]
            print(score)
            #
            # print(hospital_name, address, tel, score)

    # with open("/Users/caosai/Desktop/test.html") as f:
    #     html = f.read()
    #     soup = BeautifulSoup(html, features="lxml")
    #
    #     # print(soup.select("h1"))
    #
    #     # print(soup.find_all("h1", attrs={"class":"shop-name"}))
    #     #
    #     # print(soup.find_all("h1", "shop-name"))
    #
    #     # for ll in soup.find_all("h1", "shop-name"):
    #     #     print(type(ll))
    #     #     a, b = ll.stripped_strings
    #     #     print(a)
    #
    #     # 医院名
    #     hospital_name, b = soup.find("h1").stripped_strings
    #     print(hospital_name)
    #
    #     # 地址
    #     address = ""
    #     for i in soup.find("div", attrs={"class":"expand-info address"}).find_all("span"):
    #         address += (str(i.get_text()).strip())
    #     print(address)
    #
    #     # 电话
    #     tel = str(soup.find("p", attrs={"class":"expand-info tel"}).find("span", "item").get_text()).strip()
    #     print(tel)
    #
    #     print(hospital_name, address, tel)

def parse_dianping_url2(url2):
    """
    解析大众点评 url
    :return:
    """
    response = spider_utils.requests_dianping2(url2)

    # print(response.text)
    # if response is not None and response.text.find("地址不对") == -1:
    #     print('---161----')
    #     soup = BeautifulSoup(response.text, features="lxml")
    #     # print(response.text)
    #
    #     # 如果counter_none 值为4,则说明IP被封,需更换IP
    #     counter_none = 0
    #
    #     # 医院名
    #     hospital_name = ""
    #     try:
    #         hospital_name, b = soup.find("h1").stripped_strings
    #         # print(hospital_name)
    #     except Exception as e:
    #         counter_none += 1
    #         print(e)
    #
    #     # 地址
    #     address = ""
    #     try:
    #         for i in soup.find("div", attrs={"class":"expand-info address"}).find_all("span"):
    #             address += (str(i.get_text()).strip())
    #         # print(address)
    #     except Exception as e:
    #         counter_none += 1
    #         print(e)
    #
    #     # 电话
    #     tel = ""
    #     try:
    #         tel = str(soup.find("p", attrs={"class":"expand-info tel"}).find("span", "item").get_text()).strip()
    #         # print(tel)
    #     except Exception as e:
    #         counter_none += 1
    #         print(e)
    #
    #     # 星级
    #     score = ""
    #     try:
    #         stars_str = soup.find("div", attrs={"class":"brief-info"}).find_all("span")[0]
    #         score = re.findall(r"\d+\.?\d*",str(stars_str).strip())[0]
    #     except Exception as e:
    #         counter_none += 1
    #         print(e)
    #
    #     # 检测IP是否被封
    #     if counter_none == 4:
    #         print("IP-Cookie失效, 更换....")
    #         # selenium
    #         # cookies = selenium_utils.no_delay_cookies(url)
    #         # print(cookies)
    #
    #         spider_utils.change_ip_cookies(url2)
    #         # spider_utils.change_ip()
    #         print("重新执行该方法")
    #         return parse_dianping_url2(url2)
    #     else:
    #         print('---217----')
    #         str2 = url2 + "^" + hospital_name + "^" + address + "^" + tel + "^" + score
    #         print(hospital_name, address, tel, score)
    #
    #         return str2
    #         # with open("dianping_data_huangpu_spa.txt", "a+") as f:
    #         #     f.write(str2 + "\n")

    # print(response.text)
    # 不能这样使用地址不对,因为本中可能存在这种值
    # if response is not None and response.text.find("地址不对") == -1:
    if response is not None:
        soup = BeautifulSoup(response.text, features="lxml")
        # print(response.text)

        # 如果counter_none 值为4,则说明IP被封,需更换IP
        counter_none = 0

        # 医院名
        hospital_name = ""
        try:
            hospital_name, b = soup.find("h1").stripped_strings
            # print(hospital_name)
        except Exception as e:
            counter_none += 1
            print(e)

        # 地址
        address = ""
        try:
            for i in soup.find("div", attrs={"class":"expand-info address"}).find_all("span"):
                address += (str(i.get_text()).strip())
                # print(address)
        except Exception as e:
            counter_none += 1
            print(e)

        # 电话
        tel = ""
        try:
            tel = str(soup.find("p", attrs={"class":"expand-info tel"}).find("span", "item").get_text()).strip()
            # print(tel)
        except Exception as e:
            counter_none += 1
            print(e)

        # 星级
        score = ""
        try:
            stars_str = soup.find("div", attrs={"class":"brief-info"}).find_all("span")[0]
            score = re.findall(r"\d+\.?\d*",str(stars_str).strip())[0]
        except Exception as e:
            counter_none += 1
            print(e)

        # 检测IP是否被封
        if counter_none == 4:
            print("IP-Cookie失效, 更换....")
            # selenium
            # cookies = selenium_utils.no_delay_cookies(url)
            # print(cookies)

            spider_utils.change_ip_cookies(url2)
            # spider_utils.change_ip()
            print("重新执行该方法")
            return parse_dianping_url2(url2)
        else:
            print('---217----')
            str2 = url2 + "^" + hospital_name + "^" + address + "^" + tel + "^" + score
            print(hospital_name, address, tel, score)

            return str2
            # with open("dianping_data_huangpu_spa.txt", "a+") as f:
            #     f.write(str2 + "\n")
    else:
        print('#'*20)
        return parse_dianping_url2(url2)

def get_all_page_links():

    url_start = "http://www.dianping.com/hangzhou/ch50/g158r8845"
    url_list = list()
    url_list.append(url_start)

    for i in range(50):
        if i > 0:
            # url_former = "http://www.dianping.com/beijing/ch50/g158r5950p" + str(i+1) + "?cpt=5220127%2C19813864"
            url_former = url_start + "p" + str(i+1)
            url_list.append(url_former)

    return url_list

def get_all_detail_page_links():
    ########### 1、将url写入文件 ###############
    province = '浙江省'
    city = '杭州市'
    region = '余杭区'

    str1 = ""
    for i in get_all_page_links():
        print(i)
        url_sets = get_dianping_url(i)

        for res in url_sets:
            print(res)
            str1 = str1 + province + '^' + city + '^' + region + '^' + res + "\n"

    with open("url_all.txt", "a+") as f:
        f.write(str1)
    ########### 1、将url写入文件 ###############

if __name__ == '__main__':
    # get_all_detail_page_links()
    # print(get_dianping_url("http://www.dianping.com/shanghai/ch50/g183"))

    # 获取所有页面主链接
    # print(get_all_page_links())

    get_all_detail_page_links()

    # with open("url_all.txt") as f:
    #     file_path = "dianping_hangzhou_data_spa.txt"
    #
    #     counter = 1
    #     flag = False
    #     last_one_url = ""
    #     if os.path.exists(file_path):
    #         with open(file_path) as f2:
    #             last_one_url = f2.readlines()[-1].split('^')[3]
    #     else:
    #         flag = True
    #
    #     for line in f.readlines():
    #         province = line.strip().split('^')[0]
    #         city = line.strip().split('^')[1]
    #         region = line.strip().split('^')[2]
    #         url = line.strip().split('^')[3]
    #
    #         if last_one_url == url:
    #             flag = True
    #             continue
    #
    #         if flag is True:
    #             print("开始解析第%d行, %s"%(counter, url))
    #             str2 = province + '^' + city + '^' + region + '^' + parse_dianping_url2(url)
    #
    #             with open(file_path, "a+") as f2:
    #                 f2.write(str2 + "\n")
    #             # print(url)
    #             # if counter%150 == 0:
    #             #     print("切换")
    #             #     spider_utils.change_ip()
    #             print("结束解析----------" + '\n')
    #
    #             counter +=1


# Title: Face detection using Python and OpenCV.
# Author: @CodeProgrammer "On Telegram" || @PythonSy "On Instagram".
"""
you need to install OpenCV by using this command in the terminal:
pip install opencv-python
for more codes you can visit our channel on Telegram: @CodeProgrammer
"""
import cv2

"""
load the cascade file from:
https://github.com/opencv/opencv/blob/master/data/haarcascades/haarcascade_frontalface_default.xml
"""
face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

img = cv2.imread('3.jpg')
img = cv2.resize(img, (500, 500))
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# Detect faces.
faces = face_cascade.detectMultiScale(gray, 1.1, 4)

# Draw rectangle around the faces.
for (x, y, w, h) in faces:
    cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)

# Show the result.
cv2.imshow('by: @CodeProgrammer', img)
cv2.waitKey()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('academy', '0005_aluno_pessoa'),
    ]

    operations = [
        migrations.CreateModel(
            name='Professor',
            fields=[
                ('pessoa_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='academy.Pessoa')),
                ('MatriculaProfessor', models.IntegerField(null=True, verbose_name=b'Numero de matricula do professor')),
                ('CursoProfessor', models.CharField(max_length=50, null=True, verbose_name=b'Nome do curso lecionado')),
            ],
            options={
            },
            bases=('academy.pessoa',),
        ),
    ]


from flask import Flask
from twilio.twiml.voice_response import VoiceResponse

app = Flask(__name__)

@app.route("/answer", methods=['GET', 'POST'])
def answer_call():
    """Respond to incoming phone calls with a brief message."""
    # Start our TwiML response
    resp = VoiceResponse()

    # Read a message aloud to the caller
    resp.say("Thank you for getting back! please reach me on my email or phone number to further discuss my application!", voice='alice')

    return str(resp)

if __name__ == "__main__":
    app.run(debug=True)


import unittest
from itertools import chain

import torch

from pytorch_metric_learning.distances import CosineSimilarity
from pytorch_metric_learning.losses import CentroidTripletLoss
from pytorch_metric_learning.reducers import MeanReducer

from .. import TEST_DEVICE, TEST_DTYPES
from ..zzz_testing_utils.testing_utils import angle_to_coord

def normalize(embeddings):
    embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=0)
    return embeddings

class TestCentroidTripletLoss(unittest.TestCase):
    def test_indices_tuple_failure(self):
        loss_fn = CentroidTripletLoss()
        for labels_arr in [[0, 0, 0, 0, 0, 0, 0, 0, 1], [0, 0, 1, 1, 1, 2, 2, 3]]:
            embeddings = torch.randn(len(labels_arr), 32, device=TEST_DEVICE)
            labels = torch.tensor(labels_arr, device=TEST_DEVICE)
            with self.assertRaises(ValueError) as context:
                loss_fn(embeddings, labels)

    def test_centroid_triplet_loss(self):
        for dtype in TEST_DTYPES:
            embedding_angles = [0, 10, 20, 30, 40, 50]
            centroid_makers = [
                [[10, 20], [30, 40, 50]],
                [[0, 20], [30, 40, 50]],
                [[0, 10], [30, 40, 50]],
                [[40, 50], [0, 10, 20]],
                [[30, 50], [0, 10, 20]],
                [[30, 40], [0, 10, 20]],
            ]
            triplets = [
                (0, (0, 0), (0, 1)),
                (1, (1, 0), (1, 1)),
                (2, (2, 0), (2, 1)),
                (3, (3, 0), (3, 1)),
                (4, (4, 0), (4, 1)),
                (5, (5, 0), (5, 1)),
            ]

            labels = torch.LongTensor([0, 0, 0, 1, 1, 1])

            self.helper(embedding_angles, centroid_makers, labels, triplets, dtype)

    def test_sorting_invariance(self):
        for dtype in TEST_DTYPES:
            centroid_makers = [
                [[10, 20], [30, 40, 50]],
                [[40, 50], [0, 10, 20]],
                [[30, 50], [0, 10, 20]],
                [[0, 20], [30, 40, 50]],
                [[0, 10], [30, 40, 50]],
                [[30, 40], [0, 10, 20]],
            ]

            embedding_angles = [0, 30, 40, 10, 20, 50]
            labels = torch.LongTensor([0, 1, 1, 0, 0, 1])

            triplets = [
                (0, (0, 0), (0, 1)),
                (1, (1, 0), (1, 1)),
                (2, (2, 0), (2, 1)),
                (3, (3, 0), (3, 1)),
                (4, (4, 0), (4, 1)),
                (5, (5, 0), (5, 1)),
            ]

            self.helper(embedding_angles, centroid_makers, labels, triplets, dtype)

    def test_imbalanced(self):
        for dtype in TEST_DTYPES:
            per_class_angles = [
                [0, 10, 20],  # class A
                [30, 40, 50, 55],  # class B
                [60, 70, 80, 90],  # class C
            ]
            embedding_angles = chain(*per_class_angles)
            labels = torch.LongTensor([0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2])

            centroid_makers = [
                [[10, 20], [30, 40, 50, 55]],
                [[10, 20], [60, 70, 80, 90]],
                [[0, 20], [30, 40, 50, 55]],
                [[0, 20], [60, 70, 80, 90]],
                [[0, 10], [30, 40, 50, 55]],
                [[0, 10], [60, 70, 80, 90]],
                [[40, 50, 55], [0, 10, 20]],
                [[40, 50, 55], [60, 70, 80, 90]],
                [[30, 50, 55], [0, 10, 20]],
                [[30, 50, 55], [60, 70, 80, 90]],
                [[30, 40, 55], [0, 10, 20]],
                [[30, 40, 55], [60, 70, 80, 90]],
                [[30, 40, 50], [0, 10, 20]],
                [[30, 40, 50], [60, 70, 80, 90]],
                [[70, 80, 90], [0, 10, 20]],
                [[70, 80, 90], [30, 40, 50, 55]],
                [[60, 80, 90], [0, 10, 20]],
                [[60, 80, 90], [30, 40, 50, 55]],
                [[60, 70, 90], [0, 10, 20]],
                [[60, 70, 90], [30, 40, 50, 55]],
                [[60, 70, 80], [0, 10, 20]],
                [[60, 70, 80], [30, 40, 50, 55]],
            ]

            triplets = [
                (0, (0, 0), (0, 1)),
                (0, (1, 0), (1, 1)),
                (1, (2, 0), (2, 1)),
                (1, (3, 0), (3, 1)),
                (2, (4, 0), (4, 1)),
                (2, (5, 0), (5, 1)),
                (3, (6, 0), (6, 1)),
                (3, (7, 0), (7, 1)),
                (4, (8, 0), (8, 1)),
                (4, (9, 0), (9, 1)),
                (5, (10, 0), (10, 1)),
                (5, (11, 0), (11, 1)),
                (6, (12, 0), (12, 1)),
                (6, (13, 0), (13, 1)),
                (7, (14, 0), (14, 1)),
                (7, (15, 0), (15, 1)),
                (8, (16, 0), (16, 1)),
                (8, (17, 0), (17, 1)),
                (9, (18, 0), (18, 1)),
                (9, (19, 0), (19, 1)),
                (10, (20, 0), (20, 1)),
                (10, (21, 0), (21, 1)),
            ]

            self.helper(embedding_angles, centroid_makers, labels, triplets, dtype)

    def helper(
        self,
        embedding_angles,
        centroid_makers,
        labels,
        triplets,
        dtype,
        ref_emb=None,
        ref_labels=None,
    ):
        embeddings = torch.tensor(
            [angle_to_coord(a) for a in embedding_angles],
            requires_grad=True,
            dtype=dtype,
        ).to(
            TEST_DEVICE
        )  # 2D embeddings

        centroids = [
            [
                torch.stack(
                    [
                        torch.tensor(
                            angle_to_coord(a), requires_grad=True, dtype=dtype
                        ).to(TEST_DEVICE)
                        for a in coords
                    ]
                ).mean(-2)
                for coords in one_maker
            ]
            for one_maker in centroid_makers
        ]

        margin = 0.2
        loss_funcA = CentroidTripletLoss(margin=margin)
        loss_funcB = CentroidTripletLoss(margin=margin, reducer=MeanReducer())
        loss_funcC = CentroidTripletLoss(margin=margin, distance=CosineSimilarity())
        loss_funcD = CentroidTripletLoss(
            margin=margin, reducer=MeanReducer(), distance=CosineSimilarity()
        )
        loss_funcE = CentroidTripletLoss(margin=margin, smooth_loss=True)

        [lossA, lossB, lossC, lossD, lossE] = [
            x(embeddings, labels, ref_emb=ref_emb, ref_labels=ref_labels)
            for x in [loss_funcA, loss_funcB, loss_funcC, loss_funcD, loss_funcE]
        ]

        correct_loss = 0
        correct_loss_cosine = 0
        correct_smooth_loss = 0
        num_non_zero_triplets = 0
        num_non_zero_triplets_cosine = 0

        for a, pc, nc in triplets:
            anchor = embeddings[a]

            positive = centroids[pc[0]][pc[1]]
            negative = centroids[nc[0]][nc[1]]

            anchor = normalize(anchor)
            positive = normalize(positive)
            negative = normalize(negative)

            ap_dist = torch.sqrt(torch.sum((anchor - positive) ** 2))
            an_dist = torch.sqrt(torch.sum((anchor - negative) ** 2))
            curr_loss = torch.relu(ap_dist - an_dist + margin)

            curr_loss_cosine = torch.relu(
                torch.sum(anchor * negative) - torch.sum(anchor * positive) + margin
            )
            correct_smooth_loss += torch.nn.functional.softplus(
                ap_dist - an_dist + margin
            )
            if curr_loss > 0:
                num_non_zero_triplets += 1
            if curr_loss_cosine > 0:
                num_non_zero_triplets_cosine += 1
            correct_loss += curr_loss
            correct_loss_cosine += curr_loss_cosine

        rtol = 1e-2 if dtype == torch.float16 else 1e-5
        self.assertTrue(
            torch.isclose(lossA, correct_loss / num_non_zero_triplets, rtol=rtol)
        )
        self.assertTrue(torch.isclose(lossB, correct_loss / len(triplets), rtol=rtol))
        self.assertTrue(
            torch.isclose(
                lossC, correct_loss_cosine / num_non_zero_triplets_cosine, rtol=rtol
            )
        )
        self.assertTrue(
            torch.isclose(lossD, correct_loss_cosine / len(triplets), rtol=rtol)
        )
        self.assertTrue(
            torch.isclose(lossE, correct_smooth_loss / len(triplets), rtol=rtol)
        )

    def test_backward(self):
        margin = 0.2
        loss_funcA = CentroidTripletLoss(margin=margin)
        loss_funcB = CentroidTripletLoss(margin=margin, reducer=MeanReducer())
        loss_funcC = CentroidTripletLoss(smooth_loss=True)
        for dtype in TEST_DTYPES:
            for loss_func in [loss_funcA, loss_funcB, loss_funcC]:
                embedding_angles = [0, 20, 40, 60, 80, 85]
                embeddings = torch.tensor(
                    [angle_to_coord(a) for a in embedding_angles],
                    requires_grad=True,
                    dtype=dtype,
                ).to(
                    TEST_DEVICE
                )  # 2D embeddings
                labels = torch.LongTensor([0, 0, 1, 1, 2, 2])

                loss = loss_func(embeddings, labels)
                loss.backward()


############### Blackjack Project #####################

############### Blackjack House Rules #####################

## The deck is unlimited in size.
## There are no jokers.
## The Jack/Queen/King all count as 10.
## The the Ace can count as 11 or 1.
## Use the following list as the deck of cards:

## The cards in the list have equal probability of being drawn.
## Cards are not removed from the deck as they are drawn.
## The computer is the dealer.

import random
from art import logo
#Exclude replit function as it will not work without replit
from replit import clear

#Deals a random card from the list
def deal_card():
  """This returns a random card from the cards list"""
  cards = [11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10]
  chosen_card = random.choice(cards)
  return chosen_card

#Compares the cards to figure out the result of the game
def compare_score(user_score, computer_score):
  if user_score > 21 and computer_score > 21:
    return "You passed over 21, you lose!"
  if user_score == computer_score:
    return "The game is a tie!"
  elif computer_score == 0:
    return "You lost. Computer has blackjack!"
  elif user_score == 0:
    return "You won with a blackjack!"
  elif user_score > 21:
    return "You passed over 21, you lose!"
  elif computer_score > 21:
    return "Computer went over 21, you win!"
  elif user_score > computer_score:
    return "You win!"
  else:
    return "You lost!"

def calculate_score(cards):
  """Outputs total score by the cards in hand"""
  #Blackjack during first deal result
  if sum(cards) == 21 and len(cards) == 2:
    return 0
  #If score goes over 21 then 11 becomes 1 (value of A)
  if 11 in cards and sum(cards) > 21:
    cards.remove(11)
    cards.append(1)
  return sum(cards)

#Starting the blackjack game
def blackjack_game():
  """Starts the entire blackjack game"""
  print(logo)
  user_cards = []
  computer_cards = []
  is_game_over = False

  for i in range(2):
    user_cards.append(deal_card())
    computer_cards.append(deal_card())

  while not is_game_over:
    #Call calculate_score(). If the computer or the user has a blackjack (0) or if the user's score is over 21, then the game ends.
    user_score = calculate_score(user_cards)
    computer_score = calculate_score(computer_cards)
    print(f"Your cards: {user_cards}, score is: {user_score}")
    print(f"Computer's first card: {computer_cards[0]}")

    if user_score > 21 or user_score == 0 or computer_score == 0:
      is_game_over = True
    else:
      user_deal = input("Type 'y' to get another card, or 'n' to pass: ")
      if user_deal == 'y':
        user_cards.append(deal_card())
      else:
        is_game_over = True

  #Once the user is done, the computer keeps drawing cards as long as it has a score less than 17.
  while computer_score != 0 and computer_score < 17:
    computer_cards.append(deal_card())
    computer_score = calculate_score(computer_cards)

  #Showing the cards at hand of player and one of computer's
  print(f"You have in your hand: {user_cards}, you scored: {user_score}")
  print(f"The computer has: {computer_cards}, the computer's score is: {computer_score}")
  print(compare_score(user_score, computer_score))

#Ask the user if they want to restart the game
while input("Want to play blackjack? Type 'y' for yes or 'n' for no. "):
  #Exclude clear function as it will not work without replit
  clear()
  blackjack_game()


from docx import Document
from docx.shared import Pt
from openpyxl import load_workbook
import os

nome_arquivo = "Alunos.xlsx"
planilhaAlunos = load_workbook(nome_arquivo)
sheet = planilhaAlunos['Nomes']

for L in range(2, len(sheet['A']) + 1):
    arquivo = Document('Certificado1.docx')

    estilo = arquivo.styles['Normal']

    nomeAluno = sheet['A%s' % L].value

    for p in arquivo.paragraphs:
        if "@nome" in p.text:
            p.text = nomeAluno
            fonte = estilo.font
            fonte.name = 'Calibri (Corpo)'
            fonte.size = Pt(24)

    certificados = f'C:\\Users\\Win\\Documents\\GitHub\\Python\\Word\\Certificados\\{nomeAluno}.docx'
    arquivo.save(certificados)


import pygame
import math

car_img = pygame.image.load('pic/car_img.png')

class Window:
    def __init__(self, win_w, win_h):

        self.win_w = win_w
        self.win_h = win_h

        self.win = pygame.display.set_mode((win_w, win_h))

        self.clock = pygame.time.Clock()
        self.speed = 30

    def redraw_win(self, cars, track, agent=False):

        self.clock.tick(self.speed)

        pygame.draw.rect(self.win, (255, 255, 255), (0, 0, self.win_w, self.win_h))

        for car in cars:
            self.draw_car(car_img, car)
        self.draw_track(track)
        self.draw_track_check(track)
        if agent:
            for car in cars:
                self.draw_radar(car, agent, track)

        pygame.display.update()

    def draw_car(self, image, car):

        w, h = image.get_size()

        blitRotate(self.win, image, (car.x + w/2, car.y + h/2), (w/2, h/2), car.angle)

    def draw_track(self, track):
        for p, point in enumerate(track.barrier_p1):
            if p > 0:
                pygame.draw.line(self.win, (0, 0, 0), (track.barrier_p1[p-1].get_x(), track.barrier_p1[p-1].get_y()), (point.get_x(), point.get_y()), 5)

        for p, point in enumerate(track.barrier_p2):
            if p > 0:
                pygame.draw.line(self.win, (0, 0, 0), (track.barrier_p2[p-1].get_x(), track.barrier_p2[p-1].get_y()), (point.get_x(), point.get_y()), 5)

    def draw_track_check(self, track):

        for p, point in enumerate(track.checkpoints):
            pygame.draw.line(self.win, point.get_col(), (point.get_x1(), point.get_y1()), (point.get_x2(), point.get_y2()))

    def draw_radar(self, car, agent, track):

        for i in range(5):

            x = car.x + math.cos(math.radians(car.angle - 90 + i * 45)) * agent.get_state(car, track)[i]
            y = car.y - math.sin(math.radians(car.angle - 90 + i * 45)) * agent.get_state(car, track)[i]

            pygame.draw.line(self.win, (i * 50, 0, 0), (car.x + car.width/2, car.y + car.height/2), (x, y))

def blitRotate(surf, image, pos, originPos, angle):
    # calcaulate the axis aligned bounding box of the rotated image
    w, h = image.get_size()
    box = [pygame.math.Vector2(p) for p in [(0, 0), (w, 0), (w, -h), (0, -h)]]
    box_rotate = [p.rotate(angle) for p in box]
    min_box = (min(box_rotate, key=lambda p: p[0])[0], min(box_rotate, key=lambda p: p[1])[1])
    max_box = (max(box_rotate, key=lambda p: p[0])[0], max(box_rotate, key=lambda p: p[1])[1])

    # calculate the translation of the pivot
    pivot = pygame.math.Vector2(originPos[0], -originPos[1])
    pivot_rotate = pivot.rotate(angle)
    pivot_move = pivot_rotate - pivot

    # calculate the upper left origin of the rotated image
    origin = (pos[0] - originPos[0] + min_box[0] - pivot_move[0], pos[1] - originPos[1] - max_box[1] + pivot_move[1])

    # get a rotated image
    rotated_image = pygame.transform.rotate(image, angle)

    # rotate and blit the image
    surf.blit(rotated_image, origin)

    # draw rectangle around the image
    pygame.draw.rect(surf, (255, 0, 0), (*origin, *rotated_image.get_size()), 2)


import argparse
import time, datetime
import os
import os.path as osp
import logging
from baselines import logger, bench
from baselines.common.misc_util import (
    set_global_seeds,
    boolean_flag,
)
import baselines.torcs_ddpg.training as training
from baselines.torcs_ddpg.models import Actor, Critic
from baselines.torcs_ddpg.memory import Memory
from baselines.torcs_ddpg.noise import *

import gym
import tensorflow as tf
from mpi4py import MPI

# from baselines.gym_torcs import TorcsEnv
import gym_torcs

def run(env_id, seed, noise_type, layer_norm, evaluation, **kwargs):
    # Configure things.
    rank = MPI.COMM_WORLD.Get_rank()

    if rank != 0:
        logger.set_level(logger.DISABLED)

    # GymTorcs parameters
    vision = False
    throttle = True
    gear_change = False
    rendering = False
    lap_limiter = 4

    # 10 Fixed Sparsed Config 2m not too much bots in corners
    race_config_path = os.path.dirname(os.path.abspath(__file__)) + \
        "/raceconfig/agent_10fixed_sparsed_4.xml"

    # The default pre process for dosssman Hybrid IL RL Experiment
    def obs_preprocess_fn(dict_obs):
        return np.hstack((dict_obs['angle'],
            dict_obs['track'],
            dict_obs['trackPos'],
            dict_obs['speedX'],
            dict_obs['speedY'],
            dict_obs['speedZ'],
            dict_obs['wheelSpinVel'],
            dict_obs['rpm'],
            dict_obs['opponents']))

    # Required to change properly build the observation space
    custom_obs_vars = [
        'angle', 'track', 'trackPos', 'speedX', 'speedY', 'speedZ',
        'wheelSpinVel',
        'rpm',
        'opponents'
    ]

    # Creating the environment
    env = gym.make( 'Torcs-v0', vision=vision, throttle=throttle, gear_change=gear_change,
        lap_limiter = lap_limiter, obs_preprocess_fn=obs_preprocess_fn, obs_vars=custom_obs_vars,
        hard_reset_interval=20, rank=rank)

    if evaluation and rank==0:
        eval_env = env
        eval_env = bench.Monitor(eval_env, os.path.join(logger.get_dir(), 'gym_eval'))
        env = bench.Monitor(env, None)
    else:
        eval_env = None

    # Parse noise_type
    action_noise = None
    param_noise = None
    nb_actions = env.action_space.shape[-1]

    for current_noise_type in noise_type.split(','):
        current_noise_type = current_noise_type.strip()
        if current_noise_type == 'none':
            pass
        elif 'adaptive-param' in current_noise_type:
            _, stddev = current_noise_type.split('_')
            param_noise = AdaptiveParamNoiseSpec(initial_stddev=float(stddev), desired_action_stddev=float(stddev))
        elif 'normal' in current_noise_type:
            _, stddev = current_noise_type.split('_')
            action_noise = NormalActionNoise(mu=np.zeros(nb_actions), sigma=float(stddev) * np.ones(nb_actions))
        elif 'ou' in current_noise_type:
            _, stddev = current_noise_type.split('_')
            action_noise = OrnsteinUhlenbeckActionNoise(mu=np.zeros(nb_actions), sigma=float(stddev) * np.ones(nb_actions))
        else:
            raise RuntimeError('unknown noise type "{}"'.format(current_noise_type))

    # Configure components.
    memory = Memory(limit=int(1e6), action_shape=env.action_space.shape, observation_shape=env.observation_space.shape)
    critic = Critic(layer_norm=layer_norm, hidden_sizes=[200,100])
    actor = Actor(nb_actions, layer_norm=layer_norm, hidden_sizes=[200,100])

    # Seed everything to make things reproducible.
    seed = seed + 1000000 * rank
    logger.info('rank {}: seed={}, logdir={}'.format(rank, seed, logger.get_dir()))
    tf.reset_default_graph()
    set_global_seeds(seed)
    env.seed(seed)
    if eval_env is not None:
        eval_env.seed(seed)

    # Disable logging for rank != 0 to avoid noise.
    if rank == 0:
        start_time = time.time()
    training.train(env=env, eval_env=eval_env, param_noise=param_noise,
        action_noise=action_noise, actor=actor, critic=critic, memory=memory, **kwargs)

    env.close()

    if eval_env is not None:
        eval_env.close()
    if rank == 0:
        logger.info('total runtime: {}s'.format(time.time() - start_time))

def parse_args():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--env-id', type=str, default='Pendulum-v0')
    boolean_flag(parser, 'render-eval', default=False)
    boolean_flag(parser, 'layer-norm', default=True)
    boolean_flag(parser, 'render', default=False)
    boolean_flag(parser, 'normalize-returns', default=False)
    boolean_flag(parser, 'normalize-observations', default=False)
    parser.add_argument('--seed', help='RNG seed', type=int, default=0)
    parser.add_argument('--critic-l2-reg', type=float, default=1e-2)
    parser.add_argument('--batch-size', type=int, default=64)  # per MPI worker
    parser.add_argument('--actor-lr', type=float, default=1e-4)
    parser.add_argument('--critic-lr', type=float, default=1e-3)
    boolean_flag(parser, 'popart', default=False)
    parser.add_argument('--gamma', type=float, default=0.99)
    parser.add_argument('--reward-scale', type=float, default=1.)
    parser.add_argument('--clip-norm', type=float, default=None)
    parser.add_argument('--nb-epochs', type=int, default=1500)  # with default settings, perform 1M steps total
    parser.add_argument('--nb-epoch-cycles', type=int, default=20)
    parser.add_argument('--nb-train-steps', type=int, default=50)  # per epoch cycle and MPI worker
    parser.add_argument('--nb-eval-steps', type=int, default=720)  # per epoch cycle and MPI worker
    parser.add_argument('--nb-rollout-steps', type=int, default=100)  # per epoch cycle and MPI worker
    parser.add_argument('--noise-type', type=str, default='adaptive-param_0.2')  # choices are adaptive-param_xx, ou_xx, normal_xx, none
    parser.add_argument('--num-timesteps', type=int, default=None)
    parser.add_argument('--pretrained_model', type=str, default=None)
    boolean_flag(parser, 'evaluation', default=True)
    args = parser.parse_args()
    # we don't directly specify timesteps for this script, so make sure that if we do specify them
    # they agree with the other parameters
    if args.num_timesteps is not None:
        assert(args.num_timesteps == args.nb_epochs * args.nb_epoch_cycles * args.nb_rollout_steps)
    dict_args = vars(args)
    del dict_args['num_timesteps']

    # Fix for Torcs snakeoil parameter overide gatekeeping
    import sys
    sys.argv = [sys.argv[0]]

    return dict_args

if __name__ == '__main__':
    args = parse_args()
    if MPI.COMM_WORLD.Get_rank() == 0:
        #Custom logging method to separate DDPG and GAIL for instance
        dir = os.path.dirname(os.path.abspath(__file__)) +  "/result"

        dir = os.path.join( dir,
            datetime.datetime.now().strftime("torcs-ddpg-%Y-%m-%d-%H-%M-%S-%f"))

        assert isinstance(dir, str)
        os.makedirs(dir, exist_ok=True)
        # args.log_dir = dir

        logger.configure( dir=dir, format_strs="stdout,log,csv,tensorboard")
        print( "# DEBUG: Logging to %s" % logger.get_dir())

    # Run actual script.
    run(**args)


import sys
import logging

sys.path.append('../../../../')

from flask import current_app, request
from flask_restful import Resource, reqparse
from spiders.selenium_spider import bloomberg

parser = reqparse.RequestParser()
parser.add_argument('spider')
parser.add_argument('action')

def test():
    print('testing')

def run_bloomberg():
    spider = bloomberg.Bloomberg()
    spider.run()

spider_list = ['run_bloomberg', ]
action_list = ['start', 'pause', 'resume']

class CustomScheduler(Resource):
    """
    http://127.0.0.1:5000/api/scheduler?spider=run_bloomberg&action=start
    http://127.0.0.1:5000/api/scheduler?spider=run_bloomberg&action=pause
    http://127.0.0.1:5000/api/scheduler?spider=run_bloomberg&action=resume
    """

    def get(self):
        args = parser.parse_args()
        spider = args['spider']
        action = args['action']

        if not spider or not action:
            error_msg = {'error': '没有指定爬虫或者操作'}
            print(error_msg)
            return error_msg

        if spider not in spider_list or action not in action_list:
            error_msg = {'error': '错误的爬虫或者操作'}
            print(error_msg)
            return error_msg

        print(spider, action)
        if action == 'start':
            try:
                result = current_app.apscheduler.add_job(func=eval(spider), id=spider, args='',
                                                         trigger='interval',
                                                         minutes=1)
                logging.info(result)
            except Exception as err:
                logging.error(err)

        elif action == 'pause':
            try:
                result = current_app.apscheduler.pause_job(spider)
                logging.info(result)
            except Exception as err:
                logging.error(err)

        elif action == 'resume':
            try:
                result = current_app.apscheduler.resume_job(spider)
                logging.info(result)
            except Exception as err:
                logging.error(err)

        return {'status': 200}


from socket import socket, AF_INET, SOCK_DGRAM
import CommTypes_pb2 as pb
import time
import datetime
import numpy as np

def recvSSLMessage(udp_sock):
        msg = pb.protoMotorsDataSSL()
        # multiple messages are received and accumulated on buffer during vision processing
        # so read until buffer socket are no longer available
        has_msg = False
        while True:
            try:
                data, _ = udp_sock.recvfrom(1024)
                msg.ParseFromString(data)
                has_msg = True
            except:
                break
        current_speeds = [msg.current_m1, msg.current_m2, msg.current_m3, msg.current_m4]
        pwms = [msg.pwm_m1, msg.pwm_m2, msg.pwm_m3, msg.pwm_m4]
        desired_speeds = [msg.desired_m1, msg.desired_m2, msg.desired_m3, msg.desired_m4]
        timestamp = msg.msgTime

        return has_msg, current_speeds, pwms, desired_speeds, timestamp

def serializeMsgToLog(current_speeds, pwms, desired_speeds, timestamp):
    print(f"current: {current_speeds[0]}, desired: {desired_speeds[0]}, pwm: {pwms[0]} | timestamp: {timestamp:.3f}")
    return current_speeds[0], current_speeds[1], current_speeds[2], current_speeds[3], \
        pwms[0], pwms[1], pwms[2], pwms[3], \
        desired_speeds[0], desired_speeds[1], desired_speeds[2], desired_speeds[3], \
        timestamp

if __name__ ==  "__main__":
    server = "199.0.1.1"
    pc2robot_port = 9600
    robot2pc_port = 9601

    msg = pb.protoMotorsPWMSSL()
    conn = socket(AF_INET, SOCK_DGRAM)
    conn.bind(('', robot2pc_port))
    conn.settimeout(0)

    # CHANGE LIST TO PSEUDO-RANDOM VALUES GENERATED BY VICTOR
    sleep_between_iterations_ms = 0.25
    maximum_time_without_messages_s = 2

    log = []

    # START MSG TIME COUNTER
    start = time.time()
    msgTimeout = time.time()
    received_first_msg = False
    time_offset = 0
    last_timestamp = 0

    while True:
        # SLEEP
        time.sleep(sleep_between_iterations_ms/1000)

        # RECV MESSAGE
        has_msg, current_speeds, pwms, desired_speeds, timestamp_data = recvSSLMessage(conn)
        if has_msg:
            if timestamp_data - last_timestamp < 0:
                time_offset += last_timestamp
            last_timestamp = timestamp_data
            log_msg = serializeMsgToLog(current_speeds, pwms, desired_speeds, timestamp_data+time_offset)
            log.append(log_msg)
            received_first_msg = True
            msgTimeout = time.time()
        elif ((time.time() - msgTimeout) > maximum_time_without_messages_s) and received_first_msg:
            break

        # COUNT TIME
        elapsed_time = time.time() - start
        if (elapsed_time > 120) and received_first_msg:
            break

        #print(f'Elapsed msgTimeout: {time.time() - msgTimeout:.3f} | First Msg: {received_first_msg}')

    if len(log)>0:
        # Get the current date and time
        current_datetime = datetime.datetime.now()

        # Format the date and time as a string
        formatted_datetime = current_datetime.strftime("%Y-%m-%d_%H-%M-%S")

        print(f"Saving motors log: {len(log)}")
        names = 'CURRENT_M1, CURRENT_M2, CURRENT_M3, CURRENT_M4, PWM_M1, PWM_M2, PWM_M3, PWM_M4, DESIRED_M1, DESIRED_M2, DESIRED_M3, DESIRED_M4, TIMESTAMP'
        np.savetxt(f'./results/motor_log_{formatted_datetime}.csv', \
                   log,
                   delimiter=',',
                   header=names)
    else:
        print("Empty log!")

    print("Finish")


import yaml
from aws_cdk import Aws, CfnOutput, Duration, Size, Stack
from aws_cdk import aws_events as events
from aws_cdk import aws_events_targets as targets
from aws_cdk import aws_iam as iam
from aws_cdk import aws_s3 as s3
from aws_cdk import aws_sns as sns
from constructs import Construct

class ProducerStack(Stack):
    def __init__(self, scope: Construct, id: str, **kwargs) -> None:
        super().__init__(scope, id, **kwargs)
        acct_config = self.get_yaml_config("../config/targets.yaml")
        resource_config = self.get_yaml_config("../config/resources.yaml")
        topic_name = resource_config["topic_name"]
        bucket_name = resource_config["bucket_name"]
        topic = self.init_get_topic(topic_name)
        self.sns_permissions(topic)
        self.init_subscribe_permissions(topic, acct_config)
        self.init_publish_permissions(topic, acct_config)
        bucket = self.init_create_bucket(bucket_name)
        self.init_cross_account_log_role(acct_config, bucket)

    def get_yaml_config(self, filepath):
        with open(filepath, "r") as file:
            data = yaml.safe_load(file)
        return data

    def init_get_topic(self, topic_name):
        topic = sns.Topic(self, "fanout-topic", topic_name=topic_name)
        return topic

    def init_rule(self, topic):
        rule = events.Rule(
            self,
            "trigger-rule",
            schedule=events.Schedule.cron(
                minute="0",
                hour="22",
                week_day="FRI",
            ),
        )
        rule.add_target(targets.SnsTopic(topic))

    def sns_permissions(self, topic):
        # Set up base Amazon SNS permissions.
        sns_permissions = iam.PolicyStatement()
        sns_permissions.add_any_principal()
        sns_permissions.add_actions(
            "SNS:AddPermission",
            "SNS:DeleteTopic",
            "SNS:GetTopicAttributes",
            "SNS:ListSubscriptionsByTopic",
            "SNS:SetTopicAttributes",
            "SNS:Subscribe",
            "SNS:RemovePermission",
            "SNS:Publish",
        )
        sns_permissions.add_resources(topic.topic_arn)
        sns_permissions.add_condition(
            "StringEquals", {"AWS:SourceOwner": Aws.ACCOUNT_ID}
        )
        topic.add_to_resource_policy(sns_permissions)

    def init_subscribe_permissions(self, topic, target_accts):
        subscribe_permissions = iam.PolicyStatement()
        subscribe_permissions.add_arn_principal(f"arn:aws:iam::{Aws.ACCOUNT_ID}:root")
        for language in target_accts.keys():
            if "enabled" in str(target_accts[language]["status"]):
                subscribe_permissions.add_arn_principal(
                    f"arn:aws:iam::{str(target_accts[language]['account_id'])}:root"
                )
        subscribe_permissions.add_actions("SNS:Subscribe")
        subscribe_permissions.add_resources(topic.topic_arn)
        topic.add_to_resource_policy(subscribe_permissions)

    def init_publish_permissions(self, topic, target_accts):
        publish_permissions = iam.PolicyStatement()
        publish_permissions.add_arn_principal(f"arn:aws:iam::{Aws.ACCOUNT_ID}:root")
        for language in target_accts.keys():
            publish_permissions.add_arn_principal(
                f"arn:aws:iam::{str(target_accts[language]['account_id'])}:root"
            )
        publish_permissions.add_actions("SNS:Publish")
        publish_permissions.add_service_principal("events.amazonaws.com")
        publish_permissions.add_resources(topic.topic_arn)
        topic.add_to_resource_policy(publish_permissions)

    def init_create_bucket(self, bucket_name):
        bucket = s3.Bucket(
            self,
            bucket_name,
            bucket_name=bucket_name,
            versioned=False,
            block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
        )
        return bucket

    def init_cross_account_log_role(self, target_accts, bucket):
        languages = target_accts.keys()
        if len(languages) > 0:
            # Define policy that allows cross-account Amazon SNS and Amazon SQS access.
            statement = iam.PolicyStatement()
            statement.add_actions("s3:PutObject", "s3:PutObjectAcl")
            statement.add_resources(f"{bucket.bucket_arn}/*")
            for language in languages:
                if "enabled" in str(target_accts[language]["status"]):
                    statement.add_arn_principal(
                        f"arn:aws:iam::{str(target_accts[language]['account_id'])}:role/LogsLambdaExecutionRole"
                    )
            statement.add_arn_principal(f"arn:aws:iam::{Aws.ACCOUNT_ID}:root")
            bucket.add_to_resource_policy(statement)


import constants
import time
import datetime
import os
import pandas as pd
import py_parser
import numpy as np
import label_perturbation_main

def giveTimeStamp():
  tsObj = time.time()
  strToret = datetime.datetime.fromtimestamp(tsObj).strftime(constants.TIME_FORMAT)
  return strToret

def generateUnitTest(algo, attack_type):
    file_name = "../../Output/attack_unit_test/test_attack_" + algo + ".py"
    with open(file_name,"w+") as file:
        file.write("import unittest\n")
        file.write("import label_perturbation_main\n")
        file.write("import " + algo + "\n")
        file.write("\n\n")
        file.write("class TestAttack( unittest.TestCase ):\n")
        file.write("\tdef test_attack(self):\n")
        file.write("\t\tchange_unit = 0.5\n")
        file.write("\t\tprecision4model1, recall4model1, fscore4model1, auc4model1 = label_perturbation_main.run_experiment(" + "algo" + ")\n")
        if (attack_type == 'random'):
            file.write("\t\tprecision4model2, recall4model2, fscore4model2, auc4model2 = label_perturbation_main.run_random_perturbation_experiment(change_unit, "+ "algo" + ")\n")
        if (attack_type == 'loss'):
            file.write("\t\tprecision4model2, recall4model2, fscore4model2, auc4model2 = label_perturbation_main.run_loss_based_perturbation_experiment(change_unit, "+ "algo" + ")\n")
        if (attack_type == 'prob'):
            file.write("\t\tprecision4model2, recall4model2, fscore4model2, auc4model2 = label_perturbation_main.run_prob_based_perturbation_experiment(change_unit, 10, 10, "+ "algo" + ")\n")
        file.write("\t\tself.assertEqual(auc4model1, auc4model2, \"DECREASE IN AUC VALUE ... POSSIBLE ATTACK?\"  )\n")

def generateAttack(inp_dir, delta):
    if os.path.exists(inp_dir):
        algo_df = pd.read_csv(inp_dir)
    else:
        return
    for index, row in algo_df.iterrows():
        row['ALGO_NAME'] = row['ALGO_NAME'].replace('[', '')
        row['ALGO_NAME'] = row['ALGO_NAME'].replace(']', '')
        algo_list = row['ALGO_NAME'].split(',')
        for algo in algo_list:
            algo = algo.replace('\'', '')
            algo = algo.replace(' ', '')
            initial_auc, random_auc, loss_auc, prob_auc = label_perturbation_main.run_label_perturbation(algo)
            random_diff = random_auc[0] -  initial_auc[0]
            loss_diff = loss_auc[0] -  initial_auc[0]
            prob_diff = prob_auc[0] -  initial_auc[0]
            if (random_diff < delta):
                generateUnitTest(algo, 'random')
            elif (loss_diff < delta):
                generateUnitTest(algo, 'loss')
            elif (prob_diff < delta):
                generateUnitTest(algo, 'prob')

if __name__=='__main__':

    delta = 0.5

    t1 = time.time()
    print('Started at:', giveTimeStamp() )
    print('*'*100 )

#     print('*'*100 )
#     algo_input_csv = 'algo_list.csv'
#     generateAttack(algo_input_csv, delta)
#     print('*'*100 )

    print('*'*100 )
    algo_input_csv = '../../Output/ALGO_SUPERVISED_OUTPUT_GITHUB.csv'
    generateAttack(algo_input_csv, delta)
    print('*'*100 )

    print('*'*100 )
    algo_input_csv = '../../Output/ALGO_SUPERVISED_OUTPUT_GITLAB.csv'
    generateAttack(algo_input_csv, delta)
    print('*'*100 )

    print('*'*100 )
    algo_input_csv = '../../Output/ALGO_SUPERVISED_OUTPUT_MODELZOO.csv'
    generateAttack(algo_input_csv, delta)
    print('*'*100 )

    print('Ended at:', giveTimeStamp() )
    print('*'*100 )

    t2 = time.time()
    time_diff = round( (t2 - t1 ) / 60, 5)
    print('Duration: {} minutes'.format(time_diff) )
    print('*'*100 )


# -*- coding: utf-8 -*-
"""
Created on Thu Dec 21 13:14:50 2017

@author: B
"""
import sys
sys.path.append('/root/asar2018psc/training/Models/')
import numpy as np
np.random.seed(123)
import argparse
import Models , PageLoadBatches
from keras.callbacks import ModelCheckpoint
from keras import optimizers
import glob
import cv2
import os

parser = argparse.ArgumentParser()
parser.add_argument("--save_weights_path", type = str,default ="bestweights"  )
parser.add_argument("--train_images", type = str, default ="ptrain/"  )
parser.add_argument("--train_annotations", type = str, default = "pltrain/"  )
parser.add_argument("--n_classes", type=int, default = 3 )
parser.add_argument("--input_height", type=int , default = 320  )
parser.add_argument("--input_width", type=int , default = 320 )

parser.add_argument('--validate',action='store_false')
parser.add_argument("--val_images", type = str , default = "pvalidation/")
parser.add_argument("--val_annotations", type = str , default = "plvalidation/")

parser.add_argument("--test_images", type = str , default = "ptest/")
parser.add_argument("--test_annotations", type = str , default = "pltest/")
parser.add_argument("--output_path", type = str , default = "pprediction/")

parser.add_argument("--epochs", type = int, default = 250 )
parser.add_argument("--batch_size", type = int, default = 16 )
parser.add_argument("--val_batch_size", type = int, default = 16 )
parser.add_argument("--test_batch_size", type = int, default = 16 )

parser.add_argument("--load_weights", type = str , default = 'bestweights')

parser.add_argument("--model_name", type = str , default = "fcn8")
parser.add_argument("--optimizer_name", type = str , default = "sgd")

args = parser.parse_args()

train_images_path = args.train_images
train_segs_path = args.train_annotations
train_batch_size = args.batch_size
n_classes = args.n_classes
input_height = args.input_height
input_width = args.input_width
validate = args.validate
save_weights_path = args.save_weights_path
epochs = args.epochs
load_weights = args.load_weights
test_images_path = args.test_images
test_segs_path = args.test_annotations
test_batch_size = args.test_batch_size

optimizer_name = args.optimizer_name
model_name = args.model_name

if validate:
	val_images_path = args.val_images
	val_segs_path = args.val_annotations
	val_batch_size = args.val_batch_size

modelFns = { 'vgg_segnet':Models.VGGSegnet.VGGSegnet , 'vgg_unet':Models.VGGUnet.VGGUnet , 'vgg_unet2':Models.VGGUnet.VGGUnet2 , 'fcn8':Models.FCN8.FCN8 , 'fcn32':Models.FCN32.FCN32   }
modelFN = modelFns[ model_name ]

m = modelFN( n_classes , input_height=input_height, input_width=input_width   )
sgd = optimizers.SGD(lr=0.0001)
#adm=optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=5e-05)

m.compile(loss='categorical_crossentropy',
      optimizer= sgd,
      metrics=['accuracy'])

if len( load_weights ) > 0:
    print("loading initial weights")
    m.load_weights(load_weights)

print ( m.output_shape)

output_height = m.outputHeight
output_width = m.outputWidth

G  = PageLoadBatches.imageSegmentationGenerator( train_images_path , train_segs_path ,  train_batch_size,  n_classes , input_height , input_width , output_height , output_width   )

if validate:
	G2  = PageLoadBatches.imageSegmentationGenerator( val_images_path , val_segs_path ,  val_batch_size,  n_classes , input_height , input_width , output_height , output_width   )

mcp=ModelCheckpoint( filepath=save_weights_path, monitor='val_loss', save_best_only=True, save_weights_only=True,verbose=1)

if not validate:
	for ep in range( epochs ):
		m.fit_generator( G , 1  , epochs=1 )
else:
    for ep in range( epochs ):
        m.fit_generator( G , 6250 , validation_data=G2 , validation_steps=930,  epochs=1,callbacks=[mcp] )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import cherrypy
import Installation
import Equipement
import Activite
import database as bd

class WebManager(object):

	@cherrypy.expose
	def index(self):
		return '''

 Installations sportives des Pays de la loire

		<input type="button" name="Installations" value="Afficher les installations" onclick="self.location.href='display_Installations'">
		<input type="button" name="Activites" value="Afficher les activites" onclick="self.location.href='display_Activites'">
		<input type="button" name="Installations" value="Afficher les equipements" onclick="self.location.href='display_Equipements'">

		'''

	@cherrypy.expose
	def display_Installations(self):
		html=""
		database = bd.Database('BD/M4105C.db')
		insts = database.read_Installations()
		html += '''
Tableau des installations

			 		Numéro
			 		ComLib
			 		ComInsee

\n'''
		for i in insts:
			html += '''
\n
					''' + str(i.InsNumeroInstall) + '''
					''' + i.ComLib + '''
					''' + str(i.ComInsee) + '''

'''
		html += '''
'''
		return html

	@cherrypy.expose
	def display_Activites(self):
		html = ""
		database = bd.Database('BD/M4105C.db')
		insts = database.read_Activites()
		html += '''
Tableau des activites

			 		ComInsee
			 		EquipementId
			 		ActCode

\n'''
		for i in insts:
			html += '''
\n
					''' + str(i.ComInsee) + '''
					''' + str(i.EquipementId) + '''
					''' + str(i.ActCode) + '''

'''
		html += '''
'''
		return html

cherrypy.quickstart(WebManager())


from django.urls import path

from api.views import (
    TaskAPIView,
    AlgorithmTestAPIView,
    TestAPIView,
    WantedPageDataAPIView,
    KreditJobAPIView,
    GoogleTrendsAPIView,
)

urlpatterns = [
    path('test/', TestAPIView.as_view(), name='test'),
    path('algo_test/', AlgorithmTestAPIView.as_view(), name='algo_test'),
    path('', TaskAPIView.as_view(), name='task'),
    path('wanted_page_data', WantedPageDataAPIView.as_view(), name='wanted_page_data'),
    path('kredit_job_data', KreditJobAPIView.as_view(), name='kredit_job_data'),
    path('google_trends_data', GoogleTrendsAPIView.as_view(), name='google_trends_data'),
]


# -*- coding: utf-8 -*-
import json

from classytags.core import Tag, Options
from cms.utils.encoder import SafeJSONEncoder
from django import template
from django.utils.safestring import mark_safe

register = template.Library()

@register.filter('json')
def json_filter(value):
    """
    Returns the JSON representation of ``value`` in a safe manner.
    """
    return mark_safe(json.dumps(value, cls=SafeJSONEncoder))

@register.filter
def bool(value):
    if value:
        return 'true'
    else:
        return 'false'

class JavascriptString(Tag):
    name = 'javascript_string'
    options = Options(
        blocks=[
            ('end_javascript_string', 'nodelist'),
        ]
    )

    def render_tag(self, context, **kwargs):
        try:
            from django.utils.html import escapejs
        except ImportError:
            from django.utils.text import javascript_quote as escapejs
        rendered = self.nodelist.render(context)
        return u"'%s'" % escapejs(rendered.strip())
register.tag(JavascriptString)