"""
Experimental Functions - In Construction !!!

Author:         Lucas Tim Grulich
Created:        October 2017
Last Update:    02. August 2019
"""

from oap.__conf__ import MARKER, MONOSCALE_SHADOWLEVEL, SLICE_SIZE
from oap.utils import barycenter

import numpy as np

from copy import copy

from matplotlib import pyplot as plt
from matplotlib.patches import Rectangle
from matplotlib.transforms import Affine2D

from math import acos, atan, atan2, cos, degrees, radians, sin, sqrt
from scipy.cluster.hierarchy import linkage, fcluster

def vanish_poisson(array):
    vanish_color(array, color=MARKER['poisson'])

def vanish_color(array, color):
    array[array == color] = 0

def __manhattan_dist(v0, v1):
    return abs(v1[0] - v0[0]) + abs(v1[1] - v0[1])

def __euclidean_dist(v0, v1):
    return sqrt(sum(pow(x - y, 2) for x, y in zip(v0, v1)))

def __vector_length(vec):
    return sqrt(vec[0] * vec[0] + vec[1] * vec[1])

def __normalize(vec):
    length = __vector_length(vec)
    return [vec[0] / length, vec[1] / length] if length > 0 else [0, 0]

def __degree(vec0, vec1):
    vec0 = __normalize(vec0)
    vec1 = __normalize(vec1)

    x = vec0[0] * vec1[0] + vec0[1] * vec1[1]
    if -1 <= x <= 1:
        return degrees(acos(x))
    return degrees(acos(round(x)))

def __vector_shift_in_radians(vec0, vec1):
    vec0 = __normalize(vec0)
    vec1 = __normalize(vec1)
    return atan2(vec0[1], vec0[0]) - atan2(vec1[1], vec1[0])

# ToDo: recursive depth
def __floodfill(array, x, y, marker, colors, diagonal=False, horizontal=True, slice_size=SLICE_SIZE):
    """
    Recursive FloodFill Alogrithm for optical arrays
    as 1 dimensional list or numpy array.

    :param array:       optical array (particle image)
    :type array:        numpy array (1 dimensional) or list or string

    :param x:           x-index
    :type x:            integer

    :param y:           y-index
    :type y:            integer

    :param marker:      color-value of the filling
    :type marker:       integer

    :param colors:      list of colors, which shall be colored
    :type colors:       list of integers

    --- optional params ---
    :param diagonal:    filling in diagonal direction (default == False)
    :type diagonal:     boolean

    :param horizontal:  filling in horizontal direction (default == True)
    :type horizontal:   boolean

    :param slice_size:   width of the optical array (number of diodes)
    :type slice_size:    integer

    :return:             this (recursive)
    """
    if 0 <= x < slice_size and 0 <= y < (len(array) / slice_size):

        if array[y*slice_size+x] not in colors:
            return

        if array[y*slice_size+x] != marker:
            array[y*slice_size+x] = marker
            try:
                if horizontal:
                    __floodfill(array, x + 1, y, marker, colors, diagonal, horizontal, slice_size)
                    __floodfill(array, x - 1, y, marker, colors, diagonal, horizontal, slice_size)
                    __floodfill(array, x, y + 1, marker, colors, diagonal, horizontal, slice_size)
                    __floodfill(array, x, y - 1, marker, colors, diagonal, horizontal, slice_size)
                if diagonal:
                    __floodfill(array, x + 1, y + 1, marker, colors, diagonal, horizontal, slice_size)
                    __floodfill(array, x + 1, y - 1, marker, colors, diagonal, horizontal, slice_size)
                    __floodfill(array, x - 1, y + 1, marker, colors, diagonal, horizontal, slice_size)
                    __floodfill(array, x - 1, y - 1, marker, colors, diagonal, horizontal, slice_size)
            except:
                return
    return

def principal_axes(array,
                   noise=False,
                   mindegree=175.0,
                   metric="cityblock",
                   method="average",
                   maxdistance=6.2,
                   merge=17.8,
                   mergingmethod="average",
                   dimensionfactor=1.3,
                   plot=(0, 0, 0),
                   slicesize=SLICE_SIZE):
    """
    This method finds parts of the particle image, which stick out (so called vertices).
    For example the individual bullets of a combined bullet or the arms of a snowflake.

    Returns the number of vertices and the number of vectors, which point from
    the barycenter to the individual vertices, which are nearly in one line
    (the threshold is the minimum degree).

    Returns also the variance of the vector lengths and the length and width of the stick out parts.

    Perfect ice crystals should have 6 vertices, 3 straight lines and a small variance.
    Needles or Columns should have 2 vertices, 1 straight line and a small variance.
    Combined bullets should have at least 3 vertices.
    Thats just an example! These values differ for different clustering methods, maximum distances,
    degrees and the method of merging vectors.

    Created: March / April 2018 in St. Louis, MO
    """

    # Copy of array.
    array_copy = copy(array)

    x_bary, y_bary = barycenter(array_copy, coordinates=False, slice_size=slicesize)
    x_bary_coord, y_bary_coord = barycenter(array_copy, slice_size=slicesize)

    if array_copy[y_bary_coord * slicesize + x_bary_coord] == 0 \
            or array_copy[y_bary_coord * slicesize + x_bary_coord] == MARKER['poisson']:
        return False

    if not noise:
        __floodfill(array_copy, x_bary_coord, y_bary_coord, MARKER['floodfill'],
                    colors=[1, 2, 3, MARKER['poisson'], MARKER['floodfill']])

        # Delete noisy components of the optical array.
        for i in range(len(array_copy)):
            if array_copy[i] and array_copy[i] != MARKER['floodfill']:
                array_copy[i] = 0

    slices = int(len(array_copy) / slicesize)

    sum_distance = 0.0
    number_of_pixels = 0

    # Calculate the average distance of every shadowed pixel to the
    # particle barycenter.
    for y in range(slices):
        for x in range(slicesize):
            if array_copy[y * slicesize + x]:
                sum_distance += __euclidean_dist((x_bary, y_bary), (x, y))
                number_of_pixels += 1

    avg_distance = sum_distance / number_of_pixels
    stick_out_pixels = []
    remaining_pixels = []

    # Find the pixels, which stick out of the particle core.
    inner_pixel_array = np.zeros(len(array))
    for y in range(slices):
        for x in range(slicesize):
            if array_copy[y * slicesize + x]:
                if avg_distance <= __euclidean_dist((x_bary, y_bary), (x, y)):
                    stick_out_pixels.append([x, y])
                else:
                    inner_pixel_array[y * slicesize + x] = 2
                    remaining_pixels.append([x, y])

    # Calculate a new barycenter for the inner pixel group  of the optical array.
    try:
        x_new_bary, y_new_bary = barycenter(inner_pixel_array,
                                            coordinates=False,
                                            slice_size=slicesize)
    except TypeError:
        return False

    stick_out_pixels = np.array(stick_out_pixels)
    remaining_pixels = np.array(remaining_pixels)

    # Ratio of pixels ticking out to the total number of pixels.
    # ratio = float(len(stick_out_pixels)) / (len(stick_out_pixels)+len(remaining_pixels))

    if not len(stick_out_pixels) >= 2:
        return False

    # Manhattan Clustering of the sticking out pixels.
    z = linkage(stick_out_pixels, method=method, metric=metric)
    # Clustering with maximum distance between clusters.
    clusters = fcluster(z, maxdistance, criterion='distance')

    number_of_stick_outs = max(clusters)
    largest_distances = np.zeros(number_of_stick_outs)

    for i, v in enumerate(stick_out_pixels):
        if __euclidean_dist((x_new_bary, y_new_bary), v) > largest_distances[clusters[i] - 1]:
            largest_distances[clusters[i] - 1] = __euclidean_dist((x_new_bary, y_new_bary), v)

    # Point the vectors to the barycenter of the different clusters.
    vectors = []
    cluster_barycenters = []

    plot_vectors = []

    for i in range(max(clusters)):

        pixels = np.zeros(len(array))
        for j, pxl in enumerate(stick_out_pixels):
            if i == clusters[j] - 1:
                pixels[pxl[1] * slicesize + pxl[0]] = 2

        x_vec, y_vec = barycenter(pixels, coordinates=False, slice_size=slicesize)
        vec = [x_vec - x_new_bary, y_vec - y_new_bary]

        cluster_barycenters.append([x_vec, y_vec])

        # Calculate scale to set the vector size to the largest distance.
        vec_length = __vector_length(vec)
        if vec_length:
            vec_scale = largest_distances[i] / vec_length
        else:
            vec_scale = 1.0

        # Update vector with the new scale.
        vec = [vec_scale * vec[0], vec_scale * vec[1]]
        vectors.append(vec)
        if plot:
            plot_vectors.append((x_new_bary, y_new_bary, vec[0], vec[1]))

    old_clusters = None
    # --- Agglomerative Clustering of close vectors ---
    #
    # The degree between vectors determines if they are close.
    # Vectors which degrees are smaller than the "merge" value get merged.
    # User can switch between single or average clustering of vectors.
    if merge:
        # First step: every vector is in its own cluster.
        joined_vectors = [[[v, i]] for i, v in enumerate(vectors)]

        while True:
            min_degree = 361
            join_index_1 = 0
            join_index_2 = 0

            for c_1, cluster_1 in enumerate(joined_vectors):
                for c_2, cluster_2 in enumerate(joined_vectors):
                    if c_1 == c_2 or len(cluster_1) == 0 or len(cluster_2) == 0:
                        continue

                    if mergingmethod.upper() == "AVERAGE":
                        # Average Clustering
                        avg_vec_1 = [0, 0]
                        for vec in cluster_1:
                            avg_vec_1[0] += vec[0][0]
                            avg_vec_1[1] += vec[0][1]
                        avg_vec_1[0] /= len(cluster_1)
                        avg_vec_1[1] /= len(cluster_1)

                        avg_vec_2 = [0, 0]
                        for vec in cluster_2:
                            avg_vec_2[0] += vec[0][0]
                            avg_vec_2[1] += vec[0][1]
                        avg_vec_2[0] /= len(cluster_2)
                        avg_vec_2[1] /= len(cluster_2)

                        deg = __degree(avg_vec_1, avg_vec_2)
                        if deg < min_degree:
                            min_degree = deg
                            join_index_1 = c_1
                            join_index_2 = c_2
                    elif mergingmethod.upper() == "SINGLE":
                        for vec_1 in cluster_1:
                            for vec_2 in cluster_2:
                                deg = __degree(vec_1[0], vec_2[0])
                                if deg < min_degree:
                                    min_degree = deg
                                    join_index_1 = c_1
                                    join_index_2 = c_2

            # Break if there are no more clusters to merge.
            if min_degree > merge:
                break
            # Join closest clusters.
            joined_vectors[join_index_1] += joined_vectors[join_index_2]
            joined_vectors[join_index_2] = []

        # Overwrite clusters with other information. The following function needs the vector lengths and their indices.
        # Hence the new clusters provide both informations.
        joined = []
        for j_vecs in joined_vectors:
            if len(j_vecs) == 0:
                continue
            new_cluster = []
            for v in j_vecs:
                new_cluster.append([__vector_length(v[0]), v[1]])
            joined.append(sorted(new_cluster, reverse=True))
        joined_vectors = joined

        # Find the new barycenter of the new joined vector.
        merged_barycenters = []
        for j_vecs in joined_vectors:
            merged_center = [0, 0]

            for v in j_vecs:
                merged_center[0] += cluster_barycenters[v[1]][0]
                merged_center[1] += cluster_barycenters[v[1]][1]
            merged_center[0] /= len(j_vecs)
            merged_center[1] /= len(j_vecs)
            merged_barycenters.append(merged_center)

        old_clusters = copy(clusters)
        for i, j_vecs in enumerate(joined_vectors):
            # Join Clusters -> Clusters get the value of the cluster with the longest vector.
            for j in range(len(clusters)):
                if clusters[j] - 1 in [v[1] for v in j_vecs]:
                    clusters[j] = j_vecs[0][1] + 1

            # Calcluate the new scale and the new Vector direction.
            vec_scale = largest_distances[j_vecs[0][1]] / __vector_length(
                [merged_barycenters[i][0] - x_new_bary, merged_barycenters[i][1] - y_new_bary])
            vectors[j_vecs[0][1]] = [vec_scale * (merged_barycenters[i][0] - x_new_bary),
                                     vec_scale * (merged_barycenters[i][1] - y_new_bary)]
            if plot:
                plot_vectors[j_vecs[0][1]] = (x_new_bary, y_new_bary,
                                              vectors[j_vecs[0][1]][0], vectors[j_vecs[0][1]][1])

            # Set merged vectors to None.
            for j in range(1, len(j_vecs)):
                vectors[j_vecs[j][1]] = None
                if plot:
                    plot_vectors[j_vecs[j][1]] = None

    # --- Dimension Calculation ---
    dimensions = []
    if dimensionfactor:
        for i in range(max(clusters)):

            # Some clusters might be empty, because of the merging process.
            empty = True
            pixels = np.zeros(len(array))
            for j, pxl in enumerate(stick_out_pixels):
                if i == clusters[j] - 1:
                    pixels[pxl[1] * slicesize + pxl[0]] = 2
                    empty = False
            if empty:
                continue

            # Calculate the width of the cluster in dependence of the vector rotation.
            theta = __vector_shift_in_radians([0, 1], vectors[i])
            pixels_x = []
            pixels_y = []
            for y in range(int(len(pixels) / slicesize)):
                for x in range(slicesize):

                    if pixels[y * slicesize + x]:
                        px = x * cos(theta) - y * sin(theta)
                        # Rotation of y is not necessary, but is needed for debugging.
                        py = x * sin(theta) + y * cos(theta)
                        pixels_x.append(px)
                        pixels_y.append(py)

            # Calculate the variance in the x direction
            sum_x = 0
            sum_xx = 0
            for j in range(len(pixels_x)):
                sum_x += pixels_x[j]
                sum_xx += pixels_x[j] * pixels_x[j]

            x_bary_rotation = sum_x / len(pixels_x)
            var_x = sum_xx / len(pixels_x) - x_bary_rotation * x_bary_rotation
            discriminant = sqrt(4 * var_x * var_x)
            width = dimensionfactor * 2 * sqrt(((var_x + var_x) + discriminant) / 2.0)

            dimensions.append([__vector_length(vectors[i]), width])

    if merge:
        # Remove objects in vector lists, which are equal to None.
        # Objects which are None got deleted in the vector and cluster merging process above.
        # The deletion must be down here, because it changes the indices of the vectors.
        # The correct vector indices are needed for the dimension calculations
        vectors = [x for x in vectors if x is not None]
        if plot:
            plot_vectors = [x for x in plot_vectors if x is not None]

    # Calculate the mean and the variance of vector lengths.
    mean = sum([__vector_length(vec) for vec in vectors]) / float(len(vectors))
    variance = sum([pow(__vector_length(vec) - mean, 2) for vec in vectors]) \
               / float(len(vectors))

    # Find the number of vectors which are nearly in a straight line.
    # In other words: Vectors which are nearly parallel to each other.
    angles = []
    nearly_straight_angles = 0
    for i, vec0 in enumerate(vectors):
        for j, vec1 in enumerate(vectors):
            if i == j:
                break
            angle = round(__degree(vec0, vec1), 2)
            if angle >= mindegree:
                nearly_straight_angles += 1
                angles.append(angle)

    if plot:

        if plot[0]:
            if plot[0] == 1:
                plt.scatter(stick_out_pixels[:, 0], stick_out_pixels[:, 1], c=clusters, cmap="jet")
            elif plot[0] == 2:
                plt.scatter(stick_out_pixels[:, 0], stick_out_pixels[:, 1], c=old_clusters, cmap="jet")
            else:
                plt.scatter(stick_out_pixels[:, 0], stick_out_pixels[:, 1], c="gray")
            plt.scatter(remaining_pixels[:, 0], remaining_pixels[:, 1], c="gray")

        ax = plt.gca()
        ax.invert_yaxis()
        ax.set_aspect(1)

        if plot[1] == 1:
            a, b, c, d = zip(*plot_vectors)
            plt.quiver(a, b, c, d, angles='xy', scale_units='xy', scale=1, width=0.01)

        elif plot[1] == 2 and dimensionfactor:

            for i, v in enumerate(vectors):
                theta = __vector_shift_in_radians([0, 1], v)
                trans = Affine2D().rotate_around(x_new_bary, y_new_bary, -theta)

                hexagon_1 = Rectangle((x_new_bary, y_new_bary),
                                      dimensions[i][1] / 2.0, dimensions[i][0],
                                      fill=True,
                                      edgecolor="black",
                                      alpha=0.3,
                                      linewidth=1)
                hexagon_2 = Rectangle((x_new_bary, y_new_bary),
                                      -dimensions[i][1] / 2.0, dimensions[i][0],
                                      fill=True,
                                      edgecolor="black",
                                      alpha=0.3,
                                      linewidth=1)

                hexagon_1.set_transform(trans + ax.transData)
                hexagon_2.set_transform(trans + ax.transData)
                ax.add_patch(hexagon_1)
                ax.add_patch(hexagon_2)

        if plot[2]:
            cluster_barycenters = np.array(cluster_barycenters)
            plt.scatter(cluster_barycenters[:, 0], cluster_barycenters[:, 1], c="black")

            plt.scatter(x_bary, y_bary, c="red")
            plt.scatter(x_new_bary, y_new_bary, c="black")

        plt.show()

    return number_of_stick_outs, \
           nearly_straight_angles, \
           angles, \
           round(variance, 3), \
           dimensions

def __mse(array_a, array_b):
    """
    Mean Squared Error: The two arrays must have the same dimension.
    """
    error = np.sum((array_a.astype("float") - array_b.astype("float")) ** 2)
    # Divide by the number of imagepixels
    error /= float(len(array_a))
    # The lower the error, the more similar the two images are.
    return error

def __polygon_intersection(verts, x, y):
    """
    Computes the intersection with a polygon.

    Algorithm from:
    W. Randolph Franklin (WRF)
    https://wrf.ecse.rpi.edu//Research/Short_Notes/pnpoly.html#The Method
    """
    intersection = False
    for i in range(len(verts)):
        j = (i + len(verts) - 1) % len(verts)
        if (verts[i][1] > y) != (verts[j][1] > y) \
                and x < (verts[j][0] - verts[i][0]) * (y - verts[i][1]) / (verts[j][1] - verts[i][1]) + verts[i][0]:
            intersection = not intersection
    return intersection

def __polygon_rotation(verts, alpha, point):
    """
    Rotates an Object around a specific point.

    Parameter: List of 2d vertices represented as tuple (x,y)
               The angle in radians
               A 2d point (x,y) as Tuple, List or Array

    Returns:   The rotated object
    """
    polygon = []
    cos_alpha = cos(alpha)
    sin_alpha = sin(alpha)
    for v in verts:
        x = v[0] * cos_alpha - sin_alpha * v[1] + point[0]
        y = v[0] * sin_alpha + cos_alpha * v[1] + point[1]
        polygon.append((x, y))
    return polygon

def principal_components(array, dimensionfactor=1.0, plot=None, slicesize=SLICE_SIZE):
    """
    Calculates the Principal Components of an Optical Array.
    """
    sum_x = 0.0
    sum_y = 0.0
    sum_xx = 0.0
    sum_yy = 0.0
    sum_xy = 0.0
    number_pix = 0.0

    for y in range(int(len(array) / slicesize)):
        for x in range(slicesize):
            if array[y * slicesize + x]:
                sum_x += x
                sum_y += y
                sum_xx += x * x
                sum_yy += y * y
                sum_xy += x * y
                number_pix += 1

    if number_pix == 0:
        return

    x_bary = sum_x / number_pix
    y_bary = sum_y / number_pix

    # Calculating the variance and the covariance.
    var_x = sum_xx / number_pix - x_bary * x_bary
    var_y = sum_yy / number_pix - y_bary * y_bary
    cov_xy = sum_xy / number_pix - x_bary * y_bary

    discriminant = (var_x - var_y) * (var_x - var_y) + 4 * cov_xy * cov_xy
    sqrt_discr = sqrt(discriminant)

    lambda_plus = ((var_x + var_y) + sqrt_discr) / 2.0
    lambda_minus = ((var_x + var_y) - sqrt_discr) / 2.0

    # --- Eigenvectors ---
    x_plus = var_x + cov_xy - lambda_minus
    y_plus = var_y + cov_xy - lambda_minus
    x_minus = var_x + cov_xy - lambda_plus
    y_minus = var_y + cov_xy - lambda_plus

    # Normalizing the vectors.
    denom_plus = sqrt(x_plus * x_plus + y_plus * y_plus)
    denom_minus = sqrt(x_minus * x_minus + y_minus * y_minus)

    # Computing the normalized vector, which is parallel to the
    # longest axis of a particle image.
    if denom_plus:
        x_parallel = x_plus / denom_plus
        y_parallel = y_plus / denom_plus
    else:
        x_parallel = x_plus
        y_parallel = y_plus
    # Computing the normalized vector, which is corresponding the
    # Normal of a particle image.
    if denom_minus:
        x_normal = x_minus / denom_minus
        y_normal = y_minus / denom_minus
    else:
        x_normal = x_minus
        y_normal = y_minus

    if lambda_plus < 0:
        lambda_plus = 0
    if lambda_minus < 0:
        lambda_minus = 0
    major_axis = dimensionfactor * 2 * sqrt(lambda_plus)
    minor_axis = dimensionfactor * 2 * sqrt(lambda_minus)

    # Computing the rotation of the principal components.
    if x_parallel:
        alpha = atan(y_parallel / x_parallel)
    else:
        alpha = radians(90.0)

    # --- Polygon ------------------------------------------------------------------------------------------------------
    """
    scale_1 = 0.75
    scale_2 = 1.0
    scale_3 = 1.0
    scale_4 = 0.5

    vert_1 = (-major_axis * scale_1,  minor_axis * scale_2)
    vert_2 = ( major_axis * scale_1,  minor_axis * scale_2)
    vert_3 = ( major_axis * scale_1, -minor_axis * scale_2)
    vert_4 = (-major_axis * scale_1, -minor_axis * scale_2)
    vert_5 = (-major_axis * scale_3,  minor_axis * scale_4)
    vert_6 = ( major_axis * scale_3,  minor_axis * scale_4)
    vert_7 = ( major_axis * scale_3, -minor_axis * scale_4)
    vert_8 = (-major_axis * scale_3, -minor_axis * scale_4)

    polygon = [vert_5, vert_1, vert_2, vert_6,
              vert_7, vert_3, vert_4, vert_8]
    polygon = __polygon_rotation(object, alpha, (x_bary, y_bary))
    """
    vert_1 = (-major_axis, minor_axis)
    vert_2 = (major_axis, minor_axis)
    vert_3 = (major_axis, -minor_axis)
    vert_4 = (-major_axis, -minor_axis)

    polygon = [vert_1, vert_2, vert_3, vert_4]
    polygon = __polygon_rotation(polygon, alpha, (x_bary, y_bary))

    cos_alpha = cos(alpha)
    sin_alpha = sin(alpha)

    # ToDo: perhaps a return would be better here
    if minor_axis == 0:
        minor_axis = 1e-8
    if major_axis == 0:
        major_axis = 1e-8

    b = minor_axis * minor_axis
    a = major_axis * major_axis

    polygon_array = np.zeros(len(array), "int")
    ellipse_array = np.zeros(len(array), "int")
    one_color_array = np.zeros(len(array), "int")

    polygon_hits = 0
    ellipse_hits = 0
    polygon_misses = 0
    ellipse_misses = 0

    points = [[], [], []]

    for y in range(int(len(array) / slicesize)):
        for x in range(slicesize):

            denom_x = cos_alpha * (x - x_bary) + sin_alpha * (y - y_bary)
            denom_y = sin_alpha * (x - x_bary) - cos_alpha * (y - y_bary)
            intersect_ellipse = ((denom_x * denom_x) / a) + ((denom_y * denom_y) / b) <= 1

            intersect_polygon = __polygon_intersection(polygon, x, y)

            if plot:
                if intersect_ellipse and array[y * slicesize + x]:
                    points[0].append([x, y])
                elif intersect_polygon and array[y * slicesize + x]:
                    points[1].append([x, y])
                elif array[y * slicesize + x]:
                    points[2].append([x, y])

            if intersect_ellipse and array[y * slicesize + x]:
                ellipse_hits += 1
            elif array[y * slicesize + x]:
                ellipse_misses += 1
            elif intersect_ellipse and array[y * slicesize + x] == 0:
                ellipse_misses += 1

            if intersect_polygon and array[y * slicesize + x]:
                polygon_hits += 1
            elif array[y * slicesize + x]:
                polygon_misses += 1

            if intersect_ellipse:
                ellipse_array[y * slicesize + x] = MONOSCALE_SHADOWLEVEL
            if intersect_polygon:
                polygon_array[y * slicesize + x] = MONOSCALE_SHADOWLEVEL
            if array[y * slicesize + x]:
                one_color_array[y * slicesize + x] = MONOSCALE_SHADOWLEVEL

    # Calculate the mean and the variance of vector lengths.
    # mean = (major_axis + minor_axis) / 2.0
    # variance = ((major_axis - mean) * (major_axis - mean) + (minor_axis - mean) * (minor_axis - mean)) / 2.0
    hit_ratio = ellipse_hits / float(ellipse_misses + ellipse_hits)
    alpha_value = degrees(alpha)
    axis_ratio = major_axis / minor_axis

    if plot is not None:

        points[0] = np.array(points[0])
        points[1] = np.array(points[1])
        points[2] = np.array(points[2])

        if plot[0] == 1 and len(points[0]):
            plt.scatter(points[0][:, 0], points[0][:, 1], c="#04466c")
        if plot[1] == 1 and len(points[1]):
            plt.scatter(points[1][:, 0], points[1][:, 1], c="#ff0000", alpha=0.75, marker="s")
        if plot[2] == 1 and len(points[2]):
            plt.scatter(points[2][:, 0], points[2][:, 1], c="#cc3f3f", alpha=0.75, marker="s")
        if plot[3] == 1 and len(points[2]) and False:
            plt.scatter(points[2][:, 0], points[2][:, 1], c="#ff0000", alpha=0.75, marker="s")

        array = np.array(array)
        vanish_poisson(array)
        image = np.reshape(array, (int(len(array) / 64), 64))

        plt.imshow(image, cmap="Greys_r")

        if y_normal < 0:
            vectors = [(x_bary, y_bary, x_parallel * major_axis, y_parallel * major_axis),
                       (x_bary, y_bary, -x_parallel * major_axis, -y_parallel * major_axis),
                       (x_bary, y_bary, x_normal * minor_axis, y_normal * minor_axis),
                       (x_bary, y_bary, -x_normal * minor_axis, -y_normal * minor_axis)]
            txt_center_x = x_bary + (x_normal * minor_axis)
            txt_center_y = y_bary + (y_normal * minor_axis)
        else:
            vectors = [(x_bary, y_bary, -x_parallel * major_axis, -y_parallel * major_axis),
                       (x_bary, y_bary, x_parallel * major_axis, y_parallel * major_axis),
                       (x_bary, y_bary, -x_normal * minor_axis, -y_normal * minor_axis),
                       (x_bary, y_bary, x_normal * minor_axis, y_normal * minor_axis)]
            txt_center_x = x_bary + (-x_normal * minor_axis)
            txt_center_y = y_bary + (-y_normal * minor_axis)

        if alpha < 0:
            horizontal = "right"
        else:
            horizontal = "left"

        # Plot Principal Components.
        x, y, u, v = zip(*vectors)
        plt.quiver(x_bary, y_bary, 0, -y_bary-0.5, angles='xy', scale_units='xy', scale=1, width=0.0075, headlength=0, headwidth=1,
                   color="grey")
        plt.quiver(x, y, u, v, angles='xy', scale_units='xy', scale=1, width=0.01, headlength=0, headwidth=1,
                   color=["#1f77b4", "#1f77b4", "#ff7f0e", "#ff7f0e"], label="Major Axis")
        plt.scatter(x_bary, y_bary, marker="o", color="#1f77b4", label="Barycenter")

        plt.text(x=txt_center_x, y=txt_center_y-0.5, s=f"{round(alpha_value, 3)}°", verticalalignment="bottom", horizontalalignment=horizontal, color="#ff7f0e")
        #plt.text(x=0, y=-10, s=f"Hit Ratio: {round(hit_ratio, 3)}\nAspect Ratio: {round(axis_ratio, 3)}",
        #         color="white", verticalalignment="bottom")

        if plot[3] == 1:
            # Plot Polygon.
            polygon_verts = len(polygon)
            for i in range(polygon_verts):
                if i == 0:
                    plt.plot((polygon[i % polygon_verts][0], polygon[(i + 1) % polygon_verts][0]),
                             (polygon[i % polygon_verts][1], polygon[(i + 1) % polygon_verts][1]),
                             linewidth=1.2, c="#d62728", label="Rectangle")
                else:
                    plt.plot((polygon[i % polygon_verts][0], polygon[(i + 1) % polygon_verts][0]),
                             (polygon[i % polygon_verts][1], polygon[(i + 1) % polygon_verts][1]),
                             linewidth=1.2, c="#d62728", )

        # Plot Ellipse.
        array_length = 1000
        theta = np.linspace(0, 2 * np.pi, array_length)

        # Transform cartesian coordinates to polar coordinates.
        x = major_axis * np.cos(theta)
        y = minor_axis * np.sin(theta)

        plt.plot((x * cos(alpha) - sin(alpha) * y) + x_bary,
                 (x * sin(alpha) + cos(alpha) * y) + y_bary,
                 linewidth=1.8, c="#2ca02c", label="Ellipse")

        from matplotlib.ticker import MultipleLocator
        ax = plt.gca()
        ax.set_facecolor("black")
        ax.set_aspect(1)

        #plt.xlabel("Diode Array")
        #plt.ylabel("Flight Direction")
        ax.xaxis.set_minor_locator(MultipleLocator(1))
        ax.yaxis.set_minor_locator(MultipleLocator(1))
        #ax.xaxis.set_major_locator(MultipleLocator(5))
        #ax.yaxis.set_major_locator(MultipleLocator(5))

        plt.xlim(plot[4])
        #ax.set_yticks(np.arange(-.5, int(len(array)/slicesize), 5))
        #ax.set_xticklabels(np.arange(0, 65+0.5, 5, dtype=int))
        #ax.set_yticklabels(np.arange(0, int(len(array)/slicesize)+0.5, 5, dtype=int))

        #plt.axes().yaxis.set_minor_locator(MultipleLocator(1))

        #plt.grid(which='minor', color='red', linestyle='-', linewidth=0.5, alpha=0.5)
        # plt.legend()
        # plt.savefig(plot[4])

    return hit_ratio, \
           __mse(one_color_array, ellipse_array), \
           polygon_hits / float(polygon_misses + polygon_hits) * 100, \
           __mse(one_color_array, polygon_array), \
           alpha_value, \
           axis_ratio, \
           (2 * major_axis, 2 * minor_axis)

def connected_components(array, slicesize=SLICE_SIZE):
    """
    Returns the number of Connected Components of an optical array
    as 1 dimensional list or numpy array.

    CAUTION: If the particle image is to big and the recursion depth of Python is
             to low, number_of_components will incremented more than once, since the
             the recursion will be interrupted and will be later continued with a
             new marker.
    """
    # Start with a high number as marker.
    marker = 100
    used_markers = []
    number_of_components = 0

    # Create copy of array. Otherwise the original
    # optical array will be colored.
    array_copy = copy(array)

    for y in range(int(len(array_copy) / slicesize)):
        for x in range(slicesize):
            if array_copy[y*slicesize+x] != 0 and not int(array_copy[y*slicesize+x]) in used_markers:
                # Color the component and then change the marker.
                __floodfill(array_copy, x, y, marker, colors=[1, 2, 3, MARKER['poisson'], marker], diagonal=True)
                used_markers.append(marker)
                number_of_components += 1
                marker += 1

    return number_of_components

# ToDo: recursive depth
def __poisson(array, x, y, counter=0):
    if counter > 64*30: # ToDo: Extremely bad solution -> think of an other algorithm for poisson detection!
        return
    if 0 <= x < 64 and 0 <= y < (len(array) / 64):
        if array[y*64+x] != 0:
            return
        if array[y*64+x] != 7:
            array[y*64+x] = 7
            try:
                __poisson(array, x + 1, y, counter+1)
                __poisson(array, x - 1, y, counter+1)
                __poisson(array, x, y + 1, counter+1)
                __poisson(array, x, y - 1, counter+1)
            except:
                return
        return
    return

# ToDo: check_y doesn't work on images with frames (not clipped)
def poisson_spot(array, sizing=False, check_y=False, slice_size=SLICE_SIZE):
    """
    If the particle has a Poisson Spot, the function fills the spot with a marker.
    Returns also the Poisson diameter, if sizing is True.

    If poisson spot is not closed in x-direction it is not possible to measure the
    spot size. Hence, the marker will be removed. If checkAlsoY is true the poisson spot
    must be closed in y direction too.

    :param array:       optical array (particle image)
    :type array:        numpy array (1 dimensional) or list or string

    --- optional params ---
    :param sizing:      if True and there is a spot -> returns spot size in pixels
    :type sizing:       boolean

    :param check_y:     if True -> checks the first and last slice, if spot is closed
    :type check_y :     boolean

    :param slice_size:  width of the optical array (number of diodes)
    :type slice_size:   integer

    :return:            True or spot size in pixels (if spot is closed) or False
    """
    y_bary, x_bary = barycenter(array, coordinates=True, slice_size=slice_size)
    __poisson(array, x_bary, y_bary)

    # Check if the Poisson Spot is closed. If the edges of a particle images are
    # colored with the Poisson Spot marker, the Spot cannot be closed.
    spot_is_closed = True

    for y in range(int(len(array) / slice_size)):
        if array[y * slice_size] == MARKER['poisson'] or array[y * slice_size + (slice_size - 1)] == MARKER['poisson']:
            spot_is_closed = False

    if spot_is_closed and check_y:
        for x in range(slice_size):
            if array[x] == MARKER['poisson']\
                    or array[int(((len(array) / slice_size) - 1) * slice_size + x)] == MARKER['poisson']:
                spot_is_closed = False

    # If the spot is not closed, the Poisson Spot marker gets deleted.
    if not spot_is_closed:
        for y in range(int(len(array) / slice_size)):
            for x in range(slice_size):
                if array[y * slice_size + x] == MARKER['poisson']:
                    array[y * slice_size + x] = 0

    # If sizing is True, the function returns the Poisson Spot diameter.
    if sizing and spot_is_closed:
        min_poisson_index = slice_size - 1
        max_poisson_index = 0
        for y in range(int(len(array) / slice_size)):
            for x in range(slice_size):
                if array[y * slice_size + x] == MARKER['poisson'] and x > max_poisson_index:
                    max_poisson_index = x
                if array[y * slice_size + x] == MARKER['poisson'] and x < min_poisson_index:
                    min_poisson_index = x
        poisson_diameter = max_poisson_index - min_poisson_index + 1
        poisson_diameter = poisson_diameter if poisson_diameter > 0 else 0
        return poisson_diameter
    else:
        if spot_is_closed:
            return True
        else:
            return False


import random
import os
import time
from fastapi import FastAPI, status, HTTPException
from fastapi.responses import RedirectResponse, JSONResponse, Response
from sqlalchemy import create_engine
from sqlalchemy.orm import declarative_base, sessionmaker
from sqlalchemy import Column, String
from  sqlalchemy.sql.expression import func
from pydantic import BaseModel

#Docker
#sudo apt install uvicorn
#sudo apt-get install build-dep python-psycopg2

baseurl = 'http://localhost:5000/'

###Create model for input
class UrlMappingRequest(BaseModel):
    url: str

###Initialize DB and ORM
engine = create_engine(
    "postgresql+psycopg2://fastapi:fastapi@postgresfastapi:5432/fastapi"
    , pool_size=50, max_overflow=0
)

Base = declarative_base()
class UrlMapping(Base):
     __tablename__ = "url_mappings"

     id = Column(String(10),  primary_key=True, index=True)
     url = Column(String, index=True)

     def __repr__(self):
         return "<UrlMapping(id='%s', url='%s')>" % (
             self.id,
             self.url
         )

while(True):
    try:
        time.sleep(1)
        Base.metadata.create_all(engine)
        break
    except(Exception):
        print("Connection to database failed. Retrying...")
Session = sessionmaker(bind=engine)
###

app = FastAPI()

@app.get('/')
async def index():
    with open(os.path.join('./index.html')) as fh:
        data = fh.read()
    return Response(content=data, media_type="text/html")

@app.post('/url')
async def shorten_url(item: UrlMappingRequest):
    url_to_shorten = item.url
    if not url_to_shorten or len(url_to_shorten)==0:
        raise HTTPException(
        status_code=status.HTTP_400_BAD_REQUEST,
        detail=f'Url not passed',
        )
    if not (url_to_shorten[0:8]=='https://' or url_to_shorten[0:7]=='http://'):
        raise HTTPException(
        status_code=status.HTTP_400_BAD_REQUEST,
        detail=f'Url must start with either http or https',
        )
    session = Session()
    url_mapping = session.query(UrlMapping).filter_by(url=url_to_shorten).first()
    if not url_mapping:
        shortened = random_string_10()
        session.add(UrlMapping(id=shortened, url=url_to_shorten))
        session.commit()
        session.close()
        return JSONResponse(content={ 'url': url_to_shorten, 'shortened' :  baseurl+shortened}, status_code=status.HTTP_201_CREATED)
    else:
        shortened = url_mapping.id
        session.commit()
        session.close()
        return JSONResponse(content={ 'url': url_to_shorten, 'shortened' :  baseurl+shortened}, status_code=status.HTTP_200_OK)

@app.get('/random')
async def random_url():
    session = Session()
    url_mapping = session.query(UrlMapping).order_by(func.random()).limit(1).first()
    if not url_mapping:
        session.commit()
        session.close()
        raise HTTPException(
        status_code=status.HTTP_404_NOT_FOUND,
        detail=f'Url not found',
        )
    else:
        url = url_mapping.url
        session.commit()
        session.close()
        return RedirectResponse(url, status_code=status.HTTP_302_FOUND)

@app.get('/{shortened_url}')
async def redirect_url(shortened_url):
    session = Session()
    url_mapping = session.query(UrlMapping).filter(UrlMapping.id==str(shortened_url)).first()
    if not url_mapping:
        session.commit()
        session.close()
        raise HTTPException(
        status_code=status.HTTP_404_NOT_FOUND,
        detail=f'Url not found',
        )
    else:
        url = url_mapping.url
        session.commit()
        session.close()
        return RedirectResponse(url, status_code=status.HTTP_302_FOUND)

#Should give us 62^10 possible combinations
def random_string_10():
    chars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
    return ''.join(random.choice(chars) for i in range(10))


"""
Runs a full AC continuation power flow.
"""
# --- run cpf ---
import logging

import numpy as np
from numpy import flatnonzero as find

import scipy.sparse as sp  # NOQA

from andes.shared import deg2rad  # NOQA
from andes.utils.misc import elapsed  # NOQA

import ams.pypower.utils as putil  # NOQA
import ams.pypower.io as pio  # NOQA
import ams.pypower.routines.opffcns as opfcn  # NOQA
from ams.pypower.idx import IDX  # NOQA
from ams.pypower.make import (makeSbus, makeYbus, dSbus_dV)  # NOQA
from ams.pypower.routines.pflow import newtonpf, pfsoln  # NOQA
from ams.pypower.core import ppoption  # NOQA
import ams.pypower.routines.cpf_callbacks as cpf_callbacks  # NOQA

logger = logging.getLogger(__name__)

def runcpf(casedata, ppopt=None, scale=1.2):
    """
    Runs a full AC continuation power flow.
    """
    ppopt = ppoption(ppopt)

    # options
    verbose = ppopt["VERBOSE"]
    step = ppopt["CPF_STEP"]
    parameterization = ppopt["CPF_PARAMETERIZATION"]
    adapt_step = ppopt["CPF_ADAPT_STEP"]
    cb_args = ppopt["CPF_USER_CALLBACK_ARGS"]

    # set up callbacks
    callback_names = ["cpf_default_callback"]
    if len(ppopt["CPF_USER_CALLBACK"]) > 0:
        if isinstance(ppopt["CPF_USER_CALLBACK"], list):
            callback_names = np.r_[callback_names, ppopt["CPF_USER_CALLBACK"]]
        else:
            callback_names.append(ppopt["CPF_USER_CALLBACK"])
    callbacks = []
    for callback_name in callback_names:
        callbacks.append(getattr(cpf_callbacks, callback_name))

    # read base case data
    ppcbase = pio.loadcase(casedata)
    nb = ppcbase["bus"].shape[0]

    # add zero columns to branch for flows if needed
    if ppcbase["branch"].shape[1] < IDX.branch.QT:
        ppcbase["branch"] = np.c_[ppcbase["branch"],
                                  np.zeros((ppcbase["branch"].shape[0],
                                            IDX.branch.QT - ppcbase["branch"].shape[1] + 1))]

    # convert to internal indexing
    ppcbase = opfcn.ext2int(ppcbase)
    baseMVAb, busb, genb, branchb = \
        ppcbase["baseMVA"], ppcbase["bus"], ppcbase["gen"], ppcbase["branch"]

    # get bus index lists of each type of bus
    ref, pv, pq = putil.bustypes(busb, genb)

    # generator info
    onb = find(genb[:, IDX.gen.GEN_STATUS] > 0)  # which generators are on?
    gbusb = genb[onb, IDX.gen.GEN_BUS].astype(int)  # what buses are they at?

    # scale the load and generation of base case as target case
    ppctarget = pio.loadcase(casedata)
    ppctarget["bus"][:, IDX.bus.PD] *= scale
    ppctarget["bus"][:, IDX.bus.QD] *= scale
    ppctarget["gen"][:, IDX.gen.PG] *= scale
    ppctarget["gen"][:, IDX.gen.QG] *= scale

    # add zero columns to branch for flows if needed
    if ppctarget["branch"].shape[1] < IDX.branch.QT:
        ppctarget["branch"] = np.c_[ppctarget["branch"],
                                    np.zeros((ppctarget["branch"].shape[0],
                                              IDX.branch.QT - ppctarget["branch"].shape[1] + 1))]

    # convert to internal indexing
    ppctarget = opfcn.ext2int(ppctarget)
    baseMVAt, bust, gent, brancht = \
        ppctarget["baseMVA"], ppctarget["bus"], ppctarget["gen"], ppctarget["branch"]

    # get bus index lists of each type of bus
    # ref, pv, pq = putil.bustypes(bust, gent)

    # generator info
    ont = find(gent[:, IDX.gen.GEN_STATUS] > 0)  # which generators are on?
    # gbust = gent[ont, IDX.gen.GEN_BUS].astype(int)  # what buses are they at?

    # -----  run the power flow  -----
    t0, _ = elapsed()  # start timer

    # initial state
    # V0    = np.ones((bus.shape[0])            ## flat start
    V0 = busb[:, IDX.bus.VM] * np.exp((1j * deg2rad * busb[:, IDX.bus.VA]))
    vcb = np.ones(V0.shape)    # create mask of voltage-controlled buses
    vcb[pq] = 0     # exclude PQ buses
    k = find(vcb[gbusb])    # in-service gens at v-c buses
    V0[gbusb[k]] = genb[onb[k], IDX.gen.VG] / abs(V0[gbusb[k]]) * V0[gbusb[k]]

    # build admittance matrices
    Ybus, Yf, Yt = makeYbus(baseMVAb, busb, branchb)

    # compute base case complex bus power injections (generation - load)
    Sbusb = makeSbus(baseMVAb, busb, genb)
    # compute target case complex bus power injections (generation - load)
    Sbust = makeSbus(baseMVAt, bust, gent)

    # scheduled transfer
    Sxfr = Sbust - Sbusb

    # Run the base case power flow solution
    lam = 0
    logger.info('Solve base case power flow')
    V, success, iterations = newtonpf(Ybus, Sbusb, V0, ref, pv, pq, ppopt)

    logger.info('Start full AC CPF')
    logger.info('%2d: lambda = %6.3f, %2d Newton steps' % (0, 0, iterations))

    lamprv = lam    # lam at previous step
    Vprv = V    # V at previous step
    continuation = 1
    cont_steps = 0

    # input args for callbacks
    cb_data = dict(ppc_base=ppcbase, ppc_target=ppctarget,
                   Sxfr=Sxfr, Ybus=Ybus, Yf=Yf, Yt=Yt,
                   ref=ref, pv=pv, pq=pq, ppopt=ppopt)
    cb_state = {}

    # invoke callbacks
    for k in range(len(callbacks)):
        cb_state, _ = callbacks[k](cont_steps, V, lam, V, lam,
                                   cb_data, cb_state, cb_args)

    if np.linalg.norm(Sxfr) == 0:
        logger.error('base case and target case have identical load and generation!')

        continuation = 0
        V0 = V
        lam0 = lam

    # tangent predictor z = [dx;dlam]
    z = np.zeros(2*len(V)+1)
    z[-1] = 1.0
    while continuation:
        cont_steps = cont_steps + 1
        # prediction for next step
        V0, lam0, z = cpf_predictor(V, lam, Ybus, Sxfr, pv, pq, step, z,
                                    Vprv, lamprv, parameterization)

        # save previous voltage, lambda before updating
        Vprv = V
        lamprv = lam

        # correction
        V, success, i, lam = cpf_corrector(Ybus, Sbusb, V0, ref, pv, pq,
                                           lam0, Sxfr, Vprv, lamprv, z,
                                           step, parameterization, ppopt)

        if not success:
            continuation = 0
            logger.warning('%2d: lambda = %6.3f, corrector did not converge in %d iterations' % (
                cont_steps, lam, i))
            break

        logger.info('%2d: lambda = %6.3f, %2d corrector Newton steps' %
                    (cont_steps, lam, i))

        # invoke callbacks
        for k in range(len(callbacks)):
            cb_state, _ = callbacks[k](cont_steps, V, lam, V0, lam0,
                                       cb_data, cb_state, cb_args)

        if isinstance(ppopt["CPF_STOP_AT"], str):
            if ppopt["CPF_STOP_AT"].upper() == "FULL":
                if abs(lam) < 1e-8:     # traced the full continuation curve
                    logger.info('Reached steady state loading limit in %d continuation steps' % cont_steps)
                    continuation = 0
                elif lam < lamprv and lam - step < 0:    # next step will overshoot
                    step = lam      # modify step-size
                    parameterization = 1    # change to natural parameterization
                    adapt_step = False      # disable step-adaptivity

            else:   # == 'NOSE'
                if lam < lamprv:    # reached the nose point
                    logger.info('Reached steady state loading limit in %d continuation steps' % cont_steps)
                    continuation = 0

        else:
            if lam < lamprv:
                logger.info('Reached steady state loading limit in %d continuation steps' % cont_steps)
                continuation = 0
            elif abs(ppopt["CPF_STOP_AT"] - lam) < 1e-8:     # reached desired lambda
                logger.info('Reached desired lambda %3.2f in %d continuation steps' % (
                    ppopt["CPF_STOP_AT"], cont_steps))
                continuation = 0
            # will reach desired lambda in next step
            elif lam + step > ppopt["CPF_STOP_AT"]:
                step = ppopt["CPF_STOP_AT"] - lam   # modify step-size
                parameterization = 1    # change to natural parameterization
                adapt_step = False      # disable step-adaptivity

        if adapt_step and continuation:
            pvpq = np.r_[pv, pq]
            # Adapt stepsize
            cpf_error = np.linalg.norm(np.r_[np.angle(V[pq]), abs(
                V[pvpq]), lam] - np.r_[np.angle(V0[pq]), abs(V0[pvpq]), lam0], np.Inf)
            if cpf_error < ppopt["CPF_ERROR_TOL"]:
                # Increase stepsize
                step = step * ppopt["CPF_ERROR_TOL"] / cpf_error
                if step > ppopt["CPF_STEP_MAX"]:
                    step = ppopt["CPF_STEP_MAX"]
            else:
                # decrese stepsize
                step = step * ppopt["CPF_ERROR_TOL"] / cpf_error
                if step < ppopt["CPF_STEP_MIN"]:
                    step = ppopt["CPF_STEP_MIN"]

    # invoke callbacks
    if success:
        cpf_results = {}
        for k in range(len(callbacks)):
            cb_state, cpf_results = callbacks[k](cont_steps, V, lam, V0, lam0,
                                                 cb_data, cb_state, cb_args, results=cpf_results, is_final=True)
    else:
        cpf_results = {}
        cpf_results["iterations"] = i

    # update bus and gen matrices to reflect the loading and generation
    # at the noise point
    bust[:, IDX.bus.PD] = busb[:, IDX.bus.PD] + lam * (bust[:, IDX.bus.PD] - busb[:, IDX.bus.PD])
    bust[:, IDX.bus.QD] = busb[:, IDX.bus.QD] + lam * (bust[:, IDX.bus.QD] - busb[:, IDX.bus.QD])
    gent[:, IDX.gen.PG] = genb[:, IDX.gen.PG] + lam * (gent[:, IDX.gen.PG] - genb[:, IDX.gen.PG])

    # update data matrices with solution
    bust, gent, brancht = pfsoln(baseMVAt, bust, gent, brancht,
                                 Ybus, Yf, Yt, V, ref, pv, pq)

    ppctarget["et"] = elapsed(t0)
    ppctarget["success"] = success

    # -----  output results  -----
    # convert back to original bus numbering & print results
    ppctarget["bus"], ppctarget["gen"], ppctarget["branch"] = bust, gent, brancht
    if success:
        n = cpf_results["iterations"] + 1
        cpf_results["V_p"] = opfcn.i2e_data(
            ppctarget, cpf_results["V_p"], np.full((nb, n), np.NaN), "bus", 0)
        cpf_results["V_c"] = opfcn.i2e_data(
            ppctarget, cpf_results["V_c"], np.full((nb, n), np.NaN), "bus", 0)
    results = opfcn.int2ext(ppctarget)
    results["cpf"] = cpf_results

    # zero out result fields of out-of-service gens & branches
    if len(results["order"]["gen"]["status"]["off"]) > 0:
        results["gen"][np.ix_(results["order"]["gen"]
                              ["status"]["off"], [IDX.gen.QF, IDX.gen.QG])] = 0

    if len(results["order"]["branch"]["status"]["off"]) > 0:
        results["branch"][np.ix_(results["order"]["branch"]
                                 ["status"]["off"], [IDX.branch.PF, IDX.branch.QF, IDX.branch.PT, IDX.branch.QT])] = 0

    if ppopt["CPF_PLOT_LEVEL"]:
        import matplotlib.pyplot as plt
        plt.show()
    sstats = dict(solver_name='PYPOWER', num_iters=cpf_results["iterations"])
    return results, success, sstats

def cpf_predictor(V, lam, Ybus, Sxfr, pv, pq,
                  step, z, Vprv, lamprv, parameterization):
    # sizes
    pvpq = np.r_[pv, pq]
    nb = len(V)
    npv = len(pv)
    npq = len(pq)

    # compute Jacobian for the power flow equations
    dSbus_dVm, dSbus_dVa = dSbus_dV(Ybus, V)

    j11 = dSbus_dVa[np.array([pvpq]).T, pvpq].real
    j12 = dSbus_dVm[np.array([pvpq]).T, pq].real
    j21 = dSbus_dVa[np.array([pq]).T, pvpq].imag
    j22 = dSbus_dVm[np.array([pq]).T, pq].imag

    J = sp.vstack([
        sp.hstack([j11, j12]),
        sp.hstack([j21, j22])
    ], format="csr")

    dF_dlam = -np.r_[Sxfr[pvpq].real, Sxfr[pq].imag].reshape((-1, 1))
    dP_dV, dP_dlam = cpf_p_jac(parameterization, z, V, lam, Vprv, lamprv, pv, pq)

    # linear operator for computing the tangent predictor
    J = sp.vstack([
        sp.hstack([J, dF_dlam]),
        sp.hstack([dP_dV, dP_dlam])
    ], format="csr")

    Vaprv = np.angle(V)
    Vmprv = abs(V)

    # compute normalized tangent predictor
    s = np.zeros(npv+2*npq+1)
    s[-1] = 1
    z[np.r_[pvpq, nb+pq, 2*nb]] = sp.linalg.spsolve(J, s)
    z = z / np.linalg.norm(z)

    Va0 = Vaprv
    Vm0 = Vmprv
    lam0 = lam

    # prediction for next step
    Va0[pvpq] = Vaprv[pvpq] + step * z[pvpq]
    Vm0[pq] = Vmprv[pq] + step * z[nb+pq]
    lam0 = lam + step * z[2*nb]
    V0 = Vm0 * np.exp((1j * Va0))

    return V0, lam0, z

def cpf_corrector(Ybus, Sbus, V0, ref, pv, pq,
                  lam0, Sxfr, Vprv, lamprv, z,
                  step, parameterization, ppopt):

    # default arguments
    if ppopt is None:
        ppopt = ppoption(ppopt)

    # options
    verbose = ppopt["VERBOSE"]
    tol = ppopt["PF_TOL"]
    max_it = ppopt["PF_MAX_IT"]

    # initialize
    converged = 0
    i = 0
    V = V0
    Va = np.angle(V)
    Vm = abs(V)
    lam = lam0

    # set up indexing for updating V
    pvpq = np.r_[pv, pq]
    npv = len(pv)
    npq = len(pq)
    nb = len(V)
    j1 = 0
    j2 = npv    # j1:j2 - V angle of pv buses
    j3 = j2
    j4 = j2 + npq   # j1:j2 - V angle of pv buses
    j5 = j4
    j6 = j4 + npq   # j5:j6 - V mag of pq buses
    j7 = j6
    j8 = j6 + 1    # j7:j8 - lambda

    # evaluate F(x0, lam0), including Sxfr transfer/loading

    mis = V * np.conj(Ybus.dot(V)) - Sbus - lam*Sxfr
    F = np.r_[mis[pvpq].real, mis[pq].imag]

    # evaluate P(x0, lambda0)
    P = cpf_p(parameterization, step, z, V, lam, Vprv, lamprv, pv, pq)

    # augment F(x, lambda) with P(x, lambda)
    F = np.r_[F, P]

    # check tolerance
    normF = np.linalg.norm(F, np.Inf)
    logger.debug('CPF correction')
    logger.debug('%2d: |F(x)| = %.10g', i, normF)
    if normF < tol:
        converged = 1

    # do Newton iterations
    while not converged and i < max_it:
        # update iteration counter
        i = i + 1

        # evaluate Jacobian
        dSbus_dVm, dSbus_dVa = dSbus_dV(Ybus, V)

        j11 = dSbus_dVa[np.array([pvpq]).T, pvpq].real
        j12 = dSbus_dVm[np.array([pvpq]).T, pq].real
        j21 = dSbus_dVa[np.array([pq]).T, pvpq].imag
        j22 = dSbus_dVm[np.array([pq]).T, pq].imag

        J = sp.vstack([
            sp.hstack([j11, j12]),
            sp.hstack([j21, j22])
        ], format="csr")

        dF_dlam = -np.r_[Sxfr[pvpq].real, Sxfr[pq].imag].reshape((-1, 1))
        dP_dV, dP_dlam = cpf_p_jac(parameterization, z, V, lam, Vprv, lamprv, pv, pq)

        # augment J with real/imag -Sxfr and z^T
        J = sp.vstack([
            sp.hstack([J, dF_dlam]),
            sp.hstack([dP_dV, dP_dlam])
        ], format="csr")

        # compute update step
        dx = -1 * sp.linalg.spsolve(J, F)

        # update voltage
        if npv:
            Va[pv] = Va[pv] + dx[j1:j2]
        if npq:
            Va[pq] = Va[pq] + dx[j3:j4]
            Vm[pq] = Vm[pq] + dx[j5:j6]
        V = Vm * np.exp((1j * Va))
        Vm = abs(V)
        Va = np.angle(V)

        # update lambda
        lam = lam + dx[j7:j8]

        # evalute F(x, lam)
        mis = V * np.conj(Ybus.dot(V)) - Sbus - lam*Sxfr
        F = np.r_[mis[pv].real, mis[pq].real, mis[pq].imag]

        # evaluate P(x, lambda)
        P = cpf_p(parameterization, step, z, V, lam, Vprv, lamprv, pv, pq)

        # augment F(x, lambda) with P(x, lambda)
        F = np.r_[F, P]

        # check for convergence
        normF = np.linalg.norm(F, np.Inf)
        logger.debug('%2d: |F(x)| = %.10g', i, normF)

        if normF < tol:
            converged = 1

    if not converged:
        logger.error('Newton''s method corrector did not converge in %d iterations.' % i)

    return V, converged, i, lam

def cpf_p_jac(parameterization, z, V, lam, Vprv, lamprv, pv, pq):
    if parameterization == 1:
        npv = len(pv)
        npq = len(pq)
        dP_dV = np.zeros(npv+2*npq)
        if lam >= lamprv:
            dP_dlam = 1.0
        else:
            dP_dlam = -1.0

    elif parameterization == 2:
        pvpq = np.r_[pv, pq]

        Va = np.angle(V)
        Vm = abs(V)
        Vaprv = np.angle(Vprv)
        Vmprv = abs(Vprv)
        dP_dV = 2 * (np.r_[Va[pvpq], Vm[pq]] -
                     np.r_[Vaprv[pvpq], Vmprv[pq]])
        if lam == lamprv:
            dP_dlam = 1.0
        else:
            dP_dlam = 2 * (lam - lamprv)

    elif parameterization == 3:
        nb = len(V)
        dP_dV = z[np.r_[pv, pq, nb+pq]]
        dP_dlam = z[2 * nb]

    return dP_dV, dP_dlam

def cpf_p(parameterization, step, z, V, lam, Vprv, lamprv, pv, pq):
    # evaluate P(x0, lambda0)
    if parameterization == 1:
        if lam >= lamprv:
            P = lam - lamprv - step
        else:
            P = lamprv - lam - step

    elif parameterization == 2:
        pvpq = np.r_[pv, pq]

        Va = np.angle(V)
        Vm = abs(V)
        Vaprv = np.angle(Vprv)
        Vmprv = abs(Vprv)
        P = sum(np.square(np.r_[Va[pvpq], Vm[pq], lam] -
                          np.r_[Vaprv[pvpq], Vmprv[pq], lamprv])) - np.square(step)

    elif parameterization == 3:
        pvpq = np.r_[pv, pq]

        nb = len(V)
        Va = np.angle(V)
        Vm = abs(V)
        Vaprv = np.angle(Vprv)
        Vmprv = abs(Vprv)
        P = np.dot(z[np.r_[pvpq, nb+pq, 2*nb]].reshape((1, -1)),
                   np.array(np.r_[Va[pvpq], Vm[pq], lam] - np.r_[Vaprv[pvpq], Vmprv[pq], lamprv]).T) - step

    return P


import time
from selenium import webdriver
from selenium.webdriver.common.by import By
import math

link = 'http://suninjuly.github.io/alert_accept.html'
browser = webdriver.Chrome()
browser.get(link)
def calc(x):
    return str(math.log(abs(12 * math.sin(int(x)))))

try:
    button = browser.find_element(By.CLASS_NAME, 'btn.btn-primary')
    button.click()
    time.sleep(3)

    confirm = browser.switch_to.alert
    confirm.accept()

    x_element = browser.find_element(By.ID, 'input_value')
    x = x_element.text
    result = calc(x)

    input1 = browser.find_element(By.ID, 'answer')
    input1.send_keys(result)

    button2 = browser.find_element(By.CSS_SELECTOR, 'button.btn')
    button2.click()

except Exception as error:
    print(f'ошибка - {error}')
finally:
    time.sleep(10)
    browser.quit()


from subprocess import IDLE_PRIORITY_CLASS
import pynput
import time
import pyperclip
import pyautogui
from colorama import Fore, Style,Back
#################################################################################################################
# we will create four shortcut keys for each entry
#id->ctrl + [
#title->ctrl + ]
#url->ctrl+ alt +;
#html-text->ctrl + m
#############################################################################################
def id_copyer():  #id
    print(Fore.GREEN+" ID Pasted-"+str(sample[0]))
    paste_it(sample[0])

def title_copyer():  #title
    print(Fore.GREEN+"Title Pasted-"+str(sample[2]))

    paste_it(sample[2])

def url_copyer():   # url
   print(Fore.GREEN+"URL  Pasted-"+str(sample[3]))
   paste_it(sample[3])

def html_copyer():  # html_text
    print(Fore.GREEN+"Html Data  Pasted -CAN'T PREVIEW ")
    paste_it(sample[1])

def paste_it(data_to_p):
    pyperclip.copy(data_to_p)
    pyautogui.hotkey("ctrl","v")

#########################################################################################

def exiter():
 print(Fore.GREEN+"thx for running me".center(80))
 time.sleep(3)
 exit()

######################################################################################

def log_keys(quant):
  global sample
  sample=quant
  with pynput.keyboard.GlobalHotKeys(
    {"<ctrl>+[" :id_copyer,  # id
    "<ctrl>+]":title_copyer,
    "<ctrl>+;":url_copyer,
    "<ctrl>+m":html_copyer,
    "<ctrl>+'":exiter,
     }) as listener:
    listener.join()

###############################################################################################

if __name__=="__main__":
    sample= []  # sample data removed for privacy concerns
    log_keys(sample)

#Done


#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import json
from collections import Counter
import matplotlib.pyplot as plt

def json_read(file_path, encoding="uft-8"):
    """ 读取json文件解析为dict返回.
    """
    with open(file_path, 'r', encoding=encoding) as f:
        return json.load(f)

def dict_save(mdict, save_path="dict.json"):
    """ 将词典保存至json文件中

        Args:
          mdict: 待保存词典
          save_path: 词典将保存在该路径下
    """
    with open(save_path, 'w', encoding="utf-8") as f:
        json.dump(mdict, f)
        print("Dict is saved in -- {0}.".format(save_path))

class HMMDataLoader:
    def __init__(self, data_path):
        # self.status2num = {'B': 0, 'M': 1, 'E': 2, 'S': 3}
        self.data_path = data_path
        self.vocab_dict = {}
        self.corpus = []

    def generate_vocab_dict(self, encoding='utf-8'):
        """
        统计语料库，获取词表
        :param encoding: 编码方式
        :return: 词表字典
        """
        with open(self.data_path, 'r', encoding=encoding) as f:
            print("Start generate vocab dict...")

            for line in f.readlines():
                for word in line.strip().split():
                    self.vocab_dict[word] = self.vocab_dict.get(word, 0) + 1
            count = len(self.vocab_dict)
            self.vocab_dict['_total_'] = count
            print("Finished. Total number of words: {0}".format(count))

        return self.vocab_dict

    def generate_corpus_status(self, encoding="utf-8"):
        """ 将原始分词数据集处理成BMES标记的数据集.

            返回格式[[('你','B'),..],[..],..]
        """
        with open(self.data_path, 'r', encoding=encoding) as f:
            for line in f:
                l = []
                for word in line.strip().split():
                    if len(word) == 1:
                        l.append((word[0], 'S'))
                        continue
                    for i in range(len(word)):
                        if i == 0:
                            l.append((word[i], 'B'))
                        elif i == len(word) - 1:
                            l.append((word[i], 'E'))
                        else:
                            l.append((word[i], 'M'))
                self.corpus.append(l)
        return self.corpus

    def index_corpus(self):
        """ 将数据集进行编码

            Args:
              corpus: 格式必须为[[(obsv, hide), (obsv,hide),...], ...]

            Returns:
              idxed_corpus: 编码后的corpus,格式不变
              (obsv2idx, idx2obsv): 两个dict用于观察值与其编码之间的转换
              (hide2idx, idx2hide): 两个dict用于隐藏值与其编码之间的转换
        """
        obsv2idx, idx2obsv = {'unk': 0}, {0: 'unk'}
        hide2idx, idx2hide = {}, {}
        obsv_idx, hide_idx = 1, 0

        # build dictionaries and indexing
        idxed_corpus = []
        for seq in self.generate_corpus_status():
            idxed_seq = []
            for obsv, hide in seq:
                if obsv not in obsv2idx.keys():
                    obsv2idx[obsv] = obsv_idx
                    idx2obsv[obsv_idx] = obsv
                    obsv_idx += 1
                if hide not in hide2idx.keys():
                    hide2idx[hide] = hide_idx
                    idx2hide[hide_idx] = hide
                    hide_idx += 1
                # indexing
                idxed_seq.append((obsv2idx[obsv], hide2idx[hide]))
            idxed_corpus.append(idxed_seq)

        return idxed_corpus, (obsv2idx, idx2obsv), (hide2idx, idx2hide)

    def draw_zipf(self):
        vocab_dict = self.generate_vocab_dict()
        del vocab_dict["_total_"]
        counter = Counter(vocab_dict.values())
        counter_top = counter.most_common()
        plt.figure('Word frequent')
        plt.rc('font', family='SimHei', size=13)
        y = list(map(lambda y: y[1], counter_top[:]))
        plt.xlabel('rank')
        plt.ylabel('frequency')
        plt.plot(range(len(y)), y)
        plt.scatter(range(len(y)), y, s=2, c='red')
        plt.title('Word frequency vs rank')
        plt.show()

    def draw_top10_hist(self):
        vocab_dict = self.generate_vocab_dict()
        del vocab_dict["_total_"]
        counter = Counter(vocab_dict)
        counter_top = counter.most_common(10)
        x = list(map(lambda x: x[1], counter_top[:]))
        y = list(map(lambda y: y[0], counter_top[:]))
        plt.rc('font', family='SimHei', size=13)
        plt.barh(range(10), x, height=0.7, color='steelblue', alpha=0.8)  # 从下往上画
        plt.yticks(range(10), y)
        plt.xlabel("频数")
        plt.title("频率前10的词语")
        for x, y in enumerate(x):
            plt.text(y + 0.2, x - 0.1, '%s' % y)
        plt.show()

# 统计语料词表
# data_path = "../seg-data/training/pku_training.utf8"
# data_path = "../seg-data/training/msr_training.utf8"
# save_path = "./dict/pku_vocab_dict.json"
# save_path = "./dict/msr_vocab_dict.json"
# hmm_dataloader = HMMDataLoader(data_path)
# dict_save(hmm_dataloader.generate_vocab_dict(), save_path)
# idxed_corpus, (obsv2idx, idx2obsv), (hide2idx, idx2hide) = hmm_dataloader.index_corpus()
# print(len(idxed_corpus))
# print(len(obsv2idx))
# print(len(hide2idx))
# print(idxed_corpus[:10])

# vocab_dict = json_read(save_path)

# 数据集统计分析图
# hmm_dataloader.draw_zipf()
# hmm_dataloader.draw_top10_hist()


#!/usr/bin/env python3

import json
import sys

offset = float(sys.argv[1])

for raw_line in sys.stdin:
    line = json.loads(raw_line)
    if isinstance(line, dict):
        print(raw_line, end="")
        continue
    updated_timestamp = line[0] - offset
    if updated_timestamp > 0:
        line[0] = updated_timestamp
    print(json.dumps(line))


# -*- coding: utf-8 -*-
"""
Created on Thu Jan 11 14:23:21 2018

@author: Abhimanyu
"""

# -*- coding: utf-8 -*-
"""
Created on Wed Jan 10 16:40:14 2018

@author: Abhimanyu
"""

# -*- coding: utf-8 -*-
"""
Created on Fri Dec 29 15:01:41 2017

@author: Abhimanyu
"""

""" Trains an agent with (stochastic) Policy Gradients on Pong. Uses OpenAI Gym. """
import numpy as np
import pickle
import gym
import tensorflow as tf
import itertools
import matplotlib
import os
os.chdir(r'C:\Users\Abhimanyu\Documents\Reinforcement_Learning\Progs')
import plotting
import collections

matplotlib.style.use('ggplot')
# hyperparameters
H = 200 # number of hidden layer neurons
batch_size = 10 # every how many episodes to do a param update?
learning_rate = 1e-3
gamma = 0.99 # discount factor for reward
decay_rate = 0.99 # decay factor for RMSProp leaky sum of grad^2
resume = False # resume from previous checkpoint?
render = True

# model initialization
D = 80 * 80 # input dimensionality: 80x80 grid

def sigmoid(x):
  return 1.0 / (1.0 + np.exp(-x)) # sigmoid "squashing" function to interval [0,1]

def prepro(I):
  """ prepro 210x160x3 uint8 frame into 6400 (80x80) 1D float vector """
  I = I[35:195] # crop
  I = I[::2,::2,0] # downsample by factor of 2
  I[I == 144] = 0 # erase background (background type 1)
  I[I == 109] = 0 # erase background (background type 2)
  I[I != 0] = 1 # everything else (paddles, ball) just set to 1
  return I.astype(np.float).ravel()

def discount_rewards(r):
  """ take 1D float array of rewards and compute discounted reward """
  discounted_r = np.zeros_like(r)
  running_add = 0
  for t in reversed(range(0, r.size)):
    if r[t] != 0: running_add = 0 # reset the sum, since this was a game boundary (pong specific!)
    running_add = running_add * gamma + r[t]
    discounted_r[t] = running_add
  return discounted_r

class PolicyEstimator():
    """
    Policy Function approximator.
    """

    def __init__(self, learning_rate=0.0003, scope="policy_estimator"):
        with tf.variable_scope(scope):
            self.state = tf.placeholder(tf.float32, [None,6400], "state")
            self.action = tf.placeholder(dtype=tf.int32,shape=[None], name="action")
            self.target = tf.placeholder(dtype=tf.float32, name="target")

            # This is just table lookup estimator

            self.hidden = tf.contrib.layers.fully_connected(
                inputs=self.state,
                num_outputs=200,
                activation_fn=tf.nn.relu)
            self.output_layer=tf.contrib.layers.fully_connected(
                inputs=self.hidden,
                num_outputs=2,
                activation_fn=tf.nn.softmax)
            self.oneHotAction=tf.one_hot(self.action,2)
            self.action_probs=self.output_layer
            self.action_probs2 = tf.reduce_sum(self.action_probs*self.oneHotAction,1)
            #self.picked_action_prob = tf.gather(self.action_probs, self.action)

            # Loss and train op
            self.loss = tf.reduce_mean(-tf.log(self.action_probs2) * self.target)

            self.optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
            self.train_op = self.optimizer.minimize(
                self.loss, global_step=tf.contrib.framework.get_global_step())

    def predict(self, state, sess=None):
        sess = sess or tf.get_default_session()
        return sess.run(self.action_probs, { self.state: np.expand_dims(state,0) })

    def update(self, state, target, action, sess=None):
        sess = sess or tf.get_default_session()
        feed_dict = { self.state: state, self.target: target, self.action: action  }
        _, loss = sess.run([self.train_op, self.loss], feed_dict)
        return loss

env = gym.make("Pong-v0")

#import matplotlib
#matplotlib.pyplot.imshow(observation)
#observation, reward, done, info= env.step(1)
#matplotlib.pyplot.imshow(observation)
#observation, reward, done, info= env.step(2)
#matplotlib.pyplot.imshow(observation)
#observation, reward, done, info= env.step(3)
#

def policy_gradient(env, estimator_policy, num_episodes, discount_factor=1.0):
    """
    Actor Critic Algorithm. Optimizes the policy
    function approximator using policy gradient.

    Args:
        env: OpenAI environment.
        estimator_policy: Policy Function to be optimized
        estimator_value: Value function approximator, used as a baseline
        num_episodes: Number of episodes to run for
        discount_factor: Time-discount factor

    Returns:
        An EpisodeStats object with two numpy arrays for episode_lengths and episode_rewards.
    """
    running_reward=None
    # Keeps track of useful statistics
    stats = plotting.EpisodeStats(
        episode_lengths=np.zeros(num_episodes),
        episode_rewards=np.zeros(num_episodes))

    Transition = collections.namedtuple("Transition", ["state", "action", "reward", "next_state", "done"])

    for i_episode in range(num_episodes):
        # Reset the environment and pick the fisrst action
        state = env.reset()

        episode = []
        reward_ep=[]
        ap=[]
        s=[]
        a=[]
        prev_x=np.zeros(D)
        # One step in the environment
        prev_x = None
        cur_x = prepro(state)
        for t in itertools.count():
            if render: env.render()

            x = cur_x - prev_x if prev_x is not None else np.zeros(D)
            prev_x = cur_x
            # Take a step
            action_probs = estimator_policy.predict(x)
            #print(action_probs,"knsnka")
            action_probs=action_probs[0]
            action = np.random.choice(np.arange(len(action_probs)), p=action_probs)
            a.append(action)
            next_state, reward, done, _ = env.step(action+2)
            ap.append(action_probs)
            next_x=(prepro(next_state))
            # Keep track of the transition
            episode.append(Transition(
              state=x, action=action, reward=reward, next_state=(next_x-cur_x), done=done))
            s.append(x)
            reward_ep.append(reward)
            # Update statistics
            stats.episode_rewards[i_episode] += (reward)
            stats.episode_lengths[i_episode] = t

            if done:
                break

            cur_x = next_x
        print("\rStep {} @ Episode {}/{} ({})".format(
                    t, i_episode + 1, num_episodes, stats.episode_rewards[i_episode]), end="")
        discounted_epr = discount_rewards(np.array(reward_ep,dtype=float))
        discounted_epr-=np.mean(discounted_epr)
        discounted_epr/=np.std(discounted_epr)
        #print(discounted_epr)
        running_reward = stats.episode_rewards[i_episode] if running_reward is None else running_reward * 0.93 + stats.episode_rewards[i_episode] * 0.07
        print(running_reward)
        s=np.vstack(s)
        a=np.vstack(a)
        a=np.squeeze(a)
        #print(s.shape)
        #episode=np.array(episode)
#        for i,k in enumerate(episode):
##                matplotlib.pyplot.imshow(k[0].reshape(80,80))
##                matplotlib.pyplot.show()
#                #print(ap[i])
#                estimator_policy.update(k[0],discounted_epr[i],k[1])
        estimator_policy.update(s,discounted_epr,a)

    return stats

tf.reset_default_graph()

global_step = tf.Variable(0, name="global_step", trainable=False)
policy_estimator = PolicyEstimator()
saver = tf.train.Saver()
with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    # Note, due to randomness in the policy the number of episodes you need to learn a good
    # policy may vary. ~300 seemed to work well for me.
    stats = policy_gradient(env, policy_estimator, 1500)
    save_path = saver.save(sess, r"C:\Users\Abhimanyu\Documents\Reinforcement_Learning\Weights\model_new.ckpt")

#with tf.Session() as sess:
#    saver.restore(sess,r"C:\Users\Abhimanyu\Documents\Reinforcement_Learning\Weights\model_2.1.ckpt")
#    stats = policy_gradient(env, policy_estimator, 1500)
#    save_path = saver.save(sess, r"C:\Users\Abhimanyu\Documents\Reinforcement_Learning\Weights\model_3.ckpt")
#
#
plotting.plot_episode_stats(stats, smoothing_window=10)


# CSV 2 flare.json
# convert a csv file to flare.json for use with many D3.js viz's
# This script creates outputs a flare.json file with 2 levels of nesting.
# For additional nested layers, add them in lines 32 - 47
# sample: http://bl.ocks.org/mbostock/1283663
# csv2shapes script largely developed by andrew heekin, additions and corrections by eric d

import pandas as pd
import json

df = pd.read_csv('shapes.csv')

# choose columns to keep, in the desired nested json hierarchical order
df = df[["parent", "child", "value"]]

# order in the groupby here matters, it determines the json nesting
# the groupby call makes a pandas series by grouping 'the_parent' and 'the_child', while summing the numerical column 'child_size'
df1 = df.groupby(['parent', 'child'])['value'].sum()
df1 = df1.reset_index()

# start a new flare.json document
flare = dict()
d = {"name":"shape", "children": []}

for line in df1.values:
    parent = line[0]
    child = line[1]
    value = line[2]

    # make a list of keys
    keys_list = []
    for item in d['children']:
        keys_list.append(item['name'])

    # if 'the_parent' is NOT a key in the flare.json yet, append it
    if not parent in keys_list:
        d['children'].append({"name":parent, "children":[{"name":child, "size":value}]})

    # if 'the_parent' IS a key in the flare.json, add a new child to it
    else:
        d['children'][keys_list.index(parent)]['children'].append({"name":child, "size":value})

flare = d

# export the final result to a json file
with open('shapes.json', 'w') as outfile:
    json.dump(flare, outfile)


import os, sys
from PyQt5.QtWidgets import QMainWindow, QFileDialog
from gui.ui_mainwindow import Ui_MainWindow
from gui.hiokitab import HiokiTab
from gui.agilenttab import AgilentTab
from gui.normaltab import NormalTab

class MainWindow(QMainWindow):
    def __init__ (self, app, translator, parent=None):
        super(MainWindow, self).__init__(parent)

        self._app = app
        self._translator = translator
        app.installTranslator(translator)

        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        self._setup_ui()

    def _setup_ui(self):
        self._hioki_tab = HiokiTab(self.ui.tab)
        self._agilent_tab = AgilentTab(self.ui.tab_2)
        self._normal_tab = NormalTab(self.ui.tab_3)

    def openFile(self):
        '''
        print(__file__) # /Users/kakalin/csv/gui/mainwindow.py
        print (os.path.dirname(__file__)) # /Users/kakalin/csv/gui
        print (os.path.abspath(__file__)) # /Users/kakalin/csv/gui/mainwindow.py
        print (os.path.abspath(os.path.dirname(__file__))) # /Users/kakalin/csv/gui
        print (os.path.dirname(os.path.abspath(__file__))) # /Users/kakalin/csv/gui
        '''

        dir = os.path.dirname(__file__)

        file_name = QFileDialog.getOpenFileName(self, 'Open file', dir)
        print(file_name)


# import all the dependencies
import argparse
import os, glfw, copy, time
import numpy as np
import matplotlib.pyplot as plt
import mujoco, mujoco_viewer
from mujoco_utils import *
import pyspacemouse

from flexIk import *
from flexIk.velIkSolver import SolverTypes as SolverTypes

# connect to 3D mouse
pyspacemouse.close()
success = pyspacemouse.open()

# set numpy print options
np.set_printoptions(precision=2, suppress=True, linewidth=100)

print("MuJoCo version:[%s]" % (mujoco.__version__))

# set up the command-line interface
parser = argparse.ArgumentParser(description="Interactive IK using 3D mouse")
parser.add_argument(
    "robot", type=str, help="name of the robot model [spot, ur5e, panda]"
)
args = parser.parse_args()

# parse the selected robot model description file
selected_robot = args.robot

# make sure the path for the mujoco_menagerie is correct for your system
# https://github.com/deepmind/mujoco_menagerie
path_to_mujoco_menagerie = "/home/apark/Projects/robot-models/mujoco_menagerie/"

if selected_robot == "spot":
    xml_path = "./model/spot/scene.xml"
elif selected_robot == "ur5e":
    xml_path = path_to_mujoco_menagerie + "universal_robots_ur5e/scene.xml"
elif selected_robot == "panda":
    xml_path = path_to_mujoco_menagerie + "franka_emika_panda/scene.xml"

# get the environment and model
env = MuJoCoParserClass(name=selected_robot, rel_xml_path=xml_path, VERBOSE=True)
print("[" + selected_robot + "] parsed.")

# initialize viewer
env.init_viewer(
    viewer_title="IK using 3D mouse",
    viewer_width=1200,
    viewer_height=800,
    viewer_hide_menus=True,
)
env.update_viewer(azimuth=180, distance=3.84, elevation=-8.32, lookat=[0.02, 0.05, 0.7])
env.reset()

# Set IK target to the last body before the grippers
if selected_robot == "panda":
    ik_body_name = env.body_names[-3]
if selected_robot == "spot":
    ik_body_name = env.body_names[-2]
else:
    ik_body_name = env.body_names[-1]

# set joint angles
joint_idx_offset = 0
if selected_robot == "spot":
    base_joint_idxs = np.arange(0, 6)
    qBase = np.zeros(6)
    qBase[2] = 0.6
    qInit = np.zeros(env.n_rev_joint)
    # set the initial configuration for the legs
    qInit[0:12] = np.array([0, 30, -60] * 4) * np.pi / 180.0
    # set the initial configuration for the arm
    qInit[12:-1] = np.array([0, -45, 90, 0, -45, 0]) * np.pi / 180.0
    joint_idx_offset = 6

elif selected_robot == "ur5e":
    qInit = np.array((-90, -45, 90, -135, -90, 0)) * np.pi / 180.0
elif selected_robot == "panda":
    qInit = np.array((0, 25, 0, -120, 0, 145, 0)) * np.pi / 180.0

qCurrent = copy.deepcopy(qInit)

if selected_robot == "spot":
    # set initial configuration for base
    env.forward(q=qBase, joint_idxs=base_joint_idxs)

# set initial configuration for arm and gripper
env.forward(
    q=qCurrent, joint_idxs=env.rev_joint_idxs + joint_idx_offset
)  # Set joint angles
if selected_robot == "panda":
    # Open gripper (prismatic joints)
    env.forward(q=[-0.04, -0.04], joint_idxs=env.pri_joint_idxs)
    gripperState = False  # gripper state (True: closed, False: open)

# set the indices of the arm joints
if selected_robot == "spot":
    # arm_jnt_idxs = env.rev_joint_idxs[-1:-7:-1]
    jnt_idxs = env.rev_joint_idxs + joint_idx_offset - 1
else:
    jnt_idxs = env.rev_joint_idxs

# initialize the input for the flex Ik solver
numDof = env.n_rev_joint
C = np.eye(numDof)
I = np.eye(numDof)
qMin = env.rev_joint_min
qMax = env.rev_joint_max
qdMin = np.array([-2] * numDof)
qdMax = np.array([2] * numDof)
qddMax = np.array([1] * numDof)
ts = 5e-1
s = 1
sPrev = s
tol = 1e-4
errNormMax = 2e-1
kNs = 2e-3

# initialize 3D mouse input
state = pyspacemouse.read()
mouse_pos_init = copy.deepcopy(env.data.body(ik_body_name).xpos)
mouse_pos = copy.deepcopy(mouse_pos_init)
mouse_ori_init = copy.deepcopy(env.get_R_body(body_name=ik_body_name))
mouse_ori = copy.deepcopy(mouse_ori_init)
scale_pos_init = 1e-4
scale_ori_init = 2e-2

# display settings
display_target = True

# IK loop
imgs, img_ticks, max_tick = [], [], 100000
while (env.tick < max_tick) and env.is_viewer_alive():
    # Update the spacemouse input
    state = pyspacemouse.read()
    scale_pos = scale_pos_init
    scale_ori = scale_ori_init

    # update mouse pose only if the scale factor is increasing above the threshold
    mouse_pos += scale_pos * np.rad2deg(np.array([-state.y, state.x, state.z]))
    mouse_ori_temp = rpy2r(
        scale_ori * np.array((-state.roll, -state.pitch, -state.yaw))
    )
    mouse_ori = np.matmul(mouse_ori_temp, mouse_ori)

    # Reset if button 1 is pressed
    if state.buttons[0] and state.buttons[1]:  # reset to initial configuration
        qCurrent = copy.deepcopy(qInit)
        mouse_pos = copy.deepcopy(mouse_pos_init)
        mouse_ori = copy.deepcopy(mouse_ori_init)
        time.sleep(0.2)

    elif state.buttons[0] and not state.buttons[1]:
        mouse_pos = copy.deepcopy(env.get_p_body(ik_body_name))
        mouse_ori = copy.deepcopy(env.get_R_body(ik_body_name))
        time.sleep(0.2)

    # Flip gripper state (only for panda robot)
    elif state.buttons[1] and not state.buttons[0] and selected_robot == "panda":
        if not gripperState:
            env.forward(q=[0.04, 0.04], joint_idxs=env.pri_joint_idxs)
            gripperState = True
        else:
            env.forward(q=[-0.04, -0.04], joint_idxs=env.pri_joint_idxs)
            gripperState = False
        time.sleep(0.2)

    # update box constraints
    dqLow = np.maximum(
        (qMin - qCurrent) / ts,
        -qdMax,
        -np.sqrt(np.maximum(2 * qddMax * (qCurrent - qMin), 0)),
    )
    dqUpp = np.minimum(
        (qMax - qCurrent) / ts,
        qdMax,
        np.sqrt(np.maximum(2 * qddMax * (qMax - qCurrent), 0)),
    )

    # Numerical IK
    J, err = env.get_ik_ingredients(
        body_name=ik_body_name, p_trgt=mouse_pos, R_trgt=mouse_ori, IK_P=True, IK_R=True
    )

    # clamp error
    errNorm = np.linalg.norm(err)
    if errNorm > errNormMax:
        errClamped = (errNormMax / errNorm) * err
    else:
        errClamped = err

    # nullspace bias
    qdNs = kNs * (qInit - qCurrent)
    qdNs = qdNs.reshape(-1, 1)

    dxGoalData = [errClamped, qdNs]

    JData = [J[:, jnt_idxs], I]

    dq, sData, exitCode = velIk(
        C, dqLow, dqUpp, dxGoalData, JData, SolverTypes.ESNS_MT, invTypes.SRINV
    )
    s = sData[0]

    # Update q and FK
    qCurrent = qCurrent + dq * ts

    # normalized joint position [0, 1]
    qNormal = np.maximum((qMax - qCurrent), (qCurrent - qMin)) / (qMax - qMin)

    if env.tick % 50 == 0 and s < 0.4:
        if np.any(qNormal >= 0.99):
            print(qNormal)
        print(s)
        print(dq.T)

    # update model
    env.forward(q=qCurrent, joint_idxs=env.rev_joint_idxs + joint_idx_offset)

    # Render
    env.plot_T(
        p=env.get_p_body(body_name=ik_body_name),
        R=env.get_R_body(body_name=ik_body_name),
        PLOT_AXIS=True,
        axis_len=0.15,
        axis_width=0.005,
        PLOT_SPHERE=False,
        sphere_r=0.05,
        sphere_rgba=[1, 0, 0, 0.9],
    )
    if display_target:
        env.plot_T(
            p=mouse_pos,
            R=mouse_ori,
            PLOT_AXIS=True,
            axis_len=0.15,
            axis_width=0.005,
            PLOT_SPHERE=False,
            sphere_r=0.05,
            sphere_rgba=[0, 0, 1, 0.9],
        )
    env.plot_T(
        p=[0, 0, 0], R=np.eye(3, 3), PLOT_AXIS=True, axis_len=0.5, axis_width=0.01
    )
    env.render()

# Close viewer
env.close_viewer()
print("Done.")


from django.shortcuts import render
from django.urls import reverse_lazy
from .models import Course, Opinion, New, Slider, Urls, Video
from django.views.generic import DetailView, CreateView
from .forms import CourseForm, VideoForm

# Create your views here.

def index(request):
    slider = Slider.objects.all()

    return render(
        request,
        'index.html',
        {
            'slider': slider,
        }
    )

def about(request):
    aboutus_image = Slider.objects.first()
    opinion = Opinion.objects.all()

    return render(request, 'about.html', {'image': aboutus_image,})

def course(request):

    query = request.GET.get('query')

    where = {}
    if query:
        where['name__icontains'] = query

    course = Course.objects.filter(**where)

    return render(
        request,
        'course.html',
        {
            'courses': course
        }
    )

def course_list(request, pk):
    video = Video.objects.filter(course=pk).select_related('course')

    return render(
        request,
        'course_list.html',
        {
            'videos': video,
        }
    )

class CourseDetailView(DetailView):
    model = Video
    template_name = 'course_page.html'
    context_object_name = 'video'

class CourseCreateView(CreateView):
    model = Course
    form_class = CourseForm
    template_name = 'course/create.html'
    success_url = reverse_lazy('course')


from flask import Blueprint, render_template, request
from werkzeug.utils import secure_filename
from static.torch_model.u2net import U2NETP
import random
import os
import numpy
from static.torch_model.toolof_ocrdetect import *
import time
import base64

"""
切换CPU与GPU 需注释
GPU服务器还应是生产服务器选择的方案
模型加载放在接口设置之前
"""
# 注册蓝图
ocr_vn = Blueprint("ocr_vn", __name__)
# 设置允许上传的文件格式
ALLOW_EXTENSIONS = ['png', 'jpg', 'jpeg']
# 保存文件地址
UPLOAD_FOLDER_OCR_VN = './static/images/ocr_detect'
# 模型参数路径
pth_data_1 = './static/torch_model/ocr_detect_model.pth'
pth_data_2 = './static/torch_model/ocr_read_model.pth'
# 模型实例化
# model_1 = U2NETP().cpu()
# model_2 = U2NETP().cpu()
model_1 = U2NETP()
model_2 = U2NETP()
# 模型初始化
model_1.eval()
model_2.eval()
model_1.cuda()
model_2.cuda()
# model_1.load_state_dict(torch.load(pth_data_1, map_location=torch.device('cpu')))  # 将参数加载到模型里(CPU)
# model_2.load_state_dict(torch.load(pth_data_2, map_location=torch.device('cpu')))  # 将参数加载到模型里(CPU)
model_1.load_state_dict(torch.load(pth_data_1))
model_2.load_state_dict(torch.load(pth_data_2))

def after_request(resp):
    """
    跨域支持
    :param resp:响应
    :return:
    """
    resp.headers['Access-Control-Allow-Origin'] = '*'
    return resp

ocr_vn.after_request(after_request)

def allowed_file(filename):
    """
    # 判断文件后缀是否在列表中
    :param filename:
    :return:
    """
    return '.' in filename and filename.rsplit('.', 1)[-1] in ALLOW_EXTENSIONS

def contours_get(img0):
    """
    图片后处理
    :param img0:
    :return:
    """
    img0 = numpy.array(img0 * 255, numpy.uint8)
    # 转换为灰度图
    gray = cv2.cvtColor(img0, cv2.COLOR_BGR2GRAY)
    # 将图片二值化
    _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY)
    # 在二值图上寻找轮廓
    contours, _ = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    return img0, contours

def base64_cv2(base64_str):
    imgString = base64.b64decode(base64_str)
    nparr = np.fromstring(imgString, np.uint8)
    image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
    print(image.shape)
    return image

# 以下是接口设置
# OCR检测(POST请求)
@ocr_vn.route("/ocr_detect", methods=['POST', 'GET'])
def detect():
    """
    模型检测接口，正常图片识别返回状态码200，并返回识别结果
    传入图片不符合要求或者其他错误返回状态码500
    :return: json响应
    """
    if request.method == 'POST':
        # 获取post过来的文件名称，从name=file参数中获取
        start_time = time.time()
        # 从这开始接受base64
        # resParm = request.data
        # resParm = str(resParm, encoding="utf-8")
        # print(resParm)
        # resParm = eval(resParm)
        # base = resParm.get('base64')
        # image_test = base64_cv2(base)
        file = request.files['file']
        # 检测文件格式
        if file and allowed_file(file.filename):
            # secure_filename方法会去掉文件名中的中文
            try:
                file_name = secure_filename(file.filename)
                # 保存图片
                file.save(os.path.join(UPLOAD_FOLDER_OCR_VN, file_name))
                img_orgin_cv, pil_img = behind_img(os.path.join(UPLOAD_FOLDER_OCR_VN, file_name))
                # img_orgin_cv, pil_img = behind_img(image_test) # base64传入
                inputs_test = tf(pil_img, 416, 416)
                if torch.cuda.is_available():
                    inputs_test = inputs_test.cuda()
                # 将数据传如模型 得到7个tensor张量（1，1，416，416）
                pred0, pred1, pred2, pred3, pred4, pred5, pred6 = model_1(inputs_test)
                # 将张量反算成（3，416，416）
                pred_mask = torch.cat((pred0, pred1, pred2), 1)[0]
                pred_mask = pred_mask
                # 将张量转化为numpy格式
                pred_mask = pred_mask.cpu().data.numpy()
                # 将c，h，w转换成h，w，c
                pred_mask = np.transpose(pred_mask, (1, 2, 0))
                # cv2.imshow('pred',pred_mask)
                # cv2.waitKey(0)
                boxs = mask2value(pred_mask)  # 得到外接矩形boxs
                img_0, img_2, img_7, img_x = get_cut_img1(img_orgin_cv, boxs)  # 得到想得到的数字切片
                img_0 = img_trans(img_0, (120, 120))
                img_0 = img_0.cuda()
                pred_idnumber = model_2(img_0)
                img_2 = img_trans(img_2, (120, 120))
                img_2 = img_2.cuda()
                pred_birth = model_2(img_2)
                img_7 = img_trans(img_7, (120, 120))
                img_7 = img_7.cuda()
                pred_expire = model_2(img_7)
                # 身份证号码
                pred_idnumber = mask_read2value(ten2cv(pred_idnumber))
                print(pred_idnumber)
                # 生日日期
                pred_birth = mask_read2value(ten2cv(pred_birth))
                print(pred_birth)
                # 到期时间
                pred_expire = mask_read2value(ten2cv(pred_expire))
                print(pred_expire)
                # 之后更新新的识别的再加入这里
                # 三个值检测不到视为识别失败
                if pred_idnumber.strip() == '' and pred_expire.strip() == '' and pred_birth.strip() == '':
                    return {"code": '500',
                            'message': '识别失败',
                            'result': {}}
                end_time = time.time()
                cost = end_time - start_time
                print("TIME:", cost * 1000)
                # 保存检测图片
                # pred_mask1, contours = contours_get(pred_mask)
                # pred_mask = pred_mask1.copy()
                # for cont in contours:
                #     # 取轮廓长度的1%为epsilon
                #     epsilon = 0.005 * cv2.arcLength(cont, True)
                #     # 预测多边形
                #     box = cv2.approxPolyDP(cont, epsilon, True)
                #     img = cv2.polylines(pred_mask, [box], True, (0, 0, 255), 2)
                # cv2.drawContours(pred_mask, contours, -1, (100, 100, 100))
                # 检测后的图片保存（看上线需求）
                # imgpath = 'static/images/ocr_detect/{}{}'.format(random.randint(1000, 9999), file_name)
                # cv2.imwrite(imgpath, pred_mask1)
                return {"code": '200',
                        # "previewpath": 'static/images/ocr_detect/' + file_name,
                        # "resultpath": imgpath,
                        "message": "上传成功",
                        'result': {
                            'idnumber': pred_idnumber.strip(),
                            'birth': pred_birth.strip(),
                            'expire': pred_expire.strip(),
                            'name': "",
                            'place': '',
                            'address': "",
                            'sex': "",
                            'nation': ""},
                        }
            except(Exception):
                return {"code": '500',
                        'message': '识别失败',
                        'result': {}}
        else:
            return {"code": 404,
                    "message": "格式错误，仅支持jpg、png、jpeg格式文件",
                    "result": {}
                    }
        # 上线时注释
    else:
        return {"code": '503',
                "message": "仅支持post方法", "result": {}}
        # return render_template('ocr_vn.html')


import os
import pandas as pd
import re
import requests
import sys

CLOUDFLARE_ACCOUNT_ID = str(os.environ.get("CLOUDFLARE_ACCOUNT_ID"))
CLOUDFLARE_API_TOKEN = str(os.environ.get("CLOUDFLARE_API_TOKEN"))

bulk_url = (
    "https://api.cloudflare.com/client/v4/accounts/"
    + CLOUDFLARE_ACCOUNT_ID
    + "/intel/domain/bulk"
)
headers = {
    "Content-Type": "application/json",
    "Authorization": "Bearer " + CLOUDFLARE_API_TOKEN,
}

if __name__ == "__main__":
    if sys.argv[1] == "cloudflare_csv_header":
        print("domain\tcloudflare_id")

    elif sys.argv[1] == "crux_top":
        """
        Generate domain list of top domains according to CrUX
        """
        top = int(sys.argv[2])
        crux_path = sys.argv[3]
        output_path = sys.argv[4]
        # CrUX
        crux = pd.read_csv(crux_path, sep=",")
        crux = crux.head(top)
        # Regex to remove http(s)://
        crux["origin"] = crux.origin.apply(lambda x: re.sub("https?:\/\/", "", x))
        crux.origin.to_csv(output_path, index=False, header=False)

    elif sys.argv[1] == "api_request":
        """
        Call Cloudflare API to classify bulk domains
        """
        domains = sys.argv[2:]
        first_domain = True
        for domain in domains:
            if first_domain:
                bulk_url += "?domain=" + domain
                first_domain = False
            else:
                bulk_url += "&domain=" + domain
        response = requests.request("GET", bulk_url, headers=headers)

        if response.status_code == 200:
            # parse json response
            json_cloudflare = response.json()
            for result in json_cloudflare["result"]:
                # No category returned by Cloudflare
                if "content_categories" not in result:
                    print("{}\t-10".format(result["domain"]) + "\n", end="")
                else:
                    content_categories = result["content_categories"]
                    for c in content_categories:
                        print("{}\t{}".format(result["domain"], c["id"]) + "\n", end="")
            exit(0)
        else:
            # print(response.status_code)
            for domain in domains:
                print("{}\terror".format(domain) + "\n", end="")
            exit(1)