import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import os
import shutil
import subprocess

from multiprocessing import Pool, Process, Queue, Manager
from PIL import Image
from scipy import ndimage
from skimage import feature, filters, measure, morphology, data, color
from skimage.transform import hough_circle, hough_circle_peaks

# high level
def applySegmentationSteps(input_img, input_mode, output_root, save_intermediate=False, overwrite=False):
    """ This is a convenience function to apply all my vision steps in one go.

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param output_root: The filepath to the directory where outputs will be created.
    :type output_root: str
    :param save_intermediate: Flag to save intermediate outputs to disk, defaults to False.
    :type save_intermediate: bool
    :param overwrite: Flag to overwrite existing outputs, defaults to False.
    :type overwrite: bool

    :return: Returns 0 if output successfully created, otherwise returns 1.
    :rtype: int
    """

    np_img = loadImage(input_img)

    if save_intermediate == False:
        np_dist = getDistanceRaster(input_img, input_mode=input_mode)
        np_ridge = ridgeDetection(np_dist, 'np', method='meijering', black_ridges=False)
        np_blobs = connectedComponentsImage(np_ridge, 'np', output_path=output_root + '_blobs.tif')
        exportBlobs(np_img, np_blobs, 'np', output_root)
        plt.imsave(output_root + 'blobs_cmap.png', np_blobs, cmap='nipy_spectral')
    else:
        np_dist = getDistanceRaster(input_img, input_mode=input_mode, output_path=output_root + '_distance.tif')
        np_ridge = ridgeDetection(np_dist, 'np', method='meijering', black_ridges=False, output_path=output_root + '_ridge.tif')
        np_blobs = connectedComponentsImage(np_ridge, 'np', output_path=output_root + '_blobs.tif')
        exportBlobs(np_img, np_blobs, 'np', output_root)
        plt.imsave(output_root + 'blobs_cmap.png', np_blobs, cmap='nipy_spectral')

    if os.path.exists(output_root + 'blobs_tif'):
        return 0
    else:
        return 1

def calculatePixelMetrics(input_img, input_df):
    """ Calculates pixel metrics for every detection in a dataframe.

    :param input_img: Filepath to the input img
    :type input_img: str
    :param input_df: Dataframe with detections
    :type input_df: pandas dataframe

    :return: Returns the dataframe augmented with pixel metrics.
    :rtype: pandas dataframe
    """

    new_cir = []
    for index, row in input_df.iterrows():
        plot = row['plot']
        x = row['x']
        y = row['y']
        r = row['r']
        weight = row['weight']
        core, inner, outer = countPixels(input_img, 'np', x, y, r)
        print(core, inner, outer)
        circle = ([plot, x, y, r, weight, core, inner, outer])
        new_cir.append(circle)
    print('circles added:', len(new_cir))
    header = ['plot', 'x', 'y', 'r', 'weight', 'outer', 'inner', 'core']
    output_df = pd.DataFrame(new_cir, columns=header)
    return output_df

# mid level

def getDistanceRaster(input_img, input_mode, output_path=None):
    """ Takes a string containing the file path to the input image and an
    optional string with the desired output filepath. Returns a np array
    with the distance transform applied. If output path is applied, the
    distance raster will be saved.
    Note: the logical not is necessary for the scipy function.
    https://stackoverflow.com/questions/44770396/how-does-the-scipy-distance-transform-edt-function-work

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param output_path: The filepath of the image to be saved.
    :type output_path: str, optional

    :return: Returns numpy array of transformed image
    :rtype: numpy array
    """

    if input_mode == 'fp':
        np_img = loadImage(input_img)
    elif input_mode == 'np':
        np_img = input_img
    else:
        return (input_mode, " is not a supported mode. Supported modes are 'np' or 'fp'.")
    np_dist = ndimage.distance_transform_edt(np.logical_not(np_img))
    if output_path is not None:
        saveImage(np_dist, output_path, mode='dist_norm')
    return(np_dist)

def getImageThreshold(input_img, input_mode, threshold, output_path=None):
    """ Takes either a string to an image or a np array as input and a float
        for thresholding. Supported input modes are: "np" (numpy array) or "fp"
        (filepath). Returns a thresholded image. If output path is applied, the
        thresholded raster will be saved.

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param threshold: The comparison value for thresholding.
    :type threshold: int or float
    :param output_path: The filepath of the image to be saved.
    :type output_path: str, optional

    :return: Returns numpy array of transformed image
    :rtype: numpy array
    """

    if input_mode == 'fp':
        np_img = loadImage(input_img)
    elif input_mode == 'np':
        np_img = input_img
    else:
        return (input_mode, " is not a supported mode. Supported modes are 'np' or 'fp'.")
    np_thresh = np.zeros_like(np_img)
    dimensions = np_img.shape
    for i in range(0, dimensions[0]):
        for j in range(0, dimensions[1]):
            pixel = np_img[i][j]
            if pixel < threshold:
                np_thresh[i][j] = 1
            else:
                np_thresh[i][j] = 0
    if output_path is not None:
        saveImage(np_thresh, output_path)
    return(np_thresh)

def edgeDetection(input_img, input_mode, sigma=1, output_path=None):
    """ Takes either a string to an image or a np array as input and an integer
        for sigma. Default settings are for thresholded images.Supported input
        modes are: "np" (numpy array) or "fp" (filepath). Returns an image of
        edge detections. If output path is applied, the edge raster will be saved.

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param sigma: Controls the scale of edge detection, defaults to 1.
    :type sigma: int
    :param output_path: The filepath of the image to be saved.
    :type output_path: str, optional

    :return: Returns numpy array of transformed image
    :rtype: numpy array
    """

    if input_mode == 'fp':
        np_img = loadImage(input_img)
    elif input_mode == 'np':
        np_img = input_img
    else:
        return (input_mode, " is not a supported mode. Supported modes are 'np' or 'fp'.")
    edges = feature.canny(np_img, sigma=sigma)
    if output_path is not None:
        saveImage(edges, output_path)
    return(edges)

def morphologicalCloseImage(input_img, input_mode, stel=50, output_path=None):
    """ Takes either a string to an image or a np array as input and an integer
        for structuring element. Applies a circular structuring element. Supported input
        modes are: "np" (numpy array) or "fp" (filepath). Returns an image with
        a closing appplied. If output path is applied, the closed raster will be saved.

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param stel: Controls the size of structuring element, defaults to 50.
    :type stel: int
    :param output_path: The filepath of the image to be saved.
    :type output_path: str, optional

    :return: Returns numpy array of transformed image
    :rtype: numpy array
    """

    if input_mode == 'fp':
        np_img = loadImage(input_img)
    elif input_mode == 'np':
        np_img = input_img
    else:
        return (input_mode, " is not a supported mode. Supported modes are 'np' or 'fp'.")
    closed_img = morphology.binary_closing(np_img, morphology.disk(stel))
    if output_path is not None:
        saveImage(closed_img, output_path)
    return(closed_img)

def skeletonizeImage(input_img, input_mode, output_path=None):
    """ Takes either a string to an image or a np array as input. Supported
        input modes are: "np" (numpy array) or "fp" (filepath). Returns a
        skeletonized image. If output path is applied, the closed raster will
        be saved.

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param output_path: The filepath of the image to be saved.
    :type output_path: str, optional

    :return: Returns numpy array of transformed image
    :rtype: numpy array
    """

    if input_mode == 'fp':
        np_img = loadImage(input_img)
    elif input_mode == 'np':
        np_img = input_img
    else:
        return (input_mode, " is not a supported mode. Supported modes are 'np' or 'fp'.")
    skeleton = morphology.skeletonize(np_img)
    if output_path is not None:
        saveImage(skeleton, output_path)
    return(skeleton)

def connectedComponentsImage(input_img, input_mode, connectivity=1, output_path=None):
    """ Applies connected components labelling to an input image.

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param connectivity: Controls the distance to connect components, defaults to 1.
    :type connectivity: int
    :param output_path: The filepath of the image to be saved.
    :type output_path: str, optional

    :return: Returns numpy array with the connected components.
    :rtype: numpy array
    """

    if input_mode == 'fp':
        np_img = loadImage(input_img)
    elif input_mode == 'np':
        np_img = input_img
    else:
        return (input_mode, " is not a supported mode. Supported modes are 'np' or 'fp'.")
    dimensions = np_img.shape
    blobs = np_img.copy()
    for i in range(0, dimensions[0]):
        for j in range(0, dimensions[1]):
            pixel = np_img[i][j]
            if pixel < 0.5:
                blobs[i][j] = 1
            else:
                blobs[i][j] = 0
    blobs_labels = measure.label(blobs, connectivity=connectivity)
    if output_path is not None:
        saveImage(blobs_labels, output_path, mode='blob_labels')
    return(blobs_labels)

def exportBlobs(input_img, input_blobs, input_mode, output_root):
    """ Input_img and input_blobs will be opened with identical modes. Output is a
    directory containing base and blobs subdirectories. Base has the mask for each blob
    applied to the image.

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_img: Either the filepath to the input blobs (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param output_root: The filepath to the directory where clusters will be saved.
    :type output_root: str

    :return: Returns 0 upon completion.
    :rtype: numpy array
    """

    if input_mode == 'fp':
        np_img = loadImage(input_img)
        np_blobs = loadImage(input_blobs)
    elif input_mode == 'np':
        np_img = input_img
        np_blobs = input_blobs
    else:
        return (input_mode, " is not a supported mode. Supported modes are 'np' or 'fp'.")
    clusters = np.unique(np_blobs)

    output_base = output_root + '/base'
    output_blobs = output_root + '/blobs'

    if not os.path.exists(output_base):
        os.makedirs(output_base)
    if not os.path.exists(output_blobs):
        os.makedirs(output_blobs)

    group_counter = 0
    print(len(clusters))
    for i in sorted(clusters):
        print("i", i)
        print("group", group_counter)
        single_blob = np.where(np_blobs == i, True, False)

        base_out = single_blob*np_img
        blob_out = single_blob*np_blobs
        trimmed_blob_out = trim2DArray(blob_out)
        ul_x, ul_y = calculateTrimOffsetForward(blob_out)
        lr_x, lr_y = calculateTrimOffsetBackward(blob_out)
        trimmed_base_out = base_out[ul_y:lr_y,ul_x:lr_x]
        print('base', trimmed_base_out.shape)
        print('blob', trimmed_blob_out.shape)
        if testVerySmallClusters(trimmed_blob_out, 'np') or np.count_nonzero(base_out) < 5: # 5 is a magic constant but there's some logic here. how many points are needed to identify a circle? at least 3 needed in theory, more needed depending on the scanner
            print('skipping cluster')
            continue
        sub_base = output_base + '/' + str(ul_x) + '_' + str(ul_y) + '_' +str(group_counter) + '.tif'
        sub_blob = output_blobs + '/' + str(ul_x) + '_' + str(ul_y) + '_' + str(group_counter) + '.tif'
        saveImage(trimmed_base_out, sub_base)
        saveImage(trimmed_blob_out, sub_blob, mode='blob_labels')
        group_counter += 1
    return 0

def ridgeDetection(input_img, input_mode, method='meijering', black_ridges=False, output_path=None):
    """ Applies ridge detection to an input image.

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param method: Ridge detection algorithm, defaults to meijering.
    :type method: str
    :param black_ridges: Controls whether black or white pixels are the ridges, defaults to False (white pixels = ridge).
    :type black_ridges: bool
    :param output_path: The filepath of the image to be saved.
    :type output_path: str, optional

    :return: Returns numpy array with the connected components.
    :rtype: numpy array
    """

    if input_mode == 'fp':
        np_img = loadImage(input_img)
    elif input_mode == 'np':
        np_img = input_img
    else:
        return (input_mode, " is not a supported mode. Supported modes are 'np' or 'fp'.")

    if method == 'meijering':
        np_ridges = filters.meijering(np_img, black_ridges=black_ridges)

    if output_path is not None:
        saveImage(np_ridges, output_path)
    return(np_ridges)

# low level

def loadImage(img_path):
    """ Returns the image from img path as a np array using PIL.

    :param img_path: Filepath to the input img
    :type img_path: str

    :return: Returns the image loaded as a numpy array.
    :rtype: numpy array
    """

    img = Image.open(img_path)
    np_img = np.array(img)
    return (np_img)

def saveImage(np_img, output_path, mode=None, scale=255.0):
    """ Takes a np array as input and saves it to output path using PIL.
    Current supported modes are:
    dist_norm - distance normalization I use for distance transforms
    blob_labels - 255 scaling I use for connected components

    :param np_img: The numpy array to be saved
    :type np_img: numpy array
    :param output_path: The filepath of the image to be saved.
    :type output_path: str
    :param mode: There are two convenience modes: 'dist_norm' (distance transform normalization) and 'blob_labels' (blob labelling), defaults to None.
    :type mode: str
    :param scale: Value used for scaling in distance normalization, defaults to 255.0
    :type scale: int or float

    :return: Returns the image loaded as a numpy array.
    :rtype: numpy array
    """

    if mode is not None:
        if mode == 'dist_norm':
            np_img *= scale/np_img.max()
            np_out = Image.fromarray((np_img).astype('uint8'))
        elif mode == 'blob_labels':
            np_out = Image.fromarray((np_img*255).astype(np.uint8))
        else:
            return (mode, " is not a supported mode.")
    else:
        np_out = Image.fromarray(np_img)
    np_out.save(output_path)

    if os.path.exists(output_path):
        return 0
    else:
        return 1

def trim2DArray(input_arr, threshold=0):
    """ Convenience function to trim a 2D array. Removes any rows/columns
        with values equal to or less than the threshold. Source:
        https://stackoverflow.com/questions/11188364/remove-zero-lines-2-d-numpy-array?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa

    :param input_arr: A 2D numpy array to be trimmed.
    :type input_arr: numpy array
    :param threshold: Any exterior row or column with a total less than or equal to threshold will be removed, defaults to 0.
    :type threshold: int or float

    :return: Returns a subset of the input array.
    :rtype: numpy array
    """

    print(input_arr.shape)
    ul_x, ul_y = calculateTrimOffsetForward(input_arr, threshold)
    lr_x, lr_y = calculateTrimOffsetBackward(input_arr, threshold)

    output_arr = input_arr[ul_y:lr_y, ul_x:lr_x]

    print(output_arr.shape)

    return (output_arr)

def calculateTrimOffsetForward(input_arr, threshold=0):
    """ This function is used to calculate the offset of the upper left corner
        after trimming. The threshold should be the same as whatever is used in
        trim2DArray.

    :param input_arr: A 2D numpy array to be trimmed.
    :type input_arr: numpy array
    :param threshold: Any exterior row or column with a total less than or equal to threshold will be removed, defaults to 0.
    :type threshold: int or float

    :return: Returns the x and y of the upper left corner.
    :rtype: int, int
    """

    ul_x, ul_y = 0, 0

    row_sum = np.sum(input_arr, axis=1)
    col_sum = np.sum(input_arr, axis=0)

    for i in range(0, len(col_sum)):
        if col_sum[i] > threshold:
            ul_x = i
            break
    for j in range(0, len(row_sum)):
        if row_sum[j] > threshold:
            ul_y = j
            break
    return (ul_x, ul_y)

def calculateTrimOffsetBackward(input_arr, threshold=0):
    """ This function is used to calculate the offset of the lower right corner
        after trimming. The threshold should be the same as whatever is used in
        trim2DArray.
        Returns the x and y of the lower right corner.

   :param input_arr: A 2D numpy array to be trimmed.
    :type input_arr: numpy array
    :param threshold: Any exterior row or column with a total less than or equal to threshold will be removed, defaults to 0.
    :type threshold: int or float

    :return: Returns the x and y of the lower right corner.
    :rtype: int, int
    """

    y,x = input_arr.shape
    lr_x, lr_y = 0, 0

    row_sum = np.sum(np.flip(input_arr), axis=1)
    col_sum = np.sum(np.flip(input_arr), axis=0)

    for i in range(0, len(col_sum)):
        if col_sum[i] > threshold:
            lr_x = i
            break
    for j in range(0, len(row_sum)):
        if row_sum[j] > threshold:
            lr_y = j
            break
    return (x-lr_x, y-lr_y)

def testVerySmallClusters(input_img, input_mode, min_rad=7.5):
    """ This functions takes a cluster as input and tests if its biggest
        circle would be too small.
        Returns True if it's smaller and False if it's bigger.

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param min_rad: the minimum radius for a valid cluster, defaults to 7.5
    :type min_rad: int or float

    :return: Returns True if the image is too small to be a valid cluster, otherwise False.
    :rtype: bool
    """

    if input_mode == 'fp':
        np_img = loadImage(input_img)
    elif input_mode == 'np':
        np_img = input_img
    else:
        return (input_mode, " is not a supported mode. Supported modes are 'np' or 'fp'.")
    dim = np_img.shape
    if dim[0] <= dim[1]:
        length = dim[0]
    else:
        length = dim[1]
    if length/2. < min_rad:
        return True
    else:
        return False

def circleMask(img, cir_x, cir_y, r, mode, filter=0):
    """ Takes an img (loaded as a np array), the x and y coordinates to center
    the mask, the radius of the mask, and the mode. Mode must be interior
    or exterior. The default filter value is 0.
    https://towardsdatascience.com/the-little-known-ogrid-function-in-numpy-19ead3bdae40

    :param input_img: An image loaded as a numpy array.
    :type input_img: numpy array
    :param cir_x: The x coordinate of the center of the circle.
    :type cir_x: int
    :param cir_y: The y coordinate of the center of the circle.
    :type cir_y: int
    :param r: The radius of the circle mask
    :param mode: Determines what is inside and outside the mask. Must be interior or exterior
    :type mode: str
    :type r: int
    :param filter: The value for pixels outside the mask, defaults to 0.
    :type filter: int or float

    :return: Returns True if the image is too small to be a valid cluster, otherwise False.
    :rtype: bool
    """

    if not mode == 'interior' and not mode == 'exterior':
        print(mode, "is not a supported mode. Please enter interior or exterior")
        return 1

    #get the dimensions of the image
    n,m = img.shape

    #create an open grid for our image
    y,x = np.ogrid[0:n, 0:m]
    #operate on a copy of the image
    copyImg = img.copy()

    #get the x and y center points of our image
    center_x = cir_x
    center_y = cir_y

    #create a circle mask
    if mode == 'interior':
        circle_mask = (x-center_x)**2 + (y-center_y)**2 <= r**2
    elif mode == 'exterior':
        circle_mask = (x-center_x)**2 + (y-center_y)**2 >= r**2

    #black out anywhere within the circle mask
    copyImg[circle_mask] = [filter]
    copyImg[copyImg != filter] = [255-filter]

    return copyImg

def countPixels(input_img, input_mode, x, y, r):
    """ Counts pixels within a radius around x, y.

    :param input_img: Either the filepath to the input img (input_mode='fp') or a numpy array (input_mode='np').
    :type input_img: str or numpy array
    :param input_mode: 'fp' if input_img is a filepath or 'np' if input_img is a numpy array.
    :type input_mode: str
    :param x: The x coordinate of the center of the circle.
    :type x: int
    :param y: The y coordinate of the center of the circle.
    :type y: int
    :param r: The radius around the center of the circle
    :type r: int

    :return: Returns the number of pixels.
    :rtype: int
    """

    if input_mode == 'fp':
        np_img = loadImage(input_img)
    elif input_mode == 'np':
        np_img = input_img
    else:
        return (input_mode, " is not a supported mode. Supported modes are 'np' or 'fp'.")

    base_img = circleMask(np_img, x, y, r, 'exterior')

    core_img = circleMask(np_img, x, y, r*0.8, 'exterior')
    core_count = np.count_nonzero(base_img*core_img)

    inner_img = circleMask(np_img, x, y, r*0.8, 'exterior')
    inner_ring = base_img - inner_img
    inner_count = np.count_nonzero(inner_ring)

    outer_img = circleMask(np_img, x, y, r*1.2, 'exterior')
    outer_ring = outer_img - base_img
    outer_count = np.count_nonzero(outer_ring)

    return (core_count, inner_count, outer_count)

# multiprocessing

def segmentationVisionQueue(q, output_dir, save_intermediate=False, overwrite=False):
    """ This is a queue for multiprocessing.

    :param q: the queue generated by segment Directory
    :type q: Multiprocessing.Queue object
    :param output_dir: the filepath to the directory where outputs will be saved.
    :type output_dir: str
    :param save_intermediate: Flag to save intermediate outputs to disk, defaults to False.
    :type save_intermediate: bool
    :param overwrite: Flag to overwrite existing outputs, defaults to False.
    :type overwrite: bool
    """

    while not q.empty():
        try:
            input_img = q.get()
            print("input img: ", input_img)
            # this is hard coded to run on my registered rasters only
            img_info = input_img.split('/')
            img_id = img_info[-1]
            img_id = img_id[:-4]
            output_root = output_dir + '/' + img_id
            applySegmentationSteps(input_img, 'fp', output_root, save_intermediate=save_intermediate, overwrite=overwrite)
        except ValueError as val_error:
            print(val_error)
        except Exception as error:
            print(error)

def segmentDirectoryMP(input_dir, output_root, num_workers, target='.tif', save_intermediate=False, overwrite=False):
    """ A multiprocessing segmenter.
    :param input_dir: Filepath to the input directory.
    :type input_dir: str
    :param output_root: The filepath to the directory where outputs will be created.
    :type output_root: str
    :param num_workers: Number of workers to assign.
    :type num_workers: int
    :param target: The file ending for valid inputs, defaults to '.tif'
    :type target: str
    :param save_intermediate: Flag to save intermediate outputs to disk, defaults to False.
    :type save_intermediate: bool
    :param overwrite: Flag to overwrite existing outputs, defaults to False.
    :type overwrite: bool

    """
    q = Queue()
    for i in sorted(os.listdir(input_dir)):
        if i.endswith(target):
            input_img = input_dir + '/' + i
            q.put(input_img)
        else:
            continue
    workers = Pool(num_workers, segmentationVisionQueue,(q, output_root, save_intermediate, overwrite))
    workers.close()
    workers.join()

def calculatePixelMetricsMP(input_img, input_df, num_workers=8):
    """ A multiproessing pixel metrics implementation.

    :param input_img: Filepath to the input img
    :type input_img: str
    :param input_df: Dataframe with detections
    :type input_df: pandas dataframe
    :param num_workers: Number of workers to assign, defaults to 8.
    :type num_workers: int

    :return: Returns the dataframe augmented with pixel metrics.
    :rtype: pandas dataframe
    """

    manager = Manager()
    new_cir = manager.list()
    q = Queue()
    for index, row in input_df.iterrows():
        plot = row['plot']
        x = row['x']
        y = row['y']
        r = row['r']
        weight = row['weight']
        info = [plot, x, y, r, weight]
        q.put(info)
    workers = Pool(num_workers, calculatePixelMetricsQueue,(q, input_img, input_df, new_cir))
    workers.close()
    workers.join()

    header = ['plot', 'x', 'y', 'r', 'weight', 'core', 'inner', 'outer']
    print(len(new_cir))
    output_df = pd.DataFrame(list(new_cir), columns=header)
    return output_df

def calculatePixelMetricsQueue(q, input_img, input_df, output_list):
    """ This is a queue for multiprocessing.

    :param q: the queue generated by calculatePixelMetricsMP
    :type q: Multiprocessing.Queue object
    :param input_img: Filepath to the input img
    :type input_img: str
    :param input_df: Dataframe with detections
    :type input_df: pandas dataframe
    :param output_list: a Manager generated by calculatePixelMetricsMP
    :type output_list: Multiprocessing.Manager object

    """

    counter = 0
    while not q.empty():
        try:
            if counter % 10000 == 0 and counter > 0:
                print(counter)
            info = q.get()
            plot = info[0]
            x = info[1]
            y = info[2]
            r = info[3]
            weight = info[4]
            core, inner, outer = countPixels(input_img, 'fp', x, y, r)
            circle = ([plot, x, y, r, weight, core, inner, outer])
            output_list.append(circle)
            counter += 1

        except ValueError as val_error:
            print(val_error)
        except Exception as error:
            print(error)


from courses.models import Program, Course, Offering, Semester, Assignment, Material, Builder
from people.models import Instructor
from django.shortcuts import render_to_response, get_object_or_404
from django.http import HttpResponseRedirect
from django.template import RequestContext
from django.core.urlresolvers import reverse
from django.contrib import messages
from courses.forms import QueryForm, OfferingIntraEditForm
from postman.models import Message, STATUS_PENDING, STATUS_ACCEPTED
from django.db.models import Q  # For search

def program_home(request):
    """
    Main course listings page
    """

    # First handle course listing selections from programs switcher dropdown
    if request.POST.get('type'):
        return HttpResponseRedirect(reverse('program_sec_courses',
            kwargs={'progslug':request.POST['type'],})) # Redirect after POST

    programs = Program.objects.all()

    return render_to_response(
        'program/index.html',
        locals(),
        context_instance=RequestContext(request)
        )

def programs_list(request):
    """
    Listing of departments
    """

    programs = Program.objects.all()

    return render_to_response(
        'courses/programs_list.html',
        locals(),
        context_instance=RequestContext(request)
        )

def programs_detail(request,slug):
    """
    Single major view
    """

    program = get_object_or_404(Program, slug=slug)

    # Get all instructors in this program. TODO:
    instructors = None

    return render_to_response(
        'courses/programs_detail.html',
        locals(),
        context_instance=RequestContext(request)
        )

def program_sec_courses(request,progslug=False):
    """
    The "Courses" sub-section for a Program of Study, e.g. Business Reporting Courses
    """

    # First handle selections from programs switcher dropdown
    # Redirect after POST
    if request.POST.get('type'):
        program = get_object_or_404(Program, slug=request.POST.get('type'))
        return HttpResponseRedirect(
                    reverse('program_sec_courses',
                            kwargs={'progslug':program.slug,}))

    program = get_object_or_404(Program, slug=progslug)
    programs = Program.objects.all()
    sem = Semester.objects.get(current=True)
    # Get courses that match the current live semester AND are associated with this view's program slug
    offerings = Offering.objects.filter(in_programs__in=(program.id,),semester=sem)

    return render_to_response(
        'program/section-courses.html',
        locals(),
        context_instance=RequestContext(request)
        )

def program_sec_faculty(request,progslug=False):
    """
    The "Faculty" subsection of a Program of Study, e.g. Business Reporting Faculty
    Subdivied into present faculty, lecturers, and previous (inactive) instructors
    """

    program = get_object_or_404(Program, slug=progslug)
    faculty = program.instructors.filter(profile__user__groups__in=(4,),profile__user__is_active=True)
    lecturers = program.instructors.filter(profile__user__groups__in=(5,),profile__user__is_active=True)
    prev_inst = program.instructors.filter(profile__user__is_active=False)

    return render_to_response(
        'program/section-faculty.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offerings_schedule(request, printable=False, sem_id=False):
    """
    Schedule of ALL course offerings - one view handles both calendar and printable view.
    """

    # Construct query params dict for search
    basic_params = {
        'q': request.GET.get('q',''),
        }

    keywords = basic_params['q']
    search_form = QueryForm(basic_params)

    # First handle selections from semester switcher dropdown
    if request.POST.get('sem'):
        sem = get_object_or_404(Semester, pk=request.POST.get('sem'))
        if printable:
            # Redirect to new semester after POST
            return HttpResponseRedirect(
                        reverse(
                            'courses_descriptions_sem',
                            kwargs={'sem_id':sem.pk}))
        else:
            # Redirect to new semester after POST
            return HttpResponseRedirect(
                        reverse(
                            'courses_schedule_sem',
                            kwargs={'sem_id':sem.pk}))

    # Current semester may come through in the URL. If not, default to current semester.
    if sem_id :
        current_sem = get_object_or_404(Semester, pk=sem_id)
    else:
        current_sem = get_object_or_404(Semester, current=True)

    semesters = Semester.objects.filter(live=True).order_by('-id')

    # To generate a unique list of courses (not offerings) for this semester,
    # get all offerings for this semester and derive the distinct internal_titles
    # from their related courses. There's probably a better way to do this query :)
    semofferings = Offering.objects.filter(sec_term=current_sem)
    courselist = semofferings.distinct('course__internal_title').values('course__internal_title')
    courses = Course.objects.filter(internal_title__in=courselist).order_by('internal_title')

    # if keywords exist, filter the results
    if keywords:

        # Use Q object for 'OR' type query
        courses = courses.filter(
                Q(internal_title__icontains=keywords) |
                Q(long_title__icontains=keywords)
            )

    # Which template? Calendar style or printable?
    if printable :
        template = 'courses/descriptions.html'
    else :
        template = 'courses/schedule.html'

    return render_to_response(
        template,
        locals(),
        context_instance=RequestContext(request)
        )

def offering_detail(request, course_sec_id):
    """
    Detail info on a particular course offering
    """

    offering = get_object_or_404(Offering, course_sec_id=course_sec_id)

    # Is this offering already in ScheduleBuilder?
    try:
        builder = Builder.objects.get(profile=request.user.profile, offering=offering)
        scheduled = True
    except:
        pass

    # Allow instructors of a specific offering to override some course details
    if request.user.profile in [i.profile for i in offering.instructors.all()]:
        user_can_edit_offering = True

        if request.method == 'POST':
            course_edit_form = OfferingIntraEditForm(request.POST, instance=offering)
            if course_edit_form.is_valid():
                course_edit_form.save()
                messages.success(request, "Course Offering details overridden")
                return HttpResponseRedirect(reverse('offering_detail',args=[offering.course_sec_id]))

        else:

            '''
            The form's initial values are tricksy because the title and body displayed
            on the *Offering* are inherited from the parent Course object. But when the
            form is saved, it saves overrides into the Offering object itself. To avoid
            presenting a blank form, show inherited values *unless* the object has
            previously been overridden.
            '''

            if not offering.title:
                init_title = offering.course.long_title
            else:
                init_title = offering.title

            if not offering.title:
                init_description_override = offering.course.description
            else:
                init_description_override = offering.description_override

            course_edit_form = OfferingIntraEditForm(
                instance=offering,
                initial={'title': init_title, 'description_override': init_description_override}
                )

    return render_to_response(
        'courses/offering_detail.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offering_schedule(request, course_sec_id):
    """
    Class schedule for a particular course offering
    """

    offering = get_object_or_404(Offering, course_sec_id=course_sec_id)

    return render_to_response(
        'courses/offering_schedule.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offering_announcements(request, course_sec_id):
    """
    Announcements displayed to students in a given course offering
    """

    offering = get_object_or_404(Offering, course_sec_id=course_sec_id)
    # announcements = Notification.objects.filter(offering=offering)

    return render_to_response(
        'courses/offering_announcements.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offering_library(request, course_sec_id):
    """
    Files shared with a course offering
    """

    offering = get_object_or_404(Offering, course_sec_id=course_sec_id)

    return render_to_response(
        'courses/offering_library.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offering_contact(request, course_sec_id):
    """
    Allows students to make private contact with instructors of a given offering
    """

    offering = get_object_or_404(Offering, course_sec_id=course_sec_id)
    recips = offering.instructors.all()

    return render_to_response(
        'courses/offering_contact.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offering_policies(request, course_sec_id):
    """
    Simple text field display for grading and policies fields on the Offering model.
    """

    offering = get_object_or_404(Offering, course_sec_id=course_sec_id)

    return render_to_response(
        'courses/offering_policies.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offering_assignments(request, course_sec_id):
    """
    List of assignments given by instructors to students in a given course offering.
    """

    offering = get_object_or_404(Offering, course_sec_id=course_sec_id)
    assignments = Assignment.objects.filter(offering=offering)

    return render_to_response(
        'courses/offering_assignments.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offering_assignment_detail(request, assign_id):
    """
    Detail view for individual Assignments (popup only)
    """
    assignment = get_object_or_404(Assignment, id=assign_id)

    return render_to_response(
        'courses/assignment_pop.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offering_materials(request, course_sec_id):
    """
    Materials required by particpants in a given offering
    """

    offering = get_object_or_404(Offering, course_sec_id=course_sec_id)
    materials = Material.objects.filter(offering=offering)

    return render_to_response(
        'courses/offering_materials.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offering_material_detail(request, material_id):
    """
    Detail view for class materials requirements (popup only)
    """
    material = get_object_or_404(Material, id=material_id)

    return render_to_response(
        'courses/material_pop.html',
        locals(),
        context_instance=RequestContext(request)
        )

def offering_participants(request, course_sec_id):
    """
    List of students, TAs and instructors associated with a given course offering.
    """

    offering = get_object_or_404(Offering, course_sec_id=course_sec_id)
    students = offering.students.all()
    instructors = offering.instructors.all()

    return render_to_response(
        'courses/offering_participants.html',
        locals(),
        context_instance=RequestContext(request)
        )

def program_detail(request,program_slug):
    """
    Detail info on a particular program
    """

    program = get_object_or_404(Program, slug=program_slug)
    instructors = program.instructors.filter(profile__user__is_active=True)

    # For use in Programs sidebar
    programs = Program.objects.all()

    return render_to_response(
        'courses/program_detail.html',
        locals(),
        context_instance=RequestContext(request)
        )

def course_detail(request, internal_title):
    """
    Detail info on a particular course
    """

    course = get_object_or_404(Course, internal_title=internal_title)
    term = Semester.objects.get(current=True)
    sections = Offering.objects.filter(course=course, sec_term=term).order_by('section')

    # For use in Programs sidebar
    # programs = Program.objects.all()

    return render_to_response(
        'courses/course_detail.html',
        locals(),
        context_instance=RequestContext(request)
        )


import json
from typing import Generator, List, Optional

import arrow
import attr
import deserialize

from fractal_python.api_client import (
    ApiClient,
    _arrow_or_none,
    _call_api,
    _get_paged_response,
)
from fractal_python.banking.api import BANKING_ENDPOINT

banks_endpoint = BANKING_ENDPOINT + "/banks"
consents = "consents"

@attr.s(auto_attribs=True)
@deserialize.auto_snake()
class Bank:
    r"""Bank.

    :attr id: unique id of the bank
    :attr name: name of the bank
    :attr logo: bank logo
    :attr logo_url: url to logo
    """
    id: int
    name: str
    logo: Optional[str]
    logo_url: Optional[str]

def new_bank(
    bank_id: int,
    name: str,
    logo: str = None,
    logo_url: str = None,
) -> Bank:
    r"""Make a new Bank object.

    :param bank_id: unique id of the bank
    :type bank_id: int
    :param name: name of the bank
    :type name: str
    :param logo: logo
    :type logo: str
    :param logo_url: url to the logo
    :type logo_url: str
    :return: new Bank object
    :rtype: Bank
    """
    return Bank(bank_id, name, logo, logo_url)

def retrieve_banks(client: ApiClient) -> Generator[List[Bank], None, None]:
    r"""Retrieve all banks.

    :param client: the client to use for the api call
    :type client: ApiClient
    :yield: Page of Banks
    :rtype: Generator[List[Bank], None, None]

    Usage::

      >>> banks =[x for y in banking.retrieve_banks(client) for x in y]
    """
    yield from _get_paged_response(
        client=client, url=banks_endpoint, cls=Bank, param_keys=None, company_id=None
    )

@attr.s(auto_attribs=True)
@deserialize.auto_snake()
class CreateBankConsentResponse:
    r"""Response for a CreateBankConsent call.

    :attr signin_url: URL user will sign in to bank at
    :attr consent_id: unique id of the consent
    :attr bank_id: unique id of the bank
    :attr type: type of consent
    :attr permission: permissions available
    """
    signin_url: str
    consent_id: str
    bank_id: int
    type: str
    permission: str

def create_bank_consent(
    client: ApiClient, bank_id: int, redirect: str, company_id: str
) -> CreateBankConsentResponse:
    r"""Create a bank consent.

    This is the first step in connecting an account.

    :param client: the client to use for the api call
    :type client: ApiClient
    :param bank_id: the unique id of the bank
    :type bank_id: int
    :param redirect: the urk to redirect the account owner to
    :type redirect: str
    :param company_id: the unique id of the company
    :type company_id: str
    :return: response from the bank
    :rtype: CreateBankConsentResponse
    """
    response = _call_api(
        client=client,
        url=f"{banks_endpoint}/{bank_id}/{consents}",
        method="POST",
        company_id=company_id,
        data=json.dumps(dict(redirect=redirect)),
    )
    json_response = json.loads(response.text)
    bank_consent_response = deserialize.deserialize(
        CreateBankConsentResponse, json_response
    )
    return bank_consent_response

@attr.s(auto_attribs=True)
@deserialize.auto_snake()
@deserialize.parser("expiry_date", _arrow_or_none)
@deserialize.parser("date_created", _arrow_or_none)
@deserialize.parser("authorised_date", _arrow_or_none)
class BankConsent:
    r"""Bank Consent for company.

    :attr company_id: unique identifier for the company
    :attr permission: permissions given by account owner
    :attr expiry_date: consent may expire
    :attr consent_id: unique id for this consent
    :attr bank_id: unique id of the bank
    :attr date_created: when this consent was created
    :attr authorised_date: when the consent was last authorised
    :attr consent_type: type of consent
    :attr status: status of consent
    """
    company_id: str
    permission: str
    expiry_date: Optional[arrow.Arrow]
    consent_id: str
    bank_id: int
    date_created: arrow.Arrow
    authorised_date: Optional[arrow.Arrow]
    consent_type: str
    status: str

def retrieve_bank_consents(
    client: ApiClient, bank_id: int, company_id: Optional[str] = None
) -> Generator[List[BankConsent], None, None]:
    r"""Retrieve consents by bank id and company id.

    :param client: the client to use for the api call
    :type client: ApiClient
    :param bank_id: the id of the bank to filter on
    :type bank_id: int
    :param company_id: the id of the company to filter on, defaults to None
    :type company_id: str, optional
    :yield: Page of Bank Consents
    :rtype: Generator[List[BankConsent], None, None]
    """
    url = f"{banks_endpoint}/{bank_id}/{consents}"
    yield from _get_paged_response(
        client=client, url=url, cls=BankConsent, param_keys=[], company_id=company_id
    )

def put_bank_consent(
    client: ApiClient,
    code: str,
    id_token: str,
    state: str,
    bank_id: int,
    consent_id: str,
    company_id: str,
):
    r"""Put a Bank Consent for a Company.

    Call this after user has been redirected back from the bank.

    :param client: the client to use for the api call
    :type client: ApiClient
    :param code: to be exchanged for access token with the bank
    :type code: str
    :param id_token: base64 encoded JSON for verifying
    :type id_token: str
    :param state: base64b encoded JSON to persist the state
    :type state: str
    :param bank_id: the id of the bank to filter on
    :type bank_id: int
    :param consent_id: id returned from create_bank_consent
    :type consent_id: str
    :param company_id:  the id of the company
    :type company_id: str
    :raises AssertionError: When response code not 204
    """
    payload = {"code": code, "id_token": id_token, "state": state}
    response = _call_api(
        client,
        f"{banks_endpoint}/{bank_id}/{consents}/{consent_id}",
        "PUT",
        company_id=company_id,
        data=json.dumps(payload),
    )
    if response.status_code != 204:
        raise AssertionError(f"status_code:{response.status_code} {response.text}")

def delete_bank_consent(
    client: ApiClient,
    bank_id: int,
    consent_id: str,
    company_id: str,
):
    r"""Delete a bank consent.

    Use this when a bank account owner wants to disconnect their account.

    :param client: the client to use for the api call
    :type client: ApiClient
    :param bank_id: the id of the bank to filter on
    :type bank_id: int
    :param consent_id: id returned from create_bank_consent
    :type consent_id: str
    :param company_id:  the id of the company
    :type company_id: str
    :raises AssertionError: When response code not 202
    """
    response = _call_api(
        client,
        f"{banks_endpoint}/{bank_id}/{consents}/{consent_id}",
        "DELETE",
        company_id=company_id,
    )
    if response.status_code != 202:
        raise AssertionError(f"status_code:{response.status_code} {response.text}")


import matplotlib.pyplot as plt
import src.features.helpers_vis as hp_vis
import src.features.helpers as hp
import logging
import pickle
from pathlib import Path
import numpy as np
import click

hp_vis.set_layout(15)

ROOT = Path(__file__).resolve().parents[3]

SOUND_FILES = ROOT / 'data/raw/sound_samples/'
# create a list of the sound files
SOUND_FILES = list(SOUND_FILES.glob('**/*.wav'))

# Parameters
@click.command()
@click.option('--save_figs', type=click.BOOL, default=False, help='Save figures')
@click.option('--save_type', default='svg', help='Define the format figures are saved.')
@click.option('--model_name', default='parameter_sweep', help='Defines the model name.')
@click.option('--exp_name', default='default', help='Defines the experiment name')
@click.option('--azimuth', default=12, help='Azimuth for which localization is done. Default is 12')
@click.option('--snr', default=0.2, help='Signal to noise ration to use. Default is 0.2')
@click.option('--freq_bands', default=128, help='Amount of frequency bands to use. Default is 128')
@click.option('--max_freq', default=20000, help='Max frequency to use. Default is 20000')
@click.option('--elevations', default=25, help='Number of elevations to use 0-n. Default is 25 which equals 0-90 deg')
@click.option('--mean_subtracted_map', default=True, help='Should the learned map be mean subtracted. Default is True')
@click.option('--ear', default='contra', help='Which ear should be used, contra or ipsi. Default is contra')
@click.option('--normalization_type', default='sum_1', help='Which normalization type should be used sum_1, l1, l2. Default is sum_1')
@click.option('--clean', is_flag=True)
def main(save_figs=False, save_type='svg', model_name='parameter_sweep', exp_name='default', azimuth=12, snr=0.2, freq_bands=128, max_freq=20000, elevations=25, mean_subtracted_map=True, ear='ipsi', normalization_type='sum_1', clean=False):

    logger = logging.getLogger(__name__)
    logger.info('Showing parameter sweep results')

    ########################################################################
    ######################## Set parameters ################################
    ########################################################################

    # make sure save type is given
    if not save_type or len(save_type) == 0:
        save_type = 'svg'

    normalize = False
    time_window = 0.1  # time window in sec

    elevations = np.arange(0, elevations, 1)

    ########################################################################
    ########################################################################

    # create unique experiment name
    exp_name_str = hp.create_exp_name([exp_name, normalization_type, mean_subtracted_map, time_window, int(
        snr * 100), freq_bands, max_freq, (azimuth - 12) * 10, normalize, len(elevations), ear])

    exp_path = ROOT / 'models' / model_name
    exp_file = exp_path / exp_name_str
    # check if model results exist already and load
    if exp_path.exists() and exp_file.is_file():
        # try to load the model files
        with open(exp_file.as_posix(), 'rb') as f:
            logger.info('Reading model data from file')
            [scores,sigma_smoothing_vals,sigma_gauss_norm_vals] = pickle.load(f)

        fig = plt.figure(figsize=(20, 5))
        axes = fig.subplots(1,3,sharex=True,sharey=True)
        ax = axes[0]
        p = ax.pcolormesh(sigma_smoothing_vals, sigma_gauss_norm_vals, np.squeeze(scores[:, :, 0]),linewidth=0,rasterized=True)
        p.set_edgecolor('face')
        ax.set_xlabel(r'Gauss Normalization $\sigma$')
        ax.set_ylabel(r'Smoothing $\sigma$')
        # ax.set_title('Gain')
        cbar = plt.colorbar(p,ax=ax)
        cbar.set_label('Gain', rotation=270)
        cbar.ax.get_yaxis().labelpad = 15

        ax = axes[1]
        p = ax.pcolormesh(sigma_smoothing_vals, sigma_gauss_norm_vals, np.squeeze(scores[:, :, 1]),linewidth=0,rasterized=True)
        p.set_edgecolor('face')
        ax.set_xlabel(r'Gauss Normalization $\sigma$')
        # ax.set_ylabel('Smoothing Sigma')
        # ax.set_title('Bias')
        cbar = plt.colorbar(p,ax=ax)
        cbar.set_label('Bias', rotation=270)
        cbar.ax.get_yaxis().labelpad = 15

        ax = axes[2]
        p = ax.pcolormesh(sigma_smoothing_vals, sigma_gauss_norm_vals, np.squeeze(scores[:, :, 2]),linewidth=0,rasterized=True)
        p.set_edgecolor('face')
        ax.set_xlabel(r'Gauss Normalization $\sigma$')
        # ax.set_ylabel('Smoothing Sigma')
        # ax.set_title('Score')
        cbar = plt.colorbar(p,ax=ax)
        cbar.set_label('Score', rotation=270)
        cbar.ax.get_yaxis().labelpad = 15

        plt.tight_layout()

        if save_figs:
            exp_name_str = hp.create_exp_name([exp_name, normalization_type, 0, 1, mean_subtracted_map, time_window, int(
            snr * 100), freq_bands, max_freq, (azimuth - 12) * 10, normalize, len(elevations), ear])
            fig_save_path = ROOT / 'reports' / 'figures' / exp_name_str / model_name
            if not fig_save_path.exists():
                fig_save_path.mkdir(parents=True, exist_ok=True)
            logger.info('Saving figures to ' + fig_save_path.as_posix())
            plt.savefig((fig_save_path / (model_name + '_' + exp_name + '_sweep.' + save_type)).as_posix(), dpi=300)
        else:
            plt.show()
    else:
        logger.error('No data set found. Run model first!')
        logger.error(exp_file)

if __name__ == '__main__':
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    main()


import os
import warnings
from urlparse import urljoin

from unidecode import unidecode

from git import Repo

from elasticutils import get_es, Q, F

from elasticgit.storage import StorageManager, RemoteStorageManager
from elasticgit.search import ESManager, S

import logging

log = logging.getLogger(__name__)

class Workspace(object):
    """
    The main API exposing a model interface to both a Git repository
    and an Elasticsearch index.

    :param git.Repo repo:
        A :py:class:`git.Repo` instance.
    :param dit es:
        A dictionary of values one would pass to elasticutils.get_es
        to get an Elasticsearch connection
    :param str index_prefix:
        The prefix to use when generating index names for Elasticsearch
    """

    def __init__(self, repo, es, index_prefix):
        self.repo = repo
        self.sm = StorageManager(repo)
        self.es_settings = es
        self.im = ESManager(
            self.sm, get_es(**self.es_settings), index_prefix)
        self.working_dir = self.repo.working_dir
        self.index_prefix = index_prefix

    def setup(self, name, email):
        """
        Setup a Git repository & ES index if they do not yet exist.
        This is safe to run if already existing.

        :param str name:
            The name of the committer in this repository.
        :param str email:
            The email address of the committer in this repository.
        """
        if not self.sm.storage_exists():
            self.sm.create_storage()

        self.sm.write_config('user', {
            'name': name,
            'email': email,
        })

        if not self.im.index_exists(self.sm.active_branch()):
            self.im.create_index(self.sm.active_branch())

    def exists(self):
        """
        Check if the Git repository or the ES index exists.
        Returns ``True`` if either of them exist.

        :returns: bool
        """
        if self.sm.storage_exists():
            return self.im.index_exists(self.sm.active_branch())

        return False

    def destroy(self):
        """
        Removes an ES index and a Git repository completely.
        Guaranteed to remove things completely, use with caution.
        """
        if self.sm.storage_exists():
            if self.im.index_exists(self.sm.active_branch()):
                self.im.destroy_index(self.sm.active_branch())
            self.sm.destroy_storage()

    def save(self, model, message, author=None, committer=None):
        """
        Save a :py:class:`elasticgit.models.Model` instance in Git and add it
        to the Elasticsearch index.

        :param elasticgit.models.Model model:
            The model instance
        :param str message:
            The commit message to write the model to Git with.
        :param tuple author:
            The author information (name, email address)
            Defaults repo default if unspecified.
        :param tuple committer:
            The committer information (name, email address).
            Defaults to the author if unspecified.
        """
        if isinstance(message, unicode):
            message = unidecode(message)
        self.sm.store(model, message, author=author, committer=committer)
        self.im.index(model)

    def delete(self, model, message, author=None, committer=None):
        """
        Delete a :py:class`elasticgit.models.Model` instance from Git and
        the Elasticsearch index.

        :param elasticgit.models.Model model:
            The model instance
        :param str message:
            The commit message to remove the model from Git with.
        :param tuple author:
            The author information (name, email address)
            Defaults repo default if unspecified.
        :param tuple committer:
            The committer information (name, email address).
            Defaults to the author if unspecified.
        """
        if isinstance(message, unicode):
            message = unidecode(message)
        self.sm.delete(model, message, author=author, committer=committer)
        self.im.unindex(model)

    def fast_forward(self, branch_name='master', remote_name='origin'):
        warnings.warn('This method is deprecated, use pull() instead',
                      DeprecationWarning)
        return self.pull(branch_name=branch_name, remote_name=remote_name)

    def index_diff(self, diff_index):
        # NOTE: This is probably more complicated than it needs to be
        #       If we have multiple remotes GitPython gets confused about
        #       deletes. It marks things as deletes because it may not
        #       exist on another remote.
        #
        #       Here we loop over all changes, track the models that've
        #       changed and then reindex fully to make sure we're in sync.
        if len(self.repo.remotes) > 1 and any(diff_index):
            return self.reindex_diff(diff_index)

        # NOTE: There's a very unlikely scenario where we're dealing with
        #       renames. This generally can only happen when a repository
        #       has been manually modififed. If that's the case then
        #       reindex everything as well
        if any(diff_index.iter_change_type('R')):
            return self.reindex_diff(diff_index)

        # unindex deleted blobs
        for diff in diff_index.iter_change_type('D'):
            path_info = self.sm.path_info(diff.a_blob.path)
            if path_info is None:
                continue
            self.im.raw_unindex(*path_info)

        # reindex added blobs
        for diff in diff_index.iter_change_type('A'):
            path_info = self.sm.path_info(diff.b_blob.path)
            if path_info is None:
                continue
            obj = self.sm.get(*path_info)
            self.im.index(obj)

        # reindex modified blobs
        for diff in diff_index.iter_change_type('M'):
            path_info = self.sm.path_info(diff.a_blob.path)
            if path_info is None:
                continue
            obj = self.sm.get(*path_info)
            self.im.index(obj)

    def reindex_diff(self, diff_index):
        changed_model_set = set([])
        for diff in diff_index:
            if diff.new_file:
                path_info = self.sm.path_info(diff.b_blob.path)
                if path_info is not None:
                    changed_model_set.add(path_info[0])
            elif diff.renamed:
                path_info = self.sm.path_info(diff.a_blob.path)
                if path_info is not None:
                    changed_model_set.add(path_info[0])
            else:
                path_info = self.sm.path_info(diff.a_blob.path)
                if path_info is not None:
                    changed_model_set.add(path_info[0])

        for model_class in changed_model_set:
            self.reindex(model_class)

    def pull(self, branch_name='master', remote_name='origin'):
        """
        Fetch & Merge in an upstream's commits.

        :param str branch_name:
            The name of the branch to fast forward & merge in
        :param str remote_name:
            The name of the remote to fetch from.
        """
        changes = self.sm.pull(branch_name=branch_name,
                               remote_name=remote_name)
        return self.index_diff(changes)

    def reindex_iter(self, model_class, refresh_index=True):
        """
        Reindex everything that Git knows about in an iterator

        :param elasticgit.models.Model model_class:
        :param bool refresh_index:
            Whether or not to refresh the index after everything has
            been indexed. Defaults to ``True``

        """
        if not self.im.index_exists(self.sm.active_branch()):
            self.im.create_index(self.sm.active_branch())
        iterator = self.sm.iterate(model_class)
        for model in iterator:
            yield self.im.index(model)

        if refresh_index:
            self.refresh_index()

    def reindex(self, model_class, refresh_index=True):
        """
        Same as :py:func:`reindex_iter` but returns a list instead of
        a generator.
        """
        return list(
            self.reindex_iter(model_class, refresh_index=refresh_index))

    def refresh_index(self):
        """
        Manually refresh the Elasticsearch index. In production this is
        not necessary but it is useful when running tests.
        """
        self.im.refresh_indices(self.sm.active_branch())

    def index_ready(self):
        """
        Check if the index is ready

        :returns: bool
        """
        return self.im.index_ready(self.sm.active_branch())

    def sync(self, model_class, refresh_index=True):
        """
        Resync a workspace, it assumes the Git repository is the source
        of truth and Elasticsearch is made to match. This involves two
        passes, first to index everything that Git knows about and
        unindexing everything that's in Elastisearch that Git does not
        know about.

        :param elasticgit.models.Model model_class:
            The model to resync
        :param bool refresh_index:
            Whether or not to refresh the index after indexing
            everything from Git

        """
        reindexed_uuids = set([])
        removed_uuids = set([])

        for model_obj in self.reindex_iter(model_class,
                                           refresh_index=refresh_index):
            reindexed_uuids.add(model_obj.uuid)

        for result in self.S(model_class).everything():
            if result.uuid not in reindexed_uuids:
                self.im.raw_unindex(model_class, result.uuid)
                removed_uuids.add(result.uuid)

        return reindexed_uuids, removed_uuids

    def setup_mapping(self, model_class):
        """
        Add a custom mapping for a model_class

        :param elasticgit.models.Model model_class:
        :returns: dict, the decoded dictionary from Elasticsearch
        """
        return self.im.setup_mapping(self.sm.active_branch(), model_class)

    def setup_custom_mapping(self, model_class, mapping):
        """
        Add a custom mapping for a model class instead of accepting
        what the model_class defines.

        :param elasticgit.models.Model model_class:
        :param dict: the Elastisearch mapping definition
        :returns: dict, the decoded dictionary from Elasticsearch
        """

        return self.im.setup_custom_mapping(
            self.sm.active_branch(), model_class, mapping)

    def get_mapping(self, model_class):
        """
        Get a mapping from Elasticsearch for a model_class
        :param elasticgit.models.Model model_class:
        :returns: dict
        """
        return self.im.get_mapping(self.sm.active_branch(), model_class)

    def S(self, model_class):
        """
        Get a :py:class:`elasticutils.S` object for the given
        model class. Under the hood this dynamically generates a
        :py:class:`elasticutils.MappingType` and
        :py:class:`elasticutils.Indexable` subclass which maps the
        Elasticsearch results to :py:class:`elasticgit.models.Model`
        instances on the UUIDs.

        :param elasticgit.models.Model model_class:
            The class to provide a search interface for.
        """
        return S(
            self.im.get_mapping_type(model_class)).es(**self.es_settings)

class RemoteWorkspace(Workspace):
    """
    A workspace that connects to a unicore.distribute server hosted
    somewhere on the network.

    This is a read only version of the :py:class:`Workspace`
    """
    def __init__(self, url, es=None, index_prefix=None):
        """
        :param str url:
            The URL of the unicore.distribute server.
        :param dict es:
            The parameters for connecting to Elasticsearch to. If not specified
            then the default unicore.distribute ES proxy would be used.
            This defaults to ``/esapi`` on the host of the ``url`` parameter
            provided.
        :param str index_prefix:
            The prefix to use when generating index names for Elasticsearch
        """
        self.sm = RemoteStorageManager(url)
        self.index_prefix = index_prefix or self.sm.repo_name
        self.es_settings = es or {'urls': urljoin(url, '/esapi')}
        self.im = ESManager(
            self.sm,
            es=get_es(**self.es_settings),
            index_prefix=self.index_prefix)

    def reindex_changes(self, changes):
        changed_model_set = set([])
        for change in changes:
            if change['type'] == 'A':
                path_info = self.sm.path_info(change['path'])
                if path_info is not None:
                    changed_model_set.add(path_info[0])
            elif change['type'] == 'R':
                path_info = self.sm.path_info(change['rename_to'])
                if path_info is not None:
                    changed_model_set.add(path_info[0])
            else:
                path_info = self.sm.path_info(change.get('path'))
                if path_info is not None:
                    changed_model_set.add(path_info[0])

        for model_class in changed_model_set:
            self.reindex(model_class)

    def pull(self, branch_name='master', remote_name='origin'):
        changes = self.sm.pull(branch_name=branch_name,
                               remote_name=remote_name)

        def pick_type(change_type):
            return filter(
                lambda change: change['type'] == change_type, changes)

        # NOTE: There's a very unlikely scenario where we're dealing with
        #       renames. This generally can only happen when a repository
        #       has been manually modififed. If that's the case then
        #       reindex everything as well
        if pick_type('R'):
            return self.reindex_changes(changes)

        # unindex deleted blobs
        for diff in pick_type('D'):
            path_info = self.sm.path_info(diff['path'])
            if path_info is None:
                continue
            self.im.raw_unindex(*path_info)

        # reindex added blobs
        for diff in pick_type('A'):
            path_info = self.sm.path_info(diff['path'])
            if path_info is None:
                continue
            obj = self.sm.get(*path_info)
            self.im.index(obj)

        # reindex modified blobs
        for diff in pick_type('M'):
            path_info = self.sm.path_info(diff['path'])
            if path_info is None:
                continue
            obj = self.sm.get(*path_info)
            self.im.index(obj)

class EG(object):

    """
    A helper function for things in ElasticGit.

    """
    @classmethod
    def workspace(cls, workdir, es={}, index_prefix=None):
        """
        Create a workspace

        :param str workdir:
            The path to the directory where a git repository can
            be found or needs to be created when
            :py:meth:`.Workspace.setup` is called.
        :param dict es:
            The parameters to pass along to :func:`elasticutils.get_es`
        :param str index_prefix:
            The index_prefix use when generating index names for
            Elasticsearch
        :returns:
            :py:class:`.Workspace`
        """
        index_prefix = index_prefix or os.path.basename(workdir)
        repo = (cls.read_repo(workdir)
                if cls.is_repo(workdir)
                else cls.init_repo(workdir))
        return Workspace(repo, es, index_prefix)

    @classmethod
    def dot_git_path(cls, workdir):
        return os.path.join(workdir, '.git')

    @classmethod
    def is_repo(cls, workdir):
        return cls.is_dir(cls.dot_git_path(workdir))

    @classmethod
    def is_dir(cls, workdir):
        return os.path.isdir(workdir)

    @classmethod
    def read_repo(cls, workdir):
        return Repo(workdir)

    @classmethod
    def init_repo(cls, workdir, bare=False):
        return Repo.init(workdir, bare=bare)

    @classmethod
    def clone_repo(cls, repo_url, workdir):
        return Repo.clone_from(repo_url, workdir)

Q
F


# Self-supervised Spatiotemporal Learning via Video Clip Order Prediction
import torch
import numpy as np
import math
import itertools

def order_class_index(order):
    """Return the index of the order in its full permutation.

    Args:
        order (tensor): e.g. [0,1,2]
    """
    classes = list(itertools.permutations(list(range(len(order)))))
    return classes.index(tuple(order.tolist()))

def train(args, model, criterion, optimizer, device, train_dataloader, writer, epoch):
    torch.set_grad_enabled(True)
    model.train()

    running_loss = 0.0
    correct = 0
    if args.use_clip:
        tag = 'clip'
    elif args.use_front:
        tag = 'front'
    elif args.use_360:
        tag = '360'
    print("Start training...")

    target_list = []
    pred_list = []

    for i, data in enumerate(train_dataloader, 1):
        # get inputs
        if args.dataset == '360x':
            tuple_orders = data[tag]["order"]
        else:
            tuple_clips, tuple_orders = data

        # inputs = tuple_clips.to(device)
        targets = [order_class_index(order) for order in tuple_orders]
        targets = torch.tensor(targets).to(device)

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward and backward
        outputs = model(data) # return logits here
        loss = criterion(outputs, targets)
        loss.backward()
        optimizer.step()
        # compute loss and acc
        running_loss += loss.item()

        pts = torch.argmax(outputs, dim=1)
        correct += torch.sum(targets == pts).item()

        target_list.extend(targets.cpu().detach().numpy())
        pred_list.extend(pts.cpu().detach().numpy())

        # print statistics and write summary every N batch
        if i % args.pf == 0:
            avg_loss = running_loss / args.pf
            p = np.asarray(pred_list).flatten()

            print("pred_list:", pred_list)

            avg_acc = np.sum(target_list == p)/p.shape[0]
            target_list = []
            pred_list = []

            # avg_acc = # correct / (args.pf * args.bs)
            print('[TRAIN] epoch-{}, batch-{}, loss: {:.3f}, acc: {:.3f}'.format(epoch, i, avg_loss, avg_acc))
            step = (epoch -1 ) *len(train_dataloader) + i
            writer.add_scalar('train/CrossEntropyLoss', avg_loss, step)
            writer.add_scalar('train/Accuracy', avg_acc, step)
            running_loss = 0.0
            correct = 0

    # summary params and grads per eopch
    for name, param in model.named_parameters():
        try:
            writer.add_histogram('params/{}'.format(name), param, epoch)
            writer.add_histogram('grads/{}'.format(name), param.grad, epoch)
        except:
            pass

def validate(args, model, criterion, device, val_dataloader, writer, epoch):
    torch.set_grad_enabled(False)
    model.eval()

    total_loss = 0.0
    correct = 0
    if args.use_clip:
        tag = 'clip'
    elif args.use_front:
        tag = 'front'
    elif args.use_360:
        tag = '360'

    target_list = []
    pred_list = []
    for i, data in enumerate(val_dataloader):
        # get inputs
        if args.dataset == '360x':
            tuple_orders = data[tag]["order"]

        else:
            tuple_clips, tuple_orders = data

        # inputs = tuple_clips.to(device)
        targets = [order_class_index(order) for order in tuple_orders]
        targets = torch.tensor(targets).to(device)

        # forward
        outputs = model(data) # return logits here
        loss = criterion(outputs, targets)
        # compute loss and acc
        total_loss += loss.item()

        pts = torch.argmax(outputs, dim=1)
        correct += torch.sum(targets == pts).item()

        target_list.extend(targets.cpu().detach().numpy())
        pred_list.extend(pts.cpu().detach().numpy())
        # print('correct: {}, {}, {}'.format(correct, targets, pts))

    target_list = np.asarray(target_list).flatten()
    pred_list = np.asarray(pred_list).flatten()

    print("pred_list:", pred_list)

    avg_loss = total_loss / len(val_dataloader)

    avg_acc = np.sum(target_list == pred_list)/pred_list.shape[0] #correct / len(val_dataloader.dataset)

    writer.add_scalar('val/CrossEntropyLoss', avg_loss, epoch)
    writer.add_scalar('val/Accuracy', avg_acc, epoch)

    print('[VAL] loss: {:.3f}, acc: {:.3f}'.format(avg_loss, avg_acc))

    return avg_loss

def test(args, model, criterion, device, test_dataloader):
    torch.set_grad_enabled(False)
    model.eval()

    total_loss = 0.0
    correct = 0
    if args.use_clip:
        tag = 'clip'
    elif args.use_front:
        tag = 'front'
    elif args.use_360:
        tag = '360'

    for i, data in enumerate(test_dataloader, 1):
        # get inputs
        if args.dataset == '360x':
            tuple_clips = data[tag]["f"]
            tuple_orders = data[tag]["order"]
        else:
            tuple_clips, tuple_orders = data

        inputs = tuple_clips.to(device)
        targets = [order_class_index(order) for order in tuple_orders]
        targets = torch.tensor(targets).to(device)
        # forward
        outputs = model(inputs)
        loss = criterion(outputs, targets)
        # compute loss and acc
        total_loss += loss.item()
        pts = torch.argmax(outputs, dim=1)
        correct += torch.sum(targets == pts).item()
        # print('correct: {}, {}, {}'.format(correct, targets, pts))
    avg_loss = total_loss / len(test_dataloader)
    avg_acc = correct / len(test_dataloader.dataset)
    print('[TEST] loss: {:.3f}, acc: {:.3f}'.format(avg_loss, avg_acc))
    return avg_loss


"""
Based on https://morvanzhou.github.io/tutorials/
"""

import gym
import multiprocessing
import threading
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

np.random.seed(2)
tf.set_random_seed(2)
GAME = 'Pendulum-v0'
UPDATE_GLOBAL_ITER = 10
GLOBAL_RUNNING_R = []
GLOBAL_EP = 0
MAX_EP_STEP = 200  # Pendulum never terminated, so set a max_loop
MAX_GLOBAL_EP = 2000
N_WORKERS = multiprocessing.cpu_count()
OUTPUT_GRAPH = True

#
# def _build_session(graph):
#     config = tf.ConfigProto()
#     config.gpu_options.per_process_gpu_memory_fraction = 0.8  # 程序最多只能占用指定gpu80%的显存
#     config.gpu_options.allow_growth = True  # 程序按需申请内存
#     sess = tf.Session(graph=graph, config=config)
#     return sess

class ACNet(object):
    def __init__(self, *, scope: str, n_actions: int, n_features: int, action_bounds: list, reward_decay=0.9,
                 entropy_beta=0.01, actor_learning_rate=0.0001, critic_learning_rate=0.001,
                 is_global: bool=False, globalAC=None):
        self.scope = scope
        self.n_actions = n_actions  # 1
        self.n_features = n_features
        self.action_low, self.action_high = action_bounds
        self.gamma = reward_decay
        self.entropy_beta = entropy_beta
        self.actor_lr = actor_learning_rate
        self.critic_lr = critic_learning_rate
        self.is_global = is_global
        self.globalAC = globalAC
        self._build_graph()

    def _build_net(self):
        with tf.variable_scope("critic"):
            c_l1 = tf.layers.dense(inputs=self.s, units=100, activation=tf.nn.relu6,
                                   kernel_initializer=self.w_init,
                                   bias_initializer=self.b_init,
                                   name="c_l1")
            v = tf.layers.dense(inputs=c_l1, units=1, activation=None,
                                kernel_initializer=self.w_init,
                                bias_initializer=self.b_init,
                                name="V")

        with tf.variable_scope("actor"):
            a_l1 = tf.layers.dense(inputs=self.s, units=200, activation=tf.nn.relu6,
                                   kernel_initializer=self.w_init,
                                   bias_initializer=self.b_init,
                                   name="a_l1")
            # [None, n_actions]
            mu = tf.layers.dense(inputs=a_l1, units=self.n_actions, activation=tf.nn.tanh,
                                 kernel_initializer=self.w_init,
                                 bias_initializer=self.b_init,
                                 name="mu")
            # [None, n_actions]
            sigma = tf.layers.dense(inputs=a_l1, units=self.n_actions, activation=tf.nn.softplus,
                                    kernel_initializer=self.w_init,
                                    bias_initializer=self.b_init,
                                    name="sigma")

        actor_params = tf.get_collection(key=tf.GraphKeys.TRAINABLE_VARIABLES,
                                         scope=self.scope + "/actor")
        critic_params = tf.get_collection(key=tf.GraphKeys.TRAINABLE_VARIABLES,
                                          scope=self.scope + "/critic")
        return mu, sigma, v, actor_params, critic_params

    def _build_graph(self):
        self.w_init = tf.random_normal_initializer(mean=0, stddev=0.1)
        self.b_init = tf.constant_initializer(0.0)
        if self.is_global:
            with tf.variable_scope(self.scope):
                self.s = tf.placeholder(dtype=tf.float32, shape=[None, self.n_features], name="S")
                _, _, _, self.actor_params, self.critic_params = self._build_net()
        else:
            with tf.variable_scope(self.scope):
                self.s = tf.placeholder(dtype=tf.float32, shape=[None, self.n_features], name="S")
                self.a_his = tf.placeholder(dtype=tf.float32, shape=[None, self.n_actions], name="A")  # 分布中取概率密度，需要float32
                self.v_target = tf.placeholder(dtype=tf.float32, shape=[None, 1], name="V_target")
                mu, sigma, self.v, self.actor_params, self.critic_params = self._build_net()

                td = tf.subtract(self.v_target, self.v, name="td_error")  # tf.subtract element-wise x - y
                with tf.name_scope("critic_loss"):
                    # mse tf.reduce_mean(tf.squared_difference(x, y))
                    critic_loss = tf.reduce_mean(tf.square(td))

                with tf.name_scope("wrap_action_out"):
                    mu = mu * self.action_high
                    sigma = sigma + 1e-4

                normal_dist = tf.distributions.Normal(loc=mu, scale=sigma)  # action的概率分布

                with tf.name_scope("actor_loss"):
                    log_prob = normal_dist.log_prob(value=self.a_his)
                    weighted_log_prob = log_prob * tf.stop_gradient(td)
                    entropy = normal_dist.entropy()
                    actor_loss = tf.reduce_mean(-(entropy * self.entropy_beta + weighted_log_prob))

                with tf.name_scope("choose_action"):
                    self.A = tf.clip_by_value(tf.squeeze(normal_dist.sample(sample_shape=1), axis=[0, 1]),
                                              clip_value_min=self.action_low,
                                              clip_value_max=self.action_high)

                with tf.name_scope("local_grad"):
                    self.actor_grads = tf.gradients(actor_loss, self.actor_params)
                    self.critic_grads = tf.gradients(critic_loss, self.critic_params)

                with tf.name_scope("opt"):
                    actor_opt = tf.train.RMSPropOptimizer(learning_rate=self.actor_lr)
                    critic_opt = tf.train.RMSPropOptimizer(learning_rate=self.critic_lr)

            with tf.name_scope("sync"):
                with tf.name_scope("pull"):
                    """从global同步参数"""
                    self.actor_params_pull_op = [local_p.assign(global_p) for local_p, global_p in
                                                 zip(self.actor_params, self.globalAC.actor_params)]
                    self.critic_params_pull_op = [local_p.assign(global_p) for local_p, global_p in
                                                  zip(self.critic_params, self.globalAC.critic_params)]
                with tf.name_scope("push"):
                    """将梯度更新至global"""
                    self.actor_params_push_op = \
                        actor_opt.apply_gradients(zip(self.actor_grads, self.globalAC.actor_params))
                    self.critic_params_push_op = \
                        critic_opt.apply_gradients(zip(self.critic_grads, self.globalAC.critic_params))

    def update_global(self, *, sess, s, a, v_target):
        s = np.asarray(s).reshape(-1, self.n_features)
        a = np.asarray(a).reshape(-1, self.n_actions)
        v_target = np.asarray(v_target).reshape(-1, 1)
        feed_dict = {self.s: s, self.a_his: a, self.v_target: v_target}
        sess.run([self.actor_params_push_op, self.critic_params_push_op], feed_dict=feed_dict)

    def sync_with_global(self, sess):
        sess.run([self.actor_params_pull_op, self.critic_params_pull_op])

    def choose_action(self, sess: tf.Session, s):
        s = np.asarray(s).reshape(-1, self.n_features)
        return sess.run(self.A, feed_dict={self.s: s})

class Worker(object):
    def __init__(self, scope, globalAC: ACNet):
        self.scope = scope
        self.env: gym.Env = gym.make(GAME).unwrapped
        self.globalAC = globalAC
        self.AC = ACNet(scope=scope,
                        n_actions=self.env.action_space.shape[0],
                        n_features=self.env.observation_space.shape[0],
                        action_bounds=[self.env.action_space.low[0], self.env.action_space.high[0]],
                        is_global=False,
                        globalAC=globalAC)
        self._reset_buffer()

    def _reset_buffer(self):
        self.buffer_s = []
        self.buffer_a = []
        self.buffer_r = []

    def work(self, COORD, SESS):
        global GLOBAL_RUNNING_R, GLOBAL_EP
        total_step = 1
        while not COORD.should_stop() and GLOBAL_EP < MAX_GLOBAL_EP:
            s = self.env.reset()
            ep_r = 0

            for ep_t in range(MAX_EP_STEP):
                a = self.AC.choose_action(sess=SESS, s=s)
                s_, r, done, info = self.env.step(a)  # need a (1,) shape
                done = True if ep_t == MAX_EP_STEP - 1 else False  # 最后一个循环为done
                ep_r += r
                self.buffer_s.append(s)
                self.buffer_a.append(a)
                self.buffer_r.append((r + 8) / 8)  # normalized to (-1, 1)

                if total_step % UPDATE_GLOBAL_ITER == 0 or done:
                    v_s_ = 0 if done else np.asscalar(SESS.run(self.AC.v, feed_dict={self.AC.s: [s_]}))
                    # trans buffer_r to buffer_v_target by v(s) = gamma * v(s_) + r circularly
                    for i in range(len(self.buffer_r) - 1, -1, -1):
                        v_s_ = v_s_ * self.AC.gamma + self.buffer_r[i]
                        self.buffer_r[i] = v_s_
                    # maybe misleading buffer_r is buffer_v_target now
                    self.AC.update_global(sess=SESS, s=self.buffer_s, a=self.buffer_a, v_target=self.buffer_r)  # 更新
                    self.AC.sync_with_global(sess=SESS)
                    self._reset_buffer()
                s = s_
                total_step += 1

                if done:
                    if not GLOBAL_RUNNING_R:
                        GLOBAL_RUNNING_R.append(ep_r)
                    else:
                        GLOBAL_RUNNING_R.append(0.9 * GLOBAL_RUNNING_R[-1] + 0.1 * ep_r)
                    print("%s  E.P.: %d  EP_r: %i" % (self.scope, GLOBAL_EP, GLOBAL_RUNNING_R[-1]))
                    GLOBAL_EP += 1
                    break

if __name__ == "__main__":
    env = gym.make(GAME).unwrapped
    n_actions = env.action_space.shape[0]
    action_bounds = [env.action_space.low[0], env.action_space.high[0]]
    n_features = env.observation_space.shape[0]

    tf.reset_default_graph()
    graph = tf.Graph()
    with tf.device("/CPU:0"):  # 都放在内存里
        with graph.as_default():
            GLOBAL_AC = ACNet(scope="GLOBAL", n_actions=n_actions, action_bounds=action_bounds,
                              n_features=n_features, is_global=True)
            workers = []
            for i in range(N_WORKERS):
                scope = "Worker_%d" % i
                workers.append(Worker(scope=scope, globalAC=GLOBAL_AC))
            init = tf.global_variables_initializer()

    SESS = tf.Session(graph=graph, config=tf.ConfigProto(device_count={'GPU': 0}))  # 使用cpu计算
    SESS.run(init)
    COORD = tf.train.Coordinator()

    if OUTPUT_GRAPH:
        tf.summary.FileWriter("F:/board/A3C/", graph)

    worker_threads = []
    for worker in workers:
        t = threading.Thread(target=lambda: worker.work(COORD=COORD, SESS=SESS))
        t.start()
        worker_threads.append(t)
    COORD.join(threads=worker_threads)

    plt.plot(np.arange(len(GLOBAL_RUNNING_R)), GLOBAL_RUNNING_R)
    plt.xlabel("step")
    plt.ylabel("Total moving reward")
    plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri May 19 12:15:08 2023

@author: aaa
"""

import os
import numpy as np
import shutil
import matplotlib.pyplot as plt
from mpl_toolkits import mplot3d
from matplotlib.gridspec import GridSpec
plt.rcParams.update({'figure.max_open_warning': 0})
from PIL import Image as im
from scipy.optimize import curve_fit as fit
import warnings
warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning)

w_ps=8.002

a1=1/5**0.5
a2=2*a1
a21=2
def exp_w1p(x,x0):
    return alpha*((1/(1+a21*np.exp(-1j*(w_ps*(x-x0)))))).real

def fit_cos(x,A,B,C,D):
    return A+B*np.cos(C*x-D)

def I_px_co(beta, chi, C, alpha, gamma):
    d=((alpha+beta)**2+gamma**2)**0.5
    e=(beta**2+gamma**2)**0.5
    return C*((a1*np.cos(d/2))**2+(a2*np.cos(e/2))**2+2*a1*a2*np.cos(d/2)*np.cos(e/2)*np.cos(chi))/4

def I_px_in(beta, chi, eta, alpha, gamma):
    d=((alpha+beta)**2+gamma**2)**0.5
    e=(beta**2+gamma**2)**0.5
    return eta*(np.cos(d/2)**2+(a2/a1)**2*np.cos(e/2)**2)/4

def I_mx_co(beta, chi, C, alpha, gamma):
    d=((alpha+beta)**2+gamma**2)**0.5
    e=(beta**2+gamma**2)**0.5
    r=np.arctan(gamma/beta)
    p=np.arctan(gamma/(beta+gamma))
    return C*((a1*np.sin(d/2))**2+(a2*np.sin(e/2))**2+2*a1*a2*np.sin(d/2)*np.sin(e/2)*np.cos(chi+r-p))/4

def I_mx_in(beta, chi, eta, alpha, gamma):
    d=((alpha+beta)**2+gamma**2)**0.5
    e=(beta**2+gamma**2)**0.5
    return eta*(np.sin(d/2)**2+(a2/a1)**2*np.sin(e/2)**2)/4
beta=0
g=np.linspace(-np.pi/2,np.pi/2,100)
c=np.linspace(-2*np.pi,2*np.pi,25)
alpha=np.pi/2

def I_px(x, gamma0, chi0, C, eta):
    gamma=g-gamma0
    chi=c-chi0
    gamma, chi = np.meshgrid(gamma, chi)
    fit_I_px=I_px_co(beta, chi, C, alpha, gamma) #+ I_px_in(0, chi, eta, alpha, gamma)
    return fit_I_px

def I_mx(x, gamma0, chi0, C, eta):
    gamma=g-gamma0
    chi=c-chi0
    gamma, chi = np.meshgrid(gamma, chi)
    fit_I_mx=I_mx_co(beta, chi, C, alpha, gamma) #+ I_mx_in(0, chi, eta, alpha, gamma)
    return fit_I_mx

p=[0,0,1,0]
# beta = exp_w1p(c[i], 0)
Z= I_px(0,*p)
# Z0= I_px(0,0,0,1,0)
# Z2p= I_px(0,2*np.pi,0,1,0.2)
# Z= (I_px(0,*p)-I_mx(0,*p))/(I_px(0,*p)+I_mx(0,*p))

# fig = plt.figure(figsize=(10,10))
# ax = plt.axes(projection='3d')
# x, y = np.meshgrid(g, c)
# ax.contour3D(x, y, Z, 30, cmap='plasma')#cmap='Blues')
# ax.set_xlabel('$\\gamma$')
# ax.set_ylabel('$\chi$')
# ax.set_zlabel('z')
# ax.view_init(40, 45)
wg=np.zeros((len(c)))
for i in range(len(c)):
    fig = plt.figure(figsize=(5,5))
    ax = fig.add_subplot(111)
    ax.set_title(str(c[i]))
    ax.plot(g,Z[0])
    ax.vlines(0,0,np.amax(Z[0]))
    ax.plot(g,Z[i])
    # ax.set_ylim([0,0.05])
    # ax.set_xlim([2.5,5])
    # ax.plot(g,g*0+1)
#     wg[i]=Z2p[i,0]-Z0[i,0]

# fig = plt.figure(figsize=(5,5))
# ax = fig.add_subplot(111)
# ax.plot(c,g)
plt.show()

Grenoble 1st round/Scripts/Imperfect interferometer sim.py


from django.shortcuts import render, get_object_or_404
from django.contrib.auth.decorators import login_required
from django.http import JsonResponse
from .models import organisation, product
from taggit.models import Tag
from django.db.models import Count

@login_required(login_url='/login')
def products_request(request, tag_slug=None):
    products_list = product.objects.all()
    tag = None

    if tag_slug:
        tag = get_object_or_404(Tag, slug=tag_slug)
        products_list = products_list.filter(product_tags__in=[tag])

    return render(request, 'organisations/productsList.html', {"products": products_list,
                                                               "tag": tag})

@login_required(login_url='/login')
def product_request(request, publish, prod):
    print(prod)
    product_item = get_object_or_404(product, product_name=prod,
                                     publish__year=publish.split('.')[2],
                                     publish__month=publish.split('.')[1],
                                     publish__day=publish.split('.')[0])
    product_tags_ids = product_item.product_tags.values_list('id', flat=True)
    similar_products = product.objects.filter(product_tags__in=product_tags_ids).exclude(id_product=product_item.id_product)
    print('Похожее:', similar_products)
    similar_products = similar_products.annotate(same_tags=Count('product_tags')).order_by('-same_tags', '-publish')[:4]

    return render(request, 'organisations/product.html', context={'product_item': product_item,
                                                                  'similar_products': similar_products})

@login_required(login_url='/login')
def update_product(request):
    if request.method == 'POST':
        name = request.POST['product']
        new_product_name = request.POST['newproductname']
        new_product_description = request.POST['newproductdescription']
        username = str(request.user)

        if ("/" in new_product_name):
            return JsonResponse({}, status=405)

        try:
            if (new_product_name != name): # если название отличное от текущего
                existing_product = product.objects.get(
                        product_name=new_product_name)
                if (existing_product): # если такой продукт уже есть
                    return JsonResponse({}, status=402)
        except BaseException:
            pass

        try:
            product_to_update = product.objects.get(product_name=name)
            product_to_update.product_name = new_product_name
            product_to_update.product_description = new_product_description
            product_to_update.save()
            return JsonResponse({}, status=200)
        except BaseException:
            print('error')
            return JsonResponse({}, status=400)
    return JsonResponse({}, status=400)

def organisation_request(request, inn):
    org_item = get_object_or_404(organisation, organisation_inn=inn)
    return render(request, 'organisations/organisation.html', context={'organisation_item': org_item})

@login_required(login_url='/login')
def organisations_request(request):
    organisations_list = organisation.objects.all()
    return render(request, 'organisations/organisationsList.html', {"organisations": organisations_list})

@login_required(login_url='/login')
def employee_json(request):
    employees = organisation.objects.all()
    data = [employee.get_data() for employee in employees]
    response = {'data': data}
    return JsonResponse(response)