__author__='Rakesh Nath'
__all__=['measureSpatialSpec',
          'applyFilter']
import numpy as np
from astropy.io import fits
from matplotlib import pyplot as plt
from photutils import CircularAperture,aperture_photometry
import vip_hci
from scipy.signal import savgol_filter
from vip_hci import pca
from sklearn.decomposition import pca
import matplotlib as mpl
#from parallelCompareTemplates import CrossCorr
import pandas as pd
import glob
from vip_hci.pca.svd import svd_wrapper
from vip_hci.var.shapes import frame_center
from ._utils import find_nearest
from .removeTelluric import removeTelluric

font = {'family' : 'serif',
                'weight' : 'bold'}

mpl.rc('font', **font)
def measureSpatialSpec(datcube,loc,fwhm):
    """
    This is a function to measure the the spectrum spatially

    Parameters
    ----------
    datcube : 3d asarray
        The 3d data cube which has the format wavelength,xx,yy
    loc : list
        location at which the spectrum needs to be measured in [yy,xx]
    fwhm : list or array
        List of full-width half max

    Returns
    -------
    array :
        Measured spectrum

    """
    slices=[]
    for wl in range(datcube.shape[0]):
         apertures=CircularAperture([loc[0],loc[1]],r=fwhm[wl]/2)
    #    #print(aperture_p)
         slices.append(np.float64(aperture_photometry(datcube[wl,:,:],apertures)['aperture_sum']))
    return np.asarray(slices)
def applyFilter(flux,window_size,order):
    """
    Function  to apply the Savitzky-Golay filter and subtract the filtered
    spectrum from the target.

    Parameters
    ----------
    flux : array
        Spectrum to which this needs to be applied
    window_size: int
        The size of the savgol filter in bins
    order : int
        The polynomial order of the filter

    Returns
    -------
    array :
        The spectrum after subtracting the savgol filter from flux
    """
    flux_filt=savgol_filter(flux,window_size,order)
    flux_dat=flux-flux_filt
    return flux_dat
class SINFONI:
    """
    This is a class that defines data process for a SINFONI like instrument

    Attributes
    -----------
        datpath : str
            location where your data resides.

        filename : str
            filename of the data file
        fwhm : str
            filename of the FWHM file
        wavelen : str
            filename of the wavelength solution

        vels : array
            velocity array should have at least upto 1100 km/s

        wmin_max : list
                The min max frequencies that need to be set up for this
        sz : int
                The size to which the cube needs to be cropped down to. Inherits VIP crop_sz
    Methods
    -------
       preProcessSINFONI(polyorder,window_size,n_comps)
            Does the pre processing with basic processing. Subtracting stellar spectra
            and returning a cube after PC is subtracted if n_comps>0
       valentinPreProcess(perc,polyorder,window_size,n_comps)
            Does pre process as per Haffert et al with more accurate stellar modeling. Returns
            cube as before
    """
    def __init__(self,*args,**kwargs):
        self.datpath=kwargs.get('datpath',
                                "/mnt/disk4tb/Users/rnath/Data/HD142527/")
        self.filename=kwargs.get('filename',
                                  'ASDI_cube_derot_med.fits')
        self.cube=fits.open(self.datpath+self.filename)
        self.fwhm=fits.open(self.datpath+kwargs.get('fwhm',"fwhm_vec.fits"))
        self.wavelen=fits.open(self.datpath+kwargs.get('wavelen',"lbda_vec.fits"))
        self.vels=kwargs.get('vels',np.arange(-2000,2000,20))
        self.wmin_max=kwargs.get('wmin_max',[1.47,1.6])
        self.crop_sz=np.int(kwargs.get('sz',71))
        self.cube[0].data=vip_hci.preproc.cube_crop_frames(
        self.cube[0].data,self.crop_sz)
        self.normalized_cube=0
        self.waves_dat=0
        self.fwhm_final=0
        self.sum_spax=0.
        self.wmin_wmax_tellurics=kwargs.get("wmin_wmax_tellurics",[1.75,2.1])
        self.V=0
        self.spec=0

    def preProcessSINFONI(self,*args,**kwargs):
        """
        Performs some basic stellar subtraction on the cube. if n_comps>0
        performs the PCA

        Parameters
        ----------
        polyorder (int optional, default = 1):
                Order of the Savitzky-Golay filter. See applyFilter.
        window_size (int option, default =101):
                Window size has to be odd
        n_comps (int, default=2):
                number of components to be subtractd.

        Returns
        -------
        float :
            Returns a 3D matrix formatted (wl,RA,DEC)
        """

        idx_wmin = find_nearest(self.wavelen[0].data,self.wmin_max[0])
        idx_wmax = find_nearest(self.wavelen[0].data,self.wmin_max[1])
        print("We are reading %s"%(self.datpath+self.filename))
        cube_whole = self.cube[0].data[idx_wmin:idx_wmax,:,:]
        wavelen = self.wavelen[0].data[idx_wmin:idx_wmax]
        self.waves_dat=wavelen
        fwhm = self.fwhm[0].data[idx_wmin:idx_wmax]
        self.fwhm_final=fwhm
        #sum the spaxels
        cube_notell=np.zeros_like(cube_whole)
        for xx in range(cube_notell.shape[1]):
            for yy in range(cube_notell.shape[2]):
                self.spec=(cube_whole[:,xx,yy])
                self.spec=removeTelluric(wavelen,self.spec,self.wmin_wmax_tellurics[0],self.wmin_wmax_tellurics[1])
                cube_notell[:,xx,yy]=self.spec
                #cube_notell[:,xx,yy]=removeTelluric(wavelen,cube_whole[:,xx,yy],self.wmin_wmax_tellurics[0],self.wmin_wmax_tellurics[1])
        #center=int(self.crop_sz/2)
        center=frame_center(cube_notell)
        self.sum_spax=measureSpatialSpec(cube_notell,center,self.fwhm_final)
        print("Wavelength spans from %2.1f to %2.1fmum"%(wavelen.min(),wavelen.max()))

        #two steps now, divide the whole cube by the ref_spec and also filter
        #for lower order residuals using savgol filter. option to subtract the reference also.
        polyorder=kwargs.get('polyorder',1)
        window_size=kwargs.get('window_size',101)
        self.normalized_cube=np.zeros(cube_notell.shape)
        filt=np.zeros(cube_notell.shape)
        norm_cube=np.zeros(cube_notell.shape)
        for xx in range(cube_notell.shape[1]):
            for yy in range(cube_notell.shape[2]):
                    norm_cube[:,xx,yy]=cube_notell[:,xx,yy]/self.sum_spax
                    self.normalized_cube[:,xx,yy]=applyFilter(norm_cube[:,xx,yy],
                    window_size,polyorder)
                    self.normalized_cube[:,xx,yy]=removeTelluric(wavelen,self.normalized_cube[:,xx,yy],self.wmin_wmax_tellurics[0],self.wmin_wmax_tellurics[1])
        #self.normalized_cube[np.where(filt!=0)]=norm_cube[np.where(filt!=0)]-filt[np.where(filt!=0)]
        recon_3dcube=np.zeros_like(self.normalized_cube)
        ncomp=kwargs.get("n_comps",2)
        if(ncomp==0):
            recon_3dcube=self.normalized_cube
            print("no PCA ")
        else:
            PCA_matrix=np.reshape(np.zeros(len(wavelen)*cube_notell.shape[1]*cube_notell.shape[2]),
                    (len(wavelen),cube_notell.shape[1]*cube_notell.shape[2]))

            for wl in range(len(wavelen)):
                for xx in range(cube_notell.shape[1]):
                    for yy in range(cube_notell.shape[2]):
                        PCA_matrix[wl,xx*cube_notell.shape[1]+yy]=self.normalized_cube[wl,xx,yy]

            self.V=svd_wrapper(PCA_matrix.T,'lapack',ncomp,verbose=True)
            transformed=np.dot(self.V,PCA_matrix)
            reconstructed=np.dot(transformed.T,self.V)
            residuals=PCA_matrix-reconstructed.T

            #re-wrap into a cube
            for wl in range(idx_wmax-idx_wmin):
                for xx in range(cube_notell.shape[1]):
                    for yy in range(cube_notell.shape[2]):
                        recon_3dcube[wl,xx,yy]=residuals[wl,cube_notell.shape[1]*xx+yy]
        #waves=np.load("/mnt/diskss/home/rnath/Numpy_PDS70/wavelens_new28.npy")
        #locs=np.where(self.wavelen[0].data>=0)

        return recon_3dcube
    def valentinPreProcessSINFONI(self,*args,**kwargs):
        """
        Performs stellar subtraction on the cube based on Haeffert et al. if n_comps>0
        performs the PCA

        Parameters
        ----------
        perc (float optional, default = 95):
                Percentage of pixels to consider around center.

        polyorder (int optional, default = 1):
                Order of the Savitzky-Golay filter. See applyFilter.
        window_size (int option, default =101):
                Window size has to be odd
        n_comps (int, default=2):
                number of components to be subtractd.

        Returns
        -------
        float :
            Returns a 3D matrix formatted (wl,RA,DEC)
        """

        idx_wmin = find_nearest(self.wavelen[0].data,self.wmin_max[0])
        idx_wmax = find_nearest(self.wavelen[0].data,self.wmin_max[1])
        cube = self.cube[0].data[idx_wmin:idx_wmax,:,:]
        wavelen = self.wavelen[0].data[idx_wmin:idx_wmax]
        self.waves_dat=wavelen
        fwhm = self.fwhm[0].data[idx_wmin:idx_wmax]
        cube_nt=np.zeros(cube.shape)
        for xx in range(cube.shape[1]):
            for yy in range(cube.shape[2]):
                cube_nt[:,xx,yy]=removeTelluric(wavelen,cube[:,xx,yy],wmin=self.wmin_wmax_tellurics[0],wmax=self.wmin_wmax_tellurics[1])
                #sum the spaxels
        med_frame= np.median(cube_nt, axis=0)
        perc=0.95

        thr = np.percentile(med_frame,perc)
        idx_high = np.where(med_frame>thr)

        # take the spectrum of the star at the location of the brightest pixels
        spec_high= []
        for wl in range(len(wavelen)):
            spec_high.append(cube_nt[wl][idx_high])
        spec_high = np.array(spec_high)
        norm_specs_high = np.ones_like(spec_high)
        sum_spax_high= np.sum(spec_high,axis=0)
        # normalized spectra for each of the top percentile spaxels:
        for wl in range(len(wavelen)):
            norm_specs_high[wl] = spec_high[wl]/sum_spax_high

        # averaging spatially among the perc% brightest pixels:
        ref_spec=[]
        for wl in range(len(wavelen)):
            ref_spec.append(np.median(norm_specs_high[wl]))
        sum_spax_high= np.sum(spec_high,axis=0)
        norm_cube = cube_nt.copy()
        spat_sum = np.sum(cube_nt,axis=0)

        for zz in range(cube_nt.shape[0]):
            norm_cube[zz]=cube_nt[zz]/(spat_sum*ref_spec[zz])
        res_cube = np.zeros_like(norm_cube)
        norm_res_cube = np.zeros_like(norm_cube)
        filt = np.zeros_like(norm_cube)
        polyorder=kwargs.get('polyorder',1)
        window_size=kwargs.get('window_size',101)
        for i in range(cube.shape[1]):
            for j in range(cube.shape[2]):
                filt[:,i,j]=savgol_filter(norm_cube[:,i,j],window_size,polyorder=polyorder)

        norm_res_cube = norm_cube-filt
        self.normalized_cube=norm_res_cube

        for wl in range(cube.shape[0]):
            good= np.where(filt[wl]!=0)
            res_cube[wl][good] = cube_nt[wl][good] - (filt[wl][good]*spat_sum[good]*ref_spec[wl])
        new_matrix=np.zeros([wavelen.shape[0],cube.shape[1]*cube.shape[2]])
        counter=0
        ncomp=int(kwargs.get("n_comps",2))
        print("Subtract %d comps"%ncomp)
        if(ncomp==0):
            #recon_3dcube=res_cube
            print("no PCA ")
            return res_cube

        for j in range(cube.shape[1]):
            for k in range(cube.shape[2]):
                if res_cube[0,j,k] != 0:
                    for i in range(wavelen.shape[0]):
                        new_matrix[i,j*cube.shape[1]+k]=res_cube[i,j,k]
                    counter+=1
        new_matrix=np.asanyarray(pd.DataFrame(new_matrix).fillna(0))
        new_matrix = new_matrix[:,:]
        self.V=svd_wrapper(new_matrix.T,'lapack',ncomp,verbose=True)
        transformed=np.dot(self.V,new_matrix)
        reconstructed=np.dot(transformed.T,self.V)
        residuals=new_matrix-reconstructed.T

        new_counter = 0
        resi_3dcube=np.zeros_like(res_cube)

        for j in range(cube.shape[1]):
            for k in range(cube.shape[2]):
                if res_cube[0,j,k] != 0:
                    for i in range(wavelen.shape[0]):
                        resi_3dcube[i,j,k] = residuals[i,new_counter]
                    new_counter+=1
        return resi_3dcube


import falcon

from ..schemas.core import CoreListSchema, CoreGetSchema  # NOQA
from .core import ListResource
from ..services.meta import _declarations, _ownership_types

class DeclarationListResource(ListResource):
    _service = _declarations
    auth = {
        'exempt_methods': ['OPTIONS', 'GET']
    }

    def on_get(self, req: falcon.Request, resp: falcon.Response) -> None:
        """
        ---
        summary: Get all Declarations from the DB
        tags:
            - Declaration
        parameters:
            - in: query
              schema: CoreListSchema
        produces:
            - application/json
        responses:
            200:
                description: All Declarations
                schema:
                    type: array
                    items: Declaration
            401:
                description: Unauthorized
        """
        super().on_get(req, resp)

class OwnershipTypeListResource(ListResource):
    _service = _ownership_types
    auth = {
        'exempt_methods': ['OPTIONS', 'GET']
    }

    def on_get(self, req: falcon.Request, resp: falcon.Response) -> None:
        """
        ---
        summary: Get all OwnershipTypes from the DB
        tags:
            - OwnershipType
        parameters:
            - in: query
              schema: CoreListSchema
        produces:
            - application/json
        responses:
            200:
                description: All OwnershipTypes
                schema:
                    type: array
                    items: OwnershipType
            401:
                description: Unauthorized
        """
        super().on_get(req, resp)


from django.shortcuts import render
from django.utils import timezone
from django.http import HttpResponse, HttpResponseRedirect
from django.urls import reverse, reverse_lazy

from main.views import ray_randomiser

from study_resource.models import *
from app_user.models import AppUser
from django.contrib import messages
import random
import string

# Create your views here.

def SetupShareView(request):
	if request.user.is_active == True:
		if request.method == "POST":

		 	category = request.POST.get("category")
		 	level = request.POST.get("level")

		 	study_resources = sorted(StudyResource.objects.filter(study_category=category, study_level=level), key=lambda x: random.random())

		 	context = {"study_resources": study_resources}
		 	return render(request, 'study_resource/select_study.html', context)

		else:
			context = {}
			return render(request, 'study_resource/setup_study.html', context)

	else:
		return HttpResponseRedirect(reverse("main:sign_up"))

def ShareResourceView(request):
	if request.user.is_active == True:
		if request.method == "POST":

			title = request.POST.get("title")
			description = request.POST.get("description")

			study_level = request.POST.get("level")
			study_category = request.POST.get("category")

			content = request.POST.get("content")
			link = request.POST.get("link")

			app_user = AppUser.objects.get(user__pk=request.user.id)
			creator = app_user

			study_resource = StudyResource.objects.create(creator=creator, title=title, description=description, study_level=study_level, study_category=study_category)

			if content != "":
				title = study_resource.title
				content = Content.objects.create(title=title, content=content, contributor=app_user.user.username)
				content.save()
				study_resource_content = StudyResourceContentConnector(study_resource=study_resource, content=content)
				study_resource_content.save()

			study_resource.save()

			messages.warning(request, "Resources Successfully Added")
			return HttpResponseRedirect(reverse("main:index"))

		else:
			context = {}
			return render(request, "study_resource/share.html", context)
	else:
		return HttpResponseRedirect(reverse("main:sign_up"))

def ResourceDetailView(request, slug):
	#if request.user.is_active == True:
	study_resource = StudyResource.objects.get(slug=slug)
	context = {"study_resource": study_resource}
	return render(request, "study_resource/detail.html", context)

#	else:
#		return HttpResponseRedirect(reverse("main:sign_up"))

def ContentDetailView(request, content_id):
	if request.user.is_active == True:
		#resource_content = study_resource.contents.get(id=content_id)
		resource_content = Content.objects.get(id=content_id)

		#return HttpResponse(str(resource_content))

		context = {"resource_content": resource_content}
		return render(request, "study_resource/content_detail.html", context)

	else:
		return HttpResponseRedirect(reverse("main:sign_up"))

def ContributeVideoView(request, id):
	if request.user.is_active == True:
		study_resource = StudyResource.objects.get(id=id)
		study_resources = sorted(StudyResource.objects.all(), key=lambda x: random.random())[:10]

		if request.method == "POST":

			app_user = AppUser.objects.get(user__pk=request.user.id)
			contributor = app_user.user.username
			title = study_resource.title

			try:
				file = request.FILES["study_file"]
				video = Video.objects.create(title=title, video=file, contributor=contributor)
				video.save()

				study_resource_video = StudyResourceVideoConnector(study_resource=study_resource, video=video)
				study_resource_video.save()

			except:
				pass

			messages.warning(request, "thanks for sharing this video, Do you have more to share?")
			return HttpResponseRedirect(reverse("study_resource:contribute_video", args=(study_resource.id,)))

		else:

			context = {"study_resource": study_resource, "study_resources":study_resources}
			return render(request, "study_resource/contribute_video.html", context)

	else:
		return HttpResponseRedirect(reverse("main:sign_up"))

def ContributeImageView(request, id):
	if request.user.is_active == True:
		study_resource = StudyResource.objects.get(id=id)

		if request.method == "POST":

			app_user = AppUser.objects.get(user__pk=request.user.id)
			contributor = app_user.user.username
			title = study_resource.title

			try:
				file = request.FILES["study_file"]
				image = Image.objects.create(title=title, image=file, contributor=contributor)
				image.save()

				study_resource_image = StudyResourceImageConnector(study_resource=study_resource, image=image)
				study_resource_image.save()

			except:
				pass

			messages.warning(request, "thanks for sharing this image, Do you have more to share?")
			return HttpResponseRedirect(reverse("study_resource:contribute_image", args=(study_resource.id,)))

		else:
			context = {"study_resource": study_resource}
			return render(request, "study_resource/contribute_image.html", context)

	else:
		return HttpResponseRedirect(reverse("main:sign_up"))

def ContributeEbookView(request, id):
	if request.user.is_active == True:
		study_resource = StudyResource.objects.get(id=id)

		if request.method == "POST":

			app_user = AppUser.objects.get(user__pk=request.user.id)
			contributor = app_user.user.username

			title = study_resource.title

			try:
				file = request.FILES["study_file"]
				ebook = Ebook.objects.create(title=title, ebook=file, contributor=contributor)
				ebook.save()

				study_resource_ebook = StudyResourceEbookConnector(study_resource=study_resource, ebook=ebook)
				study_resource_ebook.save()

			except:
				pass

			messages.warning(request, "thanks for sharing this Ebook, Do you have more to share?")
			return HttpResponseRedirect(reverse("study_resource:contribute_ebook", args=(study_resource.id,)))

		else:
			context = {"study_resource": study_resource}
			return render(request, "study_resource/contribute_ebook.html", context)

	else:
		return HttpResponseRedirect(reverse("main:sign_up"))

def ContributeContentView(request, id):
	if request.user.is_active == True:
		study_resource = StudyResource.objects.get(id=id)

		if request.method == "POST":

			content = request.POST.get("content")

			app_user = AppUser.objects.get(user__pk=request.user.id)
			contributor = app_user.user.username

			title = study_resource.title

			if content != "":
				content = Content.objects.create(title=title, content=content, contributor=contributor)
				content.save()
				study_resource_content = StudyResourceContentConnector(study_resource=study_resource, content=content)
				study_resource_content.save()

			study_resource.save()

			messages.warning(request, "thanks for sharing this content, Do you have more to share?")
			return HttpResponseRedirect(reverse("study_resource:contribute_content", args=(study_resource.id,)))

		else:
			context = {"study_resource": study_resource}
			return render(request, "study_resource/contribute_content.html", context)

	else:
		return HttpResponseRedirect(reverse("main:sign_up"))

def ContributeLinkResourceView(request, id):
	if request.user.is_active == True:
		study_resource = StudyResource.objects.get(id=id)

		if request.method == "POST":

			link = request.POST.get("link")

			app_user = AppUser.objects.get(user__pk=request.user.id)
			contributor = app_user.user.username
			title = study_resource.title

			if link != "":
				link = Link.objects.create(title=title, link=link, contributor=contributor)
				link.save()
				study_resource_link = StudyResourceLinkConnector(study_resource=study_resource, link=link)
				study_resource_link.save()

			study_resource.save()

			messages.warning(request, "thanks for sharing this link, Do you have more to share?")
			return HttpResponseRedirect(reverse("study_resource:contribute_link", args=(study_resource.id,)))

		else:
			context = {"study_resource": study_resource}
			return render(request, "study_resource/contribute_to_links.html", context)

	else:
		return HttpResponseRedirect(reverse("main:sign_up"))

def ContributeChatView(request, id):
	if request.user.is_active == True:
		study_resource = StudyResource.objects.get(id=id)

		if request.method == "POST":

			chat = request.POST.get("chat")

			app_user = AppUser.objects.get(user__pk=request.user.id)
			contributor = app_user

			title = study_resource.title

			if chat != "":
				chat = Chat.objects.create(title=title, chat=chat, contributor=contributor)
				chat.save()
				study_resource_chat = StudyResourceChatConnector(study_resource=study_resource, chat=chat)
				study_resource_chat.save()

			study_resource.save()

			return HttpResponseRedirect(reverse("study_resource:contribute_chat", args=(study_resource.id,)))

		else:
			context = {"study_resource": study_resource}
			return render(request, "study_resource/contribute_chat.html", context)

	else:
		return HttpResponseRedirect(reverse("main:sign_up"))


# -*- encoding: utf-8 -*-
from experimental.tools.newabjadbooktools.ImageOutputProxy \
    import ImageOutputProxy

class GnuplotOutputProxy(ImageOutputProxy):
    r'''Output proxy for Gnuplot images of Abjad datastructures:

    ::

        >>> bpf = datastructuretools.BreakPointFunction({
        ...     0.:   0.,
        ...     0.75: (-1, 1.),
        ...     1.:   0.25,
        ...     })

    ::

        >>> output_proxy = newabjadbooktools.GnuplotOutputProxy(bpf)

    Returns output proxy.
    '''

    ### INITIALIZER ###

    def __init__(self, payload):
        if isinstance(payload, str):
            self._payload = payload
        else:
            gnuplot_format = payload.gnuplot_format
            self._payload = gnuplot_format

    ### PUBLIC METHODS ###

    def handle_html_document_environment(self, document_handler):
        r'''Handle an HTML document environment:

        ::

            >>> document_handler = newabjadbooktools.HTMLDocumentHandler([])
            >>> output_proxy.handle_html_document_environment(document_handler)
            ['<img alt="" src="assets/gnuplot-14219af1a9b714fc222091106ca83f50.png"/>']

        Returns list.
        '''
        return ImageOutputProxy.handle_html_document_environment(
            self,
            document_handler,
            )

    def handle_latex_document_environment(self, document_handler):
        r'''Handle a LaTeX document environment:

        ::

            >>> document_handler = newabjadbooktools.LaTeXDocumentHandler([])
            >>> output_proxy.handle_latex_document_environment(document_handler)
            ['\\includegraphics{assets/gnuplot-14219af1a9b714fc222091106ca83f50.pdf}']

        Returns list.
        '''
        return ImageOutputProxy.handle_latex_document_environment(
            self,
            document_handler,
            )

    def handle_rest_document_environment(self, document_handler):
        r'''Handle an ReST document environment:

        ::

            >>> document_handler = newabjadbooktools.ReSTDocumentHandler([])
            >>> output_proxy.handle_rest_document_environment(document_handler)
            ['.. image:: assets/gnuplot-14219af1a9b714fc222091106ca83f50.png']

        Returns list.
        '''
        return ImageOutputProxy.handle_rest_document_environment(
            self,
            document_handler,
            )


IS_MULTIPROCESS = True

if IS_MULTIPROCESS:
    import os

    os.environ['prometheus_multiproc_dir'] = 'prometheus_tmp'

import asyncio
from time import time
from enum import Enum

from prometheus_client import Counter, Histogram
from sanic import Sanic
from sanic.response import json

from prometheus_sanic import monitor

app = Sanic()

class MetricsType(Enum):
    COUNT = 'sanic_count'
    TIME_LATENCY = 'sanic_latency_sec'

async def ping(request):
    start_time = time()

    await asyncio.sleep(1)
    request.app.ctx.metrics[MetricsType.COUNT.name].labels(
        label_1='ping',
        label_2='GET',
    ).inc()

    request.app.ctx.metrics[MetricsType.TIME_LATENCY.name].labels(
        label_1='ping',
        label_2='GET',
    ).observe(round(float(time() - start_time), 3))

    return json({'success': 'you are home'})

if __name__ == "__main__":
    monitor(
        app,
        multiprocess_mode='all',
        metrics_path='/metrics',
        is_middleware=False,
        metrics_list=[(
            MetricsType.COUNT.name, Counter(
                name=MetricsType.COUNT.value,
                documentation='Total count',
                labelnames=['label_1', 'label_2']
            )), (
            MetricsType.TIME_LATENCY.name, Histogram(
                name=MetricsType.TIME_LATENCY.value,
                documentation='Gauge',
                labelnames=['label_1', 'label_2'],
            )
        )]
    ).expose_endpoint()

    app.add_route(ping, 'ping', methods=['GET'])
    app.run(host="127.0.0.1", port=8000, workers=2)


# -*- coding: utf-8 -*-
# @Time    : 7/3/2021 8:50 AM
# @Author  : Joseph Chen
# @Email   : josephchenhk@gmail.com
# @FileName: order.py
# @Software: PyCharm

from datetime import datetime

from qtrader.core.constants import Direction, Offset, OrderType
from qtrader.core.security import Stock

class Order:
    """订单"""

    def __init__(self,
                 security: Stock,
                 price: float,
                 volume: float,
                 direction: Direction,
                 offset: Offset,
                 order_type: OrderType,
                 place_time: datetime,
                 filled_time: datetime=None
    ):
        self.security = security
        self.price = price
        self.volume = volume
        self.direction = direction
        self.offset = offset
        self.order_type = order_type
        self.place_time = place_time
        self.filled_time = filled_time

    def __str__(self):
        return f"Order[{self.security}, price={self.price}, volume={self.volume}, direction={self.direction}, " + \
               f"offset={self.offset}, ordertype={self.order_type}, place_time={self.place_time}, filled_time={self.filled_time}]"
    __repr__ = __str__


import iso8601
import csv

COLUMN_ITEM_ID = 0
COLUMN_COUNTRY = 1
COLUMN_CITY = 2
COLUMN_TS = 3
COLUMN_PRICE = 4

KEY_ITEM_ID = 'item_id'
KEY_COUNTRY = 'country'
KEY_CITY = 'city'
KEY_TS = 'ts'
KEY_PRICE = 'price'

def load_sales(filename: str) -> list:

    with open(filename, 'r', encoding='utf-8') as f:
        reader = csv.reader(f)
        for row in reader:
            sale = {}
            sale[KEY_ITEM_ID] = row[COLUMN_ITEM_ID]
            sale[KEY_COUNTRY] = row[COLUMN_COUNTRY]
            sale[KEY_CITY] = row[COLUMN_CITY]
            sale[KEY_TS] = iso8601.parse_date(row[COLUMN_TS])
            sale[KEY_PRICE] = float(row[COLUMN_PRICE])
            yield sale


import json
import pytest
from flask_jwt_extended import create_access_token
from app import models
from ..test_helpers import get_access_token

def test_post_valid_activity_definition(test_client, test_db, basic_case_definition, basic_test_survey):
    count_before = test_db.session.query(models.ActivityDefinition).count()

    activity_definition = {
        "name": "Test Activity Definition",
        "description": "Test Activity Definition Description",
        "case_definition_id": basic_case_definition.id,
        "surveys": [basic_test_survey.id],
        "documents": [{
            "name": "Activity Document 1",
            "description": "Activity Document 1 Description",
            "is_required": True
        }, {
            "name": "Activity Document 2",
            "description": "Activity Document 2 Description",
            "is_required": False
        }],
        "custom_fields": [
            {
                "name": "Household Short Text",
                "field_type": "text",
                "selections": None,
                "validation_rules": None,
                "custom_section_id": None,
                "sort_order": 1,
                "help_text": "This is for short answers"
            },
            {
                "name": "Household Long Text",
                "field_type": "textarea",
                "selections": None,
                "validation_rules": None,
                "custom_section_id": None,
                "help_text": "This is for longer answers",
                "sort_order": 2
            },
            {
                "name": "Household Check Box",
                "field_type": "check_box",
                "selections": [
                    {
                        "id": 1,
                        "value": "Household Check Box Option A"
                    },
                    {
                        "id": 2,
                        "value": "Household Check Box Option B"
                    },
                    {
                        "id": 3,
                        "value": "Household Check Box Option C"
                    }
                ],
                "validation_rules": None,
                "custom_section_id": None,
                "help_text": "This allows for multiple selection of choices among many",
                "sort_order": 3
            },
            {
                "name": "Household Radio Button",
                "field_type": "radio_button",
                "selections": [
                    {
                        "id": 1,
                        "value": "Household Radio Option A"
                    },
                    {
                        "id": 2,
                        "value": "Household Radio Option B"
                    },
                    {
                        "id": 3,
                        "value": "Household Radio Option C"
                    }
                ],
                "validation_rules": None,
                "custom_section_id": None,
                "help_text": "This is for a single selection among many",
                "sort_order": 4
            },
            {
                "name": "Household Selection Field",
                "field_type": "select",
                "selections": [
                    {
                        "id": "1",
                        "value": "Household Selection Option A"
                    },
                    {
                        "id": "2",
                        "value": "Household Selection Option B"
                    },
                    {
                        "id": "3",
                        "value": "Household Selection Option C"
                    }
                ],
                "validation_rules": None,
                "custom_section_id": None,
                "help_text": "This is for a single selection among many",
                "sort_order": 5
            },
            {
                "name": "Household Numeric Field",
                "field_type": "number",
                "selections": None,
                "validation_rules": None,
                "custom_section_id": None,
                "help_text": "This is for numbers",
                "sort_order": 6
            },
            {
                "name": "Household Date Field",
                "field_type": "date",
                "selections": None,
                "validation_rules": None,
                "custom_section_id": None,
                "help_text": "This is for a date",
                "sort_order": 7
            },
            {
                "name": "Household Rank List",
                "field_type": "rank_list",
                "selections": [
                    {
                        "id": 1,
                        "value": "Household Rank Option A"
                    },
                    {
                        "id": 2,
                        "value": "Household Rank Option B"
                    },
                    {
                        "id": 3,
                        "value": "Household Rank Option C"
                    }
                ],
                "validation_rules": None,
                "custom_section_id": None,
                "help_text": "This is for ranking a list of options",
                "sort_order": 8
            }
        ]
    }

    access_token = get_access_token(test_client)
    response = test_client.post('/activity_definitions', data=json.dumps(activity_definition),
                                headers={'Content-Type': 'application/json', 'Authorization': f"Bearer {access_token}"})

    assert (count_before + 1) == test_db.session.query(models.ActivityDefinition).count()
    assert response.status_code == 200
    json_data = response.get_json()
    assert json_data['id']
    assert json_data['name'] == activity_definition['name']
    assert len(json_data['surveys']) == 1
    assert len(json_data['documents']) == 2
    assert len(json_data['custom_fields']) == 8
    ad = models.ActivityDefinition.query.get(json_data['id'])
    assert ad
    assert len(ad.surveys.all()) == 1
    assert len(ad.documents) == 2


from flask import Flask, request, jsonify, Blueprint, session
from flask_restful import Resource
from Model import Graph
import networkx as nx
from networkx.readwrite import json_graph

from utils import *
import json

# TODO - REMOVE OR UPDATE THIS
class NetworkStats(Resource):

    def get(self):
        G = Graph.get_graph()

        json_data = json_graph.node_link_data(G, {'link': 'edges', 'source': 'from', 'target': 'to'})
        node_list = json_data['nodes']
        edge_list = json_data['edges']

        return({'Nodes': len(node_list), 'Edges': len(edge_list)})

class GetNodeProperties(Resource):

    def get(self):

        # list of nodes on which to do action
        node_id_list = request.args.get('node_id_list', '', type=str).split(',')

        # load current graph from session
        curr_graph_json = session.get('graph_json')
        curr_focal_node_id_list = session.get('focal_node_id_list')
        curr_graph = json_graph.node_link_graph(curr_graph_json) #, {'edges': 'links', 'source': 'from', 'target': 'to'})
        curr_node_list = list(curr_graph.nodes)

        # get full graph data
        all_graph_data = Graph.get_graph()

        response = {}
        for node in node_id_list:
            response[node] = all_graph_data.nodes[node]

        return(response)

class GetEdgeProperties(Resource):

    # TODO: TEST THIS FUNCTION

    def get(self):

        # list of nodes on which to do action
        edge_list = request.args.get('edge_list', '', type=str).split(',')

        # load current graph from session
        curr_graph_json = session.get('graph_json')
        curr_focal_node_id_list = session.get('focal_node_id_list')
        curr_graph = json_graph.node_link_graph(curr_graph_json) #, {'edges': 'links', 'source': 'from', 'target': 'to'})
        curr_node_list = list(curr_graph.nodes)

        # get full graph data
        all_graph_data = Graph.get_graph()

        response = {}
        for edge in edge_list:
            response[edge] = all_graph_data.edges[edge]

        return(response)

class InitGraph(Resource):

    def get(self):
        G = nx.Graph()
        response = graph_to_json(G)
        print('init default graph')
        return(response)

class GetGraph(Resource):

    def get(self):

        # load current graph from session
        curr_graph_json = session.get('graph_json')
        curr_focal_node_id_list = session.get('focal_node_id_list')
        curr_graph = json_graph.node_link_graph(curr_graph_json) #, {'edges': 'links', 'source': 'from', 'target': 'to'})
        curr_node_list = list(curr_graph.nodes)

        if 'graph_json' in session:
            curr_graph_json = session.get('graph_json')
        else:
            G = nx.Graph()
            response = graph_to_json(G)

        response = graph_to_json(curr_graph, focal_node_id_list = curr_focal_node_id_list)

        return(response)

class InduceGraphWithHops(Resource):

    def get(self):

        # get args:
        # action = request.args.get('action', 0, type=str)
        focal_node_id_list = request.args.get('focal_node_id_list', '', type=str).split(',')
        hops = request.args.get('hops', 0, type=int)
        # node_list = request.args.get('node_list', '', type=str)

        # if node_list != '':
        #     node_list = json.loads(node_list)

        # used to keep track of all nodes and their x,y coords of all nodes currently on display
        # curr_node_pos_dict = request.args.get('curr_node_positions', '', type=str)
        # if curr_node_pos_dict != '':
        #     curr_node_pos_dict = json.loads(curr_node_pos_dict)
        #     curr_node_list = list(curr_node_pos_dict.keys())

        # used to retain current scaling and view position to pass back to client-side (to maintain view)
        # current_scale = request.args.get('curr_scale')
        # current_viewPos = request.args.get('curr_viewPos')

        print('ACTION: HOPS INDUCTION')

        all_graph_data = Graph.get_graph()

        node_list = []
        print(focal_node_id_list)
        for node in focal_node_id_list:
            print('node', node)
            node_list = node_list + list(nx.ego_graph(all_graph_data, node, radius=hops, center=True, undirected=True, distance=None).nodes)

        G = all_graph_data.subgraph(node_list)

        response = graph_to_json(G, focal_node_id_list = focal_node_id_list)

        return response

class ExpandGraph(Resource):

    def get(self):

        # return a graph based on all current nodes + add n hop nodes (default = 1) from each node
        hops = request.args.get('hops', 1, type=int) # not needed, as defaults to 1

        # load current graph from session
        curr_focal_node_id_list = session.get('focal_node_id_list')
        curr_graph_json = session.get('graph_json')
        curr_graph = json_graph.node_link_graph(curr_graph_json) #, {'edges': 'links', 'source': 'from', 'target': 'to'})
        curr_node_list = list(curr_graph.nodes)

        # get full graph data
        all_graph_data = Graph.get_graph()

        # find and expand all nodes by 1-hop
        add_node_list = []
        for n in curr_graph.nodes:
            exploded_n_G = nx.ego_graph(all_graph_data, n, radius=hops, center=True, undirected=True, distance=None)
            add_node_list += list(exploded_n_G.nodes)

        G = all_graph_data.subgraph(curr_node_list + add_node_list)
        print('ADDING THESE NODES', add_node_list)

        response = graph_to_json(G, focal_node_id_list = curr_focal_node_id_list)

        return response

class ContractGraph(Resource):

    def get(self):

        # TODO:
        # if multiple focal entities that are connected, contract will remove nodes that do not connect these two entities
        # therefore, can't contract until left with just unconnected focals
        # so, maybe should be able to move those nodes too?
        # e.g. add nodes 1, 50 (2 components), expand, expand -> 1 component, now contracting down will keep 1, 50 and connecting nodes

        # need to know focal node list as contracting nodes furthest away from these
        focal_node_id_list = request.args.get('focal_node_id_list', '', type=str).split(',')

        # load current graph from session
        curr_graph_json = session.get('graph_json')
        curr_focal_node_id_list = session.get('focal_node_id_list')
        curr_graph = json_graph.node_link_graph(curr_graph_json) #, {'edges': 'links', 'source': 'from', 'target': 'to'})
        curr_node_list = list(curr_graph.nodes)

        # get full graph data
        all_graph_data = Graph.get_graph()

        # get existing graph, undirected
        H = all_graph_data.subgraph(curr_node_list)
        G = H.to_undirected()

        remove_nodes_list = []
        # for each focal node:
        for focal_node_id in focal_node_id_list:
            p = {} # store distance each node is from the focal entity node
            for n in G.nodes:
                try:
                    p[n] = nx.shortest_path_length(G, source=focal_node_id, target=n, weight=None)
                except (nx.exception.NetworkXNoPath, nx.exception.NodeNotFound):
                    print('no path between nodes {} & {}, skipping'.format(focal_node_id, n))

            # max_hops = max(p.values())
            # remove_nodes_list = filter(lambda x: x[1] == max_hops, p.items())
            max_hops = max(p.values())  # maximum value
            remove_nodes_list = remove_nodes_list + [k for k, v in p.items() if v == max_hops] # getting all keys containing the maximum

            #if focal_node_id in remove_nodes_list: remove_nodes_list.remove(focal_node_id)

        combined_remove_node_list = [n for n in remove_nodes_list if n not in focal_node_id_list]

        print('all nodes', G.nodes)
        print('need to remove', combined_remove_node_list)
        print('will end up with', [n for n in curr_node_list if n not in combined_remove_node_list])

        G = all_graph_data.subgraph([n for n in curr_node_list if n not in combined_remove_node_list])

        # p = {}
        # for n in G.nodes:
        #     p[n] = nx.shortest_path_length(G, source=focal_node_id, target=n, weight=None)

        # max_hops = max(p.values())
        # remove_nodes_list = filter(lambda x: x[1] == max_hops, p.items())

        # max_hops = max(p.values())  # maximum value
        # remove_nodes_list = [k for k, v in p.items() if v == max_hops] # getting all keys containing the `maximum`

        # if focal_node_id in remove_nodes_list: remove_nodes_list.remove(focal_node_id)

        # print('all nodes', G.nodes)
        # print('need to remove', remove_nodes_list)
        # print('will end up with', [n for n in curr_node_list if n not in remove_nodes_list])

        # G = all_graph_data.subgraph([n for n in curr_node_list if n not in remove_nodes_list])

        response = graph_to_json(G, focal_node_id_list = curr_focal_node_id_list)

        return response

class RemoveNodes(Resource):

    def get(self):

        # list of nodes on which to do action
        node_id_list = request.args.get('node_id_list', '', type=str).split(',')

        # load current graph from session
        curr_graph_json = session.get('graph_json')
        curr_focal_node_id_list = session.get('focal_node_id_list')
        curr_graph = json_graph.node_link_graph(curr_graph_json) #, {'edges': 'links', 'source': 'from', 'target': 'to'})
        curr_node_list = list(curr_graph.nodes)

        # get full graph data
        all_graph_data = Graph.get_graph()

        # remove nodes from node list and focal node list if either list contains them
        new_node_id_list = [n for n in curr_node_list if n not in node_id_list]
        new_focal_node_id_list = [n for n in curr_focal_node_id_list if n not in node_id_list]
        G = all_graph_data.subgraph(new_node_id_list)

        response = graph_to_json(G, focal_node_id_list = new_focal_node_id_list)

        return response

class AddNodes(Resource):

    def get(self):

        #TODO: should added nodes become focal nodes??

        # list of nodes on which to do action
        node_id_list = request.args.get('node_id_list', '', type=str).split(',')
        print(node_id_list)

        # load current graph from session
        curr_graph_json = session.get('graph_json')
        curr_focal_node_id_list = session.get('focal_node_id_list')
        curr_graph = json_graph.node_link_graph(curr_graph_json) #, {'edges': 'links', 'source': 'from', 'target': 'to'})
        curr_node_list = list(curr_graph.nodes)

        # get full graph data
        all_graph_data = Graph.get_graph()

        add_node_list = [n for n in node_id_list if n not in curr_node_list]

        G = all_graph_data.subgraph(curr_node_list + add_node_list)

        response = graph_to_json(G, focal_node_id_list = curr_focal_node_id_list)

        return response

class ExplodeNodes(Resource):

    def get(self):

        # expands each node selected by 1 hop
        # node must be in current node list

        # list of nodes on which to do action
        node_id_list = request.args.get('node_id_list', '', type=str).split(',')

        print('ACTION: EXPLODE NODES:', node_id_list)

        # load current graph from session
        curr_graph_json = session.get('graph_json')
        curr_focal_node_id_list = session.get('focal_node_id_list')
        curr_graph = json_graph.node_link_graph(curr_graph_json) #, {'edges': 'links', 'source': 'from', 'target': 'to'})
        curr_node_list = list(curr_graph.nodes)

        # get full graph data
        all_graph_data = Graph.get_graph()

        add_node_list = []

        for n in node_id_list:
            if n in curr_node_list:
                exploded_node_subgraph = nx.ego_graph(all_graph_data, n, radius=1, center=True, undirected=True, distance=None)
                add_node_list += list(exploded_node_subgraph.nodes)
            else:
                print('node', n, 'not in current node list')

        print('ADDING THESE NODES', add_node_list)
        G = all_graph_data.subgraph(curr_node_list + add_node_list)

        response = graph_to_json(G, focal_node_id_list = curr_focal_node_id_list)

        return response

class MergeNodes(Resource):

    def get(self):

        # on the fly combine two nodes into one - only a temporary adjustment

        # merge node_move into merge_into node
        node_move = request.args.get('node_move', '', type=str).split(',')
        node_into = request.args.get('node_into', '', type=str).split(',')

        # load current graph from session
        curr_graph_json = session.get('graph_json')
        curr_focal_node_id_list = session.get('focal_node_id_list')
        curr_graph = json_graph.node_link_graph(curr_graph_json) #, {'edges': 'links', 'source': 'from', 'target': 'to'})
        curr_node_list = list(curr_graph.nodes)

        # get full graph data
        all_graph_data = Graph.get_graph()

        # TODO: copy node_move properties into node_into (into a separate dictionary that can keep all future dragged into info)
        # TODO: move node_move's edges (to and from) into node_into <-- directly edit the side of edge rather than creating a new edge and deleting old one?
        # TODO: drop node_move
        # TODO: must keep track of this merge in a reference table somewhere so as to action it every time another graph changing action occurs!!

        response = graph_to_json(G)

        return response


"""
GZCL

7/29/2017

Project_Practice

This file is a project in Python, specifically a small game "Alien Invasion" that is similar to space invaders,

This uses a library called Pygame
"""

# pygame is used for making the game
import pygame

from settings import Settings
from ship import Ship
import game_functions as gf

# 'run_game' is a function that creates the display window and set the logic for exiting
def run_game():
    # Initialize game and create a screen object
    pygame.init()
    # Makes variable 'ai_settings' equal to 'Settings'
    ai_settings = Settings()
    # Makes variable screen equal to the display and imports width and height from 'settings.py'
    screen = pygame.display.set_mode((ai_settings.screen_width, ai_settings.screen_height))
    # Sets top part of screen to say "Alien Invasion"
    pygame.display.set_caption("Alien Invasion")

    # Make a ship
    ship = Ship(ai_settings, screen)
    # Start the main loop for the game
    while True:
        # checks for movement and other user inputs
        gf.check_events(ship)
        # Updates screen each time through the loop
        gf.update_screen(ai_settings, screen, ship)
        # Watch for keyboard and mouse events
        ship.update()

run_game()


"""
utility functions related to queries things from internal
script databases (objdb, roomdb, zonedb, mobdb, etc...)
"""

from typeclasses.objs.custom import CUSTOM_OBJS
from evennia import GLOBAL_SCRIPTS

def search_objdb(criteria, **kwargs):
    """
    the criteria can either be name or type of object
    first checks to see if it a valid object type before trying names

    The kwargs if for searching by 'extra' which is specific depending
    on object type. If the extras field are not existent, it will ignore the
    filter.

    ex:

    search_objdb('book', {'category': 'fiction'})
    # will return all objects of books that are in category of fiction.

    search_objdb('all') # returns all objects in database

    search_objdb('fire') # returns objects with the name fire
    """
    results = dict()
    objdb = GLOBAL_SCRIPTS.objdb.vnum

    if criteria == 'all':
        results.update(dict(objdb))
        return results

    use_type = True if criteria in CUSTOM_OBJS.keys() else False
    for vnum, data in objdb.items():

        if use_type and data['type'] == criteria:
            if not kwargs:
                results[vnum] = data
            elif kwargs.items() <= data['extra'].items():
                results[vnum] = data

        elif criteria in data['key']:
            results[vnum] = data
    return results


from django.urls import path
from .import views as v

app_name = 'ind_e'

urlpatterns = [
    path('escolaridade/', v.IndEscolaridadeList.as_view(),
         name='url_indescolaridade_list'),
    path('escolaridade/<int:pk>/', v.IndEscolaridadeDetail.as_view(),
         name='url_indescolaridade_detail'),
    path('escolaridade/create/', v.IndEscolaridadeCreate.as_view(),
         name='url_indescolaridade_form'),
    path('escolaridade/update/<int:pk>/', v.IndEscolaridadeUpdate.as_view(),
         name='url_indescolaridade_update'),
    path('escolaridade/delete/<int:pk>/', v.IndEscolaridadeDelete.as_view(),
         name='url_indescolaridade_delete'),
]


import concurrent
from copy import deepcopy
import multiprocessing
import os
from sklearn import manifold
from sklearn.decomposition import PCA
import matplotlib
from tqdm import tqdm
import matplotlib.pyplot as plt

from mpl_toolkits.mplot3d import Axes3D  # noqa
from scipy.spatial import distance_matrix
from numpy.random import choice
from scipy.sparse.linalg import eigs, eigsh
import sys
from sklearn.metrics import confusion_matrix
from gwd_utils import *
import time
import torch; print('torch', torch.__version__)
import random

import platform; print(platform.platform())
import sys; print("Python", sys.version)
import numpy as np; print("NumPy", np.__version__)
import scipy; print("SciPy", scipy.__version__)
import ot; print("POT", ot.__version__)
print("Is Cuda available: {}".format(torch.cuda.is_available()))

if __name__ == '__main__':
    seed = 0
    np.random.seed(seed)
    random.seed(seed) #seed=0 => idx1=24
    parallel_computation = False
    threshold = 0.99
    num_samples_to_check = 0  #number of samples per class to check for poisoning attack. if ==0 => take all samples
    max_iter = 20  # number of maximum iterations in kmeans algorithm
    n_samples = 10  # number of points in barycenter:
    de = 18 # Paramter um das Auswahlfenster der Länge num_samples_to_check zu verschieben
    eps_init = 0.1 #5e-4'MyFile5.txt'
    eps_update = 0.1

    class_to_check = 5
    verbose = False

    # Ein- und Ausgabe

    model_names = ['SPA_incV3_s3_pp005'] #, 'SPA_incV3_s1_pp015', 'SPA_incV3_s1_pp01','SPA_incV3_s1_pp005', 'SPA_incV3_s1_pp002', 'SPA_incV3_s1_pp001']#['SPA_incV3_s2_pp001']#
    model_names = [s + 'e-rule' for s in model_names]

    for model_name in model_names:
        print(model_name)

        file_name = 'clustering_reg01' + model_name + '.txt'

        # Erstelle Ordner für die Ausgaben
        path = os.getcwd()
        path_new = path + "/Clustering_Outputs/"
        if not os.path.exists(path_new):
            os.makedirs(path_new)
        file_name = path_new + file_name

        # Open images of suspicious class
        path = './LRP_Outputs/' + str(model_name) + '/relevances/' + str(class_to_check).zfill(5) + '/'

        relevances = []
        heatmaps = []
        poison_labels = []

        for root, dirs, files in os.walk(path):

            for name in files:

                #print(name)

                # Save poison labels
                if name.endswith("_poison.npy"):
                    poison_labels.append(int(1))
                else:
                    poison_labels.append(int(0))

                # Load .npy file
                with open(str(root) + "/" + str(name), 'rb') as f:

                    d = np.load(f)

                # Sum over color channels
                d = np.sum(d, axis=0)

                # Save 2D heatmap for input to OT library
                dd = d
                # Transform 32x32 Heatmap into 1024x1 Heatmap
                d = d.reshape((d.shape[0] * d.shape[1]))

                # Save all Heatmaps in one array
                heatmaps.append(dd)
                relevances.append(d)

        # Convert list of relevances to numpy array
        rel_array = np.asarray(relevances).astype(np.float64)

        if num_samples_to_check > 0:
            rel_array = rel_array[de: de + num_samples_to_check]
            heatmaps = heatmaps[de: de + num_samples_to_check]
            poison_labels = poison_labels[de: de + num_samples_to_check]

        print('poisonLabels: ', poison_labels)
        print('Anzahl an korrumpierten Datenpunkten in der Teilmenge: ', np.asarray(poison_labels).sum())
        #Plot heatmap
        #for i in range(20):
        #    plt.imshow(heatmaps[i])
        #    plt.show()

        #Normalize relevance maps to [0,1] before computing pw. L2 dist:
        rel_array_normalized = []
        for i in range(rel_array.shape[0]):
            r = rel_array[i]
            min = r.min()
            max = r.max()

            # Normalize "by hand"
            r = (r-min)/(max-min)

            # Division by total mass
            r = r/r.sum()

            rel_array_normalized.append(r)

        rel_array_normalized = np.asarray(rel_array_normalized)

        # Zeige normalisierte Heatmap
        #plt.imshow(rel_array_normalized[0].reshape(32,32))
        #plt.show()
        # Remark: Heatmap und normalisierte HEatmap sehen im plot identisch aus

        #Wähle 2 images, für die ein Barycenter berechnet werden soll:
        #Wähle erste und zweite Heatmap aus der Liste aus:
        heatmap_array = np.asarray(heatmaps).astype(np.float64)

        #### kmeans++ ####
        ##### Initialization ##### -----------------------------------------------------------------------------------------
        print('#### kmeans++ ####')
        # Initialisierung
        # Lege gewünschte Cluster-Anzahl k fest:
        num_cluster_kmeans = 2

        #cluster_centers = np.empty(shape=(k, 2))
        #cluster_centers[:][:] = np.nan
        cluster_centers = [[[], []], [[], []]]
        #print('cc: ', cluster_centers)

        # Wähle im ersten Schritt eine zufällige Heatmap als erstes Zentrum
        n = heatmap_array.shape[0]  # number of heatmaps to cluster
        cluster = np.zeros(n)

        seq = list(range(0, n))
        #print(seq)

        idx_1 = random.sample(seq, k=1)[0]
        #print('idx1: ', idx_1)

        # Compute distances of every other sample to the chosen first center
        #measured_distances = np.asarray([heatmap_to_distance_matrix(im) for im in heatmap_array])

        CC = []
        pp = []
        max_dim = 0
        min_dim = np.square(heatmap_array[0].shape[0])

        for i in range(n):
            c_, p_ = heatmap_to_distance_matrix(heatmap_array[i])
            CC.append(c_)
            pp.append(p_)
            if c_.shape[0] > max_dim:
                max_dim = c_.shape[0]
            if c_.shape[0] < min_dim:
                min_dim = c_.shape[0]

        CC = np.asarray(CC)
        pp = np.asarray(pp)

        #print('max_dim: ', max_dim)
        #print('min_dim:', min_dim)

        def my_function_star(args):
            return compute_GWD_to_index(*args)

        print(' ==> Compute distances to first mean')
        start_time = time.time()
        if parallel_computation:

            # https://stackoverflow.com/questions/57354700/starmap-combined-with-tqdm/57364423#57364423

            tuples_input = []
            #md = measured_distances
            for i in range(n):
                #tuples_input.append(((md[:, 0][idx_1], md[:, 1][idx_1])
                #                     , (md[:, 0][i], md[:, 1][i])))

                tuples_input.append(((CC[idx_1],pp[idx_1]),(CC[i],pp[i])))

            process_pool = multiprocessing.Pool(32)
            data = tuples_input
            distances = process_pool.starmap(compute_GWD_to_index, data)
            #distances = tqdm.tqdm()
            #for _ in tqdm(process_pool.istarmap(compute_GWD_to_index, data),
             #                  total=len(data)):
              #  pass
            #inputs = zip(param1, param2, param3)
            #with multiprocessing.Pool(10) as pool:
                #distances = pool.starmap(compute_GWD_to_index, tqdm(data, total=len(data)))
            #    distances = list(tqdm(pool.imap(my_function_star, data), total=len(data)))
            #mapped_values = list(tqdm.tqdm(pool.imap_unordered(do_work, range(num_tasks)), total=len(values)))

        else:
            distances = []
            for i in range(0, n):
                print('i: ', i)

                gw, log = ot.gromov.entropic_gromov_wasserstein2(
                        CC[idx_1], CC[i],
                        pp[idx_1], pp[i], 'square_loss', epsilon=eps_init, log=True)
                #print(log['gw_dist'])
                distances.append(log['gw_dist'])

                #distances.append(1.0)

            distances = np.asarray(distances)
            distances_for_iter0 = distances
            distances /= distances.sum()
        print("--- %s seconds ---" % (time.time() - start_time))
        print(' ==> Distances computed.')

        # Choose probabilities of choosing next k-1 centers:
        p = np.square(distances)
        p /= p.sum()
        #print(p)

        p[idx_1] = 0
        p /= p.sum()
        #print(p)
        idx_2 = choice(seq, size=1, p=p)[0]
        #print('idx_2: ', idx_2)

        clustering = np.empty(shape=(n,))
        clustering[:] = np.nan

        clustering[idx_1] = 0
        clustering[idx_2] = 1

        #Use dict for cluster centers cc:
        cc = {0: {'dist_m': CC[idx_1], 'weights': pp[idx_1]},
              1: {'dist_m': CC[idx_2], 'weights': pp[idx_2]}}

        # ------------------------------------------------------------------------------------------------------------------
        file_clustering = open(file_name,"w+")
        file_clustering.write('eps_init: ' + str(eps_init))
        file_clustering.write('\n')
        file_clustering.write('eps_update: ' + str(eps_update))
        file_clustering.write('\n')
        file_clustering.write('n_samples_bary: ' + str(n_samples))
        file_clustering.write('\n')

        iter_kmeans = 0
        print(' ==> Starting k-means++-iterations')

        while iter_kmeans < max_iter:

            print('iter: ', iter_kmeans, ' |----------------------------------------------------------------------------------------')
            ##### Zuordnung #####
            # --- Distanzberechnung zu den aktuellen Baryzentren -----------------------------------------------------------
            #print('==> Recalculate distances to each barycenter')
            # Compute for every point, which is not part of a cluster the distance to the cluster centers 1 and 2, take the minimum and assign the point accordingly.
            distances_to_cluster_centers = np.empty(shape=(n, 2))
            distances_to_cluster_centers[:][:] = np.nan

            # Compute distances to all barycenters per data point
            start_time = time.time()
            #print('...starting.')
            for i in range(0, n):

                if (iter_kmeans == 0) and (i == idx_1 or i == idx_2):
                    # Im ersten Iterationsschritt muss für die gewählten Baryzentren nicht der Abstand zu sich selbst berechnet werden
                    continue
                #print('i: ', i)

                for j in range(num_cluster_kmeans):

                    if j == 0 and iter_kmeans == 0:
                        # Distanzen zwischen idx1 und allen anderen Matrizen wurde oben schon berechnet und ist in distances_for_iter0 abgespeichert:
                        distances_to_cluster_centers[i][j] = distances_for_iter0[i]
                    else:
                        # Die Abstände zwischen idx2 und allen anderen Punkten muss noch berechnet werden
                        # Auch für iter_kmeans > 0 müssen die Distanzen neu berechnet werden
                        gw, log = ot.gromov.entropic_gromov_wasserstein2(
                                cc[j]['dist_m'], CC[i],
                                cc[j]['weights'], pp[i], 'square_loss', epsilon=eps_update, log=True, verbose=verbose)#method=method_update
                        distances_to_cluster_centers[i][j] = log['gw_dist']

                #print('Distances to centers: ', distances_to_cluster_centers)

                #print(distances_to_cluster_centers[i])
                #print('argmin: ', distances_to_cluster_centers[i].argmin())
                # --- Cluster Update ---------------------------------------------------------------------------------------
                #Choose minimum distance and set cluster label accordingly:
                clustering[i] = distances_to_cluster_centers[i].argmin()

                #if distances_to_cluster_centers[i].argmin() == 0:
                #    clustering[i] = 0
                #if distances_to_cluster_centers[i].argmin() == 1:
                #    clustering[i] = 1

                #print('==> Update Clustering:' + str(clustering))
                # ----------------------------------------------------------------------------------------------------------
            #  Die einzelnen Punkte sind den Clustern neu zugeordnet

            if iter_kmeans > 0:
                file_clustering = open(file_name, "a")

            file_clustering.write('iter: ' + str(iter_kmeans))
            file_clustering.write('\n')
            #file_clustering.write(str(clustering))
            #file_clustering.write('\n')
            file_clustering.close()

            # Evaluation inside each iteration: ----------------------------------------------------------------------------
            labels_pred = clustering

            # Change clustering labels, if the bigger class is labeled with '1':
            if labels_pred.sum() > labels_pred.shape[0]/2:
                labels_pred = 1 - labels_pred
            tn, fp, fn, tp = confusion_matrix(poison_labels, labels_pred).ravel()
            #print('(tn, fp, fn, tp): ', a, b, c, d)
            acc = round(100*(tp+tn)/(tn+fn+tp+fp), 2)
            tpr = round(100*tp/(tp+fn), 2)
            tnr = round(100*tn/(tn+fp), 2)

            file_clustering = open(file_name,"a")
            file_clustering.write('(tn, fp, fn, tp): ' + str(tn) + ',' + str(fp) + ',' + str(fn) + ',' + str(tp))
            file_clustering.write('\n')
            file_clustering.write('acc: ' + str(acc))
            file_clustering.write('\n')
            file_clustering.write('tpr: ' + str(tpr))
            file_clustering.write('\n')
            file_clustering.write('tnr: ' + str(tnr))
            file_clustering.write('\n')
            file_clustering.close()

            # Abbruchkriterium ---------------------------------------------------------------------------------------------
            if iter_kmeans > 0:
                print('Old Clustering', clustering_old)

            print('Clustering: ', clustering)

            if iter_kmeans > 0:
                #Check if clustering has changed in the last iteration:
                if np.equal(clustering_old, clustering).all():
                    print('k-means Clustering didnt change and is being stopped.')
                    break

            clustering_old = deepcopy(clustering)

            #print("--- %s seconds ---" % (time.time() - start_time))

            ##### Aktualisierung #####--------------------------------------------------------------------------------------
            # Recalculate barycenters

            start_time = time.time()
            print('==> Recalculate barycenters per cluster')

            for j in range(num_cluster_kmeans):
                idd = np.where(clustering==id)
                print('idd:', idd[0], end=" ")
                bary = compute_barycenter_from_Cp(CC, pp, clustering=clustering, id=j, n_samples=n_samples, entropic=True, eps=eps_update)
                p = ot.unif(bary.shape[0])

                cc[j]['dist_m'] = bary
                cc[j]['weights'] = p

            iter_kmeans +=1

            print("--- %s seconds ---" % (time.time() - start_time))

        # --- ENDE kMEans --------------------------------------------------------------------------------------------------

        # Change clustering labels, if the bigger class is labeled with '1':
        if clustering.sum() > clustering.shape[0]/2:
            print('Cluster got flipped')
            clustering = 1 - clustering

        print('final_CLustering: ', clustering)
        # Evaluate clustering:
        a, b, c, d = confusion_matrix(poison_labels, clustering).ravel()
        #specificity = tn / (tn+fp)
        #print(tn, fp, fn, tp)
        # 1650 0 386 427
        print('(tn, fp, fn, tp): ', a, b, c, d)


# -*- coding: utf-8 -*-
"""
music资源下载
@file: thread_downloader.py
@time: 2018/9/3 22:36
Created by Junyi.
"""

import os
import threading
import requests
from .progress_bar import MyProgressBar

class ThreadDownloader(object):
    """
    多线程下载器
    """
    def __init__(self, url, filename, file_path=None, num_thread=10):
        self.url = url
        self.filename = filename
        self.file_path = file_path
        self.num_thread = num_thread
        self.target_path = os.path.join(self.file_path, self.filename) if self.file_path else self.filename
        self.resource_size = self.get_resource_size()
        self.my_pb = MyProgressBar.get_progressbar_object(maxval=self.resource_size)
        self.lock = threading.Lock()

    def handler(self, startpos, endpos, fp):
        """
        单个进程发起请求
        :param startpos: 开始字节的序号
        :param endpos: 结束字节的序号
        :param fp: 写文件对象
        :return: None
        """
        headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Encoding': 'gzip,deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Host': 'dl.stream.qqmusic.qq.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' +
                          'Chrome/67.0.3396.99 Safari/537.36',
            'Range': 'bytes={}-{}'.format(startpos, endpos),
        }
        response = requests.get(self.url, headers=headers, stream=True)
        with self.lock:                         # 写入数据时启用线程锁
            fp.seek(startpos)
            fp.write(response.content)
            self.my_pb.update(endpos)

    def download(self):
        """
        多线程下载音乐资源
        :return: None
        """
        with open(self.target_path, 'wb') as fp:
            piece_of_data = self.resource_size // self.num_thread
            thread_list = []
            self.my_pb.start()
            for i in range(self.num_thread):
                startpos = piece_of_data * i
                if i == self.num_thread - 1:
                    endpos = self.resource_size
                else:
                    endpos = startpos + piece_of_data
                thread = threading.Thread(target=self.handler, kwargs={'startpos': startpos, 'endpos': endpos, 'fp': fp})
                thread.setDaemon(True)
                thread.start()
                thread_list.append(thread)
            for thread in thread_list:
                thread.join()
        self.my_pb.finish()

    @staticmethod
    def get_file_size(file_path):
        """
        返回指定文件路径下的文件的大小
        :param file_path: 文件的路径
        :return: 文件的大小(字节数)<int>
        """
        file_size = os.path.getsize(file_path)
        return file_size

    def get_resource_size(self):
        """
        获取资源的总大小
        :return: int(字节数)
        """
        response = requests.head(self.url)
        resource_size = int(response.headers['content-length'])
        return resource_size


from typing import Dict, Optional
from app_facebook.events.abstract_event import AbstractEvent
from app_facebook.message_objects.buttons.postback_button import PostbackButton
from app_facebook.message_objects.buttons.quick_button import QuickButton
from app_facebook.message_objects.elements.default_action import DefaultAction
from app_facebook.message_objects.elements.element import Element
from app_facebook.message_objects.elements.element_definition import ElementDefinition
from app_facebook.message_objects.elements.webview_height_ratio_enum import WebviewHeightRatioEnum
from app_facebook.message_objects.reply import Reply
from app_facebook.message_objects.templates.generic_template import GenericTemplate
from app_facebook.view import View

class DummyEvent(AbstractEvent):
    def __init__(self, expected_payload: Optional[Dict] = None):
        super().__init__()
        self.expected_payload = expected_payload

    EXPECTED_PAYLOAD_DICT = { 'test-key': 'test-value' }
    EXPECTED_JSON = {
            'message': {
                'attachment': {
                    'type': 'template',
                    'payload': {
                        'template_type': 'generic',
                        'elements': [
                            {
                                'buttons': [
                                    {
                                        'payload': 'test_payload_button',
                                        'title': 'test_title_button',
                                        'type': 'postback'
                                    }
                                ],
                                'default_action': {
                                    'messenger_extensions': False,
                                    'type': 'web_url',
                                    'url': 'http://default_action.com',
                                    'webview_height_ratio': 'FULL'
                                },
                                'image_url': 'http://image.com',
                                'subtitle': 'test_subtitle_element_definition',
                                'title': 'test_title_element_definition'
                            }
                        ],
                    },
                },
                'quick_replies': [
                    {
                        'content_type': 'text',
                        'payload': 'test_payload_quick_button',
                        'title': 'test_title_quick_button',
                    }
                ]
            },
            'recipient': {
                'id': '123'
            }
        }

    def raise_view(self) -> View:
        assert self.payload().payload_dict == (self.expected_payload or self.EXPECTED_PAYLOAD_DICT)

        quick_button = QuickButton(
            title='test_title_quick_button',
            payload='test_payload_quick_button'
        )

        button = PostbackButton(
            title='test_title_button',
            payload='test_payload_button'
        )

        action = DefaultAction(
            url='http://default_action.com',
            webview_height_ratio=WebviewHeightRatioEnum.FULL
        )

        element_definition = ElementDefinition(
            title='test_title_element_definition',
            image_url='http://image.com',
            subtitle='test_subtitle_element_definition',
            action=action
        )

        element = Element(
            element_definition=element_definition,
            buttons=[button]
        )

        template = GenericTemplate(
            [element]
        )

        reply = Reply(
            recipient_id='123',
            template=template,
            quick_buttons=[quick_button],
        )

        return View(reply)


import time
import operator

class Node():
    """docstring for Node,if it is explored,explored = True.
    out_edge is a list for out edge"""
    def __init__(self,name):
        self.name = name
        self.explored = False
        self.out_edge = []
        self.leader = None

    def append(self,head):
        self.out_edge.append(head)

    def explore(self):
        self.explored = True

    def n_print(self):
        print(self.name,self.explored)
        print(self.out_edge)

def Graph_read(file_name):
    G = {}
    G_rev = {}
    with open(file_name) as f:
        for line in f:
            words = line.split()
            if int(words[0]) not in G:
                G[int(words[0])] = Node(int(words[0]))
            if int(words[1]) not in G_rev:
                G_rev[int(words[1])] = Node(int(words[1]))
            G[int(words[0])].append(int(words[1]))
            G_rev[int(words[1])].append(int(words[0]))
    return G,G_rev

def DFS_Loop1(G): #外层循环，t在第一遍DFS中起作用，判断节点的finishing time
  #s在第二次中起作用，判断leader，即SCC节点
    global t,s,f_time,SCC_Num_lis
    t = 0
    s = None
    name_lis = [node for node in G]
    #print(sorted(name_lis,reverse = True))
    f_time = {}
    for i in sorted(name_lis,reverse = True):
        if not G[i].explored:
            #print(i)
            #print(i)
            DFS_Stack(G,i)

def DFS_Loop2(G): #外层循环，t在第一遍DFS中起作用，判断节点的finishing time
  #s在第二次中起作用，判断leader，即SCC节点头
    global t,s,f_time,SCC_Num_lis
    s = None
    SCC_Num_lis = {} #统计leader s中节点的个数
    #print(f_time)
    sorted_x = sorted(f_time.items(), key=operator.itemgetter(1),reverse = True)
    #print(sorted_x)
    #按照f_time中value降序排序
    for i,f_time_num in sorted_x:
        #print(i,f_time_num)
        #print(i,00000)
        if i not in G:
            G[i] = Node(i)
            s = i
            DFS_Stack(G,i)
        else:
            if not G[i].explored:
                #print(123)
                s = i
                DFS_Stack(G,i)

'''def DFS(G,i):
    global t,s,f_time,SCC_Num_lis
    G[i].explore()
    G[i].leader = s
    if s != None:
        SCC_Num_lis[s] = SCC_Num_lis.get(s,0)+1
    for j in G[i].out_edge:
        if j in G and not G[j].explored:
             DFS(G,j)
        t = t + 1     #弹栈的视乎时间加一
        f_time[i] = t #记录时间'''

def DFS_Stack(G,i):
    global t,s,f_time,SCC_Num_lis
    #print(s)
    Stack = []
    Stack.append(i)
    while len(Stack)>0:
        #print(s,len(Stack))
        while True: #疯狂压栈
            #print(Stack)
            num = Stack.pop()
            Stack.append(num)
            G[num].explore()
            explore_left = False
            #print(len(Stack),num,G[num].out_edge)
            for j in G[num].out_edge:
                #print(len(Stack))
                #print(0,j)
                if j in G :
                    if not G[j].explored:
                        Stack.append(j)
                        G[j].explore()
                        #print(1)
                        explore_left = True
                        #print('sadasdasd',111111111111111)
                else:
                    G[j] = Node(j)
                    Stack.append(j)
                    G[j].explore()
                    #print(2)
                    explore_left = True
                    #print(explore_left,2222222222222222222222)
            if not explore_left:
                #print('break')
                break
        '''压栈结束！'''
        num = Stack.pop() #弹栈访问
        G[num].leader = s
        #print(num,t)
        #print(s)
        if s != None:
            SCC_Num_lis[s] = SCC_Num_lis.get(s,0)+1
        #else:
        #    print(i,t,1111)
            #print(SCC_Num_lis)
        else:
            t = t + 1     #弹栈的视乎时间加一
            f_time[num] = t #记录时间

tic = time.clock()
global t,s,f_time,SCC_Num_lis
#G = Graph_read('SCC.txt')
G,G_rev = Graph_read('SCC.txt') #同时读出G和arc反转
print(time.clock()-tic)
print('Loading Complete')
print('''begin DFS_search''')
DFS_Loop1(G_rev)
print(time.clock()-tic)
print('DFS_Loop1 Complete')
DFS_Loop2(G)
print(time.clock()-tic)
print('DFS_Loop2 Complete')
#print(SCC_Num_lis)
sorted_SCC = sorted(SCC_Num_lis.values(),reverse = True)
#再一次按value降序排列
print('Top 5:')
print(sorted_SCC[:5])
#for name,node in G.items():
#    node.n_print()
print(time.clock()-tic)


from rest_framework import serializers

from db.models.bookmarks import Bookmark

class BookmarkedSerializerMixin(serializers.Serializer):
    bookmarked_model = None

    bookmarked = serializers.SerializerMethodField()

    def get_bookmarked(self, obj):
        bookmarks = self.context.get('bookmarks', None)

        if bookmarks is not None:
            return obj.id in bookmarks
        else:
            # Get the requesting user if set in the context
            request = self.context.get('request', None)
            if request:
                return Bookmark.objects.filter(
                    user=request.user,
                    content_type__model=self.bookmarked_model,
                    object_id=obj.id,
                    enabled=True).exists()
        return False


import sys
sys.stdin = open('input.txt','r')

k = int(input())
li = list(map(str,input().split()))

# 백트래킹
# max 값 구하기 :
# (첫숫자~끝숫자)에 들어갈 수 있는 가장 큰 수 넣어보면서
# 틀리면 되돌아온다.

# min 값 구하기 :
# (첫숫자~끝숫자)에 들어갈 수 있는 가장 작은 수 넣어보면서
# 틀리면 되돌아온다.

#######   답이 021인 경우 출력도 021

def isright(n1,n2,mark):
    if mark == '>':
        if n1 > n2:
            return True
    if mark == '<':
        if n1 < n2:
            return True
    return False
#가장 큰 수 구하기
# li = [< < < > < ]
def findmax(startno,li,n,check):
    num = [0,1,2,3,4,5,6,7,8,9]
    touse = [0,1,2,3,4,5,6,7,8,9]
    for a in check:
        touse.pop(a)
        print(check, a, touse)
    if startno in check:
        start = max(touse)
    else:
        start = startno
    check.append(start)
    while len(check) != n+1:
        now = touse.pop()
        check.append(now)
        mark = li[len(check)-2]
        right = isright(check[-2],check[-1],mark)
        if not right and start > 0:
            print(start)
            print(check)
            findmax(num[num.index(start)-1],li,n,check[-2:])
            #num을 줄여서 다시시작
    return check

print(findmax(9,li,k,[]))


# Author: Kyle Kastner
# License: BSD 3-clause
import numpy as np
import theano
from theano import tensor
from theano.scan_module.scan_utils import infer_shape
from theano.gof.fg import MissingInputError
from collections import OrderedDict
from .training_utils import _get_file_matches, _unpickle
from .training_utils import _set_shared_variables_in_function
from .training_utils import get_checkpoint_dir

TAG_ID = "_dagbldr_"
DATASETS_ID = "__datasets__"
RANDOM_ID = "__random__"

def safe_zip(*args):
    """Like zip, but ensures arguments are of same length.

       Borrowed from pylearn2 - copied from utils to avoid circular import
    """
    base = len(args[0])
    for i, arg in enumerate(args[1:]):
        if len(arg) != base:
            raise ValueError("Argument 0 has length %d but argument %d has "
                             "length %d" % (base, i+1, len(arg)))
    return zip(*args)

def as_shared(arr, name=None):
    """ Quick wrapper for theano.shared """
    if type(arr) in [float, int]:
        if name is not None:
            return theano.shared(np.cast[theano.config.floatX](arr))
        else:
            return theano.shared(np.cast[theano.config.floatX](arr), name=name)
    if name is not None:
        return theano.shared(value=arr, borrow=True)
    else:
        return theano.shared(value=arr, name=name, borrow=True)

def concatenate(tensor_list, graph, name, axis=0, force_cast_to_float=True):
    """
    Wrapper to `theano.tensor.concatenate`.
    """
    if force_cast_to_float:
        tensor_list = cast_to_float(tensor_list)
    out = tensor.concatenate(tensor_list, axis=axis)
    # Temporarily commenting out - remove when writing tests
    # conc_dim = int(sum([calc_expected_dim(graph, inp)
    #                for inp in tensor_list]))
    # This may be hosed... need to figure out how to generalize
    # shape = list(expression_shape(tensor_list[0]))
    # shape[axis] = conc_dim
    # new_shape = tuple(shape)
    # tag_expression(out, name, new_shape)
    return out

def theano_repeat(arr, n_repeat, stretch=False):
    """
    Create repeats of 2D array using broadcasting.
    Shape[0] incorrect after this node!
    """
    if arr.dtype not in ["float32", "float64"]:
        arr = tensor.cast(arr, "int32")
    if stretch:
        arg1 = arr.dimshuffle((0, 'x', 1))
        arg2 = tensor.alloc(1., 1, n_repeat, arr.shape[1])
        arg2 = tensor.cast(arg2, arr.dtype)
        cloned = (arg1 * arg2).reshape((n_repeat * arr.shape[0], arr.shape[1]))
    else:
        arg1 = arr.dimshuffle(('x', 0, 1))
        arg2 = tensor.alloc(1., n_repeat, 1, arr.shape[1])
        arg2 = tensor.cast(arg2, arr.dtype)
        cloned = (arg1 * arg2).reshape((n_repeat * arr.shape[0], arr.shape[1]))
    shape = expression_shape(arr)
    name = expression_name(arr)
    # Stretched shapes are *WRONG*
    tag_expression(cloned, name + "_stretched", (shape[0], shape[1]))
    return cloned

def cast_to_float(list_of_inputs):
    """ A cast that preserves name and shape info after cast """
    input_names = [inp.name for inp in list_of_inputs]
    cast_inputs = [tensor.cast(inp, theano.config.floatX)
                   for inp in list_of_inputs]
    for n, inp in enumerate(cast_inputs):
        cast_inputs[n].name = input_names[n]
    return cast_inputs

def interpolate_between_points(arr, n_steps=50):
    """ Helper function for drawing line between points in space """
    assert len(arr) > 2
    assert n_steps > 1
    path = [path_between_points(start, stop, n_steps=n_steps)
            for start, stop in safe_zip(arr[:-1], arr[1:])]
    path = np.vstack(path)
    return path

def path_between_points(start, stop, n_steps=100, dtype=theano.config.floatX):
    """ Helper function for making a line between points in ND space """
    assert n_steps > 1
    step_vector = 1. / (n_steps - 1) * (stop - start)
    steps = np.arange(0, n_steps)[:, None] * np.ones((n_steps, len(stop)))
    steps = steps * step_vector + start
    return steps.astype(dtype)

def make_shapename(name, shape):
    if len(shape) == 1:
        # vector, primarily init hidden state for RNN
        return name + TAG_ID + str(shape[0]) + "x"
    else:
        return name + TAG_ID + "x".join(map(str, list(shape)))

def parse_shapename(shapename):
    try:
        # Bracket for scan
        shape = shapename.split(TAG_ID)[1].split("[")[0].split("x")
    except AttributeError:
        raise AttributeError("Unable to parse shapename. Has the expression "
                             "been tagged with a shape by tag_expression? "
                             " input shapename was %s" % shapename)
    if "[" in shapename.split(TAG_ID)[1]:
        # inside scan
        shape = shape[1:]
    name = shapename.split(TAG_ID)[0]
    # More cleaning to handle scan
    shape = tuple([int(s) for s in shape if s != ''])
    return name, shape

def names_in_graph(list_of_names, graph):
    """ Return true if all names are in the graph """
    return all([name in graph.keys() for name in list_of_names])

def add_arrays_to_graph(list_of_arrays, list_of_names, graph, strict=True):
    assert type(graph) is OrderedDict
    arrays_added = []
    for array, name in safe_zip(list_of_arrays, list_of_names):
        if name in graph.keys() and strict:
            raise ValueError("Name %s already found in graph!" % name)
        shared_array = as_shared(array, name=name)
        graph[name] = shared_array
        arrays_added.append(shared_array)

def add_fixed_to_graph(list_of_fixed_numpy, list_of_shapes,
                       list_of_names, graph, strict=True):
    assert type(graph) is OrderedDict
    shared_added = []
    if RANDOM_ID not in graph.keys():
        graph[RANDOM_ID] = []
    for n, (fixed, shape, name) in enumerate(safe_zip(list_of_fixed_numpy,
                                                      list_of_shapes,
                                                      list_of_names)):
        shared_array = as_shared(fixed, name=name)
        tag_expression(shared_array, name, shape)
        shared_added.append(shared_array)
    graph[RANDOM_ID] += shared_added
    return shared_added

def add_random_to_graph(list_of_random, list_of_shapes, list_of_names,
                        graph, strict=True):
    assert type(graph) is OrderedDict
    random_added = []
    if RANDOM_ID not in graph.keys():
        graph[RANDOM_ID] = []
    for n, (random, shape, name) in enumerate(safe_zip(list_of_random,
                                                       list_of_shapes,
                                                       list_of_names)):
        tag_expression(random, name, shape)
        random_added.append(random)
    graph[RANDOM_ID] += random_added
    return random_added

def add_embedding_datasets_to_graph(list_of_embedding_vectors, list_of_masks,
                                    base_name, graph, strict=True):
    assert type(list_of_masks) is list
    assert type(list_of_embedding_vectors) is list
    assert len(list_of_embedding_vectors) == len(list_of_masks)
    embeddings = []
    masks = []
    for n, (v, m) in enumerate(zip(list_of_embedding_vectors, list_of_masks)):
        assert type(v) is list
        assert type(m) is np.ndarray
        embedding_names = [base_name + "_%s_%s" % (n, i) for i in range(len(v))]
        mask_names = [base_name + "_%s_mask" % i for i in range(len([m]))]
        r1 = add_datasets_to_graph(v + [m],
                                   embedding_names + mask_names,
                                   graph, strict=True)
        embeddings.append(r1[:len(embedding_names)])
        masks += r1[len(embedding_names):]
    return embeddings, masks

def add_datasets_to_graph(list_of_datasets, list_of_names, graph, strict=True,
                          list_of_test_values=None):
    assert type(graph) is OrderedDict
    datasets_added = []
    for n, (dataset, name) in enumerate(safe_zip(list_of_datasets,
                                                 list_of_names)):
        if dataset.dtype != "int32":
            if len(dataset.shape) == 1:
                sym = tensor.vector()
            elif len(dataset.shape) == 2:
                sym = tensor.matrix()
            elif len(dataset.shape) == 3:
                sym = tensor.tensor3()
            elif len(dataset.shape) == 4:
                sym = tensor.tensor4()
            else:
                raise ValueError("dataset %s has unsupported shape" % name)
        elif dataset.dtype == "int32":
            if len(dataset.shape) == 1:
                sym = tensor.ivector()
            elif len(dataset.shape) == 2:
                sym = tensor.imatrix()
            elif len(dataset.shape) == 3:
                sym = tensor.itensor3()
            elif len(dataset.shape) == 4:
                sym = tensor.itensor4()
            else:
                raise ValueError("dataset %s has unsupported shape" % name)
        else:
            raise ValueError("dataset %s has unsupported dtype %s" % (
                name, dataset.dtype))
        if list_of_test_values is not None:
            sym.tag.test_value = list_of_test_values[n]
        tag_expression(sym, name, dataset.shape)
        datasets_added.append(sym)
    if DATASETS_ID not in graph.keys():
        graph[DATASETS_ID] = []
    graph[DATASETS_ID] += datasets_added
    if len(list_of_datasets) == 1:
        # Make it easier if you only added a single dataset
        datasets_added = datasets_added[0]
    return datasets_added

def tag_expression(expression, name, shape):
    expression.name = make_shapename(name, shape)

def expression_name(expression):
    return parse_shapename(expression.name)[0]

def expression_shape(expression):
    return parse_shapename(expression.name)[1]

def alt_shape_of_variables(inputs, outputs, input_shapes):
    # Thanks to P. Lamblin, F. Bastien for help to make this work
    # mapping from initial to cloned var
    equiv = theano.gof.graph.clone_get_equiv(inputs, outputs)
    cloned_inputs = [equiv[inp] for inp in inputs]
    cloned_outputs = [equiv[out] for out in outputs]
    cloned_shapes = {equiv[k]: v for k, v in input_shapes.items()}
    fgraph = theano.FunctionGraph(cloned_inputs, cloned_outputs, clone=False)
    if not hasattr(fgraph, 'shape_feature'):
        fgraph.attach_feature(tensor.opt.ShapeFeature())

    kept_input = [n for n, f in enumerate(fgraph.inputs)
                  if f in fgraph.shape_feature.shape_of.keys()]

    input_dims = [dimension for kept_idx in kept_input
                  for dimension in fgraph.shape_feature.shape_of[
                      fgraph.inputs[kept_idx]]]
    """
    # The old way
    output_dims = [dimension for f, shape in
                   fgraph.shape_feature.shape_of.items()
                   for dimension in shape
                   if f in fgraph.outputs]
    """

    output_dims = list(*infer_shape(cloned_outputs, cloned_inputs,
                                    [input_shapes[k] for k in inputs]))

    try:
        compute_shapes = theano.function(input_dims,
                                         output_dims,
                                         mode=theano.Mode(optimizer=None),
                                         on_unused_input="ignore")

        numeric_input_dims = [dim for kept_idx in kept_input
                              for dim in cloned_shapes[fgraph.inputs[kept_idx]]]

        numeric_output_dims = compute_shapes(*numeric_input_dims)
    except MissingInputError:
        # need to add fake datasets and masks to input args for ?? reasons
        # unfortunate things might start happening if intermediate vars named
        # example that activated this code path
        # data -> linear -> tanh_rnn_layer
        dataset_and_mask_indices = [n for n, f in enumerate(fgraph.inputs)
                                    if f.name is not None]
        compute_shapes = theano.function(
            [fgraph.inputs[i] for i in dataset_and_mask_indices] + input_dims,
            output_dims,
            mode=theano.Mode(optimizer=None),
            on_unused_input="ignore")

        numeric_input_dims = [dim for kept_idx in kept_input
                              for dim in cloned_shapes[fgraph.inputs[kept_idx]]]
        fake_numeric_data = [np.ones(
            cloned_shapes[fgraph.inputs[i]]).astype(fgraph.inputs[i].dtype)
            for i in dataset_and_mask_indices]

        numeric_inputs = fake_numeric_data + numeric_input_dims

        numeric_output_dims = compute_shapes(*numeric_inputs)

    final_shapes = {}
    # This assumes only 1 OUTPUT!!!
    for var in fgraph.outputs:
        final_shapes[var] = np.array(numeric_output_dims).ravel()

    shapes = dict((outputs[n], tuple(np.array(final_shapes[co]).ravel()))
                  for n, co in enumerate(cloned_outputs))
    return shapes

def calc_expected_dims(graph, expression):
    # Intertwined with add_datasets_to_graph and add_random_to_graph
    # Expect variables representing datasets, shared, and random vars in graph
    # Named shape format is HxWxZ
    if expression in graph[DATASETS_ID]:
        # The expression input is a datastet - use tagged info directly
        dims = expression_shape(expression)
    elif RANDOM_ID in graph.keys() and expression in graph[RANDOM_ID]:
        # The expression input is a random variable - use tagged info directly
        dims = expression_shape(expression)
    else:
        # Assume anything not a random value or dataset is shared
        # Use short-circuit AND to avoid key-error if no random values in graph
        all_shared = [s for s in graph.values()
                      if s != graph[DATASETS_ID] and (
                          RANDOM_ID not in graph.keys()
                          or s != graph[RANDOM_ID])]
        #  == may not be good comparison in all cases
        all_random = [r for r in graph.values()
                      if (RANDOM_ID in graph.keys() and r == graph[RANDOM_ID])]
        # Flatten list of lists for random
        all_random = [ri for r in all_random for ri in r]
        all_inputs = graph[DATASETS_ID] + all_shared + all_random
        # Get shapes or fake shapes for all of the inputs
        dataset_shapes = [expression_shape(d) for d in graph[DATASETS_ID]]
        shared_shapes = [s.get_value().shape for s in all_shared]
        random_shapes = [expression_shape(r) for r in all_random]
        all_input_shapes = dataset_shapes + shared_shapes + random_shapes
        # Fake length of 2
        fake_shapes = [(2,) + s[1:] if len(s) <= 3 else s
                       for s in all_input_shapes]
        all_outputs = [expression]
        fake_dict = dict(zip(all_inputs, fake_shapes))
        calc_shapes = alt_shape_of_variables(all_inputs, all_outputs, fake_dict)
        dims = calc_shapes[expression]
    return dims

def fetch_from_graph(list_of_names, graph):
    """ Returns a list of shared variables from the graph """
    if DATASETS_ID not in graph.keys():
        # Check for dataset in graph
        raise AttributeError("No dataset in graph! Make sure to add "
                             "the dataset using add_datasets_to_graph")
    return [graph[name] for name in list_of_names]

def get_weights_from_graph(graph, verbose=False):
    weights = []
    for k, v in graph.items():
        if "_W" in k:
            weights.append(v)
            if verbose:
                print("Fetching weight %s from graph" % k)
    return weights

def get_params_and_grads(graph, cost, verbose=False):
    params = []
    for k, p in graph.items():
        if k == DATASETS_ID:
            # skip datasets
            continue
        if k == RANDOM_ID:
            # skip random
            continue
        params.append(p)

    if verbose:
        grads = []
        for k, p in graph.items():
            if k == DATASETS_ID:
                # skip datasets
                continue
            if k == RANDOM_ID:
                # skip random
                continue
            print("Computing grad w.r.t %s" % k)
            grad = tensor.grad(cost, p)
            grads.append(grad)
    else:
        grads = tensor.grad(cost, params)
    return params, grads

def create_checkpoint_dict(lcls):
    """
    Create checkpoint dict that contains all local theano functions

    Example usage:
        create_checkpoint_dict(locals())

    Parameters
    ----------
    lcls : dict
        A dictionary containing theano.function instances, normally the
        result of locals()

    Returns
    -------
    checkpoint_dict : dict
        A checkpoint dictionary suitable for passing to a training loop

    """
    print("Creating new checkpoint dictionary")
    checkpoint_dict = {}
    for k, v in lcls.items():
        if isinstance(v, theano.compile.function_module.Function):
            checkpoint_dict[k] = v
    if len(checkpoint_dict.keys()) == 0:
        raise ValueError("No theano functions in lcls!")
    return checkpoint_dict

def create_or_continue_from_checkpoint_dict(lcls, append_name="best"):
    """
    Create or load a checkpoint dict that contains all local theano functions

    Example usage:
        create_or_load_checkpoint_dict(locals(), append_name="best")

    Parameters
    ----------
    lcls : dict, default locals()
        A dictionary containing theano.function instances, normally the
        result of locals()

    append_name : string, default "best"
        The append name to use for the checkpoint

    Returns
    -------
    checkpoint_dict : dict
        A checkpoint dictionary suitable for passing to a training loop

    """
    sorted_paths = _get_file_matches("*.npz", append_name)
    if len(sorted_paths) < 1:
        print("No saved results found in %s, creating!" % get_checkpoint_dir())
        return create_checkpoint_dict(lcls)

    last_weights_path = sorted_paths[-1]
    print("Loading in weights from %s" % last_weights_path)
    last_weights = np.load(last_weights_path)

    checkpoint_dict = {}

    sorted_paths = _get_file_matches("*_results_*.pkl", append_name)
    last_results_path = sorted_paths[-1]
    print("Loading in results from %s" % last_results_path)

    checkpoint_dict["previous_results"] = _unpickle(
        last_results_path)

    for k, v in lcls.items():
        if isinstance(v, theano.compile.function_module.Function):
            matches = [name for name in last_weights.keys() if k in name]
            sorted_matches = sorted(
                matches, key=lambda x: int(x.split("_")[-1]))
            matching_values = [last_weights[s] for s in sorted_matches]
            _set_shared_variables_in_function(v, matching_values)
            checkpoint_dict[k] = v
    return checkpoint_dict


from parmed.amber import *
import os,sys

base = AmberParm(sys.argv[1],sys.argv[2])

res = base.residues[0]

base.LJ_depth = [0,0]
base.LJ_radius = [0,0]

base.recalculate_LJ()

base.write_parm("test.parm7")
base.write_rst7("test.rst7")