from __future__ import absolute_import
from __future__ import unicode_literals
from django.conf import settings
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse
from django.contrib.auth.models import User, Group
from django.contrib.auth import login
import requests
from django.http import HttpResponse, JsonResponse
from apps.accounts.models import UserProfile
from apps.fhir.bluebutton.models import Crosswalk
from apps.fhir.bluebutton.utils import get_resourcerouter, FhirServerAuth
import urllib.request as req
import random
from .models import AnonUserState
import logging
from django.shortcuts import render
from django.views.decorators.cache import never_cache

__author__ = "Alan Viars"

logger = logging.getLogger('hhs_server.%s' % __name__)

@never_cache
def callback(request):
    token_endpoint = getattr(
        settings, 'SLS_TOKEN_ENDPOINT', 'https://test.accounts.cms.gov/v1/oauth/token')
    redirect_uri = getattr(settings, 'SLS_REDIRECT_URI',
                           'http://localhost:8000/mymedicare/sls-callback')
    userinfo_endpoint = getattr(
        settings, 'SLS_USERINFO_ENDPOINT', 'https://test.accounts.cms.gov/v1/oauth/userinfo')
    verify_ssl = getattr(settings, 'SLS_VERIFY_SSL', False)
    code = request.GET.get('code')
    state = request.GET.get('state')
    try:
        aus = AnonUserState.objects.get(state=state)
    except AnonUserState.DoesNotExist:
        return JsonResponse({"error": "your OAuth2 client application must supply a state code."},
                            status=400)
    next_uri = aus.next_uri
    token_dict = {
        "grant_type": "authorization_code",
        "code": code,
        "redirect_uri": redirect_uri}
    logger.debug("token_endpoint %s" % (token_endpoint))
    logger.debug("redirect_uri %s" % (redirect_uri))
    # Call SLS token API
    r = requests.post(token_endpoint, json=token_dict, verify=verify_ssl)
    token_response = {}
    if r.status_code != 200:
        logger.error("Token request response error %s" % (r.status_code))
        return HttpResponse("Error: HTTP %s from the token response." % (r.status_code), status=r.status_code)
    token_response = r.json()
    # Create the Bearer
    bt = "Bearer %s" % (token_response['access_token'])
    # build a headers dict with Authorization
    headers = {"Authorization": bt}
    # Call SLS userinfo: ",
    # Authorization Bearer token in header.
    r = requests.get(userinfo_endpoint, headers=headers, verify=verify_ssl)
    # print("Status", r.status_code)
    if r.status_code != 200:
        logger.error("User info request response error %s" % (r.status_code))
        return HttpResponse("Error: HTTP %s response from userinfo request." % (r.status_code), status=r.status_code)
    # Get the userinfo response object
    user_info = r.json()
    try:
        user = User.objects.get(username=user_info['sub'][9:36])
        if not user.first_name:
            user.first_name = user_info['given_name']
        if not user.last_name:
            user.last_name = user_info['family_name']
        if not user.email:
            user.email = user_info['email']
        user.save()
    except User.DoesNotExist:
        # Create a new user. Note that we can set password
        # to anything, because it won't be checked.
        user = User(username=user_info['sub'][9:36], password='',
                    first_name=user_info['given_name'],
                    last_name=user_info['family_name'],
                    email=user_info['email'])
        user.save()
    up, created = UserProfile.objects.get_or_create(
        user=user, user_type='BEN')
    group = Group.objects.get(name='BlueButton')
    user.groups.add(group)
    # Log in the user
    user.backend = 'django.contrib.auth.backends.ModelBackend'
    login(request, user)
    # Determine patient_id
    fhir_source = get_resourcerouter()
    cx, g_o_c = Crosswalk.objects.get_or_create(
        user=user, fhir_source=fhir_source)
    hicn = user_info.get('hicn', "")
    cx.user_id_hash = hicn
    cx.save()
    auth_state = FhirServerAuth(None)
    certs = (auth_state['cert_file'], auth_state['key_file'])

    # URL for patient ID.
    url = fhir_source.fhir_url + \
        "Patient/?identifier=http%3A%2F%2Fbluebutton.cms.hhs.gov%2Fidentifier%23hicnHash%7C" + \
        cx.user_id_hash + \
        "&_format=json"
    response = requests.get(url, cert=certs, verify=False)

    if 'entry' in response.json():
        identifiers = response.json()['entry'][0]['resource']['identifier']
        fhir_id = ""
        for i in identifiers:
            if i['system'] == 'http://bluebutton.cms.hhs.gov/identifier#bene_id':
                fhir_id = i['value']
            if fhir_id:
                cx.fhir_id = fhir_id
                cx.save()
    # Get first and last naem from FHIR if not in OIDC Userinfo response.
    if user_info['given_name'] == "" or user_info['family_name'] == "":
        if 'entry' in response.json():
            if 'name' in response.json()['entry'][0]['resource']:
                names = response.json()['entry'][0]['resource']['name']
                first_name = ""
                last_name = ""
                for n in names:
                    if n['use'] == 'usual':
                        last_name = n['family']
                        first_name = n['given'][0]
                    if last_name or first_name:
                        user.first_name = first_name
                        user.last_name = last_name
                        user.save()

    # Delete the AUS since its not longer needed or valid:
    aus.delete()
    return HttpResponseRedirect(next_uri)

def generate_nonce(length=26):
    """Generate pseudo-random number."""
    return ''.join([str(random.randint(0, 9)) for i in range(length)])

@never_cache
def mymedicare_login(request):
    redirect = getattr(settings, 'MEDICARE_REDIRECT_URI',
                       'http://localhost:8000/mymedicare/sls-callback')
    mymedicare_login_url = getattr(settings, 'MEDICARE_LOGIN_URI',
                                   'https://impl1.account.mymedicare.gov/?scope=openid%20profile&client_id=bluebutton')
    redirect = req.pathname2url(redirect)
    state = generate_nonce()
    state = req.pathname2url(state)
    request.session['state'] = state
    mymedicare_login_url = "%s&state=%s&redirect_uri=%s" % (
        mymedicare_login_url, state, redirect)
    next_uri = request.GET.get('next', "")
    if request.user.is_authenticated():
        return HttpResponseRedirect(next_uri)
    AnonUserState.objects.create(state=state, next_uri=next_uri)
    if 'apps.testclient' in settings.INSTALLED_APPS:
        return HttpResponseRedirect(reverse('mymedicare-choose-login'))
    return HttpResponseRedirect(mymedicare_login_url)

@never_cache
def mymedicare_choose_login(request):
    mymedicare_login_uri = getattr(settings, 'MEDICARE_LOGIN_URI',
                                   'https://impl1.account.mymedicare.gov/?scope=openid%20profile&client_id=bluebutton')
    redirect = getattr(settings, 'MEDICARE_REDIRECT_URI',
                       'http://localhost:8000/mymedicare/sls-callback')
    redirect = req.pathname2url(redirect)
    aus = AnonUserState.objects.get(state=request.session['state'])
    mymedicare_login_uri = "%s&state=%s&redirect_uri=%s" % (
        mymedicare_login_uri, aus.state, redirect)
    context = {'next_uri': aus.next_uri,
               'mymedicare_login_uri': mymedicare_login_uri}
    return render(request, 'design_system/login.html', context)


from numpy import *
import sys
import shutil
import subprocess
import os
from itertools import product as iterproduct
aut=24.2#attoseconds per atomic unit of time
Hrt=27.21

#this script is designed to write input files for Dan Haxton's MCTDHF program
#corresponding to cosine squared pulses.  An abstract pulse class is defined
#which is capable of calculating the various quantities needed to create the
#input file

################################################################################
#pulse class contains all information needed to characterize a pulse and create
#the relevant input files

class pulse:
    def __init__(self, I0=1.e-4, pulsestrength=None, w0=.0565, wenv=.014,
                 CEPtheta=0, jtheta=0, jphiz=0, propeuler0=0, propeuler1=-pi/2,
                 propeuler2=-pi/2, tcenter=0, tstart=None , pulsetype=2,
                 chirp=0, ramp=0, longstep=0, rightcircular=False,
                 leftcircular=False):
        self.I0=I0#intensity divided by 10^16 W/cm^2
        if(pulsestrength==None):
            self.pulsestrength=sqrt(I0/3.51)/w0#definition in MCTDHF code
            #(hard coded 3.51 not explained in MCTDHF code  )
        else:
            self.pulsestrength=pulsestrength
            self.I0=pow(w0*pulsestrength,2)*3.51
        self.w0=w0#central frequency in atomic units
        self.wenv=wenv#envelope frequency in atomic units
        self.CEPtheta=CEPtheta#phase at peak of envelope
        self.jtheta=jtheta# jones theta: vec{E} = E0(zhat cos(jtheta) exp(i jphiz)+
                                #xhat sin(jtheta))
        self.jphiz=jphiz#jones phiz: vec{E} = E0(zhat cos(jtheta) exp(i jphiz)+
                                # xhat sin(jtheta))
        #given euler angles, define vectors for propagation direction and 2
        #polarization directions s.t. vpol1 x vpol2 = vprop

        #default values for right- or left-circularly polarized pulses
        if(rightcircular):
            self.jtheta=pi/4
            self.jphiz=-pi/2
        if(leftcircular):
            self.jtheta=pi/4
            self.jphiz=pi/2
        self.propeuler0=propeuler0
        self.propeuler1=propeuler1
        self.propeuler2=propeuler2
        self.initpolvectors()
        self.tcenter=tcenter
        if(tstart!=None):
            self.settstart(tstart)
        self.pulsetype=pulsetype
        self.chirp=chirp
        self.ramp=ramp
        self.longstep=longstep
        self.initMCTDHF()

    def initpolvectors(self):
        vprop=[0,0,1]
        vpol1=[1,0,0]
        vpol2=[0,1,0]
        rotmat0=Rx(self.propeuler0)
        rotmat1=Rz(self.propeuler1)
        rotmat2=Rx(self.propeuler2)
        self.vprop=dot(rotmat2,dot(rotmat1,vprop))
        self.vpol1=dot(rotmat2,dot(rotmat1,vpol1))
        self.vpol2=dot(rotmat2,dot(rotmat1,vpol2))
#        print("self.vprop\t"+str(self.vprop))
#        print("self.vpol1\t"+str(self.vpol1))
#        print("self.vpol2\t"+str(self.vpol2))

    def initMCTDHF(self):
        #set up two MCTDHF pulses containing the same information
        E0=self.E0()
        E1=E0*cos(self.jtheta)
        I1=E1**2
        E2=E0*sin(self.jtheta)
        I2=E2**2
        pulsestrength1=self.pulsestrength*cos(self.jtheta)
        pulsestrength2=self.pulsestrength*sin(self.jtheta)

        #MCTDHF takes I1, I2 as inputs rather than E1, E2.  Negative signs for
        #E1, E2 #correspond to a phase shift of pi
        if(sign(E1)<0):
            signphase1=pi
        else:
            signphase1=0
        if(sign(E2)<0):
            signphase2=pi
        else:
            signphase2=0
        pulser1, pulsetheta1, pulsephi1=topolar(self.vpol1)
        pulser2, pulsetheta2, pulsephi2=topolar(self.vpol2)

        strtphase=self.startphase()
        strtphase1=mod(strtphase+self.jphiz+signphase1, 2*pi)
        strtphase2=mod(strtphase+signphase2, 2*pi)
#apologies for using I1, I2, etc to define MCTDHFpulse0, MCTDHFpulse1, but it's
#better to name the pulses by their positions in the pulse list than otherwise
        #print ("self.pulsetype\t"+str(self.pulsetype))
        self.MCTDHFpulse0=MCTDHFpulse(pulsetype=self.pulsetype, w0=self.w0,
                                      wenv=self.wenv, I0=I1,
                                      pulsestrength=pulsestrength1,
                                      pulsetheta=pulsetheta1,
                                      tstart=self.tstart(),
                                      phaseshift=strtphase1, chirp=self.chirp,
                                      ramp=self.ramp, longstep=self.longstep)
        self.MCTDHFpulse1=MCTDHFpulse(pulsetype=self.pulsetype, w0=self.w0,
                                      wenv=self.wenv, I0=I2,
                                      pulsestrength=pulsestrength2,
                                      pulsetheta=pulsetheta2,
                                      tstart=self.tstart(),
                                      phaseshift=strtphase2, chirp=self.chirp,
                                      ramp=self.ramp, longstep=self.longstep)
        self.MCTDHFpulselist=[self.MCTDHFpulse0, self.MCTDHFpulse1]

    def twidth(self):
        return pi/self.wenv

    def settstart(self, tstart):
        self.tcenter=tstart+self.twidth()/2

    def E0(self):
        #print("self.I0\t"+str(self.I0))
        return sqrt(self.I0)

    def startphase(self):
        #MCTDHF code has pulse of the form A(t) sin(wenv t)^2
        #sin(wosc*t+startphase) however, it's often more convenient to
        #think about a pulse of the form
        #A(t) cos(wenv(t-tmid))^2 cos(w(t-tmid)+centerphase).

        #This subroutine calculates startphase corresponding to a chosen
        #centerphase -- ie, so that
        #sin(wosc*(t-tstart)+startphase)= cos(wosc*(t-tcenter)+centerphase),
        #where tcenter=tstart+(pi/2)/wenv
        accumulatedphase=self.twidth()/2*self.w0
        starttheta=mod(self.CEPtheta-accumulatedphase,2*pi)
        return starttheta

    def tstart(self):
        return self.tcenter-self.twidth()/2

    def tostringarray(self):
        #return list containing variable names and strings corresponding to their values
        return array([['I0', str(self.I0)], ['w0', str(self.w0)],
                      ['wenv',str(self.wenv)], ['CEPtheta',str(self.CEPtheta)],
                      ['jtheta',str(self.jtheta)], ['jphiz',str(self.jphiz)],
                      ['propeuler0',str(self.propeuler0)],
                      ['propeuler1',str(self.propeuler1)],
                      ['propeuler2',str(self.propeuler2)],
                      ['vprop',str(self.vprop)],
                      ['vpol1',str(self.vpol1)],
                      ['vpol2',str(self.vpol2)],
                      ['tcenter',str(self.tcenter)],
                      ['pulsetype',str(self.pulsetype)],
                      ['chirp',str(self.chirp)],
                      ['ramp',str(self.ramp)],
                      ['longstep',str(self.longstep)],
                     ])
#    def todict(self):
#        return {
#            'I0':str(self.I0),
#            'wenv':str(self.wenv),
#            'jtheta':str(self.jtheta),
#            'propeuler0':str(self.propeuler0),
#            'propeuler1':str(self.propeuler1),
#            'propeuler2':str(self.propeuler2),
#            'vprop':str(self.vprop),
#            'vpol1':str(self.vpol1),
#            'vpol2':str(self.vpol2),
#            'tcenter':str(self.tcenter),
#            'pulsetype':str(self.pulsetype),
#            'chirp':str(self.chirp),
#            'ramp':str(self.ramp),
#            'longstep':str(self.longstep),
#            'pulsestrength':str(self.pulsestrength)
#        }

    def todict(self):
        dictlist=[]
        for i in range(len(self.MCTDHFpulselist)):
            MCTDHFpulse=self.MCTDHFpulselist[i]
            dictlist.append(MCTDHFpulse.todict())
        retdict=dictjoin(dictlist)
        return retdict

    def tostring(self):
        tmparray=self.tostringarray()
        return "\t".join(tmparray[:,1])

    def stringkey(self, pulseindx=0):
        tmparray=self.tostringarray()
        joinstr="_"+str(pulseindx)+"\t"
        return joinstr.join(tmparray[:,0])+"_"+str(pulseindx)

################################################################################
#exponential pulse class gives a pulse of the form
#cos(wenv t)**2 e^(1j w2 t)=cos(wenv t)**2 * (cos(w2 t)+1j*sin(w2 t))
class exponential_pulse:
    def __init__(self, **kwargs):
#cosine component
        self.pulse0=pulse(**kwargs)
#sine component
        CEPtheta=self.pulse0.CEPtheta
        pulsestrength=self.pulse0.pulsestrength

        self.pulse1=pulse(**kwargs)
        self.pulse1.CEPtheta-=pi/2
        self.pulse1.CEPtheta=mod(self.pulse1.CEPtheta, 2*pi)
        self.pulse1.pulsestrength*=1j
        self.pulse1.initMCTDHF()

#make list of component pulses, MCTDHF pulses
        self.pulselist=[self.pulse0, self.pulse1]
        self.MCTDHFpulselist=self.pulse0.MCTDHFpulselist+self.pulse1.MCTDHFpulselist

    def todict(self):
        dictlist=[]
        for i in range(len(self.MCTDHFpulselist)):
            MCTDHFpulse=self.MCTDHFpulselist[i]
            dictlist.append(MCTDHFpulse.todict())
        retdict=dictjoin(dictlist)
        return retdict

################################################################################
#MCTDHF pulse class contains all information needed to write an MCTDHF input file
class MCTDHFpulse:
    def __init__(self, pulsetype=2, w0=.0565, wenv=.014, I0=1.e-4, pulsestrength=0, pulsetheta=0,
                 tstart=0., phaseshift=0, chirp=0, ramp=0, longstep=0):
#this class contains the information that can go into an MCTDHF pulse and
#returns a dictionary which maps the names of the variables to strings
#containing their value
        self.pulsetype=pulsetype
        self.pulsestrength=pulsestrength
        self.w0=w0
        self.wenv=wenv
        self.I0=I0
        self.pulsetheta=pulsetheta
        self.tstart=tstart
        self.phaseshift=phaseshift
        self.chirp=chirp
        self.longstep=longstep

    def todict(self):
        return {'#pulsetypelist#':str(self.pulsetype), '#omega2list#':str(self.w0),
                '#pulsestartlist#':str(self.tstart),
                '#omegalist#':str(self.wenv), '#intensitylist#':str(self.I0),
                '#pulsethetalist#':str(self.pulsetheta),
                '#CEPlist#':str(self.phaseshift), '#chirplist#':str(self.chirp),
                '#longsteplist#':str(self.longstep),
                '#pulsestrengthlist#':fortran_cmplx_str(self.pulsestrength)}

################################################################################
#Helper functions

def fortran_cmplx_str(zinp):
    rval=real(zinp)
    imval=imag(zinp)
    return "("+str(rval)+", "+str(imval)+")"

def FWHM_to_wenv(FWHM_fs):
    return pi/(FWHM_fs*2/aut)

def wenv_to_tmid(wenv):
    return (pi/2)/wenv

def phasearray(nphasepoints):
    return arange(0.,2.,2./nphasepoints)

#functions to choose dt,npts for desired energy resolution
def tparams(dE, maxE):
    #dE is desired energy resolution, maxE is maximum energy we wish to resolve
    dEHrt=dE/Hrt
    maxEHrt=maxE/Hrt
    ErangeHrt=2*maxEHrt
    npts=int(floor(ErangeHrt/dEHrt))

    #critical sampling is 2 samples per period
    dt=(2*pi)/(ErangeHrt)
    Tmax=dt*npts
    return dt,Tmax,npts

def pulselistkey(pulselist):
    retstr=''
    for i in range(len(pulselist)):
        retstr+=pulselist[i].stringkey(pulseindx=i)+"\t"
    return retstr+"\n"
def printloopkey(resultdir, loopnamelist, looplist, filename='loopkey.txt'):
    tmpfile=open(resultdir+filename,'w')
    tmpfile.write("arrays looped over in this calculation (outer loops first, inner loops last)\n")
    for i in range(len(loopnamelist)):
#        tmpfile.write(loopnamelist[i]+"\t"+str(looplist[i])+"\n")
        tmpfile.write(loopnamelist[i]+"\t"+"\t".join(map(str,looplist[i]))+"\n")
    tmpfile.close

def printparameterkey_full(resultdir, pulsearray, filename='parameterkey_full.txt'):
    tmpfile=open(resultdir+filename,'w')
    tmpfile.write(pulselistkey(pulsearray[0]))
    for i in range(len(pulsearray)):
        tmpfile.write(pulselisttostring(pulsearray[i]))
    tmpfile.close

def dictjoin(dictlist, joinstr=", "):
#join the strings saved in different pulse dictionaries
    retdict={}
    for key in dictlist[0]:
        tmpstr=''
        for i in range(len(dictlist)):
            tmpstr+=str(dictlist[i][key])+", "
        retdict[key]=tmpstr[:-2]
    return retdict

def pulselistdictionary(pulselist):
#first, make a list of the MCTHDF pulse list dictionaries
    pulsedictlist=[]
    for i in range(len(pulselist)):
        for j in range(len(pulselist[i].MCTDHFpulselist)):
            pulsedictlist+=[pulselist[i].MCTDHFpulselist[j].todict()]
    dict0=dictjoin(pulsedictlist)
    return dict0

def makeinputfiles(pulsearray, resultdir):
    masterscriptfile="fullcalculation.sh"
    for calcindx in range(len(pulsearray)):
        pulselist=pulsearray[calcindx]
        dirstr=resultdir+str(calcindx)+"/"
        subprocess.call(["mkdir", dirstr])

        initialrelaxation(masterscriptfile,dirstr)
        multiplepulses(masterscriptfile,dirstr,1,pulselist)

def initialrelaxation(masterscriptfile,dirstr):
    bashtemplate="templates/Relax.Bat.template"
    inputtemplate="templates/Input.Inp.Relax.template"

    steproot=stepstr(0)

    bashfilename=steproot+".Bat"
    inpfilename="Input.Inp."+steproot
    outfilename="Out.States."+steproot

    #list of replacement rules
    dictlist=[inpoutdictionary(steproot,steproot),filedictionary(inpfilename,outfilename)]

    #use dictlist to convert template files to usable input & script files
    templatereplace(bashtemplate,dirstr+bashfilename,dictlist)
    templatereplace(inputtemplate,dirstr+inpfilename,dictlist)

    masterscript=open(dirstr+masterscriptfile,'w')
    masterscript.write("bash "+bashfilename+"\n")
    masterscript.close()

def multiplepulses(masterscriptfile, dirstr, stepindx, pulselist):
    dict0=pulselistdictionary(pulselist)

    #next, make a dictionary corresponding to the values which aren't contained
    #in the previous dictionary
    numpulses=sum(map(lambda pls: len(pls.MCTDHFpulselist), pulselist))#2*len(pulselist)
    if(velocitygauge):
        velstr="1"
    else:
        velstr="0"
    dict1={'#numpulses#':str(numpulses), '#timestep#':str(timestep),
           '#measurementtimestep#':str(measurementtimestep),
           '#tfinal#':str(tfinal), "#velflag#":velstr}

    inpstr=stepstr(stepindx-1)
    outstr=stepstr(stepindx)
    steproot=stepstr(stepindx)

    bashtemplate="templates/Multiplepulses.Bat.template"
    inputtemplate="templates/Input.Inp.Multiplepulses.template"

    bashfilename=steproot+".Bat"
    inpfilename="Input.Inp."+steproot
    outfilename="Out.States."+steproot

    dictlist=[dict0, dict1, inpoutdictionary(inpstr, outstr),
              filedictionary(inpfilename, outfilename)]
    #use dictlist to convert template files to usable input & script files
    templatereplace(bashtemplate,dirstr+bashfilename,dictlist)
    templatereplace(inputtemplate,dirstr+inpfilename,dictlist)

    masterscript=open(dirstr+masterscriptfile,'a')
    masterscript.write("bash "+bashfilename+"\n")
    masterscript.close()

def inpoutdictionary(inpstr,outstr):
    dict={'#inpstr#':inpstr,'#outstr#':outstr}
    return dict

def filedictionary(inpfilename,outfilename):
    dict={'#inpfilename#':inpfilename,'#outfilename#':outfilename,'#gridstr#':gridstring,'#atomstr#':atomstring}
    return dict

def replacementdict(pulselist):
    dictlist=[]
    for i in range(len(pulselist)):
        for j in range(len(pulselist[i].MCTDHFpulselist)):
            dictlist+=pulselist[i].MCTDHFpulselist[j].todict()

def templatereplace(oldfilename,newfilename,dictlist):
    oldfile=open(oldfilename)
    newfilestring=oldfile.read()

    for dict in dictlist:
        newfilestring=dictionaryreplace(newfilestring,dict)

    newfile=open(newfilename,'w+')
    newfile.write(newfilestring)
    #os.chmod(newfilename,stat. S_IXUSR)
    oldfile.close()
    newfile.close()

def stepstr(n):
    return "step_"+str(n)

def Rx(theta):
#rotation matrix about x axis
    return [[1,0,0],[0,cos(theta),-sin(theta)],[0,sin(theta),cos(theta)]]

def Rz(theta):
    #rotation matrix about z axis
    return [[cos(theta),-sin(theta),0],[sin(theta),cos(theta),0],[0,0,1]]

def Ry(theta):
#rotation matrix about y axis
    return [[cos(theta),0,sin(theta)],[0,1,0],[-sin(theta),0,cos(theta)]]

def topolar(cartvec):
#convert cartesian vector to polar coords
    vx=cartvec[0]
    vy=cartvec[1]
    vz=cartvec[2]
    r=sqrt(vx**2+vy**2+vz**2)
    theta=arccos(vz/r)
    phi=angle(vx+1j*vy)
    return r, theta, phi

def nm_to_eV(nm):
    hc=1239.841#planck's constant in eV*nm
    eV=hc/nm
    return eV

def eV_to_nm(eV):
    hc=1239.841#planck's constant in eV*nm
    nm=hc/eV
    return nm

def dictionaryreplace(text,dictionary):
    newtext=text
    for i,j in dictionary.items():#.iteritems():
        newtext=newtext.replace(i,j)
    return newtext

################################################################################
#Main program

##test initiation
#pls1=pulse()
#print("check to see that pls1 is initiated")
#print("pls1 vprop\t"+str(pls1.vprop))
#print("pls1 vpol1\t"+str(pls1.vpol1))
#print("pls1 vpol2\t"+str(pls1.vpol2))

evstr=sys.argv[-1]
if(len(sys.argv)<2):
    evstr="22"
evfloat=float(evstr)
resultdir="results_"+str(evstr)+"eV/"
#resultdir=sys.argv[-2]#"results_tmp/"
if(resultdir[-1]!="/"):
    resultdir=resultdir+"/"

#copy this program into resultdir
subprocess.call(['mkdir',resultdir])
shutil.copy('./corotating_counterrotating_streaking.py',resultdir)

#copy templates directory to resultdir
templatetargetstr=resultdir+"templates"
if(os.path.exists(templatetargetstr)):
    shutil.rmtree(templatetargetstr)
shutil.copytree("./templates",templatetargetstr)

#read grid parameters from file
tmpfile=open("templates/gridfile.txt")
gridstring="".join(tmpfile.readlines())
tmpfile.close()
#read atom parameters from file
tmpfile=open("templates/atomfile.txt")
atomstring="".join(tmpfile.readlines())
tmpfile.close()

################################################################################
#main program

velocitygauge=False#True#True: use velocity gauge.  False: use length gauge

timestep=0.05
wir=nm_to_eV(800)/Hrt
wxuv=evfloat/Hrt

Iir=3e-4
Ixuv=1.e-6
IRduration=11e3#pulse FWHM in attoseconds
IRwenv=FWHM_to_wenv(IRduration)
IRtmid_atomic=wenv_to_tmid(IRwenv)#midpoint of IR pulse in atomic units
XUVduration=11e3
XUVwenv=FWHM_to_wenv(XUVduration)

dE=.25#.08#desired energy resolution in eV
maxE=40.#20.#desired maximum energy range in eV
dtdip,Tdipmax,ndtdip=tparams(dE,maxE)
dtdip=ceil(dtdip/timestep)*timestep#make interpulse delays an integral
                               #number of timesteps
measurementtimestep=dtdip#interval between calculating dipole
tfinal=Tdipmax#total interval to propagate when calculating dipole

#make array corresponding to delay between the centers of the two pulses
dtstart=0.
dtstop=0.
deltadt=100#
dtarray=arange(dtstart,dtstop+deltadt,deltadt)/aut
ndt=len(dtarray)

#number of phase points for XUV and IR pulses
nxuvphase=5
xuvphasearray=phasearray(nxuvphase)

ncophase=5
cophasearray=phasearray(ncophase)

ncounterphase=5
counterphasearray=phasearray(ncounterphase)

paramindx=0
pulsearray=[]
loopnamelist=['dtarray','xuvphasearray','cophasearray','counterphasearray']
looplist=[dtarray, xuvphasearray, cophasearray, counterphasearray]
printloopkey(resultdir,loopnamelist, looplist)
for i,j,k,l in iterproduct(range(ndt), range(nxuvphase), range(ncophase),
                             range(ncounterphase)):
    dt=dtarray[i]
    xuvphase=xuvphasearray[j]*pi
    cophase=cophasearray[k]*pi
    counterphase=counterphasearray[l]*pi
    #print("sumphase\t"+str(sumphasearray[l])+"\t"+str(sumphase))
    #IR pulse is split into two pulses: E(t)= 1/sqrt(2)*E0*(cos(wt+IRphase+sumphase)+cos(wt+IRphase-sumphase))
    pulselist=[exponential_pulse(I0=Iir/sqrt(2), w0=wir, wenv=IRwenv,
                                 CEPtheta=cophase,tstart=0),
               exponential_pulse(I0=Iir/sqrt(2), w0=-wir, wenv=IRwenv,
                                 CEPtheta=counterphase,tstart=0),
               pulse(I0=Ixuv, w0=wxuv, wenv=XUVwenv, CEPtheta=xuvphase,
                     tcenter=IRtmid_atomic+dt)]
#    pulselist=[pulse(I0=Iir/sqrt(2),  w0=wir, wenv=IRwenv,
#                     CEPtheta=IRphase+sumphase, tstart=0),
#               pulse(I0=Iir/sqrt(2),  w0=wir, wenv=IRwenv,
#                     CEPtheta=IRphase-sumphase, tstart=0),
#               pulse(I0=Ixuv, w0=wxuv, wenv=XUVwenv,
#                     CEPtheta=xuvphase, tcenter=IRtmid_atomic+dt)]
    pulsearray.append(pulselist)

#make input files corresponding to this calculation
makeinputfiles(pulsearray, resultdir)


from django.conf.urls import patterns, include, url
from django.contrib import admin

urlpatterns = patterns('',
    # Examples:

    # url(r'^blog/', include('blog.urls')),
    # url(r'^view', 'Obrazki_django.views.home', name='home'),
    url(r'^$' ,       include('img_bro.urls')), # kiedy nie ma nic w pasku poza localhostem

    url(r'admin/', include(admin.site.urls)),
    url(r'', include('img_bro.urls')),
)


from .config import (COOKIE_NAME, COOKIE_REMEMBER_ME, COOKIE_DURATION,
                     COOKIE_SECURE, COOKIE_HTTPONLY, LOGIN_REQUIRED_MESSAGE, LOGIN_REQUIRED_ROUTE,
                     LOGIN_PROHIBITED_MESSAGE, LOGIN_PROHIBITED_ROUTE, DISABLE_RUNTIME_WARNING)

def UNAUTHORIZED(req, resp, *_, **__):
    if LOGIN_REQUIRED_ROUTE:
        resp.headers.update({"Location": LOGIN_REQUIRED_ROUTE})
    resp.text = LOGIN_REQUIRED_MESSAGE
    resp.status_code = 401

def AUTHORIZED(req, resp, *_, **__):
    if LOGIN_PROHIBITED_ROUTE:
        resp.headers.update({"Location": LOGIN_PROHIBITED_ROUTE})
    resp.text = LOGIN_PROHIBITED_MESSAGE
    resp.status_code = 403


import datetime as dt
import logging
import shutil

# from tensorflow.python.summary.writer import writer_cache
# import tensorflow.contrib.keras as kr
import keras as kr
import numpy as np
import pandas as pd
import tensorflow as tf
from phdlib.data.api import get_monthly_data, LocalResults, LoadLocalResult, \
    Sector, r2_trans
from phdlib.metrics.deep_learn import HPGenerator, DNNBuilder, DNNTrainer, \
    LinearModel, TensorPandas, hyper_para_string, HP1DGrid
from phdlib.metrics.tools import KFoldCV
from phdlib.misc.api import log_set_ups_runs, load_settings
from phdlib.settings import RESULTS_PATH

__author__ = 'mmess'

logger = logging.getLogger(__name__)

FILE_PATTERN_HYPER = 'hp_simulation'

def load_data(train=True, para=None, dates=None):

    if para is None:
        para = dict(
            size='large',
            demean_ret=True,
            fill_na=True,
            winsorize_prc=None,  # 0.01,
            drop_na=True,
            drop_columns=['cfp_ia', 'roic', 'pchemp_ia', 'ipo'],
            per_rank=True,
        #    sample_balance=4000,
        )
    if dates is None:
        dates = dict(train_start='19700101',
                     train_end='19791231',
                     valid_start='19800101',
                     valid_end='19950131', )

    if train:
        para = {**dict(start_date=dates['train_start'],
                       end_date=dates['train_end'],
                       fc_lead=1,
                       ), **para}
    else:
        para = {**dict(start_date=dates['valid_start'],
                       end_date=dates['valid_end'],
                       fc_lead=0), **para}

    df = get_monthly_data(**para)
    df.dropna(inplace=True)
    df.set_index(['date', 'PERMNO'], append=True, inplace=True)
    X = df.drop(['ret'], axis=1)
    y = df['ret'] * 1000
    # y = y.groupby(level='date').rank(pct=True)*100

    return X, y, para

def hyper_optimization(X, y, hyper_para, para, file_pattern=None):
    """ Performs a hyper-parameter optimization

    Parameters
    ----------
    X: pd.DataFrame
    y: pd.DataFrame
    hyper_para: pd.DataFrame
        with a set of hyper-parameters
    para: dict
        model parameter (size)
    file_pattern: str
        which pattern should be used to save the data

    Returns
    -------
    LocalResults
    """
    cv_runs = para['cv_depth']
    # If each model should be saved
    delete_mode_dir = True
    # If loss data should be saved
    save_performance_results = True

    model_dir = RESULTS_PATH.joinpath('TB').joinpath('keras_debug_tb') # +
    #      dt.datetime.now().strftime('%Y-%m-%d_%H-%M'))

    tf_pd_data = TensorPandas(X.columns)
    # ensure training and validation are from same period (compatibility
    # across runs)
    np.random.seed(10)
    training_data = [(train, valid) for train, valid in KFoldCV(X, y)]

    # initiate DF for result collection
    loss_df = pd.DataFrame(index=hyper_para.index,
                           columns=['dnn', 'linear', 'zero']).fillna(0)
    std_err = pd.DataFrame(index=hyper_para.index,
                           columns=['dnn_std', 'linear_std',
                                    'zero_std']).fillna(0)

    input_columns = len(X.columns)

    if save_performance_results:
        res_saver = LocalResults()
    tmp_file_name = None

    for idx, draw in hyper_para.iterrows():

        count = 0
        try:
            for train_, valid_ in training_data[:cv_runs]:
                count += 1
                logger.info('\n Current run {}/{} \n CV {}/5'.format(idx + 1,
                                                           len(hyper_para),
                                                           count))
                logger.info(draw)
                # current_model_dir = str(
                #     model_dir.joinpath(hyper_para_string(draw) + str(count)))
                current_model_dir = None

                model = DNNBuilder(input_cols=input_columns, output_dim=1,
                                   hidden_units=draw['hidden_units'],
                                   layers=draw['layers'],
                                   dropout=draw['dropout'],
                                   l_2=draw['l2_strength'],
                                   l_1=draw['l1_strength'],
                                   input_dropout=True).model

                DNNTrainer(model, learning_rate=draw['learning_rate'],
                           model_dir=current_model_dir,
                           optimizer=draw['keras_optimizer'],
                           patience=draw['patience'],
                           # loss='mean_absolute_error',
                           ).fit(*tf_pd_data.xy_values(train_),
                                 validation_data=tf_pd_data.xy_values(valid_),
                                 verbose=0)

                loss_df.loc[idx, 'dnn'] += model.evaluate(
                    *tf_pd_data.xy_values(valid_))
                std_err.loc[idx, 'dnn_std']+= model.predict(
                    x=tf_pd_data.x_values(valid_[0])).std()
                kr.backend.clear_session()
                if delete_mode_dir:
                    if current_model_dir is not None:
                        shutil.rmtree(current_model_dir)

        except tf.errors.InvalidArgumentError:
            logger.error('Error occured at run {} cv {}'.format(idx, count))
            loss_df.loc[idx, 'dnn'] = np.nan
            kr.backend.clear_session()
            shutil.rmtree(current_model_dir, ignore_errors=True)

        if save_performance_results:
            tmp_res = pd.concat([loss_df, hyper_para], axis=1)
            res_saver.save(tmp_res, parameters=para,
                           results_name=file_pattern)
            # if old tmp_files are present -> delete as new res are saved)
            if tmp_file_name is not None:
                res_saver.delete(tmp_file_name, raise_error=False)
            tmp_file_name = res_saver.file

    # train linear model and evaluate zero prediction model
    for train_, valid_ in training_data[:cv_runs]:
        linear_model = LinearModel(train_, valid_)
        loss_df['linear'] += linear_model.evaluate()
        loss_df['zero'] += np.mean(valid_[1].values ** 2)
        std_err['linear_std'] += linear_model.predict(X=valid_[0]).std()
        std_err['zero_std']=valid_[1].values.std()

    res = pd.concat([loss_df, hyper_para, std_err], axis=1)
    logger.info('Hyper-search complete, the results are:\n{}'.format(res))

    if save_performance_results:
        res_saver.save(res, parameters=para, results_name=file_pattern,
                       overwrite=True)
        if tmp_file_name is not None and res_saver.file != tmp_file_name:
            res_saver.delete(tmp_file_name, raise_error=False)

    return res

def random_search(which=None):
    """ Performs random search of deep learning

    Parameters
    ----------
    which

    Returns
    -------

    """
    para = load_settings(FILE_PATTERN_HYPER)
    num_draws = para['simulation']['num_draws']
    cv_runs = para['simulation']['cv_depth']
    # load data
    X, y, para = load_data(para=para['data'], dates=para['dates'])
    para['cv_depth'] = cv_runs

    # Draw hyper parameter
    # make random again, otherwise same parameters are run
    np.random.seed(int(dt.datetime.now().timestamp()))
    if which is None:
        which = ['learning_rate', 'l2_strength', 'l1_strength', 'dropout',
                 'hidden_units', 'layers', 'keras_optimizer', 'patience',]
    hyper_para = HPGenerator(which=which, size=num_draws,
                             num_input_units=len(X.columns)).generate()
    logger.info('The following hyper-parameter sets are used: \n {}'.format(
        hyper_para))

    hyper_optimization(X=X, y=y, hyper_para=hyper_para, para=para,
                       file_pattern=FILE_PATTERN_HYPER)

def grid_search(which=None):
    """ 2nd Step Grid search algorithm

    Returns
    -------

    """
    para = load_settings(FILE_PATTERN_HYPER)
    num_grid_points = 20
    cv_runs = para['simulation']['cv_depth']
    # load data
    X, y, para = load_data(para=para['data'], dates=para['dates'])
    para['cv_depth'] = cv_runs

    if which is None:
        which = ['learning_rate', 'l2_strength', 'l1_strength', 'dropout',
                 'hidden_units', 'layers', 'keras_optimizer', 'patience', ]
    #  load optimal parameters
    opt_para = inspect_results(size=para['size'])
    lr = HP1DGrid(which='learning_rate', size=num_grid_points,
                  low_bound=2, upper_bound=6).get()
    hyper_para = pd.DataFrame(columns=which, index=lr.index)

    for parameter in which:
        hyper_para[parameter] = opt_para.loc[0, parameter]

    hyper_para['learning_rate'] = lr['learning_rate']
    logger.info('The following learning rates are used for {}'
                'size Grid search: \n {}'.format(para['size'], hyper_para))

    hyper_optimization(X=X, y=y, hyper_para=hyper_para, para=para,
                       file_pattern='HP_grid_search')

def tensorboard_tuning(size='large'):
    """ Tensorboard tuning

    Returns
    -------

    """

    X, y, data_para = load_data()
    training_data = [(train, valid) for train, valid in KFoldCV(X, y)]
    model_dir = RESULTS_PATH.joinpath('TB').joinpath('tf_debug_tb_' +
                            dt.datetime.now().strftime('%Y-%m-%d_%H-%M'))
    which = ['learning_rate', 'l2_strength', 'l1_strength', 'dropout',
             'hidden_units', 'layers', 'keras_optimizer', 'patience']

    #  load optimal parameters
    draw = inspect_results(size=size, search='grid').loc[
        0, which].to_dict()

    current_model_dir = str(model_dir)
    tf_pd_data = TensorPandas(X.columns)
    input_columns = len(X.columns)

    train_ = training_data[0][0]
    valid_ = training_data[0][1]
    model = DNNBuilder(input_cols=input_columns, output_dim=1,
                       hidden_units=draw['hidden_units'],
                       layers=draw['layers'],
                       dropout=draw['dropout'],
                       l_2=draw['l2_strength'],
                       l_1=draw['l1_strength'],
                       input_dropout=True).model

    DNNTrainer(model, learning_rate=draw['learning_rate'],
               model_dir=current_model_dir,
               optimizer=draw['keras_optimizer'],
               patience=draw['patience'],
               # loss='mean_absolute_error',
               ).fit(*tf_pd_data.xy_values(train_),
                     validation_data=tf_pd_data.xy_values(valid_),
                     verbose=1)

    # hyper_para['hidden_units'] = 38  # 31
    # hyper_para['layers'] = 5  # 12
    # hyper_para['l2_strength'] = 0.01565639704941852  # 0.002044
    # hyper_para['dropout'] = 0.050349424919385655  # 0.138813
    # hyper_para['learning_rate'] = 0.013861455351410994  # 0.000199
    # # hyper_para['keras_optimizer'] = 'Adadelta'
    # hyper_para['l1_strength'] = 0.0010869141938006771
    #
    # optimizer = dict(Nadam=tf.train.keras.optimizers.Nadam)
    #
    # optimizer = optimizer[hyper_para['keras_optimizer']](
    #     lr=float(hyper_para['learning_rate']),
    #     l1_regularization_strength=float(hyper_para['l1_strength']),
    #     l2_regularization_strength=float(hyper_para['l2_strength'])
    # )
    #
    # tf_pandas = TensorPandas(X.columns)
    #
    # losses = {'dnn': 0, 'linear': 0, 'zero_mse': 0}
    #
    # for train_, valid_ in training_data[:1]:
    #     # build model
    #     model = tf.contrib.learn.DNNRegressor(
    #         hidden_units=[hyper_para['hidden_units']] *
    #                      hyper_para['layers'],
    #         model_dir=str(model_dir),
    #         feature_columns=tf_pandas.input_columns(X),
    #         optimizer=optimizer,
    #         dropout=hyper_para['dropout'])
    #
    #     # training_data
    #     input_train = lambda: tf_pandas.input_fn(train_[0], train_[1])
    #     input_valid = lambda: tf_pandas.input_fn(valid_[0], valid_[1])
    #     for _ in range(500):
    #         print('--------{}-----------'.format(_))
    #         model.fit(input_fn=input_train, steps=10)
    #         model.evaluate(input_fn=input_valid, steps=1)
    #         a = model.predict(input_fn=input_valid,as_iterable=False)
    #         print('Std y_hat: {} \n'.format(np.array(a).std()))
    #         print('Std y: {} \n'.format(y.std()))
    #
    # for train_, valid_ in training_data:
    #     losses['linear'] += LinearModel(train_, valid_).train()
    #     losses['zero_mse'] += np.mean(valid_[1].values ** 2)

def inspect_results(size='large', search='random', winsorize=0.01,
                    per_rank=False):
    """ Returns tables of random search

    Parameters
    ----------
    size: str
        which size to use "large", "mid", "small" - none uses all

    Returns
    -------

    """

    conditions = {
        "size": size,
        "start_date": "19700101",
        "end_date": "19811231",
        "demean_ret": True,
        "drop_columns": ["cfp_ia", "roic", "pchemp_ia", "ipo"],
        "fc_lead": 1,
        "drop_na": True,
        "winsorize_prc": winsorize,
        "fill_na": True,
        "per_rank": per_rank
    }

    if search == 'random':
        file_pattern = FILE_PATTERN_HYPER
    elif search == 'grid':
        file_pattern = 'HP_grid_search'

    # results
    files = LoadLocalResult(file_pattern=file_pattern,
                            since='20170821',
                            conditions=conditions).files_true

    results_coll = list()
    for file in files:
        print(file)
        tmp_res = LoadLocalResult(file_pattern=file)
        res = tmp_res.data
        res['Host'] = tmp_res.host
        res['start_date'] = tmp_res.parameter['start_date']
        res['end_date'] = tmp_res.parameter['end_date']
        res['run_time_end'] =  tmp_res.end_time
        res['cv_depth'] = tmp_res.parameter['cv_depth']

        results_coll.append(res)

    res = pd.concat(results_coll)
    print(len(res))
    res['R2_dnn'] = 1 - res['dnn'].divide(res['zero'])
    res['R2_lin'] = 1 - res['linear'].divide(res['zero'])
    res['Rank'] = res['R2_dnn'].rank(ascending=False)

    # add "patience" once new search is over
    cols = ['Rank', 'R2_dnn', 'R2_lin', 'learning_rate', 'l2_strength',
            'l1_strength', 'dropout', 'hidden_units', 'layers',
            'keras_optimizer', 'patience'  ]
    # res.dropna(inplace=True)
    res = res[cols].sort_values('R2_dnn', ascending=False).reset_index(
        drop=True)
    trans_fun = lambda x: r2_trans(x, 12)
    res['R2_dnn'] = res['R2_dnn'].apply(trans_fun)
    res['R2_lin'] = res['R2_lin'].apply(trans_fun)
    logger.info('Top Ten Networks for {} cap stocks are : \n {}'.format(
        size, res.head(10)))

    logger.info('In total {} show R^2 lower than 0.\n'
                'A total of {} have NA values'.format(
        (res['R2_dnn'] < 0).sum(), (res['R2_dnn'].isnull()).sum()))

    return res

def top_hyper_to_json(df, path_or_buf=None):
    """ Selects top optimization result and returns/saves as json

    Parameters
    ----------
    df: pd.DataFrame
        with hyper-parameter columns and a column named "Rank"

    Returns
    -------

    """

    which = ['learning_rate', 'l2_strength', 'l1_strength', 'dropout',
             'hidden_units', 'layers', 'keras_optimizer', 'patience']

    included_columns = list(set(which) & set(df.columns))

    return df.loc[df['Rank'] == 1, included_columns].to_json(
        path_or_buf=path_or_buf, orient='records')

if __name__=='__main__':
    logging.config.dictConfig(log_set_ups_runs)
    logging.root.addHandler(logging.getLogger('run_log').handlers[0])
    logger.setLevel('INFO')
    # tensorboard_tuning()
    # random_search()
    # grid_search()
    # res = inspect_results(search='grid', size='large_mid')
    res = top_hyper_to_json(inspect_results(search='grid', size='large_mid'))
    # tensorboard_tuning()
    # res = inspect_results()


import base64
import json
import requests
import re
import os
from typing import Dict, Optional
from data import Data, JobState, JsonPayload
from _logging import get_logger
from googleapiclient.discovery import build

ml = build('ml', 'v1')
GCP_PROJECT = os.environ.get('GCP_PROJECT')

def check_job_state(data: Data) -> Optional[JobState]:

    if isinstance(data.textPayload, str):
        if "completed successfully" in data.textPayload:
            return JobState.SUCCEEDED
        elif "failed" in data.textPayload:
            return JobState.FAILED
        elif "cancelled" in data.textPayload:
            return JobState.CANCELLED

    elif isinstance(data.jsonPayload, JsonPayload):
        if "queued" in data.jsonPayload.message:
            return JobState.QUEUED

    else:
        return None

def get_job_json(data):
    logger = get_logger()
    projectName = data.resource.labels.project_id
    projectId = 'projects/{}'.format(projectName)
    jobName = data.resource.labels.job_id
    jobId = '{}/jobs/{}'.format(projectId, jobName)
    request = ml.projects().jobs().get(name=jobId)
    response = None
    try:
        response = request.execute()
    except Exception as err:
        # Something went wrong. Handle the exception in an appropriate
        #  way for your application.
        raise err
    if response is None:
        raise Exception(f'Got no response for {jobId}')
    logger.info(f'Response: {json.dumps(response)}')
    return response

def get_slack_user_name(email: str) -> str:
    bot_token = os.environ.get('SLACK_BOT_TOKEN')
    response = requests.get(f'https://slack.com/api/users.lookupByEmail?email={email}', headers= {'Authorization': f'Bearer {bot_token}'})

    if response.status_code != 200:
        raise ValueError(
            'Request to slack returned an error %s, the response is:\n%s'
            % (response.status_code, response.text)
        )
    print(f'getting slack name {response.json()}')
    slack_user_name = response.json()['user']['name']
    return slack_user_name

def get_message(message: Dict) -> str:
    global GCP_PROJECT
    slack_message_template = "{slack_user_name} => JOB *{job_id}* state is: `{job_state}`{details}\n{job_link} "
    email = message.get('email')
    slack_user_name = ''
    if email:
        slack_user_name = f'@{get_slack_user_name(email)}'
    details = message.get('job_error_message')
    if details:
        details = f'\n> `{details}`\n'
    else:
        details = ''
    job_link = f'https://console.cloud.google.com/ai-platform/jobs/{message["job_id"]}/charts/cpu?project={GCP_PROJECT}'
    return slack_message_template.format(slack_user_name=slack_user_name, job_id=message['job_id'], job_state=message['job_state'], details=details, job_link=job_link)

def send_message(message: Dict) -> None:
    # Set the webhook_url to the one provided by Slack when you create the webhook at https://my.slack.com/services/new/incoming-webhook/
    webhook_url = os.environ.get('SLACK_WEBHOOK_URI')
    channel_name = os.environ.get('SLACK_CHANNEL_NAME')
    response = requests.post(
        webhook_url, data=json.dumps({
            'parse': 'full',
            'link_names': 1,
            'channel': channel_name,
            'text': get_message(message)
        }),
        headers={'Content-Type': 'application/json'}
    )
    if response.status_code != 200:
        raise ValueError(
            'Request to slack returned an error %s, the response is:\n%s'
            % (response.status_code, response.text)
        )

def extract_email_from_job(job):
    args = job['trainingInput']['args']
    for arg in args:
        result = re.search('--email(?:=| )(.+)', arg)
        if result:
            return result.group(1)
    return None

def main(event, context):
    """Triggered from a message on a Cloud Pub/Sub topic.
    Args:
         event (dict): Event payload.
         context (google.cloud.functions.Context): Metadata for the event.
    """
    logger = get_logger()
    pubsub_message = base64.b64decode(event['data']).decode('utf-8')
    logger.info(f'Event: {json.dumps(event)}')
    event_data = json.loads(pubsub_message)
    data = Data(**event_data)
    logger.info(f'data: {json.dumps(event_data)}')
    job = get_job_json(data)
    job_state = check_job_state(data)
    if not job_state:
        logger.info('Job state is none, not sending a message')
        return
    email = extract_email_from_job(job)
    message = {'email': email, 'job_id': job['jobId'], 'job_error_message': job.get('errorMessage'), 'job_state': job_state}
    send_message(message)

# main({"@type": "type.googleapis.com/google.pubsub.v1.PubsubMessage", "attributes": {"logging.googleapis.com/timestamp": "2021-05-23T23:57:10.343201317Z"}, "data": "eyJpbnNlcnRJZCI6IjEweDJhcmtjdGM1IiwibGFiZWxzIjp7Im1sLmdvb2dsZWFwaXMuY29tL2VuZHBvaW50IjoiIn0sImxvZ05hbWUiOiJwcm9qZWN0cy9zd3ZsLXNhbmRib3gvbG9ncy9tbC5nb29nbGVhcGlzLmNvbSUyRnRlc3RfZHVyYXRpb25zX3BhcmFsbGVsXzEiLCJyZWNlaXZlVGltZXN0YW1wIjoiMjAyMS0wNS0yM1QyMzo1NzoxMS4xMzE3OTIyMjFaIiwicmVzb3VyY2UiOnsibGFiZWxzIjp7ImpvYl9pZCI6InRlc3RfZHVyYXRpb25zX3BhcmFsbGVsXzEiLCJwcm9qZWN0X2lkIjoic3d2bC1zYW5kYm94IiwidGFza19uYW1lIjoic2VydmljZSJ9LCJ0eXBlIjoibWxfam9iIn0sInNldmVyaXR5IjoiSU5GTyIsInRleHRQYXlsb2FkIjoiV2FpdGluZyBmb3IgdHJhaW5pbmcgcHJvZ3JhbSB0byBzdGFydC4iLCJ0aW1lc3RhbXAiOiIyMDIxLTA1LTIzVDIzOjU3OjEwLjM0MzIwMTMxN1oifQ=="}, None)


from rebus.tools.registry import Registry
from rebus.tools.config import get_output_altering_options
from rebus.bus import DEFAULT_DOMAIN
from collections import defaultdict
import logging
import time
import json
import cPickle
log = logging.getLogger("rebus.agent")

class AgentRegistry(Registry):
    pass

class AgentLogger(logging.LoggerAdapter):
    def process(self, msg, kargs):
        return "[%s] %s" % (self.extra["agent_id"], msg), kargs

class ProcessingError(Exception):
    def __init__(self, retries=3, wait_time=30, msg=""):
        self.retries = retries
        self.wait_time = wait_time
        self.msg = msg

    def __str__(self):
        return (
            "Processing error. Retrying at most %d times for this descriptor. "
            "Waiting %d seconds between attempts.%s" %
            (self.retries, self.wait_time, "\n"+self.msg if self.msg else ""))

class Agent(object):
    _name_ = "Agent"
    _desc_ = "N/A"
    _process_slots_ = None

    #: Supported operation modes. Actual operation mode is chosen at launch;
    #: may be changed on master bus' order.
    #: default mode as 1st element.
    _operationmodes_ = ('automatic', 'interactive', 'idle')
    #: tuple of names of options that influence output values. If not
    #: overridden, every option except 'operationmode' will be considered as
    #: influencing the output.
    _output_altering_options_ = None

    @staticmethod
    def register(f):
        return AgentRegistry.register_ref(f, key="_name_")

    @staticmethod
    def parallelize(max_thread=0):
        """
        Use this decorator on an agent class to authorize concurrent runs if
        possible. Experimental, call to bus method must be performed on bus
        thread... Tread with caution.
        """
        def deco(agt):
            agt._parallelize_ = {"max_thread": max_thread}
            return agt
        return deco

    def __init__(self, bus, options, name=None, domain='default'):
        self.name = name if name else self._name_
        self.domain = domain
        self.bus = bus
        #: {key: value} containing relevant parameters that may influence the
        #: agent's outputs
        self.config = vars(options)
        # Chosen operation mode
        if len(self._operationmodes_) == 1:
            self.config['operationmode'] = self._operationmodes_[0]
        if self.config['operationmode'] == 'idle':
            self.for_idle = []
        if self._output_altering_options_ is None:
            self.config['output_altering_options'] = self.config.keys()
            self.config['output_altering_options'].remove('operationmode')
        else:
            self.config['output_altering_options'] = \
                self._output_altering_options_
        self.id = self.bus.join(self, domain)
        self.log = AgentLogger(log, dict(agent_id=self.id))
        self.log.info('Agent {0.name} registered on bus {1._name_} '
                      'with id {0.id}'.format(self, self.bus))
        self.process_slots = defaultdict(dict)
        #: List of currently held locks, for descriptors that are being
        #: processed. Used by Bus when SystemExit or KeyboardInterrupt is
        #: received. Contains tuples of unlock() arguments
        self.held_locks = []
        # Updated when starting processing
        self.processing_start_time = 0
        self.init_agent()
        self.restore_internal_state()

    def push(self, descriptor):
        if descriptor.processing_time == -1:
            descriptor.processing_time = time.time()-self.processing_start_time
        result = self.bus.push(self.id, descriptor)
        self.log.debug(
            "pushed {0}, not already present: {1}".format(descriptor,
                                                          not result))
        return result

    def get(self, desc_domain, selector):
        return self.bus.get(self.id, desc_domain, selector)

    def find(self, domain, selector_regex, limit):
        return self.bus.find(self.id, domain, selector_regex, limit)

    def list_uuids(self, desc_domain):
        return self.bus.list_uuids(self.id, desc_domain)

    def list_agents(self):
        return self.bus.list_agents(self.id)

    def processed_stats(self, desc_domain):
        return self.bus.processed_stats(self.id, desc_domain)

    def lock(self, desc_domain, selector, slots, request_id):
        """
        :param selectorstr: describes the selectors being processed
        """
        #: describes the agent & its configuration
        lockid = self.name + get_output_altering_options(self.config_txt)
        lockid += '-reqid-%d-' % request_id

        # In case of slots, lock on all the selectors at once, so that if one
        # optional selector is missing at the time of the lock, another lock
        # will be taken when this selector is received and processing the
        # complete set of slots will not be blocked.
        if self._process_slots_:
            #: describes selectors that are considered for this lock
            selectorsstr = "!".join(slots.get(s, "?") for s in
                                    self._process_slots_)
        else:
            selectorsstr = selector

        self.held_locks.append((desc_domain, selector, slots, False, 0, 0,
                                request_id))
        return self.bus.lock(self.id, lockid, desc_domain, selectorsstr)

    def unlock(self, desc_domain, selector, slots, processing_failed, retries,
               wait_time, request_id):
        lockid = self.name + get_output_altering_options(self.config_txt)
        lockid += '-reqid-%d-' % request_id
        if self._process_slots_:
            selectorsstr = "!".join(slots.get(s, "?") for s in
                                    self._process_slots_)
        else:
            selectorsstr = selector
        self.bus.unlock(self.id, lockid, desc_domain, selectorsstr,
                        processing_failed, retries, wait_time)

    def slots_are_processable(self, slots):
        """
        Test if a set of slots is ready to be processed. By default, this
        methods checks that the slot set is complete, but an agent can overload
        it to have incomplete sets to processed anyway, for instance if one
        slot is not mandatory.
        """
        return len(slots) == len(self._process_slots_)

    def on_idle(self):
        """
        on_idle is called by the bus when all descriptors have been processed
        or marked as processable, allowing agents in the 'idle' operation mode
        to only start processing when all the other agents have finished.
        it should return True if it processed descriptors, False if not.
        """
        if self.config['operationmode'] != 'idle':
            return False
        if len(self.for_idle) == 0:
            return False

        self.log.info("START on_idle bulk processing %d descriptors",
                      len(self.for_idle))
        self.processing_start_time = time.time()
        self.call_bulk_process(self.for_idle)
        self.for_idle = []
        self.log.info("END  on_idle bulk processing  |%f|",
                      time.time()-self.processing_start_time)
        self.held_locks = []
        return True

    def on_new_descriptor(self, sender_id, desc_domain, uuid, selector,
                          request_id=0):
        """
        request_id is 0 for automatic processing.
        A unique id should be used for each interactive (user-requested)
        processing.
        """
        self.log.debug("Received from %s descriptor [%s:%s] for UUID %s",
                       sender_id, desc_domain, selector, uuid)
        if self.domain != DEFAULT_DOMAIN and desc_domain != self.domain:
            # this agent only processes descriptors whose domain is self.domain
            self.bus.mark_processed(self.id, desc_domain, selector)
            return
        fres = self.selector_filter(selector)
        if not fres:
            # not interested in this
            self.bus.mark_processed(self.id, desc_domain, selector)
            return
        slots = {}
        if self._process_slots_:
            assert fres in self._process_slots_
            slots = self.process_slots[uuid]
            slots[fres] = selector
            self.log.info("Filling slot %s for %s. Filling level %i/%i." %
                          (fres, uuid, len(slots), len(self._process_slots_)))
            if not self.slots_are_processable(slots):
                self.bus.mark_processable(self.id, desc_domain, selector)
                return

        if self.config['operationmode'] == 'interactive' and not request_id:
            self.bus.mark_processable(self.id, desc_domain, selector)
            return
        elif self.config['operationmode'] == 'idle':
            self.bus.mark_processable(self.id, desc_domain, selector)
            self.for_idle.append((sender_id, desc_domain, selector, slots))
            return

        self.bus.agent_process(self, sender_id, desc_domain, selector, slots,
                               request_id)
        self.held_locks = []

    def _pre_process(self, sender_id, desc_domain, selector, slots,
                     request_id=0):
        """
        To be run before processing a descriptor. Takes care of the following:
        * acquire locks
        * fetch descriptors
        * call descriptor_filter
        * returns False if processing should not be performed, a list of
        arguments suitable for process() otherwise.
        """
        if not self.lock(desc_domain, selector, slots, request_id):
            # processing has already been started by another instance of
            # the same agent having the same configuration
            return False
        desc = self.get(desc_domain, selector)
        if desc is None:
            # that would be a bug
            self.log.warning(
                "Descriptor %s:%s sent by %s does not exist (user request: "
                "%s)", desc_domain, selector, sender_id, request_id)
            return False

        additional_descs = {k: self.get(desc_domain, s)
                            if s != selector else
                            desc for k, s in slots.iteritems()}
        if not self.descriptor_filter(desc, **additional_descs):
            return False
        # TODO detect infinite loops ?
        return (desc, sender_id, additional_descs)

    def _post_process(self, desc_domain, selector, additional_descs):
        """
        Marks required selectors as processed.
        """
        if additional_descs:
            for adesc in additional_descs.itervalues():
                self.bus.mark_processed(self.id, desc_domain, adesc.selector)
        else:
            self.bus.mark_processed(self.id, desc_domain, selector)

    def call_bulk_process(self, processlist):
        """
        :param paramlist: list of descriptors (+ associated slots) to be
        processed)
        """
        # pre-process descriptors
        descriptors = []
        senders = []
        additional_descs = []
        for args in processlist:
            res = self._pre_process(*args)
            if res:
                d, s, a = res
                descriptors.append(d)
                senders.append(s)
                additional_descs.append(a)
        # process
        self.log.info("START Bulk processing %d descriptors", len(descriptors))
        self.processing_start_time = time.time()
        try:
            self.bulk_process(descriptors, senders, additional_descs)
        except ProcessingError as e:
            # release locks
            for args in processlist:
                sender_id, desc_domain, selector, slots, request_id = args
                self.log.warning(
                    "PROCESSING_ERROR (bulk) for %s" % selector)
                self.log.exception(e)
                self.unlock(desc_domain, selector, slots, True, e.retries,
                            e.wait_time, request_id)
        except Exception as e:
            for args in processlist:
                self.log.warning(
                    "EXCEPTION while bulk processing %s, will not retry." %
                    selector)
                self.log.exception(e)
                sender_id, desc_domain, selector, slots, request_id = args
                self.unlock(desc_domain, selector, slots, True, 0, 0,
                            request_id)
        done = time.time()
        self.log.info("END   Bulk processing |%f|",
                      done-self.processing_start_time)
        # post-process - mark as processed
        for idx, desc in enumerate(descriptors):
            self._post_process(desc.domain, desc.selector,
                               additional_descs[idx])

    def call_process(self, sender_id, desc_domain, selector, slots,
                     request_id=0):

        # pre-process descriptors
        res = self._pre_process(sender_id, desc_domain, selector, slots,
                                request_id)
        if res is False:
            return

        # process
        desc, sender_id, additional_descs = res
        self.log.info("START Processing %r", desc)
        self.processing_start_time = time.time()
        try:
            self.process(desc, sender_id, **additional_descs)
        except ProcessingError as e:
            self.log.warning("PROCESSING_ERROR for %s" % selector)
            self.log.exception(e)
            self.unlock(desc_domain, selector, slots, True, e.retries,
                        e.wait_time, request_id)
            return
        except Exception as e:
            # mark as failed, do not retry
            self.log.warning(
                "EXCEPTION while processing %s, will not retry." % selector)
            self.log.exception(e)
            self.unlock(desc_domain, selector, slots, True, 0, 0, request_id)
            return
        done = time.time()
        self.log.info("END   Processing |%f| %r",
                      done-self.processing_start_time, desc)
        # post-process - mark as processed
        self._post_process(desc_domain, selector, additional_descs)

    @property
    def config_txt(self):
        return json.dumps(self.config, sort_keys=True)

    def declare_link(self, desc1, desc2, linktype, reason, isSymmetric=False):
        """
        Helper function.
        Requests two new /link/ descriptors, then pushes them.
        :param desc1: Descriptor instance
        :param desc2: Descriptor instance
        :param linktype: word describing the type of the link, that will be
          part of the selector
        :param reason: Text description of the link reason
        :param isSymmetric: indicates whether the relationship is symmetric
        """
        link1, link2 = desc1.create_links(
            desc2, self.name, linktype, reason, isSymmetric
        )
        self.push(link1)
        self.push(link2)

    def get_value(self, descriptor):
        if hasattr(descriptor, 'value'):
            return descriptor.value
        else:
            # TODO request from storage if locally available - implement when
            # agent has a reference to possibly existent local storage
            return self.bus.get_value(self.id, descriptor.domain,
                                      descriptor.selector)
            # possible trade-off: store now-fetched value in descriptor

    def get_processable(self, domain, selector):
        return self.bus.get_processable(self.id, domain, selector)

    def request_processing(self, domain, selector, targets):
        return self.bus.request_processing(self.id, domain, selector, targets)

    def save_internal_state(self):
        """
        Send internal state to storage. Called at agent shutdown, if persistent
        storage is in use.
        """
        state = self.get_internal_state()
        if state or self._process_slots_:
            complete_state = (state, self.process_slots)
            self.log.info("Save internal state %r" % (complete_state,))
            # TODO move serialization to bus
            self.bus.store_internal_state(self.id,
                                          cPickle.dumps(complete_state))

    def restore_internal_state(self):
        """
        Retrieve internal state from storage.
        """
        state_ps = self.bus.load_internal_state(self.id)
        self.log.info("Restore state: %r" % state_ps)
        if state_ps:
            # TODO move serialization to bus
            state, ps = cPickle.loads(state_ps)
            if self._process_slots_:
                self.log.info("Restore process slot state: %r" % ps)
                self.process_slots = ps
            if state:
                self.log.info("Restore internal state: %r" % state)
                self.set_internal_state(state)

    # These are the main methods that agents may overload
    def init_agent(self):
        """
        Called to initialize the agent, after it has joined the bus.

        The internal state will be restored (set_internal_state) afterwards if
        relevant.
        """
        pass

    def selector_filter(self, selector):
        return True

    def descriptor_filter(self, descriptor, **kwargs):
        return True

    def bulk_process(self, descriptors, senders, *args):
        """
        Called in idle mode. Calls process() if not overridden.
        :param descriptors: a list of descriptors
        :param senders: a list of senders
        :param args: a list of dictionaries, containing additional descriptors
        if slots are in use
        All 3 lists must have the same length.
        """
        for i in range(descriptors):
            self.process(descriptors[i], senders[i], **args[i])

    def sleep(self, t):
        """
        Should be called by the agent instead of the usual time.sleep()
        It allows REbus to do things while waiting. This function is
        really useful when waiting for a long time and using a bus
        implementation forbidding such long sleep.
        (e.g. RabbitMQ based bus)
        :param t: sleep duration (s)
        """
        self.bus.sleep(t)

    def process(self, descriptor, sender_id, **kwargs):
        pass

    def run_and_catch_exc(self):
        try:
            self.run()
        except Exception as e:
            self.log.exception(e)

    def run(self):
        """
        Overriden by agents that do not consume descriptors.

        process() and bulk_process() will not be called if run() is overridden
        """
        pass

    def get_internal_state(self):
        """
        Should be overridden by agents that have an internal state, which
        should be persistent stored when an agent is stopped.

        Return a picklable data structure that contains the internal agent
        state (ex. serialized data structures)
        """
        return

    def set_internal_state(self, state):
        """
        Should be overridden by agents that have an internal state, which
        should be persistently stored when an agent is stopped.

        :param state: data structure that contains the internal agent state
        """
        return

    def __repr__(self):
        return self.id

    @classmethod
    def add_agent_arguments(cls, subparser):
        """
        Method to add operation modes to the list of agent options. Should not
        be overridden, called by rebus_agent.
        """
        if len(cls._operationmodes_) > 1:
            subparser.add_argument('--mode', default=cls._operationmodes_[0],
                                   dest='operationmode',
                                   choices=cls._operationmodes_)
        cls.add_arguments(subparser)

    @classmethod
    def add_arguments(cls, subparser):
        """
        Overridden by agents that have configuration parameters

        Call add_argument on the received object to add options.
        """
        pass


from typing import List, Dict, Any
import os

from fxwebgen.context import Context
from fxwebgen.postprocessor import PostProcessor
from fxwebgen.templater import Templater
from fxwebgen import generator

class Generator(generator.Generator):
    distributions: List[Dict[str, Any]]
    apps: List[Dict[str, Any]]
    team: Dict[str, Any]

    def __init__(self, distributions: List[Dict[str, Any]], apps: List[Dict[str, Any]], team: Dict[str, Any],
                 output_dir: str, static_dirs: List[str], templater: Templater, pages_dir: str = None):
        ctx = Context(templater, output_dir,
                      pages_dir=pages_dir,
                      static_dirs=static_dirs,
                      interlinks=None,
                      datasets={'distributions': distributions, 'apps': apps, 'team': team})
        super().__init__(ctx, post_processor=PostProcessor())
        self.distributions = distributions
        self.apps = apps
        self.team = team
        self.repo_pages_kind = self.resources.add_kind('repo_pages')
        self.flatpakrefs_kind = self.resources.add_kind('flatpakrefs')
        maintainers = {}
        for app in apps:
            maintainer_name = app['maintainer']
            try:
                maintainer = maintainers[maintainer_name]
            except KeyError:
                maintainer = {
                    'name': maintainer_name,
                    'link': app['maintainer_link'],
                    'apps': [],
                }
                maintainers[maintainer_name] = maintainer
            maintainer['apps'].append({
                'name': app['name'],
                'link': f'/app/{app["id"]}/',
            })
        team['maintainers'] = [maintainers[key] for key in sorted(maintainers)]
        for maintainer in team['maintainers']:
            maintainer['apps'].sort(key=lambda item: item['name'])

    def before_building_pages(self) -> None:
        self.resources.remove_by_kind(self.repo_pages_kind)
        self.build_index()
        self.build_apps()
        self.build_nuvola()
        self.build_team()
        self.resources.remove_by_kind(self.flatpakrefs_kind)
        self.build_flatpak_refs()

    def build_index(self):
        self.build_index_for_distro(None, None)

        for distro in self.distributions:
            releases = distro.get("releases")
            if releases:
                for release in releases:
                    self.build_index_for_distro(distro, release)
            else:
                self.build_index_for_distro(distro, None)

    def build_index_for_distro(self, distro: Dict[str, Any] = None, release: Dict[str, Any] = None):
        print(f'Index: {distro["id"] if distro else None} {release["id"] if release else None}')
        if not distro:
            distro_spec = ""
            distro_name = None
        elif not release:
            distro_spec = "/%s" % distro["id"]
            distro_name = distro["name"] if not "releses" not in distro else None
        else:
            distro_spec = "/%s/%s" % (distro["id"], release["id"])
            if release["name"].startswith(distro["name"]):
                distro_name = release["name"]
            else:
                distro_name = "%s %s" % (distro["name"], release["name"])

        canonical_path = "/index/"
        target = os.path.join(
            self.ctx.output_dir,
            "index%s/index.html" % distro_spec if distro_spec else "index/index.html")
        os.makedirs(os.path.dirname(target), exist_ok=True)
        self.resources.add(self.repo_pages_kind, None, target)
        with open(target, "wt") as f:
            f.write(self.ctx.templater.render("index.html", {
                'navbar': 'install',
                "distributions": self.distributions,
                "distro_name": distro_name,
                "apps": self.apps,
                "distro_spec": distro_spec,
                "canonical_path": canonical_path
            }))

    def build_apps(self):
        self.build_apps_for_distro(None, None)
        for distro in self.distributions:
            self.build_apps_for_distro(distro, None)
            for release in distro.get("releases", []):
                self.build_apps_for_distro(distro, release)

    def build_apps_for_distro(self, distro: Dict[str, Any] = None, release: Dict[str, Any] = None):
        for app in self.apps:
            self.build_app_for_distro(app, distro, release)

    def build_app_for_distro(self, app, distro: Dict[str, Any] = None, release: Dict[str, Any] = None):
        print(f'App: {app["id"]} {distro["id"] if distro else None} {release["id"] if release else None}')
        templates = ["app.html"]
        if not distro:
            target = ""
            distro = self.distributions[0]
            releases = distro.get("releases")
        else:
            releases = distro.get("releases")
            if not release:
                if releases:
                    release = releases[0]
                target = "/%s" % distro["id"]
            else:
                target = "/%s/%s" % (distro["id"], release["id"])

        if not release and releases:
            release = releases[0]

        templates.insert(0, "app_%s.html" % distro["id"])
        if not release:
            distro_spec = "/%s" % distro["id"]
        else:
            distro_spec = "/%s/%s" % (distro["id"], release["id"])
            templates.insert(0, "app_%s_%s.html" % (distro["id"], release["id"]))

        canonical_path = "/app/%s%s/" % (app["id"], distro_spec)
        target = os.path.join(self.ctx.output_dir, "app/%s%s/index.html" % (app["id"], target))
        os.makedirs(os.path.dirname(target), exist_ok=True)
        self.resources.add(self.repo_pages_kind, None, target)
        with open(target, "wt") as f:
            f.write(self.ctx.templater.render(templates, {
                'navbar': 'install',
                "tab_target": "/app/" + app["id"],
                "apps": self.apps,
                "app": app,
                "distributions": self.distributions,
                "distro": distro,
                "releases": distro.get("releases"),
                "release": release,
                "distro_spec": distro_spec,
                "canonical_path": canonical_path
            }))

    def build_nuvola(self):
        self.build_nuvola_for_distro(None, None)
        for distro in self.distributions:
            self.build_nuvola_for_distro(distro, None)
            for release in distro.get("releases", []):
                self.build_nuvola_for_distro(distro, release)

    def build_nuvola_for_distro(self, distro: Dict[str, Any] = None, release: Dict[str, Any] = None):
        print(f'Nuvola: {distro["id"] if distro else None} {release["id"] if release else None}')
        if not distro:
            target = ""
            distro = self.distributions[0]
            releases = distro.get("releases")
        else:
            releases = distro.get("releases")
            if not release:
                if releases:
                    release = releases[0]
                target = "/%s" % distro["id"]
            else:
                target = "/%s/%s" % (distro["id"], release["id"])

        if not release and releases:
            release = releases[0]

        templates = []
        if not release:
            distro_spec = "/%s" % distro["id"]
        else:
            distro_spec = "/%s/%s" % (distro["id"], release["id"])
            templates.append("nuvola_%s_%s.html" % (distro["id"], release["id"]))
        templates.append("nuvola_%s.html" % distro["id"])
        templates.append("nuvola.html")

        canonical_path = "/nuvola%s/" % distro_spec
        target = os.path.join(self.ctx.output_dir, "nuvola%s/index.html" % target)
        self.resources.add(self.repo_pages_kind, None, target)
        os.makedirs(os.path.dirname(target), exist_ok=True)
        with open(target, "wt") as f:
            f.write(self.ctx.templater.render(templates, {
                'navbar': 'install',
                "tab_target": "/nuvola",
                "distributions": self.distributions,
                "apps": self.apps,
                "distro": distro,
                "releases": releases,
                "release": release,
                "distro_spec": distro_spec,
                "canonical_path": canonical_path
            }))

    def build_flatpak_refs(self):
        self.build_flatpak_ref(self.ctx.templater.env.globals["nuvola"]["uid"], "nuvola")
        for app in self.apps:
            self.build_flatpak_ref(app["uid"], "app", app=app)

    def build_flatpak_ref(self, uid, template, **data):
        print(f'Flatpakref: {uid}')
        target = os.path.join(self.ctx.output_dir, "%s.flatpakref" % uid)
        self.resources.add(self.flatpakrefs_kind, None, target)
        with open(target, "wt") as f:
            f.write(self.ctx.templater.render(template + ".flatpakref", data))

    def build_team(self):
        canonical_path = "/team/"
        target = os.path.join(self.ctx.output_dir, "team/index.html")
        os.makedirs(os.path.dirname(target), exist_ok=True)
        self.resources.add(self.repo_pages_kind, None, target)
        with open(target, "wt") as f:
            f.write(self.ctx.templater.render(['team.html'], {
                'navbar': 'team',
                "team": self.team,
                "canonical_path": canonical_path
            }))


#!/usr/bin/env python3
"""Day 12: JSAbacusFramework.io

Santa's Accounting-Elves need help balancing the books after a recent order.
Unfortunately, their accounting software uses a peculiar storage format.

They have a JSON document which contains a variety of things:
arrays (`[1,2,3]`), objects (`{"a":1, "b":2}`), numbers, and strings.
Your first job is to simply find all of the numbers throughout the document
and add them together.

Part 2:
The Accounting-Elves have realized that they double-counted everything red.
Ignore any object (and all of its children) which has any property
with the value "red".

"""
import json

def all_nums(data, ignore_reds=False):
    """Get all the integer values from `data`.

    If `ignore_reds` is set, any dicts containing the value 'red' are ignored.

    Args:
      data (loaded json structure)
      ignore_reds (bool)

    Yields:
      int: Integer values inside `data`.

    """
    if isinstance(data, int):
        yield data

    elif isinstance(data, list):
        for value in data:
            yield from all_nums(value, ignore_reds)

    elif isinstance(data, dict):
        if ignore_reds and 'red' in data.values():
            return
        for value in data.values():
                yield from all_nums(value, ignore_reds)

def main():
    import sys

    with open(sys.argv[1]) as f:
        data = json.load(f)

    print(sum(all_nums(data)))
    print(sum(all_nums(data, ignore_reds=True)))

if __name__ == '__main__':
    main()


import gym
import time
import numpy as np
import argparse

from accord.amp.evaluatorsingle import evaluator

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Evaluate Single Amp model')
    parser.add_argument('-env', dest='env_str', help="Environment string.")
    parser.add_argument('-id',
                        dest='wid',
                        type=int,
                        help="Designated model id.")
    parser.add_argument('-start',
                        dest='start',
                        type=int,
                        help="Eval start in 100K, e.g., start = 10 for 1M")
    parser.add_argument('-end',
                        dest='end',
                        type=int,
                        help="Eval end in 100K, e.g., end = 10 for 1M")
    args = parser.parse_args()

    env_str = args.env_str
    evaluator(wid=args.wid, env_str=env_str, start=args.start, end=args.end)


#!/usr/bin/env python3

"""View files or directories with the same command."""

import argparse
import os
import subprocess
import sys

def l(path='.', dcmd='ls', fcmd='less'):  # pylint: disable=invalid-name
    """
    Run a command based on the type of target.

    :param path: Specify a target.
    :type path: str

    :param dcmd: Specify a command to run in case the target is a directory.
    :type dcmd: str

    :param fcmd: Specify a command to run in case the target is a file.
    :type fcmd: str

    :returns: the exit code of the command, or a non-zero integer
    :rtype: int

    """
    if os.path.isdir(path):
        return subprocess.call(dcmd.split()+[path])
    if os.path.isfile(path):
        return subprocess.call(fcmd.split()+[path])
    real = os.path.realpath(path)
    print(
        'Special files are not supported.'
        if os.path.exists(real) else
        '"{0}"{1} does not exist.'.format(
            path,
            '' if path == real else ' ({0})'.format(real)
        ),
        file=sys.stderr
    )
    return 1

def ll(paths):  # pylint: disable=invalid-name
    """
    Preview files or directories with less or ls -h -l, repectively.

    :param paths: Specify paths to preview.
    :type paths: list

    :returns: the sum of the exit codes of each command
    :rtype: int

    """
    dcmd = 'ls -h -l'
    if sys.platform.startswith('linux'):
        dcmd += ' --color=always'
    elif sys.platform.startswith('darwin'):
        dcmd += ' -G'
    status = 0
    for path in paths:
        status += l(path=path, dcmd=dcmd)
    return status

def main():
    """Parse the CLI and execute."""
    parser = argparse.ArgumentParser(add_help=False)
    parser.add_argument('paths', metavar='path', nargs='*', default=['.'])
    return ll(paths=parser.parse_args().paths)

if __name__ == '__main__':
    sys.exit(main())


import flask
import qrcode
import io

app = flask.Flask(__name__)

@app.route("/")
def home():
    # # 第一步：获取要生成二维码的数据
    # data = flask.request.args.get("data")

    # # 第二步：生成二维码图片
    # img = qrcode.make(data)
    # img.save(r"E:\纺织大学Python实训\day6\代码\qrcode_tool_online\static\qr.png")

    # # 第三步：在页面上显示二维码图片
    # return '<img src="/static/qr.png" />'

    return flask.render_template('qr_tool.html')

@app.route("/qr")
def qr():
    # 第一步：获取要生成二维码的数据
    #data = flask.request.form.get("data")
    data =flask.request.args.get("data")

    # 第二步：生成二维码图片
    img = qrcode.make(data)
    bi = io.BytesIO() # 创建一个BytesIO对象，用于在内存中存储二维码图像数据
    img.save(bi,"png") # 调用img对象的save方法将二维码图像数据以PNG编码格式写入bi对象管理的内存空间
    bi.seek(0) # 将bi对象内部的位置指针移到图像数据的起始位置

    # 第三步：返回二维码图像数据
    return flask.send_file(bi,"image/png")

if __name__ == '__main__':
    app.run(debug=True,)


# coding=utf-8
# Copyright 2018-2020 EVA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import unittest

import cv2
import numpy as np

from src.catalog.models.df_metadata import DataFrameMetadata
from src.loaders.video_loader import VideoLoader
from test.util import custom_list_of_dicts_equal

NUM_FRAMES = 10

class VideoLoaderTest(unittest.TestCase):

    def create_dummy_frames(self, num_frames=NUM_FRAMES, filters=[]):
        if not filters:
            filters = range(num_frames)
        for i in filters:
            yield {'id': i,
                   'frame_data': np.array(np.ones((2, 2, 3)) * 0.1 * float(i + 1) * 255,
                                          dtype=np.uint8)}

    def create_sample_video(self):
        try:
            os.remove('dummy.avi')
        except FileNotFoundError:
            pass

        out = cv2.VideoWriter('dummy.avi',
                              cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 10,
                              (2, 2))
        for i in range(NUM_FRAMES):
            frame = np.array(np.ones((2, 2, 3)) * 0.1 * float(i + 1) * 255,
                             dtype=np.uint8)
            out.write(frame)

    def setUp(self):
        self.create_sample_video()

    def tearDown(self):
        os.remove('dummy.avi')

    def test_should_return_batches_equivalent_to_number_of_frames(self):
        video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
        video_loader = VideoLoader(video_info)
        batches = list(video_loader.load())
        dummy_frames = list(self.create_dummy_frames())
        self.assertEqual(len(batches), NUM_FRAMES)
        expected = [batch.frames.to_dict('records')[0] for batch in batches]
        self.assertTrue(custom_list_of_dicts_equal(dummy_frames, expected))

    def test_should_return_half_then_number_of_batches_with_skip_of_two(self):
        video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
        video_loader = VideoLoader(video_info, skip_frames=2)
        batches = list(video_loader.load())
        dummy_frames = list(
            self.create_dummy_frames(
                filters=[i * 2 for i in range(NUM_FRAMES // 2)]))
        self.assertEqual(len(batches), NUM_FRAMES / 2)
        expected = [batch.frames.to_dict('records')[0] for batch in batches]
        self.assertTrue(custom_list_of_dicts_equal(dummy_frames, expected))

    def test_should_skip_first_two_frames_with_offset_two(self):
        video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
        video_loader = VideoLoader(video_info, offset=2)
        dummy_frames = list(
            self.create_dummy_frames(
                filters=[i for i in range(2, NUM_FRAMES)]))
        batches = list(video_loader.load())
        self.assertEqual(NUM_FRAMES - 2, len(batches))
        expected = [batch.frames.to_dict('records')[0] for batch in batches]
        self.assertTrue(custom_list_of_dicts_equal(dummy_frames, expected))

    def test_should_return_only_few_frames_when_limit_is_specified(self):
        video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
        limit = 4
        video_loader = VideoLoader(video_info, limit=limit)
        dummy_frames = list(
            self.create_dummy_frames(filters=[i for i in range(limit)]))
        batches = list(video_loader.load())
        self.assertEqual(limit, len(batches))
        expected = [batch.frames.to_dict('records')[0] for batch in batches]
        self.assertTrue(custom_list_of_dicts_equal(dummy_frames, expected))

    def test_should_return_single_batch_if_batch_size_equal_to_no_of_frames(
            self):
        video_info = DataFrameMetadata("dataset_1", 'dummy.avi')
        video_loader = VideoLoader(video_info, batch_size=NUM_FRAMES)
        dummy_frames = list(
            self.create_dummy_frames(filters=[i for i in range(NUM_FRAMES)]))
        batches = list(video_loader.load())
        self.assertEqual(1, len(batches))
        expected = batches[0].frames.to_dict('records')
        self.assertTrue(custom_list_of_dicts_equal(dummy_frames, expected))