# -*- coding: utf-8 -*-
"""
Created on Wed Apr 20 14:55:35 2016

@author: M
"""

# -*- coding: utf-8 -*-
"""
Created on Wed Mar 30 08:34:54 2016

@author: John
"""
import csv
from gurobipy import *

H = {}
myFile = open('Opponents Season 2016.csv','rt')
myReader = csv.reader(myFile)
for row in myReader:
    if row[0] != 'Away Team':

        if row[1] in H:
            H[row[1]].append(row[0])
        else:
            H[row[1]] = [row[0]]

myFile.close()

A = {}
myFile = open('Opponents Season 2016.csv','rt')
myReader = csv.reader(myFile)
for row in myReader:
    if row[0] != 'Away Team':

        if row[0] in A:
            A[row[0]].append(row[1])
        else:
            A[row[0]] = [row[1]]

myFile.close()

S = {}
myFile= open('Slots.csv')
myReader = csv.reader(myFile)
for row in myReader:
    for cell in row:
        if len(cell)!=0 and cell!=row[0]:
            if int(row[0]) in S:
                S[int(row[0])].append(cell)
            else:
                S[int(row[0])]=[cell]

del(row,cell)
myFile.close()

T = ['DAL', 'NYG', 'PHI','WAS','CHI', 'DET', 'GB','MIN','ATL','CAR','NO','TB',
     'ARZ','LAR','SF','SEA','BUF','MIA','NE','NYJ','BAL','CIN','CLE','PIT',
     'HOU','IND','JAC','TEN','DEN','KC','OAK','SD']

DIVISION = {'NFC':{'EAST':['DAL', 'NYG', 'PHI','WAS'],
                   'NORTH':['CHI', 'DET', 'GB','MIN'],
                   'SOUTH':['ATL', 'CAR','NO','TB'],
                   'WEST':['ARZ','LAR','SF','SEA']
                   },
            'AFC':{'EAST': ['BUF','MIA','NE','NYJ'],
                   'NORTH': ['BAL','CIN','CLE','PIT'],
                   'SOUTH': ['HOU','IND','JAC','TEN'],
                   'WEST': ['DEN','KC','OAK','SD']
                   }
            }
CONFERENCE = {'NFC':['DAL', 'NYG', 'PHI','WAS','CHI', 'DET', 'GB','MIN','ATL','CAR','NO','TB','ARZ','LAR','SF','SEA'],
              'AFC':['BUF','MIA','NE','NYJ','BAL','CIN','CLE','PIT','HOU','IND','JAC','TEN','DEN','KC','OAK','SD']
            }

myModel = Model()
myModel.update()

##OBJECTIVE FUNCTION##
myGames = {}
for h in T:
    for a in H[h]:
        for w in range(1,18):
            for s in S[w]:
                myGames[a,h,s,w] = myModel.addVar(obj =1, vtype=GRB.BINARY,
                                    name='games_%s_%s_%s_%s' % (a,h,s,w))

for h in T:
    for w in range(4,13):
        myGames['BYE',h,'SUNB_NFL',w] = myModel.addVar(obj =1, vtype=GRB.BINARY,
                                        name='games_BYE_%s_%s_%s' % (h,s,w))

myModel.update()

myModel.setObjective(quicksum(myGames[a,h,s,w,] for w in range(1,18) for h in T for a in H[h] for s in S[w]),GRB.MINIMIZE)
########################################################################################################################

## CONSTRAINTS
myConstr = {}

#constraint 1: every game played once
for h in T: #iterate over all 32 teams (a,h)
    for a in H[h]: #each away team
        constrName = '1_game_once_%s_%s' % (a,h)
        myConstr[constrName] = myModel.addConstr(quicksum(myGames[a,h,s,w]
                                        for w in range (1,18) for s in S[w]) == 1,
                                        name = constrName)
myModel.update()

#constraint 2: teams play one game each week (takes care  of everything but bye games)
for t in T:
    for w in [1,2,3,12,13,14,15,16,17]:
        constrName = '1_in_w%s_by_%s' % (w,t)
        myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,t,s,w] for a in H[t] for s in S[w])
                            + quicksum(myGames[t,h,s,w] for h in A[t] for s in S[w]) == 1,
                            name=constrName)
myModel.update()

#constraint 3: teams play one game each week (takes care of bye games)
for t in T:
    for w in range(4,12):
        constrName = '1_bye_in_w%s_by_%s' %(w,t)
        myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,t,s,w] for a in H[t] for s in S[w] if s != 'SUNB_NFL')
        + quicksum(myGames[t,h,s,w] for h in A[t] for s in S[w] if s != 'SUNB_NFL')
        + myGames['BYE',t, 'SUNB_NFL', w] == 1, name=constrName)
myModel.update()

#constraint 4:No more than 6 bye games in a given a week
for w in range(4,12):
    constrName = '4_Bye_game_by_%s_in_w%s' % (t,w)
    myConstr[constrName]=myModel.addConstr(quicksum(myGames['BYE',t,'SUNB_NFL',w] for t in T)<=6, name=constrName)

w = None
myModel.update()

#contraint 5: No team that had an early bye game (week 4) in 2015 can have an early bye game (week 4) in 2016
constrName = 'TEN,NE do not play in week 4'
myConstr[constrName]=myModel.addConstr(myGames['BYE','NE','SUNB_NFL',4] + myGames['BYE','TEN','SUNB_NFL',4]==0,
name=constrName)
myModel.update()

#constraint 6: Exactly 1 Thursday game every week upto week 16, week 17 has no thursday games
Thursday = ['THUN_NBC' , 'THUN_NFL' , 'THUN_NFL' , 'THUE_FOX' ,'THUL_CBS', 'THUN_NBC' , 'THUN_CBS'  ]
for w in range(1,17):
    for s in S[w]:
        if s in Thursday:
            constrName='6_one_Thursady_in_w%s' %(w)
            myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w] for h in T for a in H[h] ) ==1 ,name=constrName)
myModel.update()

#constraint 7:There are two Saturday Night Games in Week 15 (one SatE and one SatL)
for s in ['SATE_NFL','SATL_NFL']:
    constrName='2_Games_on_Sat_s%s' %(s)
    myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,15] for h in T for a in H[h]) == 1,name=constrName)
myModel.update()

#constraint 8: There is only one “double header” game in weeks 1 through 16 (and two in week 17)
#Week 1:16
for w in range(1,17):
    for s in ['SUNDH_CBS','SUNDH_FOX']:
        constrName= '1_DH_in_w%s%s' %(w,s)
        myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w] for h in T for a in H[h]) == 1, name=constrName)
myModel.update()

#Week 17
constrName='2_DH_in_17'
myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,17] for h in T for a in H[h] for s in ['SUNDH_CBS','SUNDH_FOX']) == 2,name=constrName)
myModel.update()

#constraint 9: There is exactly one Sunday Night Game in weeks 1 through 16 (no Sunday Night Game in week 17)
#Week 1:16
for w in range(1,17):
    constrName='1_SundayNight_in_w%s' %w
    myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,'SUNN_NBC',w] for h in T for a in H[h]) == 1,name=constrName)
myModel.update()

#constraint 10 Part 1: There are two Monday night games in week 1
WC=['SD', 'SF', 'SEA', 'OAK', 'LAR']

constrName='2_Mondays_in_w1'
myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,1]for h in T for a in H[h] for s in ['MON1_ESPN','MON2_ESPN']) == 2,name=constrName)
myModel.update()

#constraint 10 Part 2: The late Monday Night Game must be hosted by a West Coast Team (SD, SF, SEA, OAK, LAR)
 #List of west coast teams

for w in range(1,3):
    if s in S[w]:
        constrName='%s_slot_w%s' %(s,w)
        myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,'MON2_ESPN',w] for h in WC for a in H[h]) == 1, name=constrName)
myModel.update()

#constraint 10 Part 3: There in exactly one Monday night game in weeks 2 through 16 (no Sunday Night Game in week 17))
# Week 2:16
for w in range(2,17):
    constrName='1_Monday_in_w%s' %(w)
    myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,'MON1_ESPN',w] for h in T for a in H[h]) ==1 , name=constrName)
myModel.update()

#constraint 11: West Coast (SD, SF, SEA, OAK, LAR) and Mountain Teams (DEN, ARZ) cannot play at home in the early Sunday time slot
MT=['DEN','ARZ']
WCMT=WC+MT

for w in range(1,18):
    constrName='WstCst_MtTm_cannot_SUNE_w%s' %(w)
    myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w] for h in WCMT for a in H[h] for s in ['SUNE_CBS','SUNE_FOX'])==0,name=constrName)
myModel.update()

#constraint 12_Home_Games: No team plays 4 consecutive home/away games in a season (treat a BYE game as an away game)
for w in range(1,15):
    for h in H:
       constrName ='no_more_than_4_consecutive_games_in_w%s_at_h%s' %(w,h)
       myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w1] for a in H[h] for w1 in range(w, w+4) for s in S[w1]) <= 3, name = constrName)
myModel.update()

#constraint 12_Away_Games:
for w in range(1,15):
    for a in A:
       constrName ='no_more_than_4_consecutive_games_w%s_at_a%s' %(w,a)
       myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w1] for h in A[a] for w1 in range(w, w+4) for s in S[w1]) +
                                              quicksum(myGames[a,'BYE',s,w1] for w1 in range(w, w+4) for s in S[w1] if (a,'BYE',s,w1) in myGames) <= 3, name = constrName)
myModel.update()

#constraint 13a_Home_Games: No team plays 3 consecutive home/away games during the weeks 1, 2, 3, 4, 5 and 15, 16, 17 (treat a BYE game as an away game)
for w in range(1,4):
    for h in H:
        constrName ='no_more_than_4_consecutive_games_1_5_w%s_at_h%s' %(w,h)
        myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w1] for a in H[h] for w1 in range(w, w+3) for s in S[w1]) <= 2, name = constrName)
myModel.update()

#Constraint 13a_Away_Games:
for w in range(1,4):
    for a in A:
        constrName ='no_more_than_4_consecutive_games_1_5_w%s_at_a%s' %(w,a)
        myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w1] for h in A[a] for w1 in range(w, w+3) for s in S[w1]) +
                                               quicksum(myGames[a,'BYE',s,w1] for w1 in range(w, w+3) for s in S[w1] if (a,'BYE',s,w1) in myGames)<= 2, name = constrName)
myModel.update()

#constraint 13b_Home_Games:
for h in H:
    constrName ='no_more_than_4_consecutive_games_15_18_w15_%s' %h
    myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w] for a in H[h] for w in range(15, 18) for s in S[w]) <= 2, name = constrName)
    #need to add bye games
myModel.update()

#constraint 13b_Away_Games:
for a in A:
    constrName ='no_more_than_4_consecutive_games_15_18_w15_%s' %a
    myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w] for h in A[a] for w in range(15, 18) for s in S[w]) +
                                           quicksum(myGames[a,'BYE',s,w] for w in range(15, 18) for s in S[w] if (a,'BYE',s,w) in myGames)<= 2, name = constrName)
myModel.update()

#constraint 14_Home_Games: Each team must play at least 2 home/away games every 6 weeks
for w in range(1,13):
    for h in H:
        constrName ='at_least_2games_per6weeks_w%s_at_h%s' %(w,h)
        myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w1] for a in H[h] for w1 in range(w, w+6) for s in S[w1]) >= 2, name = constrName)
myModel.update()

#constraint 14_Away_Games
for w in range(1,13):
    for a in A:
        constrName ='at_least_2games_per6weeks_w%s_at_a%s' %(w,a)
        myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w1] for h in A[a] for w1 in range(w, w+6) for s in S[w1]) +
                                               quicksum(myGames[a,'BYE',s,w1] for w1 in range(w, w+6) for s in S[w1] if (a,'BYE',s,w1) in myGames)>= 2, name = constrName)
myModel.update()

#constraint 15: Each team must play at least 4 home/away games every 10 weeks
for w in range(1,8): #adding 10 weeks goes beyond week 17
    for h in H:
        constrName ='at_least_4home_every10weeks_w%s_h%s' %(w,h)
        myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w1] for a in H[h] for w1 in range(w, w+11) for s in S[w1]) >= 4, name = constrName)
myModel.update()

for w in range(1,8):
    for a in A:
        constrName ='at_least_4away_every10weeks_w%s_h%s' %(w,a)
        myConstr[constrName]=myModel.addConstr(quicksum(myGames[a,h,s,w1] for h in A[a] for w1 in range(w, w+11) for s in S[w1]) +
                                               quicksum(myGames[a,'BYE',s,w1] for w1 in range(w, w+11) for s in S[w1] if (a,'BYE',s,w1) in myGames) >= 4, name = constrName)
myModel.update()

myModel.update()
myModel.optimize()


# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, x):
#         self.val = x
#         self.next = None

def print_list(node):
    if node is None:
        print("NULL")
    else:
        print(node.val, end="->")
        print_list(node.next)

class Solution:
    def addTwoNumbers(self, l1: ListNode, l2: ListNode) -> ListNode:
        curr1 = l1
        curr2 = l2
        carry = False
        temp = ListNode(0)
        temp.next = l1
        while curr1 is not None and curr2 is not None:
            curr1.val += curr2.val
            if carry:
                curr1.val += 1
            carry = curr1.val >= 10
            curr1.val %= 10
            temp = curr1
            curr1 = curr1.next
            curr2 = curr2.next

        if curr1 is None and curr2 is None:
            if carry:
                temp.next = ListNode(0)
        elif curr1 is None:
            temp.next = curr2

        print_list(l1)
        while carry:
            temp = temp.next
            temp.val += 1
            carry = temp.val >= 10
            if temp.next is None and carry:
                temp.next = ListNode(0)
            temp.val %= 10

        return l1


__author__ = 'Irwan Fathurrahman <irwan@kartoza.com>'
__date__ = '17/05/17'

import json
import os
import shutil
from campaign_manager.insights_functions._abstract_insights_function import (
    AbstractInsightsFunction
)
from campaign_manager.data_providers.shapefile_provider import (
    ShapefileProvider
)

class UploadCoverage(AbstractInsightsFunction):
    function_name = "Upload shapefile for creating coverage"
    manager_only = True

    def initiate(self, additional_data):
        """ Initiate function

        :param additional_data: additional data that needed
        :type additional_data:dict
        """
        if self.campaign:
            self.run()

    def get_ui_html_file(self):
        """ Get ui name in templates
        :return: string name of html
        :rtype: str
        """
        return "upload_coverage"

    def get_summary_html_file(self):
        """ Get summary name in templates
        :return: string name of html
        :rtype: str
        """
        return ""

    def get_details_html_file(self):
        """ Get summary name in templates
        :return: string name of html
        :rtype: str
        """
        return ""

    def get_data_from_provider(self):
        """ Get data provider function
        :return: data from provider
        :rtype: dict
        """
        coverage_folder = self.campaign.get_coverage_folder()
        shapefile_file = "%s/%s.shp" % (
            coverage_folder, self.campaign.uuid
        )
        return ShapefileProvider().get_data(
            shapefile_file
        )

    def process_data(self, raw_datas):
        """ Get geometry of campaign.
        :param raw_datas: Raw data that returns by function provider
        :type raw_datas: dict

        :return: processed data
        :rtype: dict
        """
        return raw_datas

    def delete_coverage_files(self):
        """Delete coverage files"""
        coverage_folder = self.campaign.get_coverage_folder()
        if os.path.exists(coverage_folder):
            shutil.rmtree(coverage_folder)

    def get_coverage_files(self):
        """ Get coverage files
        :return: coverage in geojson
        :rtype: []
        """
        coverage_folder = self.campaign.get_coverage_folder()
        output_files = []
        for root, dirs, files in os.walk(coverage_folder):
            for file in files:
                output_files.append(file)
        return output_files

    def post_process_data(self, data):
        """ Process data regarding output.
        This needed for processing data for counting or grouping.

        :param data: Data that received from open street map
        :type data: dict

        :return: Processed data
        :rtype: dict
        """

        output = {
            'uuid': self.campaign.uuid,
            'files': self.get_coverage_files(),
        }
        if self.campaign.coverage:
            output['coverage'] = {
                'last_uploader': self.campaign.coverage['last_uploader'],
                'last_uploaded': self.campaign.coverage['last_uploaded'],
                'geojson': json.dumps(data)
            }
        return output


import sys
sys.path.append('../../')

from npcnn import read_onnx
from skimage import io
from matplotlib import pyplot as plt
import numpy as np
from time import time

img = io.imread('test.jpg')
img = img.astype(np.float32)
x = img[None, None, :, :]/255

net = read_onnx('unet')

y = net(x)
start = time()
for i in range(10):
    y = net(x)
print('unet npcnn time:', time()-start)

plt.subplot(121)
plt.title('gray')
plt.imshow(img, 'gray')
plt.subplot(122)
plt.title('segment')
plt.imshow(y[0, 0, :, :])
plt.show()


import sys,os,math,time
import json,re
import contextlib
import webbrowser
import time,threading

# from subprocess import Popen, PIPE, STDOUT, check_output
from subprocess import Popen, PIPE, STDOUT, check_output, DEVNULL
from glob import glob
from shutil import copytree

import tkinter as tk
from tkinter import Toplevel
from tkinter import Tk, END, StringVar, LEFT, TOP, BOTTOM, X, BOTH, YES, INSERT, W, E
from tkinter import Text, Entry, OptionMenu

from tkinter import ttk
from tkinter.ttk import Label, Button, LabelFrame, Scrollbar, Frame, Notebook, Style

from tkinter.filedialog import askopenfilename, asksaveasfilename, askdirectory
from tkinter.messagebox import showerror, showinfo, showwarning

from pathlib import Path
from os.path import join, exists
from os import makedirs, listdir

from gui.frame import PipelineFrame
from gui.utils import ftypes, filetypes, filetype, efiletype, batchsize, customRules
from gui.utils import USER_HOME, PIPELINER_HOME, PIPELINER_CONF

class scRNASeqFrame( PipelineFrame ) :
    def __init__(self, pipepanel, pipeline_name, *args, **kwargs) :
        PipelineFrame.__init__(self, pipepanel, pipeline_name, *args, **kwargs)
        self.info = None

        eframe = self.eframe = LabelFrame(self,text="Options")
        #,fg=textLightColor,bg=baseColor)
        eframe.grid( row=5, column=1, sticky=W, columnspan=7, padx=10, pady=5 )

        label = Label(eframe,text="Pipeline")#,fg=textLightColor,bg=baseColor)
        label.grid(row=3,column=0,sticky=W,padx=10,pady=5)
        PipelineLabels=["CellRanger","Initial/QC","Clustering","Multi-Sample Clustering" ]
        Pipelines=["cellranger","scrnaseqinit","scrnaseqcluster", "scrnaseqmulticluster"]

        self.label2pipeline = { k:v for k,v in zip(PipelineLabels, Pipelines)}

        PipelineLabel = self.PipelineLabel = StringVar()
        self.Pipeline = StringVar()

        PipelineLabel.set(PipelineLabels[0])
        om = OptionMenu(eframe, PipelineLabel, *PipelineLabels, command=self.option_controller)
        om.config()#bg = widgetBgColor,fg=widgetFgColor)
        om["menu"].config()#bg = widgetBgColor,fg=widgetFgColor)
        #om.pack(side=LEFT,padx=20,pady=5)
        om.grid(row=3,column=1,sticky=W,padx=10,pady=5)

        self.crOpts = crOpts = LabelFrame( eframe,
                                              text="CellRanger Settings" )
        self.scrCRID = scrCRID = StringVar()
        scrCRID.set("SPECIFY_PREFIX_HERE")
        self.scrExpected = scrExpected = StringVar()
        scrExpected.set("3000")

        scrcridL = Label(crOpts, text="CellRanger Sample ID: ")
        scrcridE = Entry(crOpts, bd =2, width=25, textvariable=scrCRID)
        scrcridL.grid(row=9,column=1,sticky=W,padx=10,pady=5)
        scrcridE.grid(row=9,column=2,sticky=W,padx=0,pady=5)

        screxpectedL = Label(crOpts, text="Expected number of cells: ")
        screxpectedE = Entry(crOpts, bd =2, width=8, textvariable=scrExpected)
        screxpectedL.grid(row=10,column=1,sticky=W,padx=10,pady=5)
        screxpectedE.grid(row=10,column=2,sticky=W,padx=0,pady=5)

        self.clusterOpts = clusterOpts = LabelFrame( eframe,
                                              text="Clustering and tSNE Options" )

        self.scrPCs = scrPCs = StringVar()
        scrPCs.set("12")
        self.scrRes = scrRes = StringVar()
        scrRes.set("0.6")

        #scrPCs.trace('w', lambda a,b,c,x="scrPCs": makejson(x))

        #Filter out genes < [5] read counts in < [2] samples
        scrpcsL = Label(clusterOpts, text="Include principal components 1 through ")
        scrpcsE = Entry(clusterOpts, bd =2, width=3, textvariable=scrPCs)
        scrresL = Label(clusterOpts, text="with clustering resolution: ")
        scrresE = Entry(clusterOpts, bd =2, width=3, textvariable=scrRes)

        scrpcsL.grid(row=9,column=1,sticky=W,padx=10,pady=5)
        scrpcsE.grid(row=9,column=2,sticky=W,padx=0,pady=5)
        scrresL.grid(row=9,column=3,sticky=W,padx=5,pady=5)
        scrresE.grid(row=9,column=4,sticky=W,padx=0,pady=5)
        #scrRes.trace('w', lambda a,b,c,x="scrPCs": makejson(x))

        clusterOpts.grid( row=8, column=0, columnspan=4, sticky=W, padx=20, pady=10 )

        self.multiclusterOpts = multiclusterOpts = LabelFrame( eframe,
                                                text = "Multi-Sample Clustering and tSNE Options")

        scrccsL = Label(multiclusterOpts, text="Include canonical components 1 through ")
        scrccsE = Entry(multiclusterOpts, bd =2, width=3, textvariable=scrPCs)
        scrmcresL = Label(multiclusterOpts, text="with clustering resolution: ")
        scrmcresE = Entry(multiclusterOpts, bd =2, width=3, textvariable=scrRes)

        scrccsL.grid(row=9,column=1,sticky=W,padx=10,pady=5)
        scrccsE.grid(row=9,column=2,sticky=W,padx=0,pady=5)
        scrmcresL.grid(row=9,column=3,sticky=W,padx=5,pady=5)
        scrmcresE.grid(row=9,column=4,sticky=W,padx=0,pady=5)

        self.qcOpts = qcOpts = LabelFrame( eframe,
                                              text="Initial Settings" )
        countL = Label( qcOpts, text="Counts/Matrix Dir:" )
        countL.grid(row=9, column=1, sticky=W, padx=10, pady=5 )
        countpath=StringVar()
        self.countpath = countpath
        count_entry = Entry(qcOpts,
                           bd =2,
                           width = 50,
                           #bg = entryBgColor,
                           #fg = entryFgColor,
                           textvariable = countpath, state='normal'
                          )
        count_entry.grid( row=9, column=2, columnspan=3 )
        self.count_button = count_button = Button( qcOpts,
                             text="Open Directory",
                             command=self.set_count_directory )
        count_button.grid( row=9, column=5 )

        self.mattype = mattype = StringVar()
        mattypeL = Label(qcOpts, text="Count matrix format: ")
        scrMatTypeDropdown = ["cellranger", "cellranger_raw", "zumi", "biorad"]
        mattype.set(scrMatTypeDropdown[0])
        mattype_om = OptionMenu(qcOpts, mattype, *scrMatTypeDropdown)
        mattypeL.grid(row=10,column=1,sticky=W,padx=10,pady=5)
        mattype_om.grid(row=10,column=2,sticky=W,padx=0,pady=5)

        self.docycleregress = docycleregress = StringVar()
        docycleregressL = Label(qcOpts, text="Do cell cycle regression? ")
        scrCycleDropdown = ["TRUE", "FALSE"]
        docycleregress.set(scrCycleDropdown[0])
        cycle_om = OptionMenu(qcOpts, docycleregress, *scrCycleDropdown)
        docycleregressL.grid(row=11,column=1,sticky=W,padx=10,pady=5)
        cycle_om.grid(row=11,column=2,sticky=W,padx=0,pady=5)

        usecycleregressL_c = Label(clusterOpts, text="Use cell cycle regressed data? ")
        docycleregress.set(scrCycleDropdown[0])
        cycle_om_c = OptionMenu(clusterOpts, docycleregress, *scrCycleDropdown)
        usecycleregressL_c.grid(row=10,column=1,sticky=W,padx=10,pady=5)
        cycle_om_c.grid(row=10,column=2,sticky=W,padx=0,pady=5)

        usecycleregressL_mc = Label(multiclusterOpts, text="Use cell cycle regressed data? ")
        docycleregress.set(scrCycleDropdown[0])
        cycle_om_mc = OptionMenu(multiclusterOpts, docycleregress, *scrCycleDropdown)
        usecycleregressL_mc.grid(row=10,column=1,sticky=W,padx=10,pady=5)
        cycle_om_mc.grid(row=10,column=2,sticky=W,padx=0,pady=5)

        groups_buttonL = Label(qcOpts, text="SAMPLE INFORMATION: ")
        groups_button = Button(qcOpts,
                                            text="Set Groups",
                                            command = self.popup_groups )
        groups_buttonL.grid(row=12,column=1,sticky=W,padx=10,pady=5)
        groups_button.grid(row=12,column=2,sticky=W,padx=0,pady=5)
        #####################

        self.option_controller()

    def popup_groups( self ) :
        self.popup_window( "Groups Information", "groups.tab" )

    def popup_window( self, text, filename ) :
        top = Toplevel()

        info = LabelFrame(top, text=text )#"Group Information")
        info_text = Text(info,
                              width=50,
                              height=8,
                              #bg=projectBgColor,
                              #fg=projectFgColor,
                              font=("nimbus mono bold","11")
                             )

        def savefunc() :
            self.writepaste( filename, info_text )

        def loadfunc() :
            self.readpaste( filename, info_text )

        info_save_button = Button(info,
                                  text="Save",
                                  command = savefunc )
        info_load_button = Button(info,
                                  text="Load",
                                  command = loadfunc )

        #self.pairs_load_button.pack( side=BOTTOM, padx=5, pady=5 )
        #self.pairs_save_button.pack( side=BOTTOM, padx=5, pady=5 )

        info_load_button.grid( row=5, column=5, padx=10, pady=5 )
        info_save_button.grid( row=5, column=6, padx=10, pady=5 )
        info_text.grid(row=1,  rowspan=3,
                       column=1,  columnspan=7,
                       padx=5, pady=5 )

        info.grid(row=7,column=0, columnspan=6, sticky=W, padx=20, pady=10 )
        top.focus_force()

    def set_count_directory( self ):
        fname = askdirectory( initialdir = USER_HOME,
                             title="Select Data Directory")
        self.countpath.set(fname)

    def option_controller( self, *args, **kwargs ) :

        PipelineFrame.option_controller( self )

        self.Pipeline.set( self.label2pipeline[self.PipelineLabel.get()] )
        print( self.Pipeline.get() )

        if self.Pipeline.get() == 'cellranger' :
            self.clusterOpts.grid_forget()
            self.crOpts.grid(row=8,column=0, columnspan=6, sticky=W, padx=20, pady=10 )
            self.qcOpts.grid_forget()
            self.multiclusterOpts.grid_forget()
        elif self.Pipeline.get() == 'scrnaseqcluster' :
            self.clusterOpts.grid( row=8, column=0, columnspan=4, sticky=W, padx=20, pady=10 )
            self.crOpts.grid_forget()
            self.qcOpts.grid_forget()
            self.multiclusterOpts.grid_forget()
        elif self.Pipeline.get() == 'scrnaseqinit' :
            self.clusterOpts.grid_forget()
            self.crOpts.grid_forget()
            self.qcOpts.grid( row=8, column=0, columnspan=4, sticky=W, padx=20, pady=10 )
            self.multiclusterOpts.grid_forget()
        elif self.Pipeline.get() == 'scrnaseqmulticluster' :
            self.clusterOpts.grid_forget()
            self.crOpts.grid_forget()
            self.qcOpts.grid_forget()
            self.multiclusterOpts.grid( row=8, column=0, columnspan=4, sticky=W, padx=20, pady=10 )

    def makejson_wrapper( self, *args, **kwargs ) :
        self.makejson(*args, **kwargs)

    def makejson(self, *args):
        ###################################################### FROM RNASEQ.PY, NOT SURE WHAT ALL THIS DOES #############################3
        #print(args[0])
        caller=args[0]
        #global PD
        #global UnitsBak
        #global RGbak
        D=dict()
        try:
            F=open(self.workpath.get()+"/samples","r")
            f=F.read().split("\n")
            F.close()
            for line in f:
                L=line.split()
                a=L.pop(0)
                D[a]=L
                samples=D
        except:
            samples={"na":"na"}

        D=dict()
        try:
            F=open(self.workpath.get()+"/pairs","r")
            f=F.read().split()
            F.close()
            for i in range(0,len(f),2):
    #            a=f[i].split(".")[0]
    #            b=f[i+1].split(".")[0]
                a=f[i]
                b=f[i+1]

                D[a+"+"+b]=[a,b]

            pairs=D
        except:
            pairs={"na":"na"}

        D=dict()
        try:
            F=open(self.workpath.get()+"/contrasts.tab","r")
    #        f=F.read().split('\n')
    #        F.close()
    #        D["rsamps"]=f[0].split()
    #        D["rgroups"]=f[1].split()
    #        D["rcontrasts"]=f[2].split()
    #        D["rlabels"]=f[3].split()
            f=F.readlines()
            F.close()
    ##        sampl=[]
    ##        grp=[]
            cont=[]
    ##        lbl=[]
            for x in f:
                  if len(x.split()) == 2:
                     cont.append(x.split()[0])
                     cont.append(x.split()[1])
            D["rcontrasts"]=cont
    #        contrasts["rcontrasts"]=cont
            contrasts=D
        except:
            contrasts={"rcontrasts":"na"}
    ##        contrasts={"rsamps":"na","rgroups":"na","rcontrasts":"na","rlabels":"na"}
    ##------
        D=dict()
        try:
            F=open(self.workpath.get()+"/groups.tab","r")
            f=F.readlines()
            F.close()
            sampl=[]
            grp=[]
    #        cont=[]
            lbl=[]
            for x in f:
    #           if len(x.split()) == 4 or len(x.split()) == 3:
                if len(x.split()) == 3:
                    sampl.append(x.split()[0])
                    grp.append(x.split()[1])
                    lbl.append(x.split()[2])
            D["rsamps"]=sampl
            D["rgroups"]=grp
            D["rlabels"]=lbl
    #        D["rcontrasts"]="na"
    #        contrasts=D
            groups=D
        except:
    #        contrasts={"rsamps":"na","rgroups":"na","rcontrasts":"na","rlabels":"na"}
            groups={"rsamps":"na","rgroups":"na","rlabels":"na"}
    ##------
        D=dict()
        FT = filetype#.get()
    #    p = Popen("ls "+workpath.get()+"/*."+FT, shell=True, stdin=PIPE, stdout=PIPE, stderr=DEVNULL, close_fds=True)
        p = Popen("find "+self.workpath.get()+" -maxdepth 1 -type l -printf '%f\n' ", shell=True, stdin=PIPE, stdout=PIPE, stderr=DEVNULL, close_fds=True)
        a = p.stdout.read().decode(encoding='UTF-8').split("\n")

        RG=dict()
        b=a.pop()
    #    tkinter.messagebox.showerror("",a)
    #    if freezeunits.get()=="no":
        for i in a:

            key=re.sub(".realign","",i.split("/")[-1])
            key=re.sub(".bai","",key)
            key=re.sub(".bam","",key)
            key=re.sub(".sam","",key)
            key=re.sub(".recal","",key)
            key=re.sub(".dedup","",key)
            key=re.sub(".sorted","",key)
            key=re.sub(".fin","",key)
            key=re.sub("\.R[12]","",key)
            key=re.sub("_R[12]","",key)
            key=re.sub(".fastq","",key)
            key=re.sub(".gz","",key)
    #        key=re.sub("[\._](R[12]\.)*"+FT+"$","",i.split("/")[-1])
    #        key=re.sub(".R[12]."+FT+"$","",i.split("/")[-1])
    #        key=re.sub("([._]R[12][._])*([fin|sorted|dedup|recal|realign])*\.{0}$".format(FT),"",i.split("/")[-1])
            D[key]=key
            RG[key]={'rgsm':key,'rglb':'na','rgpu':'na','rgpl':'ILLUMINA','rgcn':'na'}
        units=D
        UnitsBak=D

        try:
            F=open(self.workpath.get()+"/rg.tab","r")
            f=F.read().splitlines()
            F.close()
            for theLine in f:
                if not re.match("^ID",theLine):
                    (rgid,rgsm,rglb,rgpl,rgpu,rgcn)=theLine.split("\t")
                    RG[rgid]={'rgsm':rgsm,'rglb':rglb,'rgpu':rgpu,'rgpl':rgpl,'rgcn':rgcn}
        except:
            pass

        RGbak=RG
    #     else:
    #         units=UnitsBak
    #         RG=RGbak
    # ###########################################################################################################################

        PD=dict()

        smparams=[]

        for i in range(len(self.parameters)):

            if cp[i].var.get()=="1":
                smparams.append(parameters[i])

        AD=eval( open( join(PIPELINER_HOME,
                            self.annotation.get()+".json"
                           ), "r"
                     ).read()
               )
        gi = self.global_info
        PD={
            'project': {
                'pfamily': gi.pfamily.get(),
                'units': units,
                'samples': samples,
                'pairs': pairs,
                'id': gi.eprojectid.get(),
                'pi': gi.epi.get(),
                'organism': gi.eorganism.get(),
                'analyst': gi.eanalyst.get(),
                'poc': gi.epoc.get(),
                'pipeline': self.Pipeline.get(),
                'version':"4.0",
                'annotation': gi.annotation.get(),
                'datapath': self.datapath.get(),
                'targetspath': self.targetspath.get(),
                'filetype': filetype ,
                'binset': "standard-bin",
                'username': gi.euser.get(),
                'flowcellid': gi.eflowcell.get(),
                'platform': gi.eplatform.get(),
                'custom': customRules,
                'efiletype': efiletype,
                'workpath': self.workpath.get(),
                'batchsize': batchsize,
                "smparams": smparams,
                "rgid": RG,
                "cluster": "cluster_medium.json",
                "description": gi.description.get('1.0',END),
                "technique" : gi.technique.get(),
                "CRID": self.scrCRID.get(),
                "EXPECTED": self.scrExpected.get(),
                "COUNTSPATH": self.countpath.get(),
                "MATTYPE": self.mattype.get(),
                "DOCYCLEREGRESS": self.docycleregress.get(),
                "RESOLUTION": self.scrRes.get(),
                "PCS": self.scrPCs.get(),
                "groups": groups
             }
        }

        J=json.dumps(PD, sort_keys = True, indent = 4, ensure_ascii=True)
        gi.jsonconf.delete("1.0", END)
        gi.jsonconf.insert(INSERT, J)
        self.saveproject(gi.jsonconf.get("1.0",END))


from pathlib import Path
from torch.utils.data import DataLoader
import torch
import numpy as np
from torch.nn import CrossEntropyLoss
from pgdataset.s3_handcraft import PgdHandcraft
from constants.enum_keys import PG
from models.gesture_recognition_model import GestureRecognitionModel
from torch import optim
from constants import settings

class Trainer:
    def __init__(self, is_unittest=False):
        self.is_unittest = is_unittest
        self.batch_size = 1
        self.clip_len = 5*7
        pgd = PgdHandcraft(Path.home() / 'intentlong', True, (512, 512), clip_len=self.clip_len)
        self.data_loader = DataLoader(pgd, batch_size=self.batch_size, shuffle=False, num_workers=settings.num_workers)
        self.model = GestureRecognitionModel(batch=self.batch_size)
        self.model.train()
        self.loss = CrossEntropyLoss()
        self.opt = optim.Adam(self.model.parameters(), lr=1e-3)

    def train(self):
        step = 0
        self.model.load_ckpt()
        for epoch in range(100000):
            for ges_data in self.data_loader:
                features = torch.cat((ges_data[PG.BONE_LENGTH], ges_data[PG.BONE_ANGLE_COS],
                                      ges_data[PG.BONE_ANGLE_SIN]), dim=2)
                features = features.permute(1, 0, 2)
                features = features.to(self.model.device, dtype=torch.float32)
                h0, c0 = self.model.h0(), self.model.c0()

                _, h, c, class_out = self.model(features, h0, c0)
                target = ges_data[PG.GESTURE_LABEL]
                target = target.to(self.model.device, dtype=torch.long)
                target = target.permute(1, 0)
                target = target.reshape((-1))
                loss_tensor = self.loss(class_out, target)
                self.opt.zero_grad()
                loss_tensor.backward()
                self.opt.step()

                if step % 100 == 0:
                    print("Step: %d, Loss: %f" % (step, loss_tensor.item()))
                if step % 5000 == 0 and step != 0:
                    self.model.save_ckpt()
                if self.is_unittest:
                    break
                step = step + 1
            if self.is_unittest:
                break
#p1 = Trainer()
#p1.train()


from base import BaseHandler

class OpenSearchHandler(BaseHandler):
  def get(self):
    user = self.models.users.get(username=self.breadcrumbs["profile"])[0]

    if not user:
      raise tornado.web.HTTPError(404)

    self.display["user"] = user

    self.set_header("Content-Type", "application/opensearchdescription+xml")
    self.fill_template("opensearch.html")


import numpy as np
import cv2

def split_gt(gt):
    gt = gt.data.cpu().numpy()
    kp = gt[:5,:,:]
    short = np.transpose(gt[5:5+10,:,:], (1,2,0))
    mid = np.transpose(gt[5+10:,:,:], (1,2,0))
    return kp, short ,mid

def show_input(input):
    colors = [(0,0,0.9),(0.9,0,0),(0.9,0,0.9),(0.9,0.9,0), (0.2,0.9,0.9)]
    img, gt_c0, gt_c1, gt_c2, gt_c3, instance_masks, bboxes_c0 = input

    kp0, short0, mid0 = split_gt(gt_c0)
    kp1, short1, mid1 = split_gt(gt_c1)
    kp2, short2, mid2 = split_gt(gt_c2)
    kp3, short3, mid3 = split_gt(gt_c3)

    img = (img+0.5)*255
    img = np.transpose(img.data.cpu().numpy(), (1,2,0))

    map_s0 = np.zeros((short0.shape[:2]), dtype=np.float64)
    for i_kp in range(5):
        offset = short0[:, :, i_kp * 2:i_kp * 2 + 2]
        map_s0 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    cv2.imshow('short0', np.uint8((map_s0 * 200)))

    # map_m0 = np.zeros((mid0.shape[:2]), dtype=np.float64)
    # for i_kp in range(20):
    #     offset = mid1[:, :, i_kp * 2:i_kp * 2 + 2]
    #     map_m0 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    # cv2.imshow('mid0', np.uint8((map_m0 * 200)))

    map_s1 = np.zeros((short1.shape[:2]), dtype=np.float64)
    for i_kp in range(5):
        offset = short1[:, :, i_kp * 2:i_kp * 2 + 2]
        map_s1 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    cv2.imshow('short1', np.uint8((map_s1 * 200)))

    # map_m1 = np.zeros((mid1.shape[:2]), dtype=np.float64)
    # for i_kp in range(20):
    #     offset = mid1[:, :, i_kp * 2:i_kp * 2 + 2]
    #     map_m1 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    # cv2.imshow('mid1', np.uint8((map_m1 * 200)))

    map_s2 = np.zeros((short2.shape[:2]), dtype=np.float64)
    for i_kp in range(5):
        offset = short2[:, :, i_kp * 2:i_kp * 2 + 2]
        map_s2 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    cv2.imshow('short2', np.uint8((map_s2 * 200)))

    # map_m2 = np.zeros((mid2.shape[:2]), dtype=np.float64)
    # for i_kp in range(20):
    #     offset = mid2[:, :, i_kp * 2:i_kp * 2 + 2]
    #     map_m2 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    #     cv2.imshow('mid2', np.uint8((map_m2 * 200)))

    map_s3 = np.zeros((short3.shape[:2]), dtype=np.float64)
    for i_kp in range(5):
        offset = short3[:, :, i_kp * 2:i_kp * 2 + 2]
        map_s3 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    cv2.imshow('short3', np.uint8((map_s3 * 200)))

    # map_m3 = np.zeros((mid3.shape[:2]), dtype=np.float64)
    # for i_kp in range(20):
    #     offset = mid3[:, :, i_kp * 2:i_kp * 2 + 2]
    #     map_m3 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    # cv2.imshow('mid3', np.uint8((map_m3 * 200)))

    cv2.imshow('img', np.uint8(img))

    k = cv2.waitKey(0)
    if k & 0xFF == ord('q'):
        cv2.destroyAllWindows()
        exit()


# -*- coding: utf-8 -*-
from road import road
from auths import road_engine

from sqlalchemy.orm import sessionmaker

road_Session = sessionmaker(bind = road_engine)

_road_session = road_Session()

from sqlalchemy import distinct, asc, and_
from datetime import datetime, timedelta
import simplekml

def get_tmc_by_id_timedelta(_rdid, _headtime, _timedelta):

	head_time = _headtime
	tail_time = _headtime + _timedelta

	for tmc in _tmc_session.query(tmc_point.Speed, tmc_point.Record_Time).filter(and_(tmc_point.RoadID == _rdid, tmc_point.Record_Time >= head_time, tmc_point.Record_Time <= tail_time)).all():
		if len(tmc) >= 0:
			speed, _time = tmc
			yield _time, speed

def get_geo(rdid):
	from sqlalchemy import text
	rdid = abs(int(rdid))
	cmd = "select Geom.ToString() from Table_RoadData where RoadID = {0}".format(rdid)
	linestring = road_engine.execute(cmd).fetchone()
	if linestring == None:
		cmd = "select Geom.ToString() from Table_RoadData where RoadID = {0}".format(int(rdid))
		linestring = road_engine.execute(cmd).fetchone()
		if linestring == None:
			return None
	return str(linestring[0])

def parse_linestring_to_coords(linestring):
	_return = []
	import re
	match = re.search(r".*\((.*)\)", linestring)
	string_coords = match.group(1).split(', ')

	for coord in string_coords:
		lon, lat = coord.split(" ")
		_return.append((float(lon), float(lat)))
	return _return

def set_LOS_color(speed):

	speed = int(speed)
	if speed >= 60:
		return simplekml.Color.green
	elif speed < 60 and speed >= 40:
		return simplekml.Color.yellow
	elif speed < 40 and speed >= 20:
		return simplekml.Color.orange
	elif speed < 20:
		return simplekml.Color.red

def draw_kml_color_by_speed(kml, speed, rdid, coords):
	ln = kml.newlinestring(name=str(rdid))
	ln.description = "RoadID:{0} speed:{1}".format(rdid, speed)
	ln.coords =coords
	ln.extrude=1
	ln.linestyle.width = 5
	ln.linestyle.color = set_LOS_color(speed)

def get_roadids_by_chiname(chiname):
	'''
	to get roadid by (chinese name)
	'''
	from sqlalchemy import and_, or_
	chiname = "%{0}%".format(chiname)
	return [rd[0] for rd in _road_session.query(road.RoadID).filter(and_(road.Name.like(chiname)), or_(road.City==0, road.City==1)).all()]

def get_roadname_by_roadid(rdid):
	rdid = _road_session.query(road.Name).filter(road.RoadID == rdid).all()
	return rdid[0][0]


import subprocess
import logging
import argparse

from getpass import getuser
from shlex import split

logging.basicConfig(
    format='[%(asctime)s : %(levelname)s] %(message)s',
    level=logging.INFO,
    datefmt='%I:%M:%S')
LOGGER = logging.getLogger()

def execute_cmd_on_host(_user, _host, _script):
    LOGGER.info('Running script on behalf of %s user on host %s', _user, _host)
    cmd = 'ssh {}@{} \'{}\''.format(_user, _host, _script)
    # Method subprocess.check_output() returns byte sequence starting with "b'" symbols
    # and ending with "\n'".
    # decode() will turn byte sequence into string deleting "b'", strip() will delete "\n"
    # at the end of sequence
    cmd_output = str(subprocess.check_output(split(cmd), timeout=30).decode()).strip().split('\n')
    LOGGER.info('Command executing: %s', cmd)
    LOGGER.info('Command output: \n%s', '\n'.join(cmd_output))

if __name__ == '__main__':

    PARSER = argparse.ArgumentParser()
    PARSER.add_argument('--host', type=str, default='localhost')
    PARSER.add_argument('--user', type=str, default=getuser())
    PARSER.add_argument('--script', type=str, default='echo TEST_ME > ~/Desktop/test.txt')
    ARGS = PARSER.parse_args()

    HOST = ARGS.host
    USER = ARGS.user
    SCRIPT = ARGS.script

    execute_cmd_on_host(USER, HOST, SCRIPT)
    execute_cmd_on_host(USER, HOST, 'cd Desktop && ls')


import glob
import logging

from schema_salad.validate import ValidationException

from .step import Step

def load_steps(steps_dir=None, step_file=None):
    """Return a dictionary containing Steps read from file.

    Args:
        steps_dir (str, optional): path to directory containing CWL files.
        step_file (str, optional): path to a single CWL file.

    Return:
        dict containing (name, Step) entries.

    """
    if steps_dir is not None:
        step_files = glob.glob('{}/*.cwl'.format(steps_dir))
    elif step_file is not None:
        step_files = [step_file]
    else:
        step_files = []

    steps = {}
    for f in step_files:
        try:
            s = Step(f)
            steps[s.name] = s
        except (NotImplementedError, ValidationException) as e:
            logging.warning(e)

    return steps


import random

random.seed()
L=[]
for x in range(50):
    L.append(int(random.random()*20))

for i in range(50):
    j=i+1
    index=i
    while j < 50:
        if L[j] > L[index]:
            index=j
        j+=1

    x=L[i]
    L[i]=L[index]
    L[index]=x


"""Views for `spot` app."""

# third party
from brasil_municipios.models import Municipio
from django.conf import settings
from django.contrib.gis.db.models.functions import Distance
from django.contrib.gis.geos import Point
from django.core.urlresolvers import reverse
# project
from .models import Spot
from utils.http import JsonResponse
from utils.pagination import paginate

def spots_list(request, api_version):
    response_headers = {}
    spots = Spot.objects.all()

    #
    # Sorting
    #
    sort = 'name'

    if 'lat' in request.GET and 'lng' in request.GET:
        try:
            lat = float(request.GET['lat'])
            lng = float(request.GET['lng'])
        except ValueError:  # pragma: no cover
            pass  # invalid values for lator lng: use default ordering
        else:
            origin = Point(lng, lat, srid=settings.SRID)
            spots = spots.annotate(distance=Distance('coords', origin))
            sort = 'distance'
    else:
        descending_sort = False  # descending sort: '-field'
        sort = request.GET.get('sort', '')
        if sort.startswith('-'):
            descending_sort = True
            sort = sort[1:]  # '-field' -> 'field'

        # maps url 'sort' param value to the queryset .order() value
        sort_field_map = {'name': 'name', 'created_at': 'created_at'}

        if sort in sort_field_map:
            sort = sort_field_map[sort]
            if descending_sort:
                sort = '-' + sort
        else:
            sort = 'name'

    spots = spots.order_by(sort)

    #
    # City filtering
    #
    if 'city' in request.GET:
        city_id = request.GET['city']
        try:
            city = Municipio.objects.defer('geometry').get(pk=city_id)
        except (Municipio.DoesNotExist, ValueError):
            pass  # Invalid city id, return all spots
        else:
            spots = spots.filter(city=city)

    #
    # Searching
    #
    query = request.GET.get('q', '')
    if query:
        spots = spots.filter(name__unaccent__icontains=query)

    #
    # Pagination
    #
    base_url = '{}://{}{}'.format(
        request.scheme,
        request.get_host(),
        reverse('spots', kwargs={'api_version': api_version})
    )
    pagination = paginate(queryset=spots,
                          query_dict=request.GET,
                          base_url=base_url)
    response_headers.update(pagination['response_headers'])
    page = pagination['page']

    #
    # Construct response
    #
    data = []
    for spot in page:
        spot_data = spot.to_dict()
        spot_data['url'] = spot.get_full_url(request.scheme,
                                             request.get_host(),
                                             api_version)
        if spot.directions:
            spot_data['directionsUrl'] = spot.get_directions_full_url(
                request.scheme, request.get_host(), api_version)
        else:
            spot_data['directionsUrl'] = None
        data.append(spot_data)

    prettify = request.GET.get('pretty') != 'false'
    response = JsonResponse(data, safe=False, pretty=prettify)
    for (key, val) in response_headers.items():
        response[key] = val
    return response

def spot_details(request, api_version, spot_id=None, spot_slug=None):
    assert (
        (spot_id is not None and spot_slug is None) or
        (spot_id is None and spot_slug is not None)
    ), 'Either `spot_id` or `spot_slug` required, but not both'

    spots_qs = Spot.objects.select_related('city').defer('city__geometry')

    try:
        if spot_id is not None:
            spot = spots_qs.get(pk=spot_id)
        else:
            spot = spots_qs.get(slug=spot_slug)
    except Spot.DoesNotExist:
        status_code = 404
        data = {
            'status_code': status_code,
            'message': 'Spot {} not found'.format(spot_id or spot_slug)
        }
    else:
        status_code = 200
        data = spot.to_dict(all_details=True)
        data['url'] = spot.get_full_url(
            request.scheme, request.get_host(), api_version)
        if spot.directions:
            data['directionsUrl'] = spot.get_directions_full_url(
                request.scheme, request.get_host(), api_version)
        else:
            data['directionsUrl'] = None

    prettify = request.GET.get('pretty') != 'false'
    response = JsonResponse(data, status=status_code, safe=False,
                            pretty=prettify)
    return response

def spots_endpoint(request, api_version, spot_id=None, spot_slug=None):
    if spot_id is None and spot_slug is None:
        return spots_list(request, api_version)
    else:
        return spot_details(request,
                            api_version,
                            spot_id=spot_id,
                            spot_slug=spot_slug)

def directions_endpoint(request, api_version, spot_slug):
    try:
        spot = Spot.objects.get(slug=spot_slug)
    except (Spot.DoesNotExist):
        status_code = 404
        data = {
            'status_code': status_code,
            'message': 'Spot {} not found'.format(spot_slug)
        }
    else:
        if spot.directions is None:
            status_code = 404
            data = {
                'status_code': status_code,
                'message': 'Directions not available for spot {}'.format(
                    spot_slug)
            }
        else:
            status_code = 200
            data = spot.directions

    prettify = request.GET.get('pretty') != 'false'
    response = JsonResponse(data, status=status_code, safe=False,
                            pretty=prettify)
    return response


class DLinkedNode:
    def __init__(self, key=0, value=0):
        self.key = key
        self.value = value
        self.pre = None
        self.next = None

class LRUCache:
    def __init__(self, capacity: int):
        self.capacity = capacity
        self.size = 0  # 现有数据数
        self.cache = {}
        self.head = DLinkedNode()
        self.tail = DLinkedNode()
        self.head.next = self.tail
        self.tail.pre = self.head

    def get(self, key: int) -> int:
        if key not in self.cache:
            return -1
        node = self.cache[key]
        self.move2head(node)
        return node.value

    def put(self, key: int, value: int) -> None:
        if key in self.cache:
            node = self.cache[key]
            node.value = value
            self.move2head(node)
        else:
            node = DLinkedNode(key, value)
            self.add2head(node)
            self.cache[key] = node
            self.size += 1
            # 如果大于最大容量，删除最旧数据
            if self.size > self.capacity:
                removed_node = self.remove_tail()
                self.cache.pop(removed_node.key)
                self.size -= 1

    # 将节点移动至头节点后一位位置，表示为最新操作数据
    def move2head(self, node):
        self.remove(node)
        self.add2head(node)

    # 删除节点
    def remove(self, node):
        node.pre.next = node.next
        node.next.pre = node.pre

    # 头节点后添加节点
    def add2head(self, node):
        self.head.next.pre = node
        node.next = self.head.next
        node.pre = self.head
        self.head.next = node

    # 删除尾部的前一个节点，表示为删除最旧操作数据
    def remove_tail(self):
        node = self.tail.pre
        self.remove(node)
        return node

if __name__ == '__main__':
    cache = LRUCache(2)
    print(cache.put(1, 1))
    print(cache.put(2, 2))
    print(cache.get(1))
    print(cache.put(3, 3))
    print(cache.get(2))
    print(cache.put(4, 4))
    print(cache.get(1))
    print(cache.get(3))
    print(cache.get(4))


from bs4 import BeautifulSoup
import requests

def scrape_results(search_query):
    html = requests.get(f"https://www.midiworld.com/search/?q={search_query}").text
    soup = BeautifulSoup(html, "html.parser")
    error = soup.find("div", {"id": "page"}).text
    if "found nothing!" in error:
        return None
    unordered_lists = soup.find_all("ul")[1]
    lists = unordered_lists.find_all("li")
    results = [{"name": lst.text.replace("\n", "").split(" - download")[0], "url": lst.find("a")["href"]} for lst in lists]
    return results

def save_midi(url, name):
    song = requests.get(url)
    statuscode = song.status_code
    if statuscode == 200:
        open(f"songs/{name}.mid", "wb").write(song.content)
        return True # changed from name
    else:
        return False

def main(query):
    results = scrape_results(query)
    return results

if __name__ == '__main__':
    main("")


import os
os.environ['THEANO_FLAGS'] = 'device=cpu'

import argparse
from ast import literal_eval
import random
import shutil

import numpy as np

import config
from model import build_model_test, Agent
from state import NormState
from environments import RunEnv2

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--accuracy', dest='accuracy', action='store', default=5e-5, type=float)
    parser.add_argument('--modeldim', dest='modeldim', action='store', default='3D',
            choices=('3d', '2d', '3D', '2D'), type=str)
    parser.add_argument('--prosthetic', dest='prosthetic', action='store', default=True, type=bool)
    parser.add_argument('--difficulty', dest='difficulty', action='store', default=0, type=int)
    parser.add_argument('--episodes', type=int, default=10, help="Number of test episodes.")
    parser.add_argument('--critic_layers', type=str, default='(64,32)', help="critic hidden layer sizes as tuple")
    parser.add_argument('--actor_layers', type=str, default='(64,64)', help="actor hidden layer sizes as tuple")
    parser.add_argument('--layer_norm', action='store_true', help="Use layer normalization.")
    parser.add_argument('--weights', type=str, default=None, help='weights to load')
    parser.add_argument('--skip_frames', type=int, default=config.skip_frames, help='how many frames to execute each action for')
    args = parser.parse_args()
    args.modeldim = args.modeldim.upper()
    return args

def test_agent(args, state_transform, num_test_episodes,
               actor, weights, global_step, save_dir):
    env = RunEnv2(state_transform, visualize=True, integrator_accuracy=args.accuracy,
                  model=args.modeldim, prosthetic=args.prosthetic, difficulty=args.difficulty,
                  skip_frame=config.skip_frames)
    test_rewards = []

    actor.set_actor_weights(weights)
    if args.weights is not None:
        actor.load(args.weights)

    for ep in range(num_test_episodes):
        seed = random.randrange(2**32-2)
        state = env.reset(seed=seed, difficulty=2)
        test_reward = 0
        while True:
            state = np.asarray(state, dtype='float32')
            action = actor.act(state)
            state, reward, terminal, _ = env._step(action)
            test_reward += reward
            if terminal:
                break
        test_rewards.append(test_reward)
    mean_reward = np.mean(test_rewards)
    std_reward = np.std(test_rewards)

    test_str ='global step {}; test reward mean: {:.2f}, std: {:.2f}, all: {} '.\
        format(global_step.value, float(mean_reward), float(std_reward), test_rewards)

    print(test_str)
    with open(os.path.join('test_report.log'), 'a') as f:
        f.write(test_str + '\n')

def main():
    args = get_args()
    args.critic_layers = literal_eval(args.critic_layers)
    args.actor_layers = literal_eval(args.actor_layers)

    save_dir = os.path.join('tests')
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    state_transform = NormState(args.prosthetic)
    # state_transform = StateVelCentr(obstacles_mode='standard',
    #                                 exclude_centr=True,
    #                                 vel_states=[])
    env = RunEnv2(state_transform, integrator_accuracy=args.accuracy, model=args.modeldim,
                  prosthetic=args.prosthetic, difficulty=args.difficulty, skip_frame=1)
    env.change_model(args.modeldim, args.prosthetic, args.difficulty)
    num_actions = env.get_action_space_size()
    del env

    model_params = {
            'state_size': state_transform.state_size,
            'num_act': num_actions,
            'gamma': 0,
            'actor_layers': args.actor_layers,
            'critic_layers': args.critic_layers,
            'actor_lr': 0,
            'critic_lr': 0,
            'layer_norm': args.layer_norm
    }
    actor_fn, params_actor, params_crit, actor_lr, critic_lr = \
            build_model_test(**model_params)
    actor = Agent(actor_fn, params_actor, params_crit)

    actor.load(args.weights)

    weights = [p.get_value() for p in params_actor]

    global_step = 0
    test_agent(args, state_transform, args.episodes,
            actor, weights, global_step, save_dir)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
#################################################################################
# Copyright 2018 ROBOTIS CO., LTD.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#################################################################################

# Authors: Gilbert #

import rospy
import numpy as np
import math
from math import pi
from geometry_msgs.msg import Twist, Point, Pose
from sensor_msgs.msg import LaserScan
from nav_msgs.msg import Odometry
from std_srvs.srv import Empty
from tf.transformations import euler_from_quaternion, quaternion_from_euler
from respawnGoal import Respawn

class Env():
    def __init__(self, action_size):
        self.goal_x = 0
        self.goal_y = 0
        self.heading = 0
        self.action_size = action_size
        self.initGoal = True
        self.get_goalbox = False
        self.position = Pose()
        self.pub_cmd_vel = rospy.Publisher('cmd_vel', Twist, queue_size=5)
        self.sub_odom = rospy.Subscriber('odom', Odometry, self.getOdometry)
        self.reset_proxy = rospy.ServiceProxy('gazebo/reset_simulation', Empty)
        self.unpause_proxy = rospy.ServiceProxy('gazebo/unpause_physics', Empty)
        self.pause_proxy = rospy.ServiceProxy('gazebo/pause_physics', Empty)
        self.respawn_goal = Respawn()

    def getGoalDistace(self):
        goal_distance = round(math.hypot(self.goal_x - self.position.x, self.goal_y - self.position.y), 2)

        return goal_distance

    def getOdometry(self, odom):
        self.position = odom.pose.pose.position
        orientation = odom.pose.pose.orientation
        orientation_list = [orientation.x, orientation.y, orientation.z, orientation.w]
        _, _, yaw = euler_from_quaternion(orientation_list)

        goal_angle = math.atan2(self.goal_y - self.position.y, self.goal_x - self.position.x)

        heading = goal_angle - yaw
        if heading > pi:
            heading -= 2 * pi

        elif heading < -pi:
            heading += 2 * pi

        self.heading = round(heading, 2)

    def getState(self, scan):
        scan_range = []
        heading = self.heading
        min_range = 0.2 # Threshold distance to obstacle
                        # Initially was 0.13 (save_model data)
        done = False

        for i in range(len(scan.ranges)):
            if scan.ranges[i] == float('Inf'):
                scan_range.append(3.5)
            elif np.isnan(scan.ranges[i]):
                scan_range.append(0)
            else:
                scan_range.append(scan.ranges[i])

        obstacle_min_range = round(min(scan_range), 2)
        obstacle_angle = np.argmin(scan_range)
        if min_range > min(scan_range) > 0:
            done = True

        current_distance = round(math.hypot(self.goal_x - self.position.x, self.goal_y - self.position.y), 2)
        if current_distance < 0.2:
            self.get_goalbox = True

        return scan_range + [heading, current_distance, obstacle_min_range, obstacle_angle], done

    def setReward(self, state, done, action):
        yaw_reward = []
        current_distance = state[-3]
        heading = state[-4]

        for i in range(5):
            angle = -pi / 4 + heading + (pi / 8 * i) + pi / 2
            tr = 1 - 4 * math.fabs(0.5 - math.modf(0.25 + 0.5 * angle % (2 * math.pi) / math.pi)[0])
            yaw_reward.append(tr)

        distance_rate = 2 ** (current_distance / self.goal_distance)
        reward = ((round(yaw_reward[action] * 5, 2)) * distance_rate)

        if done:
            rospy.loginfo("Collision!!")
            reward = -150
            self.pub_cmd_vel.publish(Twist())

        if self.get_goalbox:
            rospy.loginfo("Goal!!")
            reward = 200
            self.pub_cmd_vel.publish(Twist())
            self.goal_x, self.goal_y = self.respawn_goal.getPosition(True, delete=True)
            self.goal_distance = self.getGoalDistace()
            self.get_goalbox = False

        return reward

    def step(self, action):
        max_angular_vel = 1.5
        ang_vel = ((self.action_size - 1)/2 - action) * max_angular_vel * 0.5

        vel_cmd = Twist()
        vel_cmd.linear.x = 0.15
        vel_cmd.angular.z = ang_vel
        self.pub_cmd_vel.publish(vel_cmd)

        data = None
        while data is None:
            try:
                data = rospy.wait_for_message('scan', LaserScan, timeout=5)
            except:
                pass

        state, done = self.getState(data)
        reward = self.setReward(state, done, action)

        return np.asarray(state), reward, done

    def reset(self):
        rospy.wait_for_service('gazebo/reset_simulation')
        try:
            self.reset_proxy()
        except (rospy.ServiceException) as e:
            print("gazebo/reset_simulation service call failed")

        data = None
        while data is None:
            try:
                data = rospy.wait_for_message('scan', LaserScan, timeout=5)
            except:
                pass

        if self.initGoal:
            self.goal_x, self.goal_y = self.respawn_goal.getPosition()
            self.initGoal = False

        self.goal_distance = self.getGoalDistace()
        state, done = self.getState(data)

        return np.asarray(state)


import typing
from typing import Any, Callable, List, Tuple, Union

from .abc_interpreter import Interpreter
from ..data_processor.readers import preprocess_image, read_image, restore_image
from ..data_processor.visualizer import visualize_grayscale

import IPython.display as display
import cv2
import numpy as np
import os, sys
from PIL import Image

class GradShapInterpreter(Interpreter):
    """
    Gradient SHAP Interpreter.

    More details regarding the Gradient SHAP method can be found in the original paper:
    http://papers.nips.cc/paper/7062-a-unified-approach-to-interpreting-model-predictions
    """

    def __init__(self,
                 paddle_model,
                 trained_model_path,
                 class_num,
                 use_cuda,
                 model_input_shape=[3, 224, 224]) -> None:
        """
        Initialize the GradShapInterpreter.

        Args:
            paddle_model (callable): A user-defined function that gives access to model predictions.
                It takes the following arguments:

                - data: Data inputs.
                and outputs predictions. See the example at the end of ``interpret()``.
            trained_model_path (str): The pretrained model directory.
            class_num (int): Number of classes for the model.
            use_cuda (bool, optional): Whether or not to use cuda. Default: True
            model_input_shape (list, optional): The input shape of the model. Default: [3, 224, 224]
        """
        Interpreter.__init__(self)
        self.paddle_model = paddle_model
        self.trained_model_path = trained_model_path
        self.use_cuda = use_cuda
        self.model_input_shape = model_input_shape
        self.paddle_prepared = False

    def interpret(self,
                  data,
                  label=None,
                  baseline=None,
                  n_samples=5,
                  noise_amout=0.1,
                  visual=True,
                  save_path=None):
        """
        Main function of the interpreter.

        Args:
            data (str or numpy.ndarray): The image filepath or processed image.
            label (int, optional): The target label to analyze. If None, the most likely label will be used. Default: None.
            baseline (numpy.ndarray, optional): The baseline input. If None, all zeros will be used. Default: None
            n_samples (int, optional): The number of randomly generated samples. Default: 5.
            noise_amount (float, optional): Noise level of added noise to the image.
                                            The std of Guassian random noise is noise_amount * (x_max - x_min). Default: 0.1
            visual (bool, optional): Whether or not to visualize the processed image. Default: True.
            save_path (str, optional): The filepath to save the processed image. If None, the image will not be saved. Default: None

        Returns:
            numpy.ndarray: avg_attributions

        Example::

            def paddle_model(data):
                import paddle.fluid as fluid
                class_num = 1000
                model = ResNet50()
                logits = model.net(input=image_input, class_dim=class_num)
                probs = fluid.layers.softmax(logits, axis=-1)
                return probs
            gs = GradShapInterpreter(predict_fn, "assets/ResNet50_pretrained", 1000, True)
            avg_attributions = gs.interpret(
                                img_path,
                                label=None,
                                noise_amout=0.1,
                                n_samples=5,
                                visual=True,
                                save_path='grad_shap_test.jpg')
        """

        def add_noise_to_inputs():
            std = noise_amout * (np.max(data) - np.min(data))
            noise = np.random.normal(
                0.0, std,
                (n_samples, ) + data.shape[1:]).astype(self.data_type)
            return data.repeat(n_samples, axis=0) + noise

        # Read in image
        if isinstance(data, str):
            img = read_image(data, crop_size=self.model_input_shape[1])
            data = preprocess_image(img)
        else:
            img = restore_image(data.copy())

        self.data_type = np.array(data).dtype

        data_with_noise = add_noise_to_inputs()

        if baseline is None:
            baseline = np.zeros_like(data)
        baseline = baseline.repeat(n_samples, axis=0)

        if not self.paddle_prepared:
            self._paddle_prepare()

        if label is None:
            _, out = self.predict_fn(data, np.array([[0]]))
            label = np.argmax(out[0])
        label = np.array([[label]]).repeat(n_samples, axis=0)

        rand_scales = np.random.uniform(0.0, 1.0,
                                        (n_samples, 1)).astype(self.data_type)

        input_baseline_points = np.array([
            d * r + b * (1 - r)
            for d, r, b in zip(data_with_noise, rand_scales, baseline)
        ])

        gradients, _ = self.predict_fn(input_baseline_points, label)

        input_baseline_diff = data_with_noise - baseline

        attributions = gradients * input_baseline_diff
        avg_attributions = np.mean(attributions, axis=0, keepdims=True)

        visualize_grayscale(avg_attributions, visual, save_path)

        return avg_attributions

    def _paddle_prepare(self, predict_fn=None):
        if predict_fn is None:
            import paddle.fluid as fluid
            startup_prog = fluid.Program()
            main_program = fluid.Program()
            with fluid.program_guard(main_program, startup_prog):
                with fluid.unique_name.guard():
                    data_op = fluid.data(
                        name='data',
                        shape=[None] + self.model_input_shape,
                        dtype=self.data_type)
                    label_op = fluid.data(
                        name='label', shape=[None, 1], dtype='int64')

                    data_plus = data_op + fluid.layers.zeros_like(data_op)
                    probs = self.paddle_model(data_plus)

                    for op in main_program.global_block().ops:
                        if op.type == 'batch_norm':
                            op._set_attr('use_global_stats', True)
                        elif op.type == 'dropout':
                            op._set_attr('dropout_prob', 0.0)

                    class_num = probs.shape[-1]
                    one_hot = fluid.layers.one_hot(label_op, class_num)
                    one_hot = fluid.layers.elementwise_mul(probs, one_hot)
                    target_category_loss = fluid.layers.reduce_sum(one_hot)

                    p_g_list = fluid.backward.append_backward(
                        target_category_loss)
                    gradients_map = fluid.gradients(one_hot, data_plus)[0]

            if self.use_cuda:
                gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
                place = fluid.CUDAPlace(gpu_id)
            else:
                place = fluid.CPUPlace()
            exe = fluid.Executor(place)
            fluid.io.load_persistables(exe, self.trained_model_path,
                                       main_program)

            def predict_fn(data, label):
                gradients, out = exe.run(main_program,
                                         feed={'data': data,
                                               'label': label},
                                         fetch_list=[gradients_map, probs])
                return gradients, out

        self.predict_fn = predict_fn
        self.paddle_prepared = True


# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the BSD 3-Clause License  (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import torch
import pickle
import numpy as np
import pandas as pd
from config import model_config as config

from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score

import itertools
import matplotlib.pyplot as plt

def load_data(batched=True, test=False, file_dir='./inputs/data_url_0/combined/combined_features.pkl'):
    bs = config['batch_size']
    ftype = 'test' if test else 'train'

    with open('{}'.format(file_dir), 'rb') as f:
        features = pickle.load(f)

    x = features['x_{}'.format(ftype)]
    y = features['y_{}'.format(ftype)]
    data = (x, y)
    if test or not batched:
        return [torch.FloatTensor(data[0]), torch.LongTensor(data[1])]
    data = list(zip(data[0], data[1]))
    n_iters = len(data) // bs
    batches = []
    for i in range(1, n_iters + 1):
        input_batch = []
        output_batch = []
        for e in data[bs * (i-1):bs * i]:
            input_batch.append(e[0])
            output_batch.append(e[1])
        batches.append([torch.FloatTensor(input_batch),
                        torch.LongTensor(output_batch)])
    return batches

def evaluate(targets, predictions):
    performance = {
        'acc': accuracy_score(targets, predictions),
        'f1': f1_score(targets, predictions, average='macro'),
        'precision': precision_score(targets, predictions, average='macro'),
        'recall': recall_score(targets, predictions, average='macro')}
    return performance

def plot_confusion_matrix(targets, predictions, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    # plt.figure(figsize=(8,8))
    cm = confusion_matrix(targets, predictions)
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')


from logging import basicConfig, getLogger, INFO
import pandas as pd
from const import RESULT_DIR, PICKLE_DIR
from preprocessing import load_train_data, load_test_data
from submission import create_submission
import datetime
import numpy as np
from sklearn.metrics import f1_score, accuracy_score
from sklearn.model_selection import StratifiedKFold
from tqdm import tqdm
from models import ModelLgbm

logger = getLogger(__name__)

# 2020/05/09
# CV max f1: 0.9360586378833148  / PL 0.880
# スタッキング用にベストパラメータをきつく調整した。過学習ぎみになると予想。
# 予想と違って過学習気味にならない。緩くした分類器と評価はあんまり変わらない。
# 個々の予測は変わってるが全体として変わってないのか、そもそも個々の予測が変わってないのか。
# 個々がかわってなければスタッキングしても意味ない。パラメータを8割で調整したの甘かった？

def train_model():
    x_train, y_train = load_train_data()
    x_test = load_test_data()

    sf = StratifiedKFold(n_splits=5,
                         shuffle=True,
                         random_state=0)

    params = {
        'objective': 'multiclass',
        'n_estimators': 10000,
        'silent': False,
        'metric': 'multi_logloss',
        'early_stopping_rounds': 100,
        'lambda_l1': 6.83751010327766,
        'lambda_l2': 7.991823346452801,
        'num_leaves': 164,
        'feature_fraction': 0.42,
        'bagging_fraction': 1.0,
        'bagging_freq': 0,
        'min_child_samples': 8,
        'seed': 0
    }
    pred_probas = []
    preds_test_probas = []
    va_idxes = []
    max_score = 0
    list_f1_score = []
    list_accuracy_score = []
    for train_idx, valid_idx in tqdm(list(sf.split(x_train, y_train))):
        trn_x = x_train.iloc[train_idx, :]
        val_x = x_train.iloc[valid_idx, :]

        trn_y = y_train[train_idx]
        val_y = y_train[valid_idx]

        model = ModelLgbm()
        model.fit(trn_x, trn_y, val_x, val_y, params)
        pred = model.predict(val_x)

        pred_proba = model.predict_proba(val_x)
        pred_probas.append(pred_proba)
        pred_test_proba = model.predict_proba(x_test)
        preds_test_probas.append(pred_test_proba)
        va_idxes.append(valid_idx)

        sc_f1 = f1_score(val_y, pred, average='macro')
        sc_accuracy = accuracy_score(val_y, pred)

        list_f1_score.append(sc_f1)
        list_accuracy_score.append(sc_accuracy)
        logger.info('     f1: {}, accuracy: {}'.format(sc_f1, sc_accuracy))

    f1_mean = np.mean(list_f1_score)
    accuracy_mean = np.mean(list_accuracy_score)
    logger.info('f1: {}, accuracy: {}'.format(f1_mean, accuracy_mean))

    if max_score < f1_mean:
        max_score = f1_mean

    logger.info('max f1: {}'.format(max_score))

    va_idxes = np.concatenate(va_idxes)
    pred_probas = np.concatenate(pred_probas, axis=0)
    order = np.argsort(va_idxes)
    stacking_train = pred_probas[order]
    stacking_test = np.mean(preds_test_probas, axis=0)
    pd.to_pickle(stacking_train, PICKLE_DIR + 'stacking_lightgbm_3_train.pkl')
    pd.to_pickle(stacking_test, PICKLE_DIR + 'stacking_lightgbm_3_test.pkl')
    return model

if __name__ == '__main__':
    log_format = '%(asctime)s %(name)s %(lineno)d [%(levelname)s][%(funcName)s] %(message)s '
    file_prefix = datetime.datetime.now().strftime('%m%d_%H%M')
    basicConfig(filename=RESULT_DIR + file_prefix + '_train_lightgbm_st3.py.log', level=INFO, format=log_format)

    logger.info('##### Begin single model of lightGBM #####')

    model = train_model()

    # 提出時のみコメントOFF
    create_submission(model)

    logger.info('##### single model of lightGBM END #####')


import re
from janome.tokenizer import Tokenizer

with open("before_jp_message.txt", mode="r", encoding="utf-8") as f:
    message_original = f.read()

message = re.sub("[｜ 　「」\n]", "", message_original) # remove 2-byte space, 1-byte space, and return char

separator = "。"  # Japanese sentence separator

message_list = message.split(separator)
message_list.pop()
message_list = [x+separator for x in message_list]

t = Tokenizer()

message_words = []

for sentence in message_list:
    message_words.append(t.tokenize(sentence, wakati=True))

print(message_words)

all_words = ''

for sentence in message_words:
    for words in sentence:
        all_words = all_words + " " + words

with open('after_jp_message.txt', mode='w') as f:
    f.write(all_words)


#
# insert.py
#
# by - sam nnodim (son2105)
#
# adapted from: https://github.com/mikedewar/RealTimeStorytelling/blob/master/2/insert.py

# import some useful libs
import json
import sys
import redis

# Connect a redis database
conn = redis.Redis()

while True:
    # read a line from stdin
    line = sys.stdin.readline()

    # parse the data as a json obj.
    data = json.loads( line )
    distance = data["stormDist"]
    delta = data["delta"]

    # Add object to database w/ an expiration date
    # This is so that we can keep a window of measurements
    # For which to calculate rates
    conn.setex(distance, delta, 120)

    # Print the data to stdout so it can be
    # resued in the pipeline
    print ( json.dumps({"stormDist":distance, "delta":delta}) )
    sys.stdout.flush()


# -*- coding: utf-8 -*-
"""
Routine to detect last date a transmitter reported
originally coded by JiM in July 2016
Note: It reads output of "getap2s.py" which is /net/pubweb_html/drifter/emolt.dat
Note: In May 2017, I added emolt_ap3.dat
Note: In May 2018, I started adding "day last fished"
Note: In Jan 2019, I added DOPPIO (currently subtracts 4 days to pick up file needed)
Note: In Feb 2019, I made a temp file "lastfix_temp.html" until routine was complete
Note: In Mar 2019, I added GOMOFS and fixed "getlastwifi" and revised gomofs_modules method
Note: In Mar 2019, I added "mindistfromharbor" to avoid request from dock to appear in lastfix website
Note: In Apr 2019, I added a) a check on depth compared to NGDC depth and b) a fix to check harborlist
Note: In May 2019, I fixed the getlastwifi function
Note: In Aug 2020, I started implementing the "datatables" sort function
Note: In Jan 2021, I moved to my new laptop at home, made ready for Python 3,
"""

from datetime import datetime as dt
from datetime import timedelta
from dateutil import parser
import pandas as pd
import pytz
import glob
import numpy as np
import netCDF4
import os
from conversions import c2f,dd2dm # converts celcius to fahrenheight and decimal degrees to degrees-minutes format
#import zlconversions as zl # Lei Zhao's conversion routines
#import gomofs_modules_zl2 as gomofs_modules # lei Zhao's GOMOFS functions
import gomofs_modules # lei Zhao's final quick version GOMOFS functions
import doppio_modules
import warnings
from pandas.core.common import SettingWithCopyWarning
warnings.simplefilter(action="ignore", category=SettingWithCopyWarning)
warnings.simplefilter(action="ignore", category=RuntimeWarning)
#
# HARDCODES:
maxdiff=10. # maximum model error to report
mindistfromharbor=0.4 # mimimum distance in miles from harbor to be considered real
mindist_allowed=0.4 # minimum distance from nearest NGDC depth in km

# function to get vessel_name and probe serial number given vessel_number
def getclim(lat1=43.,lon1=-73.0,yrday=dt.now().strftime('%j').lstrip('0'),var='Bottom_Temperature/BT_'):
  # gets climatology of Bottom_Temperature  (note:Surface_Temperature, Bottom_Salinity, or Surface_Salinity also available)
  # as calculated by Chris Melrose from 30+ years of NEFSC CTD data on the NE Shelf
  # where "lat1", "lon1", and "yrday" are the position and yearday of interest
  # where "var" is the variable of interest (usually Bottom_Temperature)
  inputdir='/net/data5/jmanning/clim/' # hardcoded climatology directory name "/home/pi/clim/"
  dflat=pd.read_csv(inputdir+'LatGrid.csv',header=None)
  dflon=pd.read_csv(inputdir+'LonGrid.csv',header=None)
  lat=np.array(dflat[0])   # gets the first col (35 to 45)
  lon=np.array(dflon.ix[0])# gets the first row (-75 to -65)
  clim=pd.read_csv(inputdir+var+str(int(yrday))+'.csv',header=None) # gets bottom temp for this day of year
  idlat = np.abs(lat - lat1).argmin() # finds the neareast lat to input lat1
  idlon = np.abs(lon - lon1).argmin() # finds the neareast lon to input lon1
  return clim[idlon][idlat]

def getvesselname(vessel_number):
  #df=pd.read_csv('/net/data5/jmanning/drift/codes_temp.dat',sep='\s+',header=None)
  df=pd.read_csv('codes_temp.dat',sep='\s+',header=None)
  vn=df[df[0]==vessel_number][3].values[0] #
  sn=df[df[0]==vessel_number][4].values[-1]# gets the last one
  form=df[df[0]==vessel_number][5].values[-1]# fixed or mobile as "f" or "m"
  #sn=sn[2:4]# just use the last two characters
  return vn,sn,form

def getlastwifi(sn):
  #if (sn[0:2]=='7a') or (sn[0:2]=='4d'): # case of Lowell Instruments
  if (sn[0]!='1'): # case of Lowell Instruments where serial numbers starting with "1" are aquatec (change made 3/13/19)
    #filenames=glob.glob('/net/data5/jmanning/li/Matdata/temporary/li_*.csv')
    #filenames=glob.glob('/home/jmanning/py/aq_main/aqmain_and_raw_check/raw_data_check/result/Matdata/li_*.csv') # change made after lei Zhao's method was implemented in Aug 2019
    filenames=glob.glob('li_*.csv') # change made after lei Zhao's method was implemented in Aug 2019
    #mrf=[s for s in filenames if "li_"+sn[2:4] in s]
    #mrf=[s for s in filenames if "li_"+sn in s] # change made 3/13/2019
    #mrf=[s for s in filenames if "li_"+sn[-5:-1] in s] # change made 4/9/2019 to account for some new sn having 6 characters but only four in filenames
    mrf=[s for s in filenames if "li_"+sn[-4:] in s] # change made 5/16/2019
  else: # case of Aquatec
    filenames=glob.glob('/home/jmanning/py/api/aqu_data/Logger_sn_*.csv')
    mrf=[s[-19:] for s in filenames if "Logger_sn_"+sn in s]
    #print sn
  if len(mrf)==0:# empty string
    last_wifi_date='never'
  else:
    mrf=max(mrf)
    #print mrf
    try:
      mrf=mrf.split('Matdata/')[1] # this gets the last file according to date
    except:
      pass

    if sn[0]!='1': # case of Lowell Instruments
      i=[x for x, v in enumerate(mrf) if v == '_'][1]# finds index of 2nd underscore in the filename

      last_wifi_date=dt(int(mrf[i+1:i+5]),int(mrf[i+5:i+7]),int(mrf[i+7:i+9]),0,0,0).strftime("%b %d,%Y")
    else: # aquatec case
      #print mrf
      last_wifi_date=dt(int(mrf[0:4]),int(mrf[4:6]),int(mrf[6:8]),0,0,0).strftime("%b %d,%Y")
  return last_wifi_date

def getlastfished(vn,mindistfromharbor):
   # function to determine when the last date the vessel "reported" away from the dock
   # assuming the dock position is entered in "harborlist.txt" file
   #df=pd.read_csv('/net/pubweb_html/drifter/emolt_ap3_reports.dat',sep=',',header=None) # this file is created during the "getap3.py" run
   #df=pd.read_csv('emolt_ap3_reports.dat',sep=',',header=None) # this file is created during the "getap3.py" run
   df=pd.read_csv('http://nefsc.noaa.gov/drifter/emolt_ap3_reports.dat',sep=',',header=None) # this file is created during the "getap3.py" run
   df1=df[df[0]==vn] # this assumes the vessel names in "codes_temp.dat" are exactly the same as those on the AssetLink site but with the "F/V " prefix removed
   if len(df1)>0:
     yorn=[] # yes  or no near a harbor
     for k in range(len(df1)): # get positions away from the dock
        [la,lo]=dd2dm(list(df1[2])[k],list(df1[3])[k])# converts to ddmm format
        yorn.append(gps_compare_JiM(la,lo,mindistfromharbor))
     df1[4]=yorn # this is a list of yes and no's
     df2=df1[df1[4]=='no'] # forms a new dataframe with only those positions away from the dock
     list_of_dates=list(df2[1])
     if len(list_of_dates)==0:
        last_fished_date='unknown'
     else:
        last_fished_date=parser.parse(max(list_of_dates)).strftime("%b %d,%Y") # last date away from the dock
   else:
     last_fished_date='unknown'
   if vn=='Lady_Jane':
     last_fished_date='retired'
   return last_fished_date# return the last date the vessel fished

def gps_compare_JiM(lat,lon,mindistfromharbor): #check to see if the boat is in the harbor derived from Huanxin's "wifipc.py" functions
    # function returns yes if this position is with "mindistfromharbor" miles of a dock
    #file='/home/jmanning/py/harborlist.txt' # has header line lat, lon, harbor
    file='harborlist.txt' # has header line lat, lon, harbor
    df=pd.read_csv(file,sep=',')
    indice_lat=[i for i ,v in enumerate(abs(np.array(df['lat'])-lat)<mindistfromharbor) if v]
    indice_lon=[i for i ,v in enumerate(abs(np.array(df['lon'])-lon)<mindistfromharbor) if v]
    harbor_point_list=[i for i, j in zip(indice_lat,indice_lon) if i==j]
    if len(harbor_point_list)>0:
       near_harbor='yes'
    else:
       near_harbor='no'
    return near_harbor #yes or no

def get_depth(loni,lati,mindist_allowed):
    # routine to get depth (meters) using vol1 from NGDC
    if lati>=40.:
        url='https://www.ngdc.noaa.gov/thredds/dodsC/crm/crm_vol1.nc'
    else:
        url='https://www.ngdc.noaa.gov/thredds/dodsC/crm/crm_vol2.nc'
    nc = netCDF4.Dataset(url).variables
    lon=nc['x'][:]
    lat=nc['y'][:]
    xi,yi,min_dist= nearlonlat_zl(lon,lat,loni,lati)
    if min_dist>mindist_allowed:
      depth=np.nan
    else:
      depth=nc['z'][yi,xi]
    return depth#,min_dist

def nearlonlat_zl(lon,lat,lonp,latp): # needed for the next function get_FVCOM_bottom_temp
    """
    used in "get_depth"
    """
    # approximation for small distance
    cp=np.cos(latp*np.pi/180.)
    dx=(lon-lonp)*cp
    dy=lat-latp
    xi=np.argmin(abs(dx))
    yi=np.argmin(abs(dy))
    min_dist=111*np.sqrt(dx[xi]**2+dy[yi]**2)
    return xi,yi,min_dist

def nearlonlat(lon,lat,lonp,latp): # needed for the next function get_FVCOM_bottom_temp
    """
    i=nearlonlat(lon,lat,lonp,latp) change
    find the closest node in the array (lon,lat) to a point (lonp,latp)
    input:
        lon,lat - np.arrays of the grid nodes, spherical coordinates, degrees
        lonp,latp - point on a sphere
        output:
            i - index of the closest node
            For coordinates on a plane use function nearxy
            Vitalii Sheremet, FATE Project
    """
    cp=np.cos(latp*np.pi/180.)
    # approximation for small distance
    dx=(lon-lonp)*cp
    dy=lat-latp
    dist2=dx*dx+dy*dy
    i=np.argmin(dist2)
    return i

def get_FVCOM_bottom_temp(lati,loni,dtime,layer): # gets modeled temp using GOM3 forecast
        '''
        Taken primarily from Rich's blog at: http://rsignell-usgs.github.io/blog/blog/2014/01/08/fvcom/ on July 30, 2018
        where lati and loni are the position of interest, dtime is the datetime, and layer is "-1" for bottom
        '''
        #urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/fvcom/hindcasts/30yr_gom3'
        #urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_FVCOM_OCEAN_MASSBAY_FORECAST.nc'
        urlfvcom = 'http://www.smast.umassd.edu:8080/thredds/dodsC/FVCOM/NECOFS/Forecasts/NECOFS_GOM3_FORECAST.nc'
        nc = netCDF4.Dataset(urlfvcom).variables
        #first find the index of the grid
        lat = nc['lat'][:]
        lon = nc['lon'][:]
        inode = nearlonlat(lon,lat,loni,lati)
        #second find the index of time
        time_var = nc['time']
        itime = netCDF4.date2index(dtime,time_var,select='nearest')# where startime in datetime
        if itime==0:
            print('time is before fvcom forecast')
            return np.nan
        else:
            return nc['temp'][itime,layer,inode]

#MAIN PROGRAM

# open output html file and write header line
outfile=open('lastfix_temp.html','w')
#outfile=open('/net/pubweb_html/drifter/lastfix_temp.html','w')
#outfile=open('/net/pubweb_html/drifter/lastfix_temp4.html','w')
#outfile=open('/net/newfish_www/html/nefsc/emolt/lastfix_temp.html','w')

outfile.write('\n')
outfile.write('\n')
outfile.write('\n')
outfile.write('<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.10.21/css/jquery.dataTables.min.css">\n')
outfile.write('\n')

outfile.write('
commercial fishing vessels telemetering realtime bottom temperatures from the Northeast Shelf
\n')
outfile.write('
\n')
outfile.write('

vessel	date_last_telemetered	last_obs (degF)	FVCOM	DOPPIO	GOMFS (degF)	Climatology	#Transmitted	probe SN	last_wifi_upload	last_fished

\n')
outfile.write('
\n')

# read input file assuming that emolt_ap3.dat has been appended to this at 10, 40 minutes after even hour
df=pd.read_csv('http://emolt.org/emoltdata/emolt.dat',sep='\s+',header=None)
#df=pd.read_csv('https://nefsc.noaa.gov/drifter/emolt.dat',sep='\s+',header=None)
#df=pd.read_csv('https://apps-nefsc.fisheries.noaa.gov/drifter/emolt.dat',sep='\s+',header=None)           # assetlink results and assumes rockblock hauls already appended to AL by "getap3.py" 8/19/2020

# create a datetime for all fixes
datet=[]
for k in range(len(df)): # for each fix
  datet.append(dt(df[16][k],df[2][k],df[3][k],df[4][k],df[5][k]))
df['datet']=datet

# for each vessel find the last datetime and write to file
ves=list(set(df[0])) # list of unique vessels numbers

sumnum=0
anomaly,moderror,moderror_doppio,moderror_gomofs=[],[],[],[]# intitialize a few variables for appending later
#for j in range(len(ves)): # for each vessel
for j in [0]:
  print(ves[j])
  df1=df[df[0]==ves[j]]# gets all the records for this one vessel in new dataframe "df1"
  # added the following 6 lines 29 Mar 2019
  yorn=[] # yes  or no near a harbor
  for k in range(len(df1)): # get positions away from the dock
        #[la,lo]=dd2dm(list(df1[2])[k],list(df1[3])[k])# converts to ddmm format
        [la,lo]=dd2dm(list(df1[8])[k],list(df1[7])[k])# converts to ddmm format change indexes 4/9/2019
        yorn.append(gps_compare_JiM(la,lo,mindistfromharbor))
  df1[4]=yorn # this is a list of yes and no's
  df1=df1[df1[4]=='no'] # forms a new dataframe with only those positions away from the dock
  ####
  if len(df1)>0:# note in some cases the vessel may have only reported temps from the dock
    [vn,sn,form]=getvesselname(ves[j])#gets vessel name, probe serial number, and fixed_or_mobile for this vessel number
    print(vn)
    last_wifi_date=getlastwifi(sn)# gets date of last wifi for this serieal number
    last_fishing_date=getlastfished(vn,mindistfromharbor)
    obst=str(round(c2f(float(df1[14][-1:].values[0]))[0],1)) # last observation for this vessel
    #climt=str(round(c2f(getclim(df1[8][-1:].values[0],df1[7][-1:].values[0],str(int(max(df1.datet).to_pydatetime().strftime('%j')))))[0],1))# climatology for this day based on Melrose
    climt='0.0'
    print('looking for NGDC depth')
    depth_ngdc=get_depth(df1[7][-1:].values[0],df1[8][-1:].values[0],mindist_allowed)# ngdc bottom depth estimate
    if abs(df1[11][-1:].values[0]-depth_ngdc)/depth_ngdc>0.15:# big difference in observed and NGDC depth
      obst=obst+'*'
    #get FVCOM
    print('looking for FVCOM')
    try:
        # JiM modified the "str(round(" prefix on 7/8/2020
        modt=str(round(c2f(get_FVCOM_bottom_temp(df1[8][-1:].values[0],df1[7][-1:].values[0],max(df1.datet).to_pydatetime(),-1))[0],1))
        if abs(float(obst)-float(modt))>maxdiff: #something must be wrong
          modt='nan'
    except:
        print('FVCOM is down?')
        modt='nan' # the FVCOM must be down
    # get DOPPIO
    if max(df1.datet).to_pydatetime()>=dt(2017,11,1,0,0,0):
      print('looking for DOPPIO')
      try:
        modt_doppio=str(round(c2f(doppio_modules.get_doppio(lat=df1[8][-1:].values[0],lon=df1[7][-1:].values[0],depth='bottom',time=max(df1.datet).to_pydatetime()-timedelta(days=4),fortype='temperature'))[0],1))
      except:
        modt_doppio=np.nan
      print('looking for GOMOFS')
      try:
        modt_gomofs=str(round(c2f(gomofs_modules.get_gomofs(max(df1.datet).to_pydatetime()-timedelta(days=0),df1[8][-1:].values[0],df1[7][-1:].values[0],'bottom'))[0],1))
      except:
        modt_gomofs=np.nan
    if (last_fishing_date!='unknown') and (last_fishing_date!='retired'):
      if form=='f':
        days_to_worry=10 # fixed gear case because they do not always haul the trap each trip
      else:
        days_to_worry=1 # mobile case
      if max(df1.datet).to_pydatetime()<parser.parse(last_fishing_date)-timedelta(days=days_to_worry): # makes sure there are at least two days difference
         add_color=' class=redtext'
      else:
         add_color=''
    else:
      add_color=''
    outfile.write('
'+vn+'	'+''+max(df1.datet).strftime("%Y-%m-%d")+'	'+obst+'	'+modt+'	'+str(modt_doppio)+'	'+str(modt_gomofs)+'	'+climt+'	'+str(len(df1))+'	'+sn+'	'+last_wifi_date+'	'+last_fishing_date+'
\n')
    #outfile.write('
'+vn+'	'+''+max(df1.datet).strftime("%b %d,%Y")+'	'+obst+'	'+modt+'	'+modt_doppio+'	coming soon	'+climt+'	'+str(len(df1))+'	'+sn+'	'+last_wifi_date+'	'+last_fishing_date+'
\n')

    sumnum=sumnum+len(df1)
    if ~np.isnan(float(climt)):
       anomaly.append(float(obst)-float(climt))
    if ~np.isnan(float(modt)):
       moderror.append(float(obst)-float(modt))
    if ~np.isnan(float(modt_doppio)):
       moderror_doppio.append(float(obst)-float(modt_doppio))
    if ~np.isnan(float(modt_gomofs)):
       moderror_gomofs.append(float(obst)-float(modt_gomofs))

outfile.write('
TOTAL							'+str(sumnum)+'
\n')
outfile.write('

')
outfile.write('

 * indicates maximum observed depth is more than 15% different from what the bottom depth is at that location (according to NGDC)')
outfile.write('

 Most recent mean observed minus Melrose-30-year-climatology = '+str(round(np.mean(anomaly),1))+' degF')
if str(round(np.mean(moderror),1))=='nan':
  outfile.write('

 Note: FVCOM model is evidently down.')
else:
  outfile.write('

 Most recent mean observed minus FVCOM GOM3 forecast model = '+str(round(np.mean(moderror),1))+' degF')
outfile.write('

 Most recent mean observed minus DOPPIO forecast model = '+str(round(np.mean(moderror_doppio),1))+' degF')
outfile.write('

 Most recent mean observed minus GOMOFS forecast model = '+str(round(np.mean(moderror_gomofs),1))+' degF')
outfile.write('

 Note: Vessels highlighted in red may require some attention.')
outfile.write('

 Note: To see a map of the last month of observations, click here.')
outfile.close()
#os.system('cp /net/pubweb_html/drifter/lastfix_temp.html /net/pubweb_html/drifter/lastfix.html')# changed from "mv" to "cp" on 4/9/2019
#os.system('cp /net/newfish_www/html/nefsc/emolt/lastfix_temp.html /net/newfish_www/html/nefsc/emolt/lastfix.html')# changed from "mv" to "cp" on 4/9/2019