###############################################################################
###
### p1_data
### This file is part of CorePressure
### This file was created by Dr Daniel Parker
### includes a set of fuctions to process walking data from pedar
###
### Copyright (C) 2018 University of Salford - All Rights Reserved
### You may use, distribute and modify this code under the terms of MIT Licence
### See <filename> or go to <url> for full licence details
###
###############################################################################

import os
import sys
import importlib
import itertools
import numpy as np
import pandas as pd

import local

from matplotlib import pyplot	 as plt
import pedar_walk; importlib.reload(pedar_walk)
from pedar_walk import PedarWalkData as pdat
import AProcessing as ap; importlib.reload(ap)

def closeall(): # this closes all figures in reverse order
        l = plt.get_fignums()
        l.reverse()
        for a in l:
            plt.close(a)

def join_l(l, sep):
    out_str = ''
    for el in l:
        out_str += str(el) + str(sep)
    return out_str[:-1]

def grp_by_prefix(filelist,seperator='_',level=0):
    pref = []
    used = set()
    for file in filelist:
        pref.append(file.split(seperator)[level])
    unique = [x for x in pref if x not in used and (used.add(x) or True)]
    groups = []
    for i in range(0,len(unique)):
        grp = []
        for j in range(0,len(pref)):
            if pref[j] == unique[i]:
                grp.append(filelist[j])
        groups.append(grp)
    return(groups)

def clean_data(data,fname,num):
    # plt.figure()
    # plt.subplot(411)
    # plt.plot(data.max(axis=1),color='r')
    # plt.plot(data.mean(axis=1),color='b')
    #
    for col in data.columns:
        # plt.subplot(412)
        # plt.plot(data[col])
        if data[col].max() < 10: # Sensors below low limit
            # plt.subplot(413)
            # plt.plot(data[col])
            data[col] = 0
        elif data[col].max() < data[col].mean()*1.8: # Sensors with constant load
            if data[col].max() < 100:
                # plt.subplot(413)
                # plt.plot(data[col])
                data[col] = 0
    # plt.subplot(414)
    # plt.plot(data.max(axis=1),color='r')
    # plt.plot(data.mean(axis=1),color='b')

    # plt.savefig('C:/Temp/CleanPlots/'+figname+'.png')
    return(data)

def find_blocks(dataset,figname):
        """
        takes 2d cyclic dataset and chunks file to blocks
        returns start and end locations of each block
        """

        Condit = max(dataset) *0.75
        PPeaks = ap.PeakDetect(dataset,span=30,condit=Condit)
        MCondit = 0.5*Condit
        PMins = ap.MinDetect(dataset,span=30,condit=MCondit)

        plt.figure()

        ax = plt.subplot(211)
        ax.plot(dataset)
        ax.plot(PPeaks[0],PPeaks[1],'*')
        ax.plot(PMins[0],PMins[1],'*')

        ### Zero between cycles:
        for Pmin in PMins[1]:
            dataset[dataset == Pmin] = 0

        ### Ave gap:
        gap = 0
        for j in range(1, len(PPeaks[0])):
            gap = gap + (PPeaks[0][j]-PPeaks[0][j-1])
        gap = gap/len(PPeaks[0])

        ### Zero start:
        dataset[:PPeaks[0][0]] = 0
        for j in range(1, len(PPeaks[0])):
            if PPeaks[0][j] - PPeaks[0][j-1] > (gap*1.3):
                if j < 5:
                    dataset[:PPeaks[0][j]] = 0
                else:
                    break
            elif j > 5:
                if PPeaks[0][j] - PPeaks[0][j-1] < gap:
                    break

        ### Zero end:
        dataset[PPeaks[0][-1]:] = 0

        ### Zero between blocks:
        step = [0]
        for j in range(1, len(PPeaks[0])):
            if PPeaks[0][j] - PPeaks[0][j-1] > (gap*1.45): ### ADJUSTABLE LEVEL
                dataset[PPeaks[0][j-1]:PPeaks[0][j]] = 0
                step.append(PPeaks[0][j])
        step.append(len(dataset))

        ## Remove small blocks:
        for j in range(1,len(step)):
            nsteps = PPeaks[0][(PPeaks[0]>step[j-1]) & (PPeaks[0]<step[j])],
            if len(nsteps[0]) < 4:
                dataset[step[j-1]:step[j]] = 0
        ax.plot(step,[0]*len(step),'*',color='r')

        ax = plt.subplot(212)
        ax.plot(dataset)

        count= 0
        first= 0
        start = []

        for j in range(0,len(dataset)):
            count+=1
            if dataset[j] != 0:
                if count > 50:
                    start.append(j)
                    count=0
                else:
                    first+=1
                    count=0
                    if first == 1:
                        start.append(j)

        level = [50]*len(start)
        ax.plot(start,level,'*',color='g')

        end = []

        for st in start:
            count = 0
            for j in range(st,len(dataset)):
                count+=1
                if dataset[j] == 0:
                    if count > 50:
                        end.append(j)
                        break
                else:
                    count=0
        if len(end) < len(start):
            end.append(len(dataset))

        level = [50]*len(start)

        ax.plot(end,level,'*',color='r')
        plt.savefig('C:/Temp/BlockPlots/'+figname+'.png')
        return (start,end)

def data_blocks(data,figname,good_walks):
        """
        Cleans and stacks blocks of data for each insole
        """

        # Setup Mean Cycles and find blocks
        cyc_finder = pd.DataFrame()
        for col in data:
            if  data[col].max() > 100:
                cyc_finder[col] = insole_df[col]
        cyc_filt = cyc_finder.mean(axis=1)
        start,end = find_blocks(cyc_filt,figname)

        plt.figure()

        ax = plt.subplot(211)
        ax.plot(data.mean(axis=1))
        plt.savefig('C:/Temp/BlockCutPlots/'+figname+'.png')
        walk_set = []

        if len(start) > 0:
            # Cut raw data to blocks
            # walks = []
            for j in range(1,len(start)):
                walk = data.iloc[start[j]:end[j],:]
                steps_data = walk.mean(axis=1)

                Condit = steps_data.max()*0.4
                stp = ap.PeakDetect(steps_data,span=30,condit=Condit)
                PMins = ap.MinDetect(steps_data,span=45,condit=15)

                for Pmin in PMins[1]:
                    steps_data[steps_data == Pmin] = 0

                if len(stp[0]) >= 3:
                    ax.plot(steps_data,color='y')
                    ax.plot(stp[0]+start[j],stp[1],'*',color='k')
                    ax.plot(PMins[0]+start[j],PMins[1],'*',color='b')
                    plt.savefig('C:/Temp/BlockCutPlots/'+figname+'.png')
                    part_a = steps_data.iloc[:stp[0][1]][::-1]
                    trip_a = np.argwhere(part_a==0)[0]
                    cut_a = int(stp[0][1] - trip_a + start[j])
                    ax.plot(cut_a,0,'*',color='r')
                    plt.savefig('C:/Temp/BlockCutPlots/'+figname+'.png')
                    part_b = steps_data.iloc[stp[0][-3]:]
                    trip_b = np.argwhere(part_b==0)[0]
                    cut_b = int(stp[0][-3]+trip_b+start[j])
                    ax.plot(cut_b,0,'*',color='r')
                    plt.savefig('C:/Temp/BlockCutPlots/'+figname+'.png')
                    trim = data.iloc[cut_a:cut_b,:]
                    walktrim = trim.mean(axis=1)
                    ax.plot(walktrim,color='r')
                    walk_set.append(trim)
                    plt.savefig('C:/Temp/BlockCutPlots/'+figname+'.png')

        if len(walk_set) != len(good_walks):
            print('Walks :' + str(len(walk_set)))
            print('GoodWalks :' + str(len(good_walks)))

        toproc = []
        for j in range(len(good_walks)):
            if good_walks[j] == 1:
                ax.plot(walk_set[j].mean(axis=1),color='g')
                toproc.append(walk_set[j])

        if len(toproc) > 1:
            ins = toproc[0].append(toproc[1:])
            ins = ins.reset_index(drop=True)
        else:
            print('NUMBER OF WALKS IS LOW')

        ax = plt.subplot(212)
        ax.plot(ins.mean(axis=1))

        plt.savefig('C:/Temp/BlockCutPlots/'+figname+'.png')

        return(ins)

def cut_to_steps(single_insole,figname):
        """
        Cuts clean/stacked data to individual cycles or steps for a given walk
        input is dataframe for a single insole only.

        returns:
            Array containing individual steps each as a dataframe

        """

        SMean = single_insole.mean(axis=1)

        Condit = max(SMean)*0.5
        CPeaks = ap.PeakDetect(SMean,span=20,condit=Condit)
        # MCondit = 10
        # CMins = ap.MinDetect(SMean,span=10,condit=MCondit)

        ### NEED MINIMUM SETUP
        CM = [np.argmin(SMean[:CPeaks[0][0]])]
        for i in range(1,len(CPeaks[0])):
            CM.append(np.argmin(SMean[CPeaks[0][i-1]:CPeaks[0][i]]))
        CM.append(np.argmin(SMean[CPeaks[0][-1]:]))

        Mins = np.zeros(len(CM))
        CMins = [CM,Mins]

        plt.figure()
        ax = plt.subplot(211)
        ax.plot(SMean)
        ax.plot(CPeaks[0],CPeaks[1],'*')
        ax.plot(CMins[0],CMins[1],'*')

        steps = []
        ax = plt.subplot(212)

        for i in range(len(CPeaks[0])):

            peak = CPeaks[0][i]
            min_b_peak = max([x for x in CMins[0] if x <= peak])
            min_a_peak = min([x for x in CMins[0] if x >= peak])

            Cycle = single_insole.iloc[min_b_peak:min_a_peak,:]
            Cycle = Cycle.reset_index(drop=True)
            plt.plot(Cycle)
            steps.append(Cycle)

        plt.savefig('C:/Temp/CycPlots/'+figname+'.png')

        return(steps)

def peak_pressure(steps):
        """
        Gives peak pressure variables from pedar insole

        input is array containing individual steps

        returns:
            Array of peak pressure for each sensor for each cycle
            Array of highest overall peak pressure
            Array of average peak pressure across all steps
        """

        sensors = [str(x) for x in range(1,100)]
        peaks = pd.DataFrame(columns=sensors)
        # print(steps)

        # plt.figure()

        for i in range(len(steps)):
            tag = 'step_'+str(i+1)
            step_df = steps[i]
            # plt.plot(step_df.max())
            peaks.loc[tag] = step_df.max()

        # plt.savefig('C:/Temp/CycPlots/'+figname+'Peaks.png')

        return(peaks)

def bld_str(f):
    a = ''
    for i in f:
        a = a+str(i)+'_'
    return a[:-1]

###############################################################################
# Test Script
###############################################################################

closeall()
mypath = 'C:/Temp/ToRun/'
filelist = [os.path.join(r,file) for r,d,f in os.walk(mypath) for file in f
            if file.endswith('.asc')]

groups = grp_by_prefix(filelist,seperator='\\')

blocks = pd.read_excel('C:/Temp/Blocks.xlsx', sheet_name=None)
block_keys = list(blocks.keys())
for group in groups:
    grp_tags = os.path.splitext(os.path.basename(group[0]))[0].split('_')
    grp_tag = join_l(grp_tags[0:2],'_')
    grp_num = int(grp_tags[2])

    key = 'AM_P1 ('+str(grp_num)+')'
    ofile = 'C:/Temp/OutDir/' +key+'.xlsx'
    block = blocks[key]

    print('Processing - '+key)
    # fig = plt.figure(key+ '_StackBlock')
    num = 0
    grp_dat = {}

    for fname in group:
        num +=1
        ffname = bld_str(fname.split('.')[0].split('_')[-2:])
        g_w = block.loc[block['Condition'] == ffname,:]
        gws = list(g_w.values)[0][1:]

        good_walks = []
        for w in gws:
            try:
                good_walks.append(int(w))
            except:
                continue
        print(good_walks)
        data = pdat.openPD(fname)

        #Split to Left and Right Insole Raw Data:
        heads = list(data.head(0))
        Left = data.loc[:,heads[1]:heads[99]]
        try:
            Right = data.loc[:,heads[100]:heads[198]]

        except IndexError:
            Right = data.loc[:,heads[100]:]
        pair = {'Left':Left , 'Right':Right}

        sensors = [str(x) for x in range(1,100)]

        for insole in pair:
            insole_df = pd.DataFrame(data=pair[insole])
            insole_df.columns = sensors
            figname = os.path.splitext(fname)[0].split('\\')[-1] + '_' + insole
            print(figname)

            cdat = clean_data(insole_df,figname,num)
            walks = data_blocks(cdat,figname,good_walks)

            steps = cut_to_steps(walks,figname)
            print(len(steps))
            peaks = peak_pressure(steps)
            grp_dat[figname] = peaks
            peaks.to_csv('C:/Temp/OutDir/'+figname+'.txt',header=False,index=False,sep='\t')

    writer = pd.ExcelWriter(ofile)
    for dat in grp_dat:
        grp_dat[dat].to_excel(writer,dat)
    writer.save()

    # plt.tight_layout()
    # plt.savefig('C:/Temp/'+key+'.png')
    # plt.close()


import string
sentence = input()
sentence = sentence.lower()
letters = list(sentence)
alphabet = list(string.ascii_lowercase)

x = ([item for item in alphabet if item not in letters])
if len(x) == 0:
    print ('pangram')
else:
    print ('not pangram')


load("@npm//google-closure-compiler:index.bzl", "google_closure_compiler")

def closure_js_library(**kwargs):
    pass

def closure_js_binary(name, **kwargs):
  srcs = kwargs.pop("deps", []) + [
      "@com_google_javascript_closure_library//:closure/goog/base.js",
  ]
  entry_points = kwargs.pop("entry_points", [])
  args = [
      "--js=$(location %s)" % s for s in srcs
  ] + kwargs.pop("defs", [])
  if "compilation_level" in kwargs:
      args.append("--compilation_level=" + kwargs.pop("compilation_level"))
  dependency_mode = kwargs.pop("dependency_mode", "")
  language = kwargs.pop("language", "")
  output_wrapper = kwargs.pop("output_wrapper", "")

  google_closure_compiler(
      name = name,
      data = srcs,
      args = args + [
          "--js_output_file=$(location %s.compiled.js)" % name,
      ],
      outs = ["%s.compiled.js" % name],
      **kwargs
  )

def closure_js_binary_set(name, **kwargs):
  """Defines two closure_js_binary targets for name.

  The first, :name, builds with ADVANCED optimizations.
  The second, :name_simple, builds with SIMPLE optimizations and pretty printed code (for easier
  debugging).
  """
  closure_js_binary(
      name = name,
      compilation_level = "ADVANCED",
      **kwargs)

  defs = kwargs.get("defs", [])
  defs.append("--formatting=PRETTY_PRINT")

  closure_js_binary(
      name = name + "_simple",
      compilation_level = "SIMPLE",
      **kwargs)


import os
import sys
from io import BytesIO, IOBase

_str = str
str = lambda x=b"": x if type(x) is bytes else _str(x).encode()

BUFSIZE = 8192

class FastIO(IOBase):
    newlines = 0

    def __init__(self, file):
        self._buffer = BytesIO()
        self._fd = file.fileno()
        self._writable = "x" in file.mode or "r" not in file.mode
        self.write = self._buffer.write if self._writable else None

    def read(self):
        if self._buffer.tell():
            return self._buffer.read()
        return os.read(self._fd, os.fstat(self._fd).st_size)

    def readline(self):
        while self.newlines == 0:
            b = os.read(self._fd, max(os.fstat(self._fd).st_size, BUFSIZE))
            self.newlines = b.count(b"\n") + (not b)
            ptr = self._buffer.tell()
            self._buffer.seek(0, 2), self._buffer.write(b), self._buffer.seek(ptr)
        self.newlines -= 1
        return self._buffer.readline()

    def flush(self):
        if self._writable:
            os.write(self._fd, self._buffer.getvalue())
            self._buffer.truncate(0), self._buffer.seek(0)

def print(*args, sep=b" ", end=b"\n", file=sys.stdout, flush=False):
    at_start = True
    for x in args:
        if not at_start:
            file.write(sep)
        file.write(str(x))
        at_start = False
    file.write(end)
    if flush:
        file.flush()

sys.stdin, sys.stdout = FastIO(sys.stdin), FastIO(sys.stdout)
input = lambda: sys.stdin.readline().rstrip(b"\r\n")


# -*- coding: utf-8 -*-
"""Interface to Lightnet object proposals."""
import copy
import logging
import os
import random
import time
from os.path import expanduser, join

import cv2
import numpy as np
import PIL
import tqdm
import utool as ut

from wbia import constants as const

(print, rrr, profile) = ut.inject2(__name__, '[densenet]')
logger = logging.getLogger('wbia')

PARALLEL = not const.CONTAINERIZED
INPUT_SIZE = 224

ARCHIVE_URL_DICT = {
    'canonical_zebra_grevys_v1': 'https://wildbookiarepository.azureedge.net/models/classifier.canonical.zebra_grevys.v1.zip',
    'canonical_zebra_grevys_v2': 'https://wildbookiarepository.azureedge.net/models/classifier.canonical.zebra_grevys.v2.zip',
    'canonical_zebra_grevys_v3': 'https://wildbookiarepository.azureedge.net/models/classifier.canonical.zebra_grevys.v3.zip',
    'canonical_zebra_grevys_v4': 'https://wildbookiarepository.azureedge.net/models/classifier.canonical.zebra_grevys.v4.zip',
    'canonical_giraffe_reticulated_v1': 'https://wildbookiarepository.azureedge.net/models/classifier.canonical.giraffe_reticulated.v1.zip',
    'ryan_densenet_v1': 'https://wildbookiarepository.azureedge.net/models/classifier.cameratrap.ryan.densenet.v1.zip',
    'ryan_densenet_v2': 'https://wildbookiarepository.azureedge.net/models/classifier.cameratrap.ryan.densenet.v2.zip',
    'megan_argentina_v1': 'https://wildbookiarepository.azureedge.net/models/classifier.cameratrap.megan.argentina.densenet.v1.zip',
    'megan_kenya_v1': 'https://wildbookiarepository.azureedge.net/models/classifier.cameratrap.megan.kenya.densenet.v1.zip',
    'megan_kenya_v2': 'https://wildbookiarepository.azureedge.net/models/classifier.cameratrap.megan.kenya.densenet.v2.zip',
    'laterals_v0': 'https://wildbookiarepository.azureedge.net/models/classifier.cameratrap.laterals.densenet.v0.zip',
    'belly_v0': 'https://wildbookiarepository.azureedge.net/models/classifier.cameratrap.right_whale_belly.densenet.v0.zip',
    'zebra_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.zebra_grevys-zebra_plains.v1.zip',
    'zebra_mountain_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.zebra_mountain.v0.zip',
    'giraffe_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.giraffe.v1.zip',
    'jaguar_v3': 'https://wildbookiarepository.azureedge.net/models/labeler.jaguar.v3.zip',
    'lynx_v3': 'https://wildbookiarepository.azureedge.net/models/labeler.lynx.v3.zip',
    'manta_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.manta_ray_giant.v1.zip',
    'seaturtle_v3': 'https://wildbookiarepository.azureedge.net/models/labeler.seaturtle.v3.zip',
    'hendrik_dorsal_v2': 'https://wildbookiarepository.azureedge.net/models/labeler.hendrik_dorsal.v2.zip',
    'spotted_dolphin_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.dolphin_spotted.v0.zip',
    'spotted_skunk_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.skunk_spotted.v0.zip',
    'humpback_dorsal': 'https://wildbookiarepository.azureedge.net/models/labeler.whale_humpback.dorsal.v0.zip',
    'orca_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.whale_orca.v0.zip',
    'whale_sperm_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.whale_sperm.v0.zip',
    'fins_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.fins.v0.zip',
    'fins_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.fins.v1.1.zip',
    'fins_enforcement_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.fins.enforcement.v0.zip',
    'wilddog_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.wild_dog.v0.zip',
    'wilddog_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.wild_dog.v1.zip',
    'wilddog_v2': 'https://wildbookiarepository.azureedge.net/models/labeler.wild_dog.v2.zip',
    'wilddog_v3': 'https://wildbookiarepository.azureedge.net/models/labeler.wild_dog.v3.zip',
    'leopard_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.leopard.v0.zip',
    'cheetah_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.cheetah.v1.zip',
    'cheetah_v2': 'https://wildbookiarepository.azureedge.net/models/labeler.cheetah.v2.zip',
    'hyaena_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.hyaena.v0.zip',
    'wild_horse_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.wild_horse.v0.zip',
    'seadragon_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.seadragon.v0.zip',
    'seadragon_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.seadragon.v1.zip',
    'seadragon_v2': 'https://wildbookiarepository.azureedge.net/models/labeler.seadragon.v2.zip',
    'jaguar_v4': 'https://wildbookiarepository.azureedge.net/models/labeler.jaguar.v4.zip',
    'lynx_v4': 'https://wildbookiarepository.azureedge.net/models/labeler.lynx.v4.zip',
    'manta_v2': 'https://wildbookiarepository.azureedge.net/models/labeler.manta_ray_giant.v2.zip',
    'seaturtle_v4': 'https://wildbookiarepository.azureedge.net/models/labeler.seaturtle.v4.zip',
    'nassau_grouper_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.grouper_nassau.v0.zip',
    'nassau_grouper_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.grouper_nassau.v1.zip',
    'nassau_grouper_v2': 'https://wildbookiarepository.azureedge.net/models/labeler.grouper_nassau.v2.zip',
    'nassau_grouper_v3': 'https://wildbookiarepository.azureedge.net/models/labeler.grouper_nassau.v3.zip',
    'salanader_fire_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.salamander_fire.v0.zip',
    'salanader_fire_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.salamander_fire.v1.zip',
    'salamander_fire_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.salamander_fire.v0.zip',
    'salamander_fire_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.salamander_fire.v1.zip',
    'salamander_fire_v2': 'https://wildbookiarepository.azureedge.net/models/labeler.salamander_fire.v2.zip',
    'spotted_dolphin_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.dolphin_spotted.v1.zip',
    'spotted_skunk_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.skunk_spotted.v1.zip',
    'monk_seal_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.mediterranean_monk_seal.v0.zip',
    'iot_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.iot.v0.zip',
    'right_whale_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.rightwhale.v0.zip',
    'whale_shark_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.whale_shark.v0.zip',
    'flukebook_v1': 'https://wildbookiarepository.azureedge.net/models/classifier2.flukebook.v1.zip',
    'rightwhale_v5': 'https://wildbookiarepository.azureedge.net/models/labeler.rightwhale.v5.zip',
    'snow_leopard_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.snow_leopard.v0.zip',
    'grey_whale_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.whale_grey.v0.zip',
    'beluga_whale_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.whale_beluga.v0.zip',
    'beluga_whale_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.whale_beluga.v1.zip',
    'sea_turtle_v4': 'https://wildbookiarepository.azureedge.net/models/labeler.sea_turtle.v4.zip',
    'seals_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.seals.v0.zip',
    'seals_v1': 'https://wildbookiarepository.azureedge.net/models/labeler.seals.v1.zip',
    'lions_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.lions.v0.zip',
    'spotted_eagle_ray_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.spotted_eagle_ray.v0.zip',
    'scout-d3e8bf43-boost0': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.d3e8bf43.0.zip',
    'scout-d3e8bf43-boost1': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.d3e8bf43.1.zip',
    'scout-d3e8bf43-boost2': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.d3e8bf43.2.zip',
    'scout-d3e8bf43-boost3': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.d3e8bf43.3.zip',
    'scout-d3e8bf43-boost4': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.d3e8bf43.4.zip',
    'scout-d3e8bf43-boost5': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.d3e8bf43.5.zip',
    'scout-d3e8bf43-boost6': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.d3e8bf43.6.zip',
    'scout-d3e8bf43-boost7': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.d3e8bf43.7.zip',
    'scout-5fbfff26-boost0': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.5fbfff26.0.zip',
    'scout-5fbfff26-boost1': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.5fbfff26.1.zip',
    'scout-5fbfff26-boost2': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.5fbfff26.2.zip',
    'scout-5fbfff26-boost3': 'https://wildbookiarepository.azureedge.net/models/classifier2.scout.5fbfff26.3.zip',
    'whale_fin_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.whale_fin.v0.zip',
    'leopard_shark_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.leopard_shark.v10.zip',
    'tigershark_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.tigershark.v0.zip',
    'snail_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.snail.v0.zip',
    'deer_v0': 'https://wildbookiarepository.azureedge.net/models/labeler.deer.v0.zip',

}

if not ut.get_argflag('--no-pytorch'):
    try:
        import torch
        import torch.nn as nn
        import torch.optim as optim
        import torchvision

        logger.info('PyTorch Version: {}'.format(torch.__version__))
        logger.info('Torchvision Version: {}'.format(torchvision.__version__))
    except ImportError:
        logger.info('WARNING Failed to import pytorch.  PyTorch is unavailable')
        if ut.SUPER_STRICT:
            raise

    try:
        import imgaug  # NOQA

        class Augmentations(object):
            def __call__(self, img):
                img = np.array(img)
                return self.aug.augment_image(img)

        class TrainAugmentations(Augmentations):
            def __init__(self, blur=True, flip=False, rotate=10, shear=10, **kwargs):
                from imgaug import augmenters as iaa

                sequence = []

                sequence += [
                    iaa.Scale((INPUT_SIZE, INPUT_SIZE)),
                    iaa.ContrastNormalization((0.75, 1.25)),
                    iaa.AddElementwise((-10, 10), per_channel=0.5),
                    iaa.AddToHueAndSaturation(value=(-20, 20), per_channel=True),
                    iaa.Multiply((0.75, 1.25)),
                ]
                sequence += [
                    iaa.PiecewiseAffine(scale=(0.0005, 0.005)),
                    iaa.Affine(
                        rotate=(-rotate, rotate), shear=(-shear, shear), mode='symmetric'
                    ),
                    iaa.Grayscale(alpha=(0.0, 0.5)),
                ]
                if flip:
                    sequence += [
                        iaa.Fliplr(0.5),
                    ]
                if blur:
                    sequence += [
                        iaa.Sometimes(0.01, iaa.GaussianBlur(sigma=(0, 1.0))),
                    ]
                self.aug = iaa.Sequential(sequence)

        class ValidAugmentations(Augmentations):
            def __init__(self, **kwargs):
                from imgaug import augmenters as iaa

                self.aug = iaa.Sequential([iaa.Scale((INPUT_SIZE, INPUT_SIZE))])

        AUGMENTATION = {
            'train': TrainAugmentations,
            'val': ValidAugmentations,
            'test': ValidAugmentations,
        }
    except ImportError:
        AUGMENTATION = {}
        logger.info(
            'WARNING Failed to import imgaug. '
            'install with pip install git+https://github.com/aleju/imgaug'
        )
        if ut.SUPER_STRICT:
            raise

def _init_transforms(**kwargs):
    TRANSFORMS = {
        phase: torchvision.transforms.Compose(
            [
                AUGMENTATION[phase](**kwargs),
                torchvision.transforms.Lambda(PIL.Image.fromarray),
                torchvision.transforms.ToTensor(),
                torchvision.transforms.Normalize(
                    [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
                ),
            ]
        )
        for phase in AUGMENTATION.keys()
    }

    return TRANSFORMS

class ImageFilePathList(torch.utils.data.Dataset):
    def __init__(self, filepaths, targets=None, transform=None, target_transform=None):
        from torchvision.datasets.folder import default_loader

        self.targets = targets is not None

        args = (filepaths, targets) if self.targets else (filepaths,)
        self.samples = list(zip(*args))

        if self.targets:
            self.classes = sorted(set(ut.take_column(self.targets, 1)))
            self.class_to_idx = {self.classes[i]: i for i in range(len(self.classes))}
        else:
            self.classes, self.class_to_idx = None, None

        self.loader = default_loader
        self.transform = transform
        self.target_transform = target_transform

    def __getitem__(self, index):
        """
        Args:
            index (int): Index
        Returns:
            tuple: (sample, target) where target is class_index of the target class.
        """
        sample = self.samples[index]

        if self.targets:
            path, target = sample
        else:
            path = sample[0]
            target = None

        sample = self.loader(path)

        if self.transform is not None:
            sample = self.transform(sample)

        if self.target_transform is not None:
            target = self.target_transform(target)

        result = (sample, target) if self.targets else (sample,)

        return result

    def __len__(self):
        return len(self.samples)

    def __repr__(self):
        fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
        fmt_str += '    Number of samples: {}\n'.format(self.__len__())
        tmp = '    Transforms (if any): '
        fmt_str += '{}{}\n'.format(
            tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))
        )
        tmp = '    Target Transforms (if any): '
        fmt_str += '{}{}'.format(
            tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))
        )
        return fmt_str

class StratifiedSampler(torch.utils.data.sampler.Sampler):
    def __init__(self, dataset, phase, multiplier=1.0):
        self.dataset = dataset
        self.phase = phase
        self.training = self.phase == 'train'

        self.labels = np.array(ut.take_column(dataset.samples, 1))
        self.classes = set(self.labels)

        self.indices = {
            cls: list(np.where(self.labels == cls)[0]) for cls in self.classes
        }
        self.counts = {cls: len(self.indices[cls]) for cls in self.classes}
        self.min = min(self.counts.values())
        self.min = int(np.around(multiplier * self.min))

        if self.training:
            self.total = 0
            for cls in self.indices:
                num_in_class = len(self.indices[cls])
                num_samples = min(self.min, num_in_class)
                self.total += num_samples
        else:
            self.total = len(self.labels)

        args = (
            self.phase,
            len(self.labels),
            len(self.classes),
            self.min,
            self.total,
            multiplier,
        )
        logger.info(
            'Initialized Sampler for %r (sampling %d for %d classes | min %d per class, %d total, %0.02f multiplier)'
            % args
        )

    def __iter__(self):
        if self.training:
            ret_list = []
            for cls in self.indices:
                num_in_class = len(self.indices[cls])
                num_samples = min(self.min, num_in_class)
                ret_list += random.sample(self.indices[cls], num_samples)
            random.shuffle(ret_list)
        else:
            ret_list = range(self.total)
        assert len(ret_list) == self.total
        return iter(ret_list)

    def __len__(self):
        return self.total

def finetune(model, dataloaders, criterion, optimizer, scheduler, device, num_epochs=128):
    phases = ['train', 'val']

    start = time.time()

    best_accuracy = 0.0
    best_model_state = copy.deepcopy(model.state_dict())

    last_loss = {}
    best_loss = {}

    for epoch in range(num_epochs):
        start_batch = time.time()

        lr = optimizer.param_groups[0]['lr']
        logger.info('Epoch {}/{} (lr = {:0.06f})'.format(epoch, num_epochs - 1, lr))
        logger.info('-' * 10)

        # Each epoch has a training and validation phase
        for phase in phases:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()  # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            seen = 0
            for inputs, labels in tqdm.tqdm(dataloaders[phase], desc=phase):
                inputs = inputs.to(device)
                labels = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    # Get model outputs and calculate loss
                    outputs = model(inputs)
                    loss = criterion(outputs, labels)

                    _, preds = torch.max(outputs, 1)

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                seen += len(inputs)
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

            epoch_loss = running_loss / seen
            epoch_acc = running_corrects.double() / seen

            last_loss[phase] = epoch_loss
            if phase not in best_loss:
                best_loss[phase] = np.inf

            flag = epoch_loss < best_loss[phase]
            if flag:
                best_loss[phase] = epoch_loss

            logger.info(
                '{:<5} Loss: {:.4f} Acc: {:.4f} {}'.format(
                    phase, epoch_loss, epoch_acc, '!' if flag else ''
                )
            )

            # deep copy the model
            if phase == 'val' and epoch_acc > best_accuracy:
                best_accuracy = epoch_acc
                logger.info('\tFound better model!')
                best_model_state = copy.deepcopy(model.state_dict())
            if phase == 'val':
                scheduler.step(epoch_loss)

                time_elapsed_batch = time.time() - start_batch
                logger.info(
                    'time: {:.0f}m {:.0f}s'.format(
                        time_elapsed_batch // 60, time_elapsed_batch % 60
                    )
                )

                ratio = last_loss['train'] / last_loss['val']
                logger.info('ratio: {:.04f}'.format(ratio))

        logger.info('\n')

    time_elapsed = time.time() - start
    logger.info(
        'Training complete in {:.0f}m {:.0f}s'.format(
            time_elapsed // 60, time_elapsed % 60
        )
    )
    logger.info('Best val Acc: {:4f}'.format(best_accuracy))

    # load best model weights
    model.load_state_dict(best_model_state)
    return model

def visualize_augmentations(dataset, augmentation, tag, num_per_class=10, **kwargs):
    import matplotlib.pyplot as plt

    samples = dataset.samples
    flags = np.array(ut.take_column(samples, 1))
    logger.info('Dataset %r has %d samples' % (tag, len(flags)))

    indices = []
    for flag in set(flags):
        index_list = list(np.where(flags == flag)[0])
        random.shuffle(index_list)
        indices += index_list[:num_per_class]

    samples = ut.take(samples, indices)
    paths = ut.take_column(samples, 0)
    flags = ut.take_column(samples, 1)

    images = [np.array(cv2.imread(path)) for path in paths]
    images = [image[:, :, ::-1] for image in images]

    images_ = []
    for image, flag in zip(images, flags):
        image_ = image.copy()
        color = (0, 255, 0) if flag else (255, 0, 0)
        cv2.rectangle(image_, (1, 1), (INPUT_SIZE - 1, INPUT_SIZE - 1), color, 3)
        images_.append(image_)
    canvas = np.hstack(images_)
    canvas_list = [canvas]

    augment = augmentation(**kwargs)
    for index in range(len(indices) - 1):
        logger.info(index)
        images_ = [augment(image.copy()) for image in images]
        canvas = np.hstack(images_)
        canvas_list.append(canvas)
    canvas = np.vstack(canvas_list)

    canvas_filepath = expanduser(
        join('~', 'Desktop', 'densenet-augmentation-{}.png'.format(tag))
    )
    plt.imsave(canvas_filepath, canvas)

def train(
    data_path,
    output_path,
    batch_size=48,
    class_weights={},
    multi=PARALLEL,
    sample_multiplier=4.0,
    allow_missing_validation_classes=False,
    **kwargs,
):
    # Detect if we have a GPU available
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    using_gpu = str(device) != 'cpu'

    phases = ['train', 'val']

    logger.info('Initializing Datasets and Dataloaders...')

    # Create training and validation datasets
    transforms = _init_transforms(**kwargs)
    datasets = {
        phase: torchvision.datasets.ImageFolder(
            os.path.join(data_path, phase), transforms[phase]
        )
        for phase in phases
    }

    # Create training and validation dataloaders
    dataloaders = {
        phase: torch.utils.data.DataLoader(
            datasets[phase],
            sampler=StratifiedSampler(
                datasets[phase], phase, multiplier=sample_multiplier
            ),
            batch_size=batch_size,
            num_workers=batch_size // 8,
            pin_memory=using_gpu,
        )
        for phase in phases
    }

    train_classes = datasets['train'].classes
    val_classes = datasets['val'].classes

    if not allow_missing_validation_classes:
        assert len(train_classes) == len(val_classes)
    num_classes = len(train_classes)

    logger.info('Initializing Model...')

    # Initialize the model for this run
    model = torchvision.models.densenet201(pretrained=True)
    num_ftrs = model.classifier.in_features
    model.classifier = nn.Linear(num_ftrs, num_classes)

    # Send the model to GPU
    model = model.to(device)

    # Multi-GPU
    if multi:
        logger.info('USING MULTI-GPU MODEL')
        model = nn.DataParallel(model)

    logger.info('Print Examples of Training Augmentation...')

    for phase in phases:
        visualize_augmentations(datasets[phase], AUGMENTATION[phase], phase, **kwargs)

    logger.info('Initializing Optimizer...')

    # logger.info('Params to learn:')
    params_to_update = []
    for name, param in model.named_parameters():
        if param.requires_grad:
            params_to_update.append(param)
            # logger.info('\t', name)

    # Observe that all parameters are being optimized
    optimizer = optim.SGD(params_to_update, lr=0.001, momentum=0.9)

    scheduler = optim.lr_scheduler.ReduceLROnPlateau(
        optimizer, 'min', factor=0.5, patience=10, min_lr=1e-6
    )

    # Get weights for the class
    class_index_list = list(dataloaders['train'].dataset.class_to_idx.items())
    index_class_list = [class_index[::-1] for class_index in class_index_list]
    weight = torch.tensor(
        [class_weights.get(class_, 1.0) for index, class_ in sorted(index_class_list)]
    )
    weight = weight.to(device)

    # Setup the loss fxn
    criterion = nn.CrossEntropyLoss(weight=weight)

    logger.info('Start Training...')

    # Train and evaluate
    model = finetune(model, dataloaders, criterion, optimizer, scheduler, device)

    ut.ensuredir(output_path)
    weights_path = os.path.join(output_path, 'classifier.densenet.weights')
    weights = {
        'state': copy.deepcopy(model.state_dict()),
        'classes': train_classes,
    }
    torch.save(weights, weights_path)

    return weights_path

def test_single(filepath_list, weights_path, batch_size=1792, multi=PARALLEL, **kwargs):

    # Detect if we have a GPU available
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    using_gpu = str(device) != 'cpu'

    logger.info('Initializing Datasets and Dataloaders...')

    # Create training and validation datasets
    transforms = _init_transforms(**kwargs)
    dataset = ImageFilePathList(filepath_list, transform=transforms['test'])

    # Create training and validation dataloaders
    dataloader = torch.utils.data.DataLoader(
        dataset, batch_size=batch_size, num_workers=0, pin_memory=using_gpu
    )

    logger.info('Initializing Model...')
    try:
        weights = torch.load(weights_path)
    except RuntimeError:
        weights = torch.load(weights_path, map_location='cpu')
    state = weights['state']
    classes = weights['classes']

    num_classes = len(classes)

    # Initialize the model for this run
    model = torchvision.models.densenet201()
    num_ftrs = model.classifier.in_features
    model.classifier = nn.Linear(num_ftrs, num_classes)

    # Convert any weights to non-parallel version
    from collections import OrderedDict

    new_state = OrderedDict()
    for k, v in state.items():
        k = k.replace('module.', '')
        new_state[k] = v

    # Load state without parallel
    model.load_state_dict(new_state)

    # Add softmax
    model.classifier = nn.Sequential(model.classifier, nn.LogSoftmax(), nn.Softmax())

    # Make parallel at end
    if multi:
        logger.info('USING MULTI-GPU MODEL')
        model = nn.DataParallel(model)

    # Send the model to GPU
    model = model.to(device)

    model.eval()

    start = time.time()

    counter = 0
    outputs = []
    for (inputs,) in tqdm.tqdm(dataloader, desc='test'):
        logger.info('Loading batch %d from disk' % (counter,))
        inputs = inputs.to(device)
        logger.info('Moving batch %d to GPU' % (counter,))
        with torch.set_grad_enabled(False):
            logger.info('Pre-model inference %d' % (counter,))
            output = model(inputs)
            logger.info('Post-model inference %d' % (counter,))
            outputs += output.tolist()
            logger.info('Outputs done %d' % (counter,))
        counter += 1

    time_elapsed = time.time() - start
    logger.info(
        'Testing complete in {:.0f}m {:.0f}s'.format(
            time_elapsed // 60, time_elapsed % 60
        )
    )

    result_list = []
    for output in outputs:
        result = dict(zip(classes, output))
        result_list.append(result)

    return result_list

def test_ensemble(
    filepath_list,
    weights_path_list,
    classifier_weight_filepath,
    ensemble_index,
    ibs=None,
    gid_list=None,
    multiclass=False,
    **kwargs,
):

    if ensemble_index is not None:
        assert 0 <= ensemble_index and ensemble_index < len(weights_path_list)
        weights_path_list = [weights_path_list[ensemble_index]]
        assert len(weights_path_list) > 0

    cached = False
    try:
        assert ensemble_index is None, 'Do not use depc on individual model computation'
        assert None not in [ibs, gid_list], 'Needs to have access to depc'
        assert len(filepath_list) == len(gid_list)

        results_list = []
        for model_index in range(len(weights_path_list)):
            if multiclass:
                classifier_two_weight_filepath_ = '%s:%d' % (
                    classifier_weight_filepath,
                    model_index,
                )
                config = {
                    'classifier_two_algo': 'densenet',
                    'classifier_two_weight_filepath': classifier_two_weight_filepath_,
                }
                scores_list = ibs.depc_image.get_property(
                    'classifier_two', gid_list, 'scores', config=config
                )
                result_list = []
                for score_dict in scores_list:
                    result = score_dict
                    result_list.append(result)
            else:
                classifier_weight_filepath_ = '%s:%d' % (
                    classifier_weight_filepath,
                    model_index,
                )
                config = {
                    'classifier_algo': 'densenet',
                    'classifier_weight_filepath': classifier_weight_filepath_,
                }
                prediction_list = ibs.depc_image.get_property(
                    'classifier', gid_list, 'class', config=config
                )
                confidence_list = ibs.depc_image.get_property(
                    'classifier', gid_list, 'score', config=config
                )
                result_list = []
                for prediction, confidence in zip(prediction_list, confidence_list):
                    # DO NOT REMOVE THIS ASSERT
                    assert prediction in {
                        'negative',
                        'positive',
                    }, 'Cannot use this method, need to implement classifier_two in depc'
                    if prediction == 'positive':
                        pscore = confidence
                        nscore = 1.0 - pscore
                    else:
                        nscore = confidence
                        pscore = 1.0 - nscore
                    result = {
                        'positive': pscore,
                        'negative': nscore,
                    }
                    result_list.append(result)
            assert len(result_list) == len(gid_list)
            results_list.append(result_list)
        assert len(results_list) == len(weights_path_list)
        cached = True
    except AssertionError:
        cached = False

    if not cached:
        # Use local implementation, due to error or not valid config
        results_list = []
        for weights_path in weights_path_list:
            result_list = test_single(filepath_list, weights_path)
            results_list.append(result_list)

    for result_list in zip(*results_list):
        merged = {}
        for result in result_list:
            for key in result:
                if key not in merged:
                    merged[key] = []
                merged[key].append(result[key])
        for key in merged:
            value_list = merged[key]
            merged[key] = sum(value_list) / len(value_list)

        yield merged

def test(
    gpath_list,
    classifier_weight_filepath=None,
    return_dict=False,
    multiclass=False,
    **kwargs,
):
    from wbia.detecttools.directory import Directory

    # Get correct weight if specified with shorthand
    archive_url = None

    ensemble_index = None
    if classifier_weight_filepath is not None and ':' in classifier_weight_filepath:
        assert classifier_weight_filepath.count(':') == 1
        classifier_weight_filepath, ensemble_index = classifier_weight_filepath.split(':')
        ensemble_index = int(ensemble_index)

    if classifier_weight_filepath in ARCHIVE_URL_DICT:
        archive_url = ARCHIVE_URL_DICT[classifier_weight_filepath]
        archive_path = ut.grab_file_url(archive_url, appname='wbia', check_hash=True)
    else:
        logger.info(
            'classifier_weight_filepath {!r} not recognized'.format(
                classifier_weight_filepath
            )
        )
        raise RuntimeError

    assert os.path.exists(archive_path)
    archive_path = ut.truepath(archive_path)

    ensemble_path = archive_path.strip('.zip')
    if not os.path.exists(ensemble_path):
        ut.unarchive_file(archive_path, output_dir=ensemble_path)

    assert os.path.exists(ensemble_path)
    direct = Directory(ensemble_path, include_file_extensions=['weights'], recursive=True)
    weights_path_list = direct.files()
    weights_path_list = sorted(weights_path_list)
    assert len(weights_path_list) > 0

    kwargs.pop('classifier_algo', None)

    logger.info(
        'Using weights in the ensemble, index %r: %s '
        % (ensemble_index, ut.repr3(weights_path_list))
    )
    result_list = test_ensemble(
        gpath_list,
        weights_path_list,
        classifier_weight_filepath,
        ensemble_index,
        multiclass=multiclass,
        **kwargs,
    )
    for result in result_list:
        best_key = None
        best_score = -1.0
        for key, score in result.items():
            if score > best_score:
                best_key = key
                best_score = score
        assert best_score >= 0.0 and best_key is not None
        if return_dict:
            yield best_score, best_key, result
        else:
            yield best_score, best_key

def test_dict(gpath_list, classifier_weight_filepath=None, return_dict=None, **kwargs):
    result_gen = test(
        gpath_list,
        classifier_weight_filepath=classifier_weight_filepath,
        return_dict=True,
        **kwargs,
    )

    for result in result_gen:
        best_score, best_key, result_dict = result
        best_key = best_key.split(':')
        if len(best_key) == 1:
            best_species = best_key
            best_viewpoint = None
        elif len(best_key) == 2:
            best_species, best_viewpoint = best_key
        else:
            raise ValueError('Invalid key {!r}'.format(best_key))

        yield (
            best_score,
            best_species,
            best_viewpoint,
            'UNKNOWN',
            0.0,
            result_dict,
        )

def features(filepath_list, batch_size=512, multi=PARALLEL, **kwargs):
    # Detect if we have a GPU available
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    using_gpu = str(device) != 'cpu'

    logger.info('Initializing Datasets and Dataloaders...')

    # Create training and validation datasets
    transforms = _init_transforms(**kwargs)
    dataset = ImageFilePathList(filepath_list, transform=transforms['test'])

    # Create training and validation dataloaders
    dataloader = torch.utils.data.DataLoader(
        dataset, batch_size=batch_size, num_workers=0, pin_memory=using_gpu
    )

    # Initialize the model for this run
    model = torchvision.models.densenet201(pretrained=True)

    # Send the model to GPU
    model = model.to(device)

    if multi:
        logger.info('USING MULTI-GPU MODEL')
        model = nn.DataParallel(model)

    model.eval()

    start = time.time()

    outputs = []
    for (inputs,) in tqdm.tqdm(dataloader, desc='test'):
        inputs = inputs.to(device)
        with torch.set_grad_enabled(False):
            output = model(inputs)
            outputs += output.tolist()

    outputs = np.array(outputs, dtype=np.float32)
    time_elapsed = time.time() - start
    logger.info(
        'Testing complete in {:.0f}m {:.0f}s'.format(
            time_elapsed // 60, time_elapsed % 60
        )
    )

    return outputs


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def zigzagLevelOrder(self, root: TreeNode) -> List[List[int]]:

        return Solution.solve(self,root)

    def height(self,root):

        if root is None:
            return 0

        return max(Solution.height(self,root.left),Solution.height(self,root.right))+1

    def solve(self,root):
        s1 = []
        s1.append(root)
        s2 = []
        res = []

        h = Solution.height(self,root)

        for order in range(h):
            temp = []

            if order%2==0:
                while len(s1)!=0:

                    t = s1.pop()
                    temp.append(t.val)
                    if t.left is not None:
                        s2.append(t.left)

                    if t.right is not None:
                        s2.append(t.right)
            else:
                while len(s2)!=0:

                    t = s2.pop()
                    temp.append(t.val)
                    if t.right is not None:
                        s1.append(t.right)

                    if t.left is not None:
                        s1.append(t.left)
            res.append(temp)

        return res

Challenges/2020/July Challenge/Week 4/Binary Tree Zigzag Level Order Traversal.py


"""
Author: Jing (https://github.com/gnijuohz)

Wildcard Matching: https://oj.leetcode.com/problems/wildcard-matching

Implement wildcard pattern matching with support for '?' and '*'.

'?' Matches any single character.
'*' Matches any sequence of characters (including the empty sequence).

The matching should cover the entire input string (not partial).

The function prototype should be:
bool isMatch(const char *s, const char *p)

Some examples:
isMatch("aa","a") â false
isMatch("aa","aa") â true
isMatch("aaa","aa") â false
isMatch("aa", "*") â true
isMatch("aa", "a*") â true
isMatch("ab", "?*") â true
isMatch("aab", "c*a*b") â false
Tags
Dynamic Programming, Backtracking, Greedy, String
"""

class Solution:
    # @param s, an input string
    # @param p, a pattern string
    # @return a boolean
    def isMatch(self, s, p):
        lenS, lenP = len(s), len(p)
        i, j = 0, 0
        star = None
        starPos = 0
        while i < lenS:
            if j < len(p) and (s[i] == p[j] or p[j] == "?"):
                i += 1
                j += 1
            elif j < len(p) and p[j] == "*":
                star = j
                j += 1
                starPos = i
            elif star is not None:
                j = star + 1
                starPos += 1
                i = starPos
            else:
                return False
        while j < lenP and p[j] == "*":
            j += 1
        return j >= lenP


# -*- coding: utf-8 -*-

import os
basedir = os.path.abspath(os.path.dirname(__file__))

# admin list
ADMINS = [os.environ.get('ADMIN_EMAIL_LIST')]

# OpenID integration
OPENID_PROVIDERS = [
    {'name': 'Google', 'url': 'https://www.google.com/accounts/o8/id'},
    {'name': 'Yahoo', 'url': 'https://me.yahoo.com'}
]

# database configuration
# this configuration is taken from Flask Megatutorial
# http://blog.miguelgrinberg.com/post/the-flask-mega-tutorial-part-iv-database
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'app.db')
SQLALCHEMY_MIGRATE_REPO = os.path.join(basedir, 'db_repository')


#calss header
class _BEAUTICIAN():
	def __init__(self,):
		self.name = "BEAUTICIAN"
		self.definitions = [u"a trained person whose job it is to improve the appearance of a customer's face, body, and hair, using make-up and beauty treatments, often in a beauty salon"]

		self.parents = []
		self.childen = []
		self.properties = []
		self.jsondata = {}

		self.specie = 'nouns'

	def run(self, obj1 = [], obj2 = []):
		return self.jsondata


from threading import Thread
from selenium import webdriver
import time
from datetime import date

DISEASES = []
databaseManager = None
THREAD_COUNT = 8
GERIYE_BAKILACAK_SAYFA = 4
listOfLists = []

import sqlite3 as sql

import os.path

def GetDialogues():
    try:
        connection = sql.connect("C:/Users/berkayalcin/Desktop/DiseaseDetectAI/DiseaseDetectAI/HealthCare")
        query = """select * from Dialogues"""
        cursor = connection.cursor()
        cursor.execute(query)
        result = cursor.fetchall()
        connection.commit()
        return result
    except Exception as e:
        print("query failed {}".format(e))
        pass
    finally:
        try:
            connection.cursor().close()
            connection.close()
        except Exception as e:
            pass
    return

def GetMaxPage():
    try:
        connection = sql.connect("C:/Users/berkayalcin/Desktop/DiseaseDetectAI/DiseaseDetectAI/HealthCare")
        query = """select IFNULL(MAX(CAST(Page as INTEGER)),1) from Dialogues"""
        cursor = connection.cursor()
        cursor.execute(query)
        result = cursor.fetchone()
        connection.commit()
        return result[0]
    except Exception as e:
        print("query failed {}".format(e))
        pass
    finally:
        try:
            connection.cursor().close()
            connection.close()
        except Exception as e:
            pass
    return

def checkIsDialogueExists(Question, Answer):
    try:
        connection = sql.connect("C:/Users/berkayalcin/Desktop/DiseaseDetectAI/DiseaseDetectAI/HealthCare")
        query = "select EXISTS(select * from Dialogues where Question=? and Answer=?)"
        cursor = connection.cursor()
        cursor.execute(query, (Question, Answer))
        result = cursor.fetchone()
        connection.commit()
        return result[0] == 1
    except Exception as e:
        print("query failed {}".format(e))
        pass
    finally:
        try:
            connection.cursor().close()
            connection.close()
        except Exception as e:
            pass
    return

def InsertDialogue(Question, Answer, Branch, Date, Sayfa):
    try:
        if (checkIsDialogueExists(Question, Answer)) == True:
            print("Kayıt Zaten Mevcut")
            return
        connection = sql.connect("C:/Users/berkayalcin/Desktop/DiseaseDetectAI/DiseaseDetectAI/HealthCare")
        query = """INSERT INTO Dialogues (Question, Answer, Branch,Page,Date) VALUES (?,?,?,?,?)"""
        cursor = connection.cursor()
        Parameters = (Question, Answer, Branch, Sayfa, Date)
        cursor.execute(query, Parameters)
        result = cursor.fetchone()
        connection.commit()
        return True

    except Exception as e:
        print("query failed {}".format(e))
    finally:
        try:
            connection.cursor().close()
            connection.close()

        except Exception as e:
            pass

def scrapdoktorsitesi(url, sayfa):
    addedRecordCountSinceStart = 0
    baslangicZamani = time.time()

    soruCevapDICT = []

    options = webdriver.FirefoxOptions()

    options.headless = True

    browser = webdriver.Firefox(executable_path="C:/Program Files/Mozilla Firefox/geckodriver.exe",
                                firefox_options=options)
    browser.get(url)

    print("Tarayacı {0} Numaralı Sayfada başladı".format(sayfa))
    delay = 3

    sorular = browser.find_elements_by_xpath("//*[contains(@class, 'question item')]")
    for i in range(0, len(sorular)):

        sorular = browser.find_elements_by_xpath("//*[contains(@class, 'question item')]")
        body = sorular[i].find_element_by_class_name("body")
        soru = body.find_element_by_tag_name("h4")
        try:
            aTag = soru.find_element_by_tag_name("a")
            href = aTag.get_attribute("href")
            browser.get(href)

            try:
                errorVarmi = browser.find_element_by_xpath("//*[contains(@class, 'error-page')]")
                if errorVarmi is not None:
                    browser.back()
                    continue
            except Exception as error:
                pass
            panelFirst = \
            browser.find_elements_by_xpath("//*[contains(@class, 'panel panel-default expert-answers t-margin-30')]")[0]
            questions = panelFirst.find_elements_by_class_name('question')
            answers = []
            try:
                answers = panelFirst.find_elements_by_class_name("answer")
            except Exception as e:
                print("Cevap Gelmemiş Answer Classı Bulunamadı")
                browser.back()
                continue
            doktorBranch = answers[0].find_element_by_xpath(
                "//*[@id=\"pageContainer\"]/div[3]/div/div[2]/div[2]/div[2]/ul[2]/li/div/div[2]/div[2]/div/div/div/h2/a")
            branchName = doktorBranch.get_attribute("innerText")
            today = date.today()
            d1 = today.strftime("%d/%m/%Y")
            for j in range(0, len(answers)):
                q = questions[j].get_attribute("innerText").replace("*", "")
                a = answers[j].find_element_by_class_name("panel-body").find_element_by_tag_name("p").get_attribute(
                    "innerText")

                eklendiMi = InsertDialogue(q, a, branchName, d1, sayfa)
                if eklendiMi == True:
                    addedRecordCountSinceStart += 1
                    print("Kayıt Eklendi")
                soruCevapDICT.append({"Q": q, "A": a, "Branch": branchName})
            browser.back()

        except Exception as e:

            print(e)
            browser.back()
            continue

    bitisZamani = time.time()
    print("{0} Kadar zamanda {1} Veri Eklendi - One One Browser".format(str(bitisZamani - baslangicZamani),
                                                                        addedRecordCountSinceStart))
    browser.close()

def main():
    sayfaSayisi = getSayfaSayisi()

    threads = []
    kacinciSayfadaKaldi = GetMaxPage()
    pageNum = int(kacinciSayfadaKaldi)
    if (pageNum > GERIYE_BAKILACAK_SAYFA):
        pageNum -= GERIYE_BAKILACAK_SAYFA
    print("Kaçıncı Sayfada Kaldı = " + str(pageNum))
    kacinciSayfadaKaldi = pageNum

    while True:
        time.sleep(1)
        RemoveDiedThreads(threads)
        if len(threads) < THREAD_COUNT:
            t1 = Thread(target=scrapdoktorsitesi,
                        args=("https://www.doktorsitesi.com/icerikler/soru?sayfa=" + str(kacinciSayfadaKaldi) + "",
                              kacinciSayfadaKaldi,))
            t1.start()
            threads.append(t1)
            kacinciSayfadaKaldi += 1
        if kacinciSayfadaKaldi == sayfaSayisi or kacinciSayfadaKaldi == sayfaSayisi - 1:
            break

def RemoveDiedThreads(threads):
    for th in threads:
        if not th.is_alive():
            threads.remove(th)

def getSayfaSayisi():
    browser = webdriver.Firefox(executable_path="C:/Program Files/Mozilla Firefox/geckodriver.exe")
    browser.get("https://www.doktorsitesi.com/icerikler/soru")
    sayfaSayisi = int(browser.find_element_by_xpath("/html/body/div[1]/main/div/div[2]/div/div/div/b[1]").get_attribute(
        "innerText").split("/")[1])
    browser.close()
    return sayfaSayisi

if __name__ == "__main__":
    main()


import datetime
from pupil import BlinkDetector

from util.observer import Observer
from notification import notification

class ConcentrationClassifier(Observer):
    def __init__(self):
        self.previous_average = 0
        self.decreasing_timestamp = None
        self.last_notification_timestamp = datetime.datetime.now()
        self.blink_detector = BlinkDetector()
        self.blink_detector.register(self)
        self.blink_detector.start()

    def update(self):
        if self.previous_average > self.blink_detector.average():
            if self.decreasing_timestamp:
                if (datetime.datetime.now() - self.decreasing_timestamp).total_seconds() > 1:
                    if (datetime.datetime.now() - self.last_notification_timestamp).total_seconds() > 5:
                        print("Making a notification")
                        notification('Try to focus on the task at hand', title="Focus")
                        self.last_notification_timestamp = datetime.datetime.now()
                    self.decreasing_timestamp = None
            else:
                self.decreasing_timestamp = datetime.datetime.now()
        else:
            self.decreasing_timestamp = None
        self.previous_average = self.blink_detector.average()


#Recommended to remove the following tables before starting: CovidMutations, CovidMutatedPosMutation, CovidSequencedGene, CovidSequence , CovidIsolate - in this order
from django.core.management.base import BaseCommand, CommandError
import os
import pandas as pd
from io import StringIO
from covid19.models import *
import datetime
import re
import csv
import tqdm
import gc
from django.conf import settings

class Command(BaseCommand):
    help = "Saves info from GSAID to the database."
    def add_arguments(self, parser):
        parser.add_argument(
            '--skip_isolates',
            action='store_true',
            dest='skip_isolates',
            default=False,
            help='Do not save isolates data.',
        )
        parser.add_argument(
            '--skip_sequences',
            action='store_true',
            dest='skip_sequences',
            default=False,
            help='Do not save sequences data.',
        )

    def handle(self, *args, **options):

        self.stdout.write(self.style.NOTICE("Loading data..."))
        #data_toni:
            ##genes=pd.read_csv( os.path.join( gsaid_path, "genes.csv.xz"))
            #genes_path=os.path.join( gsaid_path, "genes.csv.xz")
            ##seq=pd.read_csv( os.path.join( gsaid_path, "seq_2.csv.xz"),index_col=0)
            #seq_path= os.path.join( gsaid_path, "seq_2.csv.xz")
            ##isolates=pd.read_csv( os.path.join( gsaid_path, "isolates.csv.xz"))
            #isolates_path=os.path.join( gsaid_path, "isolates.csv.xz")

        gsaid_path=settings.MEDIA_ROOT + "Covid19Data/Data/gisaid/tables"

        genes_path=os.path.join( gsaid_path, "genes.csv")
        isolates_path=os.path.join( gsaid_path, "isolates.csv")

        if not options["skip_sequences"]:
            seq_path= os.path.join( gsaid_path, "seq_2.csv.xz")
            seq_all=pd.read_csv( os.path.join( gsaid_path, "seq_2.csv.xz"),index_col=0)
            #seq_all_iter = pd.read_csv('train_2011_2012_2013.csv', sep=';', iterator=True, chunksize=20000000, low_memory = False)
            #seq_all = pd.concat(seq_all_iter, ignore_index=True)
            seq=seq_all.loc[(seq_all["mutation_description"] != "X Too different")& (seq_all["mutation_description"] != "X Undefined AA")]
            del seq_all
            gc.collect()

        #Saving all isolates

        if not options["skip_isolates"]:
            self.stdout.write(self.style.NOTICE("Saving isolates..."))
            iso_skip_rows=0
            nrow=0
            total_iso=2095134 #just for the log
            with open(isolates_path, 'r') as read_obj:
                csv_reader = csv.reader(read_obj)
                header=False
                for row in tqdm.tqdm(csv_reader,total=total_iso):
                    if not header:
                        header=row
                    else:
                        if nrow < iso_skip_rows:
                            nrow+=1
                            continue
                        nrow+=1
                        myisolate= pd.DataFrame([row],columns=header).iloc[0]
                        isolate_id=myisolate["isolate_id"]

                        (y,m,d)=myisolate["ymd"].split("-")
                        if int(m)==0:
                            m=1
                        if int(d)==0:
                            d=1
                        isolate_date=datetime.date(int(y),int(m),int(d))

                        #In case an entry with this isolate ID altrady existed, this would get it and update the info (some fields may have changed slightly)
                        isolateobj, created_isolateobj= CovidIsolate.objects.get_or_create(
                            isolate_name = myisolate["isolate_name"],
                            isolate_date = isolate_date,
                            isolate_id = myisolate["isolate_id"],
                            history = myisolate["history"],
                            tloc = myisolate["tloc"],
                            host = myisolate["host"],
                            originating_lab = myisolate["originating_lab"],
                            submitting_lab = myisolate["submitting_lab"],
                            submitter = myisolate["submitter"],
                            location = myisolate["location"],
                            isolate_type = myisolate["type"]
                        )

        if not options["skip_sequences"]:
            self.stdout.write(self.style.NOTICE("Saving sequences and mutations..."))
            #Saving all sequences except those with mut_desctiption 'X Too different' or 'X Undefined AA' (discarded before)
            for i,myseq in tqdm.tqdm(seq.iterrows(),total=len(seq)):
                mut_descr=myseq["mutation_description"]
                is_wt=False
                if mut_descr =="WT":
                    is_wt=True
                    continue
    #            elif mut_descr in {'X Too different', 'X Undefined AA'}:
    #                #Has unknown residues - Not considered
    #                continue

                seqobj, created_seqobj =CovidSequence.objects.get_or_create(
                    seq = myseq["seq"],
                    is_wt = is_wt,
                    seq_idx_table=int(myseq["seq_idx"])
                )

                if not is_wt:
                    mut_li=mut_descr.split(",")
                    adj=0
                    for mut in mut_li:
                        mutmatch=re.match("([A-Z])(\d+)([A-Z])",  mut)
                        if mutmatch:
                            fromres=mutmatch.group(1)
                            mutpos=int(mutmatch.group(2))
                            tores=mutmatch.group(3)
                            corrected_mutpos=mutpos+adj

#                            mutobj, created_mutobj=CovidMutations.objects.get_or_create(
#                                resid =corrected_mutpos,
#                                resletter_from = fromres,
#                                resletter_to = tores,
#                                id_sequence =seqobj
#                                )

                            mutobj, created_mutobj=CovidMutation.objects.get_or_create(
                                resletter_from = fromres,
                                resletter_to = tores,
                                )

                            mutposobj, created_mutposobj=CovidMutatedPos.objects.get_or_create(
                                resid =corrected_mutpos,
                                id_sequence =seqobj
                                )

                            mutrelobj, created_mutrelobj=CovidMutatedPosMutation.objects.get_or_create(
                                id_mutated_pos= mutposobj,
                                id_mutation_fromto= mutobj,
                                )

                        elif mut.startswith("del"):
                            #print("deletion found at seq %s" % myseq["seq_idx"])
                            mutmatch=re.match("del(\d+)([A-Z]+)",  mut)
                            if mutmatch:
                                mutpos=mutmatch.group(1)
                                fromres=mutmatch.group(2)
                                tores=""
                            else:
                                self.stdout.write(self.style.NOTICE("Error in mutation %s of seq %s" % (mut,myseq["seq_idx"])))
                        elif mut.startswith("ins"):
                            #print("insertion found at seq %s" % myseq["seq_idx"])
                            mutmatch=re.match("ins(\d+)([A-Z]+)",  mut)
                            if mutmatch:
                                mutpos=mutmatch.group(1)
                                fromres=""
                                tores=mutmatch.group(2)
                                adj-=len(tores)
                            else:
                                self.stdout.write(self.style.NOTICE("Error in mutation %s of seq %s" % (mut,myseq["seq_idx"])))
                        else:
                            self.stdout.write(self.style.NOTICE("Unknown mutation %s of seq %s" % (mut,myseq["seq_idx"])))

            del seq
            gc.collect()

        #Saving all genes except those that are 'bad' and those for which the sequence was not saved (seq was 'X Too different' or 'X Undefined AA')
        self.stdout.write(self.style.NOTICE("Saving genes..."))
        total_genes=55784470 #just for the log
        gene_skip_rows=0
        nrow=0
        with open(genes_path, 'r') as read_obj:
            csv_reader = csv.reader(read_obj)
            header=False
            for row in tqdm.tqdm(csv_reader,total=total_genes):
                if not header:
                    header=row
                else:
                    if nrow < gene_skip_rows:
                        nrow+=1
                        continue
                    nrow+=1

                    mygene= pd.DataFrame([row],columns=header).iloc[0]

                    if  mygene["bad"]=="True":
                        continue

                    #NSx should be ORFx - change this
                    genename=mygene["gene"]
                    nsnum=re.match("NS(\d+[a-zA-Z]?)",  genename)
                    alt_name=None
                    if nsnum:
                        alt_name=genename
                        genename="ORF" + nsnum.group(1)

                    finalprotobj, created_finalprotobj=CovidFinalProtein.objects.get_or_create(name=genename)

                    gene_isolate_filt=CovidIsolate.objects.filter(isolate_id= mygene["isolate_id"])
                    if gene_isolate_filt:
                        gene_isolate=gene_isolate_filt[0]
                    else:
                        continue
                    gene_seq_filt=CovidSequence.objects.filter(seq_idx_table=int(mygene["seq_idx"]))
                    if gene_seq_filt:
                        gene_seq=gene_seq_filt[0]
                    else:
                        continue
                    seqgeneobj,created_seqgeneobj=CovidSequencedGene.objects.get_or_create(
                            id_final_protein=finalprotobj,
                            alt_name=alt_name,
                            id_isolate=gene_isolate,
                            id_sequence=gene_seq
                    )


# coding=utf-8

from database.base import Session, engine, Base
from database.ipeds_survey_dimension import IpedsSurveyDimension

import numpy as np
import pandas as pd
from pandas import DataFrame
from sqlalchemy import sql

print("Reading ipeds_survey_dimension data")
df = pd.read_csv('data/ipeds_survey_dimension.csv')

session = Session()

try:
    print('\nPopulating dimension tables.')
    # insert state data
    record_deletes = session.query(IpedsSurveyDimension).delete(synchronize_session=False)
    df_inserts = session.bulk_insert_mappings(mapper = IpedsSurveyDimension,
                                              mappings = df.to_dict(orient='records')),
    print('\tFinished populating ipeds_employee_dimension table.')

except Exception as e:
    session.rollback()
    print("\nAn error occurred and no data were changed in the database.\n\nError:\n{}".format(str(e)))
else:
    session.commit()
    print('\nChanges committed to database.')

session.close()

print("\nAll Done.")


"""Common utilities for file operations."""

import logging
import os
import errno
import shutil

class FileUtilsError(Exception):
    pass

class FileUtils(object):

    @staticmethod
    def rm_rf(path):
        """Util to delete path"""
        if not os.path.exists(path):
            return
        if os.path.isfile(path) or os.path.islink(path):
            os.unlink(path) # also removes a file
        else:
            shutil.rmtree(path)

    @staticmethod
    def symlink(linkto, dstname):
        """Util to symlink path"""
        if os.path.exists(dstname):
            if os.path.abspath(linkto) == os.path.abspath(dstname):
                return
            os.unlink(dstname)
        os.symlink(linkto, dstname)

    @staticmethod
    def mkdir_p(start_path):
        """Util to make path"""
        try:
            os.makedirs(start_path)
        except OSError as exc: # Python >2.5
            if exc.errno == errno.EEXIST and os.path.isdir(start_path):
                pass
            else:
                raise

    @staticmethod
    def dir_size(start_path):
        """Util to get total size of path in MB"""
        total_size = 0
        for dirpath, dirnames, filenames in os.walk(start_path):
            for f in filenames:
                fp = os.path.join(dirpath, f)
                if os.path.exists(fp):
                    try:
                        total_size += os.path.getsize(fp)
                    except:
                        continue
                        # convert to MB
                        return int(total_size * 1.0 / 10000000)


import unittest

from typing import Any, Dict

from . import common

from edgin_around_api import craft

class CraftTest(common.SerdeTest):
    def test_ingredient_filter_items(self) -> None:
        """Checks if filtering works correctly. Filtered list should contain only items compatible
        with the given ingredient/material."""

        ingredient = craft.Ingredient(craft.Material.WOOD, 1)

        input_items = {
            craft.Item(0, craft.Essence.ROCKS, 1),
            craft.Item(1, craft.Essence.LOGS, 1),
        }

        expected_items = {
            craft.Item(1, craft.Essence.LOGS, 1),
        }

        result_items = ingredient.filter_items(input_items)
        self.assertEqual(result_items, expected_items)

    def test_assembly_filter_items(self) -> None:
        """Checks if filtering works correctly. Quantities of items in the filtered list should be
        reduced by quantities of corresponding items/entities in the assembly. If the new quantity
        is zero, the item should be removed."""

        assembly = craft.Assembly(
            recipe_codename="test_recipe",
            sources=[
                [
                    craft.Item(0, craft.Essence.ROCKS, 2),
                    craft.Item(4, craft.Essence.ROCKS, 3),
                ],
                [
                    craft.Item(3, craft.Essence.STICKS, 3),
                    craft.Item(1, craft.Essence.LOGS, 2),
                ],
            ],
        )

        input_items = {
            craft.Item(0, craft.Essence.ROCKS, 3),
            craft.Item(1, craft.Essence.LOGS, 2),
            craft.Item(2, craft.Essence.GOLD, 2),
        }

        expected_items = {
            craft.Item(0, craft.Essence.ROCKS, 1),
            craft.Item(2, craft.Essence.GOLD, 2),
        }

        result_items = assembly.filter_items(input_items)
        self.assertEqual(result_items, expected_items)

    def test_assembly_find_item(self) -> None:
        """Checks if items lookup in `Assembly` works correctly."""

        rocks1 = craft.Item(0, craft.Essence.ROCKS, 2)
        rocks2 = craft.Item(4, craft.Essence.ROCKS, 3)
        sticks = craft.Item(3, craft.Essence.STICKS, 3)
        logs = craft.Item(1, craft.Essence.LOGS, 2)

        assembly = craft.Assembly(
            recipe_codename="test_recipe",
            sources=[[rocks1, rocks2], [sticks, logs]],
        )

        self.assertEqual(assembly.find_item(3, None), sticks)
        self.assertEqual(assembly.find_item(3, 0), None)
        self.assertEqual(assembly.find_item(5, None), None)

    def test_assembly_update_item(self) -> None:
        """Updating the assembly items should work correctly."""

        assembly = craft.Assembly(
            recipe_codename="test_recipe",
            sources=[
                [
                    craft.Item(1, craft.Essence.GOLD, 3),
                    craft.Item(2, craft.Essence.LOGS, 3),
                    craft.Item(3, craft.Essence.STICKS, 3),
                ],
                [],
            ],
        )

        expected = craft.Assembly(
            recipe_codename="test_recipe",
            sources=[
                [
                    craft.Item(1, craft.Essence.GOLD, 2),
                    craft.Item(2, craft.Essence.LOGS, 5),
                    craft.Item(3, craft.Essence.STICKS, 3),
                    craft.Item(4, craft.Essence.ROCKS, 2),
                ],
                [],
            ],
        )

        # Negative change on not existing item is a noop
        self.assertFalse(assembly.update_item(1, craft.Item(3, craft.Essence.STICKS, 0), -1))

        # Removing too many existing items is a noop
        self.assertFalse(assembly.update_item(0, craft.Item(3, craft.Essence.STICKS, 0), -5))

        # Increasing quantity of not existing items should create them
        self.assertTrue(assembly.update_item(0, craft.Item(4, craft.Essence.ROCKS, 0), 2))

        # Increasing quantity of existing items should work correctly
        self.assertTrue(assembly.update_item(0, craft.Item(2, craft.Essence.LOGS, 0), 2))

        # Increasing quantity of existing items should work correctly
        self.assertTrue(assembly.update_item(0, craft.Item(1, craft.Essence.GOLD, 0), -1))

        self.assertEqual(assembly, expected)

    def test_serde_assembly(self) -> None:
        original: Dict[str, Any] = {
            "recipe_codename": "my_recipe",
            "sources": [
                [
                    {
                        "actor_id": 4,
                        "essence": "GOLD",
                        "quantity": 6,
                    },
                    {
                        "actor_id": 7,
                        "essence": "ROCKS",
                        "quantity": 5,
                    },
                ]
            ],
        }

        self.assert_serde(original, craft.Assembly.Schema(), craft.Assembly)


from woqlclient import WOQLClient, WOQLQuery
import pandas as pd
import json

server_url = "http://localhost:6363"
key = "root"
dbId = "schema_dot_org"

SIMPLE_TYPE_MAP={"http://schema.org/Boolean": "boolean",
             "http://schema.org/Text": "string",
             "http://schema.org/Date": "dateTime",
             "http://schema.org/DateTime": "dateTime",
             "http://schema.org/URL": "string",
             "http://schema.org/XPathType": "string",
             "http://schema.org/Integer": "integer",
             "http://schema.org/Number": "integer",
             "http://schema.org/Float": "decimal"
             }

def construct_objects(series):
    if series.id in SIMPLE_TYPE_MAP:
        result = WOQLQuery().\
                 doctype(series.id).\
                 label(series.label).\
                 description(series.comment).\
                 property(series.id+"Value", "xsd:"+SIMPLE_TYPE_MAP[series.id])
    else:
        result = WOQLQuery().\
                 doctype(series.id).\
                 label(series.label).\
                 description(series.comment)
    return result

def construct_prop_dr(series):
    result = []
    if (type(series.domainIncludes)==str) and \
       (',' in series.domainIncludes):
        result.append(WOQLQuery().doctype(series.id+"Domain"))
    if (type(series.rangeIncludes)==str) and \
       (',' in series.rangeIncludes):
        result.append(WOQLQuery().doctype(series.id+"Range"))
    return result

def construct_type_addon(series, type_id):
    result = []
    if type(series.subTypes) == str:
        for kid in series.subTypes.split(','):
            kid = kid.strip()
            if kid in list(type_id):
                result.append(WOQLQuery().add_quad(kid, "subClassOf", series.id, "schema"))
    if type(series.subTypeOf) == str:
        for parent in series.subTypeOf.split(','):
            parent = parent.strip()
            if parent in list(type_id):
                result.append(WOQLQuery().add_quad(series.id, "subClassOf", parent, "schema"))
    return result

def construct_property_addon(series, type_id):
    result = [WOQLQuery().add_quad(series.id, "rdf:type", "owl:ObjectProperty", "schema")]
    if (type(series.domainIncludes)==str):
        if (',' in series.domainIncludes):
            result.append(WOQLQuery().add_quad(series.id, "domain", series.id+"Domain", "schema"))
            for domain in series.domainIncludes.split(','):
                domain = domain.strip()
                if domain in list(type_id):
                    result.append(WOQLQuery().add_quad(domain, "subClassOf", series.id+"Domain", "schema"))
        else:
            result.append(WOQLQuery().add_quad(series.id, "domain", series.domainIncludes, "schema"))
    if (type(series.rangeIncludes)==str):
        if (',' in series.rangeIncludes):
            result.append(WOQLQuery().add_quad(series.id, "range", series.id+"Range", "schema"))
            for range in series.rangeIncludes.split(','):
                range = range.strip()
                if range in list(type_id):
                    result.append(WOQLQuery().add_quad(range, "subClassOf", series.id+"Range", "schema"))
        else:
            result.append(WOQLQuery().add_quad(series.id, "range", series.rangeIncludes, "schema"))
    if len(result) == 1:
        print(series)
        return []
    return result

def create_schema_from_queries(client, queries):
    schema = WOQLQuery().when(True).woql_and(*queries)
    return schema.execute(client)

def create_schema_from_addon(client, queries):
    new_queries = []
    for query in queries:
        if len(query) > 0:
            new_queries.append(WOQLQuery().woql_and(*query))
    schema = WOQLQuery().when(True).woql_and(*new_queries)
    output_json(schema, filename)
    return schema.execute(client)

print("read types from csv and construct WOQL objects")
types = pd.read_csv("all-layers-types.csv")
types["WOQLObject"] = types.apply(construct_objects, axis=1)

print("read perperties from csv and construct WOQL objects")
properties = pd.read_csv("all-layers-properties.csv")
properties["WOQLObject"] = properties.apply(construct_objects, axis=1)

print("create Domain and Range object for properties")
properties["WOQLObject_DR"] = properties.apply(construct_prop_dr, axis=1)

print("create types addon")
types["addon"] = types.apply(construct_type_addon,
                             axis=1,
                             type_id=types.id)

print("create properties addon")
properties["addon"] = properties.apply(construct_property_addon,
                                       axis=1,
                                       type_id=types.id)

print("create db and schema")
client = WOQLClient()
client.connect(server_url, key)
client.deleteDatabase(dbId)
client.createDatabase(dbId, "Schema.org")
create_schema_from_queries(client, list(types["WOQLObject"]))
create_schema_from_queries(client, list(properties["WOQLObject"]))
create_schema_from_addon(client, list(properties["WOQLObject_DR"]))
create_schema_from_addon(client, list(types["addon"]))
create_schema_from_addon(client, list(properties["addon"]))