############################################
# Tech Check 4 - Provided Starter File
############################################

#FUNCTION
def main():

    print("Grade Point Calculator\n")
    print("Valid letter grades that can be entered: A, B, C, D, F.")
    print("Valid grade modifiers are +, - or nothing.")
    print("All letter grades except F can include a + or - symbol.")
    print("Calculated grade point value cannot exceed 4.0.\n")

    numericGrade = 0.0

    letterGrade = input("Please enter a letter grade : ").upper()
    modifier = input("Please enter a modifier (+, - or nothing) : ")

    # Determine base numeric value of the grade
    if letterGrade == "A":
        numericGrade = 4.0
    elif letterGrade == "B":
        numericGrade = 3.0
    elif letterGrade == "C":
        numericGrade = 2.0
    elif letterGrade == "D":
        numericGrade = 1.0
    elif letterGrade == "F":
        numericGrade = 0.0
    else:
        print("You entered an invalid letter grade.")

    # Determine whether to apply a modifier
    if modifier == "+":
        if letterGrade != "A" and letterGrade != "F": # Plus is not valid on A or F
            numericGrade += 0.3
    elif modifier == "-":
        if letterGrade != "F":     # Minus is not valid on F
            numericGrade -= 0.3

    # Output final message and result, with formatting
    print("The numeric value is: {0:.1f}".format(numericGrade))

#PROGRAM STARTS HERE
if __name__ == "__main__":
    main()


import os.path
import maya.cmds as MCmds
import U4_Utils
from U4_SceneScriptsWindow import U4_SceneScriptsWindow

def getScriptsInDirectory(directory):
	"""
	Find *.py (python) scripts in a given directory.
	@returns: List of tuples (scriptName, absolutePath)
	"""
	scripts = []
	pythonScriptNames = MCmds.getFileList(folder = directory, filespec = "*.py")
	if pythonScriptNames:
		for name in pythonScriptNames:
			absolutePath = os.path.join(directory, name)
			scripts.append((name, absolutePath))
	return scripts

def getSceneScripts():
	"""
	Find *.py scripts neighboring saved scene file.
	@returns: List of tuples (scriptName, scriptPath).
	"""
	sceneFile = U4_Utils.getSceneFile()
	if not sceneFile:
		return None

	sceneFileDir = sceneFile[0]
	return getScriptsInDirectory(sceneFileDir)

def readScriptContents(path):
	"""
	Read contents of a python script file given the absolute file path.
	@param path: Absolute path to a .py (python) file.
	@returns: File contents as string.
	"""
	if not path.endswith(".py"):
		raise ValueError("path should lead to a .py (python) file")

	# Using execfile(path) didn't load modules in maya's scripts path correctly,
	# so we read it in ourselves and pass it through the python mel command.
	fileHandle = open(path)
	fileContents = fileHandle.read()
	fileHandle.close()
	return fileContents

def execPython(script):
	"""
	Execute python code.
	@param script: String of python code.
	"""

	# Using execfile(path) didn't load modules in maya's scripts path correctly,
	# so we read it in ourselves and pass it through the python mel command.
	MCmds.python(script)

def runScriptFromFile(path):
	"""
	Run a python script given the absolute file path.
	"""
	scriptContents = readScriptContents(path)
	execPython(scriptContents)

def showSceneScriptsWindow():
	U4_SceneScriptsWindow()

def runDefaultScript():
	scripts = getSceneScripts()
	if not scripts:
		return

	runScriptFromFile(scripts[0][1])


'''
Created on 2013-11-28

@author: dongmingchang
'''

import traceback
import Global
import subprocess
import locale
import re
import platform
import pythoncom
import win32com.client
import os
import win32process
import win32event
import SDK.FileFunc as FileFunc
import codecs
import time
import sys

from framework import (
     Decorators)

def isXPPlatform():
    version =  platform.platform()
    verReg = re.search('Windows-(\w+)-', version)
    if verReg != None :
        if verReg.group(1) == 'XP'  or verReg.group(1) == '2003Server':
            return True
    return False

@Decorators.Step
def createScheduler(errorHandler, name, exePath):

    #SCHTASKS /Create /SC DAILY /TN gaming /TR c:\freecell /ST 12:00 /ET 14:00

    try:

        version =  platform.platform()
        rootFolder = None

        command = "SCHTASKS /Create /SC DAILY /TN \""+name+"\" /TR "+exePath+" /ST 12:00 /ET 14:00 /F"

        verReg = re.search('Windows-(\w+)-', version)
        if verReg != None :
            if verReg.group(1) == 'XP'  or verReg.group(1) == '2003Server':
                command = "SCHTASKS /Create /SC DAILY /TN \""+name+"\" /TR "+exePath+" /RU SYSTEM"

        p = subprocess.Popen(command ,stdin = subprocess.PIPE, \
                     stdout = subprocess.PIPE, \
                     stderr = subprocess.PIPE, \
                     shell=True)

        encoding = locale.getdefaultlocale()[1]
       	'''
        for line in p.stdout :
            columns = line.decode(encoding)
            print(columns)
        '''

    except Exception:

        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);

    return True

@Decorators.Step
def deleteScheduler(errorHandler, name):

    try:
        command = "SCHTASKS /Delete /TN \""+name +"\" /F"

        p = subprocess.Popen(command ,stdin = subprocess.PIPE, \
                     stdout = subprocess.PIPE, \
                     stderr = subprocess.PIPE, \
                     shell=True)

        '''
        encoding = locale.getdefaultlocale()[1]

        for line in p.stderr :
            columns = line.decode(encoding).split()
            if columns:
                print(columns[-1])
        '''

    except Exception:

        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);

    return True

@Decorators.Step
def checkSchedulerStart(errorHandler, name, isExisted):

    errorFile = "error.log"
    listFile = "schList.log"

    try:

        XPMode = False
        version =  platform.platform()
        verReg = re.search('Windows-(\w+)-', version)
        if verReg != None :
            if verReg.group(1) == 'XP' or verReg.group(1) == '2003Server':
                XPMode = True

        if XPMode :
            filePath = os.getenv('windir')+os.sep+"Tasks"+os.sep+name+".job"
            FileFunc.checkFileExist(errorHandler, filePath, isExisted)

        else:

            command = "cmd.exe /c chcp 437 | SCHTASKS /QUERY /TN "+"\""+name+"\" 2>"+errorFile+" 1>"+listFile

            exePath = "C:\Windows\System32\cmd.exe"
            handle = win32process.CreateProcess(exePath, command, \
                                       None , None , 0 ,win32process.CREATE_NEW_CONSOLE , None , None , \
                                       win32process.STARTUPINFO())

            win32event.WaitForSingleObject(handle[0], -1)
            time.sleep(1)

            if isExisted and os.path.getsize(errorFile) > 0:
                raise Exception("The job \"%s\" does not exist"%(name))
            elif not isExisted and os.path.getsize(errorFile) == 0 :
                disableFlag = False

                if os.path.exists(listFile) :
                    fList = codecs.open(listFile,'r',encoding='utf8')

                    for aLine in fList.readlines():
                        if aLine.find("Disabled") > -1:
                            disableFlag = True
                            break

                    fList.close()

                if not disableFlag:
                    raise Exception("The job \"%s\" exists"%(name))

            #os.remove(errorFile)
            #os.remove(listFile)

    except Exception:

        if os.path.exists(errorFile) : os.remove(errorFile)

        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);

    return True

@Decorators.Step
def createService(errorHandler, srcName, exePath):

    try:
        command = "sc create \""+srcName+"\" binpath= \""+exePath+"\" type= share start= auto"
        p = subprocess.Popen(command ,stdin = subprocess.PIPE, \
                     stdout = subprocess.PIPE, \
                     stderr = subprocess.PIPE, \
                     shell=True)

        encoding = locale.getdefaultlocale()[1]

        for line in p.stdout :
            outStr = line.decode(encoding)

            failedCodeReg = re.search('(\d+):', outStr)
            if failedCodeReg != None and failedCodeReg.group(1) != "1073" :
                raise Exception(outStr)

    except Exception:

        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);

    return True

@Decorators.Step
def deleteService(errorHandler, srvName):

    try:
        command = "sc delete \""+srvName+"\""
        p = subprocess.Popen(command ,stdin = subprocess.PIPE, \
                     stdout = subprocess.PIPE, \
                     stderr = subprocess.PIPE, \
                     shell=True)

        encoding = locale.getdefaultlocale()[1]

        for line in p.stdout :
            outStr = line.decode(encoding)

            failedCodeReg = re.search('(\d+):', outStr)
            if failedCodeReg != None and failedCodeReg.group(1) != "1060" :
                raise Exception(outStr)

    except Exception:

        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);

    return True

@Decorators.Step
def checkServiceExist(errorHandler, srvName, isExisted):

    try:
        command = "sc query \""+srvName+"\""
        p = subprocess.Popen(command ,stdin = subprocess.PIPE, \
                     stdout = subprocess.PIPE, \
                     stderr = subprocess.PIPE, \
                     shell=True)

        encoding = locale.getdefaultlocale()[1]

        errorMatch = False
        for line in p.stdout :
            outStr = line.decode(encoding)

            failedCodeReg = re.search('(\d+):', outStr)
            if failedCodeReg != None and \
                failedCodeReg.group(1) == "1060" :
                errorMatch = True

        if not isExisted and not errorMatch:
            raise Exception("The service \"%s\" exists"%srvName)
        elif isExisted and errorMatch:
            raise Exception("The service \"%s\" does not exist"%srvName)

    except Exception:

        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);

    return True

@Decorators.Step
def checkServiceStartMode(errorHandler, srvName, statusStr):

    try:
        pythoncom.CoInitialize()
        strComputer = "."
        objWMIService = win32com.client.Dispatch("WbemScripting.SWbemLocator")
        objSWbemServices = objWMIService.ConnectServer(strComputer,"root\cimv2")
        colItems = objSWbemServices.ExecQuery("SELECT * FROM Win32_Service")
        for objItem in colItems :

            if objItem.Name == srvName :
                if objItem.StartMode != statusStr :
                    raise Exception("The status of service \"%s\" is %s, not %s"%(srvName,objItem.StartMode,statusStr))
                break

    except Exception:

        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);

    return True

@Decorators.Step
def createShortcut(errorHandler, exePath, linkPath, linkName, descrp=""):
    try:
        localeStr = locale.getdefaultlocale()[1]
        if linkPath.find(os.getenv('USERPROFILE')+"\\Desktop\\") > -1:
            if localeStr == 'cp950' and isXPPlatform():
                linkPath = linkPath.replace('Desktop', '桌面')

        pythoncom.CoInitialize()
        ws = win32com.client.Dispatch("wscript.shell")
        scut = ws.CreateShortcut(linkPath+os.sep+linkName+'.lnk')
        scut.TargetPath = exePath
        scut.Description = descrp
        scut.WindowStyle = 4
        scut.Save()

        FileFunc.checkFileExist(errorHandler, linkPath+"\\"+linkName+".lnk", True)

    except Exception:

        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);

    return linkPath

@Decorators.Step
def getWindodwsVersion(errorHandler):
    try:
        version =  platform.platform()
        verReg = re.search('Windows-(\w+)-', version)
        if verReg != None and verReg.group(1) != None:
            return verReg.group(1)

    except Exception:
        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);
    return ''

@Decorators.Step
def executeCommand(errorHandler, command, errJudge=None, waitFlag=True):
    try:

        p = subprocess.Popen(command ,stdin = subprocess.PIPE, \
                     stdout = subprocess.PIPE, \
                     stderr = subprocess.PIPE, \
                     shell=True)

        encoding = locale.getdefaultlocale()[1]
        outStr = ''
        if waitFlag:
            for line in p.stdout :
                outStr = outStr+line.decode(encoding)

        if errJudge != None:
            errReg = re.search(errJudge, outStr)
            if errReg != None :
                raise Exception("%s match error string %s"%(outStr, errJudge))

    except Exception:

        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);

    return True

def getStartupPath(errorHandler):

    rootFolder = ''
    try:

        version =  platform.platform()
        verReg = re.search('Windows-(\w+)-', version)
        if verReg != None :
            if verReg.group(1) == 'XP' or verReg.group(1) == '2003Server':
                localeStr = locale.getdefaultlocale()[1]
                if localeStr == 'cp950':
                    rootFolder = os.getenv('USERPROFILE')+"\\「開始」功能表\\程式集\\啟動\\"
                else:
                    rootFolder = os.getenv('USERPROFILE')+"\\Start Menu\\Programs\\Startup\\"

            else:
                rootFolder = os.getenv('USERPROFILE')+"\\AppData\\Roaming\\Microsoft\\Windows\\Start Menu\\Programs\\Startup\\"

    except Exception:

        errorInfo={Global.ERROR_CALLSTACK:traceback, Global.ERROR_MESSAGE:traceback.format_exc()}
        errorHandler.handler(errorInfo);

    return rootFolder

def getPythonPath():
    for pyPath in sys.path:
        if os.path.exists(pyPath+"\\python.exe"):
            return pyPath+"\\python.exe"
    return None


from apiclient.discovery import build
from oauth2client.service_account import ServiceAccountCredentials
from httplib2 import Http
import os
import logging
from pandas_bigquery.exceptions import *
from pandas_bigquery.datasets import Datasets
from pandas_bigquery.tables import Tables
from pandas_bigquery.tabledata import Tabledata
from pandas_bigquery.jobs import Jobs
from pandas import DataFrame, concat
from pandas.compat import lzip
from datetime import datetime
from random import randint
import numpy as np
from time import sleep

try:
    from googleapiclient.errors import HttpError
except:
    from apiclient.errors import HttpError

log = logging.getLogger()

class Bigquery:
    def __init__(self, project_id=os.getenv('BIGQUERY_PROJECT'), private_key_path=os.getenv('BIGQUERY_KEY_PATH')):

        if private_key_path is None:
            raise RuntimeError('Invalid bigquery key path')

        self.project_id = project_id
        self.private_key_path = private_key_path

        credentials = ServiceAccountCredentials.from_json_keyfile_name(
            private_key_path, ['https://www.googleapis.com/auth/bigquery'])

        http_auth = credentials.authorize(Http())
        self._service = build('bigquery', 'v2', http=http_auth)

        with open(private_key_path) as data_file:
            self.private_key = data_file.read()

        self._tables = Tables(self.project_id, private_key=self.private_key_path)
        self._datasets = Datasets(self.project_id, private_key=self.private_key_path)
        self._jobs = Jobs(self.project_id, private_key=self.private_key_path)
        self._tabledata = Tabledata(self.project_id, private_key=self.private_key_path)

    @staticmethod
    def _parse_data(schema, rows):
        # see:
        # http://pandas.pydata.org/pandas-docs/dev/missing_data.html
        # #missing-data-casting-rules-and-indexing
        dtype_map = {'FLOAT': np.dtype(float),
                     'TIMESTAMP': 'M8[ns]'}

        fields = schema['fields']
        col_types = [field['type'] for field in fields]
        col_names = [str(field['name']) for field in fields]
        col_dtypes = [dtype_map.get(field['type'], object) for field in fields]
        page_array = np.zeros((len(rows),), dtype=lzip(col_names, col_dtypes))
        for row_num, raw_row in enumerate(rows):
            entries = raw_row.get('f', [])
            for col_num, field_type in enumerate(col_types):
                field_value = Bigquery._parse_entry(entries[col_num].get('v', ''),
                                                    field_type)
                page_array[row_num][col_num] = field_value

        return DataFrame(page_array, columns=col_names)

    @staticmethod
    def _parse_entry(field_value, field_type):
        if field_value is None or field_value == 'null':
            return None
        if field_type == 'INTEGER':
            return int(field_value)
        elif field_type == 'FLOAT':
            return float(field_value)
        elif field_type == 'TIMESTAMP':
            timestamp = datetime.utcfromtimestamp(float(field_value))
            return np.datetime64(timestamp)
        elif field_type == 'BOOLEAN':
            return field_value == 'true'
        return field_value

    @property
    def datasets(self):
        return self._datasets

    @property
    def tables(self):
        return self._tables

    @property
    def jobs(self):
        return self._jobs

    @property
    def tabledata(self):
        return self._tabledata

    def query_async(self, query, dialect='standard', priority='BATCH', strict=True, **kwargs):

        if Bigquery._check_strict(query, strict):
            raise Exception('Strict mode error',
                            "partition reference not found in query, "
                            "please add a partitiondate, _partitiontime or _table_suffix restriction "
                            "in the where-clause or set strict = False if you are confident in what you're doing.")

        config = kwargs.get('configuration')
        if config is not None and 'query' in config:
            config['query']['priority'] = priority
        else:
            config = {
                'query': {
                    'priority': priority
                }
            }
        config['query']['useLegacySql'] = dialect == 'legacy'

        return self.jobs.query_async(query, configuration=config)

    def query(self, query, dialect='standard', priority='INTERACTIVE', strict=True, **kwargs):

        if Bigquery._check_strict(query, strict):
            raise Exception('Strict mode error',
                            "partition reference not found in query, "
                            "please add a partitiondate, _partitiontime or _table_suffix restriction "
                            "in the where-clause or set strict = False if you are confident in what you're doing.")

        if dialect not in ('legacy', 'standard'):
            raise ValueError("'{0}' is not valid for dialect".format(dialect))

        if priority not in ('BATCH', 'INTERACTIVE'):
            raise ValueError("'{0}' is not valid for priority".format(dialect))

        config = kwargs.get('configuration')
        if config is not None and 'query' in config:
            config['query']['priority'] = priority
        else:
            config = {
                'query': {
                    'priority': priority
                }
            }
        config['query']['useLegacySql'] = dialect == 'legacy'

        schema, pages = self.jobs.query(query, configuration=config)
        dataframe_list = []
        while len(pages) > 0:
            page = pages.pop()
            dataframe_list.append(Bigquery._parse_data(schema, page))

        if len(dataframe_list) > 0:
            final_df = concat(dataframe_list, ignore_index=True)
        else:
            final_df = Bigquery._parse_data(schema, [])

        # cast BOOLEAN and INTEGER columns from object to bool/int
        # if they dont have any nulls
        type_map = {'BOOLEAN': bool, 'INTEGER': int}
        for field in schema['fields']:
            if field['type'] in type_map and \
                    final_df[field['name']].notnull().all():
                final_df[field['name']] = \
                    final_df[field['name']].astype(type_map[field['type']])

        self.jobs.print_elapsed_seconds(
            'Total time taken',
            datetime.now().strftime('s.\nFinished at %Y-%m-%d %H:%M:%S.'),
            0
        )

        return final_df

    def upload(self, dataframe, destination_table, if_exists='fail', chunksize=500):
        if if_exists not in ('fail', 'replace', 'append'):
            raise ValueError("'{0}' is not valid for if_exists".format(if_exists))

        if '.' not in destination_table:
            raise NotFoundException(
                "Invalid Table Name. Should be of the form 'datasetId.tableId' ")

        dataset_id, table_id = destination_table.rsplit('.', 1)

        table_schema = Tables.generate_schema_from_dataframe(dataframe)

        if Tables.contains_partition_decorator(table_id):
            root_table_id, partition_id = table_id.rsplit('$', 1)

            if not self.tables.exists(dataset_id, root_table_id):
                self.tables.insert(dataset_id, root_table_id, table_schema, body={'timePartitioning': {'type': 'DAY'}})

            if not self.tables.schema_is_subset(dataset_id,
                                                root_table_id,
                                                table_schema):
                raise InvalidSchema("Please verify that the structure and "
                                    "data types in the DataFrame match "
                                    "the schema of the destination table.")

            table_resource = self.tables.get(dataset_id, root_table_id)

            if 'timePartitioning' not in table_resource:
                raise InvalidSchema("Could not write to the partition because "
                                    "the table is not partitioned.")

            partition_exists = self.query("SELECT COUNT(*) AS num_rows FROM {0}"
                                          .format(destination_table),
                                          strict=False,
                                          priority='INTERACTIVE',
                                          dialect='legacy')['num_rows'][0] > 0

            if partition_exists and if_exists == 'fail':
                raise TableCreationError("Could not create the partition "
                                         "because it already exists. "
                                         "Change the if_exists parameter to "
                                         "append or replace data.")

            if -30 < (datetime.today() - datetime.strptime(partition_id, '%Y%m%d')).days < 360 \
                    and not (partition_exists and if_exists == 'replace'):
                self.tabledata.insert_all(dataframe, dataset_id, table_id, chunksize)

            else:
                write_disposition = 'WRITE_APPEND' if if_exists == 'append' else 'WRITE_TRUNCATE'
                temporary_table_id = '_'.join(
                    [root_table_id + '_' + partition_id,
                     str(randint(1, 100000))])
                self.tables.insert(dataset_id, temporary_table_id, table_schema)

                try:
                    self.tabledata.insert_all(dataframe, dataset_id, temporary_table_id,
                                              chunksize)
                    sleep(30)  # <- Curses Google!!!
                    self.jobs.query('select * from {0}.{1}'
                                    .format(dataset_id, temporary_table_id),
                                    configuration={
                                        'query': {
                                            'destinationTable': {
                                                'projectId': self.project_id,
                                                'datasetId': dataset_id,
                                                'tableId': table_id
                                            },
                                            'createDisposition': 'CREATE_IF_NEEDED',
                                            'writeDisposition': write_disposition,
                                            'allowLargeResults': True
                                        }
                                    })
                    self.tables.delete(dataset_id, temporary_table_id)
                except Exception as e:
                    self.tables.delete(dataset_id, temporary_table_id)
                    raise e

        else:
            if self.tables.exists(dataset_id, table_id):
                if if_exists == 'fail':
                    raise TableCreationError(
                        "Could not create the table because it "
                        "already exists. "
                        "Change the if_exists parameter to "
                        "append or replace data.")
                elif if_exists == 'replace':
                    self.tables.delete_and_recreate_table(
                        dataset_id, table_id, table_schema)
                elif if_exists == 'append':
                    if not self.tables.schema_is_subset(dataset_id,
                                                        table_id,
                                                        table_schema):
                        raise InvalidSchema("Please verify that the structure and "
                                            "data types in the DataFrame match "
                                            "the schema of the destination table.")
            else:
                self.tables.insert(dataset_id, table_id, table_schema)

            self.tabledata.insert_all(dataframe, dataset_id, table_id, chunksize)

    @staticmethod
    def _check_strict(query, strict):
        return strict and \
               not any([x in query.lower() for x in ('partitiondate', '_partitiontime', '_table_suffix', '$')])

    @staticmethod
    def run_with_retry(func, max_tries=10, **kwargs):
        for i in range(0, max_tries):
            try:
                sleep(i)
                return func(**kwargs), i + 1
            except Exception as err:
                log.warning("run_with_retry error ({2}), trying again {0}/{1}".format(i + 1, max_tries, str(err)))
                if i == max_tries - 1:
                    raise err

    # Legacy methods

    def copy(self, source_dataset, source_table, destination_dataset, destination_table):
        return self.jobs.copy(source_dataset, source_table, destination_dataset, destination_table)

    def generate_schema(self, df, default_type='STRING'):
        return Tables.generate_schema_from_dataframe(df, default_type)

    def schema(self, dataset_id, table_id):
        return self.tables.get_schema(dataset_id, table_id)

    def resource(self, dataset_id, table_id):
        return self.tables.get(dataset_id, table_id)

    def verify_schema(self, dataset_id, table_id, schema):
        return self.tables.schema_matches(dataset_id, table_id, schema)

    def table_copy(self, source_dataset_id, source_table_id, destination_dataset_id, destination_table_id):
        return self.jobs.copy(source_dataset_id, source_table_id, destination_dataset_id, destination_table_id)

    def table_create(self, dataset_id, table_id, schema, **kwargs):
        return self.tables.insert(dataset_id, table_id, schema, **kwargs)

    def table_delete(self, dataset_id, table_id):
        return self.tables.delete(dataset_id, table_id)

    @property
    def dataset_list(self):
        return self.datasets.list()

    def dataset_delete(self, dataset_id, delete_contents=False):
        return self.datasets.delete(dataset_id, delete_contents)

    def query_batch(self, query, dialect='standard', strict=True, **kwargs):
        return self.query(query=query, dialect=dialect, priority='BATCH', strict=strict, **kwargs)


#! /usr/bin/env python

import os, re, sys
import io
import setuptools
from setuptools import setup, find_packages

def has_environment_marker_support():
    """
    Tests that setuptools has support for PEP-426 environment marker support.

    The first known release to support it is 0.7 (and the earliest on PyPI seems to be 0.7.2
    so we're using that), see: http://pythonhosted.org/setuptools/history.html#id142

    References:

    * https://wheel.readthedocs.org/en/latest/index.html#defining-conditional-dependencies
    * https://www.python.org/dev/peps/pep-0426/#environment-markers
    """
    import pkg_resources
    try:
        return pkg_resources.parse_version(setuptools.__version__) >= pkg_resources.parse_version('0.7.2')
    except Exception as exc:
        sys.stderr.write("Could not test setuptool's version: %s\n" % exc)
        return False

def get_changelog():
    with io.open(os.path.join(here, 'CHANGELOG'), encoding="utf-8") as f:
        text = f.read()
    header_matches = list(re.finditer('^-+$', text, re.MULTILINE))
    # until fifth header
    text = text[:header_matches[5].start()]
    # all lines without fifth release number
    lines = text.splitlines()[:-1]
    return "Changelog\n=========\n\n" + "\n".join(lines)

if __name__ == "__main__":
    here = os.path.abspath(".")
    README = open(os.path.join(here, 'README.rst')).read()
    CHANGELOG = get_changelog()

    install_requires = ["py>=1.4.23",
                        "devpi_common<4,>=3.1.0.dev0",
                        "itsdangerous>=0.24",
                        "execnet>=1.2",
                        "pyramid>=1.5.1",
                        "waitress>=1.0.1",
                        "repoze.lru>=0.6",
                        "passlib[argon2]",
                        "pluggy>=0.3.0,<1.0",
                        ]
    extras_require = {}
    if has_environment_marker_support():
        extras_require[':python_version=="2.6"'] = ['argparse']
    elif sys.version_info < (2,7):
        install_requires.append("argparse>=1.2.1")

    setup(
      name="devpi-server",
      description="devpi-server: reliable private and pypi.python.org caching server",
      keywords="pypi realtime cache server",
      long_description="\n\n".join([README, CHANGELOG]),
      url="http://doc.devpi.net",
      version='4.2.2.dev0',
      maintainer="Holger Krekel, Florian Schulze",
      maintainer_email="holger@merlinux.eu",
      packages=find_packages(),
      include_package_data=True,
      zip_safe=False,
      license="MIT",
      classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Web Environment",
        "Intended Audience :: Developers",
        "Intended Audience :: System Administrators",
        "License :: OSI Approved :: MIT License",
        "Programming Language :: Python",
        "Topic :: Internet :: WWW/HTTP",
        "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
        ] + [
            ("Programming Language :: Python :: %s" % x) for x in
                "2.7 3.4 3.5".split()] + [
            ("Programming Language :: Python :: Implementation :: PyPy")
        ],
      install_requires=install_requires,
      extras_require=extras_require,
      entry_points = {
        'console_scripts':
                    ["devpi-server = devpi_server.main:main"],
        'devpi_server': [
            "devpi-server-auth-basic = devpi_server.auth_basic",
            "devpi-server-auth-devpi = devpi_server.auth_devpi",
            "devpi-server-sqlite = devpi_server.keyfs_sqlite",
            "devpi-server-sqlite-fs = devpi_server.keyfs_sqlite_fs"],
        'devpi_web': [
            "devpi-server-status = devpi_server.views"]})


import os

import cv2
import tensorflow as tf
import tensorflow_hub as hub

INPUT_DIR_PATH = 'input'
OUTPUT_DIR_PATH = 'output'
LINE_COLOR = (0, 255, 0)
LINE_THICKNESS = 2
PERIOD = 25  # a frame goes into the detector once every [period] frames
MIN_SCORE = 0.5  # the minimal score to detect a person

module_handle = "https://tfhub.dev/google/faster_rcnn/openimages_v4/inception_resnet_v2/1"
detector = hub.load(module_handle).signatures['default']

class VideoManager:
    def __init__(self, video_path, ouput_dir):
        """try to read the file to check it's a video and store some data"""
        self.video_path = video_path
        self.video_name = os.path.splitext(os.path.basename(video_path))[0]
        self.fourcc = cv2.VideoWriter_fourcc(*'DIVX')
        self.ouput_video_path = os.path.join(ouput_dir, self.video_name) + '.avi'

        try:
            cap = cv2.VideoCapture(self.video_path)
            ret, frame = cap.read()
            self.height, self.width, self.channels = frame.shape
            self.fps = cap.get(cv2.CAP_PROP_FPS)
            self.total_frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
            self.video_can_be_read = True

        except Exception:
            self.video_can_be_read = False
            print("Unable to read video: ", self.video_name)

    def create_video(self, period=25, min_score=0.5):
        """create an output video with boxes around humans"""
        print("="*50)
        if not self.video_can_be_read:
            print("Unable to read video: ", self.video_name)
            return

        print("processing video:", self.video_name)
        print("Number of frames: ", self.total_frame_count)
        print("dimensions: ", self.width, self.height, self.channels)
        print("frame rate: ", self.fps)

        video = cv2.VideoCapture(self.video_path)
        out = cv2.VideoWriter(self.ouput_video_path, self.fourcc, self.fps, (self.width, self.height))

        frame_count = 0
        boxes = []
        while video.isOpened():

            ret, frame = video.read()
            if not ret:  # no frame anymore, end of the video reached
                break

            if frame_count % period == 0:  # detect boxes once every [period] frames
                boxes = self.detect_box(frame, min_score)

            for ymin, xmin, ymax, xmax in boxes:  # draw a box around every person detected
                min_corner = (int(xmin * self.width), int(ymin * self.height))
                max_corner = (int(xmax * self.width), int(ymax * self.height))
                cv2.rectangle(frame, min_corner, max_corner, LINE_COLOR, LINE_THICKNESS)

            out.write(frame)
            frame_count += 1

            # display how much is done in real time
            print("{0}% done".format(int(frame_count / self.total_frame_count * 100)), end='\r')

        out.release()
        video.release()

    def detect_box(self, img, min_score=0.5):
        """Detect any person in an image with the given minimal score and return the bounding boxes"""
        converted_img = tf.image.convert_image_dtype(img, tf.float32)[tf.newaxis, ...]
        result = detector(converted_img)

        result = {key: value.numpy() for key, value in result.items()}
        boxes = []
        for entity, box, score in zip(result["detection_class_entities"],
                                      result["detection_boxes"],
                                      result["detection_scores"]):
            if entity in [b'Person', b'Man', b'Woman'] and score >= min_score:
                boxes.append(box)
        return boxes

if __name__ == '__main__':
    if not os.path.exists(OUTPUT_DIR_PATH):
        os.makedirs(OUTPUT_DIR_PATH)
    for root, _, files in os.walk(INPUT_DIR_PATH):
        for name in files:
            file_path = os.path.join(root, name)
            video_manager = VideoManager(file_path, OUTPUT_DIR_PATH)
            video_manager.create_video(PERIOD, MIN_SCORE)


import torch
import torch.nn as nn

def pixel_unshuffle(input, upscale_factor):
    r"""Rearranges elements in a Tensor of shape :math:`(C, rH, rW)` to a
    tensor of shape :math:`(*, r^2C, H, W)`.
    written by: Zhaoyi Yan, https://github.com/Zhaoyi-Yan
    and Kai Zhang, https://github.com/cszn/FFDNet
    01/01/2019
    """
    batch_size, channels, in_height, in_width = input.size()

    out_height = in_height // upscale_factor
    out_width = in_width // upscale_factor

    input_view = input.contiguous().view(
        batch_size, channels, out_height, upscale_factor,
        out_width, upscale_factor)

    channels *= upscale_factor ** 2
    unshuffle_out = input_view.permute(0, 1, 3, 5, 2, 4).contiguous()
    return unshuffle_out.view(batch_size, channels, out_height, out_width)

class PixelUnShuffle(nn.Module):
    r"""Rearranges elements in a Tensor of shape :math:`(C, rH, rW)` to a
    tensor of shape :math:`(*, r^2C, H, W)`.
    written by: Zhaoyi Yan, https://github.com/Zhaoyi-Yan
    and Kai Zhang, https://github.com/cszn/FFDNet
    01/01/2019
    """

    def __init__(self, upscale_factor):
        super(PixelUnShuffle, self).__init__()
        self.upscale_factor = upscale_factor

    def forward(self, input):
        return pixel_unshuffle(input, self.upscale_factor)

    def extra_repr(self):
        return 'upscale_factor={}'.format(self.upscale_factor)


from .layout.grid.elements.background import Background_Element
from .layout.grid.elements.border import Border_Element
from ....utils.checks import is_mouse_in_quad

from ..widget.header_controller import Widget_Header_Controller

import copy
import time
import math

class Scroll_Widget():

    def __init__(self, db, show_bar=True):

        # Data
        self.db = db
        self.win_key = ""

        # Containers
        self.layout = None
        self.layout_copy = None

        # Header
        self.collapsable = False
        self.header_layout = None
        self.header_controller = Widget_Header_Controller(db=self.db)
        self.header_height_percent = 0
        self.header_height = 0
        self.min_max_height = (0,0)

        # Layout Dims
        self.layout_width = 0
        self.bottom_left = (0, 0)
        self.bottom_right = (0, 0)
        self.top_left = (0, 0)
        self.top_right = (0, 0)

        # Layout Subdivide
        self.split_count_override = False
        self.split_count = 1
        self.start_slice = 0
        self.end_slice = 0

        # Scroll Bar
        self.hide_scroll_bar_override = not show_bar
        self.scroll_bar = Scroll_Bar(db=self.db, show_bar=show_bar)
        self.bar_width = 0
        self.body_height = 0
        self.show_scroll_bar = True
        self.scroll_speed = 0

        # Scroll Bar Reset
        self.first_cycle = True
        self.last_row_count = 0

        # Scroll bar behaviors
        self.full_body_scroll_detection = True

    def setup(self, bottom_left=(0,0), bottom_right=(0,0), top_left=(0,0), top_right=(0,0)):
        '''The head function for the dimensions setup across the layouts.'''

        self.bottom_left = bottom_left
        self.bottom_right = bottom_right
        self.top_left = top_left
        self.top_right = top_right

        self.header_dims_setup()

        self.subdivide_body()

        # Body / Scroll Bar
        if self.layout != None:
            # Adjust the rendering layout
            self.layout_alter()
            self.body_setup()

            if self.show_scroll_bar:
                self.scroll_bar_setup()

    def header_dims_setup(self):

        self.header_height = (self.header_height_percent * (self.top_left[1] - self.bottom_left[1])) * .01

        if self.header_height > self.min_max_height[1]:
            self.header_height = self.min_max_height[1]

        elif self.header_height < self.min_max_height[0]:
            self.header_height = self.min_max_height[0]

        # Widget header controller adds the drop down toggle
        self.header_controller.header_layout = self.header_layout
        self.header_controller.collapsable = self.collapsable

        # Header
        if self.header_layout != None:
            self.header_setup()
        else:
            self.body_height = self.top_left[1] - self.bottom_left[1]

    def header_setup(self):
        '''Set the dimensions on the layout's header. '''

        self.body_height = (self.top_left[1] - self.bottom_left[1]) - self.header_height
        self.header_controller.body_height = self.body_height
        self.header_controller.bottom_left = self.bottom_left
        self.header_controller.bottom_right = self.bottom_right
        self.header_controller.top_left = self.top_left
        self.header_controller.top_right = self.top_right

        self.header_controller.setup()

    def subdivide_body(self):
        '''Set the split count for the body.'''

        # Subdivide the window
        height = self.top_left[1] - self.bottom_left[1]
        row_count = len(self.layout.rows)
        row_height = height / row_count if row_count > 0 else 1

        cell_height_split = self.db.ui_event.min_height * self.db.scale_factor

        if row_height > cell_height_split:
            if self.split_count_override == True:
                self.show_scroll_bar = True

            else:
                self.split_count = len(self.layout.rows)
                self.show_scroll_bar = False

        else:
            if self.split_count_override == False:
                self.split_count = int(height / cell_height_split)
            self.show_scroll_bar = True

        self.scroll_speed = row_height

    def layout_alter(self):
        '''Make a copy of the original layout. The copy will be drawn.'''

        self.layout_copy = copy.copy(self.layout)

        self.slice_rows()

        # Set row heights
        height_percent = 0
        if self.split_count < len(self.layout.rows):
            height_percent = 100 / self.split_count
        elif len(self.layout.rows) == 0:
            height_percent = 100
        else:
           height_percent = 100 / len(self.layout.rows)

        for row in self.layout_copy.rows:
            row.height_percent = height_percent

    def slice_rows(self):
        '''Slice the rows out of the original layout and insert them into the copy.'''

        self.reset_scroll_bar_on_change()

        row_height = self.body_height / len(self.layout.rows) if len(self.layout.rows) > 0 else self.body_height
        bottom_offset = self.scroll_bar.bottom_offset
        index = bottom_offset / row_height

        if index > int(index) + .5:
            index = math.ceil(index)
        else:
            index = int(index)

        # Prevent the scroll bar being at the top of the window slicing incorrectly when scaling window
        if self.split_count == len(self.layout.rows):
            index = 0

        self.start_slice = index
        self.end_slice = self.split_count + self.start_slice

        self.layout_copy.rows = self.layout_copy.rows[self.start_slice : self.end_slice]

    def reset_scroll_bar_on_change(self):
        '''Resets the scroll bar bottom offset if the list is rebuilt and the length is different.'''

        if self.first_cycle == False:
            if len(self.layout.rows) != self.last_row_count:
                self.last_row_count = len(self.layout.rows)
                self.scroll_bar.bottom_offset = 0

        if self.first_cycle == True:
            self.last_row_count = len(self.layout.rows)
            self.first_cycle = False

    def body_setup(self):
        '''Set the dimensions on the layouts body.'''

        self.layout_width = self.bottom_right[0] - self.bottom_left[0]

        if self.show_scroll_bar:
            self.bar_width = (self.layout_width * self.scroll_bar.bar_width_percent) * .01

        else:
            self.bar_width = 0

        if self.hide_scroll_bar_override:
            self.layout_copy.setup(
                bottom_left  =(self.bottom_left[0], self.bottom_left[1]),
                bottom_right =(self.bottom_right[0], self.bottom_right[1]),
                top_left     =(self.top_left[0], self.top_left[1] - self.header_height),
                top_right    =(self.top_right[0], self.top_right[1] - self.header_height))
        else:
            self.layout_copy.setup(
                bottom_left  =(self.bottom_left[0], self.bottom_left[1]),
                bottom_right =(self.bottom_right[0] - self.bar_width, self.bottom_right[1]),
                top_left     =(self.top_left[0], self.top_left[1] - self.header_height),
                top_right    =(self.top_right[0] - self.bar_width, self.top_right[1] - self.header_height))

    def scroll_bar_setup(self):
        '''Set the dimensions on the scroll bar.'''

        body_width = self.layout_width - self.bar_width

        row_height = self.body_height / len(self.layout.rows) if len(self.layout.rows) > 0 else self.body_height

        bar_height = row_height * self.split_count if len(self.layout.rows) > self.split_count else self.body_height

        self.scroll_bar.bar_height = bar_height
        self.scroll_bar.bottom_left = (self.bottom_left[0] + body_width, self.bottom_left[1])
        self.scroll_bar.bottom_right = (self.bottom_right[0], self.bottom_right[1])
        self.scroll_bar.top_left = (self.top_left[0] + body_width, self.top_left[1] - self.header_height)
        self.scroll_bar.top_right = (self.top_right[0], self.top_right[1] - self.header_height)

        self.scroll_bar.setup()

    def event(self):
        '''Run the event layer on the layouts and the scroll bar.'''

        # Header layout
        self.header_controller.event()

        # Body layout
        if not self.header_controller.collapsed:
            if self.layout != None:
                self.layout.event()

        # Scroll Bar
        if not self.header_controller.collapsed:
            self.scroll_bar.event()

        self.mouse_scrolling()

    def mouse_scrolling(self):

        mouse_over_scroll_box = False

        # Does the mouse scroll work on the entire window.
        if self.full_body_scroll_detection:
            mouse_over_scroll_box = is_mouse_in_quad(
                quad=(self.top_left, self.bottom_left, self.top_right, self.bottom_right),
                mouse_pos=self.db.event.mouse_pos,
                tolerance=0)
        # Only scrolls with mouse over scroll bar
        else:
            mouse_over_scroll_box = is_mouse_in_quad(
                quad=(self.scroll_bar.top_left, self.scroll_bar.bottom_left, self.scroll_bar.top_right, self.scroll_bar.bottom_right),
                mouse_pos=self.db.event.mouse_pos,
                tolerance=0)

        if mouse_over_scroll_box:
            if self.db.event.wheel_up:
                self.scroll_bar.bottom_offset += self.scroll_speed
                self.scroll_bar.clamp_offset()
            elif self.db.event.wheel_down:
                self.scroll_bar.bottom_offset -= self.scroll_speed
                self.scroll_bar.clamp_offset()

    def draw(self):
        '''Draw all the layouts.'''

        # Header layout
        self.header_controller.draw()

        if not self.header_controller.collapsed:

            # Body layout
            if self.layout_copy != None:
                self.layout_copy.draw()

            # Scroll Bar
            if not self.hide_scroll_bar_override:
                if self.show_scroll_bar:
                    self.scroll_bar.draw()

class Scroll_Bar():

    def __init__(self, db, show_bar=True):

        self.db = db

        # Dims
        self.bar_width_percent = 6
        self.bar_height = 0
        self.bottom_offset = 0

        self.bottom_left = (0, 0)
        self.bottom_right = (0, 0)
        self.top_left = (0, 0)
        self.top_right = (0, 0)

        # Bar overrides
        self.show_bar = show_bar

        # Event
        self.dragging = False
        self.mouse_is_over = False

        # Elements
        self.elements()

    def elements(self):

        self.border = Border_Element()
        self.border.db = self.db

        self.bar = Background_Element()
        self.bar.db = self.db

        self.bar_border = Border_Element()
        self.bar_border.db = self.db

    def setup(self):
        '''Setup the dimensions for the drawable elements.'''

        self.setup_border()
        self.setup_bar()

    def setup_border(self):

        self.border.top_left = self.top_left
        self.border.top_right = self.top_right
        self.border.bottom_left = self.bottom_left
        self.border.bottom_right = self.bottom_right
        self.border.line_width = 2

    def setup_bar(self):

        self.bar.top_left = (self.bottom_left[0], self.bottom_left[1] + self.bar_height + self.bottom_offset)
        self.bar.top_right = (self.bottom_right[0], self.bottom_right[1] + self.bar_height + self.bottom_offset)
        self.bar.bottom_left = (self.bottom_left[0], self.bottom_left[1] + self.bottom_offset)
        self.bar.bottom_right = (self.bottom_right[0], self.bottom_right[1] + self.bottom_offset)
        self.bar.primary = False

        self.bar_border.top_left = (self.bottom_left[0], self.bottom_left[1] + self.bar_height + self.bottom_offset)
        self.bar_border.top_right = (self.bottom_right[0], self.bottom_right[1] + self.bar_height + self.bottom_offset)
        self.bar_border.bottom_left = (self.bottom_left[0], self.bottom_left[1] + self.bottom_offset)
        self.bar_border.bottom_right = (self.bottom_right[0], self.bottom_right[1] + self.bottom_offset)

        self.clamp_bar()

    def clamp_bar(self):

        if self.bar.top_left[1] > self.top_left[1]:

            self.bar.top_left = (self.bar.top_left[0], self.top_left[1])
            self.bar.top_right = (self.bar.top_right[0], self.top_right[1])
            self.bar.bottom_left = (self.bar.top_left[0], self.bar.top_left[1] - self.bar_height)
            self.bar.bottom_right = (self.bar.top_right[0], self.bar.top_right[1] - self.bar_height)

            self.bar_border.top_left = (self.bar.top_left[0], self.top_left[1])
            self.bar_border.top_right = (self.bar.top_right[0], self.top_right[1])
            self.bar_border.bottom_left = (self.bar.top_left[0], self.bar.top_left[1] - self.bar_height)
            self.bar_border.bottom_right = (self.bar.top_right[0], self.bar.top_right[1] - self.bar_height)

        elif self.bar.bottom_left[1] < self.bottom_left[1]:

            self.bar.top_left = (self.bottom_left[0], self.bottom_left[1] + self.bar_height)
            self.bar.top_right = (self.bottom_right[0], self.bottom_right[1] + self.bar_height)
            self.bar.bottom_left = self.bottom_left
            self.bar.bottom_right = self.bottom_right

            self.bar_border.top_left = (self.bottom_left[0], self.bottom_left[1] + self.bar_height)
            self.bar_border.top_right = (self.bottom_right[0], self.bottom_right[1] + self.bar_height)
            self.bar_border.bottom_left = self.bottom_left
            self.bar_border.bottom_right = self.bottom_right

    def event(self):

        if self.dragging:
            self.drag()

        else:

            mouse_over_scroll_box = False
            if not self.db.ui_event.window_transforming:
                mouse_over_scroll_box = is_mouse_in_quad(
                    quad=(self.top_left, self.bottom_left, self.top_right, self.bottom_right),
                    mouse_pos=self.db.event.mouse_pos,
                    tolerance=0)

            # Mouse is over the bar
            if mouse_over_scroll_box:
                if self.db.event.mouse_dragging:
                    self.db.ui_event.cell_blocking = True
                    self.bar.drag_highlight = True
                    self.bar.highlight = False
                    self.dragging = True

                else:
                    self.db.ui_event.cell_blocking = False
                    self.bar.drag_highlight = False
                    self.bar.highlight = True

            else:
                self.bar.highlight = False

            self.mouse_is_over = mouse_over_scroll_box

        if self.db.event.left_click_released:
            self.db.ui_event.cell_blocking = False
            self.bar.drag_highlight = False
            self.dragging = False

        # Prevent bottom offset from staying at its value before scaling
        if self.db.ui_event.window_transforming:
            self.clamp_offset()

    def drag(self):

        self.bottom_offset += self.db.event.mouse_frame_diff[1]
        self.clamp_offset()

    def clamp_offset(self):

        if self.bottom_offset > self.top_left[1] - self.bottom_left[1] - self.bar_height:
            self.bottom_offset = self.top_left[1] - self.bottom_left[1] - self.bar_height

        elif self.bottom_offset < 0:
            self.bottom_offset = 0

    def draw(self):

        if self.show_bar == False:
            if self.dragging or self.mouse_is_over:
                self.bar.draw()
                self.bar_border.draw()
                self.border.draw()

        else:
            self.bar.draw()
            self.bar_border.draw()
            self.border.draw()


# todo: ignore video if there is no next action
# todo: model3, output both goal and next action
# import
from sklearn import metrics
import scipy.io
import os
import numpy as np
import copy
from operator import add

# import data
def import_data():
    data_dict = {}
    path = 'gtea/groundTruth'

    import os
    mapping = {"take": 0, "open": 1, "pour": 2, "close": 3, "shake": 4, "scoop": 5, "stir": 6, "put": 7, "fold": 8,
               "spread": 9, "background":10}

    for filename in os.listdir(path):
        file = open(path + "/" + filename, 'r')
        Lines = file.readlines()

        data_dict[filename] = []
        # Strips the newline character
        for line in Lines:
            action = line.strip()
            if action != "background":
                data_dict[filename].append(mapping[action])
    return data_dict

# find max action length
def find_max_action_length(data_dict):
    # to minimize input size, convert frames to actions first
    temp_dict = copy.deepcopy(data_dict)
    temp_dict = frames_to_action(temp_dict)
    max_action_len = 0
    for filename, action in temp_dict.items():
        if len(action) > max_action_len:
            max_action_len = len(action)
    return max_action_len

# get trainY (action class) for trainX
def get_input_y(data_dict, input_dict):
    train_y = []
    for filename, frames in data_dict.items():
        frame_len = len(input_dict[filename])
        y = [input_dict[filename][frame_len - 1][0]]
        for frame in frames[frame_len:]:
            if frame[0] != y[0]:
                y[0] = frame[0]
                break
        train_y.append(y)
    return train_y

def get_input_x_y(data_dict, trainPortion):
    new_dict = {}
    train_y = []
    train_goal_y = []
    train_action_y = []
    for filename, frames in data_dict.items():
        has_next = 0
        frameLen = len(frames)
        inputLen = round(frameLen * trainPortion)
        inputFrames = frames[:inputLen]
        y = frames[inputLen - 1]
        for frame in frames[inputLen:]:
            if frame != y:
                has_next = 1
                y = frame

                break

        if has_next:
            activity = [get_activity(filename)]
            train_goal_y.append(activity)
            train_action_y.append(y)
            new_dict[filename] = inputFrames
    return new_dict, train_goal_y, train_action_y

def add_activity(wo_activity_input,encoded_goal_y):

    w_activity_input = {}
    count = 0
    for filename, actions in wo_activity_input.items():
        w_activity_input[filename] = []
        for i in range(len(actions)):
            if actions[i] == 10 * [0]:
                w_activity_input[filename].append(actions[i] + 7 * [0])
            else:
                w_activity_input[filename].append(actions[i] + encoded_goal_y[count].tolist())
        count += 1
    return w_activity_input

def get_activity(filename):
    # convert filename to activities
    activities = {"Cheese": 0, "Coffee": 1, "CofHoney": 2, "Hotdog": 3, "Pealate": 4, "Peanut": 5, "Tea": 6}
    activity = activities[((filename.split("_", 1)[-1]).split("_",1)[0])]
    return activity

def frames_to_action(input_dict):

    for filename, frames in input_dict.items():
        action_list = []
        for frame in frames:
            if len(action_list) == 0:
                action_list.append(frame)
            else:
                if frame != action_list[-1]:
                    action_list.append(frame)
            input_dict[filename] = action_list
    return input_dict

# add padding, each video has same length of action input
def add_padding_to_x(data_dict, maxLen):
    for filename, frames in data_dict.items():
        data_dict[filename] = (maxLen - len(frames)) * [-1] + frames
    return data_dict

def get_action_label():
    # create frame_label for one hot encoding
    action_label = {}
    for i in range(0, 10):
        action_label[i] = 10 * [0]
        action_label[i][i] = 1
    action_label[-1] = 10 * [0]
    return action_label

# perform one-hot encoding to x
def feature_encoding(input_dict, action_label):
    for filename, frames in input_dict.items():
        # get corresponding one-hot encode
        new = []
        for each in frames:
            new.append(action_label[each])
        input_dict[filename] = new
    return input_dict

def label_encoding(train_y, action_label, is_goal):
    new_trainY = []
    if is_goal:
        one_hot = {}
        temp_y = copy.deepcopy(train_y)
        for i in range(7):
            encode = 7 * [0]
            encode[i] = 1
            one_hot[i] = encode

        for i in range(len(temp_y)):
            temp_y[i] = one_hot[temp_y[i][0]]
        return temp_y

    else:
        for frames in train_y:
            new_trainY.append(action_label[frames])
        return new_trainY

def get_sample_weight(train_x):
    sample_weight = []
    for actions in train_x:
        sample_weight.append([])
        for each in actions:
            if each == len(each) * [0]:
                sample_weight[-1].append(0)
            else:
                sample_weight[-1].append(1)
    return sample_weight

from keras.models import Sequential
from keras.layers import LSTM
from tcn import TCN
from tensorflow.keras.layers import Dense
from tensorflow.keras import Input, Model

def lstm_model(max_action_len, with_activity):
    if not with_activity:
        data_dimension = 10
    else:
        data_dimension = 17
    model = Sequential()
    model.add(LSTM(128, input_shape=(max_action_len, data_dimension), return_sequences=False))
    model.add(Dense(10, activation='softmax'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    model.summary()
    return model

def tcn_model(max_action_len, with_activity):
    if not with_activity:
        data_dimension = 10
    else:
        data_dimension = 17
    i = Input(shape=(max_action_len, data_dimension))
    o = TCN(128, return_sequences=True)(i)  # The TCN layers are here.
    o = TCN(64, return_sequences=False)(i)
    o = Dense(10, activation="softmax")(o)
    model = Model(inputs=[i], outputs=[o])
    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

    model.summary()
    return model

# transformer

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM, RepeatVector
from keras.layers import Dropout, Masking, TimeDistributed, Activation
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers

class MultiHeadSelfAttention(tf.keras.layers.Layer):
    def __init__(self, embed_dim, num_heads=8):
        super(MultiHeadSelfAttention, self).__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        if embed_dim % num_heads != 0:
            raise ValueError(
                "embedding dimension = {embed_dim} should be divisible by number of heads = {num_heads}"
            )
        self.projection_dim = embed_dim // num_heads
        self.query_dense = Dense(embed_dim)
        self.key_dense = Dense(embed_dim)
        self.value_dense = Dense(embed_dim)
        self.combine_heads = Dense(embed_dim)

    def attention(self, query, key, value):
        score = tf.matmul(query, key, transpose_b=True)
        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)
        scaled_score = score / tf.math.sqrt(dim_key)
        weights = tf.nn.softmax(scaled_score, axis=-1)
        output = tf.matmul(weights, value)
        return output, weights

    def separate_heads(self, x, batch_size):
        x = tf.reshape(
            x, (batch_size, -1, self.num_heads, self.projection_dim)
        )
        return tf.transpose(x, perm=[0, 2, 1, 3])

    def call(self, inputs):
        batch_size = tf.shape(inputs)[0]
        query = self.query_dense(inputs)
        key = self.key_dense(inputs)
        value = self.value_dense(inputs)
        query = self.separate_heads(query, batch_size)
        key = self.separate_heads(key, batch_size)
        value = self.separate_heads(value, batch_size)

        attention, weights = self.attention(query, key, value)
        attention = tf.transpose(attention, perm=[0, 2, 1, 3])
        concat_attention = tf.reshape(
            attention, (batch_size, -1, self.embed_dim)
        )
        output = self.combine_heads(concat_attention)
        return output

class TransformerBlock(layers.Layer):
    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):
        super(TransformerBlock, self).__init__()
        self.att = MultiHeadSelfAttention(embed_dim, 1)
        self.ffn = keras.Sequential(
            [layers.Dense(ff_dim, activation="relu"), layers.Dense(embed_dim), ]
        )
        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)
        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)
        self.dropout1 = layers.Dropout(rate)
        self.dropout2 = layers.Dropout(rate)

    def call(self, inputs, training):
        attn_output = self.att(inputs)
        attn_output = self.dropout1(attn_output, training=training)
        out1 = self.layernorm1(inputs + attn_output)
        ffn_output = self.ffn(out1)
        ffn_output = self.dropout2(ffn_output, training=training)
        return self.layernorm2(out1 + ffn_output)

# class PositionEmbedding(layers.Layer):
#     def __init__(self, maxlen, embed_dim):
#         super(PositionEmbedding, self).__init__()
#         self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)
#
#     def call(self, x):
#         maxlen = tf.shape(x)[-1]
#         positions = tf.range(start=0, limit=maxlen, delta=1)
#         positions = self.pos_emb(positions)
#         return positions

def transformer_model(max_action_len, with_activity):
    if not with_activity:
        data_dimension = 10
    else:
        data_dimension = 17
    ff_dim = 32  # Hidden layer size in feed forward network inside transformer

    inputs = layers.Input(shape=(max_action_len, data_dimension))
    transformer_block = TransformerBlock(data_dimension, 1, ff_dim)
    x = transformer_block(inputs)
    # x = layers.GlobalAveragePooling1D()(x)
    # x = layers.Dropout(0.1)(x)
    x = layers.Dense(128, activation="relu")(x)
    x = layers.Dense(64, activation="relu")(x)
    x = layers.Flatten()(x)
    outputs = layers.Dense(10, activation="softmax")(x)
    model = keras.Model(inputs=inputs, outputs=outputs)
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'],
                  sample_weight_mode="temporal")
    # sample_weight_mode = "temporal"
    model.summary()
    return model

# cross validation
def train_model(trainX, trainY, model, sample_weight):
    print(np.array(trainX).shape, np.array(trainY).shape)
    model.fit(np.array(trainX), np.array(trainY), epochs=20, batch_size=128, shuffle = True)

# def evaluation(testX, testY, model, max_timesteps):
#     predictions = model.predict(testX)
#     results = []
#     count = 0
#     for i in range(len(predictions)):
#         result = np.array([0] * 47)
#         index = predictions[i].argmax(axis=-1)
#         result[index] = 1
#         pre = metrics.accuracy_score(result, testY[i])
#         if pre == 1:
#             count += 1
#     print(count / len(testY))
#     return count / len(testY)

def evaluation(testX, testY, model, max_timesteps):

    length = 10
    predictions = model.predict(testX)
    #     print(predictions)
    results = []
    accuracy_list = []
    for i in range(len(predictions)):
        each_video = []
        result = np.array([0] * length)
        index = predictions[i].argmax(axis=-1)
        result[index] = 1
        results.append(result)
    correct = 0
    # loop each video
    for j in range(len(testY)):

        if (testY[j] == results[j]).all():
            correct += 1

    accuracy_list.append(correct / len(results))
    print("accuracy", ": ", correct, "/", len(results), "=", correct / len(results))
    return correct / len(results), results

def cross_validation(input_dict, action_y, model, max_timesteps):

    file_count = 0
    s1_train_x, s2_train_x, s3_train_x, s4_train_x = [], [], [], []
    s1_train_action_y, s2_train_action_y, s3_train_action_y, s4_train_action_y = [], [], [], []

    s1_test_x,s2_test_x,s3_test_x,s4_test_x = [], [], [], []
    s1_test_action_y,s2_test_action_y,s3_test_action_y,s4_test_action_y = [], [], [], []
    count = 0

    for filename, frames in input_dict.items():
        if filename[0:2] == "S1":
            s1_test_x.append(input_dict[filename])
            s1_test_action_y.append(action_y[file_count])

            s2_train_x.append(input_dict[filename])
            s2_train_action_y.append(action_y[file_count])

            s3_train_x.append(input_dict[filename])
            s3_train_action_y.append(action_y[file_count])

            s4_train_x.append(input_dict[filename])
            s4_train_action_y.append(action_y[file_count])

        elif filename[0:2] == "S2":
            s2_test_x.append(input_dict[filename])
            s2_test_action_y.append(action_y[file_count])

            s1_train_x.append(input_dict[filename])
            s1_train_action_y.append(action_y[file_count])

            s3_train_x.append(input_dict[filename])
            s3_train_action_y.append(action_y[file_count])

            s4_train_x.append(input_dict[filename])
            s4_train_action_y.append(action_y[file_count])

        elif filename[0:2] == "S3":
            s3_test_x.append(input_dict[filename])
            s3_test_action_y.append(action_y[file_count])

            s1_train_x.append(input_dict[filename])
            s1_train_action_y.append(action_y[file_count])

            s2_train_x.append(input_dict[filename])
            s2_train_action_y.append(action_y[file_count])

            s4_train_x.append(input_dict[filename])
            s4_train_action_y.append(action_y[file_count])

        elif filename[0:2] == "S4":
            s4_test_x.append(input_dict[filename])
            s4_test_action_y.append(action_y[file_count])

            s1_train_x.append(input_dict[filename])
            s1_train_action_y.append(action_y[file_count])

            s2_train_x.append(input_dict[filename])
            s2_train_action_y.append(action_y[file_count])

            s3_train_x.append(input_dict[filename])
            s3_train_action_y.append(action_y[file_count])

        file_count += 1

    splits_train_x = [s1_train_x, s2_train_x, s3_train_x, s4_train_x]
    splits_train_action_y = [s1_train_action_y, s2_train_action_y, s3_train_action_y, s4_train_action_y]
    splits_test_x = [s1_test_x, s2_test_x, s3_test_x, s4_test_x]
    splits_test_action_y = [s1_test_action_y, s2_test_action_y, s3_test_action_y, s4_test_action_y]

    action_acc = 0
    predicted_goal_list = []
    predicted_action_list = []
    for i in range(4):
        trainX = splits_train_x[i]
        trainY = splits_train_action_y[i]

        testX = splits_test_x[i]
        testY = splits_test_action_y[i]

        sample_weight = get_sample_weight(trainX)
        train_model(trainX, trainY, model, sample_weight)
        temp_action_acc, predicted_action = evaluation(testX, testY, model, max_timesteps)
        predicted_action_list.append(predicted_action)
        action_acc += temp_action_acc
    action_acc = action_acc / 4
    return action_acc

def display_acc(results, model_name, w_or_wo):
    # loop through result for each input proportion

    print("Model: ", model_name, "     With_Activity: ", w_or_wo)
    for proportion, acc in results.items():
        #    print("Input(%): ", proportion * 100)
        print(round(acc, 5), " ", end="")
    print("")

def run_model():
    data_dict = import_data()
    max_action_len = find_max_action_length(data_dict)
    action_label = get_action_label()
    wo_lstm = lstm_model(max_action_len, 0)
    w_lstm = lstm_model(max_action_len, 1)
    wo_tcn = tcn_model(max_action_len, 0)
    w_tcn = tcn_model(max_action_len, 1)
    wo_transformer = transformer_model(max_action_len, 0)
    w_transformer = transformer_model(max_action_len, 1)

    wo_lstm_results = {}
    w_lstm_results = {}
    wo_tcn_results = {}
    w_tcn_results = {}
    wo_transformer_results = {}
    w_transformer_results = {}
    # get input data with different proportion
    # input_proportion = [0.1]
    # input_proportion = [0.1, 0.2, 0.3, 0.4, 0.5]
    input_proportion = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
    for proportion in input_proportion:
        #     input_dict = get_input_x(data_dict, proportion)
        #     train_y = get_input_y(data_dict, input_dict)
        input_dict, train_goal_y, train_action_y = get_input_x_y(data_dict, proportion)
        input_dict = frames_to_action(input_dict)
        input_dict = add_padding_to_x(input_dict, max_action_len)
        # train_y = add_padding_to_y(train_y, max_action_len)
        encoded_goal_y = np.array(label_encoding(train_goal_y, action_label, 1))
        encoded_action_y = np.array(label_encoding(train_action_y, action_label, 0))
        wo_activity_input = feature_encoding(input_dict, action_label)
        w_activity_input = add_activity(wo_activity_input, encoded_goal_y)

        # # # # lstm
        wo_lstm_result = cross_validation(wo_activity_input, encoded_action_y, wo_lstm, max_action_len)
        wo_lstm_results[proportion] = wo_lstm_result
        w_lstm_result = cross_validation(w_activity_input, encoded_action_y, w_lstm, max_action_len)
        w_lstm_results[proportion] = w_lstm_result
        #
        # # tcn
        # wo_tcn_result = cross_validation(wo_activity_input, encoded_action_y, wo_lstm, max_action_len)
        # wo_tcn_results[proportion] = wo_tcn_result
        # w_tcn_result = cross_validation(w_activity_input, encoded_action_y, w_lstm, max_action_len)
        # w_tcn_results[proportion] = w_tcn_result
        # #
        # # # transformer
        # wo_transformer_result = cross_validation(wo_activity_input, encoded_action_y, wo_lstm, max_action_len)
        # wo_transformer_results[proportion] = wo_transformer_result
        # w_transformer_result = cross_validation(w_activity_input, encoded_action_y, w_lstm, max_action_len)
        # w_transformer_results[proportion] = w_transformer_result
    #
    display_acc(wo_lstm_results, "LSTM", "No")
    display_acc(w_lstm_results, "LSTM", "Yes")
    display_acc(wo_tcn_results, "TCN", "No")
    display_acc(w_tcn_results, "TCN", "Yes")
    display_acc(wo_transformer_results, "Transformer", "No")
    display_acc(w_transformer_results, "Transformer", "Yes")

run_model()


import numpy as np
import scipy.io as sc
from collections import deque
from random import shuffle

class NeuralNetwork:
    "Neural Network Class"

    def __init__(self, x, y, hidd_lay, units_num_in_layer, lerning_rate, lmbd):          # hidd_lay
        self.x = deque(x)
        self.y = deque(y)
        self.complete = 0
        self.m, self.n = x.shape
        self.y_vector = np.zeros(shape=(10))
        self.ny = 10
        self.hidd_lay = hidd_lay
        self.units_num_in_layer = units_num_in_layer

        self.b = list(np.random.uniform(low=0.0001, high=0.001, size=(hidd_lay, units_num_in_layer)))
        self.b.append(np.random.uniform(low=0.0001, high=0.001, size=(self.ny)))

        self.w = []
        self.w.append(np.random.uniform(low=0.0001, high=0.0002, size=(self.n, units_num_in_layer)))
        for i in range(hidd_lay - 1):
            self.w.append(np.random.uniform(low=0.0001, high=0.001, size=(units_num_in_layer, units_num_in_layer)))

        #10 is a countof units in last layer
        self.w.append(np.random.uniform(low=0.0001, high=0.001, size=(units_num_in_layer, self.ny)))

        self.a = [None] * (hidd_lay + 2)
        self.z = [None] * (hidd_lay + 1)

        self.delta_b = list(np.zeros(shape=(hidd_lay, units_num_in_layer)))
        self.delta_b.append(np.zeros(shape=(self.ny)))

        self.delta_w = []
        for i in range(hidd_lay - 1):
            self.delta_w.append(np.zeros(shape=(units_num_in_layer, units_num_in_layer)))
        self.delta_w.append(np.zeros(shape=(units_num_in_layer, self.ny)))

        self.lerning_rate = lerning_rate
        self.lmbd = lmbd
        self.counter = 0
        #print(self.b)

    def forward_prop(self):
        self.a[0] = (self.x.popleft())
        if self.complete == 1:
            print(self.a[0])

        #compute z for 1-st hidden layer
        self.z[0] = (np.asarray([i + self.b[0][j] for (i, j) in zip(np.dot(self.a[0], self.w[0]), range(self.units_num_in_layer))]).flatten())
        self.a[1] = (np.asarray([NeuralNetwork.sigm_fun(z) for z in self.z[0]]))

        for l in range(1, self.hidd_lay):
            self.z[l] = (np.asarray([i + self.b[l][j] for (i, j) in zip(np.dot(self.a[l], self.w[l]), range(self.units_num_in_layer))]).flatten())
            self.a[l + 1] = (np.asarray([NeuralNetwork.sigm_fun(z) for z in self.z[l]]))

        self.z[self.hidd_lay] = (np.asarray([i + self.b[-1][j] for (i, j) in zip(np.dot(self.a[self.hidd_lay], self.w[-1]), range(self.ny))]).flatten())
        self.a[self.hidd_lay + 1] = (np.asarray([NeuralNetwork.sigm_fun(z) for z in self.z[-1]]))

        #print('last a = ', self.a)
        if self.complete == 1:
            print('self.a = ', self.a)
        return self.a[-1]

    def back_prop(self):
        y_vector = self.y_vector
        y = self.y.popleft()
        if y == 10:
            y = 0
        print('index = ', y)
        y_vector[y] = 1

        print(y_vector)

        self.err_last = np.asarray([self.sigmoid_deriv(z) * i for (i, z) in zip(np.subtract(self.a[-1], y_vector), self.z[-1])])
        # print(self.err_last)
        # print('err_last = ', self.err_last)

        # print('a = ', self.a)
        # print('err_last = ', self.err_last)
        for l in range(self.hidd_lay, 0, -1):
            f_z = [self.sigmoid_deriv(z) for z in self.z[l - 1]]
            # print("ANOTHER ITERATION")
            # print('err_last type = ', type(self.err_last))
            # print('err_last = ', self.err_last)
            #print('err_last.shape = ', self.err_last.shape)
            # print('w[l].shape = ', self.w[l].shape)
            # print('w[l] = ', self.w[l])
            dJdW = np.dot(np.asmatrix(self.a[l]).T, np.asmatrix(self.err_last))
            dJdb = self.err_last
            self.err_last = np.multiply(np.dot(self.w[l], self.err_last), f_z) #new error
            if len(self.err_last) == 1:
                self.err_last = np.asarray(self.err_last).reshape(-1)
                # print('err_pre_last type = ', type(self.err_last))
                # print('err_pre_last = ', self.err_last)

            # print('self.delta_b[l - 1]', self.delta_b[l - 1])
            # print('dJdb = ', dJdb)
            self.delta_w[l - 1] = np.add(self.delta_w[l - 1], dJdW)
            self.delta_b[l] = np.add(self.delta_b[l], dJdb)
            # print('delta_b = ', self.delta_b)

            self.w[l] = np.subtract(self.w[l], (self.lerning_rate * np.add(((1. / self.m) * self.delta_w[l - 1]), self.lmbd * self.w[l])))

            self.b[l] = np.subtract(self.b[l], (self.lerning_rate * (1. / self.m) * self.delta_b[l]))
        self.counter += 1
        #print(self.b)
        # print('delta_w = ', self.delta_w)
        # print('w = ', self.w)

    def recognize(self, x):
        self.x = deque(x)
        return self.forward_prop()

    def training(self):
        while len(self.x) > 0:
            self.forward_prop()
            self.back_prop()
        print('training complete')
        self.complete = 1

    def sigmoid_deriv(self, x):
        return NeuralNetwork.sigm_fun(x) * (1 - NeuralNetwork.sigm_fun(x))

    def square_fun(x):
        return x ** 2

    def sigm_fun(x):
        return 1 / (1 + np.exp(-x))

    def count_fun(self, f, x):
        y = [f(i) for i in x]
        return y

class Adapter:

    def __init__(self, nn):
        self.n_n = nn

    def learning(self):
        for i in range(self.n_n.m):
            self.n_n.forward_prop()
            self.n_n.back_prop()
        return self.n_n.a3

mat = sc.loadmat('data3.mat')

x = mat['X']
y = mat['y']

print(type(x))
x_y = list(zip(x, y))
shuffle(x_y)
x1 = []
y1 = []
for (i, j) in x_y:
    x1.append(i)
    y1.append(j)

print(y1[0])
# x1 = x[:100]
nn = NeuralNetwork(x=np.asarray(x1), y=np.asarray(y1), hidd_lay=1, units_num_in_layer=15, lerning_rate=0.1, lmbd=0.1)
nn.forward_prop()
nn.back_prop()
# nn.training()
# x1 = x[600:700]
# nn.training()
# x1 = x[1100:1200]
# # print(nn.w)
# #print(nn.err_last)
#
# result = nn.recognize(x[:50])
# print(result)
#
# result = nn.recognize(x[600:999])
# print(result)
#

#print(v[:3500].count())

# x = np.random.rand(9)
# nn = NeuralNetwork(x, hidd_lay)
# result = nn.forward()
# print(result)
# nn.back_prop()


import traceback

from django.http import HttpResponse
from django.shortcuts import render

# Create your views here.
from django.views.decorators.csrf import csrf_exempt

from malloyou.SearchMain import search_shop

@csrf_exempt
def search(request):
    try:
        return search_shop(request)
    except Exception as e:

        res = {}
        error = traceback.format_exc(e)
        res["error"] = error
        res["status"] = False
        return HttpResponse(res)
@csrf_exempt
def index(request):
    res={
        "success":"1"
    }
    return HttpResponse(res)


import pytest
import logging
import struct

from . import speculos

def test_ins_with_no_payload(dongle):
    """
    Test that sending `INS_GET_PUBLIC_KEY` (0x03) APDU without payload
    returns a public key as a 32-byte long `bytes`.
    """
    try:
        apdu = struct.pack('>BBBBB', 0x80, 0x3, 0x0, 0x0, 0x0)
        key = dongle.exchange(apdu)
        assert type(key) == bytes
        assert len(key) == 32
    except speculos.CommException as e:
        logging.error(e)
        assert False

def test_ins_with_4_bytes_payload(dongle):
    """
    Test that sending `INS_GET_PUBLIC_KEY` (0x03) APDU with a
    4-byte (`uint32_t`) payload returns a public key as a
    32-byte long `bytes`.
    """
    try:
        apdu = struct.pack('>BBBBBI', 0x80, 0x3, 0x0, 0x0, 0x0, 0x0)
        key = dongle.exchange(apdu)
        assert len(key) == 32
    except speculos.CommException as e:
        logging.error(e)
        assert False

labels = {
    'verify', 'address', 'approve'
}

def getPubKey_ui_handler(event, buttons):
    logging.warning(event)
    label = sorted(event, key=lambda e: e['y'])[0]['text'].lower()
    logging.warning('label => %s' % label)
    if len(list(filter(lambda l: l in label, labels))) > 0:
        if label == "approve":
            buttons.press(buttons.RIGHT, buttons.LEFT, buttons.RIGHT_RELEASE, buttons.LEFT_RELEASE)
        else:
            buttons.press(buttons.RIGHT, buttons.RIGHT_RELEASE)

def test_ins_with_4_bytes_payload_and_user_approval(dongle):
    """
    Test that sending `INS_GET_PUBLIC_KEY` (0x03) APDU with a
    4-byte (`uint32_t`) payload returns a public key as a
    32-byte long `bytes`, after asking the user to approve the corresponding address
    """
    try:
        apdu = struct.pack('>BBBBBI', 0x80, 0x3, 0x80, 0x0, 0x0, 0x0)

        with dongle.screen_event_handler(getPubKey_ui_handler):
            key = dongle.exchange(apdu)

        assert len(key) == 32
    except speculos.CommException as e:
        logging.error(e)
        assert False

@pytest.fixture(params=[1, 2, 3, 5, 8, 14])
def invalid_size_apdu(request):
    l = request.param
    return struct.pack('>BBBBB%ds' % l, 0x80, 0x3, 0x0, 0x0, l, bytes(l))

def test_ins_with_invalid_paylod_sizes(dongle, invalid_size_apdu):
    """
    """
    with pytest.raises(speculos.CommException) as excinfo:
        dongle.exchange(invalid_size_apdu)
    assert excinfo.value.sw == 0x6a85

def test_ins_without_payload_returns_account_0_key(dongle):
    """
    """
    try:
        apdu = struct.pack('>BBBBB', 0x80, 0x3, 0x0, 0x0, 0x0)
        key1 = dongle.exchange(apdu)
        assert type(key1) == bytes
        assert len(key1) == 32
        apdu = struct.pack('>BBBBBI', 0x80, 0x3, 0x0, 0x0, 0x0, 0x0)
        key2 = dongle.exchange(apdu)
        assert type(key2) == bytes
        assert len(key2) == 32

        assert key1 == key2
    except speculos.CommException as e:
        logging.error(e)
        assert False

@pytest.fixture(params=[1, 2, 3, 5, 8, 14])
def account_apdu(request):
    account = request.param
    return struct.pack('>BBBBBI', 0x80, 0x3, 0x0, 0x0, 0x4, account)

def test_ins_with_non_0_account_does_not_return_account_0_key(dongle, account_apdu):
    """
    """
    try:
        key1 = dongle.exchange(struct.pack('>BBBBB', 0x80, 0x3, 0x0, 0x0, 0x0))
        assert type(key1) == bytes
        assert len(key1) == 32
        key2 = dongle.exchange(account_apdu)
        assert type(key2) == bytes
        assert len(key2) == 32

        assert key1 != key2
    except speculos.CommException as e:
        logging.error(e)
        assert False


"""
Functions to manage pushing and unloading remote records.

Skylark Kraken
(c) 2016 Coderouge
"""

import json

from kraken.redis_storage import storage

def push_record(name, model, user=None):
    """
    Push an updated object record to the client.

    :param str name: Object type
    :param dict model: Serialized object
    """
    storage.publish("records:push", json.dumps(
        {
            "user": user,
            name: model if isinstance(model, list) else [model]
        }
    ))

def remove_record(name, id, user=None):
    """
    Remove an object record from the client's store.

    :param str name: Object type
    :param str id: Object ID
    """
    storage.publish("records:purge", json.dumps(
        {
            "model": name, "id": id, "user": user
        }
    ))


import os

class Card:
    def __init__(self, par_dir, owner, id):
        parent_dir = par_dir
        self.zone = None
        self.text_file_loc = "this card's information is not contained in a text file"
        self.name = "this is a blank name"
        self.type = "this is a blank type"
        self.card_loc = os.path.join(parent_dir, "card_files", "sample_card_do_not_make_a_card_with_this_name" + ".txt")
        self.picture_loc = os.path.join(parent_dir,"card_pics", "Blank.png")
        self.rarity = "this is an undefined rarity"
        self.text = "this card has no text"
        self.original_power = 0
        self.current_power = 0
        self.is_face_up = False
        self.is_face_down = False #cards in the deck, hand, and death zone are False for both
        self.effect = None #an effect object, defined in util.create_effects()
        self.owner = owner #the player who owns this card
        self.id = id #the index number for this card. each number is unique to the card in this game, and all ids are recorded in the game.all_cards
        self.times_activated_this_turn = 0 #increment in game.resolve_layer_phase in layer.create_active_effects_for_all_cards_in_layer, reset in game.end_turn

    def set_id(self, leading_num, trailing_num):
        #leading num is the player/enemy number, and takes the thousands place. trailing num fills the hundreds, tens, and ones places
        thousands_int = leading_num * 1000
        id_num = thousands_int + trailing_num
        self.id = str(id_num)

    def set_face_up(self):
        self.is_face_up = True
        self.is_face_down = False

    def set_face_down(self):
        self.is_face_up = False
        self.is_face_down = True

    def set_neither_face_up_nor_face_down(self):
        self.is_face_up = False
        self.is_face_down = False


import numpy as np

def str_to_list(string):
	#print(string)
	#string= string[1:-1]
	listt= string.split(", ")

	listt2=[]
	for i in range(0,len(listt)):
		#print(listt[i])
		#listt2.append(float(listt[i][1:-1]))
		listt2.append(float(listt[i]))
	return listt2

def get_data_bumps(filename1, data_with_labels):

	file1 = open(filename1, 'r')
	while True:
	    line = file1.readline()
	    if not line:
	        break

	    line = line.strip()
	    line1 = line.split("*")

	    for i in range(1, len(line1)):
	    	temp=[]
	    	tmp_line = line1[i][1:-1]
	    	if len(tmp_line) >2:

		    	tmp_line = str_to_list(tmp_line)
		    	temp.append(tmp_line)
		    	temp.append(1)
		    	data_with_labels.append(temp)

	file1.close()

	return data_with_labels

def get_data_not_bumps(filename1,data):

	#data=[]
	file1 = open(filename1, 'r')
	while True:
	    line = file1.readline()
	    if not line:
	        break

	    temp=[]
	    line = line.strip()
	    line=line[1:-1]
	    line1= line.split(", ")
	    array=[]
	    for i in range(0,len(line1)):

	    	if(line1[i]!=""):
	    		array.append(float(line1[i]))
	    temp.append(array)
	    temp.append(0) #because not bump class label is 0
	    data.append(temp)

	file1.close()

	return data

bumps_path="/DATA/Cem_Internship/RNAproject/RNA_disorderedness_arrays_bumps4.txt"
bumps_path2="/DATA/Cem_Internship/disorderedness_arrays_bumps4.txt"

data_bump=[]
data_bump = get_data_bumps(bumps_path, data_bump)
data_bump = get_data_bumps(bumps_path2, data_bump)
print(" LEN- BUMPS == ", len(data_bump))

#######################################################################

not_bumps_path="/DATA/Cem_Internship/RNAproject/RNA_disorderedness_arrays_not_bumps4.txt"
not_bumps_path2="/DATA/Cem_Internship/disorderedness_arrays_not_bumps4.txt"
data_notbump=[]
data_notbump = get_data_not_bumps(not_bumps_path,data_notbump)
data_notbump = get_data_not_bumps(not_bumps_path2,data_notbump)

print("data_notbump[0]=",data_notbump[0])

print(" LEN- NOT_BUMPS == ", len(data_notbump))

data_with_labels= data_bump+ data_notbump

len34=0
notlen34=0
for i in range(0,len(data_bump)):
	if len(data_bump[i][0])==34:
		len34=len34+1
	else:
		notlen34=notlen34+1

print("len34= ", len34)
print("notlen34 = ", notlen34)

len34=0
notlen34=0
for i in range(0,len(data_notbump)):
	if len(data_notbump[i][0])==34:
		len34=len34+1
	else:
		notlen34=notlen34+1

print("len34= ", len34)
print("notlen34 = ", notlen34)

print("\n")
print(data_with_labels[0])
print("len (data_with_labels)", len(data_with_labels))

X=[] #disorderedness arrays
y=[] #corresponding class labels

for i in range(0,len(data_with_labels)):

	X.append(data_with_labels[i][0])
	y.append(data_with_labels[i][1])

X= np.array(X)
y= np.array(y)

from sklearn.utils import shuffle
X,y= shuffle(X,y) #shuffle the data

from sklearn.model_selection import train_test_split
#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.30, random_state=0)

from sklearn import preprocessing
#X_train_normalized = preprocessing.normalize(X_train,norm='l1')
#X_test_normalized = preprocessing.normalize(X_test,norm='l1')

#X_train= preprocessing.scale(X_train)
#X_test= preprocessing.scale(X_test)
#X_train,y_train= shuffle(X_train,y_train) #shuffle the data
#X_test,y_test= shuffle(X_test,y_test) #shuffle the data

from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.model_selection import cross_val_score

from sklearn.calibration import CalibratedClassifierCV

#model = SVC(kernel='linear', C=1, probability=True)
model = LinearSVC()
model = CalibratedClassifierCV(model)

model.fit(X, y)

#pred_class = model.predict(X_test)
#probability = model.predict_proba(X_test)

print("svm training finished   RNA+old  data-4")

#open all_dis_arrays.txt
file1 = open("/DATA/Cem_Internship/all_dis_arrays.txt", 'r')
file2= open("/DATA/Cem_Internship/results/svm_probs_RNA+olddata_network4.txt", "w+")

while True:
	# Get next line from file
	line = file1.readline()

	# if line is empty
	# end of file is reached
	if not line:
	    break

	line = line.strip()
	line = line.split("*")

	x=str_to_list(line[2][1:-1])
	y=[]
	y.append(x)

	prob= model.predict_proba(y)

	file2.writelines(str(line[0])) # add name of transcript
	file2.writelines(" ")
	file2.writelines(str(line[1])) #add index
	file2.writelines(" ")

	file2.writelines(str(prob)) #add probability
	file2.writelines("\n")

file1.close()
file2.close()

'''
print("pred_class = ",pred_class)
print("probability = ",probability)
sifir=0; bir=0
for i in range(0, len(pred_class)):
	if pred_class[i]==0:
		sifir=sifir+1
	else:
		bir=bir+1

print("bir= ",bir)
print("sifir=", sifir)

from sklearn.metrics import classification_report, confusion_matrix,accuracy_score

print("------------")
print( accuracy_score(y_test,pred_class))
print("------------")

dogri=0
yanlis=0
for i in range(0, len(probability)):
	if(probability[i][0]> probability[i][1]):
		if pred_class[i]==0:
			dogri=dogri+1
		else:
			yanlis=yanlis+1
	else:
		if pred_class[i]==1:
			dogri=dogri+1
		else:
			yanlis=yanlis+1

print(len(probability))
print("dogri = ",dogri)
print("yanlis = ",yanlis)
print((dogri)*1.0 / len(probability))

#clf = SVC(kernel='linear', C=1)
#scores = cross_val_score(clf, X_normalized, y, cv=5) #5-fold cross-validation
#print(scores)
#clf.probability=True
#clf.fit(X, y)

'''

'''
y_pred=clf.predict(X_test)
y_dec = clf.decision_function(X_test)
y_prob = clf.predict_proba(X_test)
print("y_pred = ", y_pred)
print("y_dec = ", y_dec)
print("y_prob = ", y_prob)
print("\n")

p = np.array(clf.decision_function(X)) # decision is a voting function
print("p = ",p)
prob = np.exp(p)/np.sum(np.exp(p),axis=1)

print("\n")
'''


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Sequence, Union

from .allele import Allele
from .result_with_multiple_alleles import ResultWithMultipleAlleles
from .species import Species

class Serotype(ResultWithMultipleAlleles):
    """
    Common base class for all result objects which have a species field.

    Contains many common helpers such as `is_human`.
    """
    def __init__(
            self,
            species: Species,
            name: str,
            alleles: Sequence[Allele],
            raw_string: Union[str, None] = None):
        ResultWithMultipleAlleles.__init__(
            self,
            species=species,
            name=name,
            alleles=alleles,
            raw_string=raw_string)

    def to_string(
            self,
            include_species=True,
            use_old_species_prefix=False):
        if include_species:
            return "%s-%s" % (
                self.species.to_string(
                    use_old_species_prefix=use_old_species_prefix),
                self.name)
        else:
            return self.name

    def compact_string(self, include_species=False, use_old_species_prefix=False):
        return self.to_string(
            include_species=include_species,
            use_old_species_prefix=use_old_species_prefix)

    def to_record(self):
        d = self.species.to_record()
        d["serotype"] = self.to_string()
        return d


import os

import flash
import torch
from flash.core.data.utils import download_data
from flash.image import ImageClassificationData, ImageClassifier
from pytorch_lightning import seed_everything
from pytorch_lightning.strategies import DDPStrategy

seed_everything(42)

def main():
    data_path = f"{os.getcwd()}/data"

    download_data(
        "https://pl-flash-data.s3.amazonaws.com/hymenoptera_data.zip", f"{data_path}/"
    )

    datamodule = ImageClassificationData.from_folders(
        train_folder=f"{data_path}/hymenoptera_data/train/",
        val_folder=f"{data_path}/hymenoptera_data/val/",
        test_folder=f"{data_path}/hymenoptera_data/test/",
        batch_size=4,
        transform_kwargs={
            "image_size": (196, 196),
            "mean": (0.485, 0.456, 0.406),
            "std": (0.229, 0.224, 0.225),
        },
    )

    model = ImageClassifier(backbone="resnet18", labels=datamodule.labels)

    trainer = flash.Trainer(
        max_epochs=100,
        gpus=torch.cuda.device_count(),
        # accelerator="gpu",
        # devices=torch.cuda.device_count(),
        # strategy=DDPStrategy(find_unused_parameters=False),
    )

    trainer.finetune(model, datamodule=datamodule, strategy="freeze")

if __name__ == "__main__":
    main()