import pygame
import random
import time
import WindowControl

#VARIABLES =======================================
# Enemy Values
enemy_WIDTH = 65
enemy_HEIGHT = 65
enemy_IMAGES = ['Assets/enemy_mouse_brown.png', 'Assets/enemy_mouse_grey.png', 'Assets/enemy_mouse_white.png']
enemy_minSpeed = 0.5
enemy_maxSpeed = 1.2
enemy_maxSpawned = 10
enemy_SPAWNRATE = 1500

#Used to reset enemy stats
enemy_maxSpeed_START = 1.2
enemy_SPAWNRATE_START = 1500

#CLASS CONSTRUCTOR ==============================
# Defines enemy as a Sprite
class Enemy(pygame.sprite.Sprite):
    def __init__(self):
        super().__init__()
        self.image = pygame.image.load(random.choice(enemy_IMAGES)) #Uses list to make random enemy
        self.rect = self.image.get_rect()
        self.x = WindowControl.window_WIDTH - 30
        self.y = random.randint(0, WindowControl.window_HEIGHT)
        self.rect.x = self.x
        self.rect.y = self.y
        self.speed = random.uniform(enemy_minSpeed, enemy_maxSpeed) #Sets a random speed between minSpeed value and maxSpeed value

    # Reusable code to reset enemy values (based on section)
    def resetPosition(self):
        self.x = WindowControl.window_WIDTH
        self.y = random.randint(0, WindowControl.window_HEIGHT)

    def resetImage(self):
        self.image = self.image = pygame.image.load(random.choice(enemy_IMAGES))
        self.rect = self.image.get_rect()

    def resetSpeed(self):
        self.speed = random.uniform(enemy_minSpeed, enemy_maxSpeed)

    # Functions used to return enemy position values
    def getX(self):
        return self.x

    def getY(self):
        return self.y

    # Used to define enemy movement
    def movement(self):
        self.x = self.x - self.speed #Moves self left based on speed
        self.rect.x = self.x
        self.rect.y = self.y

        # Destroys the sprite if it runs off the left side of the screen
        if (self.rect.right < 0):
            self.kill()

        # Ensures that the enemy does not spawn itself below the max window height
        if(self.rect.bottom > WindowControl.window_HEIGHT):
            self.y = self.y - enemy_HEIGHT

    def update(self):
        self.movement()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import tensorflow_transform as tft

from tfx.components.trainer.fn_args_utils import FnArgs
from tfx.examples.mnist import mnist_utils_native_keras_base as base

def _get_serve_tf_examples_fn(model, tf_transform_output):
  """Returns a function that parses a serialized tf.Example."""

  model.tft_layer = tf_transform_output.transform_features_layer()

  @tf.function
  def serve_tf_examples_fn(serialized_tf_examples):
    """Returns the output to be used in the serving signature."""
    feature_spec = tf_transform_output.raw_feature_spec()
    feature_spec.pop(base.LABEL_KEY)
    parsed_features = tf.io.parse_example(serialized_tf_examples, feature_spec)
    transformed_features = model.tft_layer(parsed_features)
    return model(transformed_features)

  return serve_tf_examples_fn

# TFX Transform will call this function.
def preprocessing_fn(inputs):
  """tf.transform's callback function for preprocessing inputs.
  Args:
    inputs: map from feature keys to raw not-yet-transformed features.
  Returns:
    Map from string feature key to transformed feature operations.
  """
  return base.preprocessing_fn(inputs)

# TFX Trainer will call this function.
def run_fn(fn_args: FnArgs):
  """Train the model based on given args.
  Args:
    fn_args: Holds args used to train the model as name/value pairs.
  """
  tf_transform_output = tft.TFTransformOutput(fn_args.transform_output)

  train_dataset = base.input_fn(fn_args.train_files, fn_args.data_accessor,
                                tf_transform_output, 40)
  eval_dataset = base.input_fn(fn_args.eval_files, fn_args.data_accessor,
                               tf_transform_output, 40)

  mirrored_strategy = tf.distribute.MirroredStrategy()
  with mirrored_strategy.scope():
    model = base.build_keras_model()

  # Write logs to path
  tensorboard_callback = tf.keras.callbacks.TensorBoard(
      log_dir=fn_args.model_run_dir, update_freq='batch')

  model.fit(
      train_dataset,
      steps_per_epoch=fn_args.train_steps,
      validation_data=eval_dataset,
      validation_steps=fn_args.eval_steps,
      callbacks=[tensorboard_callback])

  signatures = {
      'serving_default':
          _get_serve_tf_examples_fn(
              model, tf_transform_output).get_concrete_function(
                  tf.TensorSpec(shape=[None], dtype=tf.string, name='examples'))
  }
  model.save(fn_args.serving_model_dir, save_format='tf', signatures=signatures)


# Importing Libraries
import cv2
import numpy as np
from pyzbar.pyzbar import decode

# Reading Image
img = cv2.imread('1.png')

# Reading the file with authorized person's name
with open('myDataFile.txt') as f:
    myDataList = f.read().splitlines()

# Reading the QR-Code
for barcode in decode(img):
    myData = barcode.data.decode('utf-8')
    print(myData)

    pts = np.array([barcode.polygon], np.int32)
    pts = pts.reshape((-1, 1, 2))
    cv2.polylines(img, [pts], True, (255, 0, 255), 5)
    pts2 = barcode.rect
    cv2.putText(img, myData, (pts2[0], pts2[1]), cv2.FONT_HERSHEY_SIMPLEX,
                0.9, (255, 0, 255), 2)

# Displaying the output Image
cv2.imshow('Result', img)
cv2.waitKey(1)


import errno
import json
import math
import multiprocessing
import os
import re
import subprocess
import time
import traceback
import uuid
import zipfile
from pprint import pprint

from pathos.multiprocessing import ProcessingPool as Pool

import kb_cufflinks.core.contig_id_mapping as c_mapping
from AssemblyUtil.AssemblyUtilClient import AssemblyUtil
from DataFileUtil.DataFileUtilClient import DataFileUtil
from ExpressionUtils.ExpressionUtilsClient import ExpressionUtils
from GenomeFileUtil.GenomeFileUtilClient import GenomeFileUtil
from KBaseReport.KBaseReportClient import KBaseReport
from ReadsAlignmentUtils.ReadsAlignmentUtilsClient import ReadsAlignmentUtils
from SetAPI.SetAPIClient import SetAPI
from Workspace.WorkspaceClient import Workspace as Workspace

def log(message, prefix_newline=False):
    """Logging function, provides a hook to suppress or redirect log messages."""
    print(('\n' if prefix_newline else '') + '{0:.2f}'.format(time.time()) + ': ' + str(message))

class CufflinksUtils:
    CUFFLINKS_TOOLKIT_PATH = '/opt/cufflinks/'
    GFFREAD_TOOLKIT_PATH = '/opt/cufflinks/'

    def __init__(self, config):
        """

        :param config:
        :param logger:
        :param directory: Working directory
        :param urls: Service urls
        """
        # BEGIN_CONSTRUCTOR
        self.ws_url = config["workspace-url"]
        self.ws_url = config["workspace-url"]
        self.callback_url = config['SDK_CALLBACK_URL']
        self.srv_wiz_url = config['srv-wiz-url']
        self.token = config['KB_AUTH_TOKEN']
        self.shock_url = config['shock-url']
        self.dfu = DataFileUtil(self.callback_url)
        self.gfu = GenomeFileUtil(self.callback_url)
        self.au = AssemblyUtil(self.callback_url)
        self.rau = ReadsAlignmentUtils(self.callback_url)
        self.set_api = SetAPI(self.srv_wiz_url, service_ver='dev')
        self.eu = ExpressionUtils(self.callback_url)
        self.ws = Workspace(self.ws_url, token=self.token)

        self.scratch = os.path.join(config['scratch'], str(uuid.uuid4()))
        self._mkdir_p(self.scratch)

        self.tool_used = "Cufflinks"
        self.tool_version = os.environ['VERSION']
        # END_CONSTRUCTOR
        pass

    def parse_FPKMtracking_calc_TPM(self, filename):
        """
        Generates TPM from FPKM
        :return:
        """
        fpkm_dict = {}
        tpm_dict = {}
        gene_col = 0
        fpkm_col = 9
        sum_fpkm = 0.0
        with open(filename) as f:
            next(f)
            for line in f:
                larr = line.split("\t")
                gene_id = larr[gene_col]
                if gene_id != "":
                    fpkm = float(larr[fpkm_col])
                    sum_fpkm = sum_fpkm + fpkm
                    fpkm_dict[gene_id] = math.log(fpkm + 1, 2)
                    tpm_dict[gene_id] = fpkm

        if sum_fpkm == 0.0:
            log("Warning: Unable to calculate TPM values as sum of FPKM values is 0")
        else:
            for g in tpm_dict:
                tpm_dict[g] = math.log((tpm_dict[g] / sum_fpkm) * 1e6 + 1, 2)

        return fpkm_dict, tpm_dict

    def _mkdir_p(self, path):
        """
        _mkdir_p: make directory for given path
        """
        if not path:
            return
        try:
            os.makedirs(path)
        except OSError as exc:
            if exc.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                raise

    def _validate_run_cufflinks_params(self, params):
        """
        _validate_run_cufflinks_params:
                Raises an exception if params are invalid
        """

        log('Start validating run_cufflinks params')

        # check for required parameters
        for p in ['alignment_object_ref', 'workspace_name', 'genome_ref']:
            if p not in params:
                raise ValueError('"{}" parameter is required, but missing'.format(p))

    def _run_command(self, command):
        """
        _run_command: run command and print result
        """

        log('Start executing command:\n{}'.format(command))
        pipe = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
        output = pipe.communicate()[0]
        exitCode = pipe.returncode

        if (exitCode == 0):
            log('Executed command:\n{}\n'.format(command) +
                'Exit Code: {}\nOutput:\n{}'.format(exitCode, output))
        else:
            error_msg = 'Error running command:\n{}\n'.format(command)
            error_msg += 'Exit Code: {}\nOutput:\n{}'.format(exitCode, output)

            raise ValueError(error_msg)

    def _run_gffread(self, gff_path, gtf_path):
        """
        _run_gffread: run gffread script

        ref: http://cole-trapnell-lab.github.io/cufflinks/file_formats/#the-gffread-utility
        """
        log('converting gff to gtf')
        command = self.GFFREAD_TOOLKIT_PATH + '/gffread '
        command += "-E {0} -T -o {1}".format(gff_path, gtf_path)

        self._run_command(command)

    def _create_gtf_annotation_from_genome(self, genome_ref):
        """
         Create reference annotation file from genome
        """
        ref = self.ws.get_object_subset(
            [{'ref': genome_ref, 'included': ['contigset_ref', 'assembly_ref']}])
        if 'contigset_ref' in ref[0]['data']:
            contig_id = ref[0]['data']['contigset_ref']
        elif 'assembly_ref' in ref[0]['data']:
            contig_id = ref[0]['data']['assembly_ref']
        if contig_id is None:
            raise ValueError(
                "Genome at {0} does not have reference to the assembly object".format(
                    genome_ref))
        print(contig_id)
        log("Generating GFF file from Genome")
        try:
            ret = self.au.get_assembly_as_fasta({'ref': contig_id})
            output_file = ret['path']
            mapping_filename = c_mapping.create_sanitized_contig_ids(output_file)
            os.remove(output_file)
            # get the GFF
            ret = self.gfu.genome_to_gff({'genome_ref': genome_ref})
            genome_gff_file = ret['file_path']
            c_mapping.replace_gff_contig_ids(genome_gff_file, mapping_filename, to_modified=True)
            gtf_ext = ".gtf"

            if not genome_gff_file.endswith(gtf_ext):
                gtf_path = os.path.splitext(genome_gff_file)[0] + '.gtf'
                self._run_gffread(genome_gff_file, gtf_path)
            else:
                gtf_path = genome_gff_file

            log("gtf file : " + gtf_path)
        except Exception:
            raise ValueError(
                "Generating GTF file from Genome Annotation object Failed :  {}".format(
                    "".join(traceback.format_exc())))
        return gtf_path

    def _get_gtf_file(self, alignment_ref):
        """
        _get_gtf_file: get the reference annotation file (in GTF or GFF3 format)
        """
        result_directory = self.scratch
        alignment_data = self.ws.get_objects2({'objects':
                                               [{'ref': alignment_ref}]})['data'][0]['data']

        genome_ref = alignment_data.get('genome_id')
        # genome_name = self.ws.get_object_info([{"ref": genome_ref}], includeMetadata=None)[0][1]
        # ws_gtf = genome_name+"_GTF_Annotation"

        genome_data = self.ws.get_objects2({'objects':
                                            [{'ref': genome_ref}]})['data'][0]['data']

        gff_handle_ref = genome_data.get('gff_handle_ref')

        if gff_handle_ref:
            log('getting reference annotation file from genome')
            annotation_file = self.dfu.shock_to_file({'handle_id': gff_handle_ref,
                                                      'file_path': result_directory,
                                                      'unpack': 'unpack'})['file_path']
        else:
            annotation_file = self._create_gtf_annotation_from_genome(genome_ref)

        return annotation_file

    def _get_gtf_file_from_genome_ref(self, genome_ref):
        """
        _get_gtf_file: get the reference annotation file (in GTF or GFF3 format)
        """
        result_directory = self.scratch

        genome_data = self.ws.get_objects2({'objects':
                                            [{'ref': genome_ref}]})['data'][0]['data']

        gff_handle_ref = genome_data.get('gff_handle_ref')

        if gff_handle_ref:
            log('getting reference annotation file from genome')
            annotation_file = self.dfu.shock_to_file({'handle_id': gff_handle_ref,
                                                      'file_path': result_directory,
                                                      'unpack': 'unpack'})['file_path']
        else:
            annotation_file = self._create_gtf_annotation_from_genome(genome_ref)

        return annotation_file

    def _get_input_file(self, alignment_ref):
        """
        _get_input_file: get input BAM file from Alignment object
        """

        bam_file_dir = self.rau.download_alignment({'source_ref': alignment_ref})['destination_dir']

        files = os.listdir(bam_file_dir)
        bam_file_list = [file for file in files if re.match(r'.*\_sorted\.bam', file)]
        if not bam_file_list:
            bam_file_list = [file for file in files if re.match(r'.*(?<!sorted)\.bam', file)]

        if not bam_file_list:
            raise ValueError('Cannot find .bam file from alignment {}'.format(alignment_ref))

        bam_file_name = bam_file_list[0]

        bam_file = os.path.join(bam_file_dir, bam_file_name)

        return bam_file

    def _generate_command(self, params):
        """
        _generate_command: generate cufflinks command
        """
        cufflinks_command = '/opt/cufflinks/cufflinks'
        cufflinks_command += (' -q --no-update-check -p ' + str(params.get('num_threads', 1)))
        if 'max_intron_length' in params and params['max_intron_length'] is not None:
            cufflinks_command += (' --max-intron-length ' + str(params['max_intron_length']))
        if 'min_intron_length' in params and params['min_intron_length'] is not None:
            cufflinks_command += (' --min-intron-length ' + str(params['min_intron_length']))
        if 'overhang_tolerance' in params and params['overhang_tolerance'] is not None:
            cufflinks_command += (' --overhang-tolerance ' + str(params['overhang_tolerance']))

        cufflinks_command += " -o {0} -G {1} {2}".format(
            params['result_directory'], params['gtf_file'], params['input_file'])

        log('Generated cufflinks command: {}'.format(cufflinks_command))

        return cufflinks_command

    def _process_rnaseq_alignment_object(self, params):
        """
        _process_alignment_object: process KBaseRNASeq.RNASeqAlignment type input object
        """
        log('start processing RNASeqAlignment object\nparams:\n{}'.format(
            json.dumps(params, indent=1)))
        alignment_ref = params.get('alignment_ref')

        result_directory = os.path.join(self.scratch, str(uuid.uuid4()))
        self._mkdir_p(result_directory)
        params['result_directory'] = str(result_directory)

        # input files
        params['input_file'] = self._get_input_file(alignment_ref)
        if not params.get('gtf_file'):
            params['gtf_file'] = self._get_gtf_file(alignment_ref)

        if '/' not in params['genome_ref']:
            params['genome_ref'] = params['workspace_name']+'/'+params['genome_ref']

        command = self._generate_command(params)
        self._run_command(command)

        expression_obj_ref = self._save_rnaseq_expression(result_directory,
                                                   alignment_ref,
                                                   params.get('workspace_name'),
                                                   params.get('genome_ref'),
                                                   params['gtf_file'],
                                                   params['expression_suffix'])

        returnVal = {'result_directory': result_directory,
                     'expression_obj_ref': expression_obj_ref,
                     'alignment_ref': alignment_ref}

        expression_name = self.ws.get_object_info([{"ref": expression_obj_ref}],
                                                  includeMetadata=None)[0][1]

        widget_params = {"output": expression_name, "workspace": params.get('workspace_name')}
        returnVal.update(widget_params)

        return returnVal

    def _process_kbasesets_alignment_object(self, params):
        """
        _process_alignment_object: process KBaseRNASeq.RNASeqAlignment type input object
        """
        log('start processing KBaseSets object\nparams:\n{}'.format(
            json.dumps(params, indent=1)))
        alignment_ref = params.get('alignment_ref')

        result_directory = os.path.join(self.scratch, str(uuid.uuid4()))
        self._mkdir_p(result_directory)
        params['result_directory'] = str(result_directory)

        # input files
        params['input_file'] = self._get_input_file(alignment_ref)
        if not params.get('gtf_file'):
            params['gtf_file'] = self._get_gtf_file(alignment_ref)

        command = self._generate_command(params)
        self._run_command(command)

        expression_obj_ref = self._save_kbasesets_expression(result_directory,
                                                   alignment_ref,
                                                   params.get('workspace_name'),
                                                   params.get('genome_ref'),
                                                   params.get('gtf_file'),
                                                   params.get('expression_suffix'))

        returnVal = {'result_directory': result_directory,
                     'expression_obj_ref': expression_obj_ref,
                     'alignment_ref': alignment_ref}

        expression_name = self.ws.get_object_info([{"ref": expression_obj_ref}],
                                                  includeMetadata=None)[0][1]

        widget_params = {"output": expression_name, "workspace": params.get('workspace_name')}
        returnVal.update(widget_params)

        return returnVal

    def _generate_html_report(self, result_directory, obj_ref):
        """
        _generate_html_report: generate html summary report
        """
        log('Start generating html report')
        html_report = list()

        output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
        self._mkdir_p(output_directory)
        result_file_path = os.path.join(output_directory, 'report.html')

        expression_object = self.ws.get_objects2({'objects':
                                                  [{'ref': obj_ref}]})['data'][0]

        expression_object_type = expression_object.get('info')[2]

        Overview_Content = ''
        if re.match('KBaseRNASeq.RNASeqExpression-\d.\d', expression_object_type):
            Overview_Content += '
Generated Expression Object:

{}
'.format(
                expression_object.get('info')[1])
        elif re.match('KBaseRNASeq.RNASeqExpressionSet-\d.\d', expression_object_type):
            Overview_Content += '
Generated Expression Set Object:

{}
'.format(
                expression_object.get('info')[1])
            Overview_Content += '

Generated Expression Object:
'
            for expression_ref in expression_object['data']['sample_expression_ids']:
                expression_name = self.ws.get_object_info([{"ref": expression_ref}],
                                                          includeMetadata=None)[0][1]
                Overview_Content += '
{}
'.format(expression_name)
        elif re.match('KBaseSets.ExpressionSet-\d.\d', expression_object_type):
            pprint(expression_object)
            Overview_Content += '
Generated Expression Set Object:

{}
'.format(
                expression_object.get('info')[1])
            Overview_Content += '

Generated Expression Object:
'
            for expression_ref in expression_object['data']['items']:
                expression_name = self.ws.get_object_info([{"ref": expression_ref['ref']}],
                                                          includeMetadata=None)[0][1]
                condition = expression_ref['label']
                Overview_Content += '
condition:{0}; expression_name: {1}
'.format(condition, expression_name)

        with open(result_file_path, 'w') as result_file:
            with open(os.path.join(os.path.dirname(__file__), 'report_template.html'),
                      'r') as report_template_file:
                report_template = report_template_file.read()
                report_template = report_template.replace('
Overview_Content
',
                                                          Overview_Content)
                result_file.write(report_template)

        html_report.append({'path': result_file_path,
                            'name': os.path.basename(result_file_path),
                            'label': os.path.basename(result_file_path),
                            'description': 'HTML summary report for Cufflinks App'})
        return html_report

    def _save_rnaseq_expression(self, result_directory, alignment_ref,
                                workspace_name, genome_ref, gtf_file,
                                expression_suffix):
        """
        _save_rnaseq_expression: save Expression object to workspace
        """
        log('start saving Expression object')
        alignment_object_name = self.ws.get_object_info([{"ref": alignment_ref}],
                                                        includeMetadata=None)[0][1]

        # set expression name
        if re.match('.*_[Aa]lignment$', alignment_object_name):
            expression_name = re.sub('_[Aa]lignment$',
                                     expression_suffix,
                                     alignment_object_name)
        else:  # assume user specified suffix
            expression_name = alignment_object_name + expression_suffix

        expression_ref = self.eu.upload_expression({
            'destination_ref': workspace_name + '/' + expression_name,
            'source_dir': result_directory,
            'alignment_ref': alignment_ref,
            'tool_used': self.tool_used,
            'tool_version': self.tool_version
        })['obj_ref']

        return expression_ref

    def _save_kbasesets_expression(self, result_directory, alignment_ref,
                                   workspace_name, genome_ref, gtf_file,
                                   expression_suffix):
        """
        _save_kbasesets_expression: save Expression object to workspace using ExpressionUtils
        and SetAPI
        """
        log('start saving Expression object')

        alignment_info = self.ws.get_object_info3({'objects': [{"ref": alignment_ref}]})
        alignment_object_name = alignment_info['infos'][0][1]

        # set expression name
        if re.match('.*_[Aa]lignment$', alignment_object_name):
            expression_name = re.sub('_[Aa]lignment$',
                                     expression_suffix,
                                     alignment_object_name)
        else:  # assume user specified suffix
            expression_name = alignment_object_name + expression_suffix

        expression_ref = self.eu.upload_expression({
            'destination_ref': workspace_name+'/'+expression_name,
            'source_dir': result_directory,
            'alignment_ref': alignment_ref,
            'tool_used': self.tool_used,
            'tool_version': self.tool_version
        })['obj_ref']

        return expression_ref

    def _save_rnaseq_expression_set(self, alignment_expression_map, alignment_set_ref, workspace_name, expression_set_name):
        """
        _save_rnaseq_expression_set: save ExpressionSet object to workspace
        """
        log('start saving ExpressionSet object')
        if isinstance(workspace_name, int) or workspace_name.isdigit():
            workspace_id = workspace_name
        else:
            workspace_id = self.dfu.ws_name_to_id(workspace_name)

        expression_set_data = self._generate_expression_set_data(alignment_expression_map,
                                                                 alignment_set_ref,
                                                                 expression_set_name)

        object_type = 'KBaseRNASeq.RNASeqExpressionSet'
        save_object_params = {
            'id': workspace_id,
            'objects': [{
                'type': object_type,
                'data': expression_set_data,
                'name': expression_set_name
            }]
        }

        dfu_oi = self.dfu.save_objects(save_object_params)[0]
        expression_set_ref = str(dfu_oi[6]) + '/' + str(dfu_oi[0]) + '/' + str(dfu_oi[4])

        return expression_set_ref

    def _save_kbasesets_expression_set(self, alignment_expression_map,
                                       alignment_set_ref, workspace_name,
                                       expression_set_name):
        """
        _save_kbasesets_expression_set: save ExpressionSet object to workspace
        """
        log('start saving ExpressionSet object')
        if isinstance(workspace_name, int) or workspace_name.isdigit():
            workspace_id = workspace_name
        else:
            workspace_id = self.dfu.ws_name_to_id(workspace_name)

        expression_set_data = self._generate_expression_set_data(alignment_expression_map,
                                                                 alignment_set_ref,
                                                                 expression_set_name)

        object_type = 'KBaseRNASeq.RNASeqExpressionSet'
        save_object_params = {
            'id': workspace_id,
            'objects': [{
                'type': object_type,
                'data': expression_set_data,
                'name': expression_set_name
            }]
        }

        dfu_oi = self.dfu.save_objects(save_object_params)[0]
        expression_set_ref = str(dfu_oi[6]) + '/' + str(dfu_oi[0]) + '/' + str(dfu_oi[4])

        return expression_set_ref

    def _generate_report(self, obj_ref, workspace_name, result_directory,
                         exprMatrix_FPKM_ref=None, exprMatrix_TPM_ref=None):
        """
        _generate_report: generate summary report
        """

        log('creating report')

        output_files = self._generate_output_file_list(result_directory)
        output_html_files = self._generate_html_report(result_directory,
                                                       obj_ref)

        expression_object = self.ws.get_objects2({'objects':
                                                 [{'ref': obj_ref}]})['data'][0]
        expression_info = expression_object['info']
        expression_data = expression_object['data']

        expression_object_type = expression_info[2]
        if re.match('KBaseRNASeq.RNASeqExpression-\d+.\d+', expression_object_type):
            objects_created = [{'ref': obj_ref,
                                'description': 'Expression generated by Cufflinks'}]
        elif re.match('KBaseRNASeq.RNASeqExpressionSet-\d+.\d+', expression_object_type):
            objects_created = [{'ref': obj_ref,
                                'description': 'Expression generated by Cufflinks'}]
        elif re.match('KBaseSets.ExpressionSet-\d+.\d+', expression_object_type):
            objects_created = [{'ref': obj_ref,
                                'description': 'ExpressionSet generated by Cufflinks'}]
            items = expression_data['items']
            for item in items:
                objects_created.append({'ref': item['ref'],
                                        'description': 'Expression generated by Cufflinks'})
            objects_created.append({'ref': exprMatrix_FPKM_ref,
                                    'description': 'FPKM ExpressionMatrix generated by Cufflinks'})
            objects_created.append({'ref': exprMatrix_TPM_ref,
                                    'description': 'TPM ExpressionMatrix generated by Cufflinks'})

        report_params = {'message': '',
                         'workspace_name': workspace_name,
                         'file_links': output_files,
                         'objects_created': objects_created,
                         'html_links': output_html_files,
                         'direct_html_link_index': 0,
                         'html_window_height': 366,
                         'report_object_name': 'kb_cufflinks_report_' + str(uuid.uuid4())}

        kbase_report_client = KBaseReport(self.callback_url, token=self.token)
        output = kbase_report_client.create_extended_report(report_params)

        report_output = {'report_name': output['name'], 'report_ref': output['ref']}

        return report_output

    def _parse_FPKMtracking(self, filename, metric):
        result = {}
        pos1 = 0
        if metric == 'FPKM':
            pos2 = 7
        if metric == 'TPM':
            pos2 = 8

        with open(filename) as f:
            next(f)
            for line in f:
                larr = line.split("\t")
                if larr[pos1] != "":
                    try:
                        result[larr[pos1]] = math.log(float(larr[pos2]) + 1, 2)
                    except ValueError:
                        result[larr[pos1]] = math.log(1, 2)

        return result

    def _generate_output_file_list(self, result_directory):
        """
        _generate_output_file_list: zip result files and generate file_links for report
        """
        log('Start packing result files')
        output_files = list()

        output_directory = os.path.join(self.scratch, str(uuid.uuid4()))
        self._mkdir_p(output_directory)
        result_file = os.path.join(output_directory, 'cufflinks_result.zip')

        with zipfile.ZipFile(result_file, 'w',
                             zipfile.ZIP_DEFLATED,
                             allowZip64=True) as zip_file:
            for root, dirs, files in os.walk(result_directory):
                for file in files:
                    if not (file.endswith('.DS_Store')):
                        zip_file.write(os.path.join(root, file),
                                       os.path.join(os.path.basename(root), file))

        output_files.append({'path': result_file,
                             'name': os.path.basename(result_file),
                             'label': os.path.basename(result_file),
                             'description': 'File(s) generated by Cufflinks App'})

        return output_files

    def _generate_expression_data(self, result_directory, alignment_ref,
                                  gtf_file, workspace_name, expression_suffix):
        """
        _generate_expression_data: generate Expression object with cufflinks output files
        """
        alignment_data_object = self.ws.get_objects2({'objects':
                                                      [{'ref': alignment_ref}]})['data'][0]

        # set expression name
        alignment_object_name = alignment_data_object['info'][1]
        if re.match('.*_[Aa]lignment$', alignment_object_name):
            expression_name = re.sub('_[Aa]lignment$',
                                     expression_suffix,
                                     alignment_object_name)
        else:  # assume user specified suffix
            expression_name = alignment_object_name + expression_suffix

        expression_data = {
            'id': expression_name,
            'type': 'RNA-Seq',
            'numerical_interpretation': 'FPKM',
            'processing_comments': 'log2 Normalized',
            'tool_used': self.tool_used,
            'tool_version': self.tool_version
        }
        alignment_data = alignment_data_object['data']

        condition = alignment_data.get('condition')
        expression_data.update({'condition': condition})

        genome_id = alignment_data.get('genome_id')
        expression_data.update({'genome_id': genome_id})

        read_sample_id = alignment_data.get('read_sample_id')
        expression_data.update({'mapped_rnaseq_alignment': {read_sample_id: alignment_ref}})

        exp_dict, tpm_exp_dict = self.parse_FPKMtracking_calc_TPM(
            os.path.join(result_directory, 'genes.fpkm_tracking'))

        expression_data.update({'expression_levels': exp_dict})

        expression_data.update({'tpm_expression_levels': tpm_exp_dict})

        handle = self.dfu.file_to_shock({'file_path': result_directory,
                                         'pack': 'zip',
                                         'make_handle': True})['handle']
        expression_data.update({'file': handle})

        return expression_data

    def _generate_expression_set_data(self, alignment_expression_map,
                                      alignment_set_ref, expression_set_name):
        """
        _generate_expression_set_data: generate ExpressionSet object with cufflinks output files
        """
        alignment_set_data_object = self.ws.get_objects2({'objects':
                                                          [{'ref': alignment_set_ref}]})['data'][0]

        alignment_set_data = alignment_set_data_object['data']

        expression_set_data = {
            'tool_used': self.tool_used,
            'tool_version': self.tool_version,
            'id': expression_set_name,
            'alignmentSet_id': alignment_set_ref,
            'genome_id': alignment_set_data.get('genome_id'),
            'sampleset_id': alignment_set_data.get('sampleset_id')
        }

        sample_expression_ids = []
        mapped_expression_objects = []
        mapped_expression_ids = []

        for alignment_expression in alignment_expression_map:
            alignment_ref = alignment_expression.get('alignment_ref')
            expression_ref = alignment_expression.get('expression_obj_ref')
            sample_expression_ids.append(expression_ref)
            mapped_expression_ids.append({alignment_ref: expression_ref})
            alignment_name = self.ws.get_object_info([{"ref": alignment_ref}],
                                                     includeMetadata=None)[0][1]
            expression_name = self.ws.get_object_info([{"ref": expression_ref}],
                                                      includeMetadata=None)[0][1]
            mapped_expression_objects.append({alignment_name: expression_name})

        expression_set_data['sample_expression_ids'] = sample_expression_ids
        expression_set_data['mapped_expression_objects'] = mapped_expression_objects
        expression_set_data['mapped_expression_ids'] = mapped_expression_ids

        return expression_set_data

    def _process_alignment_set_object(self, params, alignment_object_type):
        """
        _process_alignment_set_object: process KBaseRNASeq.RNASeqAlignmentSet type input object
                                        and KBaseSets.ReadsAlignmentSet type object
        """
        log('start processing KBaseRNASeq.RNASeqAlignmentSet object or KBaseSets.ReadsAlignmentSet object'
            '\nparams:\n{}'.format(
            json.dumps(params, indent=1)))

        alignment_set_ref = params.get('alignment_set_ref')

        if re.match('^KBaseRNASeq.RNASeqAlignmentSet-\d*', alignment_object_type):
            params['gtf_file'] = self._get_gtf_file(alignment_set_ref)
        else:
            if not '/' in params['genome_ref']:
                params['genome_ref'] = params['workspace_name']+'/'+params['genome_ref']

            params['gtf_file'] = self._get_gtf_file_from_genome_ref(params['genome_ref'])

        alignment_set = self.set_api.get_reads_alignment_set_v1({
                                                                'ref': alignment_set_ref,
                                                                'include_item_info': 0,
                                                                'include_set_item_ref_paths': 1
                                                                })
        mul_processor_params = []
        for alignment in alignment_set["data"]["items"]:
            alignment_ref = alignment['ref_path']
            alignment_upload_params = params.copy()
            alignment_upload_params['alignment_ref'] = alignment_ref
            mul_processor_params.append(alignment_upload_params)
            # use the following when you want to run the cmd sequentially
            # self._process_kbasesets_alignment_object(mul_processor_params[0])

        cpus = min(params.get('num_threads'), multiprocessing.cpu_count())
        pool = Pool(ncpus=cpus)
        log('running _process_alignment_object with {} cpus'.format(cpus))
        alignment_expression_map = pool.map(self._process_kbasesets_alignment_object, mul_processor_params)

        result_directory = os.path.join(self.scratch, str(uuid.uuid4()))
        self._mkdir_p(result_directory)

        expression_items = list()
        for proc_alignment_return in alignment_expression_map:
            expression_obj_ref = proc_alignment_return.get('expression_obj_ref')
            alignment_ref = proc_alignment_return.get('alignment_ref')
            alignment_info = self.ws.get_object_info3({'objects': [{"ref": alignment_ref}], 'includeMetadata': 1})
            condition = alignment_info['infos'][0][10]['condition']
            expression_items.append({
                "ref": expression_obj_ref,
                "label": condition,
            })
            expression_name = self.ws.get_object_info([{"ref": expression_obj_ref}],
                                                      includeMetadata=None)[0][1]
            self._run_command('cp -R {} {}'.format(proc_alignment_return.get('result_directory'),
                                                   os.path.join(result_directory, expression_name)))

        expression_set = {
            "description": "generated by kb_cufflinks",
            "items": expression_items
        }

        expression_set_info = self.set_api.save_expression_set_v1({
            "workspace": params['workspace_name'],
            "output_object_name": params['expression_set_name'],
            "data": expression_set
        })

        returnVal = {'result_directory': result_directory,
                     'expression_obj_ref': expression_set_info['set_ref']}

        widget_params = {"output": params.get('expression_set_name'),
                         "workspace": params.get('workspace_name')}
        returnVal.update(widget_params)

        return returnVal

    def _generate_output_object_name(self, params, alignment_object_type, alignment_object_name):
        """
        Generates the output object name based on input object type and name and stores it in
        params with key equal to 'expression' or 'expression_set' based on whether the input
        object is an alignment or alignment_set.

        :param params: module input params
        :param alignment_object_type: input alignment object type
        :param alignment_object_name: input alignment object name
        :param alignment_object_data: input alignment object data
        """
        expression_set_suffix = params['expression_set_suffix']
        expression_suffix = params['expression_suffix']

        if re.match('^KBaseRNASeq.RNASeqAlignment-\d*', alignment_object_type):
            if re.match('.*_[Aa]lignment$', alignment_object_name):
                params['expression_name'] = re.sub('_[Aa]lignment$',
                                                   expression_suffix,
                                                   alignment_object_name)
            else:  # assume user specified suffix
                params['expression_name'] = alignment_object_name+expression_suffix
        if re.match('^KBaseRNASeq.RNASeqAlignmentSet-\d*', alignment_object_type):
            if re.match('.*_[Aa]lignment_[Ss]et$', alignment_object_name):
                # set expression set name
                params['expression_set_name'] = re.sub('_[Aa]lignment_[Ss]et$',
                                                       expression_set_suffix,
                                                       alignment_object_name)
            else:  # assume user specified suffix
                params['expression_set_name'] = alignment_object_name + expression_set_suffix
        if re.match('^KBaseSets.ReadsAlignmentSet-\d*', alignment_object_type):
            if re.match('.*_[Aa]lignment_[Ss]et$', alignment_object_name):

                # set expression set name
                params['expression_set_name'] = re.sub('_[Aa]lignment_[Ss]et$',
                                                       expression_set_suffix,
                                                       alignment_object_name)
            else:  # assume user specified suffix
                params['expression_set_name'] = alignment_object_name + expression_set_suffix

    def _save_expression_matrix(self, expressionset_ref, workspace_name):
        """
        _save_expression_matrix: save FPKM and TPM ExpressionMatrix
        """

        log('start saving ExpressionMatrix object')

        expression_set_name = self.ws.get_object_info([{"ref": expressionset_ref}],
                                                      includeMetadata=None)[0][1]

        output_obj_name_prefix = re.sub('_*[Ee]xpression_*[Ss]et',
                                        '',
                                        expression_set_name)

        upload_expression_matrix_params = {'expressionset_ref': expressionset_ref,
                                           'output_obj_name': output_obj_name_prefix,
                                           'workspace_name': workspace_name}

        expression_matrix_refs = self.eu.get_expressionMatrix(upload_expression_matrix_params)

        return expression_matrix_refs

    def run_cufflinks_app(self, params):
        log('--->\nrunning CufflinksUtil.run_cufflinks_app\n' +
            'params:\n{}'.format(json.dumps(params, indent=1)))

        self._validate_run_cufflinks_params(params)

        alignment_object_ref = params.get('alignment_object_ref')
        alignment_object_info = self.ws.get_object_info3({
            "objects": [{"ref": alignment_object_ref}]})['infos'][0]

        alignment_object_type = alignment_object_info[2]
        alignment_object_name = alignment_object_info[1]

        # get output object name
        self._generate_output_object_name(params, alignment_object_type, alignment_object_name)

        log('--->\nalignment object type: \n' +
            '{}'.format(alignment_object_type))

        if re.match('^KBaseRNASeq.RNASeqAlignment-\d*', alignment_object_type):
            params.update({'alignment_ref': alignment_object_ref})
            returnVal = self._process_rnaseq_alignment_object(params)
            report_output = self._generate_report(returnVal.get('expression_obj_ref'),
                                                  params.get('workspace_name'),
                                                  returnVal.get('result_directory'))
            returnVal.update(report_output)
        elif re.match('^KBaseRNASeq.RNASeqAlignmentSet-\d*', alignment_object_type) or \
             re.match('^KBaseSets.ReadsAlignmentSet-\d*', alignment_object_type):
            params.update({'alignment_set_ref': alignment_object_ref})
            returnVal = self._process_alignment_set_object(params, alignment_object_type)
            expression_matrix_refs = self._save_expression_matrix(returnVal['expression_obj_ref'],
                                                                  params.get('workspace_name'))
            returnVal.update(expression_matrix_refs)

            report_output = self._generate_report(returnVal['expression_obj_ref'],
                                                  params.get('workspace_name'),
                                                  returnVal['result_directory'],
                                                  expression_matrix_refs['exprMatrix_FPKM_ref'],
                                                  expression_matrix_refs['exprMatrix_TPM_ref'])
            returnVal.update(report_output)
        else:
            raise ValueError('None RNASeqAlignment type\nObject info:\n{}'.format(
                alignment_object_info))

        return returnVal


import morepath
from munch import Munch
from pytest import fixture

from ekklesia_portal.identity_policy import EkklesiaPortalIdentityPolicy

@fixture
def identity_policy():
    return EkklesiaPortalIdentityPolicy()

@fixture
def identity():
    user = Munch(id=1)
    return morepath.Identity(userid=user.id, user=user)

def test_remember(identity_policy, identity, req):
    req.browser_session = {}
    identity_policy.remember(None, req, identity)
    assert req.browser_session['user_id'] == 1

def test_identify(identity_policy, req, user_factory):
    user = user_factory()
    req.browser_session = {'user_id': user.id}
    identity = identity_policy.identify(req)
    assert identity.userid == user.id
    assert identity.has_global_admin_permissions is False

def test_identify_admin(identity_policy, req, user_factory, group_factory):
    group = group_factory(is_admin_group=True)
    user = user_factory()
    group.members.append(user)
    req.browser_session = {'user_id': user.id}
    identity = identity_policy.identify(req)
    assert identity.has_global_admin_permissions is True

def test_forget(identity_policy, req):
    req.browser_session = {'user_id': 1}
    identity_policy.forget(None, req)
    assert 'user_id' not in req.browser_session


"""Creating a Cartesian Communicator"""

import sys
from mpi4py import MPI

def main(parent):
    """Create a 2-d Cartesian communicator from parent communicator"""

    dims = MPI.Compute_dims(parent.size, 2)

    comm = parent.Create_cart(dims, periods = [True, False])
    rank = comm.Get_rank()
    coords = comm.Get_coords(rank)
    upx = comm.Shift(0, 1)
    upy = comm.Shift(1, 1)

    out = "Rank{:2d} coords{:2d} {:2d} upx(src,dst) {} upy(src,dst) {}\n"\
                         .format(rank, coords[0], coords[1], upx, upy)
    sys.stdout.write(out)

if __name__ == "__main__":
    """Execute in MPI.COMM_WORLD"""
    main(MPI.COMM_WORLD)


import sys
from collections import deque
sys.stdin = open('input.txt')

def BFS():
    while queue:
        x, y, breaking = queue.popleft()

        if x == N - 1 and y == M - 1:
            print(visited[x][y][breaking])
            return

        for k in range(4):
            nx, ny = x + dx[k], y + dy[k]
            if 0 <= nx < N and 0 <= ny < M :
                if Map[nx][ny] == 1 and breaking + 1 <= K and visited[nx][ny][breaking + 1] == 0:
                    visited[nx][ny][breaking + 1] = visited[x][y][breaking] + 1
                    queue.append((nx, ny, breaking + 1))
                elif Map[nx][ny] == 0 and visited[nx][ny][breaking] == 0:
                    visited[nx][ny][breaking] = visited[x][y][breaking] + 1
                    queue.append((nx, ny, breaking))

    print(-1)

dx = [-1, 1, 0, 0]
dy = [0, 0, -1, 1]
N, M, K= map(int, input().split())
Map = [list(map(int, input().rstrip())) for _ in range(N)]
visited = [[[0]*(K+1) for p in range(M)] for j in range(N)]
queue = deque()
queue.append((0, 0, 0))
visited[0][0][0] = 1
BFS()


""" Surely a fragile parser, but gets things started and will evolve over time I guess.
It seems that a lot of the json data does not have any securityData data
So these are just skipped, since there is nothing much to do with them here
"""
import json

from dojo.models import Finding

class SonatypeParser(object):
    # This parser does not deal with licenses information.

    def get_scan_types(self):
        return ["Sonatype Application Scan"]

    def get_label_for_scan_types(self, scan_type):
        return scan_type  # no custom label for now

    def get_description_for_scan_types(self, scan_type):
        return "Can be imported in JSON format"

    def get_findings(self, json_output, test):
        tree = json.load(json_output)
        return self.get_items(tree, test)

    def get_items(self, tree, test):
        items = {}
        if "components" in tree:
            vulnerability_tree = tree["components"]

            for node in vulnerability_tree:
                item = get_item(node, test)
                if item is None:
                    continue
                # TODO
                unique_key = node["hash"]
                items[unique_key] = item

        return list(items.values())

def get_item(vulnerability, test):
    # Following the CVSS Scoring per https://nvd.nist.gov/vuln-metrics/cvss
    if (
        vulnerability["securityData"] is not None
        and len(vulnerability["securityData"]["securityIssues"]) >= 1
    ):
        # there can be nothing in the array, or securityData can be null altogether. If the latter, well, nothing much to do?
        # issues is an array, and there can be 2+ of them, e.g. a cve and a sonatype entry or two cves
        # Given the current Finding class, if a cve, will be the main. If not a cve, then CVE ref will remain null due to regex.
        # Others go to references.
        main_finding = vulnerability["securityData"]["securityIssues"][0]

        if main_finding.get("source") == "cve":
            vulnerability_id = main_finding.get("reference")
        else:
            # if sonatype of else, will not match Finding model today
            vulnerability_id = None

        if main_finding["severity"] <= 3.9:
            severity = "Low"
        elif (
            main_finding["severity"] > 4.0 and main_finding["severity"] <= 6.9
        ):
            severity = "Medium"
        elif main_finding["severity"] and main_finding["severity"] <= 8.9:
            severity = "High"
        else:
            severity = "Critical"

        references = []
        if len(vulnerability["securityData"]["securityIssues"]) > 1:
            for additional_issue in vulnerability["securityData"][
                "securityIssues"
            ]:
                references.append(
                    "{}, {}, {}, {}, {} ".format(
                        additional_issue.get("reference"),
                        additional_issue.get("status"),
                        additional_issue.get("severity"),
                        additional_issue.get("threatCategory"),
                        additional_issue.get("url"),
                    )
                )

        component_id = ""
        if "componentIdentifier" in vulnerability:
            if vulnerability["componentIdentifier"]["format"] == "maven":
                component_id = "{} {} {}".format(
                    vulnerability["componentIdentifier"]["coordinates"][
                        "artifactId"
                    ],
                    vulnerability["componentIdentifier"]["coordinates"][
                        "groupId"
                    ],
                    vulnerability["componentIdentifier"]["coordinates"][
                        "version"
                    ],
                )
            elif vulnerability["componentIdentifier"]["format"] == "a-name":
                component_id = "{} {} {}".format(
                    vulnerability["componentIdentifier"]["coordinates"][
                        "name"
                    ],
                    vulnerability["componentIdentifier"]["coordinates"][
                        "qualifier"
                    ],
                    vulnerability["componentIdentifier"]["coordinates"][
                        "version"
                    ],
                )

        finding_title = "{} - {}".format(
            main_finding["reference"], component_id
        )

        finding_description = "Hash {}\n\n".format(vulnerability["hash"])
        finding_description += component_id
        finding_description += '\n\nPlease check the CVE details of this finding for a detailed description. The details of issues beginning with "SONATYPE-" can be found by contacting Sonatype, Inc. or through mechanisms they have provided in their product.'
        threat_category = main_finding.get(
            "threatCategory", "CVSS vector not provided. "
        ).title()
        status = main_finding["status"]
        main_finding.get("severity", "No CVSS score yet.")
        if "pathnames" in vulnerability:
            file_path = " ".join(vulnerability["pathnames"])[:1000]
        else:
            file_path = ""

        # create the finding object
        finding = Finding(
            title=finding_title,
            test=test,
            severity=severity,
            description=finding_description,
            mitigation=status,
            references="{}\n{}\n".format(
                main_finding["url"], "\n".join(references)
            ),
            false_p=False,
            duplicate=False,
            out_of_scope=False,
            mitigated=None,
            file_path=file_path,
            impact=threat_category,
            static_finding=True
        )
        if vulnerability_id:
            finding.unsaved_vulnerability_ids = [vulnerability_id]
        finding.description = finding.description.strip()

        return finding
    else:
        return None


import gym
import numpy as np
from actor_critic_agent import ActorCriticAgent
from ploting import plot_agent_learning

if __name__ == '__main__':
    load_agent = False
    testing_agent = False
    plot_data = True
    render_walker = True
    if render_walker:
        env = gym.make("BipedalWalker-v3", render_mode='human')
    else:
        env = gym.make("BipedalWalker-v3")
    agent = ActorCriticAgent(env=env, checkpoint_directory=r'temp/tensorflow_ddpg_3500')
    nr_of_games = 3500
    plot_path = r'plots/tensor_ddpg_' + str(nr_of_games) + '.png'

    best_score = env.reward_range[0]
    score_history = []

    if load_agent:
        nr_steps = 0
        while nr_steps <= agent.batch_size:
            env_state = env.reset()
            env_state = env_state[0]
            action = env.action_space.sample()
            new_state, reward, terminated, truncated, info = env.step(action)
            agent.add_to_buffer(env_state, action, reward, new_state, terminated or truncated)
            nr_steps += 1
        agent.learn()
        agent.load_model()

    for i in range(nr_of_games):
        env_state = env.reset()
        env_state = env_state[0]
        done = False
        score = 0
        while not done:
            action = agent.choose_action(env_state, testing_agent)
            new_state, reward, terminated, truncated, info = env.step(action)
            score += reward
            done = terminated or truncated
            agent.add_to_buffer(env_state, action, reward, new_state, done)
            if not testing_agent:
                agent.learn()
            env_state = new_state
        score_history.append(score)
        avg_score = np.mean(score_history[-100:])

        if avg_score > best_score:
            best_score = avg_score
            if not testing_agent:
                agent.save_model()
        print('Episodul', i, 'score %1.f' % score, 'avg score %1.f' % avg_score)

    if plot_data:
        plot_agent_learning(score_history, plot_path)


import json
import os
import random
import time
from pathlib import Path

import numpy as np
import torch
import torchvision
from torch import nn
from torch.utils.tensorboard import SummaryWriter
from tqdm.auto import tqdm

class Logger:
    def __init__(self, logdir: Path, step: int):
        self._logdir = Path(logdir)
        self.writer = SummaryWriter(log_dir=str(self._logdir))
        self._scalars = {}
        self._images = {}
        self._videos = {}
        self._last_step = None
        self._last_time = None
        self.step = step

    def scalar(self, name, value):
        if name in self._scalars:
            if type(self._scalars[name]) != list:
                self._scalars[name] = [self._scalars[name]]
            self._scalars[name].append(value)
        else:
            self._scalars[name] = float(value)

    @torch.no_grad()
    def image(self, name, value):
        value = value.clip(0.0, 1.0)
        self._images[name] = value

    def video(self, name, value, log_to_tensorboard=True):
        self._videos[name] = [np.array(value), log_to_tensorboard]

    def write(self, fps=False):
        scalars = list(self._scalars.items())
        if fps:
            scalars.append(("perf/fps", self._compute_fps(self.step)))
        video_fps = 5
        with (self._logdir / "metrics.jsonl").open("a") as f:
            f.write(json.dumps({"step": self.step, **dict(scalars)}) + "\n")
        for name, value in scalars:
            prefix = "" if "/" in name else "scalars/"
            self.writer.add_scalar(prefix + name, np.mean(value), self.step)
        for name, value in self._images.items():
            s_name = name.replace("/", "_")
            os.makedirs(self._logdir / "images", exist_ok=True)
            save_path = self._logdir / "images" / f"{s_name}.png"
            tqdm.write(f"Saved image to {save_path}")
            torchvision.utils.save_image(torchvision.utils.make_grid(value), save_path)
            # prefix = "" if "/" in name else "images/"
            # self.writer.add_images(prefix + name, value, self.step)
        for name, value in self._videos.items():
            write_gif(name, value[0], self._logdir, video_fps)
            if value[1] is True:
                self.writer.add_video(name, value[0], self.step, video_fps)

        self._scalars = {}
        self._videos = {}

    def _compute_fps(self, step):
        if self._last_step is None:
            self._last_time = time.time()
            self._last_step = step
            return 0
        steps = step - self._last_step
        duration = time.time() - self._last_time
        self._last_time += duration
        self._last_step = step
        return steps / duration

    def close(self):
        self.writer.close()

def write_gif(name: str, frames: np.ndarray, logdir: Path, fps: int):
    B, T, C, H, W = frames.shape
    frames = frames.transpose((1, 3, 0, 4, 2)).reshape((T, H, B * W, C))
    frames = np.clip(frames * 255.0, 0, 255.0).astype(np.uint8)
    video_dir = logdir / "gifs"
    video_dir.mkdir(parents=True, exist_ok=True)
    filename = video_dir / f"{name.replace('/', '_')}.gif"
    write_gif_to_disk(frames, filename, fps)

def write_gif_to_disk(frames: np.ndarray, filename: str, fps: int = 5):
    from moviepy.editor import ImageSequenceClip

    try:
        clip = ImageSequenceClip(list(frames), fps=fps)
        clip.write_gif(filename, fps=fps, logger=None)
        tqdm.write(f"GIF saved to {filename}")
    except Exception as e:
        tqdm.write(frames.shape)
        tqdm.write("GIF Saving failed.", e)


"""
Week 1 - Coding Assignment
Specification : Print out table name and their row count
Author : Antara Saha
"""
# Import Libraries
import duckdb

conn = duckdb.connect(database='main_DB_95797') # Connect To DuckDB Database

# To fetch table name
result = conn.execute("SHOW TABLES;").fetchall()
table_list = [row[0] for row in result]

print("Table Name : Row Count")

# To fetch table row count:
for table in table_list:
    sql_command = f"select count(1) from {table};"
    result = conn.execute(sql_command).fetchall()
    row_count = [row[0] for row in result]
    print(table, " : ",row_count)

# End Script


#Setup
# To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
def reset_graph(seed=42):
    tf.reset_default_graph()
    tf.set_random_seed(seed)
    np.random.seed(seed)

# To plot pretty figures
#%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

rep_racine = "C:/Users/domin/Google Drive/Code/git/TestTensorflowJava/TestTensorflowJava"
# Where to save the figures
#PROJECT_ROOT_DIR = "."
PROJECT_ROOT_DIR = rep_racine
CHAPTER_ID = "ann"

def save_fig(fig_id, tight_layout=True):
    path = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID, fig_id + ".png")
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format='png', dpi=300)

#Perceptrons
#import numpy as np
from sklearn.datasets import load_iris
from sklearn.linear_model import Perceptron

iris = load_iris()
X = iris.data[:, (2, 3)]  # petal length, petal width
y = (iris.target == 0).astype(np.int)

per_clf = Perceptron(random_state=42)
print("FutureWarning")
per_clf.fit(X, y)
y_pred = per_clf.predict([[2, 0.5]])

print(y_pred)

# 4
a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]
b = -per_clf.intercept_ / per_clf.coef_[0][1]

axes = [0, 5, 0, 2]

x0, x1 = np.meshgrid(
        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),
        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),
    )
X_new = np.c_[x0.ravel(), x1.ravel()]
y_predict = per_clf.predict(X_new)
zz = y_predict.reshape(x0.shape)

plt.figure(figsize=(10, 4))
plt.plot(X[y==0, 0], X[y==0, 1], "bs", label="Not Iris-Setosa")
plt.plot(X[y==1, 0], X[y==1, 1], "yo", label="Iris-Setosa")

plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], "k-", linewidth=3)
from matplotlib.colors import ListedColormap
custom_cmap = ListedColormap(['#9898ff', '#fafab0'])

plt.contourf(x0, x1, zz, cmap=custom_cmap, linewidth=5)
plt.xlabel("Petal length", fontsize=14)
plt.ylabel("Petal width", fontsize=14)
plt.legend(loc="lower right", fontsize=14)
plt.axis(axes)

save_fig("perceptron_iris_plot")
plt.show()

#Activation functions
def logit(z):
    return 1 / (1 + np.exp(-z))

def relu(z):
    return np.maximum(0, z)

def derivative(f, z, eps=0.000001):
    return (f(z + eps) - f(z - eps))/(2 * eps)

z = np.linspace(-5, 5, 200)

plt.figure(figsize=(11,4))

plt.subplot(121)
plt.plot(z, np.sign(z), "r-", linewidth=2, label="Step")
plt.plot(z, logit(z), "g--", linewidth=2, label="Logit")
plt.plot(z, np.tanh(z), "b-", linewidth=2, label="Tanh")
plt.plot(z, relu(z), "m-.", linewidth=2, label="ReLU")
plt.grid(True)
plt.legend(loc="center right", fontsize=14)
plt.title("Activation functions", fontsize=14)
plt.axis([-5, 5, -1.2, 1.2])

plt.subplot(122)
plt.plot(z, derivative(np.sign, z), "r-", linewidth=2, label="Step")
plt.plot(0, 0, "ro", markersize=5)
plt.plot(0, 0, "rx", markersize=10)
plt.plot(z, derivative(logit, z), "g--", linewidth=2, label="Logit")
plt.plot(z, derivative(np.tanh, z), "b-", linewidth=2, label="Tanh")
plt.plot(z, derivative(relu, z), "m-.", linewidth=2, label="ReLU")
plt.grid(True)
#plt.legend(loc="center right", fontsize=14)
plt.title("Derivatives", fontsize=14)
plt.axis([-5, 5, -0.2, 1.2])

save_fig("activation_functions_plot")
plt.show()

def heaviside(z):
    return (z >= 0).astype(z.dtype)

def sigmoid(z):
    return 1/(1+np.exp(-z))

def mlp_xor(x1, x2, activation=heaviside):
    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)

x1s = np.linspace(-0.2, 1.2, 100)
x2s = np.linspace(-0.2, 1.2, 100)
x1, x2 = np.meshgrid(x1s, x2s)

z1 = mlp_xor(x1, x2, activation=heaviside)
z2 = mlp_xor(x1, x2, activation=sigmoid)

plt.figure(figsize=(10,4))

plt.subplot(121)
plt.contourf(x1, x2, z1)
plt.plot([0, 1], [0, 1], "gs", markersize=20)
plt.plot([0, 1], [1, 0], "y^", markersize=20)
plt.title("Activation function: heaviside", fontsize=14)
plt.grid(True)

plt.subplot(122)
plt.contourf(x1, x2, z2)
plt.plot([0, 1], [0, 1], "gs", markersize=20)
plt.plot([0, 1], [1, 0], "y^", markersize=20)
plt.title("Activation function: sigmoid", fontsize=14)
plt.grid(True)

plt.show()

#FNN for MNIST
#using tf.learn

from tensorflow.examples.tutorials.mnist import input_data

rep_racine
rep_mnist = "{}/{}".format(rep_racine, "data/")
mnist = input_data.read_data_sets(rep_mnist)

X_train = mnist.train.images
X_test = mnist.test.images
y_train = mnist.train.labels.astype("int")
y_test = mnist.test.labels.astype("int")

import tensorflow as tf

tf.logging.set_verbosity(tf.logging.INFO)

config = tf.contrib.learn.RunConfig(tf_random_seed=42) # not shown in the config

feature_cols = tf.contrib.learn.infer_real_valued_columns_from_input(X_train)
dnn_clf = tf.contrib.learn.DNNClassifier(hidden_units=[300,100], n_classes=10,
                                         feature_columns=feature_cols, config=config)
dnn_clf = tf.contrib.learn.SKCompat(dnn_clf) # if TensorFlow >= 1.1
dnn_clf.fit(X_train, y_train, batch_size=50, steps=40000)

print("fin using tf.learn")

from sklearn.metrics import accuracy_score

y_pred = dnn_clf.predict(X_test)
print(accuracy_score(y_test, y_pred['classes']))

from sklearn.metrics import log_loss

y_pred_proba = y_pred['probabilities']
print(log_loss(y_test, y_pred_proba))


#!/usr/bin/env python3
"""
Unit tests
"""

import unittest
from stl import STL

class TestSTL(unittest.TestCase):
    """
    Unit tests
    """
    def test_load_stl(self):
        """
        loading test
        """
        stl = STL()
        stl.load("models/Tux_printable.stl")
        self.assertEqual(len(stl.triangles), 14842)
        stl.load("models/Pumpkin_whole.stl")
        self.assertEqual(len(stl.triangles), 138282)

if __name__ == "__main__":
    unittest.main(verbosity=2)


from django.urls import path
from django.conf import settings
from . import views

urlpatterns = [
    path('main/', views.main, name='main'),
    path('signup/', views.signup, name='signup'),
    path('login/', views.login, name='login'),
    path('add/', views.add, name='add'),
    path('addstudent/', views.addstudent, name='addstudent'),
    path('delete/', views.delete, name='delete'),
    path('update/', views.update, name='update'),
    path('studentupdate', views.studentupdate, name='studentupdate'),
    path('studentinfo/', views.studentinfo, name='studentinfo'),

]


import logging
import asyncio

from amqtt.client import MQTTClient, ConnectException
from amqtt.mqtt.constants import QOS_1, QOS_2

#
# This sample shows how to publish messages to broker using different QOS
# Debug outputs shows the message flows
#

logger = logging.getLogger(__name__)

config = {
    "will": {
        "topic": "/will/client",
        "message": b"Dead or alive",
        "qos": 0x01,
        "retain": True,
    }
}

async def test_coro():
    C = MQTTClient()
    await C.connect("mqtt://test.mosquitto.org/")
    tasks = [
        asyncio.ensure_future(C.publish("a/b", b"TEST MESSAGE WITH QOS_0")),
        asyncio.ensure_future(C.publish("a/b", b"TEST MESSAGE WITH QOS_1", qos=QOS_1)),
        asyncio.ensure_future(C.publish("a/b", b"TEST MESSAGE WITH QOS_2", qos=QOS_2)),
    ]
    await asyncio.wait(tasks)
    logger.info("messages published")
    await C.disconnect()

async def test_coro2():
    try:
        C = MQTTClient()
        await C.connect("mqtt://test.mosquitto.org:1883/")
        await C.publish("a/b", b"TEST MESSAGE WITH QOS_0", qos=0x00)
        await C.publish("a/b", b"TEST MESSAGE WITH QOS_1", qos=0x01)
        await C.publish("a/b", b"TEST MESSAGE WITH QOS_2", qos=0x02)
        logger.info("messages published")
        await C.disconnect()
    except ConnectException as ce:
        logger.error("Connection failed: %s" % ce)
        asyncio.get_event_loop().stop()

if __name__ == "__main__":
    formatter = (
        "[%(asctime)s] %(name)s {%(filename)s:%(lineno)d} %(levelname)s - %(message)s"
    )
    formatter = "%(message)s"
    logging.basicConfig(level=logging.DEBUG, format=formatter)
    asyncio.get_event_loop().run_until_complete(test_coro())
    asyncio.get_event_loop().run_until_complete(test_coro2())


import pytesseract
from PIL import Image
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
from selenium.webdriver.support.relative_locator import with_tag_name
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
from tabulate import tabulate

#import selenium.webdriver
stuff=pytesseract.image_to_string(Image.open(r'C:\Users\jadej\OneDrive\Pictures\okokok.jpg'))
#stuff=pytesseract.image_to_boxes(Image.open(r'C:\Users\jadej\OneDrive\Pictures\okokok.jpg'))

def func(str):
    return map[str]
stuffs=""
f=len(stuff)-1
for i in range(f):
    #print(stuff[i])
    if(stuff[i]=='\n'):
        print('hit')
        stuffs+=''
    else:
        stuffs+=stuff[i]

print(stuffs.split(','))
stuffs=stuffs.split(',')
map={}

options = Options()
options.page_load_strategy = 'eager'
driver = Chrome(options=options)

#driver.implicitly_wait(6)
for items in stuffs:
    driver.get("https://www.cosdna.com/eng/stuff.php")
    #print("h0")

    searchbar = driver.find_element(By.NAME,'q')

    #print("h1")

    searchbar.clear()
    searchbar.send_keys(items)
    #print(searchbar)
    #print("h2")

    x=driver.find_elements(with_tag_name('a').below(searchbar))
    #for i in x:
    #    print(i)
    #print(len(x))
    driver.find_element(By.TAG_NAME, 'button').click()

    #print("h3")

    searchbar = driver.find_element(By.NAME, 'q')
    x=driver.find_elements(with_tag_name('a').below(searchbar))

    driver.execute_script("arguments[0].scrollIntoView(true);", x[0])
    driver.execute_script("arguments[0].click();", x[0])

    try:
        searchbar = driver.find_element(By.NAME, 'q')
    except Exception as e:
        #print(items,' not found.')
        map[items]=-1
        print(items,' ',-1)
        continue

    x=driver.find_elements(with_tag_name('a').below(searchbar))

    #print("hp")
    #print(x[0].text)
    max=-1
    for i in x[0].text:
        if(i.isdigit()):
            max=int(i)
    map[items]=max
    print(items,' ',max)
    #print(map)
    #print("he")
    #print("h3 over")

stuffs.sort(key=func)
print('\n\n\n\n\n\nCOMPLETE TABLE')
print('________ _____')

data=[]
for i in stuffs:
    data.append([i,map[i]])

print(tabulate(data, headers=['Ingrediant name', 'safety rating']))

driver.quit()


import os
from sqlalchemy import Column, String, Integer, Date, ForeignKey
from sqlalchemy.orm import relationship
from flask_sqlalchemy import SQLAlchemy
from flask import jsonify

import json

DATABASE_NAME = "capstone_fsnd"
DATABASE_PATH = os.environ['DATABASE_URL']
db = SQLAlchemy()

'''
setup_db(app)
    binds a flask application and a SQLAlchemy service
'''

def setup_db(app, database_path=DATABASE_PATH):
    app.config["SQLALCHEMY_DATABASE_URI"] = database_path
    app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
    app.config["CORS_SUPPORTS_CREDENTIALS"] = True
    db.app = app
    db.init_app(app)

'''
db_drop_and_create_all()
    drops the database tables and starts fresh
    can be used to initialize a clean database
    !!NOTE you can change the database_filename variable to have multiple versions of a database
'''

def db_drop_and_create_all():
    db.drop_all()
    db.create_all()

'''
Crew
a persistent crew entity, extends the base SQLAlchemy Model
'''

class Crew(db.Model):
    id = Column(Integer, primary_key=True)
    name = Column(String, nullable=False)
    rank = Column(String, nullable=False)
    date_of_birth = Column(Date)
    bio = Column(String)
    base_id = Column(Integer, ForeignKey('base.id'))

    '''
    insert()
        inserts a new model into a database
        EXAMPLE
            crew = Crew(name=req_name, rank=req_rank)
            crew.insert()
    '''

    def insert(self):
        db.session.add(self)
        db.session.commit()

    '''
    delete()
        deletes a new model into a database
        the model must exist in the database
        EXAMPLE
            crew = Crew(name=req_name, rank=req_rank)
            crew.delete()
    '''

    def delete(self):
        db.session.delete(self)
        db.session.commit()

    '''
    update()
        updates a new model into a database
        the model must exist in the database
        EXAMPLE
            crew = Crew.query.filter(Crew.id == id).one_or_none()
            crew.rank = 'General'
            crew.update()
    '''

    def update(self):
        db.session.commit()

    def format(self):
        return {
            'id': self.id,
            'name': self.name,
            'rank': self.rank,
            'date_of_birth': self.date_of_birth,
            'bio': self.bio,
            'base_id': self.base_id
        }

    def __repr__(self):
        return json.dumps(self.format())

class Base(db.Model):
    id = Column(Integer, primary_key=True)
    name = Column(String, nullable=False)
    planet = Column(String, nullable=False)
    crew = relationship("Crew")

    '''
    insert()
        inserts a new model into a database
        EXAMPLE
            base = Base(name=req_name, planet=req_planet)
            base.insert()
    '''

    def insert(self):
        db.session.add(self)
        db.session.commit()

    '''
    delete()
        deletes a new model into a database
        the model must exist in the database
        EXAMPLE
            base = Base(name=req_name, planet=req_planet)
            base.delete()
    '''

    def delete(self):
        db.session.delete(self)
        db.session.commit()

    '''
    update()
        updates a new model into a database
        the model must exist in the database
        EXAMPLE
            base = Base.query.filter(Base.id == id).one_or_none()
            base.planet = 'Ilum'
            base.update()
    '''

    def update(self):
        db.session.commit()

    def format(self):
        return {
            'id': self.id,
            'name': self.name,
            'planet': self.planet,
            'crew': [c.format() for c in self.crew],
        }

    def __repr__(self):
        return json.dumps(self.format())


from dcim.choices import DeviceStatusChoices
from dcim.models import Device
from virtualization.choices import VirtualMachineStatusChoices
from virtualization.models import VirtualMachine
from ipam.choices import IPAddressStatusChoices
from extras.reports import Report

# CheckPrimaryAddress reports forked from https://gist.github.com/candlerb/5380a7cdd03b60fbd02a664feb266d44
class CheckPrimaryAddressDevice(Report):
    description = "Check that every device with an assigned IP has a primary IP address assigned"

    def test_device_primary_ips(self):
        for device in Device.objects.filter(status=DeviceStatusChoices.STATUS_ACTIVE).prefetch_related('interfaces__ip_addresses').all():
            fail = False
            intcount = 0
            all_addrs = {4: [], 6: []}
            for interface in device.interfaces.all():
                if not interface.mgmt_only:
                    intcount += 1
                    for addr in interface.ip_addresses.exclude(status=IPAddressStatusChoices.STATUS_DEPRECATED).all():
                        all_addrs[addr.address.version].append(addr)
            # There may be dumb devices with no interfaces / IP addresses, that's OK
            if not device.primary_ip4 and all_addrs[4]:
                self.log_failure(device, "Device has no primary IPv4 address (could be %s)" %
                              " ".join([str(a) for a in all_addrs[4]]))
                fail = True
            if not device.primary_ip6 and all_addrs[6]:
                self.log_failure(device, "Device has no primary IPv6 address (could be %s)" %
                              " ".join([str(a) for a in all_addrs[6]]))
                fail = True
            if not fail:
                # There may be dumb devices that are used as patch panels. Check for front/back ports
                if intcount == 0 and device.frontports.count() > 0 and device.rearports.count() > 0:
                    self.log_success(device)
                # Or dumb PDUs
                elif intcount == 0 and device.powerports.count() > 0 and device.poweroutlets.count() > 0:
                    self.log_success(device)
                elif intcount == 0:
                    self.log_warning(device, "No interfaces assigned to device")
                else:
                    if len(all_addrs[4]) + len(all_addrs[6]) == 0:
                        self.log_warning(device, "No IP assigned to device")
                    else:
                        self.log_success(device)

class CheckPrimaryAddressVM(Report):
    description = "Check that every vm with an assigned IP has a primary IP address assigned"

    def test_vm_primary_ips(self):
        for vm in VirtualMachine.objects.filter(status=VirtualMachineStatusChoices.STATUS_ACTIVE).prefetch_related('interfaces__ip_addresses').all():
            fail = False
            intcount = 0
            all_addrs = {4: [], 6: []}
            for interface in vm.interfaces.all():
                intcount += 1
                for addr in interface.ip_addresses.exclude(status=IPAddressStatusChoices.STATUS_DEPRECATED).all():
                    all_addrs[addr.address.version].append(addr)
            # A VM is useless without an IP address
            if intcount == 0:
                self.log_failure(vm, "Virtual machine has no interfaces")
                continue
            if not all_addrs[4] and not all_addrs[6]:
                self.log_failure(vm, "Virtual machine has no IP addresses")
                continue
            if not vm.primary_ip4 and all_addrs[4]:
                self.log_failure(vm, "Virtual machine has no primary IPv4 address (could be %s)" %
                              " ".join([str(a) for a in all_addrs[4]]))
                fail = True
            if not vm.primary_ip6 and all_addrs[6]:
                self.log_failure(vm, "Virtual machine has no primary IPv6 address (could be %s)" %
                              " ".join([str(a) for a in all_addrs[6]]))
                fail = True
            if not fail:
                self.log_success(vm)


from model import MLP
from optimizer import SGD
import argparse
from tqdm import tqdm
from dataset import MNIST_Dataset
import cfg
import random
import numpy as np
from utils import AverageMeter,compute_acc,Accuracy_averagemeter
from tensor import Tensor
import logging
import os
import matplotlib.pyplot as plt
import pickle

SEED = 729608
random.seed(SEED)
np.random.seed(SEED)
LOG_FMT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s"

def augment_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument('--epochs',type=int,default=cfg.epoch)
    parser.add_argument('--lr',type=float,default=cfg.lr)
    parser.add_argument('--batch-size',type=int,default=cfg.batch_size)
    parser.add_argument('--log-dir',type=str,default='./log/')
    parser.add_argument('--save-dir',type=str,default='./model/')
    parser.add_argument('--nl',type=int,default=2,help='num of layers')
    parser.add_argument('--dim-list',type=int,nargs='+',default=[28*28,512])

    return parser

def split_batch(index_list,batch_size):
    batch_index = random.sample(index_list, batch_size)
    for each in batch_index:
        index_list.remove(each)

    return np.array(batch_index)

def main():
    parser = augment_parser()
    args = parser.parse_args()
    dataset = MNIST_Dataset()
    train_data,test_data,train_label,test_label = dataset.train_data,dataset.test_data,dataset.train_label,dataset.test_label
    mlp = MLP(n_layers=args.nl,n_class=10,dim_list=args.dim_list)
    optimizer = SGD(mlp.layers[:-1], args.lr)

    log_dir = args.log_dir
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

    if not os.path.exists(args.save_dir):
        os.makedirs(args.save_dir)

    train_loss_list = []
    train_acc_list = []
    test_loss_list = []
    test_acc_list = []

    plt.figure(figsize=(12,10))

    for i in tqdm(range(args.epochs)):
        train_loss, train_acc = train_epoch(train_data,train_label,mlp,optimizer,batch_size=args.batch_size,
                                            lr=args.lr,cur_epoch=i,log_file=log_dir + 'train.log')
        test_loss, test_acc = test_epoch(test_data,test_label,mlp,i,log_file=log_dir + 'train.log')
        train_loss_list.append(train_loss)
        train_acc_list.append(train_acc)
        test_loss_list.append(test_loss)
        test_acc_list.append(test_acc)

    save_name = args.save_dir + 'mlp_epoch' + str(args.epochs)
    save_model(mlp,save_name)
    plt.subplot(1,2,1)
    plt.xlabel('epoch')
    plt.ylabel('loss')
    plt.title('loss curve')
    plt.plot(train_loss_list,color='pink',marker='o',linewidth=2,label='train_loss')
    plt.plot(test_loss_list,color='blue',marker='x',linewidth=2,label='test_loss')
    plt.legend(loc='best')
    plt.subplot(1,2,2)
    plt.xlabel('epoch')
    plt.ylabel('accuracy')
    plt.title('acc curve')
    plt.plot(train_acc_list, color='pink', marker='o', linewidth=2, label='train_acc')
    plt.plot(test_acc_list, color='blue', marker='x', linewidth=2, label='test_acc')

    plt.legend(loc='best')
    plt.savefig(log_dir+'curve.png')

def train_epoch(train_data,train_label,mlp,optimizer,batch_size,lr,cur_epoch,log_file):
    mlp.train()
    logging.basicConfig(
        level=logging.INFO,
        filename=log_file,
        filemode="w",
        format=LOG_FMT
    )
    train_loss = AverageMeter()
    train_acc = Accuracy_averagemeter()
    index_list = [i for i in range(train_data.shape[0])]

    while len(index_list) >= batch_size:
        batch_index = split_batch(index_list, batch_size)
        batch_data, batch_label = train_data[batch_index, :], train_label[batch_index, :]
        batch_data = Tensor(value=batch_data)
        output, loss = mlp(batch_data, batch_label)
        train_loss.update(val=float(loss), n=batch_size)
        train_acc.update(val= compute_acc(output.value,batch_label), n=batch_size)
        mlp.backward(target=batch_label)
        optimizer.step()

    if len(index_list):
        bs = len(index_list)
        batch_index = index_list
        batch_data, batch_label = train_data[batch_index, :], train_label[batch_index, :]
        batch_data = Tensor(value=batch_data)

        output, loss = mlp(batch_data, batch_label)
        train_loss.update(val=float(loss), n=bs)
        train_acc.update(val=compute_acc(output.value, batch_label), n=bs)
        mlp.backward(target=batch_label)
        optimizer.step()

    log_info = 'Train Epoch {0} --- train loss : {1} --- train accuracy:{2}'.format(cur_epoch + 1, train_loss.avg,train_acc.avg)
    logging.info(log_info)
    return train_loss.avg, train_acc.avg

def test_epoch(test_data,test_label,mlp,cur_epoch,log_file):
    mlp.eval()
    test_loss = AverageMeter()
    test_acc = Accuracy_averagemeter()
    logging.basicConfig(
        level=logging.INFO,
        filename=log_file,
        filemode="w",
        format=LOG_FMT
    )

    for i in range(test_data.shape[0]):
        data, label = test_data[i].reshape(1,-1),test_label[i].reshape(1,-1)
        data = Tensor(value=data)
        pred, loss = mlp(data,label)
        test_loss.update(val=float(loss))
        test_acc.update(val=int(compute_acc(pred.value,label)))

    log_info = 'Test Epoch {0} --- test loss : {1} --- test accuracy:{2}'.format(cur_epoch + 1, test_loss.avg,test_acc.avg)
    logging.info(log_info)
    return test_loss.avg, test_acc.avg

def save_model(model:MLP,name:str):
    with open(name+'.qbw','wb') as f:
        pickle.dump(model.state_dict(),f)

if __name__ == "__main__":
    main()


import json, os, logging, re, requests, urllib.parse as urlparse

# import json, os, logging, requests, datetime
logger = logging.getLogger()
logger.setLevel(logging.INFO)

session_requests = requests.session();

### Handler ###
def lambda_handler(event, context):
    global session_requests

    logger.info("## EVENT INFO ##")
    logger.info(event)

    try:
        # Get the parameters from the post
        urlparams = event.get('body', '')
        parsed_urlparams = urlparse.parse_qs(urlparams)

        # since urlparse.parse_qs returns an array
        get_param = lambda x: parsed_urlparams[x][0] if x in parsed_urlparams else None

        first_name = get_param('7ad557e1b4ecaa31cc945b87f6366cdb')
        email = get_param('a583bfe142f87e3c25b418f2bf60a515')
        source = get_param('7f19195a536f8bbc85951f5e19b8a546') # State from drop downs
        optin = get_param('5a7b17ea80f8af83b8e4e4482d592eb7') # City from drop downs
        lead_form = get_param('df2a292f42e2ff932fab665ee65547f8') # Form for exact detail

        # Clean up the source and optin
        source = re.sub(r'\?.*?$', '',  source)
        optin = re.sub(r'\?.*?$', '',  optin)
        if source: source = source.replace("+", " ")
        if optin: optin = optin.replace("+"," ")
        if not optin: optin = "emails"
        if not source: source = "unknown"

        # get the tags to add
        tags = get_tags(optin=optin, source=source)

        # Could replace this with a search to make it more robust
        if optin and optin.lower() == 'book making guide': form_id = 272097
        elif optin and optin.lower() == 'emails': form_id = 949995
        elif optin and optin.lower() == 'influence map': form_id = 458062
        elif optin and optin.lower() == 'mcdt': form_id = 341476
        elif optin and optin.lower() == 'mcdt (single opt-in)': form_id = 462329
        elif optin and optin.lower() == 'style mini-course': form_id = 459136
        elif optin and optin.lower() == 'style mini-course (single opt-in)': form_id = 462513
        elif optin and optin.lower() == 'sketchbook mini-course': form_id = 295490
        elif optin and optin.lower() == 'stsbc wl': form_id = 330122
        elif optin and optin.lower() == 'studiomates wl': form_id = 387829
        elif optin and optin.lower() == 'unknown': form_id = 949995 # Emails
        elif optin and optin.lower() == 'waf activity guide': form_id = 274068
        elif optin and optin.lower() == 'waf ebook': form_id = 272096
        elif optin and optin.lower() == 'tools': form_id = 272065
        elif optin and optin.lower() == 'test': form_id = 459136
        else:
            logging.error('Optin: ' + str(optin) + ' does not exist')
            logging.info(f'first_name: {first_name}, email: {email}, source: {source}, optin: {optin}')
            raise AttributeError('optin: ' + str(optin) + ' does not have an associated form')

        # See if the subscriber already exists, and if they have optin and source already.
        try:
            sub = get_single_subscriber(session=session_requests, email_address=email)
            # If so, then ignore those fields
            if 'lead_opt-in' in sub and sub['lead_opt-in']: optin = None
            if 'lead_source' in sub and sub['lead_source']: source = None
        except Exception as e:
            # No subscriber (or unable to id), so continue as normal
            print(f'No subscriber, {email}', e)
            pass

        #Source and optin here are for custom field data
        subscribe_to_form(
            form_id=form_id,
            email=email,
            first_name=first_name,
            source=source,
            optin=optin,
            lead_form=lead_form,
            tags=tags,
            session=session_requests
        )

        responseBody = {"Success": f'Form submitted successfully', "event":event}
        responseCode = 200

    except ValueError as e:
        logging.error(f'ValueError {e}')
        raise e
        print("Error while decoding event!")
        responseBody = {
            "Error": "Error while decoding event!",
            "Error_desc": "Bad payload",
        }
        responseCode = 400

    except AttributeError as e:
        logging.error(f'AttributeError {e}')
        raise e
        responseCode = 406 # Not acceptable
        responseBody = {
            "Error": "This optin is not supported",
            "ErrorDetail": str(a)
        }

    except Exception as e:
        logging.error(f'Exception: {e}')
        raise e
        print("Exception!", e)
        responseBody = {
            "Error": "Exception",
        }
        responseCode = 400

    # Proxy Lambda must be in this format
    returnBody = {
        "headers":{'Access-Control-Allow-Origin': '*' }, #'Content-Type': 'application/json',
        "statusCode": responseCode,
        "body": json.dumps(responseBody)
    }

    return returnBody;

### Functions ###
def subscribe_to_form(
        form_id:int,
        email:str,
        first_name:str,
        source:str,
        optin:str,
        lead_form:str,
        tags:list,
        session=None
    ):

    if not session: session = requests.session()

    logging.info(f'subscribing- form:{form_id}, name:{first_name}, email:{email}, source:{source}, optin:{optin}')
    url = f'https://api.convertkit.com/v3/forms/{form_id}/subscribe'
    fields = {'lead_form': lead_form}
    if source: fields.update({'lead_source': source}) # Only include if they aren't None
    if optin: fields.update({'lead_opt-in': optin}) # Only include if they aren't None

    data = {
        'api_key': os.environ['CK_API_KEY'],
        'email': email,
        'fields':fields,
        'tags': tags,
        'first_name': first_name
    }
    headers = {'content-type': 'application/json'}
    logging.info(f'post url:{url}, data:{json.dumps(data)}, headers:{json.dumps(headers)}')
    r = session.post(url=url, data=json.dumps(data), headers=headers)
    if r.status_code != 200: raise Exception(f'Status Code: {r.status_code}, text: {r.text}')
    return r.text

def get_single_subscriber(session=None, email_address:str = None, subscriber_id:int = None, cancelled=False):
    """Returns flat list of subscriber details (i.e. replaces fields field with the data)"""
    if not session: session = requests.session()

    headers={'content-type': 'application/json'}
    data = {'api_secret': os.environ['CK_API_SECRET']}; data

    if cancelled: data['sort_field']='cancelled_at'

    if subscriber_id:
        url = f'https://api.convertkit.com/v3/subscribers/{subscriber_id}'
    elif email_address:
        url = f'https://api.convertkit.com/v3/subscribers'
        data['email_address'] = email_address
    else:
        raise Exception('Must provide subscriber id or email address')

    with session.get(url, data=json.dumps(data), headers=headers) as resp:
        if resp.status_code != 200:
            print("FAILURE::{0}".format(url))
        json_resp = json.loads(resp.text); json_resp
        if subscriber_id:
            if 'subscriber' not in json_resp or not json_resp['subscriber']: raise Exception('no result')
            json_resp.update(json_resp['subscriber']); del json_resp['subscriber']
        elif email_address:
            if 'subscribers' not in json_resp or not json_resp['subscribers']: raise Exception('no result')
            json_resp = json_resp['subscribers'][0]

        json_resp.update(json_resp['fields']); del json_resp['fields']
        return json_resp

    raise Exception("Unable to get subscriber")

# TODO: Could get evergreen tags to make this more robust. Maybe a 1-time process
def get_tags(optin=None, source=None):
    """Provide optin and source and return the tags"""

    tags=[
            {'id': 291886, 'name': 'Opt-in: Book Making Guide'},
            {'id': 969850, 'name': 'Opt-in: Emails'},
            {'id': 291286, 'name': 'Opt-in: Influence Map'},
            {'id': 399666, 'name': 'Opt-in: MCDT'},
            {'id':1066736, 'name': 'Opt-in: MCDT (Single Opt-in)'},
            {'id': 880901, 'name': 'Opt-in: Style Mini-course'},
            {'id':1054340, 'name': 'Opt-in: Style Mini-course (Single Opt-in)'},
            {'id': 989221, 'name': 'Opt-in: Sketchbook Mini-course'},
            {'id': 880904, 'name': 'Opt-in: Studiomates WL'},
            {'id': 881005, 'name': 'Opt-in: StSBC WL'},
            {'id': 291875, 'name': 'Opt-in: Tools'},
            {'id': 881851, 'name': 'Opt-in: Unknown'},
            {'id': 294908, 'name': 'Opt-in: WAF Activity Guide'},
            {'id': 291279, 'name': 'Opt-in: WAF ebook'},
            {'id': 294894, 'name': 'Source: MailChimp'},
            {'id': 323271, 'name': 'Source: WAF Book Reading'},
            {'id': 880718, 'name': 'Source: Medium'},
            {'id': 880719, 'name': 'Source: Skillshare'},
            {'id': 880724, 'name': 'Source: Instagram'},
            {'id': 1122867, 'name': 'Source: Google Ad'},
            {'id': 880728, 'name': 'Source: FB Ad'},
            {'id': 880916, 'name': 'Source: Facebook'},
            {'id': 880917, 'name': 'Source: MCS Page'},
            {'id': 880918, 'name': 'Source: Unknown'},
            {'id': 880919, 'name': 'Source: Twitter'},
            {'id': 928112, 'name': 'Source: Manual'}
        ]

    optin_tag = [t['id'] for t in tags if optin and 'Opt-in: ' in t['name'] and optin.lower() in t['name'].lower()]
    source_tag = [t['id'] for t in tags if source and 'Source: ' in t['name'] and source.lower() in t['name'].lower()]

    return optin_tag + source_tag # Should be a list of ids


from pydantic import BaseModel

from distances.entities import Address

from .interactor import ComputeDistanceUCI

class AddressM(BaseModel):
    street: str
    house_number: int
    city: str
    country: str

class ComputeDistanceReqM(BaseModel):
    origin: AddressM
    destination: AddressM

def compute_distance_controller(account_id: int, reqm: ComputeDistanceReqM) -> ComputeDistanceUCI:
    ucin = ComputeDistanceUCI(
        account_id=account_id,
        origin=Address(
            street=reqm.origin.street,
            house_number=reqm.origin.house_number,
            city=reqm.origin.city,
            country=reqm.origin.country,
        ),
        destination=Address(
            street=reqm.destination.street,
            house_number=reqm.destination.house_number,
            city=reqm.destination.city,
            country=reqm.destination.country,
        )
    )
    return ucin


import numpy as np
import typing

from defenses.detection.fourier.FourierPeakMatrixCollector import FourierPeakMatrixCollector
from defenses.detection.DetectionDefense import DetectionDefense
from scaling.ScalingApproach import ScalingApproach
from utils.SimilarityMeasurementTool import SimilarityMeasurementTool
from utils.SimilarityMeasure import SimilarityMeasure
from defenses.prevention.PreventionDefenseType import PreventionTypeDefense
from defenses.prevention.PreventionDefenseGenerator import PreventionDefenseGenerator
from defenses.prevention.PreventionDefense import PreventionDefense

class PrevBasedFilteringDetectionDefense(DetectionDefense):
    """
    -- Clean-Filter from Paper with Options: {Median-Filter, Random-Filter} + {PSNR, SSIM} --

    Detection defense based on applying the prevention defense on an attack image;
    and then comparing input image & filtered image.
    """

    def __init__(self,
                 verbose: bool,
                 scaler_approach: ScalingApproach,
                 prevention_defense_type: PreventionTypeDefense,
                 fourierpeakmatrixcollector: FourierPeakMatrixCollector,
                 similarity_measurement: SimilarityMeasure,
                 prevention_bandwidthfactor: int = 1
                 ):
        """
        Init detection method.
        :param verbose: show more information
        :param scaler_approach: scaling settings
        :param prevention_defense_type: specify type of prevention method {median_filter, random_filter}.
        :param similarity_measurement: specify type of metric
        """
        super().__init__(verbose, scaler_approach)
        self.prevention_defense_type: PreventionTypeDefense = prevention_defense_type
        self.similarity_measurement: SimilarityMeasure = similarity_measurement
        self.fourierpeakmatrixcollector: FourierPeakMatrixCollector = fourierpeakmatrixcollector
        self.prevention_bandwidthfactor: int = prevention_bandwidthfactor

    # @Overwrite
    def detect_attack(self, att_image: np.ndarray) -> typing.Tuple[float, dict]:

        preventiondefense: PreventionDefense = PreventionDefenseGenerator.create_prevention_defense(
            defense_type=self.prevention_defense_type, scaler_approach=self.scaler_approach,
            fourierpeakmatrixcollector=self.fourierpeakmatrixcollector,
            bandwidth=self.prevention_bandwidthfactor, verbose_flag=False, usecythonifavailable=True
        )

        filtered_image = preventiondefense.make_image_secure(att_image=att_image)

        score = SimilarityMeasurementTool.sim_measure(img1=att_image, img2=filtered_image,
                                                      similarity_measurement=self.similarity_measurement)
        if self.verbose:
            print("{}, {}, {} got scores: {}".format(self.prevention_defense_type.name, self.prevention_bandwidthfactor,
                                                     self.similarity_measurement.name, score))

        return score, {} # {'filtered_image': filtered_image}