#!/usr/bin/env python3

# file    :  fuzz-webview.py
# repo    :  https://github.com/fuzzah/fuzzaide
# author  :  https://github.com/fuzzah
# license :  MIT
# check repository for more information

"""
fuzz webview: simple Flask App to aggregate and display data from fuzzer_stats files
Code here may partially overlap with fuzzman.py
"""

import os
import sys
import glob
import time
import argparse
from datetime import datetime
from threading import Thread, Lock, Event

from flask import Flask, render_template, send_from_directory

from fuzzaide.common.fuzz_stats import get_afl_stat_name, is_afl_fuzzer_stats_old

class StatsLoader(Thread):
    def __init__(self, dir, lock):
        super().__init__()
        self.dir = dir  # fuzzers sync dir
        self.stats = list()  # list of dicts
        self.common_stats = dict()
        self.lock = lock  # lock for self.stats
        self._stop_evt = Event()

        self.start()

    def get_fuzzer_stats(self, fname):
        """
        Form a dictionary from fuzzer_stats
        """

        try:
            with open(fname, "rt") as f:
                data = f.read()
        except OSError:
            return None

        if data is None:  # file is empty
            return None

        stats = dict()
        for line in data.split("\n"):
            if ":" in line:
                k, v = line.split(":", 1)
                k = k.strip()
                v = v.strip()
                stats[k] = v
        if len(stats) < 1:
            return None

        return stats

    @staticmethod
    def update_stat_timestamp(stats_dict, stat_name, saved_newest_stamp):
        """
        Use this method to update last path (crash, hang, etc) timestamp.
        Example: newest_path_stamp = update_stat_timestamp(stats, "last_path", newest_path_stamp)
        """
        stamp = stats_dict.get(stat_name)

        if stamp is None:
            return saved_newest_stamp

        try:
            stamp = int(stamp)
        except ValueError:
            return saved_newest_stamp

        if stamp > saved_newest_stamp:
            return stamp

        return saved_newest_stamp

    @staticmethod
    def format_seconds(seconds):
        s = seconds % 60
        m = (seconds // 60) % 60
        h = (seconds // 3600) % 24
        d = seconds // 86400

        if d > 0:
            return "%d days, %d hrs, %d min, %d sec" % (d, h, m, s)
        elif h > 0:
            return "%d hrs, %d min, %d sec" % (h, m, s)
        elif m > 0:
            return "%d min, %d sec" % (m, s)

        return "%d sec" % (s,)

    def load_stats(self):
        fuzzer_stats = "fuzzer_stats"
        filenames = glob.glob(os.path.join(self.dir, "*", fuzzer_stats))
        single_fuzzer_stats = os.path.join(self.dir, fuzzer_stats)
        if os.path.isfile(single_fuzzer_stats):
            filenames.append(single_fuzzer_stats)
        filenames = sorted(filenames)

        newest_path_stamp = 0
        newest_hang_stamp = 0
        newest_crash_stamp = 0

        sum_execs = 0
        sum_paths = 0
        sum_hangs = 0
        sum_crashes = 0

        common_stats = {}
        all_stats = []

        now = int(datetime.now().timestamp())

        for fname in filenames:
            stats = self.get_fuzzer_stats(fname)
            if not stats:
                continue

            use_old_style = is_afl_fuzzer_stats_old(stats)

            last_upd = int(
                stats.get(get_afl_stat_name("last_update", use_old_style), 0)
            )
            if last_upd > 0:
                delta = now - last_upd
                mess = self.format_seconds(delta) + " ago"
                if delta > 90:
                    mess += " (NOT RUNNING?)"
                stats["last_update"] = mess
            all_stats.append(stats)

            crashes = int(
                stats.get(get_afl_stat_name("unique_crashes", use_old_style), 0)
            )
            hangs = int(stats.get(get_afl_stat_name("unique_hangs", use_old_style), 0))
            paths_total = int(
                stats.get(get_afl_stat_name("paths_total", use_old_style), 0)
            )

            sum_crashes += crashes
            sum_hangs += hangs
            sum_paths += paths_total
            sum_execs += int(stats.get("execs_done", 0))

            newest_path_stamp = self.update_stat_timestamp(
                stats, get_afl_stat_name("last_path", use_old_style), newest_path_stamp
            )
            newest_hang_stamp = self.update_stat_timestamp(
                stats, get_afl_stat_name("last_hang", use_old_style), newest_hang_stamp
            )
            newest_crash_stamp = self.update_stat_timestamp(
                stats,
                get_afl_stat_name("last_crash", use_old_style),
                newest_crash_stamp,
            )

        if newest_path_stamp == 0:
            return

        e = float(sum_execs)
        c = ""
        if e >= 1000000000:
            e /= 1000000000
            c = "B"
        elif e >= 1000000:
            e /= 1000000
            c = "M"
        elif e >= 1000:
            e /= 1000
            c = "K"

        execs = "Unknown"
        if len(c) > 0:
            if c == "B":
                execs = "%.4f%c" % (e, c)
            else:
                execs = "%.2f%c" % (e, c)
        else:
            execs = "%.0f" % (e,)

        common_stats["execs"] = execs

        newest_path_delta = now - newest_path_stamp
        newest_path_fmt = self.format_seconds(newest_path_delta)

        common_stats["paths"] = sum_paths
        common_stats["last_new_path"] = newest_path_delta
        common_stats["paths_info_fmt"] = "Paths: %d.\tLast new path: %s ago" % (
            sum_paths,
            newest_path_fmt,
        )

        common_stats["hangs"] = sum_hangs

        if sum_hangs > 0:
            delta = now - newest_hang_stamp
            seconds_fmt = self.format_seconds(delta)
            common_stats["last_new_hang"] = delta
            common_stats["hangs_info_fmt"] = "Hangs: %d.\tLast new hang: %s ago" % (
                sum_hangs,
                seconds_fmt,
            )
        else:
            common_stats["last_new_hang"] = -1
            common_stats["hangs_info_fmt"] = "Hangs: 0"

        common_stats["crashes"] = sum_crashes
        if sum_crashes > 0:
            delta = now - newest_crash_stamp
            seconds_fmt = self.format_seconds(delta)
            common_stats["last_new_crash"] = delta
            common_stats[
                "crashes_info_fmt"
            ] = "Crashes: %d.\tLast new crash: %s ago" % (sum_crashes, seconds_fmt)
        else:
            common_stats["last_new_crash"] = -1
            common_stats["crashes_info_fmt"] = "Crashes: 0"

        self.lock.acquire()
        self.stats = all_stats
        self.common_stats = common_stats
        self.lock.release()

    def run(self):
        while True:
            self.load_stats()
            time.sleep(1.0)
            if self._stop_evt.is_set():
                break

    def stop(self):
        self._stop_evt.set()

class WebApp(Flask):
    def __init__(self, appname, syncdir):
        super().__init__(appname)
        self.stats_lock = Lock()
        self.stats_loader = StatsLoader(syncdir, self.stats_lock)

        @self.route("/", methods=["GET"])
        def _index():
            return self.render_index()

        @self.route("/favicon.ico")
        def _favicon():
            return send_from_directory(
                os.path.join(self.root_path, "static"),
                "favicon.ico",
                mimetype="image/x-icon",
            )

    def stop_stats_loader(self):
        self.stats_loader.stop()

    def load_stats(self):
        self.stats_lock.acquire()
        fuzz_stats = self.stats_loader.stats
        common_stats = self.stats_loader.common_stats
        self.stats_lock.release()
        return common_stats, fuzz_stats

    def render_index(self):
        now = datetime.now()
        curtime_fmt = now.strftime(r"%H:%M:%S %d.%m.%Y")
        comm_stats, fuzz_stats = self.load_stats()
        comm_stats["curtime_fmt"] = curtime_fmt
        return render_template(
            "index.html", fuzzer_stats=fuzz_stats, common_stats=comm_stats
        )

def main():
    parser = argparse.ArgumentParser(
        prog=os.path.basename(sys.argv[0]),
        description="%(prog)s - simple dashboard for AFL-like fuzzers",
        epilog="by default, listens on http://127.0.0.1:8080",
    )

    parser.add_argument(
        "-o", "--sync-dir", help="fuzzer sync directory", type=str, required=True
    )
    parser.add_argument(
        "-p", "--port", help="port to bind to", type=int, default="8080"
    )
    parser.add_argument(
        "-l", "--addr", help="address to bind to", default="127.0.0.1", type=str
    )
    parser.add_argument(
        "-v", "--verbose", help="verbose mode", action="count", default=0
    )

    if len(sys.argv) < 2:
        parser.print_help()
        return 0

    args = parser.parse_args()

    print("Using sync dir:", args.sync_dir)
    print("Trying to start web server on http://%s:%s.." % (args.addr, args.port))

    app = WebApp(__name__, args.sync_dir)
    app.run(host=args.addr, port=args.port, debug=args.verbose > 0)
    print("\nLeaving..")
    app.stop_stats_loader()
    return 0

if __name__ == "__main__":
    sys.exit(main())


"""
Given an array of integers nums and an integer target,
return indices of the two numbers such that they add up to target.

You may assume that each input would have exactly one solution,
and you may not use the same element twice.

You can return the answer in any order.
"""

import unittest
from typing import List

class Solution:
    def twoSum(self, nums: List[int], target: int) -> List[int]:

        for i in range(len(nums)-1):
            current = nums[i]

            for j in range(i+1, len(nums)):
                next_c = nums[j]

                if next_c + current == target:
                    return [i, j]

class TestSolution(unittest.TestCase):
    def testTwoSum(self):
        s = Solution()

        nums = [2,7,11,15]
        target = 9
        expected = [0,1]
        actual = s.twoSum(nums, target)
        self.assertEqual(actual, expected)

        nums = [3,2,4]
        target = 6
        expected = [1,2]
        actual = s.twoSum(nums, target)
        self.assertEqual(actual, expected)

        nums = [3,3]
        target = 6
        expected = [0,1]
        actual = s.twoSum(nums, target)
        self.assertEqual(actual, expected)

unittest.main()


#!/usr/bin/env python3

import math
from collections import defaultdict
from typing import List
from typing import Set
try:
    from _tree import *
except ImportError:
    pass

class Solution:
    def shiftGrid(self, grid: List[List[int]], k: int) -> List[List[int]]:
        if k == 0:
            return grid

        m = len(grid)
        n = len(grid[0])
        new_grid = []
        for _ in range(m):
            new_grid.append([0] * n)

        for i in range(m):
            for j in range(n):
                new_j = (j + k) % n
                new_i = (i + (j + k) // n) % m
                new_grid[new_i][new_j] = grid[i][j]

        print(new_grid)
        return new_grid

if __name__ == '__main__':
    Solution().shiftGrid(grid = [[1,2,3],[4,5,6],[7,8,9]], k = 1)
    Solution().shiftGrid(grid = [[3,8,1,9],[19,7,2,5],[4,6,11,10],[12,0,21,13]], k = 4)
    Solution().shiftGrid(grid = [[1,2,3],[4,5,6],[7,8,9]], k = 9)


'''

'''

import subprocess, os.path, logging, tempfile, warnings

from ngs.alignment import Aligner

class BWA(Aligner):

    '''
    A class to run the BWA aligner

    '''

    #
    # in order to perform a BWA alignment first an
    # sam index (sai) should be constructed. Seperately
    # for the forward and reverse read in the case of a
    # paired end
    #
    #
    #
    #
    #
    #

    def __init__( self ):
        '''

        '''
        self._locs = {}
        logging.info( "Intializing the BWA object" )

        # set the nice loc
        self._locs['nice'] = 'nice'
        self._locs['bwa']  = 'bwa'

    def is_available( self, fname ):
        '''
        Checks whether the specified index is available

        Args:
        self   -
        fname  - the file path to check for

        Returns:
        True or False
        '''
        suffixes = [ 'amb', 'ann', 'bwt', 'fai', 'pac', 'sa', 'rbwt', 'rpac', 'rsa' ]
        present = True
        for s in suffixes:
            fn = fname + '.' + s
            if not os.path.exists( fn ):
                present = False

        # return whether the index is present or not
        return present

    def align( self, index, infiles, outfile, opt={}, saifiles=[] ):
        '''
        Use the BWA aligner to align the reads in infiles
        and output the data to the outfile

        Args:
        self     -  a reference to self
        infiles  -  the input FastQ file
        index    -  the index to which to align
        outfile  -  the output SAM file
        opt      -  options to pass to the alignment

        Returns:
        A tuple with the error, samfile and error file
        '''

        retval = None
        opt    = self._add_default_options( opt )
        if len( infiles ) == 2:
            self._index   = index
            self._samfile = outfile

            # fix the output file names
            if len(saifiles) != len(infiles):
                for f in infiles:
                    if not os.path.exists( f ):
                        logging.warn( "input file %f does not exist")
                        return None
                    saifiles.append( "%s.sai" % f )

            # actually perform the alignment
            for i in range( len( infiles ) ) :
                 self._infile  = infiles[i]
                 self._saifile = saifiles[i]
                 self._aln( opt=opt['aln'] )

            # gather the results
            self._infile  = infiles
            self._saifile = saifiles
            retval = self._sampe( opt=opt['sampe'] )

        elif len(infiles) == 1:
            self._index   = index
            self._infile  = infiles[0]
            if len(saifiles) == 1:
                self._saifile = saifiles[0]
            else:
                self._saifile = "%s.sai" % infiles[0]
            self._samfile = outfile

            # perform the alignment and results gathering
            self._aln( opt=opt['aln'] )
            retval = self._samse( opt=opt['samse'] )
        else:
            logging.warn( 'Incorrect number of input files' )

        #
        return retval

    def _aln( self, opt={} ):
        '''
        Perform the alignment and make the sam index file

        Args:
        self
        opt

        Returns:
        '''
        #
        cmd = []
        cmd.append( self._locs['nice'] )
        cmd.append( self._locs['bwa'] )
        cmd.append( 'aln' )

        # add the options
        cmd = self._add_options( cmd, opt )

        # add the index
        cmd.append( self._index )
        cmd.append( self._infile )

        # run the process
        (err,pout,perr) = self._subprocess( cmd, stdout=self._saifile )

        #
        return (err,pout,perr)

    def _samse( self, opt={} ):
        '''
        Assemble the single read alignment file

        Args:

        Returns:

        '''
        #
        cmd = []
        cmd.append( self._locs['nice'] )
        cmd.append( self._locs['bwa'] )
        cmd.append( 'samse' )

        #
        cmd = self._add_options( cmd, opt )

        #
        cmd.append( self._index )
        cmd.append( self._saifile )
        cmd.append( self._infile )

        #
        (err,pout,perr) = self._subprocess( cmd, stdout=self._samfile )

        #
        return (err,pout,perr)

    def _sampe( self, opt={} ):
        '''
        Assemble a paired end alignment

        Args:

        Returns:

        '''
        #
        cmd = []
        cmd.append( self._locs['nice'] )
        cmd.append( self._locs['bwa'] )
        cmd.append( 'sampe' )

        #
        cmd = self._add_options( cmd, opt )
        cmd.append( self._index )

        #
        cmd.append( self._saifile[0] )
        cmd.append( self._saifile[1] )

        #
        cmd.append( self._infile[0] )
        cmd.append( self._infile[1] )

        #
        (err,pout,perr) = self._subprocess( cmd, stdout=self._samfile )
        return (err,pout,perr)

    def _add_default_options( self, opt ):
        '''
        This method will add 'sensible' default options.
        So NARWHAL will not crash if these are absent. This
        function will not fill fields that are defined.

        Args:
        opt   -    the options hash

        Returns:
          return the filled options
        '''
        if not opt.has_key( 'sampe' ):
            # add the sampe options
            opt['sampe'] = {}

        if not opt.has_key( 'samse' ):
            # the samse options
            opt['samse'] = {}

        if not opt.has_key( 'aln' ):
            # the alignment options
            opt['aln'] = {}

        return opt

if __name__ == '__main__':
    pass


import pytest
from dbt.tests.util import run_dbt, check_relations_equal

from dbt.tests.adapter.simple_copy.fixtures import (
    _PROPERTIES__SCHEMA_YML,
    _SEEDS__SEED_INITIAL,
    _MODELS__ADVANCED_INCREMENTAL,
    _MODELS__COMPOUND_SORT,
    _MODELS__DISABLED,
    _MODELS__EMPTY,
    _MODELS_GET_AND_REF_UPPERCASE,
    _MODELS__INCREMENTAL,
    _MODELS__INTERLEAVED_SORT,
    _MODELS__MATERIALIZED,
    _MODELS__VIEW_MODEL,
)

class TestSimpleCopyUppercase:
    @pytest.fixture(scope="class")
    def dbt_profile_target(self):
        return {
            "type": "postgres",
            "threads": 4,
            "host": "localhost",
            "port": 5432,
            "user": "root",
            "pass": "password",
            "dbname": "dbtMixedCase",
        }

    @pytest.fixture(scope="class")
    def models(self):
        return {
            "ADVANCED_INCREMENTAL.sql": _MODELS__ADVANCED_INCREMENTAL,
            "COMPOUND_SORT.sql": _MODELS__COMPOUND_SORT,
            "DISABLED.sql": _MODELS__DISABLED,
            "EMPTY.sql": _MODELS__EMPTY,
            "GET_AND_REF.sql": _MODELS_GET_AND_REF_UPPERCASE,
            "INCREMENTAL.sql": _MODELS__INCREMENTAL,
            "INTERLEAVED_SORT.sql": _MODELS__INTERLEAVED_SORT,
            "MATERIALIZED.sql": _MODELS__MATERIALIZED,
            "VIEW_MODEL.sql": _MODELS__VIEW_MODEL,
        }

    @pytest.fixture(scope="class")
    def properties(self):
        return {
            "schema.yml": _PROPERTIES__SCHEMA_YML,
        }

    @pytest.fixture(scope="class")
    def seeds(self):
        return {"seed.csv": _SEEDS__SEED_INITIAL}

    def test_simple_copy_uppercase(self, project):

        # Load the seed file and check that it worked
        results = run_dbt(["seed"])
        assert len(results) == 1

        # Run the project and ensure that all the models loaded
        results = run_dbt()
        assert len(results) == 7

        check_relations_equal(
            project.adapter, ["seed", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"]
        )


from django import template
from django.core.exceptions import ObjectDoesNotExist
from goals.models import Activity

register = template.Library()

def date_activity(id):
    activity_list = []
    try:
        activities = Activity.objects.filter(date=id)
    except ObjectDoesNotExist:
        activities = None
    for activity in activities:
        activity_list.append(activity.description)
    return activity_list
register.filter('date_activity', date_activity)


import datetime
import select
import subprocess
import time
from sys import argv
import sys
import os
from dataclasses import dataclass
from typing import Dict, DefaultDict
from collections import defaultdict
import logging
import boto3
import kubernetes
from kubernetes.client.rest import ApiException

@dataclass
class ProcessOutput:
  stderr: str
  stdout: str
  other_fds: Dict[int, str]
  proc: subprocess.Popen

def stream_while_running(cwd, args, line_delimiter='', log_output=True):
  """Helper function for streaming a process' output whilst gathering it into a string"""
  proc = subprocess.Popen(cwd=cwd,
                          args=args, stderr=subprocess.PIPE,
                          stdout=subprocess.PIPE, text=True, bufsize=1)

  stderr = []
  stdout = []
  fds: DefaultDict[str] = defaultdict(list)
  while True:
    poll = proc.poll()
    if poll is not None:
      break
    else:
      # Poll the stdout/err streams every 0.05 seconds to see if anything has been written
      rdrs, _, _ = select.select([proc.stdout, proc.stderr], [], [], 0.05)

      # figure out which was returned
      if rdrs != []:
        for r in rdrs:
          # The FD might return multiple lines, so keep calling readline()
          # until we get a '' implying nothing more to consume
          keep_reading = True
          while keep_reading:
            line = r.readline()
            if len(line) > 0:
              if line[-1] == "\n":
                line = line[:-1]
              if r.name == proc.stdout.fileno():
                stdout.append(line)
                if log_output:
                  #logging.info(f"PID {str(proc.pid)} stdout: {line}")
                  logging.info(f"{line}")
              elif r.name == proc.stderr.fileno():
                stderr.append(line)
                if log_output:
                  #logging.info(f"PID {str(proc.pid)} stderr: {line}")
                  logging.info(f"{line}")
              else:
                fds[r.name].append(line)
                if log_output:
                  #logging.info(f"PID {str(proc.pid)} fd {r.name}: {line}")
                  logging.info(f"{line}")
            else:
              keep_reading = False
      else:
        # no new output
        continue

  # Flatten the list of list of strings that are the file descriptor outputs.
  # It makes it consistent with the subprocess output capture format
  for fd in fds:
    fds[fd] = line_delimiter.join(fds[fd])

  return ProcessOutput(line_delimiter.join(stderr), line_delimiter.join(stdout), fds, proc)

def slurp_ec2_instances(client):
  next_token = None
  instances = []
  keep_going = True
  while keep_going:
    if next_token is None:
      tmp = client.describe_instances()
    else:
      tmp = client.describe_instances(NextToken=next_token)
    for reservation in tmp['Reservations']:
      for instance in reservation['Instances']:
        instances.append(instance)
      if 'NextToken' in tmp and tmp['NextToken'] is not None:
        next_token = tmp['NextToken']
      else:
        keep_going = False
  return instances

def get_asgs_names(client, target):
  next_token = None
  asgs_names_list = []
  keep_going = True
  while keep_going:
    if next_token is None:
      tmp = client.describe_auto_scaling_groups()
    else:
      tmp = client.describe_auto_scaling_groups(NextToken=next_token)
    for asg in tmp['AutoScalingGroups']:
      if target == 'dbs':
        if 'db' in asg['AutoScalingGroupName']:
          asgs_names_list.append(asg['AutoScalingGroupName'])
      elif target == 'apps':
        if 'db' not in asg['AutoScalingGroupName']:
          asgs_names_list.append(asg['AutoScalingGroupName'])
      elif target == 'all':
        asgs_names_list.append(asg['AutoScalingGroupName'])
    if 'NextToken' in tmp and tmp['NextToken'] is not None:
      next_token = tmp['NextToken']
    else:
      keep_going = False
  return asgs_names_list

def main(region, target):

  logging.basicConfig(
    level=logging.INFO,
    format='%(levelname)s: %(asctime)s: %(message)s'
  )

  ec2_client = boto3.client('ec2', region_name=region)
  asg_client = boto3.client('autoscaling', region_name=region)

  namespace = 'default'
  snooze_seconds = 30
  pods_to_ignore = ['datadog-agent']
  errored = False

  # Get ASGS
  asgs_names_list = get_asgs_names(asg_client, target)

  # Get K8s nodes information
  kubernetes.config.load_kube_config()
  k8s_core_client = kubernetes.client.CoreV1Api()
  nodes = k8s_core_client.list_node()

  # Get EC2 instance tag information
  ec2_instances = slurp_ec2_instances(ec2_client)

  for k8s_node in nodes.items:
    instance_id = k8s_node.spec.provider_id.split('/')[-1]
    node_name = k8s_node.metadata.name
    matched_ec2_node = list(filter(lambda x: x['InstanceId'] == instance_id, ec2_instances))[0]
    asg_name = [x['Value'] for x in matched_ec2_node['Tags'] if x['Key'] == 'aws:autoscaling:groupName'][0]
    if asg_name not in asgs_names_list:
      continue

    print()
    logging.info(f"Dealing with instance: {instance_id}, in ASG: {asg_name}")

    logging.info(f"Will cordon instance: {instance_id}, with node_name: {node_name} so no new pods are scheduled on it")
    stream_while_running("/", ['kubectl', 'cordon', node_name])

    # Create a dict with node and pods details
    try:
      node_and_pods_dict = {}
      kubernetes.config.load_kube_config()
      k8s_core_client = kubernetes.client.CoreV1Api()
      field_selector = 'spec.nodeName=' + k8s_node.metadata.name
      ret = k8s_core_client.list_namespaced_pod(namespace, watch=False, field_selector=field_selector)
      node_and_pods_dict[instance_id] = {}
      node_and_pods_dict[instance_id]['node_name'] = k8s_node.metadata.name
      node_and_pods_dict[instance_id]['asg_name'] = asg_name
      node_and_pods_dict[instance_id]['pods'] = []

      for i in ret.items:
        if not i.metadata.name.startswith(tuple(pods_to_ignore)):
          node_and_pods_dict[instance_id]['pods'].append(i.metadata.name)
      node_and_pods_dict[instance_id]['number_of_pods'] = len(node_and_pods_dict[instance_id]['pods'])

    except ApiException as e:
      logging.error("Exception when calling Kubernetes CoreV1Api->list_namespaced_pod: %s\n" % e)

    logging.info(f"Instance and pod info: {node_and_pods_dict}")

    pod_controller_and_controller_kind_tuple_list = []

    # Create a list of tuples with controller name (derived from pod name) and controller kind
    for pod in node_and_pods_dict[instance_id]['pods']:
      try:
        kubernetes.config.load_kube_config()
        k8s_core_client = kubernetes.client.CoreV1Api()
        api_response = k8s_core_client.read_namespaced_pod(pod, namespace)
        controller_kind = (api_response.__dict__['_metadata'].__dict__['_owner_references'][0].__dict__['_kind'])
        pod_controller = pod.rpartition('-')[0]
        pod_controller_and_controller_kind_tuple = (pod_controller, controller_kind)
        pod_controller_and_controller_kind_tuple_list.append(pod_controller_and_controller_kind_tuple)
      except ApiException as e:
        logging.error("Exception when calling Kubernetes CoreV1Api->read_namespaced_pod: %s\n" % e)

    # Scale up ASG DesiredCapacity by 1, and also possibly increase MaxSize by 1 if DesiredCapacity + 1 > MaxSize
    logging.info(f"Dealing with ASG: {asg_name}")
    get_asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name])

    # Current ASG values
    current_desired_capacity = get_asg['AutoScalingGroups'][0]['DesiredCapacity']
    logging.info(f"Current ASG DesiredCapacity: {current_desired_capacity}")
    current_max_size = get_asg['AutoScalingGroups'][0]['MaxSize']
    logging.info(f"Current ASG MaxSize: {current_max_size}")

    # New ASG values
    new_desired_capacity = current_desired_capacity + 1
    logging.info(f"New ASG DesiredCapacity: {new_desired_capacity}")
    new_max_size = new_desired_capacity if new_desired_capacity > current_max_size else current_max_size
    logging.info(f"New ASG MaxSize: {new_max_size}")

    try:
      logging.info(f"Updating ASG: {asg_name} with new DesiredCapacity: {new_desired_capacity} and MaxSize: {new_max_size}")
      update_asg = asg_client.update_auto_scaling_group(AutoScalingGroupName=asg_name, DesiredCapacity=new_desired_capacity, MaxSize=new_max_size)

      # Wait for new instance to be InService
      logging.info("Waiting for new instance to be InService...")
      new_instance_not_up = True
      while new_instance_not_up:
        get_asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name])
        instances = get_asg['AutoScalingGroups'][0]['Instances']
        instances_lifecycle_states = [instance['LifecycleState'] for instance in instances]
        number_of_in_service_instances = instances_lifecycle_states.count("InService")
        if number_of_in_service_instances == new_desired_capacity:
          logging.info("New instance ready. Moving along...")
          new_instance_not_up = False
        else:
          logging.info(f"Still waiting for new instance to be InService. Will snooze for {snooze_seconds} seconds and try again...")
          time.sleep(snooze_seconds)

        # Set a 5 minute limit on the startup, in case something goes wrong
        start_time = datetime.datetime.now()
        if datetime.datetime.now() > (start_time + datetime.timedelta(minutes=5)):
          raise RuntimeError("Failed to get new node in 5 minutes. Bailing out!")

      # Drain the old node, and wait till it's fully drained
      logging.info(f"Draining old instance {instance_id}/{node_name} in ASG {asg_name}")
      stream_while_running("/", ['kubectl', 'drain', '--ignore-daemonsets', '--delete-local-data', node_name])
      all_pods_not_drained = True

      while all_pods_not_drained:
        try:
          node_and_pods_drained_node_dict = {}
          kubernetes.config.load_kube_config()
          k8s_core_client = kubernetes.client.CoreV1Api()
          field_selector = 'spec.nodeName=' + k8s_node.metadata.name
          ret = k8s_core_client.list_namespaced_pod(namespace, watch=False, field_selector=field_selector)
          node_and_pods_drained_node_dict[instance_id] = {}
          node_and_pods_drained_node_dict[instance_id]['node_name'] = k8s_node.metadata.name
          node_and_pods_drained_node_dict[instance_id]['asg_name'] = asg_name
          node_and_pods_drained_node_dict[instance_id]['pods'] = []

          for i in ret.items:
            if not i.metadata.name.startswith(tuple(pods_to_ignore)):
              node_and_pods_drained_node_dict[instance_id]['pods'].append(i.metadata.name)
          node_and_pods_drained_node_dict[instance_id]['number_of_pods'] = len(node_and_pods_drained_node_dict[instance_id]['pods'])

        except ApiException as e:
          logging.error("Exception when calling Kubernetes CoreV1Api->list_namespaced_pod: %s\n" % e)

        if node_and_pods_drained_node_dict[instance_id]['number_of_pods'] == 0:
          logging.info(f"All pods have been drained off instance {instance_id}. Moving along...")
          all_pods_not_drained = False
        else:
          logging.info(f"Still waiting for all pods to be drained off instance {instance_id}. {node_and_pods_drained_node_dict[instance_id]['number_of_pods']} pod(s) ({node_and_pods_drained_node_dict[instance_id]['pods']}) left. Will snooze for {snooze_seconds} seconds and try again...")
          time.sleep(snooze_seconds)

      # Wait and keep checking Kubernetes API to see if there are 100% replicas available for each ReplicaSet and StatefulSet
      logging.info(f"Will wait for all replicas to be available for each ReplicaSet/StatefulSet, while ignoring DaemonSet/Ad Hoc Pods ({pods_to_ignore})...")

      set_counter_list = []

      while len(pod_controller_and_controller_kind_tuple_list) > len(set_counter_list):
        for pod_controller, controller_kind in pod_controller_and_controller_kind_tuple_list:
          logging.info(f"Dealing with {pod_controller}, {controller_kind}")
          all_replicas_not_available = True
          while all_replicas_not_available:
            try:
              kubernetes.config.load_kube_config()
              k8s_app_client = kubernetes.client.AppsV1Api()
              if controller_kind == 'ReplicaSet':
                api_response = k8s_app_client.read_namespaced_replica_set(pod_controller, namespace)
              elif controller_kind == 'StatefulSet':
                api_response = k8s_app_client.read_namespaced_stateful_set(pod_controller, namespace)
              else:
                set_counter_list.append("UnknownController")
                continue
              number_of_replicas = api_response.__dict__['_status'].__dict__['_replicas']
              number_of_ready_replicas = api_response.__dict__['_status'].__dict__['_ready_replicas']
              if number_of_ready_replicas is None or number_of_ready_replicas < number_of_replicas:
                logging.info(f"{pod_controller} {controller_kind} does not have {number_of_replicas}/{number_of_replicas} replicas ready (only {number_of_ready_replicas or 0}/{number_of_replicas} ready). Will snooze for {snooze_seconds} seconds and try again...")
                time.sleep(snooze_seconds)
              elif number_of_ready_replicas >= number_of_replicas:
                logging.info(f"{pod_controller} {controller_kind} has {number_of_ready_replicas}/{number_of_replicas} replicas ready. Moving along...")
                all_replicas_not_available = False
                set_counter_list.append(pod_controller)
              else:
                logging.info(f"{pod_controller} {controller_kind} does not have {number_of_replicas}/{number_of_replicas} replicas ready (only {number_of_ready_replicas}/{number_of_replicas} ready). Will snooze for {snooze_seconds} seconds and try again...")
                time.sleep(snooze_seconds)
            except ApiException as e:
              logging.error("Exception when calling Kubernetes AppsV1Api: %s\n" % e)

      logging.info("100% replicas available for each ReplicaSet/StatefulSet. Moving along...")

      # Terminate the old node
      logging.info("Terminating old instance...")
      terminate_instance = ec2_client.terminate_instances(InstanceIds=[instance_id])

      # Scale ASG DesiredCapacity and MaxSize back down to original values so we are consistent with Terraform
      logging.info(f"Reverting ASG: {asg_name} with original DesiredCapacity: {current_desired_capacity} and MaxSize: {current_max_size}")
      update_asg = asg_client.update_auto_scaling_group(AutoScalingGroupName=asg_name, DesiredCapacity=current_desired_capacity, MaxSize=current_max_size)

    except (KeyboardInterrupt, TypeError) as e:
      errored = True
      logging.error(e)
      logging.error(f"Signal/TypeError caught! Will revert ASG: {asg_name} back to original DesiredCapacity: {current_desired_capacity} and MaxSize: {current_max_size} before exiting!")
      stream_while_running("/", ['kubectl', 'uncordon', node_name])
      update_asg = asg_client.update_auto_scaling_group(AutoScalingGroupName=asg_name, DesiredCapacity=current_desired_capacity, MaxSize=current_max_size)
      logging.error(f"ASG reverted back to original values. Instance {instance_id} has been uncordoned. Check state of Kubernetes and AWS before running the script again. Exiting!")
      try:
        sys.exit(0)
      except SystemExit:
        os._exit(0)

    except:
      errored = True
      logging.error("Unexpected error:", sys.exc_info()[0])
      logging.error(f"Will revert ASG: {asg_name} back to original DesiredCapacity: {current_desired_capacity} and MaxSize: {current_max_size} before exiting!")
      stream_while_running("/", ['kubectl', 'uncordon', node_name])
      update_asg = asg_client.update_auto_scaling_group(AutoScalingGroupName=asg_name, DesiredCapacity=current_desired_capacity, MaxSize=current_max_size)
      logging.error(f"ASG reverted back to original values. Instance {instance_id} has been uncordoned. Check state of Kubernetes and AWS before running the script again. Exiting!")
      try:
        sys.exit(0)
      except SystemExit:
        os._exit(0)
      raise

    try:
      if errored:
        logging.error(f"An error occurred above, and the ASG: {asg_name} has already been reverted to original DesiredCapacity: {current_desired_capacity} and MaxSize: {current_max_size}. Exiting!")
      else:
        # Wait for number of instances in ASG to scale back down
        logging.info("Waiting for number of instances in ASG to scale back down...")
        asg_instances_not_scaled_down = True
        while asg_instances_not_scaled_down:
          get_asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name])
          instances = get_asg['AutoScalingGroups'][0]['Instances']
          instances_lifecycle_states = [instance['LifecycleState'] for instance in instances]
          number_of_in_service_instances = instances_lifecycle_states.count("InService")
          if number_of_in_service_instances == current_desired_capacity:
            logging.info(f"Number of instances in ASG has scaled back down to original DesiredCapacity: {current_desired_capacity}")
            print("--------------------------------------------------------------------------------------------")
            print("--------------------------------------------------------------------------------------------")
            asg_instances_not_scaled_down = False
          else:
            logging.info(f"Still waiting for number of instances in ASG to scale back down. Will snooze for {snooze_seconds} seconds and try again...")
            time.sleep(snooze_seconds)
    except Exception as e:
      logging.error(e)
      logging.error(f"An error occurred above, and the infrastructure is possibily in an unstable state now. Check state of Kubernetes and AWS before running the script again. Exiting!")

if __name__ == '__main__':
  region = argv[1]
  target = argv[2]
  main(region, target)


import sys
import os
import argparse
import torch
from itertools import product
from torch import nn
from torch.nn import functional as F
from torch.autograd import Variable
from nltk.translate.bleu_score import SmoothingFunction, corpus_bleu
from verb_preprocess import load_data
from arguments import s2s_inter as parseParams
from time import time

class model(nn.Module):
  def __init__(self,args):
    super().__init__()
    self.args = args

    # encoder decoder stuff
    self.encemb = nn.Embedding(args.svsz,args.hsz,padding_idx=0)
    self.enc = nn.LSTM(args.hsz,args.hsz//2,bidirectional=True,num_layers=args.layers,batch_first=True)
    self.decemb = nn.Embedding(args.vsz,args.hsz,padding_idx=0)
    self.dec = nn.LSTM(args.hsz*2,args.hsz,num_layers=args.layers,batch_first=True)
    self.gen = nn.Linear(args.hsz,args.vsz)

    self.vemb = nn.Embedding(args.verbvsz,args.hsz,padding_idx=0)
    self.vdec = nn.LSTM(args.hsz,args.hsz,num_layers=args.layers,batch_first=True)
    self.venc = nn.LSTM(args.hsz,args.hsz,num_layers=args.layers,bidirectional=True,batch_first=True)
    self.vgen = nn.Linear(args.hsz,args.verbvsz)
    self.vlinin = nn.Linear(args.hsz,args.hsz,bias=False)
    self.vlinout = nn.Linear(args.hsz*2,args.hsz,bias=False)

    # attention stuff
    self.linin = nn.Linear(args.hsz,args.hsz,bias=False)
    self.dvlinin = nn.Linear(args.hsz,args.hsz,bias=False)
    self.sm = nn.Softmax()
    self.linout = nn.Linear(args.hsz*2,args.hsz,bias=False)
    self.tanh = nn.Tanh()
    self.drop = nn.Dropout(args.drop)
    self.pretrain = False
    self.beamsize = args.beamsize

  def forward(self,inp,out=None,verbs=None):
    encenc = self.encemb(inp)
    enc,(he,ce) = self.enc(encenc)

    #enc hidden has bidirection so switch those to the features dim
    h = torch.cat([he[0:he.size(0):2], he[1:he.size(0):2]], 2)
    c = torch.cat([ce[0:ce.size(0):2], ce[1:ce.size(0):2]], 2)
    hx = torch.cat([he[0:he.size(0):2], he[1:he.size(0):2]], 2)
    cx = torch.cat([ce[0:ce.size(0):2], ce[1:ce.size(0):2]], 2)

    #decode verbs
    #op = Variable(torch.cuda.FloatTensor(inp.size(0),self.args.hsz).zero_())
    voutputs = []
    vencoding = []

    if verbs is None:
      outp = self.args.vmaxlen
    else:
      outp = verbs.size(1)

    for i in range(outp):
      if i == 0:
        prev = Variable(torch.cuda.LongTensor(inp.size(0),1).fill_(3))
      else:
        if verbs is None:
          prev = self.vgen(op).max(2)
          prev = prev[1]
        else:
          prev = verbs[:,i-1].unsqueeze(1)
        #op = op.squeeze(1)

      decin = self.vemb(prev)
      #dembedding = self.vemb(prev)
      #decin = torch.cat((dembedding.squeeze(1),op),1).unsqueeze(1)
      decout, (h,c) = self.vdec(decin,(h,c))
      #vencoding.append(decout)

      #attend on enc
      q = self.vlinin(decout.squeeze(1)).unsqueeze(2)
      w = torch.bmm(enc,q).squeeze(2)
      w = self.sm(w)
      cc = torch.bmm(w.unsqueeze(1),enc)
      op = torch.cat((cc,decout),2)
      op = self.drop(self.tanh(self.vlinout(op)))
      voutputs.append(self.vgen(op))

    voutputs = torch.cat(voutputs,1)
    if self.pretrain:
      return voutputs

    _, vidxs = torch.max(voutputs,2)
    vembedding = self.vemb(vidxs)
    vencoding, _ = self.vdec(vembedding,(hx,cx))
    h = hx
    c = cx

    '''
    vmask = torch.cuda.ByteTensor(verbs.size(0),verbs.size(1),args.hsz).zero_()
    for i in range(verbs.size(0)):
      for j in range(verbs.size(1)):
        if verbs[i,j].data[0]==0:
          vmask[i,j,:].fill_(1)
    vencoding = torch.cat(vencoding,1)
    vencoding.data.masked_fill_(vmask,0)
    '''

    #decode text
    op = Variable(torch.cuda.FloatTensor(inp.size(0),self.args.hsz).zero_())
    outputs = []

    if out is None:
      outp = self.args.maxlen
    else:
      outp = out.size(1)

    for i in range(outp):
      if i == 0:
        prev = Variable(torch.cuda.LongTensor(inp.size(0),1).fill_(3))
      else:
        if out is None:
          prev = self.gen(op).max(2)
          prev = prev[1]
        else:
          prev = out[:,i-1].unsqueeze(1)
        op = op.squeeze(1)

      dembedding = self.decemb(prev)
      decin = torch.cat((dembedding.squeeze(1),op),1).unsqueeze(1)
      decout, (h,c) = self.dec(decin,(h,c))

      '''
      #attend on enc
      q = self.linin(decout.squeeze(1)).unsqueeze(2)
      w = torch.bmm(enc,q).squeeze(2)
      w = self.sm(w)
      cc = torch.bmm(w.unsqueeze(1),enc)
      '''

      #attend on vencoding
      vq = self.dvlinin(decout.squeeze(1)).unsqueeze(2)
      vw = torch.bmm(vencoding,vq).squeeze(2)
      vw = self.sm(vw)
      vcc = torch.bmm(vw.unsqueeze(1),vencoding)

      #op = torch.cat((cc,decout,vcc),2)
      op = torch.cat((decout,vcc),2)
      op = self.drop(self.tanh(self.linout(op)))
      outputs.append(self.gen(op))

    outputs = torch.cat(outputs,1)
    return outputs

  def beamsearch(self,inp):
    inp = inp.unsqueeze(0)
    encenc = self.encemb(inp)
    enc,(he,ce) = self.enc(encenc)

    #enc hidden has bidirection so switch those to the features dim
    hx = torch.cat([he[0:he.size(0):2], he[1:he.size(0):2]], 2)
    cx = torch.cat([ce[0:ce.size(0):2], ce[1:ce.size(0):2]], 2)
    h = [torch.cat([he[0:he.size(0):2], he[1:he.size(0):2]], 2) for i in range(self.beamsize)]
    c = [torch.cat([ce[0:ce.size(0):2], ce[1:ce.size(0):2]], 2) for i in range(self.beamsize)]
    encbeam = [enc for i in range(self.beamsize)]

    #decode verbs
    #op = Variable(torch.cuda.FloatTensor(inp.size(0),self.args.hsz).zero_())
    op = Variable(torch.FloatTensor(inp.size(0),self.args.hsz).zero_())
    voutputs = []
    vencoding = []

    outp = self.args.vmaxlen

    for i in range(outp):
      if i == 0:
        #prev = Variable(torch.cuda.LongTensor(inp.size(0),1).fill_(3))
        prev = Variable(torch.LongTensor(inp.size(0),1).fill_(3))
      else:
        prev = self.vgen(op).max(2)
        prev = prev[1]
        if prev[0,0].data[0]==1:
          break
        #op = op.squeeze(1)

      #dembedding = self.vemb(prev)
      #decin = torch.cat((dembedding.squeeze(1),op),1).unsqueeze(1)
      decin = self.vemb(prev)
      decout, (hx,cx) = self.vdec(decin,(hx,cx))
      vencoding.append(decout)

      #attend on enc
      q = self.vlinin(decout.squeeze(1)).unsqueeze(2)
      w = torch.bmm(enc,q).squeeze(2)
      w = self.sm(w)
      cc = torch.bmm(w.unsqueeze(1),enc)
      op = torch.cat((cc,decout),2)
      op = self.drop(self.tanh(self.vlinout(op)))
      voutputs.append(self.vgen(op))

    #vencoding_ = torch.cat(vencoding,1)
    #vencoding = [vencoding_ for i in range(self.beamsize)]
    voutputs = torch.cat(voutputs,1)
    _, vidxs = torch.max(voutputs,2)
    verbs = vidxs.data
    vembedding = self.vemb(vidxs)
    vencoding_, _ = self.vdec(vembedding)
    vencoding = [vencoding_ for _ in range(self.beamsize)]

    ops = [Variable(torch.FloatTensor(1,1,self.args.hsz).zero_()) for i in range(self.beamsize)]
    prev = [Variable(torch.LongTensor(1,1).fill_(3)) for i in range(self.beamsize)]
    #ops = [Variable(torch.cuda.FloatTensor(1,1,self.args.hsz).zero_()) for i in range(self.beamsize)]
    #prev = [Variable(torch.cuda.LongTensor(1,1).fill_(3)) for i in range(self.beamsize)]
    beam = [[] for x in range(self.beamsize)]
    scores = [0]*self.beamsize
    sents = [0]*self.beamsize
    done = []
    donescores = []
    doneattns = []
    attns = [[] for x in range(self.beamsize)]
    for i in range(self.args.maxlen):
      tmp = []
      for j in range(len(beam)):
        dembedding = self.decemb(prev[j])
        decin = torch.cat((dembedding.squeeze(1),ops[j].squeeze(1)),1).unsqueeze(1)
        decout, (hx,cx) = self.dec(decin,(h[j],c[j]))

        '''
        #attend on enc
        q = self.linin(decout.squeeze(1)).unsqueeze(2)
        #q = decout.view(decout.size(0),decout.size(2),decout.size(1))
        w = torch.bmm(enc,q).squeeze(2)
        w = self.sm(w)
        cc = torch.bmm(w.unsqueeze(1),enc)
        '''

        #attend on vencoding
        vq = self.dvlinin(decout.squeeze(1)).unsqueeze(2)
        vw = torch.bmm(vencoding[j],vq).squeeze(2)
        vw = self.sm(vw)
        attns[j].append(vw)
        vcc = torch.bmm(vw.unsqueeze(1),vencoding[j])

        op = torch.cat((decout,vcc),2)
        #op = torch.cat((cc,decout,vcc),2)
        op = self.drop(self.tanh(self.linout(op)))

        op2 = self.gen(op)
        op2 = op2.squeeze()
        probs = F.log_softmax(op2)
        vals, pidx = probs.topk(self.beamsize*2,0)
        vals = vals.squeeze()
        pidx = pidx.squeeze()
        for k in range(self.beamsize):
          tmp.append((vals[k].data[0]+scores[j],pidx[k],j,hx,cx,op))
      tmp.sort(key=lambda x: x[0],reverse=True)
      newbeam = []
      newscore = []
      newsents = []
      newh = []
      newc = []
      newops = []
      newprev = []
      newattns = []
      added = 0
      j = 0
      while added < len(beam):
        v,pidx,beamidx,hx,cx,op = tmp[j]
        pdat = pidx.data[0]
        new = beam[beamidx]+[pdat]
        if pdat in self.punct:
          newsents.append(sents[beamidx]+1)
        else:
          newsents.append(sents[beamidx])
        if pdat == self.endtok or newsents[-1]>4:
          if new not in done:
            done.append(new)
            donescores.append(v)
            doneattns.append(attns[beamidx])
            added += 1
        else:
          if new not in newbeam:
            newbeam.append(new)
            newscore.append(v)
            newh.append(hx)
            newc.append(cx)
            newops.append(op)
            newprev.append(pidx)
            newattns.append(attns[beamidx])
            added += 1
        j+=1
      beam = newbeam
      prev = newprev
      scores = newscore
      sents = newsents
      h = newh
      c = newc
      ops = newops
      attns = newattns
    if len(done)==0:
      done.extend(beam)
      donescores.extend(scores)
      doneattns = attns
    donescores = [x/len(done[i]) for i,x in enumerate(donescores)]
    topscore =  donescores.index(max(donescores))
    return done[topscore], doneattns[topscore], verbs

def validate(M,DS,args):
  data = DS.val_batches
  cc = SmoothingFunction()
  M.eval()
  M = M.cpu()
  refs = []
  hyps = []
  i=0
  print(time())
  for x in data:
    if args.debug:
      sources, targets, verbs = DS.pad_batch(x,targ=False)
    else:
      sources, targets = DS.pad_batch(x,targ=False,v=False)
    sources = Variable(sources.cpu(),volatile=True)
    M.zero_grad()
    logits = []
    attns = []
    inter = []
    for s in sources:
      l,a,v = M.beamsearch(s)
      logits.append(l)
      attns.append(a)
      inter.append(v)
    hyp = [x[:x.index(1)] if 1 in x else x for x in logits]
    hyp = [[DS.vocab[x] for x in y] for y in hyp]
    for b in inter:
      ihyp = [[DS.verb_vocab[x] for x in list(y)] for y in b]
      print(ihyp)
    print(hyp)
    print(targets)
    hyps.extend(hyp)
    refs.extend(targets)
  bleu = corpus_bleu(refs,hyps,emulate_multibleu=True,smoothing_function=cc.method3)
  M = M.cuda()
  M.train()
  with open(args.savestr+"hyps"+args.epoch,'w') as f:
    hyps = [' '.join(x) for x in hyps]
    f.write('\n'.join(hyps))
  try:
    os.stat(args.savestr+"refs")
  except:
    with open(args.savestr+"refs",'w') as f:
      refstr = []
      for r in refs:
        r = [' '.join(x) for x in r]
        refstr.append('\n'.join(r))
      f.write('\n'.join(refstr))
  return bleu

def train(M,DS,args,optimizer):
  data = DS.train_batches
  weights = torch.cuda.FloatTensor(args.vsz).fill_(1)
  weights[0] = 0
  criterion = nn.CrossEntropyLoss(weights)
  trainloss = []
  for x in data:
    sources, targets, verbs = DS.pad_batch(x)
    sources = Variable(sources.cuda())
    targets = Variable(targets.cuda())
    verbs = Variable(verbs,requires_grad=False)
    M.zero_grad()
    logits= M(sources,targets,verbs)
    logits = logits.view(-1,logits.size(2))
    targets = targets.view(-1)

    loss = criterion(logits, targets)
    loss.backward()
    trainloss.append(loss.data.cpu()[0])
    optimizer.step()

    if len(trainloss)%100==99: print(trainloss[-1])
  return sum(trainloss)/len(trainloss)

def pretrain(M,DS,args,optimizer):
  M.pretrain=True
  data = DS.train_batches
  weights2 = torch.cuda.FloatTensor(args.verbvsz).fill_(1)
  weights2[0] = 0
  criterion2 = nn.CrossEntropyLoss(weights2)
  trainloss = []
  for x in data:
    sources, targets,verbs = DS.pad_batch(x)
    sources = Variable(sources)
    targets = Variable(targets)
    verbs = Variable(verbs)
    M.zero_grad()
    vlogits = M(sources,out=None,verbs=verbs)
    vlogits = vlogits.view(-1,vlogits.size(2))
    verbs = verbs.view(-1)
    loss = criterion2(vlogits,verbs)
    loss.backward()
    trainloss.append(loss.data.cpu()[0])
    optimizer.step()

    if len(trainloss)%100==99:
      print(trainloss[-1])
  M.pretrain=False
  return sum(trainloss)/len(trainloss)

def main(args):
  DS = torch.load(args.datafile)
  if args.debug:
    DS.train_batches= DS.train_batches[:2]
    DS.val_batches= DS.train_batches[:2]
    args.epochs = 100
  args.vsz = DS.vsz
  args.svsz = DS.svsz
  args.verbvsz = DS.vvsz
  if args.resume:
    M,optimizer = torch.load(args.resume)
    M.enc.flatten_parameters()
    M.dec.flatten_parameters()
    e = args.resume.split("/")[-1] if "/" in args.resume else args.resume
    e = e.split('_')[0]
    e = int(e)+1
  else:
    M = model(args).cuda()
    optimizer = torch.optim.Adam(M.parameters(), lr=args.lr)
    e=0
  M.endtok = DS.vocab.index("<eos>")
  M.punct = [DS.vocab.index(t) for t in ['.','!','?']]
  print(M)
  print(args.datafile)
  print(args.savestr)
  for epoch in range(args.pretrainepochs):
    trainloss = pretrain(M,DS,args,optimizer)
    print("train loss epoch",epoch,trainloss)
  for epoch in range(e,args.epochs):
    args.epoch = str(epoch)
    trainloss = train(M,DS,args,optimizer)
    print("train loss epoch",epoch,trainloss)
    if args.debug:
      if epoch%2==1:
        b = validate(M,DS,args)
        print("valid bleu ",b)
    else:
      b = validate(M,DS,args)
      print("valid bleu ",b)
      torch.save((M,optimizer),args.savestr+args.epoch+"_bleu-"+str(b))

if __name__=="__main__":
  args = parseParams()
  main(args)


from __future__ import unicode_literals
from __future__ import absolute_import

import os
import sys
import logging

log = logging.getLogger(__name__)

__version__ = '0.1.0'

def _(msg):
    """
    Temporal but valid translation function.
    """
    log.warning('_({}) called before configuring l10n'.format(msg))
    return msg

def find_localedir(localedir):
    """
    Custom function to find the locales message catalog in a multi-platform
    application.
    """
    import sys
    import os

    if localedir is None:
        if sys.platform.startswith('linux'):
            import gettext
            return gettext.bindtextdomain('messages')

        return find_localedir('locale')

    if os.path.isabs(localedir):
        return localedir

    def parent(path):
        return os.path.normpath(
            os.path.dirname(os.path.abspath(os.path.realpath(path)))
        )

    if sys.platform.startswith('win') and \
       getattr(sys, 'frozen', '') in ('dll', 'console_exe', 'windows_exe'):
        root = parent(sys.executable)
    else:
        import inspect
        root = parent(inspect.getfile(inspect.stack()[-1][0]))

    return os.path.join(root, localedir)

def l10n(domain=None, localedir=None):
    """
    Setup localization.

    :param str domain: The name of the translation domain (your application
     name). If None is provided (the default) the default domain is used as
     determined by gettext.textdomain() which usually is `messages`.

    :param str localedir: Path to the location of the binary `.mo` messages.

     Lookup path will then be `[prefix]/[locale]/LC_MESSAGES/[domain].mo`.

     The prefix will be determined as follow:

     1. If `localedir` is an absolute path as determined by
        :py:func:`os.path.isabs` then use the path as prefix for the lookup.

     2. If `localedir` is a relative path then determine the prefix as follow:

        1. If on Windows and in a freezed environment:

           `dirname([sys.executable])` + prefix

        2. Anything else:

           `dirname([calling_script])` + prefix

           Where `calling_script` is determined by inspecting the stack and
           retrieving the location of the calling script.

     3. If `localedir` is `None` (the default), then use the default system
        locale directory.

        1. On Linux:

           `[sys.prefix]/share/locale`

        2. Anything else, like Windows, which doesn't have a default folder,
           same as calling with `localedir` as a relative path `locale`.
    """
    import locale
    import traceback

    # Set LANG environment variable if unset
    # This is required by the gettext module to work on Windows
    os_windows = sys.platform.startswith('win')
    if os_windows:
        if os.getenv('LANG') is None:
            lang, enc = locale.getdefaultlocale()
            os.environ['LANG'] = lang

    # Set locale
    try:
        current = locale.setlocale(locale.LC_ALL, '')
        log.info('Locale set to {}'.format(current))
    except Exception:
        log.error('Unable to set user locale')
        log.error(traceback.format_exc())
        return _

    # Determine text domain and localedir
    import gettext

    if domain is None:
        domain = gettext.textdomain()

    localedir = find_localedir(localedir)

    # List modules to call
    modules = []
    if os_windows:
        try:
            import ctypes
            if not hasattr(l10n, 'libintl'):
                l10n.libintl = ctypes.cdll.LoadLibrary('intl.dll')
            modules.append(l10n.libintl)
        except Exception:
            log.error('Unable to load libintl')
            log.error(traceback.format_exc())
    modules.append(locale)
    modules.append(gettext)

    for module in modules:
        try:
            if hasattr(module, 'bindtextdomain'):
                module.bindtextdomain(domain, localedir)
            # Not sure about this call and encoding
            # Also, should we set the encoding here or just use lgettext
            # if hasattr(module, 'bind_textdomain_codeset'):
            #     module.bind_textdomain_codeset(domain, 'UTF-8')
            if hasattr(module, 'textdomain'):
                module.textdomain(domain)
        except Exception:
            log.error('Unable to bind text domain in module {}'.format(module))
            log.error(traceback.format_exc())

    global _
    _ = gettext.gettext

    return gettext.gettext

__all__ = ['_', 'l10n']


from core.abstract.viewsets import AbstractViewSet
from core.comment.serializers import CommentSerializer
from core.comment.models import Comment

from rest_framework.permissions import IsAuthenticated
from rest_framework.exceptions import NotFound
from rest_framework.response import Response
from rest_framework import status
from core.permission.permission import AuthorOrReadOnly

class CommentViewSet(AbstractViewSet):
    http_method_names = ("get", "post", "put", "delete")
    permission_classes = (AuthorOrReadOnly,)
    serializer_class = CommentSerializer

    def get_queryset(self):
        post_pk = self.kwargs["post_pk"]
        if post_pk is None:
            raise NotFound(detail="Not found.")
        queryset = Comment.objects.filter(post__public_id=post_pk)

        return queryset

    def get_object(self):
        instance = Comment.objects.get_object_by_public_id(self.kwargs["pk"])

        self.check_object_permissions(self.request, instance)

        return instance

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        self.perform_create(serializer)

        return Response(serializer.data, status=status.HTTP_201_CREATED)


import cv2
import numpy as np
import operator
import os

MIN_CONTOUR_AREA = 100
RESIZED_IMAGE_WIDTH = 20
RESIZED_IMAGE_HEIGHT = 30

class ContourWithData:
    npaContour = None
    boundingRect = None
    intRectX = 0
    intRectY = 0
    intRectWidth = 0
    intRectHeight = 0
    fltArea = 0.0

    def calculateRectTopLeftPointAndWidthAndHeight(self):
        [intX, intY, intWidth, intHeight] = self.boundingRect
        self.intRectX = intX
        self.intRectY = intY
        self.intRectWidth = intWidth
        self.intRectHeight = intHeight

    def checkIfContourIsValid(self):
        if self.fltArea < MIN_CONTOUR_AREA: return False
        return True

def main():
    allContoursWithData = []
    validContoursWithData = []

    try:
        npaClassifications = np.loadtxt("classifications.txt", np.float32)
    except:
        print("Gagal membuka file Classifications.txt, Close Program\n")
        os.system("pause")
        return

    try:
        npaFlattenedImages = np.loadtxt("flattened_images.txt", np.float32)
    except:
        print("Gagal membuka file Flattened_Images.txt, Close Program\n")
        os.system("pause")
        return

    npaClassifications = npaClassifications.reshape((npaClassifications.size, 1))

    kNearest = cv2.ml.KNearest_create()

    kNearest.train(npaFlattenedImages, cv2.ml.ROW_SAMPLE, npaClassifications)

    ImageTesting = cv2.imread("Data_Training/test_5.jpg")

    if ImageTesting is None:
        print("Gagal mengimport file citra \n\n")
        os.system("pause")
        return

    ImageGray = cv2.cvtColor(ImageTesting, cv2.COLOR_BGR2GRAY)
    ImageBlurred = cv2.GaussianBlur(ImageGray, (5, 5), 0)

    ImageThresh = cv2.adaptiveThreshold(ImageBlurred, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 11, 2)

    ImageThreshCopy = ImageThresh.copy()

    npaContours, npaHierarchy = cv2.findContours(ImageThreshCopy,
                                                 cv2.RETR_EXTERNAL,
                                                 cv2.CHAIN_APPROX_SIMPLE)

    for npaContour in npaContours:
        contourWithData = ContourWithData()
        contourWithData.npaContour = npaContour
        contourWithData.boundingRect = cv2.boundingRect(contourWithData.npaContour)
        contourWithData.calculateRectTopLeftPointAndWidthAndHeight()
        contourWithData.fltArea = cv2.contourArea(contourWithData.npaContour)
        allContoursWithData.append(contourWithData)

    for contourWithData in allContoursWithData:
        if contourWithData.checkIfContourIsValid():
            validContoursWithData.append(contourWithData)

    validContoursWithData.sort(key=operator.attrgetter("intRectX"))

    strFinalString = ""

    for contourWithData in validContoursWithData:
        cv2.rectangle(ImageTesting, (contourWithData.intRectX, contourWithData.intRectY), (contourWithData.intRectX + contourWithData.intRectWidth,
                      contourWithData.intRectY + contourWithData.intRectHeight), (0, 255, 0), 2)

        imgROI = ImageThresh[contourWithData.intRectY: contourWithData.intRectY + contourWithData.intRectHeight, contourWithData.intRectX:
                 contourWithData.intRectX + contourWithData.intRectWidth]

        imgROIResized = cv2.resize(imgROI, (RESIZED_IMAGE_WIDTH, RESIZED_IMAGE_HEIGHT))

        npaROIResized = imgROIResized.reshape((1, RESIZED_IMAGE_WIDTH * RESIZED_IMAGE_HEIGHT))

        npaROIResized = np.float32(npaROIResized)

        retval, npaResults, neigh_resp, dists = kNearest.findNearest(npaROIResized, k=1)

        strCurrentChar = str(chr(int(npaResults[0][0])))

        strFinalString = strFinalString + strCurrentChar

    print("\n" + "Karakter yang di Kenali Adalah : " + strFinalString + "\n")

    cv2.imshow("Citra Test", ImageTesting)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

    return

if __name__ == "__main__":
    main()


import sqlalchemy as sa
from sqlalchemy.orm import exc

from solum.objects import parameter as abstract
from solum.objects.sqlalchemy import models as sql

class Parameter(sql.Base, abstract.Parameter):
    """Represent a parameter in sqlalchemy."""

    __resource__ = 'parameters'
    __tablename__ = 'parameter'
    __table_args__ = sql.table_args()

    id = sa.Column(sa.Integer, primary_key=True, autoincrement=True)
    plan_id = sa.Column(sa.Integer, sa.ForeignKey('plan.id'),
                        nullable=False)
    user_defined_params = sa.Column(sql.YAMLEncodedDict(65535))
    sys_defined_params = sa.Column(sql.YAMLEncodedDict(65535))

    @classmethod
    def get_by_plan_id(cls, context, p_id):
        try:
            session = sql.SolumBase.get_session()
            return session.query(cls).filter_by(plan_id=p_id).one()
        except exc.NoResultFound:
            return None

class ParameterList(abstract.ParameterList):
    """Represent a list of parameters in sqlalchemy."""

    @classmethod
    def get_all(cls, context):
        return ParameterList(sql.model_query(context, Parameter))


import requests as req
import pprint as pp
import pandas as pd
import sys, json, pymongo
from pymongo import MongoClient
import os
import csv

##Connecting to Database
conn = 'mongodb://localhost:27017'
client = pymongo.MongoClient(conn)
# Declare the database
db = client.Donor_Data
# Declare the collection
collection = db.projects

# print(collection.count())

results = db.projects.find()
for result in results:
    print(result)

url = "https://api.donorschoose.org/common/json_feed.html?APIKey=DONORSCHOOSE&newSince=1483256682&max=1000"

# Who was the director of the movie Aliens?
data = req.get(url).json()
post_id = collection.insert_many(data['proposals'])
data

# In[58]:

cursor = collection.find({},
    {'_id':0,'id': 1, 'title': 1, 'percentFunded': 1, 'numDonors': 1,'costToComplete':1,'numStudents':1,
     'totalPrice':1,'teacherId':1,'schoolTypes.id':1,'schoolName':1,'city':1,'zip':1,
     'state':1,'latitude':1,'longitude':1,'expirationDate':1,'fundingStatus':1,'gradeLevel.name':1,'povertyType.label':1,
     'subject.name':1})

flattened_records = []
for data_rec in cursor:
   #print(data_rec['gradeLevel']['name'])
    flattened_record = {
            'id': data_rec['id'],
            'title': data_rec['title'],
            'percentFunded' : data_rec['percentFunded'] if 'percentFunded' in data_rec.keys() else '',
            'numDonors': data_rec['numDonors'] if 'numDonors' in data_rec.keys() else '',
            'costToComplete' : data_rec['costToComplete'] if 'costToComplete' in data_rec.keys() else '',
            'numStudents': data_rec['numStudents'] if 'numStudents' in data_rec.keys() else '',
            'totalPrice' : data_rec['totalPrice'] if 'totalPrice' in data_rec.keys() else '',
            'schoolName': data_rec['schoolName'] if 'schoolName' in data_rec.keys() else '',
            'zip' : data_rec['zip'] if 'zip' in data_rec.keys() else '',
            'state': data_rec['state'] if 'state' in data_rec.keys() else '',
            'latitude' : data_rec['latitude'] if 'latitude' in data_rec.keys() else '',
            'longitude': data_rec['longitude'] if 'longitude' in data_rec.keys() else '',
            'expirationDate' : data_rec['expirationDate'] if 'expirationDate' in data_rec.keys() else '',
            'fundingStatus': data_rec['fundingStatus'] if 'fundingStatus' in data_rec.keys() else '',
            'gradelevel':data_rec['gradeLevel']['name'] if 'gradeLevel' in data_rec.keys() else '',
            'povertyType':data_rec['povertyType']['label'] if 'povertyType' in data_rec.keys() else '',
            'subject':data_rec['subject']['name'] if 'subject' in data_rec.keys() else ''

        }
    flattened_records.append(flattened_record)

#print(flattened_records)
# # Step 2: Iterate through the list of flattened records and write them to the csv file
with open('stack_039.csv', 'w') as outfile:
    fields = ['_id', 'id', 'title', 'percentFunded','numDonors','costToComplete','numStudents','totalPrice' ,'schoolName','zip','state'
        ,'latitude' ,'longitude','expirationDate' ,'fundingStatus','gradelevel','povertyType','subject']
    write = csv.DictWriter(outfile, fieldnames=fields)
    write.writeheader()
    for flattened_record in flattened_records:
        write.writerow(flattened_record)

# In[59]:

# print("id,title")
# cursor = collection.find()

# while (cursor.hasNext()) {
#     jsonObject = cursor.next();
#     print(jsonObject._id.valueOf() + "," + jsonObject.id + ",\"" + jsonObject.title +"\"")
# }


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('profile', '0024_market'),
    ]

    operations = [
        migrations.CreateModel(
            name='Follow',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('company', models.ForeignKey(to='profile.Company')),
            ],
        ),
        migrations.AddField(
            model_name='posting',
            name='started',
            field=models.BooleanField(default=False),
        ),
        migrations.AddField(
            model_name='follow',
            name='posting',
            field=models.ForeignKey(to='profile.Posting'),
        ),
        migrations.AddField(
            model_name='follow',
            name='student',
            field=models.ForeignKey(to='profile.Student'),
        ),
    ]


## rtaudio testing
import time
import rtaudio as rt
import struct
import numpy as np
import matplotlib.pyplot as pl
pl.ion()

class Player():
    def __init__(self):
        output_device = 'Apple Inc.: Built-in Output'
        output_device_idx = None

        dac = rt.RtAudio()
        n_devices = dac.getDeviceCount()
        for i in range(n_devices):
            info = dac.getDeviceInfo(i)
            #print(info)
            if info['name'] == output_device:
                output_device_idx = i

        self.buffer_size = 256

        oParams = {'deviceId': output_device_idx, 'nChannels': 2, 'firstChannel': 0}
        dac.openStream(oParams, {}, 44100, self.buffer_size, self.callback)
        dac.startStream()

        self._idx = 0

        t0 = time.time()
        while True:
            if time.time() - t0 > 2:
                break

        try:
            dac.stopStream()
        except:
            pass
        if dac.isStreamOpen():
            dac.closeStream()

    def callback(self, out_buffer, *args):
        for i in range(self.buffer_size):
            samp = np.sin(2*np.pi*440*self._idx/44100)
            struct.pack_into('f', out_buffer, 4*i, samp)
            self._idx += 1

class Recorder():
    def __init__(self):
        input_device = 'Apple Inc.: Built-in Microphone'

        dac = rt.RtAudio()
        input_device_idx = None
        n_devices = dac.getDeviceCount()
        for i in range(n_devices):
            info = dac.getDeviceInfo(i)
            #print(info)
            if info['name'] == input_device:
                input_device_idx = i
        print(input_device_idx)

        self.buffer_size = 512

        oParams = {'deviceId': input_device_idx, 'nChannels': 1, 'firstChannel': 0}
        dac.openStream({}, oParams, 48000, self.buffer_size, self.callback)
        dac.startStream()

        self.data = []

        t0 = time.time()
        while True:
            if time.time() - t0 > 2:
                break

        try:
            dac.stopStream()
        except:
            pass
        if dac.isStreamOpen():
            dac.closeStream()

    def callback(self, rw_buffer, r_buffer):
        #rw = rw_buffer[:]
        #r = r_buffer[:]

        n_frames = int(len(r_buffer) / 4)
        r = struct.unpack(n_frames*'f', r_buffer)
        self.data.append(r)

if __name__ == '__main__':
    #r = Recorder()
    #d = np.array(r.data, dtype=np.float32)
    #np.save('test', d.ravel())

    p = Player()
##


#!/usr/bin/env python

import argparse
import json
import os
import pathlib
import tqdm

def load_val_ids(path):
    with open(path, 'r') as fin:
        paths = [line.strip() for line in fin.readlines()]
    ids = [int(line.split('_')[-1].split('.')[0]) for line in paths]
    return ids

def split_annotations(annotation_dir, val_ids):
    with open(annotation_dir / 'instances_train2017.json', 'r') as fin:
        train_data = json.load(fin)
    with open(annotation_dir / 'instances_val2017.json', 'r') as fin:
        val_data = json.load(fin)
    annotations = train_data['annotations'] + val_data['annotations']
    images = train_data['images'] + val_data['images']

    train_annotations = []
    val_annotations = []
    for annotation in tqdm.tqdm(annotations):
        if annotation['image_id'] in val_ids:
            val_annotations.append(annotation)
        else:
            train_annotations.append(annotation)

    train_images = []
    val_images = []
    for image in tqdm.tqdm(images):
        if image['id'] in val_ids:
            val_images.append(image)
        else:
            train_images.append(image)

    train_data = {
        'images': train_images,
        'annotations': train_annotations,
        'categories': train_data['categories'],
    }
    val_data = {
        'images': val_images,
        'annotations': val_annotations,
        'categories': val_data['categories'],
    }
    return train_data, val_data

def split_image_paths(coco2017_dir, val_ids):
    train_paths = sorted((coco2017_dir / 'train2017').glob('*'))
    val_paths = sorted((coco2017_dir / 'val2017').glob('*'))
    paths = train_paths + val_paths

    train_paths = []
    val_paths = []
    for path in tqdm.tqdm(paths):
        image_id = int(path.name.split('.')[0])
        if image_id in val_ids:
            val_paths.append(path)
        else:
            train_paths.append(path)
    return train_paths, val_paths

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--val_list', type=str, required=True)
    parser.add_argument('--coco2017_dir', type=str, required=True)
    parser.add_argument('--outdir', type=str, required=True)
    args = parser.parse_args()

    val_list_path = pathlib.Path(args.val_list)
    val_ids = load_val_ids(val_list_path)

    outdir = pathlib.Path(args.outdir)
    outdir.mkdir(exist_ok=True, parents=True)

    coco2017_dir = pathlib.Path(args.coco2017_dir)
    train_data, val_data = split_annotations(coco2017_dir / 'annotations',
                                             val_ids)
    train_paths, val_paths = split_image_paths(coco2017_dir, val_ids)

    annotation_dir = outdir / 'annotations'
    annotation_dir.mkdir(exist_ok=True, parents=True)
    with open(annotation_dir / 'instances_train2014.json', 'w') as fout:
        json.dump(train_data, fout)
    with open(annotation_dir / 'instances_val2014.json', 'w') as fout:
        json.dump(val_data, fout)

    train_image_dir = outdir / 'train2014'
    val_image_dir = outdir / 'val2014'
    train_image_dir.mkdir(exist_ok=True, parents=True)
    val_image_dir.mkdir(exist_ok=True, parents=True)
    for path in train_paths:
        os.symlink(path, train_image_dir / path.name)
    for path in val_paths:
        os.symlink(path, val_image_dir / path.name)

if __name__ == '__main__':
    main()


import scapy.all as scapy
from scapy.layers import http # import module for filtering http requests
import argparse

def get_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument("-i","--iface",dest="interface",help="Specify interface to sniff data")
    value = parser.parse_args()
    if not value.interface:
        print("[-]Specify an interface")
    else:
        return value

#this method sniffs data on given interface , prn is for callback function and store = False indicates not to store in memory
def sniff(interface):
    scapy.sniff(iface=interface,store=False,prn=process_sniffed)

#this method captures the http requests "host" is the domain and "path" is the resource or web page
def get_url(packet):
    return packet[http.HTTPRequest].Host + packet[http.HTTPRequest].Path

#this method captures post requests/login information
def get_credentials(packet):
    if packet.haslayer(scapy.Raw):
        load = packet[scapy.Raw].load
        keywords=["username","login","email","user","password","pass"]
        for keyword in keywords:
            if keyword in load:
                return load

#this method is a call back function for scapy.sniff and it processes the packet captured
def process_sniffed(packet):
    if packet.haslayer(http.HTTPRequest):
        url = get_url(packet)
        print("[+]Captured HTTP Request :"+url)

    credentials =get_credentials(packet)
    if credentials:
        print("\n[+]Captured Credentials :"+credentials+"\n")

value = get_arguments()
sniff(value.interface)


import numpy as np
from PIL import Image
from sklearn import svm
import os
import pickle

import load_pickle_database
from numpy import int

[dataset_read,label_read] = load_pickle_database.load_images_labels("/Users/yiliu/Box Sync/Stanford/Classes/2017-2018 Fall/CS 229 Machine Learning/Project/CAPTCHA Database/Simple single letter dataset/50000_single_letter.p")

#Split train set and test set
nTrainPrcntg = 0.7
nTrain = int(nTrainPrcntg*dataset_read.shape[0])
trainData = dataset_read[0:nTrain,:]
trainLabel = label_read[0:nTrain,]
testData = dataset_read[nTrain:,:]
testLabel = label_read[nTrain:,]

#Train model
multiClassSVM = svm.LinearSVC(loss = 'squared_hinge', penalty = 'l2',C=1/5000 ,max_iter=8000,verbose=1) #define kernel? default should be rbf-kernel
multiClassSVM.fit(trainData,trainLabel)
saveFile = "/Users/yiliu/Box Sync/Stanford/Classes/2017-2018 Fall/CS 229 Machine Learning/Project/CAPTCHA Database/Simple single letter dataset/SVM_Model.p"
pickle.dump(multiClassSVM,open(os.path.join(saveFile),'wb')) #Save model
print("Model Saved")
# print(trainData.shape)

#Load prediction dataset
loadPath = "/Users/yiliu/Box Sync/Stanford/Classes/2017-2018 Fall/CS 229 Machine Learning/Project/CAPTCHA Database/Simple single letter dataset/prediction_data.p"
[predData,predLabel] = pickle.load(open(loadPath, 'rb'))

#Training accuracy
confidenceTrain = multiClassSVM.decision_function(trainData)
predictionTrain = np.argmax(confidenceTrain, axis=1)
print(trainLabel.shape)
comparisonTrain = np.equal(predictionTrain,trainLabel)
accuracyTrain = sum(comparisonTrain)/trainLabel.size
print('Train accuracy:')
print(accuracyTrain)

#Test accuracy
confidenceTest = multiClassSVM.decision_function(testData)
predictionTest = np.argmax(confidenceTest, axis=1)
comparisonTest = np.equal(predictionTest,testLabel)
accuracyTest = sum(comparisonTest)/testLabel.size
print('Test accuracy:')
print(accuracyTest)

#Make predictions
confidencePred = multiClassSVM.decision_function(predData)
predictionPred = np.argmax(confidencePred, axis=1)

#Prediction accuracy
temp = 0
iteration = 0
for x in range(0,10):
    iteration = iteration + 1
    if predictionPred[x] == ord(predLabel[x,])-65:
        temp = temp+1
print('Correct number:')
print(temp)

accuracyPred = temp/predLabel.size
print('Prediction accuracy:')
print(accuracyPred)


import cv2
import numpy as np

def nothing(*arg):
    pass

if __name__ == '__main__':
    # cap = cv2.VideoCapture(0)
    cap = cv2.VideoCapture("C:/Users/xiaoi/Pictures/Camera Roll/001.mp4")
    cap.set(3, 3840)
    cap.set(4, 2160)
    image = cv2.imread("image/WIN_20191107_11_18_12_Pro.jpg", 0)
    image = cv2.resize(image, (960, 540))

    # 膨胀
    r = 1
    max_r = 100
    cv2.namedWindow("dilate", 1)
    cv2.createTrackbar("r", "dilate", r, max_r, nothing)
    while 1:
        ok, frame = cap.read()
        image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        image = cv2.resize(image, (960, 540))
        d = image
        r = cv2.getTrackbarPos("r", "dilate")
        s = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * r + 1, 2 * r + 1))

        # d = cv2.erode(d, s)  # 腐蚀
        # d = cv2.dilate(d, s)  # 膨胀
        #
        # # 开运算
        # d = cv2.erode(d, s)
        # d = cv2.dilate(d, s)
        #
        # # # 闭运算
        # d = cv2.dilate(d, s)
        # d = cv2.erode(d, s)

        # # 顶帽运算
        # d = cv2.erode(d, s)
        # d = cv2.dilate(d, s)
        # d = image-d

        # 底帽运算
        # d = cv2.dilate(d, s)
        # d = cv2.erode(d, s)
        # d = image - d

        # 形态学边缘检测（形态学梯度）
        d = cv2.erode(d, s)
        d = cv2.dilate(image, s) - d

        cv2.imshow("dilate", d)
        if cv2.waitKey(5) == 27:
            break
    cv2.destroyAllWindows()


from __future__ import annotations

import difflib
import typing as t

from ..exceptions import BadRequest
from ..exceptions import HTTPException
from ..utils import cached_property
from ..utils import redirect

if t.TYPE_CHECKING:
    from _typeshed.wsgi import WSGIEnvironment
    from .map import MapAdapter
    from .rules import Rule
    from ..wrappers.request import Request
    from ..wrappers.response import Response

class RoutingException(Exception):
    """Special exceptions that require the application to redirect, notifying
    about missing urls, etc.

    :internal:
    """

class RequestRedirect(HTTPException, RoutingException):
    """Raise if the map requests a redirect. This is for example the case if
    `strict_slashes` are activated and an url that requires a trailing slash.

    The attribute `new_url` contains the absolute destination url.
    """

    code = 308

    def __init__(self, new_url: str) -> None:
        super().__init__(new_url)
        self.new_url = new_url

    def get_response(
        self,
        environ: WSGIEnvironment | Request | None = None,
        scope: dict | None = None,
    ) -> Response:
        return redirect(self.new_url, self.code)

class RequestPath(RoutingException):
    """Internal exception."""

    __slots__ = ("path_info",)

    def __init__(self, path_info: str) -> None:
        super().__init__()
        self.path_info = path_info

class RequestAliasRedirect(RoutingException):  # noqa: B903
    """This rule is an alias and wants to redirect to the canonical URL."""

    def __init__(self, matched_values: t.Mapping[str, t.Any], endpoint: str) -> None:
        super().__init__()
        self.matched_values = matched_values
        self.endpoint = endpoint

class BuildError(RoutingException, LookupError):
    """Raised if the build system cannot find a URL for an endpoint with the
    values provided.
    """

    def __init__(
        self,
        endpoint: str,
        values: t.Mapping[str, t.Any],
        method: str | None,
        adapter: MapAdapter | None = None,
    ) -> None:
        super().__init__(endpoint, values, method)
        self.endpoint = endpoint
        self.values = values
        self.method = method
        self.adapter = adapter

    @cached_property
    def suggested(self) -> Rule | None:
        return self.closest_rule(self.adapter)

    def closest_rule(self, adapter: MapAdapter | None) -> Rule | None:
        def _score_rule(rule: Rule) -> float:
            return sum(
                [
                    0.98
                    * difflib.SequenceMatcher(
                        None, rule.endpoint, self.endpoint
                    ).ratio(),
                    0.01 * bool(set(self.values or ()).issubset(rule.arguments)),
                    0.01 * bool(rule.methods and self.method in rule.methods),
                ]
            )

        if adapter and adapter.map._rules:
            return max(adapter.map._rules, key=_score_rule)

        return None

    def __str__(self) -> str:
        message = [f"Could not build url for endpoint {self.endpoint!r}"]
        if self.method:
            message.append(f" ({self.method!r})")
        if self.values:
            message.append(f" with values {sorted(self.values)!r}")
        message.append(".")
        if self.suggested:
            if self.endpoint == self.suggested.endpoint:
                if (
                    self.method
                    and self.suggested.methods is not None
                    and self.method not in self.suggested.methods
                ):
                    message.append(
                        " Did you mean to use methods"
                        f" {sorted(self.suggested.methods)!r}?"
                    )
                missing_values = self.suggested.arguments.union(
                    set(self.suggested.defaults or ())
                ) - set(self.values.keys())
                if missing_values:
                    message.append(
                        f" Did you forget to specify values {sorted(missing_values)!r}?"
                    )
            else:
                message.append(f" Did you mean {self.suggested.endpoint!r} instead?")
        return "".join(message)

class WebsocketMismatch(BadRequest):
    """The only matched rule is either a WebSocket and the request is
    HTTP, or the rule is HTTP and the request is a WebSocket.
    """

class NoMatch(Exception):
    __slots__ = ("have_match_for", "websocket_mismatch")

    def __init__(self, have_match_for: set[str], websocket_mismatch: bool) -> None:
        self.have_match_for = have_match_for
        self.websocket_mismatch = websocket_mismatch


from multiprocessing import Pool, cpu_count
import time
import json
import os
import os.path
import sys
import shutil
import glob
import subprocess
import collections.abc

import yaml
import tomlkit
import xlwt

from gopest.common import config
from gopest.common import runtime

def nested_dict_update(d, u):
    """ update a nested dict object with an update dict
    https://stackoverflow.com/a/3233356/2368167
    """
    for k, v in u.items():
        if isinstance(v, collections.abc.Mapping):
            d[k] = nested_dict_update(d.get(k, {}), v)
        else:
            d[k] = v
    return d

def unix2dos(fname):
    import sys
    with open(fname, 'rb') as infile:
        instr = infile.read()
    outstr = instr.replace( b"\r\n", b"\n" ).replace( b"\r", b"\n" ).replace( b"\n", b"\r\n" )
    if len(outstr) == len(instr):
        return
    with open(fname, 'wb') as outfile:
        outfile.write( outstr )

def fixpcf_noptmax(fpst, noptmax=0):
    """ update PEST case file (.pst) with NOPTMAX, number of optimisation
    iterations
    """
    fpst_bk = fpst + '.backup'
    if not os.path.isfile(fpst):
        print('Error: %s does not exist.' % fpst)
        exit(1)
    if os.path.isfile(fpst_bk):
        os.remove(fpst_bk)
    os.rename(fpst, fpst_bk)

    try:
        with open(fpst_bk,'r') as fin, open(fpst, 'w') as fout:
            linesep = None # use current file's line ending
            isec = 7 # start with 7, so that it will never be 7 again
            for line in fin:
                # detect current file's line ending first
                if linesep is None:
                    if line[-2] == '\r' and line[-1] == '\n':
                        linesep = '\r\n'
                    elif line[-1] == '\n':
                        linesep = '\n'
                # find correct line, replace
                isec += 1
                oline = line
                if line.strip().startswith('*') and 'control data' in line:
                    isec = 0
                if isec == 7:
                    # replace first item in line (NOPTMAX)
                    sps = line.split()
                    oline = ' ' + ' '.join([str(noptmax)] + sps[1:] + [linesep])
                fout.write(oline)
    except Exception as e:
        fin.close()
        fout.close()
        print('fixpcf_noptmax() failed to proceed, restoring...')
        os.remove(fpst)
        os.rename(fpst_bk, fpst)
        raise(e)

def read_rec(rec_file):
    values = {}
    with open(rec_file, 'r') as f:
        reading = False
        for line in f:
            if reading:
                if empty_line:
                    empty_line = False
                    continue
                if not line.strip():
                    continue
                if '----->' in line:
                    break
                sps = line.strip().split('=')
                v = float(sps[-1])
                s = sps[0].strip()
                if s == 'Sum of squared weighted residuals (ie phi)':
                    k = 'phi'
                elif s == 'Optimised measurement objective function':
                    k = 'phi measured'
                elif s == 'Optimised regularisation objective function':
                    k =  'phi regularisation'
                elif '"' in s:
                    k = s.split('"')[1]
                else:
                    k, v = None, None
                if k is not None:
                    values[k] = v
            if line.startswith('Objective function ----->'):
                reading, empty_line = True, True
    return values

class message(object):
    """ A Waiwera log (.yaml) message object """
    def __init__(self, msg):
        self._msg = msg
        self.level, self.source, self.event, self.data = msg

def check_sim_ends(spath):
    """ extract summary of waiwera simulation from yaml log """
    def waiwera_ends(fyaml):
        """ return elapsed time in seconds of waiwera simulation time, None if not
        successful """
        try:
            with open(fyaml) as f:
                data = yaml.load(f, Loader=yaml.CSafeLoader)
                r = {''}
                if data[-3][:3] == ['info', 'timestep', 'end']:
                    time = data[-3][3]['time']
                    size = data[-3][3]['size']
                if data[-1][:3] == ['info', 'simulation', 'destroy']:
                    elapsed = data[-1][3]['elapsed_seconds']
                return {'endtime': time, 'stepsize': size, 'elapsed': elapsed}
        except:
            return {}
    cwd = os.getcwd()
    os.chdir(spath)
    print('Working in %s...' % spath)

    results = {'run': {}}
    sequence = config['model']['sequence']
    input_typ = config['simulator']['input-type']
    flsts = runtime['filename']['lst_seq']
    for i,seq in enumerate(sequence):
        results['run'][seq] = {}
        if input_typ== 'waiwera':
            fyaml = flsts[i].replace('.h5', '.yaml')
            results['run'][seq] = waiwera_ends(fyaml)
        else:
            raise NotImplementedError()

    os.chdir(cwd)
    # print('Restored directory %s' % cwd)
    return results

def check_run_status(spath):
    """ extract summary of waiwera simulation from yaml log """
    cwd = os.getcwd()
    os.chdir(spath)
    print('Working in %s...' % spath)

    results = {'run': {}}
    sequence = config['model']['sequence']
    input_typ = config['simulator']['input-type']
    flsts = runtime['filename']['lst_seq']
    for i,seq in enumerate(sequence):
        results['run'][seq] = {}
        results['run'][seq]['exists'] = os.path.exists(flsts[i])

    os.chdir(cwd)
    # print('Restored directory %s' % cwd)
    return results

def get_obj_fn(spath):
    """ Return PEST calculated objective function values by modifying config and
    PEST case file to run a dummy run.  Then read results from case .rec file.

    This is more destructive, but I have decided to leave those modification
    in place after running the command for a couple of reasons:
    1. (lazy) no need to handle undo the mod if exception is raised
    2. (safe) it's probably safer to leave as modified here, reduce possibility
       for user to wipe out existing model output files, which can be costly to
       run

    Unfortunately I need to modify the config.toml file, because gopest run-
    pest-model is involked by PEST, not this script directly.
    """
    cwd = os.getcwd()
    os.chdir(spath)
    print('Working in %s...' % spath)

    # only works if the final real model output file exists
    flst = runtime['filename']['lst_seq'][-1]
    if not os.path.exists(flst):
        print('Real model output file %s is missing, cannot get obj. fn.' % flst)
        os.chdir(cwd)
        return {}

    # modify config to skip model run
    with open('goPESTconfig.toml', 'r') as f:
        cfg = tomlkit.load(f)
    cfg['model']['skip'] = True
    with open('goPESTconfig.toml', 'w') as f:
        tomlkit.dump(cfg, f)

    # modify NOPTMAX in control data .pst file
    fpst = config['pest']['case-name'] + '.pst'
    fixpcf_noptmax(fpst, 0)

    cmd = [
        os.path.join(config['pest']['dir'], config['pest']['executable']),
        config['pest']['case-name'],
        ]
    print('Running: ', cmd)
    results = {}
    subprocess.call(cmd)
    results['obj-fn'] = read_rec('case_reg.rec')

    os.chdir(cwd)
    # print('Restored directory %s' % cwd)
    return results

def export_xls(data):
    phis, regs, others = [], [], []
    for sln,sl in data.items():
        if 'obj-fn' in sl:
            for n in sorted(sl['obj-fn'].keys()):
                if n.startswith('phi'):
                    phis.append(n)
                elif n.startswith('regul_'):
                    regs.append(n)
                else:
                    others.append(n)
            phis, regs, others = sorted(phis), sorted(regs), sorted(others)
            break
    runcs = []
    for sln,sl in data.items():
        if 'run' in sl:
            for rn,r in sl['run'].items():
                runcs += [rn+'.'+c for c in sorted(r.keys())]
            break
    cols = ['slave'] + phis + others + regs + runcs

    wb = xlwt.Workbook()
    ws = wb.add_sheet('slaves')

    i = 0
    for j,c in enumerate(cols):
        ws.write(i, j, c)

    for sln,sl in data.items():
        if 'obj-fn' in sl:
            i += 1
            j = 0
            ws.write(i, j, sln)
            for p in (phis + others + regs):
                j += 1
                if p in sl['obj-fn']:
                    ws.write(i, j, sl['obj-fn'][p])
            for r in runcs:
                j += 1
                rn,sn = r.split('.')
                if 'run' in sl and rn in sl['run'] and sn in sl['run'][rn]:
                    ws.write(i, j, sl['run'][rn][sn])
    wb.save('goPESTslaves.xls')

def init_slave(cfg, rt):
    """ used in multiprocessing Pool to initialise threads with the the updated
    gopest.common.config and gopest.common.runtime
    """
    print('Updating local thread common.config and common.runtime.')
    global config, runtime
    config = cfg
    runtime = rt

hlp = '''
Usage: gopest check-slaves [--help] [--status] [--end-time] [--obj-fn]
                           [--dir path_to_slaves] [--pest-exe pest_executable]
                           [--export-xls]

The check-slaves command searches through slave directories and obtain/collect
their running status etc.  By default, the pest.slave_dirs property from
goPESTconfig.toml is used. It is possible to specify a different directory by
using argument "--dir".

Extracted information will be dumped into a JSON file "goPESTslaves.json".  Note
that the command will only update/append results into the JSON file if exists.
It tries not to destroy whatever that is already in the file.

"--status" is the fastest option, which only checks if model output files exist
in the lsave directories.  (ie. did the runs produce any output files at all)

"--end-time" enables extraction of simulation end time (from output YAML file if
running waiwera as simulator).

"--obj-fn" is more destructive.  Within each slave directory, it will run PEST
(with the NOPTMAX set to 0) once with the internal settings set to skip actual
model runs.  This essentially runs PEST so that observations and objective
function will be extracted from model outputs within the slave directory.

"--pest-exe" is useful with the "--obj-fn" option when user have the slaves
directories in a different environment than where it was originally run.
pest_executable here can include the path to the executable if it's not already
in the system's PATH.

"--export-xls" is used to export contents of goPESTslaves.json into spreadsheet
file goPESTslaves.xls.  This command can be used alone as it will load existing
goPESTslaves.json.

This command runs locally, and does not utilise any slurm/srun/queue facilities.

This message can be printed with the "--help" argument.  It is possible to
include more than one option at a time.
'''

def check_slaves_cli(argv=[]):
    if '--help' in argv or len(argv) <= 1:
        print(hlp)
        exit(0)
    else:
        # overwrite slave_dirs
        if '--dir' in argv:
            iarg = argv.index('--dir') + 1
            try:
                spath = argv[iarg]
            except IndexError:
                raise Exception('--dir argument needs to be followed by path_to_slaves')
        else:
            spath = config['pest']['slave_dirs']
        # overwrite PEST exe
        if "--pest-exe" in argv:
            iarg = argv.index('--pest-exe') + 1
            try:
                pexe = argv[iarg]
            except IndexError:
                raise Exception('--pest-exe argument needs to be followed by pest_executable')
            # modify the shared module variable
            config['pest']['dir'] = ''
            config['pest']['executable'] = pexe
        else:
            # do nothing, keeps original setting
            pass
        # tasks
        nopts, tasks = 0, []
        if "--status" in argv:
            nopts += 1
            tasks.append('status')
        if "--end-time" in argv:
            nopts += 1
            tasks.append('end-time')
        if "--obj-fn" in argv:
            nopts += 1
            tasks.append('obj-fn')
        xls = False
        if "--export-xls" in argv:
            nopts += 1
            xls = True
        if nopts == 0:
            print(hlp)
            print('Please specify at least one task to perform.')
            exit(0)

    start_time = time.time()
    if not all([os.path.exists(spath), os.path.isdir(spath)]):
        raise Exception('Specified path_to_slaves needs to be a valid directory: %s' % spath)

    task_fn = {
        'end-time': check_sim_ends,
        'status': check_run_status,
        'obj-fn': get_obj_fn,
    }

    slave_paths = sorted([p for p in glob.glob(os.path.join(spath, '*')) if os.path.isdir(p)])
    slave_names = [os.path.basename(sp) for sp in slave_paths]

    ncpu = max(1, cpu_count() - 1)
    print('Starting %i workers to process %i slaves' % (ncpu, len(slave_paths)))
    pool = Pool(ncpu, initializer=init_slave, initargs=(config, runtime))

    fout = 'goPESTslaves.json'
    if os.path.exists(fout):
        with open(fout, 'r') as f:
            data = json.load(f)
    else:
        data = {}

    for task in tasks:
        print('Checking %s...' % task)
        results = pool.map(task_fn[task], slave_paths)
        data = nested_dict_update(data, dict(zip(slave_names, results)))

    with open(fout, 'w') as f:
        json.dump(data, f, indent=4, sort_keys=True)

    if xls:
        export_xls(data)

    print('Finished after %f seconds' % (time.time() - start_time))

if __name__ == '__main__':
    check_slaves_cli(sys.argv)