#!/usr/bin/env python3

import argparse
import itertools
import os
import re

import ninja

def _parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('input_file', help='input ninja file')
    parser.add_argument('--encoding', default='utf-8',
                        help='ninja file encoding')
    parser.add_argument('--ninja-deps', help='.ninja_deps file')

    return parser.parse_args()

def main():
    args = _parse_args()

    system_out_pattern = re.compile('out/target/product/[^/]+/system/')
    vendor_src_pattern = re.compile('(?:vendor)|(?:device)/')

    manifest = ninja.Parser().parse(
        args.input_file, args.encoding, args.ninja_deps)

    # Build lookup map
    outs = {}
    for build in manifest.builds:
        for path in build.explicit_outs:
            outs[path] = build
        for path in build.implicit_outs:
            outs[path] = build

    # Compute transitive input files
    outs_from_vendor_cache = {}

    def _are_inputs_from_vendor(build):
        # Check whether the input files are matched by vendor_src_pattern first.
        gen_paths = []
        paths = itertools.chain(
            build.explicit_ins, build.implicit_ins, build.depfile_implicit_ins)
        for path in paths:
            if vendor_src_pattern.match(path):
                return True
            if path.startswith('out/'):
                gen_paths.append(path)

        # Check whether the input files transitively depend on
        # vendor_src_pattern.
        for path in gen_paths:
            if is_from_vendor(path):
                return True

        return False

    def is_from_vendor(out_path):
        matched = outs_from_vendor_cache.get(out_path, None)
        if matched is not None:
            return matched

        build = outs.get(out_path)
        if build:
            matched = _are_inputs_from_vendor(build)
        else:
            matched = bool(vendor_src_pattern.match(path))
        outs_from_vendor_cache[out_path] = matched
        return matched

    bad_paths = [
        path for path in outs
        if is_from_vendor(path) and system_out_pattern.match(path)]

    bad_paths.sort()

    for path in bad_paths:
        print(path)

if __name__ == '__main__':
    main()


import re
from urllib.request import Request, urlopen
import pandas as pd
from bs4 import BeautifulSoup

def get_data(url,queryParams):
    pattern = re.compile(('<([ㄱ-ㅣ가-힣]+)>'))

    request_url = Request(url + queryParams)
    data_dic = {}
    request_url.get_method = lambda: 'GET'
    response_body = urlopen(request_url).read()
    soup = BeautifulSoup(response_body, 'lxml-xml')
    header_code = soup.find('resultCode').get_text()
    item_contents = soup.findAll('item')
    print(soup.find('header'))
    for content in item_contents:
        for i in range(len(content)):
            data = content.contents[i]
            tag = re.match(pattern,str(data)).group()
            replace_tag = re.sub(pattern,'\\1',tag)
            if replace_tag not in data_dic.keys():
                data_dic[replace_tag] = [data.get_text().strip()]
            else:
                data_dic[replace_tag] += [data.get_text().strip()]
    if len(data_dic) != 0 and header_code == '00':
        df = pd.DataFrame.from_dict(data_dic,orient='index')
    else:
        df = header_code
    return df


"""

Generic Jinja2 extension for all components

Jinja2 has extensions. Though our components act like Jinja2 extensions,
in that they have different tag names, they are really all the same
thing, extension wise.

Thus, we have one kind of extension which knows how to handle all
registered "components" and dispatch correctly.

"""
import dataclasses
from dataclasses import MISSING
from typing import Set

from jinja2 import nodes
from jinja2.ext import Extension

TMP_COMPONENT_DICT_NAME = '__component'
COMPONENT_DICT_NAME = 'component'

TEMPLATE_FIELD_NAME = 'template'
CHILDREN_FIELD_NAME = 'children'

CHILDREN_MACRO_NAME = 'children'

class ComponentExtension(Extension):
    tags: Set

    def parse(self, parser):
        # Get the component for the tag name that we matched on
        tag_name = parser.stream.current[2]
        component_class = self.environment.components[tag_name]
        field_names = [f.name for f in dataclasses.fields(component_class)]
        has_children = CHILDREN_FIELD_NAME in field_names

        lineno = next(parser.stream).lineno

        node = nodes.Scope(lineno=lineno)

        # list of `Pair` nodes for tag properties to update "component" dictionary
        component_dict_update_items = []

        while parser.stream.current.type != 'block_end':
            lineno = parser.stream.current.lineno
            if component_dict_update_items:
                parser.stream.expect('comma')
            name = parser.stream.expect('name')
            parser.stream.expect('assign')
            value = parser.parse_expression()
            component_dict_update_items.append(nodes.Pair(nodes.Const(name.value), value))

        # dictionary initialization in the "component" name
        prepare_component_dict = [self._initialize_component_dict(component_class, lineno)]

        if component_dict_update_items:
            component_dict_delta = nodes.Dict(component_dict_update_items)
            # `Getattr` for "update" function of the dictionary "component"
            update_component_dict_fun = nodes.Getattr(nodes.Name(TMP_COMPONENT_DICT_NAME, 'load'), 'update', 'load')
            # `Call` for `component.update(<prop name>, <prop value>)`
            call_component_dict_update = nodes.Call(update_component_dict_fun, [component_dict_delta], [], None, None)
            prepare_component_dict.append(nodes.ExprStmt(call_component_dict_update))

        # assign `component = __component` and `__component = None`
        prepare_component_dict.extend([
            nodes.Assign(nodes.Name(COMPONENT_DICT_NAME, 'store', lineno=lineno),
                         nodes.Name(TMP_COMPONENT_DICT_NAME, 'load', lineno=lineno),
                         lineno=lineno),
            nodes.Assign(nodes.Name(TMP_COMPONENT_DICT_NAME, 'store', lineno=lineno),
                         nodes.Const(None, lineno=lineno),
                         lineno=lineno)
        ])

        if has_children:
            inner_block = list(parser.parse_statements(
                ('name:end' + tag_name,),
                drop_needle=True)
            )
            # create children() macro
            children_macro = nodes.Macro()
            children_macro.name = CHILDREN_MACRO_NAME
            children_macro.args = []
            children_macro.defaults = []
            children_macro.body = inner_block
            children_macro_nodes = [children_macro]
        else:
            children_macro_nodes = []

        # include tag template
        include_tag = nodes.Include()
        # use `template` item of the "component" dictionary for template path
        include_tag.template = nodes.Getitem(nodes.Name(COMPONENT_DICT_NAME, 'load', lineno=lineno),
                                             nodes.Const(TEMPLATE_FIELD_NAME, lineno=lineno), 'load', lineno=lineno)
        include_tag.ignore_missing = False
        include_tag.with_context = True

        node.body = prepare_component_dict + children_macro_nodes + [include_tag, ]

        return node

    @staticmethod
    def _initialize_component_dict(component_class, lineno):
        items = []

        def pair_node_for_field(name, value):
            return nodes.Pair(nodes.Const(name, lineno=lineno), nodes.Const(value, lineno=lineno), lineno=lineno)

        for f in dataclasses.fields(component_class):
            if f.default is not MISSING:
                items.append(pair_node_for_field(f.name, f.default))
            elif f.default_factory is not MISSING:
                # TODO here we could use `Call` node as the assigning expression
                items.append(pair_node_for_field(f.name, f.default_factory()))

        component_dict = nodes.Dict(items, lineno=lineno)

        # `Assign` dictionary to the "component" name
        return nodes.Assign(nodes.Name(TMP_COMPONENT_DICT_NAME, 'store', lineno=lineno), component_dict, lineno=lineno)


# -*- coding: utf-8 -*-

# Copyright 2016,  Digital Reasoning
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import json
import logging

from django.conf import settings
from django.contrib.contenttypes.fields import GenericRelation
from django.core.files.base import ContentFile
from django.core.files.storage import FileSystemStorage
from django.db import models
from django_extensions.db.models import TimeStampedModel, TitleSlugDescriptionModel

from stackdio.core.fields import DeletingFileField

PROTOCOL_CHOICES = [
    ('tcp', 'TCP'),
    ('udp', 'UDP'),
    ('icmp', 'ICMP'),
    ('-1', 'all'),
]

DEVICE_ID_CHOICES = [
    ('/dev/sdb', '/dev/sdb'),
    ('/dev/sdc', '/dev/sdc'),
    ('/dev/sdd', '/dev/sdd'),
    ('/dev/sde', '/dev/sde'),
    ('/dev/sdf', '/dev/sdf'),
    ('/dev/sdg', '/dev/sdg'),
    ('/dev/sdh', '/dev/sdh'),
    ('/dev/sdi', '/dev/sdi'),
    ('/dev/sdj', '/dev/sdj'),
    ('/dev/sdk', '/dev/sdk'),
    ('/dev/sdl', '/dev/sdl'),
]

logger = logging.getLogger(__name__)

def get_props_file_path(obj, filename):
    return 'blueprints/{0}-{1}.props'.format(obj.pk, obj.slug)

_blueprint_model_permissions = (
    'create',
    'admin',
)

_blueprint_object_permissions = (
    'view',
    'update',
    'delete',
    'admin',
)

class Blueprint(TimeStampedModel, TitleSlugDescriptionModel):
    """
    Blueprints are a template of reusable configuration used to launch
    Stacks. The purpose to create a blueprint that encapsulates the
    functionality, software, etc you want in your infrastructure once
    and use it to repeatably create your infrastructure when needed.

    TODO: @params
    """
    model_permissions = _blueprint_model_permissions
    object_permissions = _blueprint_object_permissions
    searchable_fields = ('title', 'description')

    class Meta:
        ordering = ('title',)
        default_permissions = tuple(set(_blueprint_model_permissions +
                                        _blueprint_object_permissions))

    labels = GenericRelation('core.Label')

    formula_versions = GenericRelation('formulas.FormulaVersion')

    create_users = models.BooleanField('Create SSH Users')

    # storage for properties file
    props_file = DeletingFileField(
        max_length=255,
        upload_to=get_props_file_path,
        null=True,
        blank=True,
        default=None,
        storage=FileSystemStorage(location=settings.FILE_STORAGE_DIRECTORY))

    def __unicode__(self):
        return u'{0} (id={1})'.format(self.title, self.id)

    @property
    def host_definition_count(self):
        return self.host_definitions.count()

    @property
    def properties(self):
        if not self.props_file:
            return {}
        with open(self.props_file.path) as f:
            return json.loads(f.read())

    @properties.setter
    def properties(self, props):
        props_json = json.dumps(props, indent=4)
        if not self.props_file:
            self.props_file.save(self.slug + '.props', ContentFile(props_json))
        else:
            with open(self.props_file.path, 'w') as f:
                f.write(props_json)

    def get_formulas(self):
        formulas = set()
        for host_definition in self.host_definitions.all():
            for component in host_definition.formula_components.all():
                formulas.add(component.formula)

        return list(formulas)

class BlueprintHostDefinition(TitleSlugDescriptionModel, TimeStampedModel):

    class Meta:
        verbose_name_plural = 'host definitions'

        default_permissions = ()

        unique_together = (('title', 'blueprint'), ('hostname_template', 'blueprint'))

    # The blueprint object this host is owned by
    blueprint = models.ForeignKey('blueprints.Blueprint',
                                  related_name='host_definitions')

    # The cloud image object this host should use when being
    # launched
    cloud_image = models.ForeignKey('cloud.CloudImage',
                                    related_name='host_definitions')

    # The default number of instances to launch for this host definition
    count = models.PositiveIntegerField('Count')

    # The hostname template that will be used to generate the actual
    # hostname at launch time. Several template variables will be provided
    # when the template is rendered down to its final form
    hostname_template = models.CharField('Hostname Template', max_length=64)

    # The default instance size for the host
    size = models.ForeignKey('cloud.CloudInstanceSize')

    # The default availability zone for the host
    # Only for EC2 classic
    zone = models.ForeignKey('cloud.CloudZone', null=True, blank=True)

    # The subnet id for VPC enabled accounts
    # Only for EC2 VPC
    subnet_id = models.CharField('Subnet ID', max_length=32, blank=True, default='')

    # The spot instance price for this host. If null, spot
    # instances will not be used for this host.
    spot_price = models.DecimalField(max_digits=5,
                                     decimal_places=2,
                                     blank=True,
                                     null=True)

    # Grab the list of formula components
    formula_components = GenericRelation('formulas.FormulaComponent')

    @property
    def formula_components_count(self):
        return self.formula_components.count()

    def __unicode__(self):
        return self.title

class BlueprintAccessRule(TitleSlugDescriptionModel, TimeStampedModel):
    """
    Access rules are a white list of rules for a host that defines
    what protocols and ports are available for the corresponding
    machines at launch time. In other words, they define the
    firefall rules for the machine.
    """

    class Meta:
        verbose_name_plural = 'access rules'

        default_permissions = ()

    # The host definition this access rule applies to
    host = models.ForeignKey('blueprints.BlueprintHostDefinition',
                             related_name='access_rules')

    # The protocol for the access rule. One of tcp, udp, or icmp
    protocol = models.CharField('Protocol', max_length=4, choices=PROTOCOL_CHOICES)

    # The from and to ports define the range of ports to open for the
    # given protocol and rule string. To open a single port, the
    # from and to ports should be the same integer.
    from_port = models.IntegerField('Start Port')
    to_port = models.IntegerField('End Port')

    # Rule is a string specifying the CIDR for what network has access
    # to the given protocl and ports. For AWS, you may also specify
    # a rule of the form "owner_id:security_group", that will authorize
    # access to the given security group owned by the owner_id's account
    rule = models.CharField('Rule', max_length=255)

    def __unicode__(self):
        return u'{0} {1}-{2} {3}'.format(
            self.protocol,
            self.from_port,
            self.to_port,
            self.rule
        )

class BlueprintVolume(TitleSlugDescriptionModel, TimeStampedModel):

    class Meta:
        verbose_name_plural = 'volumes'

        default_permissions = ()

    # The host definition this access rule applies to
    host = models.ForeignKey('blueprints.BlueprintHostDefinition',
                             related_name='volumes')

    # The device that the volume should be attached to when a stack is created
    device = models.CharField('Device Name', max_length=32, choices=DEVICE_ID_CHOICES)

    # Where the volume will be mounted after created and attached
    mount_point = models.CharField('Mount Point', max_length=64)

    # The snapshot ID to create the volume from
    snapshot = models.ForeignKey('cloud.Snapshot',
                                 related_name='host_definitions')

    def __unicode__(self):
        return u'BlueprintVolume: {0}'.format(self.pk)


#!/usr/bin/env python
#
# call: grid_submit.py --project=ts_warm_bore --dsid=760_3000 --stage=ts1_sim  -step=gen_fcl
#-------------------------------------------------------------------------------------------------

import configparser, subprocess, shutil
import sys, string, getopt, glob, os, time, re, array

#------------------------------------------------------------------------------
class RenameLocalFilesAlaMu2eGrid:

    def __init__(self):
        self.fProject       = None
        self.fProjectDir    = None
        self.fDsid          = xxx_xxxx # just to make up
        self.fDoit          = 1
        self.fJob           = None
        self.fStage         = None
        self.fIStage        = None;
        self.fIStream       = None;
        self.fUser          = 'murat'
        self.fGridJobID     = None;

        self.fOutputPath    = {}
        self.fOutputStreams = None

        self.fVerbose   = 1

# ---------------------------------------------------------------------
    def Print(self,Name,level,Message):
        if(level>self.fVerbose): return 0;
        now = time.strftime('%Y/%m/%d %H:%M:%S',time.localtime(time.time()))
        message = now+' [ RenameLocalFilesAlaMu2eGrid::'+Name+' ] '+Message
        print(message)

#----------------------------------------------------------------------
# Parse the command-line parameters.
# the only required one is --project=$Project which defines the tiki page
# where the rest of the parameters can be found.  Set --doit=0 to
# only compute what would be done, and not actually do it.
# --verbose=0 only print necessary error messages etc.
# --verbose=1 (default) print some summary of what was done
# --verbose=2 print detailed summary of what was done
# --verbose=10 dump everything
#------------------------------------------------------------------------------
    def ParseParameters(self):
        name = 'ParseParameters'

        self.Print(name,2,'Starting')
        self.Print(name,2, '%s' % sys.argv)

        try:
            optlist, args = getopt.getopt(sys.argv[1:], '',
                     ['project=', 'verbose=', 'job=', 'doit=', 'dsid=', 'stage=' ] )

        except getopt.GetoptError:
            self.Print(name,0,'%s' % sys.argv)
            self.Print(name,0,'Errors arguments did not parse')
            return 110

        for key, val in optlist:

            # print('key,val = ',key,val)

            if key == '--project':
                self.fProject = val
            elif key == '--doit':
                self.fDoit = int(val)
            elif key == '--job':
                self.fJob = val
            elif key == '--dsid':
                self.fDsid = val
            elif key == '--stage':
                self.fStage = val
            elif key == '--verbose':
                self.fVerbose = int(val)

        self.fProjectDir = self.fProject+'/'+self.fDsid;

        self.Print(name,1,'Job        = %s' % self.fJob    )
        self.Print(name,1,'Project    = %s' % self.fProject)
        self.Print(name,1,'Verbose    = %s' % self.fVerbose)
        self.Print(name,1,'Doit       = %s' % self.fDoit)
        self.Print(name,1,'Dsid       = %s' % self.fDsid)
        self.Print(name,1,'ProjectDir = %s' % self.fProjectDir)

        if (self.fProject == None) :
            self.Print(name,0,'Error: Project not defined - exiting!')
            sys.exit(1)

        self.Print(name,1,'Done')
        return 0

#------------------------------------------------------------------------------
    def InitProject(self):
        name = 'InitProject'

        #------------------------------------------------------------------------------
        # read project config file
        #------------------------------------------------------------------------------
        config_dir = self.fProjectDir+'/config'
        name_stub  = self.fProject+'.'+self.fDsid+'.'+self.fStage;

        fn = config_dir+'/'+name_stub+'.cfg';

        self.fConfig = configparser.ConfigParser()
        self.fConfig.read(fn)

        self.Print(name,1,'self.fProject=%s, fn=%s' % (self.fProject,fn))

        # step 1: need to generate fcl files
        projectName         = self.fConfig['project']['name']
        dsid                = self.fConfig['project']['dsid']

        self.fIStage        = self.fConfig['input']['stage' ]
        self.fIStream       = self.fConfig['input']['stream']
        self.fInputDs       = self.fConfig['input']['dataset']
        self.fOutputStreams = self.fConfig['output']['streams'].split(',');

        for s in self.fOutputStreams:
            self.fOutputPath[s] = self.fConfig['path'][s];
            self.Print(name,1,'self.fOutputPath[%s] = %s' % (s,self.fOutputPath[s]));

        self.Print(name,1,'projectName   = %s' % projectName)
        self.Print(name,1,'dsid          = %s' % projectName)
        self.Print(name,1,'self.fIStage  = %s' % self.fIStage)
        self.Print(name,1,'self.fIStream = %s' % self.fIStream)

        #  read grid config, if it exists

        fn1 = config_dir+'/'+name_stub+'.jobid';

        if os.path.exists(fn1) :
            line  = open(fn1).readline()
            self.fGridJobID = line.strip()
            self.Print(name,1,'self.fGridJobID = %s' % self.fGridJobID)

#------------------------------------------------------------------------------
# check log files. asume they are copied into the output area
#------------------------------------------------------------------------------
    def CheckOutput(self):
        name = 'CheckOutput'

        input_stub = self.fIStage+'_'+self.fIStream;
        grid_dsid = self.fProject+'.'+self.fDsid+'.'+input_stub+'.'+self.fStage;
        topdir    = '/pnfs/mu2e/scratch/users/'+self.fUser+'/workflow/'+grid_dsid+'/outstage/';

        if (self.fGridJobID): topdir = topdir+self.fGridJobID+'/00/'
        # print('topdir=',topdir)

        dir = '/mu2e/data/users/murat/datasets/'+self.fProject+'/'+self.fDsid+'/'+self.fStage

        dir = '/mu2e/data/users/murat/datasets/'+self.fProject+'/'+self.fDsid+'/'+'ts1/log'

        ntot = 0;
        nev  = {}

        for stream in self.fOutputStreams:
            nev[stream] = array.array('i',[0,0])   # nfiles, nev
            # print ("stream, nev : ",stream,nev[stream][0],nev[stream][1])

        bad_files = []
        for fn in glob.glob(dir+'/*.log'):
            f = open(fn)
            #            print('-- fn: ',fn)
            lines = f.readlines();

            segment_ok = 1
            for line in lines:
                if ('Art has completed and will exit with status' in line):
                    words = line.strip().split()
                    rc = words[8].strip('.');
                    if (rc != "0"):
                        segment_ok = 0
                        # print ("rc = %s, words:"%rc,words)
                        bad_files.append((fn,rc));
                        break;

            if (segment_ok == 1):
                for line in lines:
                    words = line.strip().split()
                    if (len(words) == 0) : continue

                    # print('words:',words, 'nw=',len(words))

                    if ('TrigReport Events total' in line):
                        ntot = ntot+int(words[4]);

                    for stream in self.fOutputStreams:
                        if (words[0] == 'TrigReport') :
                            if (words[2] == 'Event') or (words[2] == 'Path'):
                                continue
                            if (len(words) == 8) and (words[7] == self.fOutputPath[stream]):
                                nev[stream][0] = nev[stream][0]+1
                                nev[stream][1] = nev[stream][1]+int(words[4])
            # else:
            #     print("failed processing %s" % fn);

        print("total input: %10i" % ntot);
        for stream in self.fOutputStreams:
            print("stream ",stream,' : processed ',stream,nev[stream][1],' events ', nev[stream][0],' files');

        for bf in bad_files:
            print("ERROR: file ", bf[0]," RC = ",bf[1])

#------------------------------------------------------------------------------
# check log files. asume they are copied into the output area
#------------------------------------------------------------------------------
    def BuildTarball(self):
        name = 'BuildTarball'

        cmd_pars = ['gridexport' ,'-E '+os.getenv('PWD')+'/grid_export', '-A ts_warm_bore/AAA_GRIDEXPORT_EXCLUDE.txt'];

        print(">>> [BuildTarball] executing :"," ".join(cmd_pars));

        process = subprocess.run(cmd_pars,capture_output=True,universal_newlines=True)

        print(">>> [BuildTarball] done executing");

        if (process.returncode == 0):
            lines       = process.stdout.split('\n')

            for line in lines: print(line)

            word        = lines[2].split();
            tarball     = word[4]
            tmp_dir     = os.path.dirname(tarball)
            new_tarball = self.fProject+'/tmp_code/'+self.fProject+'.code.'+self.fDsid+'.tbz'

            if os.path.exists(new_tarball): os.remove(new_tarball);

            shutil.copy(tarball,new_tarball);
            os.remove(tarball);

            # remove old directory
            os.rmdir(tmp_dir);

            # copy tarball to /pnfs
            grid_tarball = '/pnfs/mu2e/resilient/users/murat/'+self.fProject+'.code.tbz'
            if os.path.exists(grid_tarball):
                os.remove(grid_tarball)
            shutil.copy(new_tarball,grid_tarball);

#------------------------------------------------------------------------------
# check log files. asume they are copied into the output area
#------------------------------------------------------------------------------
    def SubmitGridJob(self):
        name = 'BuildTarball'

        input_ds = self.fConfig['input']['stage']+':'+self.fConfig['input']['stream']
        stage    = self.fConfig['job'  ]['stage']+':'+self.fConfig['job']  ['type'  ]

        parms    = ['ts_warm_bore/scripts/submit_grid_job',
                    self.fProject,
                    self.fDsid,
                    input_ds,
                    stage,
                    self.fConfig['job']['time_request'],
                    self.fConfig['job']['ifdh']
                    ,'.']  # doit

        print(parms);

        process = subprocess.run(parms, capture_output=True, universal_newlines=True)
        #
        if (process.returncode == 0):
            lines       = process.stdout.split('\n')
            print(lines)
            # extract the grid job ID and save the output
            jobid = None;
            for line in lines:
                print(line)
                if line != '':
                    w = line.split()
                    if (w[0] == 'JobsubJobId') :
                        jobid = w[4].split('.')[0]
                        break

            print('jobid:',jobid)

            # ok, now we have jobid, write it into a file
            dir = self.fProject+'/'+self.fDsid+'/config'
            bn  = self.fProject+'.'+self.fDsid+'.'+self.fConfig['job']['stage']+'_'+self.fConfig['job']['type']+'.jobid'
            fn  = dir+'/'+bn+'.jobid';
            f = open(fn,'w')
            f.write(jobid)
            f.close()

            # also want to save a grid submission loginto a separate file
            dir = self.fProject+'/'+self.fDsid+'/log'
            fn = dir+'/'+bn+'.'+jobid
            f = open(fn,'w')
            for line in lines:
                f.write(line)

            f.close();

            # finally, add one line to a grid submittion summary
            fn = dir+'/AAA_grid_submission_summary.org'

#------------------------------------------------------------------------------
# main program, just make a RenameLocalFilesAlaMu2eGrid instance and call its methods
#------------------------------------------------------------------------------
if (__name__ == '__main__'):

    x = RenameLocalFilesAlaMu2eGrid()
    x.ParseParameters()

    x.InitProject()

#    if (x.fJob == 'check_output'   ) : x.CheckOutput()
#    if (x.fJob == 'build_tarball'  ) : x.BuildTarball()
#    if (x.fJob == 'submit_grid_job') : x.SubmitGridJob()
#
#    gs.PrintConfFile(1)
#    gs.CheckForNewFiles()
#    gs.UpdateJobStatus()
#    gs.SubmitNewJobs()
#    gs.PrintConfFile(1)
#    gs.CatExit(0)
    sys.exit(0);


import jsonlines

TABLES = "data/dev.tables.jsonl"
QUERIES = "data/dev.jsonl"
PREDICTIONS = "test/my.pred.dev.jsonl"

tables = {}

# Read tables and data
with jsonlines.open(TABLES) as input_tables:
    for table in input_tables:
        tables[table["id"]] = table

# Open queries and output file
with jsonlines.open(QUERIES) as input_queries, jsonlines.open(PREDICTIONS, mode='w') as predictions_file:
    for query in input_queries:
        question = query["question"].lower()
        table = tables[query["table_id"]]

        # Determine selection
        sel = -1
        for i, field in enumerate(table["header"]):
            if field.lower() in question:
                sel = i

        # Determine conditions
        conds = []
        for row in table["rows"]:
            for i, cell in enumerate(row):
                if isinstance(cell, str):
                    if cell.lower() in question:
                        conds.append([i, 0, cell])

        prediction = {"query": {"sel": sel, "conds": conds, "agg": 0}, "error": ""}
        predictions_file.write(prediction)


import argparse
import os
import sys
from os.path import dirname, abspath

from envs.common_envs_utils.env_makers import make_CarRacing_fixed_vector_features, \
    make_CarRacing_fixed_combined_features

sys.path.append(dirname(dirname(abspath(__file__))))

from agents.actor_critic_agents.Soft_A3C import Soft_A3C
from utilities.data_structures.Config import Config

import chainerrl
import tensorflow as tf
from envs.common_envs_utils.extended_env_wrappers import ExtendedMaxAndSkipEnv, FrameCompressor, OriginalStateKeeper, \
    ImageWithVectorCombiner, ChannelSwapper, TorchTensorCaster
from envs.gym_car_intersect_fixed import CarRacingHackatonContinuousFixed

def create_config(args):
    config = Config()
    config.seed = 1
    config.environment = None
    if args.mode == 'both':
        config.environment = make_CarRacing_fixed_combined_features(args.env_settings)()
    elif args.mode == 'vector':
        config.environment = make_CarRacing_fixed_vector_features(args.env_settings)()
    else:
        raise NotImplemented
    config.env_settings = args.env_settings

    config.num_episodes_to_run = 15000
    config.file_to_save_data_results = 'result_cars'
    config.file_to_save_results_graph = 'graph_cars'
    config.show_solution_score = True
    config.visualise_individual_results = False
    config.visualise_overall_agent_results = True
    config.standard_deviation_results = 1.0
    config.runs_per_agent = 1
    config.device = args.device
    config.overwrite_existing_results_file = False
    config.randomise_random_seed = True
    config.save_model = True
    config.max_episode_steps = 300
    config.random_replay_prefill_ration = args.start_buffer_random_ratio

    config.hyperparameters = {
        "Actor_Critic_Agents": {
            "Actor": {
                "learning_rate": 3e-4,
                "linear_hidden_units": [20, 20],
                "final_layer_activation": None,
                "batch_norm": False,
                "tau": 0.005,
                "gradient_clipping_norm": 5,
                "initialiser": "Xavier"
            },
            "Critic": {
                "learning_rate": 3e-4,
                "linear_hidden_units": [20, 20],
                "final_layer_activation": None,
                "batch_norm": False,
                "buffer_size": 100000,
                "tau": 0.005,
                "gradient_clipping_norm": 5,
                "initialiser": "Xavier"
            },

            "learning_rate": 3e-4,
            "gradient_clipping_norm": 5.0,
            "epsilon_decay_rate_denominator": 1.0,
            "normalise_rewards": False,
            "exploration_worker_difference": 2.0,

            "save_frequency_episode": 500,
            "min_steps_before_learning": 50000,
            "batch_size": 128,
            "discount_rate": 0.99,
            "mu": 0.0,  # for O-H noise
            "theta": 0.15,  # for O-H noise
            "sigma": 0.25,  # for O-H noise
            "action_noise_std": 0.2,  # for TD3
            "action_noise_clipping_range": 0.5,  # for TD3
            "update_every_n_steps": 20,
            "learning_updates_per_learning_session": 10,
            "automatically_tune_entropy_hyperparameter": True,
            "entropy_term_weight": None,
            "add_extra_noise": True,
            "do_evaluation_iterations": False,
            "clip_rewards": False,

            # "mode_to_use": "normal",
            # "rlkit_mode_parameters": {
            #     "explanation_steps_per_step": 1000,
            #     "update_steps_per_step": 1000,
            # }
        }
    }
    return config

def main(args):
    agent_title = args.name
    if not os.path.exists(os.path.join('logs', agent_title)):
        os.makedirs(os.path.join('logs', agent_title))
    tf_writer = tf.summary.create_file_writer(os.path.join('logs', agent_title))
    agent_config = create_config(args)
    agent_config.name = agent_title

    # random.randint(0, 2 ** 32 - 2)
    agent_config.seed = 42

    agent_config.hyperparameters = agent_config.hyperparameters['Actor_Critic_Agents']
    print("AGENT NAME: {}".format('SAC'))

    agent = Soft_A3C(agent_config, name=args.name)

    if args.load != 'none':
        agent.load(args.load)

    print(agent.hyperparameters)

    print("RANDOM SEED ", agent_config.seed)

    game_scores, rolling_scores, time_taken = agent.run_n_episodes(tf_saver=tf_writer, visualize=True)
    print("Time taken: {}".format(time_taken), flush=True)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--name', type=str, default='test', help='name for experiment')
    parser.add_argument('--mode', type=str, default='both', help='image only, vector only, or their combination')
    parser.add_argument('--env-settings', type=str, default='test', help='path to CarRacing env settings')
    parser.add_argument('--device', type=str, default='cpu', help='path to CarRacing env settings')
    parser.add_argument('--load', type=str, default='none', help='path to load model')
    parser.add_argument('--start-buffer-random-ratio', type=float, default=1.0,
                        help='ratio of random action for replay buffer pre-fill, useful for loaded agents'
                        )
    args = parser.parse_args()

    if args.mode not in ['image', 'vector', 'both']:
        raise ValueError("mode should be one of 'image', 'vector', 'both'")
    main(args)


# -*- coding: utf-8 -*-
"""
Tencent is pleased to support the open source community by making 蓝鲸智云(BlueKing) available.
Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at http://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and limitations under the License.
"""
import json
import datetime

from blueking.component.shortcuts import get_client_by_request
from common.mymako import render_mako_context, render_json

from home_application.celery_tasks import *
from home_application.decorator import api_try_except, log_request_params
from utils import fast_execute_script, get_task_ip_log, get_business, get_host
from settings import APP_TOKEN,APP_ID

CONSTANT_PARAMS = {
    'bk_app_code': APP_ID,
    'bk_app_secret': APP_TOKEN
}

def home(request):
    """
    首页
    """
    return render_mako_context(request, '/home_application/home.html')

def test(request):
    """测试接口"""
    return render_json({
        "username": request.user.username,
        "result": "ok"
    })

def search_business(request):
    username = request.user.username
    client = get_client_by_user(username)
    params = {
        'bk_username': username
    }
    return render_json(get_business(client, params))

def search_obj(request):
    username = request.user.username
    client = get_client_by_user(username)
    request_body = json.loads(request.body)
    bk_biz_id = request_body['bk_biz_id']
    ips = request_body['ips']
    ip_list = ips.split("\n")
    params = {
        'username': username,
        'ip_list': ip_list,
        'bk_biz_id': bk_biz_id
    }
    res = get_host(client,params)
    return_data = []
    for info in res['data']:
        res_dict = {
            'ip': info['host']['bk_host_innerip'],
            'bk_biz_id':info['biz'][0]['bk_biz_id'],
            'bk_biz_name':info['biz'][0]['bk_biz_name'],
            'bk_cloud_id':info['host']['bk_cloud_id'][0]['bk_inst_id'],
            'bk_cloud_name':info['host']['bk_cloud_id'][0]['bk_inst_name'],
            'os_name': info['host']['bk_os_name'],
            'host_name': info['host']['bk_host_name'],
        }
        return_data.append(res_dict)
    real_return_data = get_host_data(return_data)
    return render_json({'result':True, 'data': real_return_data})

def get_host_data(host_list):
    for i in host_list:
        obj = HostInfo.objects.filter(ip=i['ip'])
        if not obj:
            i['disk_usage'] = "--"
            i['mem_usage'] = "--"
            i['cpu_usage'] = "--"
            i['is_monitor'] = False
            continue
        obj_data = HostData.objects.filter(host=obj[0])
        if not obj_data:
            i['disk_usage'] = "--"
            i['mem_usage'] = "--"
            i['cpu_usage'] = "--"
            i['is_monitor'] = obj[0].is_monitor
            continue
        real_obj_data = obj_data.last()
        i['disk_usage'] = real_obj_data.disk_usage
        i['mem_usage'] = real_obj_data.mem_usage
        i['cpu_usage'] = real_obj_data.cpu_usage
        i['is_monitor'] = obj[0].is_monitor
    return host_list

def modify_monitor(request):
    request_body = json.loads(request.body)
    args = request_body['args']
    if args['is_monitor']:
        is_monitor = False
    else:
        is_monitor = True
    HostInfo.objects.update_or_create(ip=args['ip'], defaults={
        "ip": args["ip"],
        "bk_cloud_id": args["bk_cloud_id"],
        "bk_cloud_name": args["bk_cloud_name"],
        "bk_biz_id": args["bk_biz_id"],
        "bk_biz_name": args["bk_biz_name"],
        "os_name": args["os_name"],
        "host_name": args['host_name'],
        "created_by": request.user.username,
        "when_created": timezone.now(),
        "is_monitor": is_monitor
    })
    return render_json({'result':True})

def search_data(request):
    request_body = json.loads(request.body)
    mes = request_body['mes']
    script_content = """
#!/bin/bash
MEMORY=$(free -m | awk 'NR==2{printf "%.2f%%", $3*100/$2 }')
DISK=$(df -h | awk '$NF=="/"{printf "%s", $5}')
CPU=$(top -bn1 | grep load | awk '{printf "%.2f%%", $(NF-2)}')
DATE=$(date "+%Y-%m-%d %H:%M:%S")
echo -e "$DATE|$MEMORY|$DISK|$CPU"
    """
    ip_list = [{
        "ip": mes['ip'],
        "bk_cloud_id": mes["bk_cloud_id"]
    }]
    username = request.user.username
    clint = get_client_by_user(username)
    res = fast_execute_script(clint,mes['bk_biz_id'], username, ip_list,
                        script_content)
    log_content = res['data'][0]['log_content'].split("|")
    return_data = {
        "mem_usage": float(log_content[1].replace("%","")),
        "disk_usage": float(log_content[2].replace("%", "")),
        "cpu_usage": float(log_content[3].replace("%", ""))
    }
    return render_json({"result":True, "data": return_data})

def search_host_mysql(request):
    hosts = HostInfo.objects.filter(is_monitor=True)
    return_data = [i.to_dict() for i in hosts]
    return render_json({"result":True, "data": return_data})

def get_host_data_mes(host):
    return_data = [
        {"name": "cpu", "data":[]},
        {"name": "mem", "data":[]},
        {"name": "disk", "data":[]},
                   ]
    for i in xrange(60):
        gt_time = timezone.now() - datetime.timedelta(minutes=i)
        lt_time = timezone.now() - datetime.timedelta(minutes=i-1)
        host_data_mes = HostData.objects.filter(when_created__gte=gt_time, when_created__lte=lt_time, host=host)
        if not host_data_mes:
            return_data[0]['data'].append(None)
            return_data[1]['data'].append(None)
            return_data[2]['data'].append(None)
        else:
            return_data[0]['data'].append(host_data_mes[0].cpu_usage)
            return_data[1]['data'].append(host_data_mes[0].mem_usage)
            return_data[2]['data'].append(host_data_mes[0].disk_usage)
    for j in return_data:
        j['data'].reverse()
    return return_data

def search_host_data(request):
    request_body = json.loads(request.body)
    ip = request_body['ip']
    categories = get_categories_minutes()
    host = HostInfo.objects.get(ip=ip)
    data = get_host_data_mes(host)
    return render_json({'result':True, 'data':{
        'data':data,
        'categories': categories
    }})


from django.contrib.staticfiles.finders import find
from django.http import Http404
from markdown import markdown
from seminar import settings
import os.path

def load_markdown(markdown_filename, directory_name="mdown"):
    path = os.path.join(settings.STATIC_ROOT, directory_name, markdown_filename)

    try:
        mdfile = open(path, 'r+')
    except IOError:
        # We must be in development! Search for the static file using staticfiles finder
        match = find(directory_name + markdown_filename)
        if (match != None):
            mdfile = open(str(match), 'r+')
        else:
            raise Http404

    text = mdfile.read()
    html = markdown(text)
    mdfile.close()

    return html

def truncate(string, num_characters=50, continuation_string="..."):
    return string[:num_characters] + (string[num_characters:] and continuation_string)


from django.urls import path
from django.conf.urls import url
from . import views

urlpatterns = [
    path('', views.home, name="member"),
    path('signup/', views.signup, name="signup"),
    path('login/', views.login, name="login"),
    path('logout/', views.logout, name="logout"),
    path('findid/', views.findid, name="findid"),
    path('findpw/', views.findpw, name="findpw"),
]


import project_functions
import numpy as np
import skimage
from skimage.morphology import disk
from PIL import Image
import numba as nb

import sys
sys.path.append('C:/Users/liuzhenye/Desktop/Project3/VSPV/model/python_layers/')
import forward_backward_map_layer
@nb.jit
def forward2backwardMap(forward_flow):
    backward_map = np.zeros((200,200))
    for x in range(0,200):
        for y in range(0,200):
            if((forward_flow[0,x,y]!=0)|(forward_flow[1,x,y]!=0)):
                uu = x+forward_flow[1,x,y]
                vv = y+forward_flow[0,x,y]
                uu = int(uu+0.5)
                vv = int(vv+0.5)

                if uu<0 or vv<0 or uu>=200 or vv>=200:
                    continue

                backward_map[uu,vv] = 1

    return backward_map

def forward_flow(path,number):
    for i in range(25,number):
        depth_map=np.fromfile(open(path+'depth/{}_9_d.bin'.format(i)), dtype = np.double)
        print(depth_map.shape)
        depth_map.resize(200,200)
        (K,Rt,C)=project_functions.getKRtC(9)
        new_xyz_image=project_functions.DepthMap2XyzImage(depth_map,K,Rt)
        for j in range(19):
            (K,Rt,C)=project_functions.getKRtC(j)
            forward_flow=project_functions.xyzImage2ForwardFlow(new_xyz_image, K, Rt)
            np.save(path+'forward_flow/{}_{}_u.bin'.format(i,j),forward_flow[0,:,:])
            np.save(path+'forward_flow/{}_{}_v.bin'.format(i,j),forward_flow[1,:,:])
        print(i)
@nb.jit
def forward_mask(path,number):
    for i in range(25,number):
        for j in range(19):
            forward_flow=np.zeros([2,200,200])
            forward_flow[0,:,:]=np.load(path+'forward_flow/{}_{}_u.bin.npy'.format(i,j))
            forward_flow[1,:,:]=np.load(path+'forward_flow/{}_{}_v.bin.npy'.format(i,j))
            flow_mask = forward2backwardMap(forward_flow)
            flow_mask = skimage.morphology.closing(flow_mask,disk(2)) * 255
            flow_mask = flow_mask
            np.save(path+'forward_mask/{}_{}'.format(i,j),flow_mask)
        print (i)

def reampped_forward_flow(path,number):
    for i in range(number):
        for j in range(19):
            bottom=np.zeros([2,200,200])
            bottom[0,:,:]=np.load(path+'forward_flow/{}_{}_u.bin.npy'.format(i,j))
            bottom[0,:,:]=np.load(path+'forward_flow/{}_{}_v.bin.npy'.format(i,j))
            top=np.zeros([2,200,200])
            f=forward_backward_map_layer.forward_backward_map_layer()
            np.save(path+'reampped_forward_flow/{}_{}_u.bin'.format(i,j),forward_flow[0,:,:])
            np.save(path+'remapped_forward_flow/{}_{}_v.bin'.format(i,j),forward_flow[1,:,:])
        print(i)
#forward_flow('H:/data/PVHM_8000-8999/',1000)
#forward_flow('H:/data/PVHM_0-999/',1000)
#forward_mask('H:/data/PVHM_0-999/',1000)
#flow_mask = np.loadtxt('H:/data/PVHM_0-999/forward_mask/{}_{}.png'.format(0,0))
#print(flow_mask)
reampped_forward_flow('H:/data/PVHM_0-999/',1000)


#!/usr/bin/env python

import argparse
import os
import subprocess

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('outdir')
    parser.add_argument('-p', '--prefix', help="add a prefix")
    parser.add_argument('inputs', nargs='+')
    return parser.parse_args()

def main():
    args = get_args()
    for inpath in args.inputs:
        name_parts = inpath.split('/')[-2:]
        if args.prefix:
            name_parts = [args.prefix] + name_parts
        outpath = os.path.join(args.outdir, '-'.join(name_parts))
        subprocess.check_call(['cp', inpath, outpath])

if __name__ == '__main__':
    main()


import csv
from StringIO import StringIO
from io import BytesIO
from zipfile import ZipFile

import traceback
from flask import Blueprint, request, abort, make_response, send_file
from flask import session
from flask.ext.login import current_user
from flask_stormpath import login_required, groups_required
from taa.services.products.plan_codes import PLAN_CODES_SIMPLE
from taa.tasks import send_admin_error_email

from taa.api import route
from taa.services import LookupService
from taa.services.enrollments.csv_export import export_hi_acc_enrollments

bp = Blueprint('enrollments', __name__, url_prefix='/enrollments')
agent_service = LookupService("AgentService")
case_service = LookupService("CaseService")
enrollment_import_service = LookupService("EnrollmentImportService")
enrollment_import_batch_service = LookupService("EnrollmentImportBatchService")
enrollment_import_batch_item_service = LookupService("EnrollmentImportBatchItemService")
enrollment_submission_service = LookupService("EnrollmentSubmissionService")
enrollment_application_service = LookupService("EnrollmentApplicationService")
user_service = LookupService("UserService")
product_service = LookupService("ProductService")

@route(bp, '/', methods=["POST"])
def submit_enrollments():
    case_token = request.args.get('case_token') or request.form.get('case_token')
    auth_token = request.args.get('auth_token') or request.form.get('auth_token')
    user_href = request.args.get('user_href') or request.form.get('user_href')
    if not user_href:
        user_href = user_service.get_current_user_href()

    data_format = request.args.get('format') or request.form.get('format', 'flat')
    upload_source = request.args.get('upload_source') or request.form.get('upload_source', 'api')
    filename = request.args.get("filename") or request.form.get('filename', '')

    if request.data:
        data = StringIO(request.data)
    elif request.files['api-upload-file']:
        data = request.files['api-upload-file']
        filename = data.filename
    else:
        raise ValueError("No data provided")

    try:
        import_results = enrollment_import_service.process_enrollment_data(
            data,
            data_format,
            case_token=case_token,
            auth_token=auth_token,
            user_href=user_href,
            data_source=upload_source,
            filename=filename,
        )
    except Exception as e:
        enrollment_import_service.send_generic_error_email(user_href)
        send_admin_error_email("Generic Import Error:<br>{}".format(e.message), [traceback.format_exc()])
        return {
           'num_processed': 0,
           'num_errors': 0,
           'errors': [{'fields':'Error', 'message': 'There was a problem reading the file'}]
         }, 400

    return {
        'num_processed': import_results.get_num_processed(),
        'num_errors': len(import_results.get_errors()),
        'errors': [error.to_json() for error in import_results.get_errors()][:20]
    }, 400 if import_results.is_error() else 200

@route(bp, '/import_batches', methods=['GET'])
@login_required
@groups_required(['admins'])
def get_batches():
    return enrollment_import_batch_service.all()

@route(bp, '/import_batches/<batch_id>', methods=['GET'])
@login_required
@groups_required(['admins'])
def get_batch_items(batch_id):
    batch = enrollment_import_batch_service.get(batch_id)
    if not batch:
        abort(404)

    return [item for item in enrollment_import_batch_service.get_batch_items(batch)]

@route(bp, '/import_batches/<batch_id>/reprocess', methods=['POST'])
@login_required
@groups_required(['admins'])
def reprocess_batch(batch_id):
    batch = enrollment_import_batch_service.get(batch_id)

    # Enqueue the batch for processing
    enrollment_submission_service.submit_import_enrollments(batch)

@route(bp, '/import_batches/<batch_id>', methods=['DELETE'])
@login_required
@groups_required(['admins'])
def delete_batch(batch_id):
    batch = enrollment_import_batch_service.get(batch_id)

    # Delete all the enrollment records, batch items, and the batch itself.
    enrollment_import_batch_service.delete_batch(batch)

# For convenience, allow lookup of enrollment records without case id for admin only
#  (if this is opened up to other users, add case permission checking)
@route(bp, '/records/<int:enrollment_record_id>', methods=['GET'])
@login_required
@groups_required(['admins', 'home_office', 'agents'], all=False)
def get_individual_enrollment_record(enrollment_record_id):
    enrollment = enrollment_application_service.get_or_404(enrollment_record_id)

    return enrollment

# Admin delete enrollment record
@route(bp, '/records/<int:enrollment_record_id>', methods=['DELETE'])
@login_required
@groups_required(['admins', 'home_office'], all=False)
def delete_individual_enrollment_record(enrollment_record_id):
    # Return a 404 if not a valid record
    enrollment_record = enrollment_application_service.get_or_404(enrollment_record_id)

    # Delete the record with associated coverages and batch items
    enrollment_application_service.delete_enrollment_record(enrollment_record)

    # Return proper DELETE response
    return None, 204

@route(bp, '/import_batches/<batch_id>/<item_id>/pdf', methods=['GET'])
@login_required
@groups_required(['admins'])
def render_batch_item_pdf(batch_id, item_id):
    item = enrollment_import_batch_item_service.get(item_id)
    if not item:
        abort(404)

    binary_pdf = enrollment_submission_service.render_enrollment_pdf(item.enrollment_record, is_stp=False, force_show_all_docs=True)

    response = make_response(binary_pdf)
    response.headers['Content-Type'] = 'application/pdf'
    response.headers['Content-Disposition'] = 'inline; filename=%s.pdf' % 'enrollment_{}'.format(item.enrollment_record_id)
    return response

@route(bp, '/import_batches/<batch_id>/<item_id>/xml', methods=['GET'])
@login_required
@groups_required(['admins'])
def render_batch_item_xml(batch_id, item_id):
    item = enrollment_import_batch_item_service.get(item_id)
    if not item:
        abort(404)

    pdf_bytes = enrollment_submission_service.render_enrollment_pdf(item.enrollment_record, is_stp=True)
    zipstream = BytesIO()
    with ZipFile(zipstream, 'w') as zip:
        for form_for in enrollment_submission_service.get_enrollees(item.enrollment_record):
            # TODO: This is not called correctly anymore.
            xml = enrollment_submission_service.render_enrollment_xml(
                item.enrollment_record, form_for, pdf_bytes)
            if xml is not None:
                fn = 'enrollment_{}-{}.xml'.format(item.enrollment_record_id, form_for)
                zip.writestr(fn, xml.encode('latin-1'))
    zipstream.seek(0)
    return send_file(zipstream, attachment_filename='enrollment_{}.zip'.format(
        item.enrollment_record_id), as_attachment=True)

@route(bp, '/records/<int:enrollment_record_id>/pdf', methods=['GET'])
def generate_enrollment_pdf(enrollment_record_id):
    enrollment = enrollment_application_service.get_or_404(enrollment_record_id)

    # Check that logged-in agent can view enrollment
    if not can_user_view_enrollment_pdf(enrollment):
        abort(403)

    binary_pdf = enrollment_submission_service.render_enrollment_pdf(enrollment, is_stp=False, force_show_all_docs=True)

    response = make_response(binary_pdf)
    response.headers['Content-Type'] = 'application/pdf'
    response.headers['Content-Disposition'] = 'inline; filename=%s.pdf' % 'enrollment_{}'.format(enrollment.id)
    return response

def can_user_view_enrollment_pdf(enrollment):
    if agent_service.can_manage_all_cases(current_user):
        return True

    agent = agent_service.get_logged_in_agent()
    if not agent:
        # Only allow if this is a preview, and we are in a self-enroll session
        can_view = (session.get('is_self_enroll') is not None and enrollment.is_preview)
    else:

        can_view = can_agent_view_enrollment(agent, enrollment)
    return can_view

def can_agent_view_enrollment(agent, enrollment):
    if not agent:
        return False
    return (case_service.can_current_user_view_case(enrollment.case) or agent.id == enrollment.agent_id)

@route(bp, '/records/<int:enrollment_record_id>/xml', methods=['GET'])
@login_required
@groups_required(['admins', 'home_office', 'agents'], all=False)
def generate_enrollment_xml(enrollment_record_id):
    app = enrollment_application_service.get_or_404(enrollment_record_id)

    xmls = enrollment_submission_service.generate_enrollment_xml_docs(app)

    zipstream = enrollment_submission_service.create_xml_zip(xmls)

    return send_file(zipstream, attachment_filename='enrollment_{}.zip'.format(
        enrollment_record_id), as_attachment=True)

def generate_xml(enrollment_record, form_for='employee'):
    # TODO: NOT called correctly anymore

    pdf_bytes = enrollment_submission_service.render_enrollment_pdf(
            enrollment_record)

    return enrollment_submission_service.render_enrollment_xml(
            enrollment_record, form_for, pdf_bytes)

@route(bp, '/export/acchi/csv/<from_>/<to_>', methods=['GET'])
@login_required
@groups_required(['admins'])
def render_acc_hi_csv(from_, to_):
    # Get cases for ACC/HI products
    case_ids = []
    for code in PLAN_CODES_SIMPLE:
        case_ids.extend([c.id for p in product_service.search(by_code=code)
                         for c in p.cases])
    case_ids = set(case_ids)

    enrollments = enrollment_application_service.get_enrollments_by_date(
            from_, to_).filter(
            enrollment_application_service.__model__.case_id.in_(case_ids)).all()
    data = export_hi_acc_enrollments(enrollments)
    response = make_response(data)
    response.headers['Content-Type'] = 'text/csv'
    response.headers['Content-Disposition'] = 'inline; filename=acc-hi_{}~{}.csv'.format(from_, to_)
    return response


#!/usr/bin/env python3.7

from collections import defaultdict
from graph_lib.edge import Edge
from graph_lib.exception import InvalidGraphException

"""
Represents the whole graph with additional functions for it
"""

class Graph:
    def __init__(self):
        self.numOfVertices = 0                  # Number of vertices
        self.uniqueVertices = []                # Storing unique vertices
        self.adjacencyList = defaultdict(list)  # Graph G(V,E)
        self.source = -1                        # Source vertex
        self.sink = -1                          # Sink vertex
        self.used = []                          # Storing used vertices

    # Adds Edge(u, v) with capacity c to the graph G(V,E)
    def addEdge(self, edge_):
        for i in range(len(self.adjacencyList[edge_.u])):
            if self.adjacencyList[edge_.u][i].u == edge_.u and self.adjacencyList[edge_.u][i].v == edge_.v:
                raise InvalidGraphException("Error: Edge ({}, {}) already exists. Skipping".format(
                                                    self.adjacencyList[edge_.u][i].u,
                                                    self.adjacencyList[edge_.u][i].v))
        self.checkIfNewVertex(edge_.u, edge_.v)                              # Does an edge contain a unique vertex?
        self.adjacencyList[edge_.u].append(Edge(edge_.u, edge_.v, edge_.c))  # Add a new edge

    # Is it a new vertex?
    def checkIfNewVertex(self, u, v):
        if u not in self.uniqueVertices:
            self.uniqueVertices.append(u)
        if v not in self.uniqueVertices:
            self.uniqueVertices.append(v)

        self.numOfVertices = len(self.uniqueVertices)

    # Checks if this edge is already in the present graph
    def doesAnEdgeAlreadyExist(self, edge_):
        for i in range(len(self.adjacencyList[edge_.u])):
            if self.adjacencyList[edge_.u][i].u == edge_.u and self.adjacencyList[edge_.u][i].v == edge_.v:
                return True
        return False

    # Sets source
    def setSource(self, source_):
        if source_ not in self.uniqueVertices:
            raise InvalidGraphException("Error: Vertex ({}) does not exist".format(source_))
        else:
            self.source = source_

    # Sets sink
    def setSink(self, sink_):
        if sink_ not in self.uniqueVertices:
            raise InvalidGraphException("Error: Vertex ({}) does not exist".format(sink_))
        else:
            self.sink = sink_

    # A recursive function that finds a path between source and sink and calculates its flow
    def pathDFS(self, source, sink, flow):
        if source == sink:
            return flow
        self.used.append(source)  # mark as visited
        for i in range(len(self.adjacencyList[source])):
            edge = self.adjacencyList[source][i]
            if self.used.count(edge.v) == 0 and edge.c > 0:  # if vertex isn't visited and has capacity > 0
                flow_through = self.pathDFS(edge.v, sink, flow if flow < edge.c else edge.c)
                if flow_through > 0:
                    self.adjacencyList[source][i].c -= flow_through
                    self.adjacencyList[source][i].f += flow_through
                    self.used.clear()
                    return flow_through
        return 0

    # A helper function to find max flow between source and sink
    def findMaxFlow(self):
        if self.source == self.sink:
            raise InvalidGraphException("Error: Source == Sink")
        max_flow = 0
        while True:  # till there's a flow that is > 0
            partial_flow = self.pathDFS(self.source, self.sink, float('inf'))
            if partial_flow == 0:
                return max_flow
            max_flow += partial_flow

    # Prints graph G(V,E)
    def print(self):
        for key, value in self.adjacencyList.items():
            for list_key in range(len(self.adjacencyList[key])):
                edge = self.adjacencyList[key][list_key]
                print("u=",edge.u, "v=",edge.v, "f=",edge.f, "c=",edge.c, "old_cap=", edge.old_cap)


# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# 'License'); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# coding: utf-8

from __future__ import absolute_import
from __future__ import print_function

from collections import Counter
import unittest

from common import assertRaises
from mxnet import ndarray as nd
from mxnet.test_utils import *
from mxnet.text import utils
from mxnet.text.glossary import Glossary
from mxnet.text.indexer import TokenIndexer
from mxnet.text.embedding import TokenEmbedding, CustomEmbedding

def _get_test_str_of_tokens(token_delim, seq_delim):
    seq1 = token_delim + token_delim.join(['Life', 'is', 'great', '!']) \
           + token_delim + seq_delim
    seq2 = token_delim + token_delim.join(['life', 'is', 'good', '.']) \
           + token_delim + seq_delim
    seq3 = token_delim + token_delim.join(['life', "isn't", 'bad', '.']) \
           + token_delim + seq_delim
    seqs = seq1 + seq2 + seq3
    return seqs

def _test_count_tokens_from_str_with_delims(token_delim, seq_delim):
    source_str = _get_test_str_of_tokens(token_delim, seq_delim)

    cnt1 = utils.count_tokens_from_str(source_str, token_delim, seq_delim,
                                       to_lower=False)
    assert cnt1 == Counter(
        {'is': 2, 'life': 2, '.': 2, 'Life': 1, 'great': 1, '!': 1, 'good': 1,
         "isn't": 1, 'bad': 1})

    cnt2 = utils.count_tokens_from_str(source_str, token_delim, seq_delim,
                                       to_lower=True)
    assert cnt2 == Counter(
        {'life': 3, 'is': 2, '.': 2, 'great': 1, '!': 1, 'good': 1,
         "isn't": 1, 'bad': 1})

    counter_to_update = Counter({'life': 2})

    cnt3 = utils.count_tokens_from_str(
        source_str, token_delim, seq_delim, to_lower=False,
        counter_to_update=counter_to_update.copy())
    assert cnt3 == Counter(
        {'is': 2, 'life': 4, '.': 2, 'Life': 1, 'great': 1, '!': 1, 'good': 1,
         "isn't": 1, 'bad': 1})

    cnt4 = utils.count_tokens_from_str(
        source_str, token_delim, seq_delim, to_lower=True,
        counter_to_update=counter_to_update.copy())
    assert cnt4 == Counter(
        {'life': 5, 'is': 2, '.': 2, 'great': 1, '!': 1, 'good': 1,
         "isn't": 1, 'bad': 1})

def test_count_tokens_from_str():
    _test_count_tokens_from_str_with_delims(' ', '\n')
    _test_count_tokens_from_str_with_delims('IS', 'LIFE')

def test_tokens_to_indices():
    counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])

    indexer = TokenIndexer(counter, most_freq_count=None, min_freq=1,
                           unknown_token='<unk>', reserved_tokens=None)

    i1 = indexer.to_indices('c')
    assert i1 == 1

    i2 = indexer.to_indices(['c'])
    assert i2 == [1]

    i3 = indexer.to_indices(['<unk>', 'non-exist'])
    assert i3 == [0, 0]

    i4 = indexer.to_indices(['a', 'non-exist', 'a', 'b'])
    assert i4 == [3, 0, 3, 2]

def test_indices_to_tokens():
    counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])

    indexer = TokenIndexer(counter, most_freq_count=None, min_freq=1,
                           unknown_token='<unknown>', reserved_tokens=None)

    i1 = indexer.to_tokens(1)
    assert i1 == 'c'

    i2 = indexer.to_tokens([1])
    assert i2 == ['c']

    i3 = indexer.to_tokens([0, 0])
    assert i3 == ['<unknown>', '<unknown>']

    i4 = indexer.to_tokens([3, 0, 3, 2])
    assert i4 == ['a', '<unknown>', 'a', 'b']

    assertRaises(ValueError, indexer.to_tokens, 100)

def test_glove():
    glove_6b_50d = TokenEmbedding.create(
        'glove', pretrained_file_name='glove.6B.50d.txt')

    assert len(glove_6b_50d) == 400001
    assert glove_6b_50d.vec_len == 50
    assert glove_6b_50d.token_to_idx['hi'] == 11084
    assert glove_6b_50d.idx_to_token[11084] == 'hi'

    first_vec_sum = glove_6b_50d.idx_to_vec[0].sum().asnumpy()[0]
    assert_almost_equal(first_vec_sum, 0)

    unk_vec_sum = glove_6b_50d.get_vecs_by_tokens(
        '<unk$unk@unk>').sum().asnumpy()[0]
    assert_almost_equal(unk_vec_sum, 0)

    unk_vecs_sum = glove_6b_50d.get_vecs_by_tokens(
        ['<unk$unk@unk>', '<unk$unk@unk>']).sum().asnumpy()[0]
    assert_almost_equal(unk_vecs_sum, 0)

def test_fasttext():
    fasttext_simple = TokenEmbedding.create(
        'fasttext', pretrained_file_name='wiki.simple.vec',
        init_unknown_vec=nd.ones)

    assert len(fasttext_simple) == 111052
    assert fasttext_simple.vec_len == 300
    assert fasttext_simple.token_to_idx['hi'] == 3241
    assert fasttext_simple.idx_to_token[3241] == 'hi'

    first_vec_sum = fasttext_simple.idx_to_vec[0].sum().asnumpy()[0]
    assert_almost_equal(first_vec_sum, fasttext_simple.vec_len)

    unk_vec_sum = fasttext_simple.get_vecs_by_tokens(
        '<unk$unk@unk>').sum().asnumpy()[0]
    assert_almost_equal(unk_vec_sum, fasttext_simple.vec_len)

    unk_vecs_sum = fasttext_simple.get_vecs_by_tokens(
        ['<unk$unk@unk>', '<unk$unk@unk>']).sum().asnumpy()[0]
    assert_almost_equal(unk_vecs_sum, fasttext_simple.vec_len * 2)

def _mk_my_pretrain_file(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.1', '0.2', '0.3', '0.4', '0.5']) + '\n'
    seq2 = token_delim.join(['b', '0.6', '0.7', '0.8', '0.9', '1.0']) + '\n'
    seqs = seq1 + seq2
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def _mk_my_pretrain_file2(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.01', '0.02', '0.03', '0.04',
                             '0.05']) + '\n'
    seq2 = token_delim.join(['c', '0.06', '0.07', '0.08', '0.09', '0.1']) + '\n'
    seqs = seq1 + seq2
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def _mk_my_pretrain_file3(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.1', '0.2', '0.3', '0.4', '0.5']) + '\n'
    seq2 = token_delim.join(['b', '0.6', '0.7', '0.8', '0.9', '1.0']) + '\n'
    seq3 = token_delim.join(['<unk1>', '1.1', '1.2', '1.3', '1.4',
                             '1.5']) + '\n'
    seqs = seq1 + seq2 + seq3
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def _mk_my_pretrain_file4(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.01', '0.02', '0.03', '0.04',
                             '0.05']) + '\n'
    seq2 = token_delim.join(['c', '0.06', '0.07', '0.08', '0.09',
                             '0.1']) + '\n'
    seq3 = token_delim.join(['<unk2>', '0.11', '0.12', '0.13', '0.14',
                             '0.15']) + '\n'
    seqs = seq1 + seq2 + seq3
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def _mk_my_invalid_pretrain_file(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.1', '0.2', '0.3', '0.4', '0.5']) + '\n'
    seq2 = token_delim.join(['b', '0.6', '0.7', '0.8', '0.9', '1.0']) + '\n'
    seq3 = token_delim.join(['c']) + '\n'
    seqs = seq1 + seq2 + seq3
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def _mk_my_invalid_pretrain_file2(path, token_delim, pretrain_file):
    path = os.path.expanduser(path)
    if not os.path.exists(path):
        os.makedirs(path)
    seq1 = token_delim.join(['a', '0.1', '0.2', '0.3', '0.4', '0.5']) + '\n'
    seq2 = token_delim.join(['b', '0.6', '0.7', '0.8', '0.9', '1.0']) + '\n'
    seq3 = token_delim.join(['c', '0.6', '0.7', '0.8']) + '\n'
    seqs = seq1 + seq2 + seq3
    with open(os.path.join(path, pretrain_file), 'w') as fout:
        fout.write(seqs)

def test_custom_embed():
    embed_root = '~/.mxnet/embeddings/'
    embed_name = 'my_embed'
    elem_delim = '/t'
    pretrain_file = 'my_pretrain_file.txt'

    _mk_my_pretrain_file(os.path.join(embed_root, embed_name), elem_delim,
                         pretrain_file)

    pretrain_file_path = os.path.join(embed_root, embed_name, pretrain_file)

    my_embed = CustomEmbedding(pretrain_file_path, elem_delim)

    assert len(my_embed) == 3
    assert my_embed.vec_len == 5
    assert my_embed.token_to_idx['a'] == 1
    assert my_embed.idx_to_token[1] == 'a'

    first_vec = my_embed.idx_to_vec[0]
    assert_almost_equal(first_vec.asnumpy(), np.array([0, 0, 0, 0, 0]))

    unk_vec = my_embed.get_vecs_by_tokens('A')
    assert_almost_equal(unk_vec.asnumpy(), np.array([0, 0, 0, 0, 0]))

    a_vec = my_embed.get_vecs_by_tokens('A', lower_case_backup=True)
    assert_almost_equal(a_vec.asnumpy(), np.array([0.1, 0.2, 0.3, 0.4, 0.5]))

    unk_vecs = my_embed.get_vecs_by_tokens(['<unk$unk@unk>', '<unk$unk@unk>'])
    assert_almost_equal(unk_vecs.asnumpy(),
                        np.array([[0, 0, 0, 0, 0],
                                  [0, 0, 0, 0, 0]]))

    # Test loaded unknown vectors.
    pretrain_file2 = 'my_pretrain_file2.txt'
    _mk_my_pretrain_file3(os.path.join(embed_root, embed_name), elem_delim,
                          pretrain_file2)
    pretrain_file_path = os.path.join(embed_root, embed_name, pretrain_file2)
    my_embed2 = CustomEmbedding(pretrain_file_path, elem_delim,
                                init_unknown_vec=nd.ones,
                                unknown_token='<unk>')
    unk_vec2 = my_embed2.get_vecs_by_tokens('<unk>')
    assert_almost_equal(unk_vec2.asnumpy(), np.array([1, 1, 1, 1, 1]))
    unk_vec2 = my_embed2.get_vecs_by_tokens('<unk$unk@unk>')
    assert_almost_equal(unk_vec2.asnumpy(), np.array([1, 1, 1, 1, 1]))

    my_embed3 = CustomEmbedding(pretrain_file_path, elem_delim,
                                init_unknown_vec=nd.ones,
                                unknown_token='<unk1>')
    unk_vec3 = my_embed3.get_vecs_by_tokens('<unk1>')
    assert_almost_equal(unk_vec3.asnumpy(), np.array([1.1, 1.2, 1.3, 1.4, 1.5]))
    unk_vec3 = my_embed3.get_vecs_by_tokens('<unk$unk@unk>')
    assert_almost_equal(unk_vec3.asnumpy(), np.array([1.1, 1.2, 1.3, 1.4, 1.5]))

    # Test error handling.
    invalid_pretrain_file = 'invalid_pretrain_file.txt'
    _mk_my_invalid_pretrain_file(os.path.join(embed_root, embed_name),
                                 elem_delim, invalid_pretrain_file)
    pretrain_file_path = os.path.join(embed_root, embed_name,
                                      invalid_pretrain_file)
    assertRaises(AssertionError, CustomEmbedding, pretrain_file_path,
                 elem_delim)

    invalid_pretrain_file2 = 'invalid_pretrain_file2.txt'
    _mk_my_invalid_pretrain_file2(os.path.join(embed_root, embed_name),
                                  elem_delim, invalid_pretrain_file2)
    pretrain_file_path = os.path.join(embed_root, embed_name,
                                      invalid_pretrain_file2)
    assertRaises(AssertionError, CustomEmbedding, pretrain_file_path,
                 elem_delim)

def test_token_indexer():
    counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])

    g1 = TokenIndexer(counter, most_freq_count=None, min_freq=1,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g1) == 5
    assert g1.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3,
                               'some_word$': 4}
    assert g1.idx_to_token[1] == 'c'
    assert g1.unknown_token == '<unk>'
    assert g1.reserved_tokens is None

    g2 = TokenIndexer(counter, most_freq_count=None, min_freq=2,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g2) == 3
    assert g2.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2}
    assert g2.idx_to_token[1] == 'c'
    assert g2.unknown_token == '<unk>'
    assert g2.reserved_tokens is None

    g3 = TokenIndexer(counter, most_freq_count=None, min_freq=100,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g3) == 1
    assert g3.token_to_idx == {'<unk>': 0}
    assert g3.idx_to_token[0] == '<unk>'
    assert g3.unknown_token == '<unk>'
    assert g3.reserved_tokens is None

    g4 = TokenIndexer(counter, most_freq_count=2, min_freq=1,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g4) == 3
    assert g4.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2}
    assert g4.idx_to_token[1] == 'c'
    assert g4.unknown_token == '<unk>'
    assert g4.reserved_tokens is None

    g5 = TokenIndexer(counter, most_freq_count=3, min_freq=1,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g5) == 4
    assert g5.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3}
    assert g5.idx_to_token[1] == 'c'
    assert g5.unknown_token == '<unk>'
    assert g5.reserved_tokens is None

    g6 = TokenIndexer(counter, most_freq_count=100, min_freq=1,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g6) == 5
    assert g6.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3,
                               'some_word$': 4}
    assert g6.idx_to_token[1] == 'c'
    assert g6.unknown_token == '<unk>'
    assert g6.reserved_tokens is None

    g7 = TokenIndexer(counter, most_freq_count=1, min_freq=2,
                      unknown_token='<unk>', reserved_tokens=None)
    assert len(g7) == 2
    assert g7.token_to_idx == {'<unk>': 0, 'c': 1}
    assert g7.idx_to_token[1] == 'c'
    assert g7.unknown_token == '<unk>'
    assert g7.reserved_tokens is None

    assertRaises(AssertionError, TokenIndexer, counter, most_freq_count=None,
                 min_freq=0, unknown_token='<unknown>',
                 reserved_tokens=['b'])

    assertRaises(AssertionError, TokenIndexer, counter, most_freq_count=None,
                 min_freq=1, unknown_token='<unknown>',
                 reserved_tokens=['b', 'b'])

    assertRaises(AssertionError, TokenIndexer, counter, most_freq_count=None,
                 min_freq=1, unknown_token='<unknown>',
                 reserved_tokens=['b', '<unknown>'])

    g8 = TokenIndexer(counter, most_freq_count=None, min_freq=1,
                      unknown_token='<unknown>', reserved_tokens=['b'])
    assert len(g8) == 5
    assert g8.token_to_idx == {'<unknown>': 0, 'b': 1, 'c': 2, 'a': 3,
                               'some_word$': 4}
    assert g8.idx_to_token[1] == 'b'
    assert g8.unknown_token == '<unknown>'
    assert g8.reserved_tokens == ['b']

    g9 = TokenIndexer(counter, most_freq_count=None, min_freq=2,
                      unknown_token='<unk>', reserved_tokens=['b', 'a'])
    assert len(g9) == 4
    assert g9.token_to_idx == {'<unk>': 0, 'b': 1, 'a': 2, 'c': 3}
    assert g9.idx_to_token[1] == 'b'
    assert g9.unknown_token == '<unk>'
    assert g9.reserved_tokens == ['b', 'a']

    g10 = TokenIndexer(counter, most_freq_count=None, min_freq=100,
                       unknown_token='<unk>', reserved_tokens=['b', 'c'])
    assert len(g10) == 3
    assert g10.token_to_idx == {'<unk>': 0, 'b': 1, 'c': 2}
    assert g10.idx_to_token[1] == 'b'
    assert g10.unknown_token == '<unk>'
    assert g10.reserved_tokens == ['b', 'c']

    g11 = TokenIndexer(counter, most_freq_count=1, min_freq=2,
                       unknown_token='<unk>', reserved_tokens=['<pad>', 'b'])
    assert len(g11) == 4
    assert g11.token_to_idx == {'<unk>': 0, '<pad>': 1, 'b': 2, 'c': 3}
    assert g11.idx_to_token[1] == '<pad>'
    assert g11.unknown_token == '<unk>'
    assert g11.reserved_tokens == ['<pad>', 'b']

    g12 = TokenIndexer(counter, most_freq_count=None, min_freq=2,
                       unknown_token='b', reserved_tokens=['<pad>'])
    assert len(g12) == 3
    assert g12.token_to_idx == {'b': 0, '<pad>': 1, 'c': 2}
    assert g12.idx_to_token[1] == '<pad>'
    assert g12.unknown_token == 'b'
    assert g12.reserved_tokens == ['<pad>']

    g13 = TokenIndexer(counter, most_freq_count=None, min_freq=2,
                       unknown_token='a', reserved_tokens=['<pad>'])
    assert len(g13) == 4
    assert g13.token_to_idx == {'a': 0, '<pad>': 1, 'c': 2, 'b': 3}
    assert g13.idx_to_token[1] == '<pad>'
    assert g13.unknown_token == 'a'
    assert g13.reserved_tokens == ['<pad>']

    counter_tuple = Counter([('a', 'a'), ('b', 'b'), ('b', 'b'),
                             ('c', 'c'), ('c', 'c'), ('c', 'c'),
                             ('some_word$', 'some_word$')])

    g14 = TokenIndexer(counter_tuple, most_freq_count=None, min_freq=1,
                       unknown_token=('<unk>', '<unk>'), reserved_tokens=None)
    assert len(g14) == 5
    assert g14.token_to_idx == {('<unk>', '<unk>'): 0, ('c', 'c'): 1,
                                ('b', 'b'): 2, ('a', 'a'): 3,
                                ('some_word$', 'some_word$'): 4}
    assert g14.idx_to_token[1] == ('c', 'c')
    assert g14.unknown_token == ('<unk>', '<unk>')
    assert g14.reserved_tokens is None

def test_glossary_with_one_embed():
    embed_root = '~/.mxnet/embeddings/'
    embed_name = 'my_embed'
    elem_delim = '/t'
    pretrain_file = 'my_pretrain_file1.txt'

    _mk_my_pretrain_file(os.path.join(embed_root, embed_name), elem_delim,
                         pretrain_file)

    pretrain_file_path = os.path.join(embed_root, embed_name, pretrain_file)

    my_embed = CustomEmbedding(pretrain_file_path, elem_delim,
                               init_unknown_vec=nd.ones)

    counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])

    g1 = Glossary(counter, my_embed, most_freq_count=None, min_freq=1,
                  unknown_token='<unk>', reserved_tokens=['<pad>'])

    assert g1.token_to_idx == {'<unk>': 0, '<pad>': 1, 'c': 2, 'b': 3, 'a': 4,
                               'some_word$': 5}
    assert g1.idx_to_token == ['<unk>', '<pad>', 'c', 'b', 'a', 'some_word$']

    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [0.6, 0.7, 0.8, 0.9, 1],
                                  [0.1, 0.2, 0.3, 0.4, 0.5],
                                  [1, 1, 1, 1, 1]])
                        )

    assert g1.vec_len == 5
    assert g1.reserved_tokens == ['<pad>']

    assert_almost_equal(g1.get_vecs_by_tokens('c').asnumpy(),
                        np.array([1, 1, 1, 1, 1])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['c']).asnumpy(),
                        np.array([[1, 1, 1, 1, 1]])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['a', 'not_exist']).asnumpy(),
                        np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
                                  [1, 1, 1, 1, 1]])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['a', 'b']).asnumpy(),
                        np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
                                  [0.6, 0.7, 0.8, 0.9, 1]])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['A', 'b']).asnumpy(),
                        np.array([[1, 1, 1, 1, 1],
                                  [0.6, 0.7, 0.8, 0.9, 1]])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['A', 'b'],
                                              lower_case_backup=True).asnumpy(),
                        np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
                                  [0.6, 0.7, 0.8, 0.9, 1]])
                        )

    g1.update_token_vectors(['a', 'b'],
                            nd.array([[2, 2, 2, 2, 2],
                                      [3, 3, 3, 3, 3]])
                            )

    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1]])
                        )

    assertRaises(ValueError, g1.update_token_vectors, 'unknown$$$',
                 nd.array([0, 0, 0, 0, 0]))

    assertRaises(AssertionError, g1.update_token_vectors, '<unk>',
                 nd.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]))

    assertRaises(AssertionError, g1.update_token_vectors, '<unk>',
                 nd.array([0]))

    g1.update_token_vectors(['<unk>'],
                            nd.array([0, 0, 0, 0, 0])
                            )
    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[0, 0, 0, 0, 0],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1]])
                        )
    g1.update_token_vectors(['<unk>'],
                            nd.array([[10, 10, 10, 10, 10]])
                            )
    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[10, 10, 10, 10, 10],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1]])
                        )
    g1.update_token_vectors('<unk>',
                            nd.array([0, 0, 0, 0, 0])
                            )
    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[0, 0, 0, 0, 0],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1]])
                        )
    g1.update_token_vectors('<unk>',
                            nd.array([[10, 10, 10, 10, 10]])
                            )
    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[10, 10, 10, 10, 10],
                                  [1, 1, 1, 1, 1],
                                  [1, 1, 1, 1, 1],
                                  [3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1]])
                        )

def test_glossary_with_two_embeds():
    embed_root = '.'
    embed_name = 'my_embed'
    elem_delim = '/t'
    pretrain_file1 = 'my_pretrain_file1.txt'
    pretrain_file2 = 'my_pretrain_file2.txt'

    _mk_my_pretrain_file(os.path.join(embed_root, embed_name), elem_delim,
                         pretrain_file1)
    _mk_my_pretrain_file2(os.path.join(embed_root, embed_name), elem_delim,
                          pretrain_file2)

    pretrain_file_path1 = os.path.join(embed_root, embed_name, pretrain_file1)
    pretrain_file_path2 = os.path.join(embed_root, embed_name, pretrain_file2)

    my_embed1 = CustomEmbedding(pretrain_file_path1, elem_delim,
                                init_unknown_vec=nd.ones)
    my_embed2 = CustomEmbedding(pretrain_file_path2, elem_delim)

    counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])

    g1 = Glossary(counter, [my_embed1, my_embed2], most_freq_count=None,
                  min_freq=1, unknown_token='<unk>', reserved_tokens=None)

    assert g1.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3,
                               'some_word$': 4}
    assert g1.idx_to_token == ['<unk>', 'c', 'b', 'a', 'some_word$']

    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
                                  [1, 1, 1, 1, 1, 0.06, 0.07, 0.08, 0.09, 0.1],
                                  [0.6, 0.7, 0.8, 0.9, 1, 0, 0, 0, 0, 0],
                                  [0.1, 0.2, 0.3, 0.4, 0.5,
                                   0.01, 0.02, 0.03, 0.04, 0.05],
                                  [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]])
                        )

    assert g1.vec_len == 10
    assert g1.reserved_tokens is None
    assert_almost_equal(g1.get_vecs_by_tokens('c').asnumpy(),
                        np.array([1, 1, 1, 1, 1, 0.06, 0.07, 0.08, 0.09, 0.1])
                        )

    assert_almost_equal(g1.get_vecs_by_tokens(['b', 'not_exist']).asnumpy(),
                        np.array([[0.6, 0.7, 0.8, 0.9, 1, 0, 0, 0, 0, 0],
                                  [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]])
                        )

    g1.update_token_vectors(['a', 'b'],
                            nd.array([[2, 2, 2, 2, 2, 2, 2, 2, 2, 2],
                                      [3, 3, 3, 3, 3, 3, 3, 3, 3, 3]])
                            )
    assert_almost_equal(g1.idx_to_vec.asnumpy(),
                        np.array([[1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
                                  [1, 1, 1, 1, 1, 0.06, 0.07, 0.08, 0.09, 0.1],
                                  [3, 3, 3, 3, 3, 3, 3, 3, 3, 3],
                                  [2, 2, 2, 2, 2, 2, 2, 2, 2, 2],
                                  [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]])
                        )

    # Test loaded unknown tokens
    pretrain_file3 = 'my_pretrain_file3.txt'
    pretrain_file4 = 'my_pretrain_file4.txt'

    _mk_my_pretrain_file3(os.path.join(embed_root, embed_name), elem_delim,
                          pretrain_file3)
    _mk_my_pretrain_file4(os.path.join(embed_root, embed_name), elem_delim,
                          pretrain_file4)

    pretrain_file_path3 = os.path.join(embed_root, embed_name, pretrain_file3)
    pretrain_file_path4 = os.path.join(embed_root, embed_name, pretrain_file4)

    my_embed3 = CustomEmbedding(pretrain_file_path3, elem_delim,
                                init_unknown_vec=nd.ones,
                                unknown_token='<unk1>')
    my_embed4 = CustomEmbedding(pretrain_file_path4, elem_delim,
                                unknown_token='<unk2>')

    g2 = Glossary(counter, [my_embed3, my_embed4], most_freq_count=None,
                  min_freq=1, unknown_token='<unk>', reserved_tokens=None)
    assert_almost_equal(g2.idx_to_vec.asnumpy(),
                        np.array([[1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.06, 0.07, 0.08, 0.09, 0.1],
                                  [0.6, 0.7, 0.8, 0.9, 1,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [0.1, 0.2, 0.3, 0.4, 0.5,
                                   0.01, 0.02, 0.03, 0.04, 0.05],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15]])
                        )

    g3 = Glossary(counter, [my_embed3, my_embed4], most_freq_count=None,
                  min_freq=1, unknown_token='<unk1>', reserved_tokens=None)
    assert_almost_equal(g3.idx_to_vec.asnumpy(),
                        np.array([[1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.06, 0.07, 0.08, 0.09, 0.1],
                                  [0.6, 0.7, 0.8, 0.9, 1,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [0.1, 0.2, 0.3, 0.4, 0.5,
                                   0.01, 0.02, 0.03, 0.04, 0.05],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15]])
                        )

    g4 = Glossary(counter, [my_embed3, my_embed4], most_freq_count=None,
                  min_freq=1, unknown_token='<unk2>', reserved_tokens=None)
    assert_almost_equal(g4.idx_to_vec.asnumpy(),
                        np.array([[1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.06, 0.07, 0.08, 0.09, 0.1],
                                  [0.6, 0.7, 0.8, 0.9, 1,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [0.1, 0.2, 0.3, 0.4, 0.5,
                                   0.01, 0.02, 0.03, 0.04, 0.05],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15]])
                        )

    counter2 = Counter(['b', 'b', 'c', 'c', 'c', 'some_word$'])

    g5 = Glossary(counter2, [my_embed3, my_embed4], most_freq_count=None,
                  min_freq=1, unknown_token='a', reserved_tokens=None)
    assert g5.token_to_idx == {'a': 0, 'c': 1, 'b': 2, 'some_word$': 3}
    assert g5.idx_to_token == ['a', 'c', 'b', 'some_word$']
    assert_almost_equal(g5.idx_to_vec.asnumpy(),
                        np.array([[1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.06, 0.07, 0.08, 0.09, 0.1],
                                  [0.6, 0.7, 0.8, 0.9, 1,
                                   0.11, 0.12, 0.13, 0.14, 0.15],
                                  [1.1, 1.2, 1.3, 1.4, 1.5,
                                   0.11, 0.12, 0.13, 0.14, 0.15]])
                        )

def test_get_embedding_names_and_pretrain_files():
    assert len(TokenEmbedding.get_embedding_and_pretrained_file_names(
        embedding_name='fasttext')) == 3

    assert len(TokenEmbedding.get_embedding_and_pretrained_file_names(
        embedding_name='glove')) == 10

    reg = TokenEmbedding.get_embedding_and_pretrained_file_names(
        embedding_name=None)

    assert len(reg['glove']) == 10
    assert len(reg['fasttext']) == 3

    assertRaises(KeyError,
                 TokenEmbedding.get_embedding_and_pretrained_file_names,
                 'unknown$$')

if __name__ == '__main__':
    import nose
    nose.runmodule()


import sys
sys.path.append('/home/cc/pytweet/analysis')
import tensorflow as tf
import gensim
import numpy as np
from gensim.models.keyedvectors import KeyedVectors
from nltk.tokenize import TweetTokenizer
from processing import pad
from processing import read_subj

from graph_builders import build_logreg

SEQ_LENGTH=16
STATE_SIZE=256
BATCH_SIZE=1
NUM_ITER = int(1e8)
NUM_CLASSES = 2

def read_csv(data_path):
  f = open(data_path, "r")
  lines = f.readlines()
  tweet_sen_tup = [(line.split(',')[0], line.split(',')[5]) for line in lines]
  return tweet_sen_tup

def string_to_vec(str_list, wv):

  result = [wv[s] for s in str_list if s in wv.vocab]
  if len(result) > 0:
    return np.concatenate(result)

def main(argv):

  data = read_subj(argv[0], argv[1])
  tknzr = TweetTokenizer(strip_handles=True)
  cleaned_data = []
#  cleaned_data = [(sent, tknzr.tokenize(tweet)) for sent, tweet in data]
  for sent, tweet in data:
    try:
      tknzd = tknzr.tokenize(tweet)
      cleaned_data.append((sent, tknzd))
    except:
      pass
  print("NUM_PASSED:", len(data) - len(cleaned_data), "TOTAL:", len(data))
  word_vectors = KeyedVectors.load_word2vec_format(argv[2], binary=True)
  # convert to word vectors
  num_els_1 = sum([len(s[1]) for s in cleaned_data])

  sent_vec = [(sent,string_to_vec(s, word_vectors)) for sent, s in cleaned_data]
  sent_vec = [s for s in sent_vec if s[1] is not None]
  num_els = sum([len(s[1]) for s in sent_vec])

  print(num_els/float(len(sent_vec)), num_els_1)
  print(len(word_vectors['computer']), len(word_vectors['pizza']))
  padded_vecs = pad(SEQ_LENGTH, [s[1] for s in sent_vec], 300)
 # x_plhr, y_plhr, loss, train_op = build_rnn(SEQ_LENGTH, BATCH_SIZE, STATE_SIZE, 300)
  x_plhr, y_plhr, loss, train_op = build_logreg(SEQ_LENGTH, BATCH_SIZE, STATE_SIZE, 300, NUM_CLASSES)

  sentiments = np.array([int(s[0]) for s in sent_vec]).reshape((len(sent_vec), 1)).astype(np.int32)

  with tf.Session() as sess:
    num_samples = len(padded_vecs)
    sess.run(tf.global_variables_initializer())
    saver = tf.train.Saver()
    for i in range(NUM_ITER):
      samples = np.random.choice(num_samples, BATCH_SIZE)
      fd = {}
      fd[x_plhr] = padded_vecs[samples]
      one_hot = np.zeros([NUM_CLASSES], dtype=np.float32)

 #     fd[y_plhr] = sentiments[samples]
      one_hot[sentiments[samples][0]] = 1.0
      fd[y_plhr] = one_hot.reshape(1, -1)
      loss_curr, _ = sess.run([loss, train_op], feed_dict=fd)
      if not i %10000:
        saver.save(sess, argv[3], global_step = i)
      if not i % 1000:
        print(loss_curr)

if __name__ == "__main__":
  main(sys.argv[1:])


import pygame
import sys
import Draw
import Controller

pygame.init()

WIDTH = 1000
HEIGHT = 1000
BACKGROUND = (255, 255, 255)

screen = pygame.display.set_mode((WIDTH, HEIGHT))
screen.fill(BACKGROUND)
pygame.display.set_caption("Super TicTacToeV2")

Draw.draw_board(screen)
controller = Controller.Controller(screen)
while True:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()
        if event.type == pygame.MOUSEBUTTONDOWN:
            controller.controller(event.pos)

    pygame.display.update()