# -*- coding: utf-8 -*-
from __future__ import absolute_import
import os
import sys
import logging
from datetime import datetime, timedelta

# add up one level dir into sys path
sys.path.append(os.path.abspath(os.path.dirname(os.path.dirname(__file__))))
os.environ['DJANGO_SETTINGS_MODULE'] = 'luckyplatform.settings'

from luckycommon.db.account import get_account
from luckycommon.account.model.account import Account
from luckycommon.stats import MG as mg
from luckycommon.db.missed_vips import batch_insert_missed_vips
from luckycommon.model.missed_vips import MissedVips
from luckycommon.model import orm
from luckycommon.db import account as account_db
from luckycommon.utils.tz import utc_to_local_str

_LOGGER = logging.getLogger(__name__)

missed_users = {
    1: set(),
    2: set(),
    3: set()
}
for k in (1, 2, 3):
    items = orm.session.query(MissedVips.uid).filter(
        MissedVips.type == k).all()
    for item in items:
        missed_users[k].add(item[0])

missed_users[1] -= missed_users[2]
missed_users[2] -= missed_users[3]

all_missed = missed_users[1] | missed_users[2] | missed_users[3]

def dump_vip_users(lost_days=3, missed_times=0):
    now = datetime.now()
    start_date = now - timedelta(days=lost_days)
    start_date = start_date.replace(hour=16, minute=0, second=0, microsecond=0)
    end_date = start_date + timedelta(days=1)

    filters = {'updated_at': {'$lte': end_date, '$gte': start_date},
               'recharge.total': {'$gt': 100}}

    if missed_times == 0:
        filters['_id'] = {'$nin': list(all_missed)}
    else:
        filters['_id'] = {'$in': list(missed_users[missed_times])}

    user_stats = mg.user_stats.find(filters).sort(
        'recharge.total', -1).limit(100)
    return user_stats

def dump_new_users():
    now = datetime.now()
    updated_at_upper = now - timedelta(days=3)
    created_at_lower = now - timedelta(days=10)
    created_at_lower = created_at_lower.replace(
        hour=16, minute=0, second=0, microsecond=0)
    created_at_upper = created_at_lower + timedelta(days=1)
    filters = {
        'created_at': {'$gte': created_at_lower, '$lt': created_at_upper},
        'recharge.total': {'$gt': 0, '$lte': 100},
        'updated_at': {'$lt': updated_at_upper}}
    user_stats = mg.user_stats.find(filters).sort(
        'recharge.total', -1)
    insert_list = []
    forbidden_users = orm.session.query(Account.id).filter(
        Account.status != 0).all()
    forbidden_users = [x[0] for x in forbidden_users]
    forbidden_users = set(forbidden_users)
    for user_stat in user_stats:
        if user_stat['_id'] not in forbidden_users:
            insert_list.append(user_stat)
    return insert_list

def get_insert_list(user_stats, lost_days, type, user_type=0):
    insert_list = []
    for user_stat in user_stats:
        user_id = int(user_stat['_id'])
        register_chn = user_stat.get('chn') or u'未知'
        created_at = user_stat.get('created_at')
        if not created_at:
            created_at = get_account(user_id).created_at
        created_at = utc_to_local_str(created_at)
        updated_at = utc_to_local_str(user_stat['updated_at'])
        recharge_amount = int(user_stat['recharge']['total'])
        pay_count = user_stat.get('pay', {}).get('count', 0)
        if 'win' in user_stat:
            win_count = user_stat['win']['count']
            win_amount = user_stat['win']['total']
        else:
            win_count = 0
            win_amount = 0
        active_count = mg.daily_stats.aggregate([
            {"$match": {"user_id": user_id}},
            {"$group": {"_id": None, "count": {"$sum": 1}}}
        ])
        active_days = active_count.next().get(
            'count', 0) if active_count.alive else 0
        account = account_db.get_account(user_id)
        user_info = {
            'uid': user_id,
            'nick_name': account.nick_name,
            'phone': account.phone[2:],
            'chn': register_chn,
            'type': type,
            'user_type': user_type,
            'created_time': created_at,
            'active_days': active_days,
            'updated_time': updated_at,
            'lost_days': lost_days,
            'recharge_amount': recharge_amount,
            'pay_count': pay_count,
            'win_count': win_count,
            'win_amount': win_amount
        }
        insert_list.append(user_info)
    return insert_list

try:
    for k in (3, 7, 14):
        user_stats = dump_vip_users(k, 0)
        insert_list = get_insert_list(user_stats, k, 1)
        batch_insert_missed_vips(insert_list)
    user_stats = dump_vip_users(7, 1)
    insert_list = get_insert_list(user_stats, 7, 2)
    batch_insert_missed_vips(insert_list)
    user_stats = dump_vip_users(30, 2)
    insert_list = get_insert_list(user_stats, 30, 3)
    batch_insert_missed_vips(insert_list)

    user_stats = dump_new_users()
    insert_list = get_insert_list(user_stats, 3, 1, 1)
    batch_insert_missed_vips(insert_list)
except Exception as e:
    _LOGGER.exception(e)


#!/usr/bin/env python3

###
# PASA Confidentiality Notice:
# This source code and information contained herewith may be legally privileged and confidential
# Any dissemination, distribution or copy of this source code is strictly prohibited.
#
# Copyright (C) 2019, Panasonic Automotive Systems Company of America
#                     All Rights Reserved
#
#
# @file: list_ssm_parameters.py
#
# @author: Panasonic, developer
#
##

import re
import asyncio
import boto3
import argparse

def get_args():
    parser = argparse.ArgumentParser(description='List AWS SSM parameters with given prefix.')
    parser.add_argument(
        '--prefix', required=True, help='Parameter name prefix')
    parser.add_argument(
        '--noop', action="store_true", help='No operations mode')
    parser.add_argument(
        '--decrypt', action="store_true", help='Decrypt secure values')
    args = parser.parse_args()
    return args

def describe_parameters(client, prefix):
    parameters = []
    response = client.describe_parameters(
        Filters=[
            {
                'Key': 'Name',
                'Values': [
                    prefix,
                ]
            },
        ],
        MaxResults=50
    )
    parameters += response["Parameters"]
    while response.get("NextToken", False):
        response = client.describe_parameters(
            Filters=[
                {
                    'Key': 'Name',
                    'Values': [
                        prefix,
                    ]
                },
            ],
            MaxResults=50,
            NextToken=response["NextToken"]
        )
        parameters += response["Parameters"]
    return parameters

def get_parameter(client, p, noop=False, decrypt=False):
    if noop:
        print("Trying to get {}".format(p["Name"]))
    else:
        r = client.get_parameters(Names=[p["Name"]], WithDecryption=decrypt)
        value = r["Parameters"][0]["Value"]
        if "\n" in r["Parameters"][0]["Value"]:
            value = "{0} // Value is truncated".format(value.split("\n")[0])
        print("{0}={1}".format(r["Parameters"][0]["Name"], value))

async def get_parameters(client, parameters, noop=False, decrypt=False):
    loop = asyncio.get_event_loop()
    properties = []
    for p in parameters:
        loop.run_in_executor(None, get_parameter, client, p, noop, decrypt)

def main():
    args = get_args()
    client = boto3.client('ssm')
    parameters = describe_parameters(client, args.prefix)
    loop = asyncio.get_event_loop()
    loop.run_until_complete(get_parameters(client, parameters, noop=args.noop, decrypt=args.decrypt))

if __name__ == "__main__":
    main()


# Brain Tumor Classification
# Script for None Preprocessing
# Author: Qixun Qu
# Create on: 2017/11/28
# Modify on: 2017/11/28

#     ,,,         ,,,
#   ;"   ';     ;'   ",
#   ;  @.ss$$$$$$s.@  ;
#   `s$$$$$$$$$$$$$$$'
#   $$$$$$$$$$$$$$$$$$
#  $$$$P""Y$$$Y""W$$$$$
#  $$$$  p"$$$"q  $$$$$
#  $$$$  .$$$$$.  $$$$'
#   $$$DaU$$O$$DaU$$$'
#    '$$$$'.^.'$$$$'
#       '&$$$$$&'

'''

Class BTCNonePreprocess

'''

import os
import numpy as np
import nibabel as nib
from btc_settings import *
from multiprocessing import Pool, cpu_count

# Helper function to do multiprocessing of
# BTCNonePreprocess._merge_to_one_volume
def unwrap_merge_to_one_volume(arg, **kwarg):
    return BTCNonePreprocess._merge_to_one_volume(*arg, **kwarg)

class BTCNonePreprocess():

    def __init__(self, input_dir, output_dir):

        if not os.path.isdir(input_dir):
            raise IOError("Cannot find input directory.")

        if not os.path.isdir(output_dir):
            os.makedirs(output_dir)

        self.volume_no = os.listdir(input_dir)
        self.full_folder = os.path.join(output_dir, FULL_FOLDER)
        self.mask_folder = os.path.join(output_dir, MASK_FOLDER)

        self._create_folders()
        self._merge_to_one_volume_multi(input_dir, output_dir)

        return

    def _merge_to_one_volume_multi(self, input_dir, output_dir):
        '''_MERGE_TO_ONE_VOLUME_MULTI

            Main function of merging four types volumes and saving outputs
            to map tasks on different cpus to accelerate processing speed.
            The number of subprocesses equals to the number of cpus.

            Inputs:
            -------
            - input_dir: path of the directory which keeps mask volumes
            - output_dir: path of directory which keeps the outputs

        '''

        print("\nMerge flair, t1, t1Gd and t2 into One Volume\n")
        volume_no_len = len(self.volume_no)
        paras = zip([self] * volume_no_len,
                    [input_dir] * volume_no_len,
                    [output_dir] * volume_no_len,
                    self.volume_no)
        pool = Pool(processes=cpu_count())
        pool.map(unwrap_merge_to_one_volume, paras)

        return

    def _create_folders(self):
        '''_CREATE_FOLDERS

            Create folders for temporary files and outputs.
            All folders are as below.

            Folders for outputs:
            ----- output_dir
              |----- full
              |----- mask

        '''

        if not os.path.isdir(self.mask_folder):
            os.makedirs(self.mask_folder)

        if not os.path.isdir(self.full_folder):
            os.makedirs(self.full_folder)

        return

    def _merge_to_one_volume(self, input_dir, output_dir, vno):
        '''_MERGE_TO_ONE_VOLUME

            Merge normalized flair, t1, t1Gd and t2 volumes of one patient
            to one volume. Remove surrounding backgrounds, and save output
            into output folder as the result of preprocessing.

            Inputs:
            -------
            - input_dir: path of the directory which keeps mask volumes
            - output_dir: path of directory which keeps the outputs
            - vno: serial number of volumes, which is also the folder name
                   of one patient's volumes

        '''

        print("NO." + vno + ": Save brain volume and mask volume")
        full_volume = np.zeros(FULL_SHAPE)
        volume_folder = os.path.join(input_dir, vno)
        for file in os.listdir(volume_folder):
            for i in range(len(VOLUME_TYPES)):
                if VOLUME_TYPES[i] in file:
                    volume_path = os.path.join(volume_folder, file)
                    volume = nib.load(volume_path).get_data()
                    volume = np.rot90(volume, 3, axes=(0, 1))
                    # full_volume[..., 0] <== flair volume
                    # full_volume[..., 1] <== t1 volume
                    # full_volume[..., 2] <== t1Gd volume
                    # full_volume[..., 3] <== t2 volume
                    full_volume[..., i] = volume
            if MASK_NAME in file:
                # Load relevant mask
                mask_path = os.path.join(volume_folder, file)
                mask_volume = nib.load(mask_path).get_data()
                mask_volume = np.rot90(mask_volume, 3, axes=(0, 1))

        # Remove surrounding backgrounds from ensemble volume and mask volume
        full_volume, mask_volume = self._keep_minimum_volume(full_volume, mask_volume)

        # Save volume into output folders
        full_volume_path = os.path.join(self.full_folder, vno + TARGET_EXTENSION)
        mask_volume_path = os.path.join(self.mask_folder, vno + TARGET_EXTENSION)

        np.save(full_volume_path, full_volume)
        np.save(mask_volume_path, mask_volume)

        return

    def _keep_minimum_volume(self, full, mask):
        '''_KEEP_MINIMUM_VOLUME

            Remove surrounding backgrounds from ensemble volume
            and mask volume to keep the minimum volume.
            Based on input volumes, compute the range of indices
            of three axes, extract sub-volume and return it back.

            Inputs:
            -------
            - full: ensemble volume
            - mask: relevant mask volume

            Outputs:
            --------
            - new_full: new ensemble volume after being processed
            - new_mask: new mask volume after being processed

        '''

        # Function to extract sub-array from given array
        # according to ranges of indices of three axes
        def sub_array(arr, index_begin, index_end):
            return arr[index_begin[0]:index_end[0],
                       index_begin[1]:index_end[1],
                       index_begin[2]:index_end[2]]

        # Compute background value of volume
        full_sum = np.sum(full, axis=3)
        min_full_sum = np.min(full_sum)

        # Compute range of indices of each axes
        non_bg_index = np.where(full_sum > min_full_sum)
        dims_begin = [np.min(nzi) for nzi in non_bg_index]
        dims_end = [np.max(nzi) + 1 for nzi in non_bg_index]

        # Add a bit more space around the minimum brain volume
        for i in range(len(dims_begin)):
            dims_begin[i] = dims_begin[i] - EDGE_SPACE
            # if the beginning index is lower than 0
            if dims_begin[i] < 0:
                dims_begin[i] = 0
            dims_end[i] = dims_end[i] + EDGE_SPACE
            # if the ending index is larger than the maximum index
            if dims_end[i] > BRAIN_SHAPE[i] - 1:
                dims_end[i] = BRAIN_SHAPE[i] - 1

        # Obtain sub-volumes from input volumes
        new_full = sub_array(full, dims_begin, dims_end)
        new_full = new_full.astype(np.float32)
        new_mask = sub_array(mask, dims_begin, dims_end)

        return new_full, new_mask

if __name__ == "__main__":

    parent_dir = os.path.dirname(os.getcwd())
    input_dir = os.path.join(parent_dir, DATA_FOLDER, ORIGINAL_FOLDER)
    output_dir = os.path.join(parent_dir, DATA_FOLDER, NONEPREPROCESSED_FOLDER)

    BTCNonePreprocess(input_dir, output_dir)


import os, requests, json, dateparser, time, yaml, boto3, slack, random
from typing import List
from pathlib import Path
from pprint import pprint
from traceback import format_exc

from missing_data import get_missing_responses, get_missing_users

BATCH_SIZE = 500

wootric_session = requests.Session()
ACCESS_TOKEN = ''
CLIENT_ID = os.getenv('WOOTRIC_CLIENT_ID')
CLIENT_SECRET = os.getenv('WOOTRIC_CLIENT_SECRET')
BASE_URL = 'https://api.wootric.com'

stitch_session = requests.Session()
STITCH_CLIENT_ID = os.getenv('STITCH_CLIENT_ID')
STITCH_TOKEN = os.getenv('STITCH_TOKEN')
STITCH_BASE_URL = 'https://api.stitchdata.com'
stitch_session.headers = {'Authorization': f'Bearer {STITCH_TOKEN}', 'Content-Type': 'application/json'}

BUCKET = os.getenv('AWS_BUCKET')

os.environ['AWS_ACCESS_KEY_ID'] = os.getenv('AWS_ACCESS_KEY_ID_', os.getenv('AWS_ACCESS_KEY_ID')) # lambda doesn't allow this reserved var
os.environ['AWS_SECRET_ACCESS_KEY'] = os.getenv('AWS_SECRET_ACCESS_KEY_', os.getenv('AWS_SECRET_ACCESS_KEY')) # lambda doesn't allow this reserved var
os.environ['AWS_SESSION_TOKEN'] = '' # lambda provides this reserved var during execution, need to set blank

STATE_KEY = 'wootric.state.json'

s3 = boto3.resource("s3").Bucket(BUCKET)

slack_client = slack.WebhookClient(url=os.getenv('SLACK_WH_TOKEN'))

# init state
state = dict(
  end_users=1420070400,
  responses=1420070400,
  declines=1420070400,
)

def get_access_token():
  global ACCESS_TOKEN
  url = f'{BASE_URL}/oauth/token'
  payload = dict(
    grant_type='client_credentials',
    client_id=CLIENT_ID,
    client_secret=CLIENT_SECRET,
  )
  resp = wootric_session.post(url, payload)
  data : dict = resp.json()
  ACCESS_TOKEN = data.get('access_token')
  if not ACCESS_TOKEN:
    raise Exception('did not find access_token')
  wootric_session.headers = dict(Authorization=f'Bearer {ACCESS_TOKEN}')

def wootric_response(user_id: str, response_id: str):
  url = f'{BASE_URL}/v1/end_users/{user_id}/responses/{response_id}'
  print(url)
  resp = wootric_session.get(url)
  return resp.json()

def wootric_user(user_id: str):
  url = f'{BASE_URL}/v1/end_users/{user_id}'
  print(url)
  resp = wootric_session.get(url)
  return resp.json()

def wootric_request(object_name: str, date_key: str, **params):
  url = f'{BASE_URL}/v1/{object_name}'
  req = requests.models.PreparedRequest()
  date_val = state[object_name]

  # put random limit because seems some get missed. an attempt to randomize sort anchors
  limit = random.randint(5,29)

  params[f"{date_key.replace('_at', '')}[gte]"] = date_val - 1
  page = 0

  all = []
  while True:
    page += 1
    if page > limit: break
    params['page'] = page
    req.prepare_url(url, params)
    print(req.url)
    try:
      resp = wootric_session.get(req.url)
      if resp is None:
        raise Exception(f'Response is for: {req.url}')
      elif not resp.ok:
        raise Exception(f'\n\nHTTP Status Code {resp.status_code} for {req.url}: \n{resp.text}')
    except Exception:
      raise Exception(f'Error for {req.url}.\n\n{format_exc()}')
    data = resp.json()
    if len(data) == 0:
      break
    all += data

  return all

def send_batch(object_name: str, schema: dict, keys: List[str], records: List[dict]):

  is_datetime = lambda k: schema['properties'].get(k, {}).get('format') == 'date-time'

  messages = []
  for record in records:
    rec = dict(
      action='upsert',
      sequence=int(time.time_ns() / 1000),
      data={},
    )
    for k, v in record.items():
      k = k.replace('.', '_')
      v = (dateparser.parse(str(v))).isoformat() if v and is_datetime(k) else v
      rec['data'][k] = v
    messages.append(rec)

  payload = dict(
    table_name=object_name,
    key_names=keys,
    schema=schema,
    messages=messages,
  )

  # with open('payload.json', 'w') as file:
  #   json.dump(payload, file)

  url = f'{STITCH_BASE_URL}/v2/import/batch'
  resp = stitch_session.post(url, json.dumps(payload))

  data : dict = resp.json()
  print(data)

  status = data.get('status')
  if status != 'OK':
    pprint(dict(status_code=resp.status_code))
    resp.raise_for_status()
  else:
    print(f'pushed {len(records)} records to "{object_name}"')

def load_state():
  global state
  state = json.loads(s3.Object(key=STATE_KEY).get()["Body"].read().decode('utf-8'))

  # re-run for past 3 days, an attempt to fill in any holes
  for k in state:
    state[k] = state.get(k, 1420070400) - 3*24*60*60

def save_state():
  global state
  s3.Object(key=STATE_KEY).put(Body=json.dumps(state))
  print(json.dumps(state))

def run(event, context):
  global state

  try:
    # load wootric access token
    get_access_token()

    # load state
    load_state()

  except Exception as E:
    slack_client.send(text=f"Error occurred for Wootric-Stitch Integration:\n{format_exc()}")
    raise E

  config_file = Path('config.yaml')
  with config_file.open() as file:
    object_configs = yaml.load(file)

  errors = []
  for object_name, object_config in object_configs.items():
    records : List[dict] = []

    try:
      print(f'Loading {object_name}')
      while True:
        date_key = object_config['date_key']
        params = object_config['params']
        schema = object_config['schema']
        keys = object_config['keys']

        data : List[dict] = wootric_request(object_name, date_key, **params)

        if len(data) == 0:
          if len(records) == 0:
            break
        else:
          records += data

        send_batch(object_name, schema, keys, records)

        record = records[-1]
        if date_key not in record:
          raise Exception(f'no datekey: {date_key}')

        records = []

        date_val = dateparser.parse(record[date_key])
        ts_val = int(date_val.timestamp())
        if date_val and ts_val > state[object_name]:
          state[object_name] = ts_val
          save_state()
        else:
          break

    except Exception as E:
      errors.append(format_exc())
    finally:
      save_state()

  # Missing users START
  # seems users are missing even with using gte. Gets the IDs from database
  try:
    users = []
    for row in get_missing_users():
      users += [wootric_user(row.end_user_id)]

    response_config = object_configs.get('end_users')
    send_batch('end_users', response_config['schema'], response_config['keys'], users)
  except Exception as E:
      errors.append(format_exc())
  # Missing users END

  # Missing responses START
  # seems some responses are missing even with using gte. Gets the IDs from database
  try:
    responses = []
    for row in get_missing_responses():
      responses += [wootric_response(row.user_id, row.last_response__id)]

    response_config = object_configs.get('responses')
    send_batch('responses', response_config['schema'], response_config['keys'], responses)
  except Exception as E:
      errors.append(format_exc())
  # Missing responses END

  if len(errors) > 0:
    e = '\n\n'.join(errors)
    slack_client.send(text=f'Error occurred for Wootric-Stitch Integration:\n{e}')
    raise Exception(e)

# run(None, None)


import tensorflow as tf
import numpy as np
from keras.utils import np_utils
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import Dense

# 데이터 처리
xy = np.loadtxt('./data/data-04-zoo.csv',encoding='utf-8',delimiter=',')

x_data = xy[:, 0:-1]
y_data = xy[:,-1:]
y_data = np_utils.to_categorical(y_data)

x_train , x_test, y_train , y_test = train_test_split(x_data, y_data, test_size=0.2, random_state= 66)

print(x_train.shape,y_train.shape) #(80,16),(80,7)
print(x_test.shape,y_test.shape) #(21,7),(21,7)

# 모델 생성
model = Sequential()
model.add(Dense(7 , activation='softmax', input_shape = (16,)))

# 훈련
model.compile(optimizer='adadelta',
                loss='categorical_crossentropy',
                metrics=['accuracy'])

model.fit(x_train,y_train, validation_data=(x_test,y_test), epochs=500, batch_size=4)

# 검증
loss, acc = model.evaluate(x_test, y_test, batch_size=3)
print('acc :', acc)


import webapp2
import logging
import urllib
from webapp2_extras import jinja2

# this library is for decoding json responses
from webapp2_extras import json

# BaseHandler subclasses RequestHandler so that we can use jinja
class BaseHandler(webapp2.RequestHandler):

    @webapp2.cached_property
    def jinja2(self):
        # Returns a Jinja2 renderer cached in the app registry.
        return jinja2.get_jinja2(app=self.app)

        # This will call self.response.write using the specified template and context.
        # The first argument should be a string naming the template file to be used.
        # The second argument should be a pointer to an array of context variables
        #  that can be used for substitutions within the template
    def render_response(self, _template, **context):
        # Renders a template and writes the result to the response.
        rv = self.jinja2.render_template(_template, **context)
        self.response.write(rv)

class Viz3(BaseHandler):
	def get(self):
		# logic for map vizualizaiton
		try:
			mapyear = int(self.request.get('mapyear'))
		except ValueError:
			mapyear = 0

		if not mapyear:
			mapyear = 0

		logging.info(mapyear)
		state_num = []
		try:
			fp = open("data/state_numbers.txt")
			lines = fp.readlines()
			line = lines[mapyear]
			line = line.split(":")
			line = line[:len(line)-1]
			state_nums = line
		except IOError:
			logging.info("failed to load file")

		if mapyear == 0:
			mapyear = '2014'
		elif mapyear == 1:
			mapyear = '2013'
		elif mapyear == 2:
			mapyear = '2012'
		elif mapyear == 3:
			mapyear = '2011'
		else:
			mapyear = '2010'

		variables = {'state_nums': state_nums,
        			 'map_year': mapyear}

		self.render_response('viz3.html', **variables)

class MainPage(BaseHandler):

	def get(self):
		#logging.info(results)
		# add it to the context being passed to jinja
		try:
			year = int(self.request.get('year'))
		except ValueError:
			year = 0

		if not year:
			year = 0

		data = []
		countries = []
		should_read = False
		try:
			fp = open("data/studies_cleaned.txt")
			for line in fp:
				if (line[0] == str(year)):
					should_read = True
					continue
				elif (len(line) == 2):
					should_read = False
				if (should_read):
					data += [line.split(",")]
			fp.close()
		except IOError:
			logging.info("failed to load file")

		for i in xrange(0,len(data)):
			countries += [data[i][0]]
			data[i] = data[i][1:len(data[i])]
			for j in xrange(0,len(data[i])):
				data[i][j] = float(data[i][j].strip("\n"))

		majors = ['Business/Mgmt.', 'Education', 'Engineering', 'Fine/Applied Arts', 'Health Professions', 'Humanities', 'Intensive English', 'Math/Computer Science', 'Physical/Life Sciences', 'Social Sciences', 'Other', 'Undeclared']
		variables = {'countries': countries,
					 'majors': majors,
				     'percentages': data,
					 'current_year': year}

		self.render_response('viz2.html', **variables)

application = webapp2.WSGIApplication([
    ('/visualization2/', MainPage),
	('/visualization3/', Viz3),
], debug=True)


# -*- coding: utf-8 -*-

from base.model import BriefMusicModel, BriefArtistModel, BriefAlbumModel, ArtistDetailModel, AlbumDetailModel, MusicModel, ArtistModel, AlbumModel
from api import NetEaseApi
from PyQt5.QtNetwork import *
from PyQt5.QtCore import *

class SearchDao(object):

    def search(text):
        api = NetEaseApi()
        keys = ['song', 'artist', 'album', 'playlist', 'user'] # 搜索歌曲，专辑，用户等的结果
        res = {}
        res.fromkeys(keys)
        # 先搜索歌曲
        songs = api.search(text)
        if songs['result']['songCount'] == 0:
            res['song'] = []
            return res
        songs = songs['result']['songs']
        for i, song in enumerate(songs):
            songs[i] = SearchDao.accessBriefMusic(song)
        res['song'] = songs
        return res

    def accessBriefMusic(songData):
        song = BriefMusicModel(songData).getModeledData()
        for i, artist in enumerate(song['artists']):
            artist = BriefArtistModel(artist).getModeledData()
            song['artists'][i] = artist
        song['album'] = BriefAlbumModel(song['album']).getModeledData()
        return song

    def accessMusic(musicData):
        musicData['url'] = musicData['mp3Url']
        song = MusicModel(musicData).getModeledData()

        for i, artist in enumerate(song['artists']):
            a = ArtistModel(artist).getModeledData()
            song['artists'][i] = a
        song['album'] = AlbumModel(song['album']).getModeledData()
        return song

    def query(id, type):
        api = NetEaseApi()
        if type == 'artist':
            data = api.queryArtistDetail(id)
            for i, track in enumerate(data['hotSongs']):
                data['hotSongs'][i] = SearchDao.accessMusic(track)
            for key in data['artist']:
                data[key] = data['artist'][key]
            model = ArtistDetailModel(data).getModeledData()
            return model
        if type == 'album':
            data = api.queryAlbumDetail(id)
            album = data['album']
            for i, track in enumerate(album['songs']):
                album['songs'][i] = SearchDao.accessMusic(track)
            model = AlbumDetailModel(album).getModeledData()
            return model

    def getCoverFromUrl(url, networkManager):
        request = QNetworkRequest(QUrl(url))
        networkManager.get(request)

    def getSongFromId(id):
        api = NetEaseApi()
        song = api.songDetail(id)
        song = SearchDao.accessMusic(song[0])
        return song


import sqlite3
import mysql.connector
import time

class SqlDB(object):
    def __init__(self, dbtype='sqlite3', **kwargs):
        """
        MySQL
            Password is given by the user at the time of installing the MySQL
            database. If you are using root then you won’t need the password.

        ref: https://pynative.com/python-mysql-insert-data-into-database-table/
        """

        self.dbtype = dbtype

        if dbtype == 'sqlite3':
            print("Connecting sqlite3 database...")
            assert 'path' in kwargs.keys()

            self.conn = sqlite3.connect(kwargs['path'])

            # Create table
            self.conn.execute('''CREATE TABLE IF NOT EXISTS PATTERNS
                (ID INTEGER PRIMARY KEY     AUTOINCREMENT,
                PATTERN        TEXT     NOT NULL,
                ENT            TEXT     NOT NULL,
                TIMESTAMP      INT      NOT NULL,
                FREQ           INT      NOT NULL);''')
            print("PATTERNS table created successfully!")
        elif dbtype == 'mysql':
            print("Connecting MySQL database...")
            assert 'host' in kwargs.keys()
            assert 'user' in kwargs.keys()
            assert 'database' in kwargs.keys()

            self.conn = mysql.connector.connect(host=kwargs['host'],
                user=kwargs['user'], database=kwargs['database'])

            print("MySQL connected:", self.conn.is_connected())

            self.cursor = self.conn.cursor()
            self.cursor.execute('''CREATE TABLE IF NOT EXISTS PATTERNS
                (ID INT PRIMARY KEY     AUTO_INCREMENT,
                PATTERN        TEXT     NOT NULL,
                ENT            TEXT     NOT NULL,
                TIMESTAMP      INT      NOT NULL,
                FREQ           INT      NOT NULL);''')
            print("PATTERNS table created successfully!")
        else:
            raise ValueError(dbtype + ' database not supported')

    @classmethod
    def fromconfig(cls, config):
        if config['DBTYPE'] == 'sqlite3':
            return cls(dbtype=config['DBTYPE'], path=config['PATH'])
        elif config['DBTYPE'] == 'mysql':
            return cls(dbtype=config['DBTYPE'],
                host=config['HOST'],
                user=config['USER'],
                database=config['DATABASE'])

        raise ValueError(config['DBTYPE'] + ' database not supported')

    def execute(self, query, params=None):
        if params is None:
            if self.dbtype == 'sqlite3':
                return self.conn.execute(query)
            elif self.dbtype == 'mysql':
                self.cursor.execute(query)
                return self._mysql_fetch()
            else:
                raise ValueError(self.dbtype + ' database not supported')

        if self.dbtype == 'sqlite3':
            return self.conn.execute(query, params)
        elif self.dbtype == 'mysql':
            # MySQL parameterized arguments uses %s instead of ?
            query = query.replace('?', '%s')
            self.cursor.execute(query, params)
            return self._mysql_fetch()

        raise ValueError(self.dbtype + ' database not supported')

    def executemany(self, query, params, max_retries=4):
        if self.dbtype == 'sqlite3':
            i = 0
            while i < max_retries:
                try:
                    return self.conn.executemany(query, params)
                except sqlite3.OperationalError:
                    time.sleep(0.5)
                    i += 1
            return

        elif self.dbtype == 'mysql':
            # MySQL parameterized arguments uses %s instead of ?
            query = query.replace('?', '%s')
            i = 0
            while i < max_retries:
                try:
                    self.cursor.executemany(query, params)
                    return self._mysql_fetch()
                except mysql.connector.errors.DatabaseError:
                    time.sleep(0.5)
                    i += 1

        raise ValueError(self.dbtype + ' database not supported')

    def commit(self):
        self.conn.commit()

    def close(self):
        self.conn.close()

        if self.dbtype == 'mysql':
            self.cursor.close()

    def delete(self):
        if self.dbtype == 'sqlite3':
            self.conn.execute("DROP TABLE IF EXISTS PATTERNS")
        elif self.dbtype == 'mysql':
            self.cursor.execute("DROP TABLE IF EXISTS PATTERNS")
        else:
            raise ValueError(self.dbtype + ' database not supported')

        print("SQL table deleted successfully!")

    def _mysql_fetch(self):
        try:
            return self.cursor.fetchall()
        except mysql.connector.errors.InterfaceError:
            return None


from pathlib import Path
from medpy.io import save, load
import dicom2nifti
import os
import click
import pandas as pd
import dask.dataframe as dd
import numpy as np
from typing import Dict, List, Optional, Tuple

from luna.common.utils import cli_runner
from luna.common.custom_logger import init_logger
from luna.common.utils import generate_uuid

logger = init_logger()

_params_ = [('raw_data_path', str), ('mapping_csv_path', str), ('output_dir', str),
            ('scan_table_path', str), ('npartitions', int), ('subset', bool)]

def find_z_nbound(dim: Tuple) -> Tuple[int, int]:
    """
    Return z, number of series bound

    Args:
        dim (Tuple): shape of the 4D bound series.
    """
    # axial (x, z, nbound, y)
    if dim[0] == dim[3]:
        return dim[1], dim[2]
    # sagittal (x, y, nbound, z)
    if dim[0] == dim[1]:
        return dim[3], dim[2]

def subset_bound(src_path: str, output_path: str, index: int):
    """
    Subset bound series and return the output file path.

    Args:
        src_path (str): path to the bound series
        output_path (str): path to save the subset output
        index (int): index to pull out from the bound 4D scan
    """
    try:
        file_path = src_path.split(':')[-1]
        data, header = load(file_path)
        subset = data[:, :, index, :]
        # re-arrange the array
        subset = np.swapaxes(np.swapaxes(subset, 1, 2), 0, 1)
        save(subset, output_path, header)
        return output_path

    except Exception as err:
        logger.error(output_path, err)
        return None

def subset_series(dim: str, src_path: str) -> str:
    """
    Subset scans if the scan is a bound series (with 4 dimensions).

    Args:
        dim (str): shape of the scan
        src_path (str): path to the scan

    Returns:
        str: path to the subset output
    """
    dim = eval(dim)
    posix_path = Path(src_path)

    if len(dim) == 4:
        # axial (x, z, n_bound_series, y)
        # sag (x, y, n_bound_series, z)
        z, nbound = find_z_nbound(dim)

        # if 3 series are bound, assume there are three post contrasts bound
        if nbound == 3:
            index = 0
        # if more than 3 series are bound, assume the first one is pre
        elif nbound > 3:
            index = 1

        output_path = os.path.join(str(posix_path.parent), f"subset_{index}_" + str(posix_path.name))
        if not os.path.exists(output_path):
            subset_bound(src_path, output_path, index)
            logger.info("Saved ", output_path)

    return output_path

def dicom_to_nifti(row: pd.DataFrame, raw_data_path: str, output_dir: str, subset=False)\
        -> pd.DataFrame:
    """
    Convert dicoms to nii.gz format

    Args:
        row (pd.DataFrame): dataframe row with path, accession_number columns
        raw_data_path (str): path to dicom files
        output_dir (str): path to save scans
        subset (Optional, boolean): True to extract first post contrast in bound series

    Returns:
        pd.DataFrame: row populated with record_uuid, dim, path, subset_path
    """
    accession_number = row.accession_number
    series_number = row.series_number

    dicom_dir = f"{raw_data_path}/{accession_number}/SCANS/{series_number}/DICOM"
    scan_dir = f"{output_dir}/{accession_number}"

    logger.info(scan_dir)
    os.makedirs(scan_dir, exist_ok=True)

    logger.info(f"Processing : {dicom_dir}")
    try:
        scan_file = None
        for fn in os.listdir(scan_dir):
            if fn.endswith("nii.gz") and fn.startswith(str(series_number)):
                scan_file = fn
                break
        if not scan_file:
            dicom2nifti.convert_directory(dicom_dir, scan_dir, reorient=False)

        print(os.listdir(scan_dir))
        for fn in os.listdir(scan_dir):
            if fn.endswith("nii.gz") and fn.startswith(str(series_number)):
                scan_file = fn
                break

        scan_file_path = os.path.join(scan_dir, scan_file)
        data, header = load(scan_file_path)

        row["record_uuid"] = generate_uuid(scan_file_path, ["SCAN"])
        row["path"] = scan_file_path
        row["dim"] = str(data.shape)

        if len(data.shape) == 4 and subset:
            row["subset_path"] = subset_series(row.dim, row.path)

    except Exception as err:
        logger.error(f"Failed {accession_number}: {err}")

    return row

@click.command()
@click.option('-r', '--raw_data_path', help="path to raw data. e.g /data/radiology/project_name/dicoms",
              type=click.Path())
@click.option('-c', '--mapping_csv_path', help="csv with AccessionNumber, SeriesNumber columns",
              type=click.Path())
@click.option('-o', '--output_dir', help="path to scan output folder")
@click.option('-t', '--scan_table_path', help="path to save scan table")
@click.option('-s', '--subset', help="extract first post contrast in bound series", is_flag=True)
@click.option('-n', '--npartitions', help="npartitions for parallelization", default=20, show_default=True)
@click.option('-m', '--method_param_path', help='path to a metadata json/yaml file with method parameters to reproduce results',
              type=click.Path())
def cli(**cli_kwargs):
    """
    Convert dicoms to nii.gz format and optionally subset bound series.

    dicom2nifti is used to convert bound series into 4D volumes.
    For more simple dicom to scan function, see luna.radiology.cli.dicom_to_itk.
    """
    cli_runner(cli_kwargs, _params_, generate_scan_table)

def generate_scan_table(raw_data_path: str, mapping_csv_path: str, output_dir: str,
                        scan_table_path: str, subset=False, npartitions=20):
    """
    Convert dicoms to nii.gz format and optionally subset bound series.
    Save scan table and method params.

    Args:
        dicom_parquet_path (str): path to dicom parquet table
        mapping_csv_path (str): csv with AccessionNumber, SeriesNumber columns
        output_dir (str): path to scan output folder
        scan_table_path (str): path to save scan table
        subset (Optional, boolean): True to extract first post contrast in bound series
        npartitions (Optional, int): number of partitions for parallelization. Default is 20
    """
    input_params = locals()

    # load accession/series mapping csv
    scan_map = pd.read_csv(mapping_csv_path)
    scan_map = scan_map[['AccessionNumber', 'SeriesNumber']]
    df = scan_map \
        .rename({'AccessionNumber': 'accession_number', 'SeriesNumber': 'series_number'}, axis=1) \
        .astype(str)

    # convert to nii.gz
    df["create_ts"] = pd.Timestamp.now()
    df["dim"] = ""
    df["path"] = ""
    df["record_uuid"] = ""
    df["subset_path"] = ""

    ddf = dd.from_pandas(df, npartitions=npartitions)
    df = ddf.apply(lambda x: dicom_to_nifti(x, raw_data_path, output_dir, subset=subset), axis=1,
                   meta=ddf).compute()
    logger.info(df)

    # save table as parquet
    df = df.replace("", np.nan)
    df = df.dropna(subset=["record_uuid"])
    df.to_parquet(scan_table_path)

    return {
        'table_path': scan_table_path,
        'n_records': len(df)
    }

if __name__ == "__main__":
    cli()


import os, sys, math, time, inspect
import numpy as np

def encode_seq(script, dic):
    """Returns decription of the script (sequence) based on dictionary dic"""

    decription = []
    ##  Convert text to indice
    for char in script:

        if isinstance(char, type(np.asarray([0]))) : char = char[0]
        ind = np.asarray([dic[char]])
        decription.append(ind)
    decription = np.asarray(decription, dtype='int32').T
    return decription

def decode_seq(script, dic):
    """Returns decription of the script (sequence) based on dictionary dic"""

    decription = ""
    ##  Convert text to indice
    for ind in script:

        #if isinstance(char, type(np.asarray([0]))) : char = char[0]
        char = dic[ind]
        decription += char

    return decription

'''This code was from
https://github.com/nyu-dl/dl4mt-tutorial/blob/master/session2/nmt.py'''
# batch preparation
def prepare_data(seqs_x, seqs_y, maxlen=None, n_words_src=30000,
                 n_words=30000):
    # x: a list of sentences
    lengths_x = [len(s) for s in seqs_x]
    lengths_y = [len(s) for s in seqs_y]

    if maxlen is not None:
        new_seqs_x = []
        new_seqs_y = []
        new_lengths_x = []
        new_lengths_y = []
        for l_x, s_x, l_y, s_y in zip(lengths_x, seqs_x, lengths_y, seqs_y):
            if l_x < maxlen and l_y < maxlen:
                new_seqs_x.append(s_x)
                new_lengths_x.append(l_x)
                new_seqs_y.append(s_y)
                new_lengths_y.append(l_y)
        lengths_x = new_lengths_x
        seqs_x = new_seqs_x
        lengths_y = new_lengths_y
        seqs_y = new_seqs_y

        if len(lengths_x) < 1 or len(lengths_y) < 1:
            return None, None, None, None

    n_samples = len(seqs_x)
    maxlen_x = np.max(lengths_x) + 1 #This will give tight bound
    maxlen_y = np.max(lengths_y) + 1 #This will give tight bound

    x = np.zeros((maxlen_x, n_samples)).astype('int64')
    y = np.zeros((maxlen_y, n_samples)).astype('int64')
    x_mask = np.zeros((maxlen_x, n_samples)).astype('float32')
    y_mask = np.zeros((maxlen_y, n_samples)).astype('float32')
    for idx, [s_x, s_y] in enumerate(zip(seqs_x, seqs_y)):
        x[:lengths_x[idx], idx] = s_x
        x_mask[:lengths_x[idx]+1, idx] = 1.
        y[:lengths_y[idx], idx] = s_y
        y_mask[:lengths_y[idx]+1, idx] = 1.

    return  x.T, x_mask.T, y.T, y_mask.T


import pandas as pd
import numpy as np

from tools import str2num

converters = {'Price':str2num,
              'Rank':str2num,
              'Rating':str2num,
              'Sales':str2num,
              'Revenue':str2num,
              'Reviews':str2num
             }
try:
    data = pd.read_csv('data.csv', converters=converters, header=7, index_col=0)
except BaseException as e:
    print(e)


from keras.preprocessing.image import img_to_array
from keras.models import load_model
import numpy as np
import argparse
import imutils
import pickle
import cv2
import os
import sys

def classify(modelPath,labelPath,count):
        if(count==0):
                modelPath = input("Where is the model?")
                labelPath = input("Where are the labels?")
                testImage = input("What image do you want to classify?")
        else:
                testImage = input("What image do you want to classify?")
        image = cv2.imread(testImage)
        output = image.copy()
        image = cv2.resize(image, (200, 200))
        image = image.astype("float") / 255.0
        image = img_to_array(image)
        image = np.expand_dims(image, axis=0)
        print("[INFO] loading network...")
        model = load_model(modelPath)
        lb = pickle.loads(open(labelPath, "rb").read())

        # classify the input image
        print("[INFO] classifying image...")
        proba = model.predict(image)[0]
        idx = np.argmax(proba)
        label = lb.classes_[idx-1]

        filename = testImage.split("/")[-2]
        correct = "correct" if filename.rfind(label) != -1 else "incorrect"

        # build the label and draw the label on the image
        label = "{}: {:.2f}% ({})".format(label, proba[idx] * 100, correct)
        output = imutils.resize(output, width=400)
        cv2.putText(output, label, (10, 25),  cv2.FONT_HERSHEY_SIMPLEX,0.7, (0, 255, 0), 2)
        if(correct=="incorrect"):
           print("[REAL] {}: {:.2f}%".format(filename,proba[int(filename)]*100))
        # show the output image
        print("[INFO] {}".format(label))
        cv2.imshow("Output", output)
        cv2.waitKey(0)
        cont = input("Do you have another image? y/n ")
        if(cont!='n'):
                classify(modelPath,labelPath,count+1)
        else:
                sys.exit(1)

classify('','',0)


from os import path

# App details
BASE_DIRECTORY = path.abspath(path.dirname(__file__))
DEBUG = True
SECRET_KEY = 'keep_it_like_a_secret'

# Database details
db_path = path.join(BASE_DIRECTORY, '../', 'app.db')
db_uri = 'sqlite:///{}'.format(db_path)
SQLALCHEMY_DATABASE_URI = db_uri

# Facebook
FB_APP_ID = ''
FB_APP_NAME = 'social-companion'
FB_APP_SECRET = ''

CACHE_TYPE = 'filesystem'
CACHE_DIR = '/tmp'


# -*- coding: utf8 -*-

class Bob:
  def hey(self, message):
    if not message:
      return "Fine. Be that way!"

    message = message.decode('cp1252')

    if not message.strip():
      return "Fine. Be that way!"
    if message.isupper():
      return "Woah, chill out!"
    elif message.endswith("?"):
        return "Sure."

    return "Whatever."


from uuid import uuid4, UUID
from enum import Enum
from json import dumps, JSONEncoder, loads
from re import compile as re_compile
from collections import OrderedDict
from functools import partial
from openvsp import vsp
try:
    from IPython.display import display_javascript, display_html
except ImportError:
    display_html, display_javascript = None, None

regex_listname = re_compile(r"^(?P<name>[a-zA-Z]+)_*(?P\d*)")
EXCLUDE_GROUPS = ('BBox',)
INCLUDE_GROUPS = ('Design',)  # Not currently used for filtering groups
PARAM_KEYS = ('upper', 'lower', 'value')

Enum0 = partial(Enum, start=0)
ParamType = Enum0('ParamType', 'PARM_DOUBLE_TYPE PARM_INT_TYPE PARM_BOOL_TYPE PARM_FRACTION_TYPE PARM_STRING_TYPE ')
ErrorCode = Enum0('ErrorCode', 'VSP_OK VSP_INVALID_PTR VSP_CANT_FIND_TYPE VSP_CANT_FIND_PARM VSP_CANT_FIND_NAME ' +
                  'VSP_INVALID_GEOM_ID VSP_FILE_DOES_NOT_EXIST VSP_FILE_WRITE_FAILURE VSP_WRONG_XSEC_TYPE ' +
                  'VSP_WRONG_FILE_TYPE VSP_INDEX_OUT_RANGE VSP_INVALID_XSEC_ID')
SymCode = Enum0('SymCode', 'SYM_XY SYM_XZ SYM_YZ SYM_ROT_X SYM_ROT_Y SYM_ROT_Z SYM_PLANAR_TYPES SYM_NUM_TYPES')
ExportCode = Enum0('ExportCode', 'EXPORT_FELISA EXPORT_XSEC EXPORT_STL EXPORT_AWAVE EXPORT_NASCART EXPORT_POVRAY ' +
                   'EXPORT_CART3D EXPORT_VORXSEC EXPORT_XSECGEOM EXPORT_GMSH EXPORT_X3D')

class VspEncoder(JSONEncoder):
    # TODO: remove keys that start with '_'
    def default(self, obj):
        if isinstance(obj, VspElement):
            return obj.json

        if isinstance(obj, Enum):
            return {"__enum__": str(obj)}

        if isinstance(obj, UUID):
            return {"__uuid__": str(obj)}

        return JSONEncoder.default(self, obj)

class VspElement(OrderedDict):
    """An object that can represent an manipulate OpenVSP elements."""

    def __init__(self, is_param=False, **kwargs):
        super().__init__()
        self.__uuid = uuid4()
        self.__is_param = is_param
        for key, value in kwargs.items():
            if isinstance(value, dict):
                value = VspElement(**value)
            self[key] = value

    def __hash__(self):
        return hash(id(self))

    def __getitem__(self, key):
        try:
            if key[0] == '_':
                return super().__getitem__(key)
        except TypeError:
            pass

        try:
            item = super().__getitem__(key)
        except KeyError:
            return None

        if isinstance(item, VspElement) and item.__is_param:
            return item.value
        else:
            return item

    def __lt__(self, other):
        if self.__is_param:
            return self.value < other
        else:
            return super().__lt__(other)

    def __gt__(self, other):
        if self.__is_param:
            return self.value > other
        else:
            return super().__gt__(other)

    def __eq__(self, other):
        if self.__is_param:
            return self.value == other
        else:
            return super().__eq__(other)

    def __setitem__(self, key, value):
        try:
            item = super().__getitem__(key)
        except KeyError:
            item = []
        if isinstance(item, VspElement) and item.__is_param:
            if value < item.lower or value > item.upper:
                msg = "{lower}<={name}<={upper}, value={_new}"
                raise ValueError(msg.format(_new=value, **item))
            item.value = value
            vsp.Update()
        else:
            super().__setitem__(key, value)

    __getattr__ = __getitem__
    __setattr__ = __setitem__
    __delattr__ = OrderedDict.__delitem__

    def __dir__(self):
        return super().__dir__() + list(self.keys())

    def __repr__(self):
        rep_dict = {'name': 'VSP Element'}
        rep_dict.update(**dict(self))
        return "{name}".format(**rep_dict)

    def _ipython_display_(self):
        if display_html is not None:
            display_html('

'.format(self.__uuid), raw=True)

            display_javascript("""
                require(["https://rawgit.com/caldwell/renderjson/master/renderjson.js"], function() {
                  document.getElementById('%s').appendChild(renderjson(%s))
                });
                """ % (self.__uuid, self.json), raw=True)

    @property
    def json(self):
        return dumps(self, cls=VspEncoder, indent=2, sort_keys=True)

class VspModel(VspElement):
    """An object that can represent and manipulate OpenVSP aircraft."""

    def __init__(self, filename=None, threejs=False, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.__filename = filename
        self.__threejs = threejs
        self.__param_map = {}
        self.design = None

        if filename:
            self._load_from_file(filename)

    def load_design(self, filename):
        design_vars = VspElement()
        vsp.ReadApplyDESFile(filename)
        for i in range(vsp.GetNumDesignVars()):
            param_id = vsp.GetDesignVar(i)
            if param_id in self.__param_map:
                param = self.__param_map[param_id]
            else:
                param = self._make_parameter(param_id)
            design_vars[param.name] = param
        self.design = design_vars

    def _make_parameter(self, param_id):
        kind = vsp.GetParmType(param_id)
        if kind < len(ParamType):
            kind = ParamType(kind)
        param = VspElement(_id=param_id,
                           name=vsp.GetParmName(param_id),
                           kind=kind,
                           value=vsp.GetParmVal(param_id),
                           lower=vsp.GetParmLowerLimit(param_id),
                           upper=vsp.GetParmUpperLimit(param_id),
                           )
        self.__param_map[param_id] = param
        return param

    @staticmethod
    def _update_list(lst, idx):
        idx = int(idx)
        if len(lst) <= idx:
            lst.extend([None] * (1 + idx - len(lst)))
        if lst[idx] is None:
            lst[idx] = VspElement()
        return lst, lst[idx]

    def _load_from_file(self, filename):
        vsp.ClearVSPModel()
        vsp.ReadVSPFile(filename)

        for geom_id in vsp.FindGeoms():
            geom_name_raw = vsp.GetGeomName(geom_id)
            geom_name, geom_idx = regex_listname.findall(geom_name_raw)[0]

            if geom_name not in self:
                if geom_idx:
                    self[geom_name] = []
                else:
                    self[geom_name] = VspElement()

            if geom_idx != '':
                geom = self._update_list(self[geom_name], geom_idx)
            else:
                geom = self[geom_name]

            geom._id = geom_id

            for param_id in vsp.GetGeomParmIDs(geom_id):
                group_name_raw = vsp.GetParmDisplayGroupName(param_id)
                group_name, group_idx = regex_listname.findall(group_name_raw)[0]
                if group_name not in EXCLUDE_GROUPS:
                    if group_name not in geom:
                        if group_idx:
                            geom[group_name] = []
                        else:
                            geom[group_name] = VspElement()

                    if group_idx != '':
                        geom[group_name], group = self._update_list(geom[group_name], group_idx)
                    else:
                        group = geom[group_name]

                    param = self._make_parameter(param_id)

                    if param['name'] in group:
                        raise ValueError("{} already in <{}:{}>".format(param.name, geom_name, group_name))

                    group[param['name']] = param

    def export(self, filename=None, vsp_set=0, file_format=ExportCode.EXPORT_STL):
        if filename is None:
            # TODO: map extension to file_format
            filename = self.__filename.split('.')[0] + '.stl'
        if isinstance(file_format, ExportCode):
            file_format = file_format.value
        vsp.ExportFile(filename, vsp_set, file_format)

    @property
    def threejs_data(self):
        if '__stl_filename' not in self:
            self.__stl_filename = '_tmp_threejs.stl'
        self.export(filename=self.__stl_filename, file_format=ExportCode.EXPORT_STL.value)
        # read in STL file
        with open(self.__stl_filename) as stl_file:
            stl_data = stl_file.read()

        return dumps([{"id": 2, "polyData": stl_data, "x": 0.0, "y": 0.0, "z": 0.0}])

    @property
    def wetted_areas(self):
        vsp.DeleteAllResults()
        results = {}
        _ = vsp.ComputeCompGeom(0, False, 0)
        result_id = vsp.FindLatestResultsID('Comp_Geom')
        components = vsp.GetStringResults(result_id, 'Comp_Name')
        wetted_areas = vsp.GetDoubleResults(result_id, 'Wet_Area')
        for comp, area in zip(components, wetted_areas):
            results[comp] = area

        results['total'] = sum(wetted_areas)
        return results

    def get_analysis_list(self):
        pass
        for a in vsp.ListAnalysis():
            pass
            print('For ', a)
            print(vsp.GetAnalysisInputNames(a))

    def get_analysis_names(self):
        pass
        print(vsp.GetNumAnalysis())

    def h(self, type='ParasiteDrag', unit='Total_CD_Total'):
        pass
        '''
        run a simulation, and return a specific value associated with the simulation
        '''
        id = vsp.ExecAnalysis(type)
        data = vsp.GetAllDataNames(id)
        results = vsp.GetDoubleResults(id, unit)[0]
        return results

    def set_param(self, id, value):
        pass
        '''
        used to set a specific parameter's value
        '''
        vsp.SetParmVal(id, value)

    def get_param(self, id):
        pass
        '''
        used to obtain a value from a parameter
        '''

    def save_file(self, filename=None):
        pass
        '''
        used to save any changes made. Saves to the same file by default
        '''
        if filename:
            pass
            vsp.WriteVSPFile(filename)
        else:
            pass
            vsp.WriteVSPFile(self.__filename)


from random import random, randint

import os, sys
directory = os.path.dirname(os.path.realpath('_file_'))
sys.path.append(os.path.join(directory, "code", "classes"))

X_MAX = 320
Y_MAX = 360

from classes import Water

def check_shortest_distance(huis1, huis2):
    """Checks if there is overlap on two axes, or one axis, or no overlap at all and returns shortest distance"""
    # if there is overlap on x and y axis, houses overlap
    if check_overlap(huis1.x1, huis1.x2, huis2.x1, huis2.x2) and \
    check_overlap(huis1.y1, huis1.y2, huis2.y1, huis2.y2) == True:
        return False
    # if there is overlap on x axis, shortest distance between two houses is on y axis
    elif check_overlap(huis1.x1, huis1.x2, huis2.x1, huis2.x2) == True:
        return calculate_shortest_distance(huis1.y1, huis1.y2, huis2.y1, huis2.y2)
    # if there is overlap on y axis, shortest distance between two houses is on x axis
    elif check_overlap(huis1.y1, huis1.y2, huis2.y1, huis2.y2) == True:
        return calculate_shortest_distance(huis1.x1, huis1.x2, huis2.x1, huis2.x2)
    # if there is no overlap, pythagoras
    else:
        return pythagoras(calculate_shortest_distance(huis1.x1, huis1.x2, huis2.x1, huis2.x2), calculate_shortest_distance(huis1.y1, huis1.y2, huis2.y1, huis2.y2))

def pythagoras(a, b):
    """Does pythagoras"""
    c_square = float(a) ** 2 + float(b) ** 2
    return int((c_square) ** (0.5))

def calculate_shortest_distance(a1, a2, b1, b2):
    """Checks shortest distance between four different points of a rectangle"""
    return min(abs(a1 - b1), abs(a1 - b2), abs(a2 - b1), abs(a2 - b2))

def check_overlap(a1, a2, b1, b2):
    """Checks if there is overlap between 4 points on the same axis"""
    if (b1 >= a1 and b1 <= a2) or (b2 >= a1 and b1 <= a2):
        return True
    return False

def check_overlap_met_water(woningen, x1, x2, y1, y2, watervlaktes):
    """Checks if water can be placed at location"""
    # iterate over list of houses and return false if water is placed over house
    for i in range(len(woningen)):
        if check_overlap(x1, x2, woningen[i].x1, woningen[i].x2) == True and check_overlap(y1, y2, woningen[i].y1, woningen[i].y2) == True:
            return False
    # iterate over bodies of water and return false if water is placed over water
    for i in range(len(watervlaktes)):
        if check_overlap(x1, x2, watervlaktes[i].x1, watervlaktes[i].x2) == True and check_overlap(y1, y2, watervlaktes[i].y1, watervlaktes[i].y2) == True:
            return False
    return True

def add_water(woningen):
    """Adds water to the map"""
    # set 4 bodies of water
    water = 4
    # empty list of bodies of water
    watervlaktes = []
    # keeps track of the number of attempts of placing water
    count = 0

    #water = 1

    while water != 0:
        # define some random configurations of water
        configuration = [[72, 80], [120, 48], [60, 96], [64, 90]]
        # choose a random configuration of water
        #random_configuration = [96, 240]
        random_configuration = configuration[randint(0, 3)]
        x2_width = random_configuration[0]
        y2_length = random_configuration[1]
        # find random location to place body of water
        x1 = randint(1, X_MAX - x2_width)
        y1 = randint(1, Y_MAX - y2_length)
        x2 = x1 + x2_width
        y2 = y1 + y2_length
        # if water can be placed at location, place it
        if check_overlap_met_water(woningen, x1, x2, y1, y2, watervlaktes) == True:
            water -= 1
            width = x2_width
            length = y2_length
            watervlakte = Water(water, x1, x2, y1, y2, width, length)
            # add body of water to a list
            watervlaktes.append(watervlakte)
        # add to count
        count += 1
        if count == 100000:
            print("Water is 10 000 keer geprobeerd te plaatsen. Zonder succes. \
            Het algoritme gaat een nieuwe configuratie proberen van de woningen.")
            return False
    return watervlaktes


# Store information for site, weather
from configobj import ConfigObj
import datetime, logging, ipdb
import ephem, time, math, os, sys, json, urllib2
class site:

    def __init__(self,site_name, night, configfile=''):

        self.name = site_name

        #set appropriate parameter based on aqawan_num
        #create configuration file object
        configObj = ConfigObj(configfile)

        try:
            siteconfig = configObj[self.name]
        except:
            print('ERROR accessing ', self.name, ".",
                self.name, " was not found in the configuration file", configfile)
            return

        self.latitude = siteconfig['Setup']['LATITUDE']
        self.longitude = siteconfig['Setup']['LONGITUDE']
        self.elevation = float(siteconfig['Setup']['ELEVATION'])
        self.enclosures = siteconfig['Setup']['ENCLOSURES']

        self.currentStatusFile = 'current_' + site_name + '.log'
        self.observing = True
        self.weather = -1
        self.startNightTime = -1
        self.night = night

        # touch a file in the current directory to enable cloud override
        self.cloudOverride = os.path.isfile('cloudOverride.txt')
        self.sunOverride = os.path.isfile('sunOverride.txt')

        logger_name = siteconfig['Setup']['LOGNAME']
        log_file = 'logs/' + night + '/' + siteconfig['Setup']['LOGFILE']

        self.obs = ephem.Observer()
        self.obs.lat = ephem.degrees(str(self.latitude)) # N
        self.obs.lon = ephem.degrees(str(self.longitude)) # E
        self.obs.elevation = self.elevation # meters

    def sunrise(self, horizon=0):

        self.obs.horizon = str(horizon)
        sunrise = self.obs.next_rising(ephem.Sun(), start=self.startNightTime, use_center=True).datetime()
        return sunrise

    def sunset(self, horizon=0):

        self.obs.horizon = str(horizon)
        sunset = self.obs.next_setting(ephem.Sun(), start=self.startNightTime, use_center=True).datetime()
        return sunset


#!/usr/bin/env python

import os
import sys
import math
import spice
from collections import OrderedDict as OD
import unittest

class TestSMAP(unittest.TestCase):

  def setUp(self):
    mydir = os.path.dirname(__file__)
    self.kernels = [ os.path.join( mydir, i.strip() ) for i in """
### Lines in this string that start with '#' are comments
### The other lines in this string are paths to kernels,
###   relative to the parent directory of this Python script, __file__
###
kernels/naif0010.tls
kernels/smap_v00.tf
kernels/pck00009.tpc
kernels/spk_drm239_WithBurn-full.bsp
kernels/smap_test.bsp
###
### kernels/SMAP_ref_150529_180529.bsp
###
### That commented SMAP SPK kernel, or one like it, should be under this URL:
###
###   http://naif.jpl.nasa.gov/pub/naif/SMAP/kernels/spk/
###
###
""".strip().split('\n') if i[0]!='#' ]

    ### Load default kernels (string variable "kernels" above")
    for kernel in self.kernels: spice.furnsh( kernel )

  def tearDown(self):
    ### Unload any kernels
    for kernel in self.kernels: spice.unload( kernel )

  def test_smap(self):

    target = 'SMAP'

    et0 = spice.utc2et( '2016-06-01T12:00:00' )

    dpr = spice.dpr()

    a,b,c = [spice.gdpool('BODY399_RADII',i,1)[1] for i in range(3)]

    ods = []

    for deltatime in [0.1 * i for i in range(1080)]:
      et = et0 + deltatime
      stSmap,lsSmap = spice.spkezr( target, et, 'IAU_EARTH', 'NONE', 'EARTH' )
      posn, veloc = stSmap[:3], stSmap[3:]
      stSun,lsSun = spice.spkezr( 'SUN', et0, 'IAU_EARTH', 'LT', 'EARTH' )
      mtx = spice.pxform( 'SMAP_REFLECTOR', 'IAU_EARTH', et)
      boreEbf = spice.mxv( mtx, [1.0,0,0] )
      point = spice.surfpt( posn, boreEbf, a, b, c)
      rsurfpt,lon,lat = spice.reclat( point )
      utc = spice.et2utc( et, 'ISOC', 3 )

      ods += [ OD(deltatime=deltatime,posn=posn,veloc=veloc,boreEbf=boreEbf
                 ,utc=utc,point=point,rsurfpt=rsurfpt
                 ,rsmap=spice.vnorm(posn),lat=lat,lon=lon
                 ,raynge=spice.vnorm(spice.vsub(point,posn))
                 ,sunsep=spice.vsep( spice.ucrss(posn,veloc), stSun[:3] )
                 )
             ]

    try:
      ### Moved matplotlib import to here so test runs to here at least
      from matplotlib import pyplot as plt
      plt.figure(1)
      keys = 'lat lon raynge'.split()
      secs = [od['deltatime'] for od in ods]
      for idx in range(len(keys)):
        scal = 1.0 if keys[idx] in 'rsurfpt rsmap raynge sunsep rp ecc t0 mu a P eccmean amean Pmean'.split() else dpr
        ordinate = [od[keys[idx]]*scal for od in ods]
        plt.subplot( 221+idx )
        plt.plot( secs, ordinate )
        plt.plot( secs, ordinate, '.')
        plt.title( keys[idx] )
        plt.ylabel( '%s%s' % (keys[idx],'' if scal==1.0 else ', deg') )
        if idx>1: plt.xlabel( 'T-T0, s' )

      abscissa = [od['lon']*dpr for od in ods]
      ordinate = [od['lat']*dpr for od in ods]
      plt.subplot( 221+idx+1 )
      plt.title( 'lon vs. lat' )
      plt.plot( abscissa, ordinate )
      plt.xlabel( 'lon, deg' )
      plt.ylabel( 'lat, deg' )
      plt.show()

    except:
      print( "Bypassed, or failed, matplotlib tests" )

if __name__=="__main__":
  unittest.main()


from datetime import timedelta
from django.utils import unittest, timezone
from django.db import models
from djangotoolbox.fields import ListField
from ikwen.core.utils import set_counters

from ikwen.core.utils import increment_history_field, calculate_watch_info, rank_watch_objects, \
    group_history_value_list

class WatchObject(models.Model):
    val1_history = models.CharField(max_length=20)
    val2_history = ListField()
    total_val1 = models.IntegerField(default=0)
    total_val2 = models.IntegerField(default=0)

    class Meta:
        app_label = 'core'

def init_watch_object():
    watch_object = WatchObject()
    watch_object.val1_history = '18,9,57,23,46'
    watch_object.val2_history = [18, 9, 57, 23, 46]
    return watch_object

class CoreUtilsTestCase(unittest.TestCase):
    """
    This test derives django.utils.unittest.TestCate rather than the default django.test.TestCase.
    Thus, self.client is not automatically created and fixtures not automatically loaded. This
    will be achieved manually by a custom implementation of setUp()
    """
    # fixtures = ['kc_setup_data.yaml', 'kc_members.yaml', 'kc_profiles.yaml']
    #
    # def setUp(self):
    #     self.client = Client()
    #     for fixture in self.fixtures:
    #         call_command('loaddata', fixture)
    #
    # def tearDown(self):
    #     wipe_test_data()

    def test_increment_history_field(self):
        """
        Last element of the report fields are incremented by the number passed as parameter
        """
        watch_object = init_watch_object()
        increment_history_field(watch_object, 'val1_history', 10)
        increment_history_field(watch_object, 'val2_history', 10)
        self.assertEqual(watch_object.val1_history, '18,9,57,23,56.0')
        self.assertListEqual(watch_object.val2_history, [18, 9, 57, 23, 56])
        self.assertEqual(watch_object.total_val1, 10)
        self.assertEqual(watch_object.total_val2, 10)

    def test_calculate_watch_info_with_less_history_values_than_period(self):
        watch_object = init_watch_object()
        watch_info0 = calculate_watch_info(watch_object.val2_history)
        watch_info1 = calculate_watch_info(watch_object.val2_history, duration=1)
        watch_info7 = calculate_watch_info(watch_object.val2_history, duration=7)
        watch_info28 = calculate_watch_info(watch_object.val2_history, duration=28)

        self.assertDictEqual(watch_info0, {'total': 46, 'change': None, 'change_rate': None})
        self.assertDictEqual(watch_info1, {'total': 23, 'change': None, 'change_rate': None})
        self.assertDictEqual(watch_info7, {'total': 107, 'change': None, 'change_rate': None})
        self.assertDictEqual(watch_info28, {'total': 107, 'change': None, 'change_rate': None})

    def test_calculate_watch_info_with_sufficient_history_values(self):
        watch_object = init_watch_object()
        watch_object.val2_history = list(range(57))
        watch_info0 = calculate_watch_info(watch_object.val2_history)
        watch_info1 = calculate_watch_info(watch_object.val2_history, duration=1)
        watch_info7 = calculate_watch_info(watch_object.val2_history, duration=7)
        watch_info28 = calculate_watch_info(watch_object.val2_history, duration=28)

        t0_28 = sum(range(28))
        t28_56 = sum(range(28, 56))
        self.assertDictEqual(watch_info0, {'total': 56, 'change': None, 'change_rate': None})
        self.assertDictEqual(watch_info1, {'total': 55, 'change': 55 - 48, 'change_rate': (55 - 48)/48.0 * 100})
        self.assertDictEqual(watch_info7, {'total': 364, 'change': 364 - 315, 'change_rate': (364 - 315)/315.0 * 100})
        self.assertDictEqual(watch_info28, {'total': t28_56, 'change': t28_56 - t0_28, 'change_rate': float(t28_56 - t0_28) / t0_28 * 100})

    def test_calculate_rank_watch_objects(self):
        wo1 = WatchObject()
        wo1.val2_history = list(range(57))
        wo2 = WatchObject()
        wo2.val2_history = list(range(56, -1, -1))
        wo3 = WatchObject()
        wo3.val2_history = list(range(0, 110, 2))

        l = [wo1, wo2, wo3]
        ranked_watch_objects0 = rank_watch_objects(l, 'val2_history')
        ranked_watch_objects1 = rank_watch_objects(l, 'val2_history', 1)
        ranked_watch_objects7 = rank_watch_objects(l, 'val2_history', 7)
        ranked_watch_objects28 = rank_watch_objects(l, 'val2_history', 28)

        self.assertListEqual([wo3, wo1, wo2], ranked_watch_objects0)
        self.assertListEqual([wo3, wo1, wo2], ranked_watch_objects1)
        self.assertListEqual([wo3, wo1, wo2], ranked_watch_objects7)
        self.assertListEqual([wo3, wo1, wo2], ranked_watch_objects28)

    def test_set_counters(self):
        watch_object = init_watch_object()
        now = timezone.now()
        yesterday = now - timedelta(days=1)
        watch_object.counters_reset_on = now
        set_counters(watch_object)
        self.assertEqual(watch_object.val1_history, '18,9,57,23,46')
        self.assertListEqual(watch_object.val2_history, [18, 9, 57, 23, 46])
        watch_object.counters_reset_on = yesterday
        set_counters(watch_object)
        self.assertEqual(watch_object.val1_history, '18,9,57,23,46,0')
        self.assertListEqual(watch_object.val2_history, [18, 9, 57, 23, 46, 0])

    # def test_group_history_value_list(self):
    #     watch_object = init_watch_object()
    #     watch_object.val2_history = list(range(57))
    #     grouped_monthly = group_history_value_list(watch_object.val2_history)
    #     grouped_weekly = group_history_value_list(watch_object.val2_history, group_unit='week')
    #
    #     # monthly = [1, sum(range(1, 29)), sum(range(29, 57))]
    #     weekly = [0, sum(range(1, 8)), sum(range(8, 15)), sum(range(15, 22)), sum(range(22, 29)),
    #                sum(range(29, 36)), sum(range(36, 43)), sum(range(43, 50)), sum(range(50, 57))]
    #
    #     # self.assertListEqual(grouped_monthly, monthly)
    #     self.assertListEqual(grouped_weekly, weekly)


from styx_msgs.msg import TrafficLight
import tensorflow as tf
import numpy as np
import datetime

from PIL import Image

class TLClassifierObjDetect(object):
    """
    This class uses the trained model from the TensorFlow Object Detection API.
    https://github.com/tensorflow/models/tree/master/research/object_detection

    For the Capstone Project, a Single Shot Detector with lightweight MobileNet
    Backbone was trained on the Bosch Traffic Light Dataset, LISA Traffic Light Dataset and
    Udacity Simulator and Site images.

    The Inference Code was adapted from:
    https://github.com/tensorflow/models/blob/master/research/object_detection/inference/detection_inference.py

    """
    def __init__(self, path_to_tensorflow_graph, confidence_thresh):

        # Threshold for detections
        self.detection_threshold = confidence_thresh

        # Create the TensorFlow session in which the graph is loaded
        self.session = tf.Session()

        # Create Tensors for results
        self.num_detections_tensor = None
        self.detected_boxes_tensor = None
        self.detected_scores_tensor = None
        self.detected_labels_tensor = None
        self.image_tensor = None

        # Load the trained and frozen model graph with respective weights
        with self.session.graph.as_default():
            od_graph_def = tf.GraphDef()
            with tf.gfile.GFile(path_to_tensorflow_graph, 'rb') as fid:
                serialized_graph = fid.read()
                od_graph_def.ParseFromString(serialized_graph)
                tf.import_graph_def(od_graph_def, name='')

                g = tf.get_default_graph()

                # Remember all the tensors we will need for inference
                # Most important: input image tensor:
                self.image_tensor = g.get_tensor_by_name('image_tensor:0')

                self.num_detections_tensor = tf.squeeze(g.get_tensor_by_name('num_detections:0'), 0)
                self.num_detections_tensor = tf.cast(self.num_detections_tensor, tf.int32)

                self.detected_boxes_tensor = tf.squeeze(g.get_tensor_by_name('detection_boxes:0'), 0)
                self.detected_boxes_tensor = self.detected_boxes_tensor[:self.num_detections_tensor]

                self.detected_scores_tensor = tf.squeeze(g.get_tensor_by_name('detection_scores:0'), 0)
                self.detected_scores_tensor = self.detected_scores_tensor[:self.num_detections_tensor]

                self.detected_labels_tensor = tf.squeeze(g.get_tensor_by_name('detection_classes:0'), 0)
                self.detected_labels_tensor = tf.cast(self.detected_labels_tensor, tf.int64)
                self.detected_labels_tensor = self.detected_labels_tensor[:self.num_detections_tensor]

    def get_classification(self, image):
        """
        Determines the color of the traffic light in the image by
        using the TensorFlow graph.

        We run the operations that will give us the boxes, scores and labels.
        Then we filter out the most probable scores (> threshold) and use the
        biggest box, since this will be the nearest traffic light.

        The graph will give us the following IDs :
        4: NA
        3: green
        2: yellow
        1: red

        Args:
            image (cv::Mat): image containing the traffic light

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """

        traffic_light_id = 4  # 4 equals to unknown

        id_mapping = {4: TrafficLight.UNKNOWN,
                      3: TrafficLight.GREEN,
                      2: TrafficLight.YELLOW,
                      1: TrafficLight.RED}

        results = []
        with self.session.graph.as_default():
            boxes, scores, labels = self.session.run([self.detected_boxes_tensor,
                                                      self.detected_scores_tensor,
                                                      self.detected_labels_tensor],
                                                     feed_dict={self.image_tensor: image})
            # Filter for probability (score) and classification
            for i, score in enumerate(scores):
                if score > self.detection_threshold and labels[i] != traffic_light_id:
                    results.append({'box': boxes[i],
                                    'score': score,
                                    'id': labels[i]})

        if len(results) > 0:
            # print('Nums: '+str(len(results))+' '+str(results[0]['score'])+ ' ' + str(results[0]['id']))

            # The boxes are encoded as xmin, xmax, ymin, ymax with normalized coordinates [0..1].
            # So lets find just the biggest box and take the traffic light state from it.
            # max_sized_result = max(results, key=lambda bb: (bb['box'][1] - bb['box'][0]) * (bb['box'][3] - bb['box'][2]))
            # traffic_light_id = max_sized_result['id']

            # Better take the best score than the biggest box !
            max_score_result = max(results, key=lambda bb: bb['score'])
            traffic_light_id = max_score_result['id']

        return id_mapping[traffic_light_id]

class TestTLClassifier(object):

    def __init__(self):
        self.detector = TLClassifier()

    def test_classification(self):
        # Load image
        image_path_green = ('light_classification/test_images/green.jpg', TrafficLight.GREEN)
        image_path_yellow = ('light_classification/test_images/yellow.jpg', TrafficLight.YELLOW)
        image_path_red = ('light_classification/test_images/red.jpg', TrafficLight.RED)
        image_path_na = ('light_classification/test_images/NA.jpg', TrafficLight.UNKNOWN)

        for image_path in [image_path_green, image_path_yellow, image_path_red, image_path_na]:
            image = np.asarray(Image.open(image_path[0]))
            image = np.expand_dims(image, 0)
            gt_result = image_path[1]
            pred_result = self.detector.get_classification(image)
            print(image_path[0])
            print('Prediction success: ' + str(gt_result == pred_result))

            if gt_result != pred_result:
                raise Exception('Prediction error.')

    def measure_time(self):
        # Load image
        image_path = 'light_classification/test_images/green.jpg'
        image = np.asarray(Image.open(image_path))
        image = np.expand_dims(image, 0)

        repeats = 25

        t0 = datetime.datetime.now()
        for i in range(repeats):
            _ = self.detector.get_classification(image)

        delta = datetime.datetime.now() - t0
        print('Time per image in ms: ' + str(delta.seconds * 100.0 / float(repeats)))

if __name__ == '__main__':
    tester = TestTLClassifier()
    tester.measure_time()
    tester.test_classification()


import datetime

from dagster import (
    PresetDefinition,
    daily_schedule,
    hourly_schedule,
    monthly_schedule,
    weekly_schedule,
)

# start_hourly_schedule

@hourly_schedule(
    pipeline_name="my_pipeline",
    start_date=datetime.datetime(2020, 1, 1),
    execution_time=datetime.time(hour=0, minute=25),
    execution_timezone="US/Central",
)
def my_hourly_schedule(date):
    return {"solids": {"process_data_for_date": {"config": {"date": date.strftime("%Y-%m-%d %H")}}}}

# end_hourly_schedule

# start_daily_schedule

@daily_schedule(
    pipeline_name="my_pipeline",
    start_date=datetime.datetime(2020, 1, 1),
    execution_time=datetime.time(hour=9, minute=0),
    execution_timezone="US/Central",
)
def my_daily_schedule(date):
    return {"solids": {"process_data_for_date": {"config": {"date": date.strftime("%Y-%m-%d")}}}}

# end_daily_schedule

# start_weekly_schedule

@weekly_schedule(
    pipeline_name="my_pipeline",
    start_date=datetime.datetime(2020, 1, 1),
    execution_day_of_week=1,  # Monday
    execution_timezone="US/Central",
)
def my_weekly_schedule(date):
    return {"solids": {"process_data_for_date": {"config": {"date": date.strftime("%Y-%m-%d")}}}}

# end_weekly_schedule

# start_monthly_schedule

@monthly_schedule(
    pipeline_name="my_pipeline",
    start_date=datetime.datetime(2020, 1, 1),
    execution_timezone="US/Central",
    execution_day_of_month=15,
    execution_time=datetime.time(hour=9, minute=0),
)
def my_monthly_schedule(date):
    return {"solids": {"process_data_for_date": {"config": {"date": date.strftime("%Y-%m")}}}}

# end_monthly_schedule

preset = PresetDefinition("test_preset", mode="basic", run_config={})

# start_preset

@daily_schedule(
    start_date=datetime.datetime(2020, 1, 1),
    pipeline_name="my_pipeline",
    solid_selection=preset.solid_selection,
    mode=preset.mode,
    tags_fn_for_date=lambda _: preset.tags,
)
def my_preset_schedule(_date):
    return preset.run_config

# end_preset

# start_modified_preset

@daily_schedule(
    start_date=datetime.datetime(2020, 1, 1),
    pipeline_name="my_pipeline",
    solid_selection=preset.solid_selection,
    mode=preset.mode,
    tags_fn_for_date=lambda _: preset.tags,
)
def my_modified_preset_schedule(date):
    modified_run_config = preset.run_config.copy()
    modified_run_config["date"] = date.strftime("%Y-%m-%d")
    return modified_run_config

# end_modified_preset


from openerp import models
from openerp.report import report_sxw
import datetime
from datetime import date, datetime, timedelta
import dateutil.parser
import time
import calendar
from ..utility.utils import hour_float_to_time as hftt
from time import gmtime, strftime
import pytz

class employee_overtime_work_inspection_report(report_sxw.rml_parse):
    _name = 'report.sisb_hr.employee_overtime_work_inspection'
    def __init__(self, cr, uid, name, context=None):
        if context is None:
            context = {}
        super(employee_overtime_work_inspection_report, self).__init__(cr, uid, name, context = context)
        self.localcontext.update({
            'get_employee_ot': self._get_employee_ot,
            'get_daily_overtime': self._get_daily_overtime,
            'get_section_code': self._get_section_code,
        })

    def _get_section_code(self, company):
        section_code = ''
        if company.school_ids:
            for rec in company.school_ids:
                section_code = '00' + str(rec.sequence) + ' :Section : SISB-' + rec.code
                break
        else:
            section_code = 'Section: SISB'
        return section_code

    def _get_daily_overtime(self, employee_id, date_from , date_to, attendance_id):
        data = []
        print('attendance_id = ', attendance_id)
        emp_obj = self.pool.get('hr.employee')
        emp_id = emp_obj.search(self.cr, self.uid, [('id', '=', employee_id)])
        employee = emp_obj.browse(self.cr, self.uid, emp_id)
        all_day = emp_obj.delta_days(self.cr, self.uid, date_from, date_to)
        for att_id in attendance_id:
            print('att_id = ', att_id)
            self.cr.execute("SELECT name + INTERVAL '8 HOURS', sign_out + INTERVAL '8 HOURS' \
                            FROM hr_attendance \
                            WHERE id = %s \
                            ",(att_id,))
            ot_att = self.cr.fetchall()
            att_obj = self.pool.get('hr.attendance').browse(self.cr, self.uid, att_id)
            sign_in_obj = datetime.strptime(ot_att[0][0], '%Y-%m-%d %H:%M:%S')
            sign_out_obj = datetime.strptime(ot_att[0][1], '%Y-%m-%d %H:%M:%S')
            sign_in = sign_in_obj.strftime('%d/%b/%Y %H:%M')
            sign_out = sign_out_obj.strftime('%d/%b/%Y %H:%M')
            date_sign_in = datetime.strptime(ot_att[0][0], '%Y-%m-%d %H:%M:%S')
            day_name = date_sign_in.strftime('%d/%b/%Y') + ' [' + date_sign_in.strftime('%a') + ']'
            att_obj = self.pool.get('hr.attendance').browse(self.cr, self.uid, att_id)
            schedule = att_obj.shift_id
            tz  = pytz.timezone(schedule.default_timezone)
            tz_time = datetime.now(tz)
            year = date_sign_in.year
            month = date_sign_in.month
            day = date_sign_in.day
            # year = int(datetime.strftime(tz_time, '%Y'))
            # month = int(datetime.strftime(tz_time, '%m'))
            # day = int(datetime.strftime(tz_time, '%d'))
            today = calendar.weekday(year, month, day)
            check_shift =  [r.options for r in schedule.day_of_wewks_ids.filtered(lambda x: int(x.day_of_week) == today)]
            holiday_shift = True if check_shift[0] == 'holiday' else False
            holiday_year = emp_obj.check_holiday(self.cr, self.uid, employee, tz_time)
            day_option = 'N'
            if holiday_shift and not holiday_year:
                day_option = 'H'
            elif (holiday_shift and holiday_year) or (not holiday_shift and holiday_year):
                day_option = 'HD'
            for ot in att_obj.overtime_ids:
                print('ot_end = ', ot.end_ot)
                ot_day = hftt(ot.ot_rounded)
                ot_type = ot.ot_type_id.name
                ot_start = hftt(ot.start_ot)
                ot_end = hftt(ot.end_ot)
                ot_len = hftt(ot.end_ot - ot.start_ot)
                ot_one = hftt(ot.rate_one) if ot.rate_one > 0.00 else ''
                ot_one_half = hftt(ot.rate_one_half) if ot.rate_one_half > 0.00 else ''
                ot_double = hftt(ot.rate_double) if ot.rate_double > 0.00 else ''
                ot_triple = hftt(ot.rate_triple) if ot.rate_triple > 0.00 else ''
                # ot_length = hftt(ot.ot_length)
                result = {
                    'type': ot_type,
                    'date': day_name,
                    'schedule': att_obj.shift_id.name,
                    'day': day_option,
                    'sign_in': sign_in,
                    'sign_out': sign_out,
                    'one': ot_one,
                    'one_half': ot_one_half,
                    'double': ot_double,
                    'triple': ot_triple,
                    'ot_start': ot_start,
                    'ot_end': ot_end,
                    'ot_one': hftt(ot.rate_one),
                    'ot_one_half': hftt(ot.rate_one_half),
                    'ot_double': hftt(ot.rate_double),
                    'ot_triple': hftt(ot.rate_triple),
                    'ot_total': ot_len
                    # 'ot_length': ot_length

                }
                data.append(result)
        if data:
            return data
        else:
            return {}

        # for day in all_day['all_day']:
        #     self.cr.execute("SELECT name + INTERVAL '8 HOURS', sign_out + INTERVAL '8 HOURS' \
        #                     FROM sisb_attendance \
        #                     WHERE name + INTERVAL '8 HOURS' >= %s \
        #                     AND name + INTERVAL '8 HOURS' <= %s \
        #                     AND employee_id = %s\
        #                     AND overtime = true\
        #                     ",(day + ' 00:00:00', day + ' 23:59:59', employee_id))

    def _get_employee_ot(self, form):
        data = []
        emp_obj = self.pool.get('hr.employee')
        date_from = form['date_from']
        print('date_from = ', date_from)
        date_to = form['date_to']
        company = form['company_id']
        all_employee = self.pool.get('hr.employee').search(self.cr, self.uid, [('company_id', '=', company[0]),('active','=', True)])
        all_employee_id = self.pool.get('hr.employee').browse(self.cr, self.uid, all_employee)
        all_day = emp_obj.delta_days(self.cr, self.uid, date_from, date_to)
        for employee in all_employee_id:
            name = ''
            if employee.gender == 'male':
                name = employee.employee_no + ' ' + 'Mr.' + employee.name
            if employee.gender == 'female':
                name = employee.employee_no + ' ' + 'Mrs.' + employee.name
            if not employee.gender:
                name = employee.employee_no + ' ' + employee.name
            print('employee_id = ', employee.id)
            self.cr.execute("SELECT id FROM hr_attendance\
                            WHERE name + INTERVAL '8 HOURS' >= %s\
                            AND name + INTERVAL '8 HOURS' <= %s\
                            AND sign_out IS NOT NULL \
                            AND employee_id = %s \
                            AND overtime = true \
                            ORDER BY name ASC \
                            ",(date_from + ' 00:00:00', date_to + ' 23:59:59', employee.id))
            emp_ot = self.cr.fetchall()
            if not emp_ot:
                continue
            emp_ot_id = []
            total_one = total_one_half = total_double = total_triple = 0.00
            one = one_half = double = triple = ''
            for ot_id in emp_ot:
                if ot_id not in emp_ot_id:
                    emp_ot_id.append(ot_id[0])
            emp_att_id = self.pool.get('hr.attendance').browse(self.cr, self.uid, emp_ot_id)
            for ot_list in emp_att_id:
                for ot in ot_list.overtime_ids:
                    total_one += ot.rate_one
                    total_one_half += ot.rate_one_half
                    total_double += ot.rate_double
                    total_triple += ot.rate_triple
            if total_one > 0.00:
                one = hftt(total_one)
            if total_one_half > 0.00:
                one_half = hftt(total_one_half)
            if total_double > 0.00:
                double = hftt(total_double)
            if total_triple > 0.00:
                triple = hftt(total_triple)

            result = {
                'name': name,
                'ot_one': one,
                'ot_one_half': one_half,
                'ot_double': double,
                'ot_triple': triple,
                'employee_id': employee.id,
                'date_from': date_from,
                'date_to': date_to,
                'attendance_id': emp_ot_id
            }
            data.append(result)
        if data:
            return data
        else:
            return {}

class report_overtime_work_inspection(models.AbstractModel):
    _name = 'report.sisb_hr.employee_overtime_work_inspection'
    _inherit = 'report.abstract_report'
    _template = 'sisb_hr.employee_overtime_work_inspection'
    _wrapped_report_class = employee_overtime_work_inspection_report


"""
@author:Liushihao
@time:2020/3/26:19:28
@email:Liushihao_1224@163.com
@describe:
"""
list = [3, 8, 11, 26, 47]
a = int(input("输入一个新元素:"))
insert_index = 0
for i in range(0, len(list)):
    if a>list[i]:
        insert_index+=1
    else:
        break

list.insert(insert_index, a)
print(list)


import cv2
import os
def format_image(img_path, size, nb_channels):
    """
    Load img with opencv and reshape
    """
    img = cv2.imread(img_path)

    img = cv2.resize(img, (size, size), interpolation=cv2.INTER_AREA)

    if nb_channels == 1:
        img = np.expand_dims(img, -1)
    img = cv2.resize(img, (size, size), interpolation=cv2.INTER_AREA)
    img = np.expand_dims(img, 0).transpose(0, 3, 1, 2)

    return img
def build_HDF5(jpeg_dir, nb_channels, size=224):
    """
    Gather the data in a single HDF5 file.
    """

    # Put train data in HDF5
    file_name = os.path.basename(jpeg_dir.rstrip("/"))
    hdf5_file = os.path.join(data_dir, "%s_data.h5" % file_name)
    with h5py.File(hdf5_file, "w") as hfw:
        for dset_type in ["train", "test", "val"]:

            list_img = [img for img in Path(jpeg_dir).glob('%s/img_*.jpg' % dset_type)]
            list_img = [str(img) for img in list_img]
           # list_img.extend(list(Path(jpeg_dir).glob('%s/*.png' % dset_type)))
            list_img = list(map(str, list_img))
            list_img = np.array(list_img)

            list_img1 = [img for img in Path(jpeg_dir).glob('%s/dep_*.jpg' % dset_type)]
            list_img1 = [str(img) for img in list_img1]
           # list_img.extend(list(Path(jpeg_dir).glob('%s/*.png' % dset_type)))
            list_img1 = list(map(str, list_img1))
            list_img1 = np.array(list_img1)

            data_full = hfw.create_dataset("%s_data_full" % dset_type,
                                           (0, nb_channels, size, size),
                                           maxshape=(None, 3, size, size),
                                           dtype=np.uint8)

            data_sketch = hfw.create_dataset("%s_data_sketch" % dset_type,
                                             (0, nb_channels, size, size),
                                             maxshape=(None, 3, size, size),
                                             dtype=np.uint8)

            num_files = len(list_img)
            chunk_size = 100
            num_chunks = num_files / chunk_size
            arr_chunks = np.array_split(np.arange(num_files), num_chunks)

            for chunk_idx in tqdm(arr_chunks):

                list_img_path = list_img[chunk_idx].tolist()
                output = parmap.map(format_image, list_img_path, size, nb_channels, pm_parallel=False)

                arr_img_full = np.concatenate([o for o in output], axis=0)

                # Resize HDF5 dataset
                data_full.resize(data_full.shape[0] + arr_img_full.shape[0], axis=0)
                data_full[-arr_img_full.shape[0]:] = arr_img_full.astype(np.uint8)

            num_files = len(list_img1)
            chunk_size = 100
            num_chunks = num_files / chunk_size
            arr_chunks = np.array_split(np.arange(num_files), num_chunks)

            for chunk_idx in tqdm(arr_chunks):
                list_img_path = list_img1[chunk_idx].tolist()
                output = parmap.map(format_image, list_img_path, size, nb_channels, pm_parallel=False)

                arr_img_dep = np.concatenate([o for o in output], axis=0)

                # Resize HDF5 dataset
                data_dep.resize(data_dep.shape[0] + arr_img_dep.shape[0], axis=0)
                data_dep[-arr_img_dep.shape[0]:] = arr_img_dep.astype(np.uint8)


import torch as T
from .module_fn import fit_loop
from ..callbacks.metrics import (MetricsAccumulator,
                                 MetricsCalculator,
                                 MetricsExecutor)
from ..callbacks.loggers import (ProgressLogger,
                                 LoggerExecutor)
from ..callbacks.schedulers import (SchedulersExecutor)
from ..callbacks import Executor
from .type_utils import (_is_list, _is_tuple, _is_dict, _check_metric, _is_str,
                         _is_optimizer_type, _is_optimizer_object, _get_optimizer,
                         _parse_optim_args)

class Module(T.nn.Module):
    def __init__(self, *args, **kwargs):
        super(Module, self).__init__(*args, **kwargs)

        self.metrics = 0
        self.metric_cb = 0

        self.logger_cb = 0
        self.sheduler_cb = 0

        self.loss_fn = 0
        self.optimizer = 0
        self.device = 'cpu'

    def init_weights(self, *args, **kwargs):
        pass

    def set_criterion(self, criterion):
        self.loss_fn = criterion

    def set_metrics(self, metrics):
        self.metrics = dict()

        if metrics is None:
            return

        if _is_list(metrics) or _is_tuple(metrics):
            for m in metrics:
                self.metrics[m.__class__.__name__] = _check_metric(m)
        elif _is_dict(metrics):
            for n, m in metrics.items():
                self.metrics[n] = _check_metric(m)

        self.metric_cb = MetricsExecutor([
            MetricsCalculator(self.metrics),
            MetricsAccumulator(self.metrics)
        ])

    def set_optimizer(self, optimizer, **kwargs):
        opt_type = None
        if _is_optimizer_object(optimizer):
            self.optimizer = optimizer
            return
        elif _is_optimizer_type(optimizer):
            opt_type = optimizer
        else:
            opt_type = _get_optimizer(optimizer)

        args = _parse_optim_args(opt_type, **kwargs)

        if 'params' in args:
            self.optimizer = opt_type(**args)
        else:
            args.update(params=self.parameters())
            self.optimizer = opt_type(**args)

    def set_schedulers(self, schedulers):
        self.scheduler_cb = SchedulersExecutor(schedulers, self)

    def compile(self, criterion, optimizer='adam', metrics=None, schedulers=None, **kwargs):
        if metrics is not None:
            self.set_metrics(metrics)

        self.set_optimizer(optimizer, **kwargs)
        self.set_criterion(criterion)

        if schedulers is not None:
            self.set_schedulers(schedulers)

        self.logger_cb = LoggerExecutor([
            ProgressLogger()
        ])

        self.callbacks = Executor(self.metric_cb.callbacks +
                                  self.logger_cb.callbacks +
                                  self.scheduler_cb.callbacks)

        self.device = T.device("cuda:0" if T.cuda.is_available() else "cpu")

    def fit(self, dataset, nb_epochs):
        # print(self)
        fit_loop(model=self,
                 train_loader=dataset.train,
                 val_loader=dataset.val,
                 nb_epochs=nb_epochs,
                 device=self.device,
                 metrics=self.metrics,
                 callbacks=self.callbacks)

    def evaluate(self, val_loader):
        pass

    def save_weights(self, filename):
        T.save(self.state_dict(), filename)

    def save_model(self, filename):
        T.save(self, filename)