import apache_beam as beam
import re
import itertools
# for datetime manipulation
from datetime import datetime
from apache_beam.options.pipeline_options import PipelineOptions, StandardOptions
from apache_beam.runners.runner import PipelineState

import argparse

def cleandata(element):
    element = re.sub('\"|\'', '', element)
    thisTuple=element.split(',')
    return thisTuple

def dict_toList(element):
    item = element[0].split(',')
    for key,value in element[1].items():
        for v in value:
            if(isinstance(v, dict)):
                for key,value in v.items():
                    item = item + [str(value[0])]
            else:
                item = item + [str(v)]
    return item

def retReview(element):
    if element is not None:
        if len(element) >= 3:
            if len(element[1]) == 32:
                return (element[1],element[2])

def retSeller(element):
    return (element[0],element[2],element[3])

def retCustomer(element):
    return (element[0],element[3],element[4])

def retProd(element):
    return (element[0],element[1])

def reTupless(element):
    l = list(element[1].keys())
    for i in range(len(l)-1):
        item = element[1].get(l[i])
        for x in item:
            for y in element[1].get(l[len(l)-1]):
                if(isinstance(y, list)):
                    yield list(x) + list(y)
                else:
                    yield list(x) + list(y )

def ListtoStr(element):
    element = ','.join(map(str, element))
    return element

parser = argparse.ArgumentParser()

parser.add_argument('--input_itens',dest='itens',required=True, help='Input itens file to process.')
parser.add_argument('--input_sellers',dest='seller',required=True, help='Input seller file to process.')
parser.add_argument('--input_products',dest='products',required=True, help='Input products file to process.')
parser.add_argument('--input_order',dest='order',required=True, help='Input order file to process.')
parser.add_argument('--input_reviews',dest='reviews',required=True, help='Input reviews file to process.')
parser.add_argument('--input_payments',dest='payments',required=True, help='Input payments file to process.')
parser.add_argument('--input_customer',dest='customer',required=True, help='Input customer file to process.')

parser.add_argument('--output',dest='output',required=True,help='Output table to write results to.')

path_args, pipeline_args = parser.parse_known_args()

input_itens = path_args.itens
input_seller = path_args.seller
input_products = path_args.products
input_order = path_args.order
input_reviews= path_args.reviews
input_payments = path_args.payments
input_customer = path_args.customer

output = path_args.output

options = PipelineOptions(pipeline_args)

p1 = beam.Pipeline(options=options)

output_header='order_id,payment_sequential,payment_type,payment_installments,payment_value'
output_header=output_header+',order_id,customer_id,order_status,order_purchase_timestamp,order_approved_at,order_delivered_carrier_date'
output_header=output_header+',order_delivered_customer_date,order_estimated_delivery_date,review_score'
output_header=output_header+',customer_id,customer_city,customer_state'
output_header=output_header+',order_id, product_id, seller_id ,shipping_limit_date, itens_count, price, freight_value'
output_header=output_header+',seller_id , seller_city,seller_state,product_id,product_category_name'

#itens orders

input_itens_orders = (
                      p1
                      | 'Read itens_orders data' >> beam.io.ReadFromText(input_itens ,skip_header_lines=1)
                      | 'Clean data itens_orders' >> beam.Map(cleandata)
                   )

itens_sum_price = (
        input_itens_orders

            | 'dict for sum prices' >> beam.Map(lambda item: (item[0]+','+item[2]
                                                                            +','+item[3]+','+item[4], float(item[5])) )
            | 'Group sum prices' >> beam.CombinePerKey(sum)
#            | 'Write sum prices' >> beam.io.WriteToText('output/sum_prices.txt')
)

itens_count = (
        input_itens_orders
            | 'dict for count itens' >> beam.Map(lambda item: (item[0]+','+item[2]
                                                                            +','+item[3]+','+item[4], int(item[1])) )
            | 'count itens' >> beam.combiners.Count.PerKey()
#            | 'Write count itens' >> beam.io.WriteToText('output/count_itens.txt')
)

join1 = ({'itens_count': itens_count, 'itens_sum_price': itens_sum_price}
           | 'itens_count + itens_sum_price' >> beam.CoGroupByKey()
#           | 'mergeddicts 1' >> beam.Map(dict_toList)
#           | 'dict for join1' >> beam.Map(lambda item: (item[0]+','+item[1]+','+item[3]+','+item[4], int(item[1])) )

#           | 'Write results' >> beam.io.WriteToText('output/result.txt')
)

freight_value = (
        input_itens_orders
            | 'dict for sum freight_value' >> beam.Map(lambda item: (item[0]+','+item[2]
                                                                            +','+item[3]+','+item[4], float(item[6])) )
            | 'Group sum freight_value' >> beam.CombinePerKey(sum)
#            | 'Write count itens' >> beam.io.WriteToText('output/freight_value.txt')
)

join2 = ({'join1': join1, 'freight_value': freight_value}
            | 'join1 + freight_value' >> beam.CoGroupByKey()
            | 'dict to list ' >> beam.Map(dict_toList)
            | 'create dict join2 ' >> beam.Map(lambda itens: (itens[2], itens))
#            | 'Write results 2' >> beam.io.WriteToText('output/result_join2.txt')
)

sellers = (
        p1
            | 'Read seller data' >> beam.io.ReadFromText(input_seller,skip_header_lines=1)
            | 'Clean data seller' >> beam.Map(cleandata)
            | 'return seller data except geo id' >> beam.Map(retSeller)
            | 'create dict from seller' >> beam.Map(lambda sellers: (sellers[0], sellers))
#            | 'Write seller data' >> beam.io.WriteToText('outputs/dept.txt')
)

join3 = ({'join2': join2, 'sellers': sellers}
            | 'join2 + sellers' >> beam.CoGroupByKey()
#            | 'dict to list sellers ' >> beam.Map(dict_toList)
            | 'dict to list sellers' >> beam.FlatMap(reTupless)
            | 'create dict join3 ' >> beam.Map(lambda itens: (itens[1], itens))#verificar essa linha
#            | 'Write results 3' >> beam.io.WriteToText('output/_dic_prod_certo.txt')
)

#"product_id","product_category_name","product_name_lenght","product_description_lenght",
#"product_photos_qty","product_weight_g","product_length_cm","product_height_cm","product_width_cm"

products = (
        p1
            | 'Read prod data' >> beam.io.ReadFromText(input_products,skip_header_lines=1)
            | 'Clean data prod' >> beam.Map(cleandata)
            | 'return prod data ' >> beam.Map(retProd)
            | 'create dict from prod' >> beam.Map(lambda prod: (prod[0], prod))
#            | 'Write seller data' >> beam.io.WriteToText('output/prod.txt')
)

join4 = ({'join3': join3, 'products': products}
            | 'join3 + products' >> beam.CoGroupByKey()
#            | 'dict to list join4' >> beam.Map(dict_toList)
            | 'dict to list join4' >> beam.FlatMap(reTupless)
            | 'dict order id key join 4 ' >> beam.Map(lambda itens: (itens[0], itens))
#            | 'Write results 4' >> beam.io.WriteToText('output/result_itenssss.txt')
)

order = (
        p1
            | 'Read orders data' >> beam.io.ReadFromText(input_order,skip_header_lines=1)
            | 'Clean data orders' >> beam.Map(cleandata)
#            | 'return seller data except geo id' >> beam.Map(retSeller)
            | 'create dict from orders' >> beam.Map(lambda order: (order[0], order))
#            | 'Write seller data' >> beam.io.WriteToText('outputs/dept.txt')
)

reviews = (
        p1
            | 'Read reviews data' >> beam.io.ReadFromText(input_reviews,skip_header_lines=1)
            | 'Clean reviews order' >> beam.Map(cleandata)
            | 'return order id and review score' >> beam.Map(retReview)
            | 'Check reviews only with id' >> beam.Filter(lambda rev: rev is not None )
            | 'create dict from reviews' >> beam.Map(lambda reviews: (reviews[0], reviews[1]))
#            | 'Write reviews data' >> beam.io.WriteToText('outputs/reviews.txt')
)

join5 = ({'order': order, 'reviews': reviews}
            | 'orders + reviews' >> beam.CoGroupByKey()
            | 'dict to list join5' >> beam.FlatMap(reTupless)
            | 'create dict join 5 ' >> beam.Map(lambda itens: (itens[0], itens))
#            | 'Write results join 1' >> beam.io.WriteToText('output/result_order.txt')
)

payments = (
        p1
            | 'Read payments data' >> beam.io.ReadFromText(input_payments,skip_header_lines=1)
            | 'Clean payments order' >> beam.Map(cleandata)
            | 'create dict from payments' >> beam.Map(lambda reviews: (reviews[0], reviews))
#            | 'Write payments' >> beam.io.WriteToText('output/payment.txt')
)

join6 = ({ 'payments': payments,'join5': join5}
            | ' payments + join5 ' >> beam.CoGroupByKey()
            | 'dict to list join6' >> beam.FlatMap(reTupless)
            | 'create dict join 6' >> beam.Map(lambda itens: (itens[6], itens))
#            | 'Write results join 2' >> beam.io.WriteToText('output/result_payments.txt')
)

customer = (
        p1
            | 'Read customer data' >> beam.io.ReadFromText(input_customer,skip_header_lines=1)
            | 'Clean customer order' >> beam.Map(cleandata)
            | 'return customer id ,state and City' >> beam.Map(retCustomer)
            | 'create dict from customer' >> beam.Map(lambda reviews: (reviews[0], reviews))
)

join7 = ({ 'join6': join6, 'customer': customer}
            | 'customer + join6' >> beam.CoGroupByKey()
            | 'dict to list join7' >> beam.FlatMap(reTupless)
            | 'dict order id key join 7  ' >> beam.Map(lambda orders: (orders[0], orders))
#            | 'Write results join 7' >> beam.io.WriteToText('output/result_orders_esse.txt')
)

join8 = ({ 'join7': join7, 'join4': join4}
            | 'join7 + join4' >> beam.CoGroupByKey()
            | 'dict to list join8' >> beam.FlatMap(reTupless)
            | 'List to string' >> beam.Map(ListtoStr)
            | 'Write results join 8' >> beam.io.WriteToText(output,file_name_suffix='.csv',header=output_header)
)

p1.run()


import logging
import json
from vwsfriend.homekit.dummy_accessory import DummyAccessory
import pyhap

from weconnect.weconnect import WeConnect

from .climatization import Climatization
# from .battery import Battery
from .charging import Charging
from .plug import Plug
from .locking_system import LockingSystem

from vwsfriend.__version import __version__

LOG = logging.getLogger("VWsFriend")

class VWsFriendBridge(pyhap.accessory.Bridge):
    """VWsfriend Bridge"""

    def __init__(self, weConnect: WeConnect, driver, displayName='VWsFriend', accessoryConfigFile=None):
        super().__init__(driver=driver, display_name=displayName, )

        self.set_info_service(__version__, "Till Steinbach", "VWsFriend", None)

        self.weConnect: WeConnect = weConnect
        self.driver = driver

        self.accessoryConfigFile = accessoryConfigFile
        self.__accessoryConfig = {}
        self.nextAID = 100
        try:
            self.readConfig()
        except FileNotFoundError:
            pass

        for identifier, accessory in self.__accessoryConfig.items():
            if 'ConfiguredName' in accessory:
                displayName = accessory['ConfiguredName']
            else:
                displayName = identifier
            placeholderAccessory = DummyAccessory(driver=driver, displayName=displayName, aid=accessory['aid'])
            self.add_accessory(placeholderAccessory)

    def persistConfig(self):
        if self.accessoryConfigFile:
            try:
                with open(self.accessoryConfigFile, 'w') as file:
                    json.dump(self.__accessoryConfig, fp=file)
                LOG.info('Writing accessory config file %s', self.accessoryConfigFile)
            except ValueError as err:  # pragma: no cover
                LOG.info('Could not write homekit accessoryConfigFile %s (%s)', self.accessoryConfigFile, err)

    def readConfig(self):
        with open(self.accessoryConfigFile, 'r') as file:
            aid = json.load(fp=file)
            self.__accessoryConfig = aid
        LOG.info('Reading homekit accessory config file %s', self.accessoryConfigFile)
        for accessoryConfig in self.__accessoryConfig.values():
            if 'aid' in accessoryConfig:
                if (accessoryConfig['aid'] + 1) > self.nextAID:
                    self.nextAID = (accessoryConfig['aid'] + 1)

    def update(self):  # noqa: C901
        configChanged = False
        for vehicle in self.weConnect.vehicles.values():
            manufacturer = 'Volkswagen'
            nickname = vehicle.nickname.value
            model = vehicle.nickname.value
            vin = vehicle.vin.value
            # Climatization
            if 'climatisationStatus' in vehicle.statuses:
                climatizationStatus = vehicle.statuses['climatisationStatus']

                if 'climatisationSettings' in vehicle.statuses:
                    climatizationSettings = vehicle.statuses['climatisationSettings']
                else:
                    climatizationSettings = None

                if 'batteryStatus' in vehicle.statuses:
                    batteryStatus = vehicle.statuses['batteryStatus']
                else:
                    batteryStatus = None

                if 'chargingStatus' in vehicle.statuses:
                    chargingStatus = vehicle.statuses['chargingStatus']
                else:
                    chargingStatus = None

                climatizationAccessory = Climatization(driver=self.driver, bridge=self, aid=self.selectAID('Climatization', vin), id='Climatization', vin=vin,
                                                       displayName=f'{nickname} Climatization', climatizationStatus=climatizationStatus,
                                                       climatizationSettings=climatizationSettings, batteryStatus=batteryStatus, chargingStatus=chargingStatus,
                                                       climatizationControl=vehicle.controls.climatizationControl)
                climatizationAccessory.set_info_service(manufacturer=manufacturer, model=model, serial_number=f'{vin}-climatization')
                if climatizationAccessory.aid not in self.accessories:
                    self.add_accessory(climatizationAccessory)
                else:
                    self.accessories[climatizationAccessory.aid] = climatizationAccessory
                configChanged = True

            # if 'batteryStatus' in vehicle.statuses:
            #     batteryStatus = vehicle.statuses['batteryStatus']

            #     if 'chargingStatus' in vehicle.statuses:
            #         chargingStatus = vehicle.statuses['chargingStatus']
            #     else:
            #         chargingStatus = None

            #     batteryAccessory = Battery(driver=self.driver, bridge=self, aid=self.selectAID('Battery', vin), id='Battery', vin=vin,
            #                                displayName=f'{nickname} Battery', batteryStatus=batteryStatus, chargingStatus=chargingStatus)
            #     batteryAccessory.set_info_service(manufacturer=manufacturer, model=model, serial_number=f'{vin}-battery')
            #     self.add_accessory(batteryAccessory)
            #     configChanged = True

            if 'chargingStatus' in vehicle.statuses:
                chargingStatus = vehicle.statuses['chargingStatus']

                if 'plugStatus' in vehicle.statuses:
                    plugStatus = vehicle.statuses['plugStatus']
                else:
                    plugStatus = None

                if 'batteryStatus' in vehicle.statuses:
                    batteryStatus = vehicle.statuses['batteryStatus']
                else:
                    batteryStatus = None

                chargingAccessory = Charging(driver=self.driver, bridge=self, aid=self.selectAID('Charging', vin), id='Charging', vin=vin,
                                             displayName=f'{nickname} Charging', chargingStatus=chargingStatus, plugStatus=plugStatus,
                                             batteryStatus=batteryStatus, chargingControl=vehicle.controls.chargingControl)
                chargingAccessory.set_info_service(manufacturer=manufacturer, model=model, serial_number=f'{vin}-charging')
                if chargingAccessory.aid not in self.accessories:
                    self.add_accessory(chargingAccessory)
                else:
                    self.accessories[chargingAccessory.aid] = chargingAccessory
                configChanged = True

            if 'plugStatus' in vehicle.statuses:
                plugStatus = vehicle.statuses['plugStatus']

                plugAccessory = Plug(driver=self.driver, bridge=self, aid=self.selectAID('ChargingPlug', vin), id='ChargingPlug', vin=vin,
                                     displayName=f'{nickname} Charging Plug', plugStatus=plugStatus)
                plugAccessory.set_info_service(manufacturer=manufacturer, model=model, serial_number=f'{vin}-charging_plug')
                if plugAccessory.aid not in self.accessories:
                    self.add_accessory(plugAccessory)
                else:
                    self.accessories[plugAccessory.aid] = plugAccessory
                configChanged = True

            if 'accessStatus' in vehicle.statuses:
                accessStatus = vehicle.statuses['accessStatus']

                lockingSystemAccessory = LockingSystem(driver=self.driver, bridge=self, aid=self.selectAID('LockingSystem', vin), id='LockingSystem', vin=vin,
                                                       displayName=f'{nickname} Locking System', accessStatus=accessStatus)
                lockingSystemAccessory.set_info_service(manufacturer=manufacturer, model=model, serial_number=f'{vin}-locking_system')
                if lockingSystemAccessory.aid not in self.accessories:
                    self.add_accessory(lockingSystemAccessory)
                else:
                    self.accessories[lockingSystemAccessory.aid] = lockingSystemAccessory
                configChanged = True

        if configChanged:
            self.driver.config_changed()
            self.persistConfig()

    def selectAID(self, id, vin):
        aid = None
        identifier = f'{vin}-{id}'
        if identifier in self.__accessoryConfig and 'aid' in self.__accessoryConfig[identifier]:
            aid = self.__accessoryConfig[identifier]['aid']
        else:
            aid = self.nextAID
            self.nextAID += 1
            if identifier in self.__accessoryConfig:
                self.__accessoryConfig[identifier]['aid'] = aid
            else:
                self.__accessoryConfig[identifier] = {'aid': aid}
        return aid

    def setConfigItem(self, id, vin, configKey, item):
        identifier = f'{vin}-{id}'
        if identifier in self.__accessoryConfig:
            self.__accessoryConfig[identifier][configKey] = item
        else:
            self.__accessoryConfig[identifier] = {configKey: item}
        self.nextAID += 1

    def getConfigItem(self, id, vin, configKey):
        identifier = f'{vin}-{id}'
        if identifier in self.__accessoryConfig and configKey in self.__accessoryConfig[identifier]:
            return self.__accessoryConfig[identifier][configKey]
        return None


from bs4 import BeautifulSoup
import requests

def scrape_results(search_query):
    html = requests.get(f"https://www.midiworld.com/search/?q={search_query}").text
    soup = BeautifulSoup(html, "html.parser")
    error = soup.find("div", {"id": "page"}).text
    if "found nothing!" in error:
        return None
    unordered_lists = soup.find_all("ul")[1]
    lists = unordered_lists.find_all("li")
    results = [{"name": lst.text.replace("\n", "").split(" - download")[0], "url": lst.find("a")["href"]} for lst in lists]
    return results

def save_midi(url, name):
    song = requests.get(url)
    statuscode = song.status_code
    if statuscode == 200:
        open(f"songs/{name}.mid", "wb").write(song.content)
        return True # changed from name
    else:
        return False

def main(query):
    results = scrape_results(query)
    return results

if __name__ == '__main__':
    main("")


# -*- coding: utf-8 -*-
from road import road
from auths import road_engine

from sqlalchemy.orm import sessionmaker

road_Session = sessionmaker(bind = road_engine)

_road_session = road_Session()

from sqlalchemy import distinct, asc, and_
from datetime import datetime, timedelta
import simplekml

def get_tmc_by_id_timedelta(_rdid, _headtime, _timedelta):

	head_time = _headtime
	tail_time = _headtime + _timedelta

	for tmc in _tmc_session.query(tmc_point.Speed, tmc_point.Record_Time).filter(and_(tmc_point.RoadID == _rdid, tmc_point.Record_Time >= head_time, tmc_point.Record_Time <= tail_time)).all():
		if len(tmc) >= 0:
			speed, _time = tmc
			yield _time, speed

def get_geo(rdid):
	from sqlalchemy import text
	rdid = abs(int(rdid))
	cmd = "select Geom.ToString() from Table_RoadData where RoadID = {0}".format(rdid)
	linestring = road_engine.execute(cmd).fetchone()
	if linestring == None:
		cmd = "select Geom.ToString() from Table_RoadData where RoadID = {0}".format(int(rdid))
		linestring = road_engine.execute(cmd).fetchone()
		if linestring == None:
			return None
	return str(linestring[0])

def parse_linestring_to_coords(linestring):
	_return = []
	import re
	match = re.search(r".*\((.*)\)", linestring)
	string_coords = match.group(1).split(', ')

	for coord in string_coords:
		lon, lat = coord.split(" ")
		_return.append((float(lon), float(lat)))
	return _return

def set_LOS_color(speed):

	speed = int(speed)
	if speed >= 60:
		return simplekml.Color.green
	elif speed < 60 and speed >= 40:
		return simplekml.Color.yellow
	elif speed < 40 and speed >= 20:
		return simplekml.Color.orange
	elif speed < 20:
		return simplekml.Color.red

def draw_kml_color_by_speed(kml, speed, rdid, coords):
	ln = kml.newlinestring(name=str(rdid))
	ln.description = "RoadID:{0} speed:{1}".format(rdid, speed)
	ln.coords =coords
	ln.extrude=1
	ln.linestyle.width = 5
	ln.linestyle.color = set_LOS_color(speed)

def get_roadids_by_chiname(chiname):
	'''
	to get roadid by (chinese name)
	'''
	from sqlalchemy import and_, or_
	chiname = "%{0}%".format(chiname)
	return [rd[0] for rd in _road_session.query(road.RoadID).filter(and_(road.Name.like(chiname)), or_(road.City==0, road.City==1)).all()]

def get_roadname_by_roadid(rdid):
	rdid = _road_session.query(road.Name).filter(road.RoadID == rdid).all()
	return rdid[0][0]


import numpy as np
import cv2

def split_gt(gt):
    gt = gt.data.cpu().numpy()
    kp = gt[:5,:,:]
    short = np.transpose(gt[5:5+10,:,:], (1,2,0))
    mid = np.transpose(gt[5+10:,:,:], (1,2,0))
    return kp, short ,mid

def show_input(input):
    colors = [(0,0,0.9),(0.9,0,0),(0.9,0,0.9),(0.9,0.9,0), (0.2,0.9,0.9)]
    img, gt_c0, gt_c1, gt_c2, gt_c3, instance_masks, bboxes_c0 = input

    kp0, short0, mid0 = split_gt(gt_c0)
    kp1, short1, mid1 = split_gt(gt_c1)
    kp2, short2, mid2 = split_gt(gt_c2)
    kp3, short3, mid3 = split_gt(gt_c3)

    img = (img+0.5)*255
    img = np.transpose(img.data.cpu().numpy(), (1,2,0))

    map_s0 = np.zeros((short0.shape[:2]), dtype=np.float64)
    for i_kp in range(5):
        offset = short0[:, :, i_kp * 2:i_kp * 2 + 2]
        map_s0 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    cv2.imshow('short0', np.uint8((map_s0 * 200)))

    # map_m0 = np.zeros((mid0.shape[:2]), dtype=np.float64)
    # for i_kp in range(20):
    #     offset = mid1[:, :, i_kp * 2:i_kp * 2 + 2]
    #     map_m0 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    # cv2.imshow('mid0', np.uint8((map_m0 * 200)))

    map_s1 = np.zeros((short1.shape[:2]), dtype=np.float64)
    for i_kp in range(5):
        offset = short1[:, :, i_kp * 2:i_kp * 2 + 2]
        map_s1 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    cv2.imshow('short1', np.uint8((map_s1 * 200)))

    # map_m1 = np.zeros((mid1.shape[:2]), dtype=np.float64)
    # for i_kp in range(20):
    #     offset = mid1[:, :, i_kp * 2:i_kp * 2 + 2]
    #     map_m1 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    # cv2.imshow('mid1', np.uint8((map_m1 * 200)))

    map_s2 = np.zeros((short2.shape[:2]), dtype=np.float64)
    for i_kp in range(5):
        offset = short2[:, :, i_kp * 2:i_kp * 2 + 2]
        map_s2 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    cv2.imshow('short2', np.uint8((map_s2 * 200)))

    # map_m2 = np.zeros((mid2.shape[:2]), dtype=np.float64)
    # for i_kp in range(20):
    #     offset = mid2[:, :, i_kp * 2:i_kp * 2 + 2]
    #     map_m2 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    #     cv2.imshow('mid2', np.uint8((map_m2 * 200)))

    map_s3 = np.zeros((short3.shape[:2]), dtype=np.float64)
    for i_kp in range(5):
        offset = short3[:, :, i_kp * 2:i_kp * 2 + 2]
        map_s3 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    cv2.imshow('short3', np.uint8((map_s3 * 200)))

    # map_m3 = np.zeros((mid3.shape[:2]), dtype=np.float64)
    # for i_kp in range(20):
    #     offset = mid3[:, :, i_kp * 2:i_kp * 2 + 2]
    #     map_m3 += np.sqrt(offset[:, :, 0] ** 2 + offset[:, :, 1] ** 2)
    # cv2.imshow('mid3', np.uint8((map_m3 * 200)))

    cv2.imshow('img', np.uint8(img))

    k = cv2.waitKey(0)
    if k & 0xFF == ord('q'):
        cv2.destroyAllWindows()
        exit()


import re
from janome.tokenizer import Tokenizer

with open("before_jp_message.txt", mode="r", encoding="utf-8") as f:
    message_original = f.read()

message = re.sub("[｜ 　「」\n]", "", message_original) # remove 2-byte space, 1-byte space, and return char

separator = "。"  # Japanese sentence separator

message_list = message.split(separator)
message_list.pop()
message_list = [x+separator for x in message_list]

t = Tokenizer()

message_words = []

for sentence in message_list:
    message_words.append(t.tokenize(sentence, wakati=True))

print(message_words)

all_words = ''

for sentence in message_words:
    for words in sentence:
        all_words = all_words + " " + words

with open('after_jp_message.txt', mode='w') as f:
    f.write(all_words)


#
# insert.py
#
# by - sam nnodim (son2105)
#
# adapted from: https://github.com/mikedewar/RealTimeStorytelling/blob/master/2/insert.py

# import some useful libs
import json
import sys
import redis

# Connect a redis database
conn = redis.Redis()

while True:
    # read a line from stdin
    line = sys.stdin.readline()

    # parse the data as a json obj.
    data = json.loads( line )
    distance = data["stormDist"]
    delta = data["delta"]

    # Add object to database w/ an expiration date
    # This is so that we can keep a window of measurements
    # For which to calculate rates
    conn.setex(distance, delta, 120)

    # Print the data to stdout so it can be
    # resued in the pipeline
    print ( json.dumps({"stormDist":distance, "delta":delta}) )
    sys.stdout.flush()


# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the BSD 3-Clause License  (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://opensource.org/licenses/BSD-3-Clause
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import torch
import pickle
import numpy as np
import pandas as pd
from config import model_config as config

from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score

import itertools
import matplotlib.pyplot as plt

def load_data(batched=True, test=False, file_dir='./inputs/data_url_0/combined/combined_features.pkl'):
    bs = config['batch_size']
    ftype = 'test' if test else 'train'

    with open('{}'.format(file_dir), 'rb') as f:
        features = pickle.load(f)

    x = features['x_{}'.format(ftype)]
    y = features['y_{}'.format(ftype)]
    data = (x, y)
    if test or not batched:
        return [torch.FloatTensor(data[0]), torch.LongTensor(data[1])]
    data = list(zip(data[0], data[1]))
    n_iters = len(data) // bs
    batches = []
    for i in range(1, n_iters + 1):
        input_batch = []
        output_batch = []
        for e in data[bs * (i-1):bs * i]:
            input_batch.append(e[0])
            output_batch.append(e[1])
        batches.append([torch.FloatTensor(input_batch),
                        torch.LongTensor(output_batch)])
    return batches

def evaluate(targets, predictions):
    performance = {
        'acc': accuracy_score(targets, predictions),
        'f1': f1_score(targets, predictions, average='macro'),
        'precision': precision_score(targets, predictions, average='macro'),
        'recall': recall_score(targets, predictions, average='macro')}
    return performance

def plot_confusion_matrix(targets, predictions, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    # plt.figure(figsize=(8,8))
    cm = confusion_matrix(targets, predictions)
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')


import os
os.environ['THEANO_FLAGS'] = 'device=cpu'

import argparse
from ast import literal_eval
import random
import shutil

import numpy as np

import config
from model import build_model_test, Agent
from state import NormState
from environments import RunEnv2

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--accuracy', dest='accuracy', action='store', default=5e-5, type=float)
    parser.add_argument('--modeldim', dest='modeldim', action='store', default='3D',
            choices=('3d', '2d', '3D', '2D'), type=str)
    parser.add_argument('--prosthetic', dest='prosthetic', action='store', default=True, type=bool)
    parser.add_argument('--difficulty', dest='difficulty', action='store', default=0, type=int)
    parser.add_argument('--episodes', type=int, default=10, help="Number of test episodes.")
    parser.add_argument('--critic_layers', type=str, default='(64,32)', help="critic hidden layer sizes as tuple")
    parser.add_argument('--actor_layers', type=str, default='(64,64)', help="actor hidden layer sizes as tuple")
    parser.add_argument('--layer_norm', action='store_true', help="Use layer normalization.")
    parser.add_argument('--weights', type=str, default=None, help='weights to load')
    parser.add_argument('--skip_frames', type=int, default=config.skip_frames, help='how many frames to execute each action for')
    args = parser.parse_args()
    args.modeldim = args.modeldim.upper()
    return args

def test_agent(args, state_transform, num_test_episodes,
               actor, weights, global_step, save_dir):
    env = RunEnv2(state_transform, visualize=True, integrator_accuracy=args.accuracy,
                  model=args.modeldim, prosthetic=args.prosthetic, difficulty=args.difficulty,
                  skip_frame=config.skip_frames)
    test_rewards = []

    actor.set_actor_weights(weights)
    if args.weights is not None:
        actor.load(args.weights)

    for ep in range(num_test_episodes):
        seed = random.randrange(2**32-2)
        state = env.reset(seed=seed, difficulty=2)
        test_reward = 0
        while True:
            state = np.asarray(state, dtype='float32')
            action = actor.act(state)
            state, reward, terminal, _ = env._step(action)
            test_reward += reward
            if terminal:
                break
        test_rewards.append(test_reward)
    mean_reward = np.mean(test_rewards)
    std_reward = np.std(test_rewards)

    test_str ='global step {}; test reward mean: {:.2f}, std: {:.2f}, all: {} '.\
        format(global_step.value, float(mean_reward), float(std_reward), test_rewards)

    print(test_str)
    with open(os.path.join('test_report.log'), 'a') as f:
        f.write(test_str + '\n')

def main():
    args = get_args()
    args.critic_layers = literal_eval(args.critic_layers)
    args.actor_layers = literal_eval(args.actor_layers)

    save_dir = os.path.join('tests')
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    state_transform = NormState(args.prosthetic)
    # state_transform = StateVelCentr(obstacles_mode='standard',
    #                                 exclude_centr=True,
    #                                 vel_states=[])
    env = RunEnv2(state_transform, integrator_accuracy=args.accuracy, model=args.modeldim,
                  prosthetic=args.prosthetic, difficulty=args.difficulty, skip_frame=1)
    env.change_model(args.modeldim, args.prosthetic, args.difficulty)
    num_actions = env.get_action_space_size()
    del env

    model_params = {
            'state_size': state_transform.state_size,
            'num_act': num_actions,
            'gamma': 0,
            'actor_layers': args.actor_layers,
            'critic_layers': args.critic_layers,
            'actor_lr': 0,
            'critic_lr': 0,
            'layer_norm': args.layer_norm
    }
    actor_fn, params_actor, params_crit, actor_lr, critic_lr = \
            build_model_test(**model_params)
    actor = Agent(actor_fn, params_actor, params_crit)

    actor.load(args.weights)

    weights = [p.get_value() for p in params_actor]

    global_step = 0
    test_agent(args, state_transform, args.episodes,
            actor, weights, global_step, save_dir)

if __name__ == '__main__':
    main()


import numpy
import logging
import sys
import multiprocessing
import matplotlib.pyplot as plt
from apgl.graph.GraphStatistics import GraphStatistics
from apgl.util.PathDefaults import PathDefaults
from apgl.util.Util import Util
from apgl.util.Latex import Latex
from apgl.predictors.ABCSMC import loadThetaArray
from exp.viroscopy.model.HIVModelUtils import HIVModelUtils
from exp.viroscopy.model.HIVVertices import HIVVertices
from exp.sandbox.GraphMatch import GraphMatch
from exp.viroscopy.model.HIVGraphMetrics2 import HIVGraphMetrics2

assert False, "Must run with -O flag"

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
numpy.set_printoptions(suppress=True, precision=4, linewidth=150)

plotStyles = ['k-', 'kx-', 'k+-', 'k.-', 'k*-']

resultsDir = PathDefaults.getOutputDir() + "viroscopy/toy/theta/"
outputDir = resultsDir + "stats/"
startDate, endDate, recordStep, M, targetGraph = HIVModelUtils.toySimulationParams()
endDate += HIVModelUtils.toyTestPeriod

saveResults = False
graphStats = GraphStatistics()

N = 16
t = 0
maxT = 10

realTheta, sigmaTheta = HIVModelUtils.toyTheta()

minVal = 10

for i in range(maxT):
    thetaArray, distArray = loadThetaArray(N, resultsDir, i)
    if thetaArray.shape[0] == N:
        t = i

logging.debug("Using population " + str(t))

#We plot some stats for the ideal simulated epidemic
#and those epidemics found using ABC.

def saveStats(args):
    i, theta = args

    featureInds= numpy.ones(targetGraph.vlist.getNumFeatures(), numpy.bool)
    featureInds[HIVVertices.dobIndex] = False
    featureInds[HIVVertices.infectionTimeIndex] = False
    featureInds[HIVVertices.hiddenDegreeIndex] = False
    featureInds[HIVVertices.stateIndex] = False
    featureInds = numpy.arange(featureInds.shape[0])[featureInds]

    matcher = GraphMatch("PATH", alpha=0.5, featureInds=featureInds, useWeightM=False)
    graphMetrics = HIVGraphMetrics2(targetGraph, 1.0, matcher, float(endDate))
    times, infectedIndices, removedIndices, graph = HIVModelUtils.simulate(thetaArray[i], startDate, endDate, recordStep, M, graphMetrics)
    times, vertexArray, removedGraphStats = HIVModelUtils.generateStatistics(graph, startDate, endDate, recordStep)
    stats = times, vertexArray, removedGraphStats, graphMetrics.dists, graphMetrics.graphDists, graphMetrics.labelDists
    resultsFileName = outputDir + "SimStats" + str(i) + ".pkl"
    Util.savePickle(stats, resultsFileName)

if saveResults:
    thetaArray = loadThetaArray(N, resultsDir, t)[0]
    logging.debug(thetaArray)

    paramList = []

    for i in range(thetaArray.shape[0]):
        paramList.append((i, thetaArray[i, :]))

    pool = multiprocessing.Pool(multiprocessing.cpu_count())
    resultIterator = pool.map(saveStats, paramList)
    #resultIterator = map(saveStats, paramList)
    pool.terminate()

    #Now save the statistics on the target graph
    stats = HIVModelUtils.generateStatistics(targetGraph, startDate, endDate, recordStep)
    resultsFileName = outputDir + "IdealStats.pkl"
    Util.savePickle(stats, resultsFileName)
else:
    #for i in range(t):
    #    thetaArray = loadThetaArray(N, resultsDir, i)[0]

    realTheta, sigmaTheta = HIVModelUtils.toyTheta()
    thetaArray, distArray = loadThetaArray(N, resultsDir, t)
    print(realTheta)
    print(thetaArray)
    print(distArray)

    meanTable = numpy.c_[realTheta, thetaArray.mean(0)]
    stdTable = numpy.c_[sigmaTheta, thetaArray.std(0)]
    table = Latex.array2DToRows(meanTable, stdTable, precision=4)
    rowNames = ["$\\|\\mathcal{I}_0 \\|$", "$\\rho_B$", "$\\alpha$", "$C$", "$\\gamma$", "$\\beta$", "$\\kappa_{max}$", "$\\lambda_H$", "$\\lambda_B$", "$\\sigma_{WM}$",  "$\\sigma_{MW}$","$\\sigma_{MB}$"]
    table = Latex.addRowNames(rowNames, table)
    print(table)

    resultsFileName = outputDir + "IdealStats.pkl"
    stats = Util.loadPickle(resultsFileName)
    timesIdeal, vertexArrayIdeal, removedGraphStats = stats

    recordStep = 100
    endDate = 1500
    times2 = numpy.arange(startDate, endDate+1, recordStep)
    times2 = times2[1:]

    graphStats = GraphStatistics()

    #First plot graphs for ideal theta
    plotInd = 0

    plt.figure(plotInd)
    plt.plot(timesIdeal, vertexArrayIdeal[:, 0], "r")
    plt.xlabel("Time (days)")
    plt.ylabel("Removed")
    plotInd += 1

    plt.figure(plotInd)
    plt.plot(timesIdeal, vertexArrayIdeal[:, 1], "r")
    plt.xlabel("Time (days)")
    plt.ylabel("Males")
    plotInd += 1

    plt.figure(plotInd)
    plt.plot(timesIdeal, vertexArrayIdeal[:, 2], "r")
    plt.xlabel("Time (days)")
    plt.ylabel("Females")
    plotInd += 1

    plt.figure(plotInd)
    plt.plot(timesIdeal, vertexArrayIdeal[:, 3], "r")
    plt.xlabel("Time (days)")
    plt.ylabel("Hetero")
    plotInd += 1

    plt.figure(plotInd)
    plt.plot(timesIdeal, vertexArrayIdeal[:, 4], "r")
    plt.xlabel("Time (days)")
    plt.ylabel("Bi")
    plotInd += 1

    plt.figure(plotInd)
    plt.plot(timesIdeal, vertexArrayIdeal[:, 5], "r")
    plt.xlabel("Time (days)")
    plt.ylabel("Random detection")
    plotInd += 1

    plt.figure(plotInd)
    plt.plot(timesIdeal, vertexArrayIdeal[:, 6], "r")
    plt.xlabel("Time (days)")
    plt.ylabel("Contact Tracing")
    plotInd += 1

    plt.figure(plotInd)
    plt.plot(timesIdeal, removedGraphStats[:, graphStats.numComponentsIndex], "r")
    plt.xlabel("Time (days)")
    plt.ylabel("Number of components")
    plotInd += 1

    plt.figure(plotInd)
    plt.xlabel("Time (days)")
    plt.ylabel("Distance")
    plotInd += 1

    plt.figure(plotInd)
    plt.xlabel("Time (days)")
    plt.ylabel("Graph distance")
    plotInd += 1

    plt.figure(plotInd)
    plt.xlabel("Time (days)")
    plt.ylabel("Label distance")
    plotInd += 1

    distsArr = []
    detectsArr = []

    for i in range(thetaArray.shape[0]):
        plotInd = 0
        resultsFileName = outputDir + "SimStats" + str(i) + ".pkl"
        stats = Util.loadPickle(resultsFileName)

        times, vertexArray, removedGraphStats, dists, graphDists, labelDists = stats

        plt.figure(plotInd)
        plt.plot(times, vertexArray[:, 0], plotStyles[0])
        plotInd += 1
        detectsArr.append(vertexArray[:, 0])

        plt.figure(plotInd)
        plt.plot(times, vertexArray[:, 1], plotStyles[0])
        plotInd += 1

        plt.figure(plotInd)
        plt.plot(times, vertexArray[:, 2], plotStyles[0])
        plotInd += 1

        plt.figure(plotInd)
        plt.plot(times, vertexArray[:, 3], plotStyles[0])
        plotInd += 1

        plt.figure(plotInd)
        plt.plot(times, vertexArray[:, 4], plotStyles[0])
        plotInd += 1

        plt.figure(plotInd)
        plt.plot(times, vertexArray[:, 5], plotStyles[0])
        plotInd += 1

        plt.figure(plotInd)
        plt.plot(times, vertexArray[:, 6], plotStyles[0])
        plotInd += 1

        plt.figure(plotInd)
        plt.plot(times, removedGraphStats[:, graphStats.numComponentsIndex], plotStyles[0])
        plotInd += 1

        plt.figure(plotInd)
        distPlotInd = plotInd
        plt.plot(times2, dists, plotStyles[0], label="Distance")
        plotInd += 1
        distsArr.append(dists)
        print(numpy.array(distsArr).mean())

        plt.figure(plotInd)
        plt.plot(times2, graphDists, plotStyles[0])
        plotInd += 1

        plt.figure(plotInd)
        plt.plot(times2, labelDists, plotStyles[0])
        plotInd += 1

    meanDetects = numpy.array(detectsArr).mean(0)
    stdDetects = numpy.array(detectsArr).std(0)
    plt.figure(plotInd)
    plt.errorbar(times, meanDetects, yerr=stdDetects)
    plt.plot(timesIdeal, vertexArrayIdeal[:, 0], "r")
    plotInd += 1

    meanDists = numpy.array(distsArr).mean(0)
    stdDists = numpy.array(distsArr).std(0)
    plt.figure(plotInd)
    plt.errorbar(times2, meanDists, yerr=stdDists)
    plotInd += 1

    plt.show()


"""Views for `spot` app."""

# third party
from brasil_municipios.models import Municipio
from django.conf import settings
from django.contrib.gis.db.models.functions import Distance
from django.contrib.gis.geos import Point
from django.core.urlresolvers import reverse
# project
from .models import Spot
from utils.http import JsonResponse
from utils.pagination import paginate

def spots_list(request, api_version):
    response_headers = {}
    spots = Spot.objects.all()

    #
    # Sorting
    #
    sort = 'name'

    if 'lat' in request.GET and 'lng' in request.GET:
        try:
            lat = float(request.GET['lat'])
            lng = float(request.GET['lng'])
        except ValueError:  # pragma: no cover
            pass  # invalid values for lator lng: use default ordering
        else:
            origin = Point(lng, lat, srid=settings.SRID)
            spots = spots.annotate(distance=Distance('coords', origin))
            sort = 'distance'
    else:
        descending_sort = False  # descending sort: '-field'
        sort = request.GET.get('sort', '')
        if sort.startswith('-'):
            descending_sort = True
            sort = sort[1:]  # '-field' -> 'field'

        # maps url 'sort' param value to the queryset .order() value
        sort_field_map = {'name': 'name', 'created_at': 'created_at'}

        if sort in sort_field_map:
            sort = sort_field_map[sort]
            if descending_sort:
                sort = '-' + sort
        else:
            sort = 'name'

    spots = spots.order_by(sort)

    #
    # City filtering
    #
    if 'city' in request.GET:
        city_id = request.GET['city']
        try:
            city = Municipio.objects.defer('geometry').get(pk=city_id)
        except (Municipio.DoesNotExist, ValueError):
            pass  # Invalid city id, return all spots
        else:
            spots = spots.filter(city=city)

    #
    # Searching
    #
    query = request.GET.get('q', '')
    if query:
        spots = spots.filter(name__unaccent__icontains=query)

    #
    # Pagination
    #
    base_url = '{}://{}{}'.format(
        request.scheme,
        request.get_host(),
        reverse('spots', kwargs={'api_version': api_version})
    )
    pagination = paginate(queryset=spots,
                          query_dict=request.GET,
                          base_url=base_url)
    response_headers.update(pagination['response_headers'])
    page = pagination['page']

    #
    # Construct response
    #
    data = []
    for spot in page:
        spot_data = spot.to_dict()
        spot_data['url'] = spot.get_full_url(request.scheme,
                                             request.get_host(),
                                             api_version)
        if spot.directions:
            spot_data['directionsUrl'] = spot.get_directions_full_url(
                request.scheme, request.get_host(), api_version)
        else:
            spot_data['directionsUrl'] = None
        data.append(spot_data)

    prettify = request.GET.get('pretty') != 'false'
    response = JsonResponse(data, safe=False, pretty=prettify)
    for (key, val) in response_headers.items():
        response[key] = val
    return response

def spot_details(request, api_version, spot_id=None, spot_slug=None):
    assert (
        (spot_id is not None and spot_slug is None) or
        (spot_id is None and spot_slug is not None)
    ), 'Either `spot_id` or `spot_slug` required, but not both'

    spots_qs = Spot.objects.select_related('city').defer('city__geometry')

    try:
        if spot_id is not None:
            spot = spots_qs.get(pk=spot_id)
        else:
            spot = spots_qs.get(slug=spot_slug)
    except Spot.DoesNotExist:
        status_code = 404
        data = {
            'status_code': status_code,
            'message': 'Spot {} not found'.format(spot_id or spot_slug)
        }
    else:
        status_code = 200
        data = spot.to_dict(all_details=True)
        data['url'] = spot.get_full_url(
            request.scheme, request.get_host(), api_version)
        if spot.directions:
            data['directionsUrl'] = spot.get_directions_full_url(
                request.scheme, request.get_host(), api_version)
        else:
            data['directionsUrl'] = None

    prettify = request.GET.get('pretty') != 'false'
    response = JsonResponse(data, status=status_code, safe=False,
                            pretty=prettify)
    return response

def spots_endpoint(request, api_version, spot_id=None, spot_slug=None):
    if spot_id is None and spot_slug is None:
        return spots_list(request, api_version)
    else:
        return spot_details(request,
                            api_version,
                            spot_id=spot_id,
                            spot_slug=spot_slug)

def directions_endpoint(request, api_version, spot_slug):
    try:
        spot = Spot.objects.get(slug=spot_slug)
    except (Spot.DoesNotExist):
        status_code = 404
        data = {
            'status_code': status_code,
            'message': 'Spot {} not found'.format(spot_slug)
        }
    else:
        if spot.directions is None:
            status_code = 404
            data = {
                'status_code': status_code,
                'message': 'Directions not available for spot {}'.format(
                    spot_slug)
            }
        else:
            status_code = 200
            data = spot.directions

    prettify = request.GET.get('pretty') != 'false'
    response = JsonResponse(data, status=status_code, safe=False,
                            pretty=prettify)
    return response


import typing
from typing import Any, Callable, List, Tuple, Union

from .abc_interpreter import Interpreter
from ..data_processor.readers import preprocess_image, read_image, restore_image
from ..data_processor.visualizer import visualize_grayscale

import IPython.display as display
import cv2
import numpy as np
import os, sys
from PIL import Image

class GradShapInterpreter(Interpreter):
    """
    Gradient SHAP Interpreter.

    More details regarding the Gradient SHAP method can be found in the original paper:
    http://papers.nips.cc/paper/7062-a-unified-approach-to-interpreting-model-predictions
    """

    def __init__(self,
                 paddle_model,
                 trained_model_path,
                 class_num,
                 use_cuda,
                 model_input_shape=[3, 224, 224]) -> None:
        """
        Initialize the GradShapInterpreter.

        Args:
            paddle_model (callable): A user-defined function that gives access to model predictions.
                It takes the following arguments:

                - data: Data inputs.
                and outputs predictions. See the example at the end of ``interpret()``.
            trained_model_path (str): The pretrained model directory.
            class_num (int): Number of classes for the model.
            use_cuda (bool, optional): Whether or not to use cuda. Default: True
            model_input_shape (list, optional): The input shape of the model. Default: [3, 224, 224]
        """
        Interpreter.__init__(self)
        self.paddle_model = paddle_model
        self.trained_model_path = trained_model_path
        self.use_cuda = use_cuda
        self.model_input_shape = model_input_shape
        self.paddle_prepared = False

    def interpret(self,
                  data,
                  label=None,
                  baseline=None,
                  n_samples=5,
                  noise_amout=0.1,
                  visual=True,
                  save_path=None):
        """
        Main function of the interpreter.

        Args:
            data (str or numpy.ndarray): The image filepath or processed image.
            label (int, optional): The target label to analyze. If None, the most likely label will be used. Default: None.
            baseline (numpy.ndarray, optional): The baseline input. If None, all zeros will be used. Default: None
            n_samples (int, optional): The number of randomly generated samples. Default: 5.
            noise_amount (float, optional): Noise level of added noise to the image.
                                            The std of Guassian random noise is noise_amount * (x_max - x_min). Default: 0.1
            visual (bool, optional): Whether or not to visualize the processed image. Default: True.
            save_path (str, optional): The filepath to save the processed image. If None, the image will not be saved. Default: None

        Returns:
            numpy.ndarray: avg_attributions

        Example::

            def paddle_model(data):
                import paddle.fluid as fluid
                class_num = 1000
                model = ResNet50()
                logits = model.net(input=image_input, class_dim=class_num)
                probs = fluid.layers.softmax(logits, axis=-1)
                return probs
            gs = GradShapInterpreter(predict_fn, "assets/ResNet50_pretrained", 1000, True)
            avg_attributions = gs.interpret(
                                img_path,
                                label=None,
                                noise_amout=0.1,
                                n_samples=5,
                                visual=True,
                                save_path='grad_shap_test.jpg')
        """

        def add_noise_to_inputs():
            std = noise_amout * (np.max(data) - np.min(data))
            noise = np.random.normal(
                0.0, std,
                (n_samples, ) + data.shape[1:]).astype(self.data_type)
            return data.repeat(n_samples, axis=0) + noise

        # Read in image
        if isinstance(data, str):
            img = read_image(data, crop_size=self.model_input_shape[1])
            data = preprocess_image(img)
        else:
            img = restore_image(data.copy())

        self.data_type = np.array(data).dtype

        data_with_noise = add_noise_to_inputs()

        if baseline is None:
            baseline = np.zeros_like(data)
        baseline = baseline.repeat(n_samples, axis=0)

        if not self.paddle_prepared:
            self._paddle_prepare()

        if label is None:
            _, out = self.predict_fn(data, np.array([[0]]))
            label = np.argmax(out[0])
        label = np.array([[label]]).repeat(n_samples, axis=0)

        rand_scales = np.random.uniform(0.0, 1.0,
                                        (n_samples, 1)).astype(self.data_type)

        input_baseline_points = np.array([
            d * r + b * (1 - r)
            for d, r, b in zip(data_with_noise, rand_scales, baseline)
        ])

        gradients, _ = self.predict_fn(input_baseline_points, label)

        input_baseline_diff = data_with_noise - baseline

        attributions = gradients * input_baseline_diff
        avg_attributions = np.mean(attributions, axis=0, keepdims=True)

        visualize_grayscale(avg_attributions, visual, save_path)

        return avg_attributions

    def _paddle_prepare(self, predict_fn=None):
        if predict_fn is None:
            import paddle.fluid as fluid
            startup_prog = fluid.Program()
            main_program = fluid.Program()
            with fluid.program_guard(main_program, startup_prog):
                with fluid.unique_name.guard():
                    data_op = fluid.data(
                        name='data',
                        shape=[None] + self.model_input_shape,
                        dtype=self.data_type)
                    label_op = fluid.data(
                        name='label', shape=[None, 1], dtype='int64')

                    data_plus = data_op + fluid.layers.zeros_like(data_op)
                    probs = self.paddle_model(data_plus)

                    for op in main_program.global_block().ops:
                        if op.type == 'batch_norm':
                            op._set_attr('use_global_stats', True)
                        elif op.type == 'dropout':
                            op._set_attr('dropout_prob', 0.0)

                    class_num = probs.shape[-1]
                    one_hot = fluid.layers.one_hot(label_op, class_num)
                    one_hot = fluid.layers.elementwise_mul(probs, one_hot)
                    target_category_loss = fluid.layers.reduce_sum(one_hot)

                    p_g_list = fluid.backward.append_backward(
                        target_category_loss)
                    gradients_map = fluid.gradients(one_hot, data_plus)[0]

            if self.use_cuda:
                gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
                place = fluid.CUDAPlace(gpu_id)
            else:
                place = fluid.CPUPlace()
            exe = fluid.Executor(place)
            fluid.io.load_persistables(exe, self.trained_model_path,
                                       main_program)

            def predict_fn(data, label):
                gradients, out = exe.run(main_program,
                                         feed={'data': data,
                                               'label': label},
                                         fetch_list=[gradients_map, probs])
                return gradients, out

        self.predict_fn = predict_fn
        self.paddle_prepared = True


from django.conf.urls import url, include
from django.contrib.auth import views as auth_views
from . import views

urlpatterns = [

    url(r'^choice_register/$', views.RegisterOption, name="register_option"),
    url(r'^$', auth_views.LoginView.as_view(template_name='users/login.html'), name='login'),
    url(r'^student_register/$', views.StudentRegister, name="student_register"),
    url(r'^tutor_register/$', views.TutorRegister, name="tutor_register"),
    url(r"^student_update/$", views.profile, name='student_update'),
    url(r'^logout/$', views.logoutUser, name="logout"),
]


#!/usr/bin/env python
#################################################################################
# Copyright 2018 ROBOTIS CO., LTD.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#################################################################################

# Authors: Gilbert #

import rospy
import numpy as np
import math
from math import pi
from geometry_msgs.msg import Twist, Point, Pose
from sensor_msgs.msg import LaserScan
from nav_msgs.msg import Odometry
from std_srvs.srv import Empty
from tf.transformations import euler_from_quaternion, quaternion_from_euler
from respawnGoal import Respawn

class Env():
    def __init__(self, action_size):
        self.goal_x = 0
        self.goal_y = 0
        self.heading = 0
        self.action_size = action_size
        self.initGoal = True
        self.get_goalbox = False
        self.position = Pose()
        self.pub_cmd_vel = rospy.Publisher('cmd_vel', Twist, queue_size=5)
        self.sub_odom = rospy.Subscriber('odom', Odometry, self.getOdometry)
        self.reset_proxy = rospy.ServiceProxy('gazebo/reset_simulation', Empty)
        self.unpause_proxy = rospy.ServiceProxy('gazebo/unpause_physics', Empty)
        self.pause_proxy = rospy.ServiceProxy('gazebo/pause_physics', Empty)
        self.respawn_goal = Respawn()

    def getGoalDistace(self):
        goal_distance = round(math.hypot(self.goal_x - self.position.x, self.goal_y - self.position.y), 2)

        return goal_distance

    def getOdometry(self, odom):
        self.position = odom.pose.pose.position
        orientation = odom.pose.pose.orientation
        orientation_list = [orientation.x, orientation.y, orientation.z, orientation.w]
        _, _, yaw = euler_from_quaternion(orientation_list)

        goal_angle = math.atan2(self.goal_y - self.position.y, self.goal_x - self.position.x)

        heading = goal_angle - yaw
        if heading > pi:
            heading -= 2 * pi

        elif heading < -pi:
            heading += 2 * pi

        self.heading = round(heading, 2)

    def getState(self, scan):
        scan_range = []
        heading = self.heading
        min_range = 0.2 # Threshold distance to obstacle
                        # Initially was 0.13 (save_model data)
        done = False

        for i in range(len(scan.ranges)):
            if scan.ranges[i] == float('Inf'):
                scan_range.append(3.5)
            elif np.isnan(scan.ranges[i]):
                scan_range.append(0)
            else:
                scan_range.append(scan.ranges[i])

        obstacle_min_range = round(min(scan_range), 2)
        obstacle_angle = np.argmin(scan_range)
        if min_range > min(scan_range) > 0:
            done = True

        current_distance = round(math.hypot(self.goal_x - self.position.x, self.goal_y - self.position.y), 2)
        if current_distance < 0.2:
            self.get_goalbox = True

        return scan_range + [heading, current_distance, obstacle_min_range, obstacle_angle], done

    def setReward(self, state, done, action):
        yaw_reward = []
        current_distance = state[-3]
        heading = state[-4]

        for i in range(5):
            angle = -pi / 4 + heading + (pi / 8 * i) + pi / 2
            tr = 1 - 4 * math.fabs(0.5 - math.modf(0.25 + 0.5 * angle % (2 * math.pi) / math.pi)[0])
            yaw_reward.append(tr)

        distance_rate = 2 ** (current_distance / self.goal_distance)
        reward = ((round(yaw_reward[action] * 5, 2)) * distance_rate)

        if done:
            rospy.loginfo("Collision!!")
            reward = -150
            self.pub_cmd_vel.publish(Twist())

        if self.get_goalbox:
            rospy.loginfo("Goal!!")
            reward = 200
            self.pub_cmd_vel.publish(Twist())
            self.goal_x, self.goal_y = self.respawn_goal.getPosition(True, delete=True)
            self.goal_distance = self.getGoalDistace()
            self.get_goalbox = False

        return reward

    def step(self, action):
        max_angular_vel = 1.5
        ang_vel = ((self.action_size - 1)/2 - action) * max_angular_vel * 0.5

        vel_cmd = Twist()
        vel_cmd.linear.x = 0.15
        vel_cmd.angular.z = ang_vel
        self.pub_cmd_vel.publish(vel_cmd)

        data = None
        while data is None:
            try:
                data = rospy.wait_for_message('scan', LaserScan, timeout=5)
            except:
                pass

        state, done = self.getState(data)
        reward = self.setReward(state, done, action)

        return np.asarray(state), reward, done

    def reset(self):
        rospy.wait_for_service('gazebo/reset_simulation')
        try:
            self.reset_proxy()
        except (rospy.ServiceException) as e:
            print("gazebo/reset_simulation service call failed")

        data = None
        while data is None:
            try:
                data = rospy.wait_for_message('scan', LaserScan, timeout=5)
            except:
                pass

        if self.initGoal:
            self.goal_x, self.goal_y = self.respawn_goal.getPosition()
            self.initGoal = False

        self.goal_distance = self.getGoalDistace()
        state, done = self.getState(data)

        return np.asarray(state)


import nltk
import numpy as np
import random
import string #For processing python std strings

# Read in the data from the txt file
# then filter it
f = open('chatbot.txt','r', errors = 'ignore')
raw = f.read()

# Convert to lowercase

raw = raw.lower()

 # converts to list of sentences
sent_tokens = nltk.sent_tokenize(raw)
# converts to list of words
word_tokens = nltk.word_tokenize(raw)

# Lemmatization essentially converts word into their base meaning, like running into run
# Define function LemTokens, takes tokens and normalizes them
lemmer = nltk.stem.WordNetLemmatizer()

# WordNet is a semantically-oriented dictionary of English included in NLTK.
def LemTokens(tokens):

    return [lemmer.lemmatize(token) for token in tokens]

remove_punct_dict = dict((ord(punct), None) for punct in string.punctuation)

# Normalizes the word
def LemNormalize(text):
    return LemTokens(nltk.word_tokenize(text.lower().translate(remove_punct_dict)))

# Keyword matching

GREETING_INPUTS = ("hello", "hi", "greetings", "sup", "what's up","hey",)

GREETING_RESPONSES = ["hi", "hey", "*nods*", "hi there", "hello", "I am glad! You are talking to me"]

# Matches a greeting with another greeting
# For generic greetings
def greeting(sentence):

    for word in sentence.split():
        if word.lower() in GREETING_INPUTS:
            return random.choice(GREETING_RESPONSES)

# Will convert document into matrix of TF-IDF features
# Term Frequency helps learn the importance of certain words
# also gives meaning to the document
from sklearn.feature_extraction.text import TfidfVectorizer
# Cosine similarity takes the dot product of the TF vectors
# Smaller the angle the more similar terms are
# Used to determine how similar a word is from the human, to the stored words
from sklearn.metrics.pairwise import cosine_similarity

# Response is needed if too many meaning are possible, or none
def response(user_response):
    robo_response = ''
    sent_tokens.append(user_response)

    # Run this through a method that will vercotize the tokens
    # Stop words are extremely common words that have little value
    TfidfVec = TfidfVectorizer(tokenizer=LemNormalize, stop_words='english')
    # Fit transform "centers" the data, normalizes it
    tfidf = TfidfVec.fit_transform(sent_tokens)
    vals = cosine_similarity(tfidf[-1], tfidf)
    idx = vals.argsort()[0][-2]
    flat = vals.flatten()
    flat.sort()
    req_tfidf = flat[-2]

    if(req_tfidf == 0):
        robo_response = robo_response + "I am sorry! I don't understand you"
        return robo_response
    else:
        robo_response = robo_response + sent_tokens[idx]
        return robo_response

# Main chat interface
flag=True
print("ROBO: My name is Robo. I will answer your queries about Chatbots. If you want to exit, type Bye!")
while(flag==True):
    user_response = input()
    user_response=user_response.lower()
    if(user_response!='bye'):
        if(user_response=='thanks' or user_response=='thank you' ):
            flag=False
            print("ROBO: You are welcome..")
        else:
            if(greeting(user_response)!=None):
                print("ROBO: "+greeting(user_response))
            else:
                print("ROBO: ",end="")
                print(response(user_response))
                sent_tokens.remove(user_response)
    else:
        flag=False
        print("ROBO: Bye! take care..")


import sys

sys.path.append("D:\\user\\pys\\Python-002\\")
print(f"current path:{sys.path}")

import scrapy

from scrapy.selector import Selector
from week02.homeWork01.homeWork01.items import Homework01Item

class Week2maoyan01Spider(scrapy.Spider):
    name = 'week2maoyan01'
    allowed_domains = ['maoyan.com']
    start_urls = ['https://maoyan.com/films?showType=3',
                  'file:///D:/user/pys/Python-002/week02/testDataFile/maoyanshowType3.html']

    def start_requests(self):
        yield scrapy.Request(url=self.start_urls[0], callback=self.parse)

    def parse(self, response):
        # 通过XPath获取电影属性
        movie_urls = Selector(response=response).xpath('//div[@class="movie-hover-info"]')[0:10]
        movie_info_item = Homework01Item()
        for movie_info in movie_urls:
            # span页签中class有的使用的是"name"，有的使用的是"name noscore"，故使用contains
            movie_name = movie_info.xpath("./div/span[contains(@class,'name ')]/text()")
            print(f"电影名称:{movie_name.extract_first()}")
            movie_info_item['movie_name'] = movie_name.extract_first()
            movie_type = movie_info.xpath("./div[contains(@class,'movie-hover-title')]/text()")
            # 不能使用normalize-space，经过查资料，该函数对单个字符串有效
            # movie_tag = movie_info.xpath("normalize-space(./div[contains(@class,'movie-hover-title')]/text())")

            # 使用strip()去除空格、\r\n，使用filter去除掉为空的字符
            movie_tag = list(filter(lambda x: x != '', [item.strip() for item in movie_type.extract()]))
            # print(f"电影类型:{movie_tag[0]}")
            movie_info_item['movie_type'] = movie_tag[0]
            # print(f"电影类型:{movie_tag[-1]}")
            movie_info_item['movie_release_time'] = movie_tag[-1]
            yield movie_info_item


"""
Version:
UCT with DP and prohibit path
5 components
analytical evaluation

Feature:
updated DP with configurable basic weight for not in approved paths
"""
import csv
import datetime
import os

from utils.util import mkdir, get_args, generate_depth_list
from Algorithms.SerialUCF import serial_UCF_test
from Algorithms.test_analytic_simulator import anay_read_test
from Algorithms.PlannerTest import Planner_test
from Algorithms.BruteForce import brute_force
from Algorithms.RandomSearch import random_search
from Algorithms.GeneticSearch import genetic_search
from Viz.VIZ import viz_test

def generate_traj_lists(trajectories, test_number):
    traj_lists = []
    for traj in trajectories:
        # TODO for multi size test
        # if traj < 800:

        try_times = test_number
        traj_list = []
        for _ in range(try_times):
            traj_list.append(traj)
        traj_lists.append(traj_list)
    return traj_lists

def main(name):
    # target_lists = []
    target_vout_list = [-200, 50, 200]
    target_vout_list = [50]

    # trajectories = [15,20]
    test_number = 2
    # trajectories = [600, 500, 400, 300, 200, 100]
    trajectories = [100, 300, 600]
    # trajectories = [10,15]
    # test_number = 4
    # trajectories = [10000]
    # test_number = 1
    # [[600,600,600,600],[500,500,500,500]]
    traj_lists = generate_traj_lists(trajectories, test_number)
    anal_output_results = [[] for i in range(test_number + 1)]
    simu_output_results = [[] for i in range(test_number + 1)]

    for try_target_vout in target_vout_list:
        for traj_list in traj_lists:
            # for try_traj in traj_list:
            mkdir("figures")
            mkdir("Results")
            # simulation_root_folder = "simu_analy/" + date_str +'-'+str(os.getpid())
            # mkdir(simulation_root_folder)
            configs = {}
            args_file_name = "config"
            get_args(args_file_name, configs)

            # try_target_vout = 50
            configs["target_vout"] = try_target_vout
            configs["fix_paras"] = None

            configs["min_vout"] = configs["target_vout"] - configs["range_percentage"] * abs(configs["target_vout"])
            configs["max_vout"] = configs["target_vout"] + configs["range_percentage"] * abs(configs["target_vout"])
            depth_list = generate_depth_list(configs["dep_start"], configs["dep_end"], configs["dep_step_len"])
            # traj_list = generate_traj_List(configs["traj_start"], configs["traj_end"], configs["traj_step_len"])
            date_str = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
            if configs["algorithm"] == "UCF" or configs["algorithm"] == "UCT":
                serial_UCF_test(depth_list, traj_list, configs, date_str)
            elif configs["algorithm"] == "GeneticSearch" or configs["algorithm"] == "GS":
                anal_results, simu_results = genetic_search(traj_list, configs, date_str)
                for i in range(test_number + 1):
                    anal_output_results[i].extend(anal_results[i])
                    simu_output_results[i].extend(simu_results[i])
            elif configs["algorithm"] == "VIZ":
                mkdir("Viz/TreeStructures")
                viz_test(depth_list, traj_list, configs, date_str)
            # elif configs["algorithm"] == "TEST":
            #     hash_read_test(depth_list, traj_list, configs, date_str)
            elif configs["algorithm"] == "ANA_TEST":
                anay_read_test(configs)
            elif configs["algorithm"] == "TEST":
                Planner_test(depth_list, traj_list, configs, date_str)
            elif configs["algorithm"] == "RS":
                random_search(traj_list, configs, date_str)
            elif configs["algorithm"] == "BF":
                anal_results, simu_results = brute_force(depth_list, traj_list, configs, date_str)
                for i in range(test_number + 1):
                    anal_output_results[i].extend(anal_results[i])
                    simu_output_results[i].extend(simu_results[i])
        anal_out_file_name = "Results/"+configs["algorithm"]+"-" + str(try_target_vout) + "-anal-" + date_str + "-" + str(os.getpid()) + ".csv"
        with open(anal_out_file_name, 'w') as f:
            csv_writer = csv.writer(f)
            csv_writer.writerows(anal_output_results)
        f.close()
        simu_out_file_name = "Results/"+configs["algorithm"]+"-" + str(try_target_vout) + "-simu-" + date_str + "-" + str(os.getpid()) + ".csv"
        with open(simu_out_file_name, 'w') as f:
            csv_writer = csv.writer(f)
            csv_writer.writerows(simu_output_results)
        f.close()

    os.system("rm *-analytic*")
    os.system("rm PCC-*.cki")
    os.system("rm PCC-*.simu")
    return

if __name__ == '__main__':
    # from SimulatorAnalysis.sim_result import conduction_analytics_result
    # conduction_analytics_result('./', 'Rb1')
    # read_result('mutitest_50-2021-04-17-16-29-41-37526.txt')
    main('PyCharm')


import numpy as np
import scipy.constants as sp
from math import pi, sin, exp

class Mesh:
    def __init__(self, ncells, ddx, epsilon_r, sigma, start_m, end_m):
        self.ncells=ncells
        self.ddx=ddx
        self.epsilon_r= epsilon_r
        self.sigma= sigma
        self.start_m= start_m
        self.end_m= end_m

    def material(self, k):
        self.k=k

        dt=self.ddx / sp.c

        ca = np.ones(self.ncells+1)
        cb = np.ones(self.ncells+1) * 0.5

        eaf = dt * self.sigma / (2 * sp.epsilon_0 * self.epsilon_r)
        ca[self.start_m : self.end_m] = (1 - eaf ) / (1 + eaf )
        cb[self.start_m : self.end_m] = 0.5 / (self.epsilon_r * (1 + eaf ))

        return ca[k], cb[k]

    def boundarymur(self, ex, time_step):
        self.ex=ex
        self.time_step=time_step

        if time_step >= 2:
                ex[time_step][0]=ex[time_step-2][1]
                ex[time_step][self.ncells]=ex[time_step-2][self.ncells-1]

        return ex

class Source:
    def __init__(self, sourcetype, t_0, s_0):
        self.sourcetype=sourcetype
        self.t_0=t_0
        self.s_0=s_0

    def pulse(self, time_step):

        self.time_step=time_step

        if self.sourcetype == 'gauss':
            pulse = exp(-0.5*( (self.t_0 - time_step) / self.s_0 )**2)

        return pulse


"""
Forms
"""
import wtforms
from gutter.client.models import Switch, Condition

from . import wtforms_extension
from .registry import operators, arguments

class ConditionForm(wtforms.Form):
    """
    Form defining the individual condition of a switch.
    """

    argument = wtforms_extension.SelectField(choices=arguments.as_choices)
    negative = wtforms.SelectField(choices=(('0', 'Is'), ('1', 'Is Not')))
    operator = wtforms_extension.SelectField(choices=operators.as_choices)

    operator_arguments = wtforms.FieldList(wtforms.StringField(),
                                           min_entries=2,
                                           max_entries=2)

class SwitchForm(wtforms.Form):
    """
    Form defining a switch.
    """
    name = wtforms.StringField(u'Name',
                               validators=[wtforms.validators.required(),
                                           wtforms.validators.length(max=100)])
    label = wtforms.StringField(u'Label')
    description = wtforms.TextAreaField(u'Description')

    state = wtforms.SelectField(
        u'State',
        choices={'1': 'Disabled',
                 '2': 'Selective',
                 '3': 'Global'}.items())

    compounded = wtforms.BooleanField()
    concent = wtforms.BooleanField(u'Consent')
    delete = wtforms.HiddenField()
    is_new = wtforms.HiddenField()

    conditions = wtforms.FieldList(wtforms.FormField(ConditionForm),
                                   min_entries=0)

    def to_object(self):
        """
        Return a Switch object based on form data.
        """
        switch = Switch(name=self.name.data,
                        label=self.label.data,
                        description=self.description.data,
                        # States must be integer to match gutter client
                        state=int(self.state.data),
                        compounded=self.compounded.data,
                        concent=self.concent.data)

        for condition in self.conditions:
            switch.conditions.append(self.make_condition(condition))

        return switch

    def make_condition(self, condition_form):
        """
        Create a condition based on form input
        """
        # Operators in the registry are the types (classes), so extract that
        # and we will construct it from the remaining data, which are the
        # arguments for the operator
        operator_type = operators[condition_form.operator.data]

        # Arguments are a Class property, so just a simple fetch from the
        # arguments dict will retreive it
        argument = arguments[condition_form.argument.data]

        # The remaining variables in the data are the arguments to the
        # operator, but they need to be cast by the argument to their
        # right type
        caster = argument.variable.to_python

        operator_kwargs = dict((k, caster(v)) for k, v in zip(
            operator_type.arguments, condition_form.operator_arguments.data))

        return Condition(argument=argument.owner,
                         attribute=argument.name,
                         operator=operator_type(**operator_kwargs),
                         negative=bool(int(condition_form.negative.data)))

    @classmethod
    def from_object(cls, switch):
        """
        Return a new form based on a Switch object.
        """
        data = dict(label=switch.label,
                    name=switch.name,
                    description=switch.description,
                    state=switch.state,
                    compounded=switch.compounded,
                    concent=switch.concent)

        instance = cls(**data)

        for condition in switch.conditions:
            instance.append_condition(condition)

        return instance

    def append_condition(self, condition):
        """
        Given a condition, append it to the list of form conditions.
        """
        fields = dict(argument='.'.join((condition.argument.__name__,
                                         condition.attribute)),
                      negative='1' if condition.negative else '0',
                      operator=condition.operator.name)

        operator_arguments = [condition.operator.variables[arg]
                              for arg in condition.operator.arguments]
        fields['operator_arguments'] = operator_arguments

        self.conditions.append_entry(fields)


import sys,os,math,time
import json,re
import contextlib
import webbrowser
import time,threading

# from subprocess import Popen, PIPE, STDOUT, check_output
from subprocess import Popen, PIPE, STDOUT, check_output, DEVNULL
from glob import glob
from shutil import copytree

import tkinter as tk
from tkinter import Toplevel
from tkinter import Tk, END, StringVar, LEFT, TOP, BOTTOM, X, BOTH, YES, INSERT, W, E
from tkinter import Text, Entry, OptionMenu

from tkinter import ttk
from tkinter.ttk import Label, Button, LabelFrame, Scrollbar, Frame, Notebook, Style

from tkinter.filedialog import askopenfilename, asksaveasfilename, askdirectory
from tkinter.messagebox import showerror, showinfo, showwarning

from pathlib import Path
from os.path import join, exists
from os import makedirs, listdir

from gui.frame import PipelineFrame
from gui.utils import ftypes, filetypes, filetype, efiletype, batchsize, customRules
from gui.utils import USER_HOME, PIPELINER_HOME, PIPELINER_CONF

class scRNASeqFrame( PipelineFrame ) :
    def __init__(self, pipepanel, pipeline_name, *args, **kwargs) :
        PipelineFrame.__init__(self, pipepanel, pipeline_name, *args, **kwargs)
        self.info = None

        eframe = self.eframe = LabelFrame(self,text="Options")
        #,fg=textLightColor,bg=baseColor)
        eframe.grid( row=5, column=1, sticky=W, columnspan=7, padx=10, pady=5 )

        label = Label(eframe,text="Pipeline")#,fg=textLightColor,bg=baseColor)
        label.grid(row=3,column=0,sticky=W,padx=10,pady=5)
        PipelineLabels=["CellRanger","Initial/QC","Clustering","Multi-Sample Clustering" ]
        Pipelines=["cellranger","scrnaseqinit","scrnaseqcluster", "scrnaseqmulticluster"]

        self.label2pipeline = { k:v for k,v in zip(PipelineLabels, Pipelines)}

        PipelineLabel = self.PipelineLabel = StringVar()
        self.Pipeline = StringVar()

        PipelineLabel.set(PipelineLabels[0])
        om = OptionMenu(eframe, PipelineLabel, *PipelineLabels, command=self.option_controller)
        om.config()#bg = widgetBgColor,fg=widgetFgColor)
        om["menu"].config()#bg = widgetBgColor,fg=widgetFgColor)
        #om.pack(side=LEFT,padx=20,pady=5)
        om.grid(row=3,column=1,sticky=W,padx=10,pady=5)

        self.crOpts = crOpts = LabelFrame( eframe,
                                              text="CellRanger Settings" )
        self.scrCRID = scrCRID = StringVar()
        scrCRID.set("SPECIFY_PREFIX_HERE")
        self.scrExpected = scrExpected = StringVar()
        scrExpected.set("3000")

        scrcridL = Label(crOpts, text="CellRanger Sample ID: ")
        scrcridE = Entry(crOpts, bd =2, width=25, textvariable=scrCRID)
        scrcridL.grid(row=9,column=1,sticky=W,padx=10,pady=5)
        scrcridE.grid(row=9,column=2,sticky=W,padx=0,pady=5)

        screxpectedL = Label(crOpts, text="Expected number of cells: ")
        screxpectedE = Entry(crOpts, bd =2, width=8, textvariable=scrExpected)
        screxpectedL.grid(row=10,column=1,sticky=W,padx=10,pady=5)
        screxpectedE.grid(row=10,column=2,sticky=W,padx=0,pady=5)

        self.clusterOpts = clusterOpts = LabelFrame( eframe,
                                              text="Clustering and tSNE Options" )

        self.scrPCs = scrPCs = StringVar()
        scrPCs.set("12")
        self.scrRes = scrRes = StringVar()
        scrRes.set("0.6")

        #scrPCs.trace('w', lambda a,b,c,x="scrPCs": makejson(x))

        #Filter out genes < [5] read counts in < [2] samples
        scrpcsL = Label(clusterOpts, text="Include principal components 1 through ")
        scrpcsE = Entry(clusterOpts, bd =2, width=3, textvariable=scrPCs)
        scrresL = Label(clusterOpts, text="with clustering resolution: ")
        scrresE = Entry(clusterOpts, bd =2, width=3, textvariable=scrRes)

        scrpcsL.grid(row=9,column=1,sticky=W,padx=10,pady=5)
        scrpcsE.grid(row=9,column=2,sticky=W,padx=0,pady=5)
        scrresL.grid(row=9,column=3,sticky=W,padx=5,pady=5)
        scrresE.grid(row=9,column=4,sticky=W,padx=0,pady=5)
        #scrRes.trace('w', lambda a,b,c,x="scrPCs": makejson(x))

        clusterOpts.grid( row=8, column=0, columnspan=4, sticky=W, padx=20, pady=10 )

        self.multiclusterOpts = multiclusterOpts = LabelFrame( eframe,
                                                text = "Multi-Sample Clustering and tSNE Options")

        scrccsL = Label(multiclusterOpts, text="Include canonical components 1 through ")
        scrccsE = Entry(multiclusterOpts, bd =2, width=3, textvariable=scrPCs)
        scrmcresL = Label(multiclusterOpts, text="with clustering resolution: ")
        scrmcresE = Entry(multiclusterOpts, bd =2, width=3, textvariable=scrRes)

        scrccsL.grid(row=9,column=1,sticky=W,padx=10,pady=5)
        scrccsE.grid(row=9,column=2,sticky=W,padx=0,pady=5)
        scrmcresL.grid(row=9,column=3,sticky=W,padx=5,pady=5)
        scrmcresE.grid(row=9,column=4,sticky=W,padx=0,pady=5)

        self.qcOpts = qcOpts = LabelFrame( eframe,
                                              text="Initial Settings" )
        countL = Label( qcOpts, text="Counts/Matrix Dir:" )
        countL.grid(row=9, column=1, sticky=W, padx=10, pady=5 )
        countpath=StringVar()
        self.countpath = countpath
        count_entry = Entry(qcOpts,
                           bd =2,
                           width = 50,
                           #bg = entryBgColor,
                           #fg = entryFgColor,
                           textvariable = countpath, state='normal'
                          )
        count_entry.grid( row=9, column=2, columnspan=3 )
        self.count_button = count_button = Button( qcOpts,
                             text="Open Directory",
                             command=self.set_count_directory )
        count_button.grid( row=9, column=5 )

        self.mattype = mattype = StringVar()
        mattypeL = Label(qcOpts, text="Count matrix format: ")
        scrMatTypeDropdown = ["cellranger", "cellranger_raw", "zumi", "biorad"]
        mattype.set(scrMatTypeDropdown[0])
        mattype_om = OptionMenu(qcOpts, mattype, *scrMatTypeDropdown)
        mattypeL.grid(row=10,column=1,sticky=W,padx=10,pady=5)
        mattype_om.grid(row=10,column=2,sticky=W,padx=0,pady=5)

        self.docycleregress = docycleregress = StringVar()
        docycleregressL = Label(qcOpts, text="Do cell cycle regression? ")
        scrCycleDropdown = ["TRUE", "FALSE"]
        docycleregress.set(scrCycleDropdown[0])
        cycle_om = OptionMenu(qcOpts, docycleregress, *scrCycleDropdown)
        docycleregressL.grid(row=11,column=1,sticky=W,padx=10,pady=5)
        cycle_om.grid(row=11,column=2,sticky=W,padx=0,pady=5)

        usecycleregressL_c = Label(clusterOpts, text="Use cell cycle regressed data? ")
        docycleregress.set(scrCycleDropdown[0])
        cycle_om_c = OptionMenu(clusterOpts, docycleregress, *scrCycleDropdown)
        usecycleregressL_c.grid(row=10,column=1,sticky=W,padx=10,pady=5)
        cycle_om_c.grid(row=10,column=2,sticky=W,padx=0,pady=5)

        usecycleregressL_mc = Label(multiclusterOpts, text="Use cell cycle regressed data? ")
        docycleregress.set(scrCycleDropdown[0])
        cycle_om_mc = OptionMenu(multiclusterOpts, docycleregress, *scrCycleDropdown)
        usecycleregressL_mc.grid(row=10,column=1,sticky=W,padx=10,pady=5)
        cycle_om_mc.grid(row=10,column=2,sticky=W,padx=0,pady=5)

        groups_buttonL = Label(qcOpts, text="SAMPLE INFORMATION: ")
        groups_button = Button(qcOpts,
                                            text="Set Groups",
                                            command = self.popup_groups )
        groups_buttonL.grid(row=12,column=1,sticky=W,padx=10,pady=5)
        groups_button.grid(row=12,column=2,sticky=W,padx=0,pady=5)
        #####################

        self.option_controller()

    def popup_groups( self ) :
        self.popup_window( "Groups Information", "groups.tab" )

    def popup_window( self, text, filename ) :
        top = Toplevel()

        info = LabelFrame(top, text=text )#"Group Information")
        info_text = Text(info,
                              width=50,
                              height=8,
                              #bg=projectBgColor,
                              #fg=projectFgColor,
                              font=("nimbus mono bold","11")
                             )

        def savefunc() :
            self.writepaste( filename, info_text )

        def loadfunc() :
            self.readpaste( filename, info_text )

        info_save_button = Button(info,
                                  text="Save",
                                  command = savefunc )
        info_load_button = Button(info,
                                  text="Load",
                                  command = loadfunc )

        #self.pairs_load_button.pack( side=BOTTOM, padx=5, pady=5 )
        #self.pairs_save_button.pack( side=BOTTOM, padx=5, pady=5 )

        info_load_button.grid( row=5, column=5, padx=10, pady=5 )
        info_save_button.grid( row=5, column=6, padx=10, pady=5 )
        info_text.grid(row=1,  rowspan=3,
                       column=1,  columnspan=7,
                       padx=5, pady=5 )

        info.grid(row=7,column=0, columnspan=6, sticky=W, padx=20, pady=10 )
        top.focus_force()

    def set_count_directory( self ):
        fname = askdirectory( initialdir = USER_HOME,
                             title="Select Data Directory")
        self.countpath.set(fname)

    def option_controller( self, *args, **kwargs ) :

        PipelineFrame.option_controller( self )

        self.Pipeline.set( self.label2pipeline[self.PipelineLabel.get()] )
        print( self.Pipeline.get() )

        if self.Pipeline.get() == 'cellranger' :
            self.clusterOpts.grid_forget()
            self.crOpts.grid(row=8,column=0, columnspan=6, sticky=W, padx=20, pady=10 )
            self.qcOpts.grid_forget()
            self.multiclusterOpts.grid_forget()
        elif self.Pipeline.get() == 'scrnaseqcluster' :
            self.clusterOpts.grid( row=8, column=0, columnspan=4, sticky=W, padx=20, pady=10 )
            self.crOpts.grid_forget()
            self.qcOpts.grid_forget()
            self.multiclusterOpts.grid_forget()
        elif self.Pipeline.get() == 'scrnaseqinit' :
            self.clusterOpts.grid_forget()
            self.crOpts.grid_forget()
            self.qcOpts.grid( row=8, column=0, columnspan=4, sticky=W, padx=20, pady=10 )
            self.multiclusterOpts.grid_forget()
        elif self.Pipeline.get() == 'scrnaseqmulticluster' :
            self.clusterOpts.grid_forget()
            self.crOpts.grid_forget()
            self.qcOpts.grid_forget()
            self.multiclusterOpts.grid( row=8, column=0, columnspan=4, sticky=W, padx=20, pady=10 )

    def makejson_wrapper( self, *args, **kwargs ) :
        self.makejson(*args, **kwargs)

    def makejson(self, *args):
        ###################################################### FROM RNASEQ.PY, NOT SURE WHAT ALL THIS DOES #############################3
        #print(args[0])
        caller=args[0]
        #global PD
        #global UnitsBak
        #global RGbak
        D=dict()
        try:
            F=open(self.workpath.get()+"/samples","r")
            f=F.read().split("\n")
            F.close()
            for line in f:
                L=line.split()
                a=L.pop(0)
                D[a]=L
                samples=D
        except:
            samples={"na":"na"}

        D=dict()
        try:
            F=open(self.workpath.get()+"/pairs","r")
            f=F.read().split()
            F.close()
            for i in range(0,len(f),2):
    #            a=f[i].split(".")[0]
    #            b=f[i+1].split(".")[0]
                a=f[i]
                b=f[i+1]

                D[a+"+"+b]=[a,b]

            pairs=D
        except:
            pairs={"na":"na"}

        D=dict()
        try:
            F=open(self.workpath.get()+"/contrasts.tab","r")
    #        f=F.read().split('\n')
    #        F.close()
    #        D["rsamps"]=f[0].split()
    #        D["rgroups"]=f[1].split()
    #        D["rcontrasts"]=f[2].split()
    #        D["rlabels"]=f[3].split()
            f=F.readlines()
            F.close()
    ##        sampl=[]
    ##        grp=[]
            cont=[]
    ##        lbl=[]
            for x in f:
                  if len(x.split()) == 2:
                     cont.append(x.split()[0])
                     cont.append(x.split()[1])
            D["rcontrasts"]=cont
    #        contrasts["rcontrasts"]=cont
            contrasts=D
        except:
            contrasts={"rcontrasts":"na"}
    ##        contrasts={"rsamps":"na","rgroups":"na","rcontrasts":"na","rlabels":"na"}
    ##------
        D=dict()
        try:
            F=open(self.workpath.get()+"/groups.tab","r")
            f=F.readlines()
            F.close()
            sampl=[]
            grp=[]
    #        cont=[]
            lbl=[]
            for x in f:
    #           if len(x.split()) == 4 or len(x.split()) == 3:
                if len(x.split()) == 3:
                    sampl.append(x.split()[0])
                    grp.append(x.split()[1])
                    lbl.append(x.split()[2])
            D["rsamps"]=sampl
            D["rgroups"]=grp
            D["rlabels"]=lbl
    #        D["rcontrasts"]="na"
    #        contrasts=D
            groups=D
        except:
    #        contrasts={"rsamps":"na","rgroups":"na","rcontrasts":"na","rlabels":"na"}
            groups={"rsamps":"na","rgroups":"na","rlabels":"na"}
    ##------
        D=dict()
        FT = filetype#.get()
    #    p = Popen("ls "+workpath.get()+"/*."+FT, shell=True, stdin=PIPE, stdout=PIPE, stderr=DEVNULL, close_fds=True)
        p = Popen("find "+self.workpath.get()+" -maxdepth 1 -type l -printf '%f\n' ", shell=True, stdin=PIPE, stdout=PIPE, stderr=DEVNULL, close_fds=True)
        a = p.stdout.read().decode(encoding='UTF-8').split("\n")

        RG=dict()
        b=a.pop()
    #    tkinter.messagebox.showerror("",a)
    #    if freezeunits.get()=="no":
        for i in a:

            key=re.sub(".realign","",i.split("/")[-1])
            key=re.sub(".bai","",key)
            key=re.sub(".bam","",key)
            key=re.sub(".sam","",key)
            key=re.sub(".recal","",key)
            key=re.sub(".dedup","",key)
            key=re.sub(".sorted","",key)
            key=re.sub(".fin","",key)
            key=re.sub("\.R[12]","",key)
            key=re.sub("_R[12]","",key)
            key=re.sub(".fastq","",key)
            key=re.sub(".gz","",key)
    #        key=re.sub("[\._](R[12]\.)*"+FT+"$","",i.split("/")[-1])
    #        key=re.sub(".R[12]."+FT+"$","",i.split("/")[-1])
    #        key=re.sub("([._]R[12][._])*([fin|sorted|dedup|recal|realign])*\.{0}$".format(FT),"",i.split("/")[-1])
            D[key]=key
            RG[key]={'rgsm':key,'rglb':'na','rgpu':'na','rgpl':'ILLUMINA','rgcn':'na'}
        units=D
        UnitsBak=D

        try:
            F=open(self.workpath.get()+"/rg.tab","r")
            f=F.read().splitlines()
            F.close()
            for theLine in f:
                if not re.match("^ID",theLine):
                    (rgid,rgsm,rglb,rgpl,rgpu,rgcn)=theLine.split("\t")
                    RG[rgid]={'rgsm':rgsm,'rglb':rglb,'rgpu':rgpu,'rgpl':rgpl,'rgcn':rgcn}
        except:
            pass

        RGbak=RG
    #     else:
    #         units=UnitsBak
    #         RG=RGbak
    # ###########################################################################################################################

        PD=dict()

        smparams=[]

        for i in range(len(self.parameters)):

            if cp[i].var.get()=="1":
                smparams.append(parameters[i])

        AD=eval( open( join(PIPELINER_HOME,
                            self.annotation.get()+".json"
                           ), "r"
                     ).read()
               )
        gi = self.global_info
        PD={
            'project': {
                'pfamily': gi.pfamily.get(),
                'units': units,
                'samples': samples,
                'pairs': pairs,
                'id': gi.eprojectid.get(),
                'pi': gi.epi.get(),
                'organism': gi.eorganism.get(),
                'analyst': gi.eanalyst.get(),
                'poc': gi.epoc.get(),
                'pipeline': self.Pipeline.get(),
                'version':"4.0",
                'annotation': gi.annotation.get(),
                'datapath': self.datapath.get(),
                'targetspath': self.targetspath.get(),
                'filetype': filetype ,
                'binset': "standard-bin",
                'username': gi.euser.get(),
                'flowcellid': gi.eflowcell.get(),
                'platform': gi.eplatform.get(),
                'custom': customRules,
                'efiletype': efiletype,
                'workpath': self.workpath.get(),
                'batchsize': batchsize,
                "smparams": smparams,
                "rgid": RG,
                "cluster": "cluster_medium.json",
                "description": gi.description.get('1.0',END),
                "technique" : gi.technique.get(),
                "CRID": self.scrCRID.get(),
                "EXPECTED": self.scrExpected.get(),
                "COUNTSPATH": self.countpath.get(),
                "MATTYPE": self.mattype.get(),
                "DOCYCLEREGRESS": self.docycleregress.get(),
                "RESOLUTION": self.scrRes.get(),
                "PCS": self.scrPCs.get(),
                "groups": groups
             }
        }

        J=json.dumps(PD, sort_keys = True, indent = 4, ensure_ascii=True)
        gi.jsonconf.delete("1.0", END)
        gi.jsonconf.insert(INSERT, J)
        self.saveproject(gi.jsonconf.get("1.0",END))


import numpy
from keras.datasets import cifar10
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Flatten
from keras.constraints import maxnorm
from keras.optimizers import SGD
from keras.layers.convolutional import Convolution2D
from keras.layers.convolutional import MaxPooling2D
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras import backend as K

K.set_image_dim_ordering('th')

seed = 7
numpy.random.seed(seed)

train_datagen = ImageDataGenerator(
    shear_range=0.2,
    zoom_range=0.2,
    rotation_range=0.1,
    channel_shift_range=0.2,
    horizontal_flip=True)

test_datagen = ImageDataGenerator()

(X_train, y_train), (X_test, y_test) = cifar10.load_data()

y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)

num_classes = y_test.shape[1]

train_generator = train_datagen.flow(X_train, y_train, batch_size=64)

validation_generator = test_datagen.flow(X_test, y_test, batch_size=64)

# (X_train, y_train), (X_test, y_test) = cifar10.load_data()
# X_train = X_train.astype('float32')
# X_test = X_test.astype('float32')
# X_train = X_train / 255.0
# X_test = X_test / 255.0

# y_train = np_utils.to_categorical(y_train)
# y_test = np_utils.to_categorical(y_test)
#
# num_classes = y_test.shape[1]

model = Sequential()
model.add(
    Convolution2D(32, 3, 3, input_shape=(3, 32, 32), border_mode='same', activation='relu'))
model.add(Dropout(0.2))
model.add(Convolution2D(32, 3, 3, activation='relu', border_mode='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(64, 3, 3, activation='relu', border_mode='same'))
model.add(Dropout(0.2))
model.add(Convolution2D(64, 3, 3, activation='relu', border_mode='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(128, 3, 3, activation='relu', border_mode='same'))
model.add(Dropout(0.2))
model.add(Convolution2D(128, 3, 3, activation='relu', border_mode='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(256, 3, 3, activation='relu', border_mode='same'))
model.add(Dropout(0.2))
model.add(Convolution2D(256, 3, 3, activation='relu', border_mode='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dropout(0.2))
model.add(Dense(1024, activation='relu', W_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu', W_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Dense(num_classes, activation='softmax'))

epochs = 80
learning_rate = 0.01
decay = learning_rate/(epochs*2)
sgd = SGD(lr=learning_rate, momentum=0.9, decay=decay, nesterov=False)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
print(model.summary())

numpy.random.seed(seed)

model.fit_generator(
    train_generator,
    samples_per_epoch=50000,
    nb_epoch=epochs,
    validation_data=validation_generator,
    nb_val_samples=300000,
    verbose=2)

scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))