import msprime
import numpy as np
import math
import os
import re
import argparse

#python3 Sample_Analysis.py --file_list list.txt

parser = argparse.ArgumentParser()
parser.add_argument('--file_list',nargs=1,type=str)

args = parser.parse_args()

file1=open(str(args.file_list[0]))

files=[]
for line in file1:
    line=line.strip().split()
    files.append(line)
file1.close()

print(len(files))

#os.system('cp ancientdatabase.geno for_merge_ancient.geno')
#os.system('cp ancientdatabase.snp for_merge_ancient.snp')
#os.system('cp ancientdatabase.ind for_merge_ancient.ind')

#os.system('cp vdata.geno for_merge_modern.geno')
#os.system('cp vdata.snp for_merge_modern.snp')
#os.system('cp vdata.ind for_merge_modern.ind')

for rep in files:
    os.system('samtools mpileup -R -B -q30 -Q30 -l /home/kluser2/datasets/wholegenome/1200Ksnps.bed  -f /home/kluser1/aDNA/Reference_Genomes/bwakit/hs37.fa  /home/kluser2/datasets/wholegenome/{} > {}_ancient'.format(rep[0],rep[0]))
    os.system('wc -l {}_ancient'.format(rep[0]))
    os.system('pileupCaller --sampleNames {}  --samplePopName TEST -f /home/kluser2/datasets/wholegenome/ancientdatabase.snp  -o EigenStrat -e {}_ancient < {}_ancient'.format(rep[1],rep[0],rep[0]))

    os.system('mv {}_ancient.geno.txt {}_ancient.geno'.format(rep[0],rep[1]))
    os.system('mv {}_ancient.snp.txt {}_ancient.snp'.format(rep[0],rep[1]))
    os.system('mv {}_ancient.ind.txt {}_ancient.ind'.format(rep[0],rep[1]))

    os.system('samtools mpileup -R -B -q30 -Q30 -l /home/kluser2/datasets/wholegenome/snpsforcallingHOA.bed  -f /home/kluser1/aDNA/Reference_Genomes/bwakit/hs37.fa  /home/kluser2/datasets/wholegenome/{} > {}_HOA'.format(rep[0],rep[0]))
    os.system('wc -l {}_HOA'.format(rep[0]))
    os.system('pileupCaller --sampleNames {}  --samplePopName TEST -f /home/kluser2/datasets/wholegenome/vdata.snp  -o EigenStrat -e {}_HOA < {}_HOA'.format(rep[1],rep[0],rep[0]))

    os.system('mv {}_HOA.geno.txt {}_HOA.geno'.format(rep[0],rep[1]))
    os.system('mv {}_HOA.snp.txt {}_HOA.snp'.format(rep[0],rep[1]))
    os.system('mv {}_HOA.ind.txt {}_HOA.ind'.format(rep[0],rep[1]))

    supp1=open('vdata.snp','r')
    supp2=open('ancientdatabase.snp','r')

    snp1=open('{}_HOA.snp'.format(rep[1]),'r')
    snp2=open('{}_ancient.snp'.format(rep[1]),'r')

    newsnp1=open('{}_HOA_new.snp'.format(rep[1]),'w')
    newsnp2=open('{}_ancient_new.snp'.format(rep[1]),'w')

    di={}
    for j in supp1:
            line=j.strip().split()
            position=str(line[1])+':'+str(line[3])
            rs=line[0]
            alt=line[4]
            ref=line[5]
            di[position]=[rs,alt,ref]

    print(len(di))

    for f in snp1:
            line=f.strip().split()
            position=str(line[1])+':'+str(line[3])
            try:
                    line[0]=di[position][0]
                    if line[4]=='<NON_REF>':
                            if line[5]==di[position][1]:
                                    line[4]=di[position][2]
                            else:
                                    line[4]=di[position][1]
            except KeyError:
                    line[0]='Missing_rs'
            newsnp1.write("\t".join(line))
            newsnp1.write("\n")
    di={}

    for j in supp2:
            line=j.strip().split()
            position=str(line[1])+':'+str(line[3])
            rs=line[0]
            alt=line[4]
            ref=line[5]
            di[position]=[rs,alt,ref]

    print(len(di))

    for f in snp2:
            line=f.strip().split()
            position=str(line[1])+':'+str(line[3])
            try:
                    line[0]=di[position][0]
                    if line[4]=='<NON_REF>':
                            if line[5]==di[position][1]:
                                    line[4]=di[position][2]
                            else:
                                    line[4]=di[position][1]
            except KeyError:
                    line[0]='Missing_rs'
            newsnp2.write("\t".join(line))
            newsnp2.write("\n")

    supp1.close()
    supp2.close()

    snp1.close()
    snp2.close()

    newsnp1.close()
    newsnp2.close()

    os.system('mv {}_HOA_new.snp {}_HOA.snp'.format(rep[1],rep[1]))
    os.system('mv {}_ancient_new.snp {}_ancient.snp'.format(rep[1],rep[1]))

    parfileancient=open('parfile_ancient','w')
    parfilemodern=open('parfile_HOA','w')

    parfilemodern.write('geno1: vdata.geno\n')
    parfilemodern.write('snp1: vdata.snp\n')
    parfilemodern.write('ind1: vdata.ind\n')
    parfilemodern.write('geno2: {}_HOA.geno\n'.format(rep[1]))
    parfilemodern.write('snp2: {}_HOA.snp\n'.format(rep[1]))
    parfilemodern.write('ind2: {}_HOA.ind\n'.format(rep[1]))
    parfilemodern.write('genooutfilename: {}_HOA_merged.geno\n'.format(rep[1]))
    parfilemodern.write('snpoutfilename: {}_HOA_merged.snp\n'.format(rep[1]))
    parfilemodern.write('indoutfilename: {}_HOA_merged.ind'.format(rep[1]))

    parfileancient.write('geno1: ancientdatabase.geno\n')
    parfileancient.write('snp1: ancientdatabase.snp\n')
    parfileancient.write('ind1: ancientdatabase.ind\n')
    parfileancient.write('geno2: {}_ancient.geno\n'.format(rep[1]))
    parfileancient.write('snp2: {}_ancient.snp\n'.format(rep[1]))
    parfileancient.write('ind2: {}_ancient.ind\n'.format(rep[1]))
    parfileancient.write('genooutfilename: {}_ancient_merged.geno\n'.format(rep[1]))
    parfileancient.write('snpoutfilename: {}_ancient_merged.snp\n'.format(rep[1]))
    parfileancient.write('indoutfilename: {}_ancient_merged.ind\n'.format(rep[1]))

    parfilemodern.close()
    parfileancient.close()

    os.system('mergeit -p parfile_ancient')
    os.system('mergeit -p parfile_HOA')

    os.system('rm parfile_HOA')
    os.system('rm parfile_ancient')

    convert_ancient=open('convert_ancient','w')
    convert_modern=open('convert_HOA','w')

    convert_ancient.write('genotypename: {}_ancient_merged.geno\n'.format(rep[1]))
    convert_ancient.write('snpname: {}_ancient_merged.snp\n'.format(rep[1]))
    convert_ancient.write('indivname: {}_ancient_merged.ind\n'.format(rep[1]))
    convert_ancient.write('outputformat: PACKEDPED\n')
    convert_ancient.write('genotypeoutname: {}_ancient_merged.bed\n'.format(rep[1]))
    convert_ancient.write('snpoutname: {}_ancient_merged.bim\n'.format(rep[1]))
    convert_ancient.write('indivoutname: {}_ancient_merged.fam'.format(rep[1]))

    convert_modern.write('genotypename: {}_HOA_merged.geno\n'.format(rep[1]))
    convert_modern.write('snpname: {}_HOA_merged.snp\n'.format(rep[1]))
    convert_modern.write('indivname: {}_HOA_merged.ind\n'.format(rep[1]))
    convert_modern.write('outputformat: PACKEDPED\n')
    convert_modern.write('genotypeoutname: {}_HOA_merged.bed\n'.format(rep[1]))
    convert_modern.write('snpoutname: {}_HOA_merged.bim\n'.format(rep[1]))
    convert_modern.write('indivoutname: {}_HOA_merged.fam'.format(rep[1]))

    convert_modern.close()
    convert_ancient.close()

    os.system('convertf -p convert_ancient')
    os.system('convertf -p convert_HOA')

    os.system('rm convert_ancient')
    os.system('rm convert_HOA')


import tqdm
import celldom
from celldom.core import cytometry
from celldom.core import acquisition
from celldom.extract import APT_IMAGES, ALL_IMAGES
from celldom.extract import cell_extraction
from celldom.exception import NoMarkerException
from skimage import io
import logging

logger = logging.getLogger(__name__)

MAX_PROC_FAILURES = 10
MAX_FILES_IN_MEM_RES = 500

def run_cytometer(exp_config, output_dir, files, max_failures=MAX_PROC_FAILURES,
                  return_results=None, sample_count=None,
                  max_invalid_acquisitions=0, dpf=APT_IMAGES, **kwargs):

    if return_results and len(files) > MAX_FILES_IN_MEM_RES:
        raise ValueError(
            'Processing cannot be run with `return_results=True` and a large number of input data files '
            '(given {} files and threshold for this error is {}).  Either set return_results=False '
            '(in which case data is saved on disk) or specify smaller batches of files yourself '
            'and deal with trimming results in memory.  Nuclear option: '
            'celldom.processing.MAX_FILES_IN_MEM_RES = float("inf")'
            .format(len(files), MAX_FILES_IN_MEM_RES)
        )
    # Default to returning results if no output directory given
    if return_results is None:
        return_results = output_dir is None

    # Group files into acquisition instances
    logger.info('Collapsing image files into acquisitions ...')
    acqs = acquisition.collapse_acquisitions(files, exp_config, max_invalid_acquisitions=max_invalid_acquisitions)
    logger.info('%s image files collapsed into %s acquisitions', len(files), len(acqs))

    # Apply sampling, if requested
    if sample_count is not None:
        if sample_count < 1:
            raise ValueError('Sample count must be >= 1 (not {})'.format(sample_count))
        logger.info('Selecting first %s acquisitions', sample_count)
        # Making this random may be useful in the future, but for now an arbitrary selection will do
        acqs = acqs[:sample_count]

    logger.info('Beginning processing of %s acquisitions ...', len(acqs))
    results = []
    with cytometry.Cytometer(exp_config, data_dir=output_dir, **kwargs) as cytometer:
        n_fail = 0
        for i, acq in tqdm.tqdm(enumerate(acqs), total=len(acqs)):
            try:
                logger.debug('Processing acquisition for path "%s"', acq.get_primary_path())

                # Analyze the image
                result = cytometer.analyze(acq, dpf=dpf)

                if return_results:
                    results.append(result)

                # Save the results
                if output_dir is not None:
                    cytometer.save(*result)

            # If there are no markers, log a more succinct message and still count this event as a failure
            except NoMarkerException:
                n_fail += 1
                logger.error(
                    'No markers found in acquisition (paths = "%s", failure threshold = %s, '
                    'current failure count = %s)', acq.paths, max_failures, n_fail
                )
            # Otherwise log whole trace
            except Exception:
                n_fail += 1
                logger.exception(
                    'A failure occurred processing acquisition (paths = "%s", failure threshold = %s, '
                    'current failure count = %s)', acq.paths, max_failures, n_fail
                )
            if n_fail >= max_failures:
                logger.error('Threshold for max number of failures exceeded; skipping any further processing')
                break

    return results if return_results else None

def run_cell_detection(exp_config, files, **kwargs):
    with cytometry.Cytometer(exp_config, data_dir=None, **kwargs) as cytometer:
        cell_model = cytometer.cell_model
        chip_config = exp_config.get_chip_config()
        for file in tqdm.tqdm(files):
            img = io.imread(file)
            result = cell_extraction.extract(img, cell_model, chip_config, dpf=ALL_IMAGES)
            result = file, img.shape, *result
            yield result


import PySimpleGUI as sg

layout = [
    [sg.Input(key = "-INPUT-"),
    sg.Spin(["km to mile", "kg to pound", "sec to min"], key = "-UNITS-"),
    sg.Button("Convert", key = "-CONVERTBUTTON-")],
    [sg.Text("Output", key = "-OUTPUT-")],
    [sg.Button("Close", key = "-CLOSEBUTTON-")]
]

window = sg.Window("Converter", layout)

while True:
    event, values = window.read()

    if event == sg.WIN_CLOSED:
        break

    if event == "-CLOSEBUTTON-":
        window.close()

    if event == "-CONVERTBUTTON-":
        inputValue = values["-INPUT-"]

        if inputValue.isnumeric():
            if values["-UNITS-"] == "km to mile":
                output = float(inputValue) * 0.06214
                outputStr = f"{inputValue} km are {round(output, 2)} miles."

            elif values["-UNITS-"] == "kg to pound":
                output = float(inputValue) * 2.20462
                outputStr = f"{inputValue} kg are {round(output, 2)} pounds."

            elif values["-UNITS-"] == "sec to min":
                output = float(inputValue) / 60
                outputStr: str = f"{inputValue} sec are {round(output, 2)} mins."

            window["-OUTPUT-"].update(outputStr)

        else:
            window["-OUTPUT-"].update("Please enter an integer")


from django.urls import path
from .views import ChatView, CreateChatView, DeleteChatView, AddUserToGroupView, CreateInvite
from django.contrib.auth.decorators import login_required

urlpatterns = [
    path('chat/<int:group_id>/', login_required(ChatView.as_view()), name='chat'),
    path('chat/create', login_required(CreateChatView.as_view()), name='create_chat'),
    path('chat/delete/<int:group_id>/', login_required(DeleteChatView.as_view()), name='delete_chat'),
    path('invite/<str:invite_str>/', login_required(AddUserToGroupView.as_view()), name='invite'),
    path('invite/create', login_required(CreateInvite.as_view()), name='create_invite')
]


from collections import OrderedDict
from transformers.transformer_base import TransformerBase

class ListReturnsTransformer(TransformerBase):

    RENAME_DATA = OrderedDict([
        ("EFiledlist.ret_prd", "return_period"),
        ("EFiledlist.mof", "method_of_filing"),
        ("EFiledlist.dof", "date_of_filing"),
        ("EFiledlist.rtntype", "return_type"),
    ])

    def rearrange_transformation(self):
        self.data = self.data['EFiledlist']


from django.conf.urls import patterns, include, url
import views

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'Fingerprints_django.views.home', name='home'),

    url(r'^$', views.home),
    url(r'^home/?$', views.home, name='home'),
    url(r'^how_to_use_api/?$', views.how_to_use_api, name='how_to_use_api'),
    url(r'^client_libs/?$', views.client_libs, name='client_libs'),
    url(r'^applications/?$', views.applications, name='applications'),
    url(r'^contact/?$', views.contact, name='contact'),
    url(r'^(?P<url>.*)$', views.not_found, name='not_found'),
)


# python3 LinearRegression.py
# SIMPLE LINEAR REGRESSION

''' IMPORT LIBRARIES  '''

import numpy as np
from matplotlib import pyplot as plt
import pandas as pd

''' IMPORT DS '''

url = 'HousePrice.csv'
DS = pd.read_csv( url )

print("\n\n-> DATASET SHAPE\n",DS.shape)
print( '\n\n-> FIRST 5 ROWS\n' )
print(DS.head(10))
print("\n\n")

''' DESCRIPTION OF DATASET '''

DS.describe()

''' EXTRACTING DEPENDENT & INDEPENDENT VARIABLES  '''

X = DS.iloc[:, :-1].values
y = DS.iloc[:, -1].values

''' SPLITTING the DS INTO TRAINING SET AND TEST SET '''

from sklearn.model_selection import train_test_split
X_Training, X_Test, Y_Training, Y_Test = train_test_split(X, y, test_size = 0.2, random_state = 0)

print("\n\n-> X TRAIN SHAPE\n",X_Training.shape)
print("\n\n-> Y TRAIN SHAPE\n",Y_Training.shape)

print("\n\n-> X TEST SHAPE\n",X_Test.shape)
print("\n\n-> Y TEST SHAPE\n",Y_Test.shape)

''' LINEAR REGRESSION MODEL : TRAINING'''

from sklearn.linear_model import LinearRegression
Regressor = LinearRegression()
Regressor.fit(X_Training, Y_Training)

''' REGRESSION INTERCEPT & COEFF '''

print("\n\n-> REGRESSION INTERCEPT : ",Regressor.intercept_)
print("\n-> REGRESSION COEFF : ",Regressor.coef_)
print("\n\n")

''' PREDICTING TEST SET RESULTS '''

y_pred = Regressor.predict(X_Test)

''' PRINTING ACTUAL VS PREDICTED PRICE '''

test_data = pd.DataFrame({'Actual': Y_Test, 'Predicted': y_pred})
print("\n\n-> TEST DATA \n")
print(test_data)

''' ERROR '''

from sklearn import metrics

print('\n\n-> MEAN ABSOLUTE ERROR:', metrics.mean_absolute_error(Y_Test, y_pred))
print('\n-> MEAN SQUARED ERROR:', metrics.mean_squared_error(Y_Test, y_pred))
print('\n-> ROOT MEAN SQUARED ERROR:', np.sqrt(metrics.mean_squared_error(Y_Test, y_pred)))

''' VISUALISING THE TRAINING SET RESULTS '''

plt.scatter(X_Training, Y_Training, color='red')
plt.plot(X_Training, Regressor.predict(X_Training), color='blue')
plt.title('PRICE vs AREA (TRAINING SET)')
plt.xlabel('Area')
plt.ylabel('Price')
plt.show()

''' VISUALISING THE TEST SET RESULTS '''

plt.scatter(X_Test, Y_Test, color='red')
plt.plot(X_Training, Regressor.predict(X_Training), color='blue')     # train values used here because regression line is made of train set
plt.title('PRICE vs AREA (TEST SET)')
plt.xlabel('Area')
plt.ylabel('Price')
plt.show()


import gym

env = gym.make('BreakoutDeterministic-v4')
state = env.reset()
count = 0
done = False
for t in range(10000) or not done:
    env.render()
    next_state, reward, done, info = env.step(env.action_space.sample())
    count += 1
    if done:
        break
print(count)


from django.shortcuts import render,HttpResponse
import io
from rest_framework.parsers import JSONParser
from .models import Student
from .serializer import CreateSerializer
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt

# Create your views here.
@csrf_exempt
def CreateStudent(request):
    if request.method == 'POST':
        b_data = request.body
        stream = io.BytesIO(b_data)
        p_data = JSONParser().parse(stream)
        c_serializer = CreateSerializer(data=p_data)
        if c_serializer.is_valid():
            c_serializer.save()
            return JsonResponse({"msg":"Data Inserted Successfully."})
        else:
            return JsonResponse(c_serializer.errors)
    else:
        return HttpResponse("It GET method")

# Get Data
@csrf_exempt
def get_data(request):
    if request.method == 'POST':
        j_data = request.body
        strem  = io.BytesIO(j_data)
        py_data = JSONParser().parse(strem)
        id = py_data.get('id',None)

        if id:
            stu = Student.objects.get(id = id)
            serializer = CreateSerializer(stu)
        else:
            stu = Student.objects.all()
            serializer = CreateSerializer(stu,many=True)
        return JsonResponse(serializer.data,safe= False )
    else:
        stu = Student.objects.all()
        # print(stu)
        serializer = CreateSerializer(stu,many=True)
        return JsonResponse(serializer.data,safe = False)

@csrf_exempt
def UpdateStudent(request):

    if request.method == 'PUT':
        r_body = request.body
        stream = io.BytesIO(r_body)
        py_data = JSONParser().parse(stream)
        id   =  py_data.get('id')
        stu  = Student.objects.get(id=id)
        serialize_up = CreateSerializer(stu,data=py_data,partial=True)
        if serialize_up.is_valid():
            serialize_up.save()
            return JsonResponse({"msg":"Data Updated Sucessfully."})
        else:
            return JsonResponse(serialize_up.errors)
    else:
        return HttpResponse("Its PUT Respone")

@csrf_exempt
def DeleteStudent(request):

    if request.method == "DELETE":
        j_body = request.body
        stream = io.BytesIO(j_body)
        py_data = JSONParser().parse(stream)
        id = py_data.get('id')
        stu = Student.objects.get(id = id)
        stu.delete()
        return JsonResponse({"msg":"Data Delted Successfully."})


import unittest
import yaml
from ABM.space_discrete.model import AirportModel

class TestModel(unittest.TestCase):
    def test_random_model(self):
        random_config = yaml.load(open('ABM/configs/default_config.yaml'), Loader=yaml.FullLoader)

        model = AirportModel()
        self.assertEqual(model.name, 'ABM')

    def test_image_view(self):
        image_config = yaml.load(open('ABM/configs/image_scenario.yaml'), Loader=yaml.FullLoader)
        model = AirportModel()
        self.assertEqual(model.name, 'ABM')


# -*- encoding: utf-8 -*-

from datetime import datetime, timedelta
from openerp.osv import fields,osv
from openerp.tools import DEFAULT_SERVER_DATE_FORMAT, DEFAULT_SERVER_DATETIME_FORMAT

class taobao_order(osv.osv):
    _name = 'taobao.order'
    _description = u"淘宝订单"

    _columns = {
        'name': fields.char(u'订单编号', required=True, copy=False, readonly=True, select=True),
        'buyer': fields.char(u'买家昵称', size=20, required=True),
        'order_date': fields.datetime(u'拍下时间', select=True),
        'pay_date': fields.datetime(u'付款时间', select=True),
        'delivery_date': fields.datetime(u'发货时间', select=True),
        'end_date': fields.datetime(u'交易结束时间', select=True),
        'freight': fields.float(u'运费'),
        'total_price': fields.float(u'总价'),
        'order_state': fields.selection([
            ('not_paid', u'等待买家付款'), ('paid', u'买家已付款'), ('send', u'卖家已发货'),
            ('success', u'交易成功'), ('drop', u'交易关闭'), ('not_paid_and_not_send', u'待付款和待发货订单'),
            ('refunding', u'退款中的订单'), ('front_paid', u'定金已付'), ('exceptional', u'异常订单')
        ], u'订单状态', required=True),
        'buyer_detail': fields.char(u'收件人信息', size=100),
        'order_line': fields.one2many('taobao.order.line', 'order_id', u'淘宝订单行', readonly=True, copy=True),
        'sync_state': fields.selection([('none', u'未同步'), ('update', u'待更新'), ('done', u'已同步')], u'同步状态', required=True, readonly=True),
        'orgin_string': fields.char(u'源数据', readonly=True)
    }

    _defaults = {
        'sync_state': 'none',
    }

    _sql_constraints = [
        ('name_uniq', 'unique(name)', 'Order Reference must be unique!'),
    ]

    def create_sale_order_line(self, cr, uid, product_id, partner_id, pricelist_id, qty, price_unit, context=None):
        line_obj = self.pool.get('sale.order.line')
        line_vals = line_obj.product_id_change(cr, uid, [], pricelist_id, product_id, qty=qty, uom=1, partner_id=partner_id, context=context)['value']
        line_vals.update({'product_id': product_id , 'price_unit':price_unit, 'product_uom_qty': qty } )
        if line_vals.get('tax_id') != None:
            line_vals['tax_id'] = [(6, 0, line_vals['tax_id'])]
        return (0, 0, line_vals)

    def create_sale_order(self, cr, uid, order, context=None):
        order_obj = self.pool.get('sale.order')
        line_obj = self.pool.get('sale.order.line')
        product_match_obj = self.pool.get('taobao.product.match')

        #如果订单已存在，则跳过创建，直接返回订单id
        order_ids = order_obj.search(cr, uid, [('name', '=', order['name'])], context = context)
        if order_ids:
            return order_ids[0]

        order_val = self.remount_sale_order_val(cr, uid, order, context = context)

        order_id = order_obj.create(cr, uid, order_val, context = context)
        return order_id

    def assets_state(self, cr, uid, taobao_order, context=None):
        state = taobao_order.order_state

        if state == 'not_paid_and_not_send' or state == 'refunding' or state == 'front_paid' or state == 'exceptional':
           raise osv.except_osv(u'订单同步失败', u'暂不支持该状态下的订单同步，请手动同步: "%s"' % (taobao_order['name']))
        return True

    def remount_sale_order_val(self, cr, uid, order, context=None):
        order_obj = self.pool.get('sale.order')
        line_obj = self.pool.get('sale.order.line')
        product_match_obj = self.pool.get('taobao.product.match')

        partner_ids = self.pool.get('res.partner').search(cr, uid, [('name', '=', u'淘宝客户')], context = context)
        partner_id = partner_ids[0]

        note = '昵称: ' + order.buyer + '\n' + '地址: ' + order.buyer_detail

        order_val = order_obj.onchange_partner_id(cr, uid, [], partner_id, context=context)['value']
        order_val.update({
            'name': order.name,
            'note': note,
            'date_order':  order.pay_date or order.order_date,      #付款时间
            'create_date': order.order_date,    #拍下时间
            'partner_id': partner_id,
            'picking_policy': 'direct',
            'order_policy': 'manual',
            'order_line': [],
        })

        for line in order.order_line:
            #添加订单明细行
            product_id = product_match_obj.find_product(cr, uid, line.product_id, line.product_code, context = context)
            qty = line.qty
            # 如果子订单行自动关闭或者已取消，则按照数量0发货
            if order.order_state != 'drop' and (line.line_state == 'close' or line.line_state == 'cancel'):
                qty = 0
            sale_line = self.create_sale_order_line(cr, uid, product_id, partner_id, order_val['pricelist_id'], qty, line.price_unit, context=context)
            order_val['order_line'].append(sale_line)

        if order.freight > 0:
            #邮费
            product_id = self.pool.get('product.product').search(cr, uid, [('name', '=', u'邮费')], context = context)[0]
            sale_line = self.create_sale_order_line(cr, uid, product_id, partner_id, order_val['pricelist_id'], 1, order.freight, context=context)
            order_val['order_line'].append(sale_line)
        return order_val

    def update_sale_order(self, cr, uid, taobao_order, context=None):
        order_obj = self.pool.get('sale.order')
        order_ids = order_obj.search(cr, uid, [('name', '=', taobao_order['name'])], context = context)
        if not order_ids:
            raise osv.except_osv(u'订单同步失败', u'无法找到对应的销售订单: "%s"' % (taobao_order['name']))
        sale_order = order_obj.browse(cr, uid, order_ids[0], context = context)

        # 更新订单基本信息，包括订单行数据
        order_val = self.remount_sale_order_val(cr, uid, taobao_order, context = context)
        sale_order_lines = len(sale_order.order_line)
        for i, line in enumerate(order_val['order_line']):
            # 更新旧的订单行
            if i < sale_order_lines:
                sale_order_line = sale_order.order_line[i]
                order_val['order_line'][i] = (1, sale_order_line.id, line[2])

        # 如果之前的行数多余现在的行数，则删除多余的订单行
        if len(order_val['order_line']) < sale_order_lines:
            for i in range(order_val['order_line'], sale_order_lines):
                order_val['order_line'].append( (2, sale_order.order_line[i].id) )

        order_obj.write(cr, uid, order_ids, order_val, context = context)

        self.assets_state(cr, uid, taobao_order, context = context)

        def datetime2day(time):
            return (datetime.strptime(time, '%Y-%m-%d %H:%M:%S') + timedelta(hours=8)).strftime(DEFAULT_SERVER_DATE_FORMAT)

        # 取消订单
        if taobao_order.order_state == 'drop':
            order_obj.action_cancel(cr, uid, order_ids, context = context)
            return True

        # 未支付，不改变状态
        if taobao_order.order_state == 'not_paid':
            return True

        # # TODO: 已支付，未发货也不改变状态，因为买家可能在发货之前退款或者退部分货? 待确认
        # if taobao_order.order_state == 'paid':
        #     return True

        # 已支付，先确认订单
        if sale_order.state == 'draft':
            order_obj.action_button_confirm(cr, uid, order_ids, context = context)

        # 服务类产品销售订单, 确认日期改为发货日期
        if not sale_order.picking_ids or len(sale_order.picking_ids) == 0:
            day_date = datetime2day(taobao_order.delivery_date)
            order_obj.write(cr, uid, [sale_order.id], { 'date_confirm': day_date }, context = context)

        if taobao_order.order_state == 'paid':
            return True

        # 检查库存可用
        stock_picking_obj = self.pool.get('stock.picking')
        stock_move_obj = self.pool.get('stock.move')
        for pick in sale_order.picking_ids:
            if pick.state == 'done':
                continue
            if pick.state == 'confirmed':
                stock_picking_obj.action_assign(cr, uid, pick.ids, context = context)
            if pick.state != 'assigned':
                to_move_names = [x.name for x in pick.move_lines if x.state not in ('draft', 'cancel', 'assigned', 'done')]
                display_name = ','.join(to_move_names)
                raise osv.except_osv(u'订单同步失败', u'下面产品无法发货: "%s"。请检查库存' % (display_name))

        # 发货
        for pick in sale_order.picking_ids:
            if pick.state == 'done':
                continue
            pick_context = context.copy()
            pick_context.update({
                'active_model': 'stock.picking',
                'active_ids': pick.ids,
                'active_id': len(pick.ids) and pick.ids[0] or False
            })
            # 设置发货时间
            stock_picking_obj.do_transfer(cr, uid, pick.ids, context = pick_context)
            stock_picking_obj.write(cr, uid, pick.ids, {'date_done': taobao_order.delivery_date}, context=context)
            for move_line in pick.move_lines:
                stock_move_obj.write(cr, uid, move_line.ids, {'date': taobao_order.delivery_date}, context=context)

        # 确认发票
        invoice_obj = self.pool.get('account.invoice')
        if not sale_order.invoice_exists:
            vals = order_obj.manual_invoice(cr, uid, order_ids, context = context)
            inv_id = vals['res_id']
        else:
            inv_id = sale_order.invoice_ids[0].id

        for inv in sale_order.invoice_ids:
            if inv.state == 'draft':
                day_date = datetime2day(taobao_order.delivery_date)
                inv.write({'date_invoice': day_date, 'date_due': day_date})
                invoice_obj.signal_workflow(cr, uid, [inv.id], 'invoice_open')

        if taobao_order.order_state == 'send':
            return True

        # 确认付款   inv.state == 'open'
        journal_pool = self.pool.get('account.journal')
        journal_id = journal_pool.search(cr, uid, [('name', '=', u'银行')], context = context)[0]
        journal = journal_pool.browse(cr, uid, [journal_id], context = context)[0]
        for inv in sale_order.invoice_ids:
            if inv.state != 'open':
                continue
            # 设置发票确认日期
            day_date = datetime2day(taobao_order.end_date)
            inv = inv.with_context(date_p=day_date)
            inv.write({'date_due': day_date})
            period = inv.period_id.with_context(context).find(day_date)[:1]
            inv.pay_and_reconcile(
                pay_amount=inv.amount_total,
                pay_account_id=journal.default_debit_account_id.id,
                period_id=period.id,
                pay_journal_id=journal_id,
                writeoff_acc_id=journal.default_debit_account_id.id,
                writeoff_period_id=period.id,
                writeoff_journal_id=journal_id,
                name='/'
            )
        return True

    def action_sync(self, cr, uid, ids, context=None):
        for taobao_order in self.browse(cr, uid, ids, context=context):
            if taobao_order.sync_state == 'none' or context['force']:
                self.create_sale_order(cr, uid, taobao_order, context=context)
            if taobao_order.sync_state != 'done' or context['force']:
                self.update_sale_order(cr, uid, taobao_order, context=context)
        self.write(cr, uid, ids, {'sync_state': 'done'}, context=context)
        return True

class taobao_order_line(osv.osv):
    _name = 'taobao.order.line'
    _description = u'淘宝订单行'
    _columns = {
        'order_id': fields.many2one('taobao.order', u'订单', required=True, ondelete='cascade', select=True, readonly=True),
        'product_id': fields.char(u'名称'),
        'product_code': fields.char(u'属性编码'),
        'qty': fields.float(u'数量'),
        'price_unit': fields.float(u'单价', required=True),
        'line_state': fields.selection([
            ('wait_pay', u'等待付款'), ('wait_send', u'等待发货'), ('send', u'已发货'),
            ('success', u'交易成功'), ('close', u'自动关闭'), ('cancel', u'已取消')
        ], u'子订单状态', required=True,  readonly=True),
    }


import base64

from django.http import HttpResponse, HttpResponseRedirect
from django.utils.encoding import force_text, force_bytes
from django.views.generic import TemplateView, View

from .digidocservice.service import DataFile
from .generic import (MobileIdSignViewMixin, MobileIdStatusViewMixin, DigidocCompleteViewMixin, IdCardPrepareViewMixin,
                      IdCardFinishViewMixin)

class SKTestView(TemplateView):
    template_name = 'esteid/test.html'

    def get_context_data(self, **kwargs):
        context = super(SKTestView, self).get_context_data(**kwargs)

        try:
            files = self.request.session['__ddoc_files']

        except KeyError:
            files = {}

        context.update({
            'files': files,
        })

        return context

    def post(self, request, *args, **kwargs):
        """ This method is only for testing and can
            be used to add some files to the session.

            actions:

                - add_file: Add another file
                - remove_file: Add another file
        """
        try:
            files = request.session['__ddoc_files']

        except KeyError:
            files = {}

        action = request.POST.get('action', 'add_file')

        if action == 'remove_file':
            file_name = request.POST.get('file_name', '')

            n_files = {}
            for key, file in files.items():
                if key != file_name:
                    n_files[key] = file

            files = n_files

        else:
            for key, file in request.FILES.items():
                file_name = file.name

                idx = 1
                while file_name in files.keys():
                    file_name = '%d_%s' % (idx, file_name)
                    idx += 1

                files[file_name] = dict(
                    content=force_text(base64.b64encode(file.read())),
                    content_type=file.content_type,
                    size=file.size,
                )

        request.session['__ddoc_files'] = files

        return HttpResponseRedirect('.')

class TestFilesMixin(object):
    def get_files(self):
        files = []

        for file_name, file in self.request.session['__ddoc_files'].items():
            files.append(
                DataFile(
                    file_name=file_name,
                    mimetype=file['content_type'],
                    content_type=None,
                    size=file['size'],
                    content=base64.b64decode(force_bytes(file['content'])),
                )
            )

        return files

class TestMobileIdSignView(TestFilesMixin, MobileIdSignViewMixin, View):
    def build_action_kwargs(self):
        return dict(
            id_code=self.request.POST.get('id_code', ''),
            phone_nr=self.request.POST.get('phone_nr', ''),
            language=self.request.POST.get('language', None),
        )

class TestMobileIdStatusView(TestFilesMixin, MobileIdStatusViewMixin, View):
    pass

class TestDownloadContainerView(TestFilesMixin, DigidocCompleteViewMixin, View):
    def get(self, request, *args, **kwargs):
        # Get the file data
        the_file = self.do_action(*args, **kwargs)

        # Destroy digidoc session
        self.destroy_digidoc_session()

        # Clear active files
        try:
            del request.session['__ddoc_files']

        except KeyError:
            pass

        # Download the file
        response = HttpResponse(the_file, content_type='application/vnd.bdoc-1.0')
        response['Content-Disposition'] = 'attachment; filename=' + 'signed.bdoc'
        return response

class TestIdCardSignView(TestFilesMixin, IdCardPrepareViewMixin, View):
    def build_action_kwargs(self):
        return {
            'certificate': self.request.POST.get('certificate', ''),
            'token_id': self.request.POST.get('token_id', ''),
        }

class TestIdCardFinishView(TestFilesMixin, IdCardFinishViewMixin, View):
    def build_action_kwargs(self):
        return {
            'signature_id': self.request.POST.get('signature_id', ''),
            'signature_value': self.request.POST.get('signature_value', ''),
        }

class AuthenticationView(TemplateView):
    template_name = 'esteid/authenticate.html'

    def __init__(self, *args, **kwargs):
        self.id_auth = None
        self.id_err = None

    def dispatch(self, request, *args, **kwargs):
        self.id_auth = getattr(request, 'id_auth', None)
        self.id_err = getattr(request, 'id_err', None)

        return super(AuthenticationView, self).dispatch(request, *args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(AuthenticationView, self).get_context_data(**kwargs)

        self.request.session['id_auth'] = self.id_auth
        context['id_auth'] = self.id_auth
        context['id_err'] = self.id_err

        return context


#! /usr/bin/env python
# -*- coding:utf8 -*-
from functools import lru_cache
import time

from framework.localconcurrent import ProcessExcecutor

@lru_cache
def my_func(num):
    print('start to call my function')
    time.sleep(5)
    print('complete calling my function')
    return 'done'

if __name__ == '__main__':
    #单进程
    # for i in range(1, 5, 1):
    #     print(my_func())

    #多进程下缓存无效
    results = ProcessExcecutor(2).execute(my_func, list(range(1, 6, 1)))
    for result in results:
        print(result)


import lnschema_bionty as lb
import pandas as pd
import pytest

import lamindb as ln  # noqa

@pytest.fixture(scope="module")
def df():
    return pd.DataFrame(
        (
            ["T cell", "CL:0000084"],
            ["hepatocyte", "CL:0000182"],
            ["my new cell type", ""],
        ),
        columns=["cell_type", "cell_type_id"],
    )

def test_from_values_name(df):
    lb.CellType.filter().delete()
    assert df["cell_type"].tolist() == ["T cell", "hepatocyte", "my new cell type"]
    # create records from bionty
    result = lb.CellType.from_values(df.cell_type, "name")
    ids = [i.ontology_id for i in result]
    assert len(result) == 2
    assert set(ids) == {"CL:0000084", "CL:0000182"}
    assert result[0].bionty_source.entity == "CellType"

    # wrong field type
    with pytest.raises(TypeError):
        result = lb.CellType.from_values(df.cell_type, field=lb.CellType)

def test_from_values_ontology_id(df):
    assert df["cell_type_id"].tolist() == ["CL:0000084", "CL:0000182", ""]
    result = lb.CellType.from_values(df.cell_type_id, "ontology_id")
    names = {i.name for i in result}
    assert len(result) == 2
    assert names == {"T cell", "hepatocyte"}
    assert result[0].bionty_source.entity == "CellType"

def test_from_values_multiple_match():
    records = lb.Gene.from_values(["ABC1", "PDCD1"], lb.Gene.symbol, organism="human")
    assert len(records) == 3

def test_from_values_organism():
    from lnschema_bionty import Gene, settings

    settings._organism = None

    with pytest.raises(AssertionError):
        Gene.from_values(["ABC1"], Gene.symbol)

    settings.organism = "human"
    values = ["ABC1"]
    curated_values = Gene.bionty().standardize(values)
    records = Gene.from_values(curated_values, Gene.symbol)
    assert records[0].ensembl_gene_id == "ENSG00000068097"

    settings.organism = "mouse"
    values = ["ABC1"]
    curated_values = Gene.bionty().standardize(values)
    records = Gene.from_values(curated_values, Gene.symbol)
    assert records[0].ensembl_gene_id == "ENSMUSG00000015243"

def test_get_or_create_records():
    names = ["ulabel" + str(i) for i in range(25)]
    labels = [ln.ULabel(name=name) for name in names]
    ln.save(labels)
    # more than 20 existing values
    labels = ln.ULabel.from_values(names, field="name")
    assert len(labels) == 25

def test_from_values_synonyms_aware():
    lb.CellType.from_bionty(name="T cell").save(parents=False)
    # existing validated values
    records = lb.CellType.from_values(["T cell"], "name")
    assert len(records) == 1
    assert records[0].name == "T cell"
    assert isinstance(records[0].bionty_source, lb.BiontySource)
    # existing validated values and synonyms
    records = lb.CellType.from_values(["T cell", "T-cell"], "name")
    assert len(records) == 1
    assert records[0].name == "T cell"
    assert isinstance(records[0].bionty_source, lb.BiontySource)
    # bionty values and synonyms
    records = lb.CellType.from_values(["B-cell", "B cell"], "name")
    assert len(records) == 1
    assert records[0].name == "B cell"
    assert isinstance(records[0].bionty_source, lb.BiontySource)
    # all possibilities of validated values
    records = lb.CellType.from_values(
        ["T cell", "T-cell", "t cell", "B cell", "B-cell"], "name"
    )
    assert len(records) == 2
    names = [r.name for r in records]
    assert set(names) == set(["T cell", "B cell"])
    assert isinstance(records[0].bionty_source, lb.BiontySource)
    assert isinstance(records[1].bionty_source, lb.BiontySource)
    # non-validated values
    records = lb.CellType.from_values(["T cell", "mycell"], "name")
    assert len(records) == 1
    assert records[0].name == "T cell"
    assert isinstance(records[0].bionty_source, lb.BiontySource)
    assert records[0].ontology_id == "CL:0000084"
    lb.CellType.filter().all().delete()


import cv2
import numpy as np

image1 = cv2.imread("./weine.jpg")
image2 = cv2.imread("./FotoJet.jpg")

M = np.ones(image1.shape,dtype="uint8")*50
added = cv2.add(image1,image2)             # 將圖像image與M相減
subtracted = cv2.subtract(image1,image2)   # 將圖像image與M相減
multiplied = cv2.multiply(image1,image2)
divided = cv2.divide(image1,image2)
cv2.imshow("weine",image1)     #示原圖
cv2.imshow("tiger",image2)     #示原圖
cv2.imshow("add",added)            # 加運算圖
cv2.imshow("sub",subtracted)  # 減運算圖
cv2.imshow("mul",multiplied)#*
cv2.imshow("div",divided)#/

cv2.waitKey(0)
cv2.destroyAllWindows()


from collections import deque
class Solution:
    def predictPartyVictory(self, senate: str) -> str:
        r=deque()
        d=deque()
        n=len(senate)
        for i in range(n):
            if senate[i]=="R":
                r.append(i)
            else:
                d.append(i)
        j=0
        while(r and d):
            pos1=r.popleft()
            pos2=d.popleft()
            if pos1<pos2:
                r.append(n+j)
            else:
                d.append(n+j)
            j+=1
        if r:
            return 'Radiant'
        else:
            return 'Dire'


import networkx as nx
import newick

def parse_graphml_file_newick_format(filename: str, digraph=True):
    """
    Parses a graphml file and return the networkx graph. Forces each node to be in the newick.Node format.
    :param filename: str; full path of the to be parsed file
    :param digraph: Bool; is the graph a digraph
    :return: nx.Graph()
    """
    graphml_graph = nx.read_graphml(filename, node_type=newick.Node)
    if digraph:
        graphml_graph = graphml_graph.to_directed()

    for current_node in graphml_graph.nodes:
        graphml_graph.add_node(current_node, name=current_node, child_position=0)

    return graphml_graph

def parse_graphml_file(filename: str, digraph=True):
    """
    Parses a graphml file and return the networkx graph.
    :param filename: str; full path of the to be parsed file
    :param digraph: Bool; is the graph a digraph
    :return: nx.Graph()
    """
    graphml_graph = nx.read_graphml(filename)
    if digraph:
        graphml_graph = graphml_graph.to_directed()

    return graphml_graph

def parse_newick_file(filename: str, digraph=True):
    """
    Parses a newick file and returns the networkx graph.
    :param filename: str; full path of the to be parsed file
    :param digraph: Bool; is the graph a digraph
    :return: nx.Graph()
    """
    tree = newick.read(filename)

    if digraph:
        graph_newick = nx.DiGraph()
    else:
        graph_newick = nx.Graph

    none_counter = 1

    # Adding root node
    graph_newick.add_node(tree[0], child_position=0)

    while tree:
        tree_node = tree[0]
        tree_node, none_counter = rename_none_node(tree_node, none_counter)
        graph_newick, descendants, none_counter = add_newick_node_and_edge(graph_newick, tree_node, none_counter)
        tree += descendants
        tree.remove(tree_node)

    return graph_newick

def add_newick_node_and_edge(nx_graph: nx.Graph, current_node, none_counter: int):
    """
    Adding the descendants of the current node to the graph and adding the according edge.
    Rename a node if the name is None.
    :param nx_graph: the network x graph to which node and edge should be added
    :param current_node: the current node from the newick graph
    :param none_counter: an integer representing the amount of None node
    :return: (graph, descendants, none_counter)
    """
    descendants = current_node.descendants
    for child_pos in range(len(descendants)):
        descendants[child_pos], none_counter = rename_none_node(descendants[child_pos], none_counter)
        nx_graph.add_node(descendants[child_pos], child_position=child_pos)
        nx_graph.add_edge(current_node, descendants[child_pos])

    return nx_graph, descendants, none_counter

def rename_none_node(node_to_rename: newick.Node, counter):
    """
    Renaming node with no name to differ from other not named node.
    :param node_to_rename: node to be checked
    :param counter: int; counter for none nodes
    :return: (Node, int)
    """
    if node_to_rename.name is None:
        node_to_rename.name = str(node_to_rename.name) + "_" + str(counter)
        counter += 1
    return node_to_rename, counter


from pathlib import Path
from pathlib import Path
from typing import Any

import numpy as np
from mesa import DataCollector
from mesa.time import RandomActivationByType
from path_planning_lib import MAPFProblemDefinition, PIBTSolver

from ABM.core.base_model import AirportModelBase
from ABM.space_continuous.agents.agv import AAGV
from ABM.space_continuous.agents.charging_station import AChargingStation
from ABM.space_continuous.agents.chute import AChute
from ABM.space_continuous.agents.infeed_station import AInfeedStation
from ABM.space_continuous.agents.obstacle import AObstacle
from ABM.space_continuous.space.my_continues_space import MyContinuousSpace

from ABM.utils import load_image_to_np

class AirportModelContinuous(AirportModelBase):
    """
    The AirportModel class represents the model for an airport in the Agent-Based Model (ABM) simulation.
    It extends the Mesa Model class and defines the simulation's grid, schedules, data collection,
    and scenario setups.
    """

    def __init__(self, config, grid_width, grid_height, *args: Any, **kwargs: Any):
        """
        Initialize an AirportModel instance with the provided configuration, grid dimensions, and other parameters.

        Args:
            config (dict): The configuration dictionary containing parameters for the simulation.
            grid_width (int): The width of the simulation grid.
            grid_height (int): The height of the simulation grid.
            *args: Variable length argument list.
            **kwargs: Arbitrary keyword arguments.
        """
        super().__init__(config, grid_width, grid_height, *args, **kwargs)

        self.grid: MyContinuousSpace = MyContinuousSpace(x_max=self._grid_width,
                                                         y_max=self._grid_height,
                                                         torus=False)  # torus is false because we don't want the agents to wrap around the grid

        # Initialize the activation schedule by type as we want to have different activation schedules for different agents
        self.schedule = RandomActivationByType(self)
        self.collision_objs = {"dynamic": [], "static": []}
        # Setting up the scenario
        if config['scenario_type'] == 'random_scenario':
            self.setup_random_scenario(config)
        elif config['scenario_type'] == 'image_scenario':
            self.setup_scenario_from_image(config)
        else:
            raise ValueError(f'Scenario type not supported: {config["scenario_type"]}')

        # Initiate data collection
        self.datacollector = DataCollector(
            {
                # "NAME OF DATA": lambda m: m.schedule.get some kind of data (AAGV)
                "Max step time": lambda m: np.max(
                    [agv.step_time for agv in m.schedule.agents_by_type[AAGV].values()]),
                "Max server step time": lambda m: m.server_step_time,
            }
        )

        # Setting up problem definition
        edge_cost = np.ones((self._grid_width, self._grid_height), dtype=int)
        self.problem_definition = MAPFProblemDefinition(_instance_name="Test",
                                                        _seed=config['seed'],
                                                        _max_comp_time=config['path_planner']['max_comp_time'],
                                                        _max_timestep=config['path_planner']['max_timestep'],
                                                        _num_agents=self.num_units,
                                                        grid_map=np.copy(self.obstacle_map).astype(int).tolist(),
                                                        edge_cost_moving_up=edge_cost.tolist(),
                                                        edge_cost_moving_down=edge_cost.tolist(),
                                                        edge_cost_moving_left=edge_cost.tolist(),
                                                        edge_cost_moving_right=edge_cost.tolist())
        # Initiate config
        self.get_and_set_config()

        s, g, p = self.problem_definition.getConfigs()

        self.pibt_solver = PIBTSolver(self.problem_definition)

        self.replan = False
        # A warmup is performed in order to allocate memory for the path planning algorithm
        # This is done as the first path planning call is much slower than the following ones
        # for agv in self.schedule.agents_by_type[AAGV].values():
        #     agv.move()

        # found_count = 0
        # while found_count < 2:
        #     start_x, start_y = self.random.randrange(grid_width), self.random.randrange(grid_height)
        #     end_x, end_y = self.random.randrange(grid_width), self.random.randrange(grid_height)
        #     # test path planner
        #     found, path = self.a_star.findPath(start_x, start_y, end_x, end_y)

        # TODO:: Here is a usecase where we don't perform well:
        # found, path, debug_points = self.find_path(122, 180, 247, 110)
        # if found:
        #     debug_map = self.obstacle_map.copy() * 255
        #     if len(debug_points) > 0:
        #         for y, x in debug_points:
        #             debug_map[y, x] = 100
        #     for y, x in path:
        #         debug_map[y, x] = 170
        #     plt.imshow(debug_map.astype(np.uint8), origin='lower')
        #     plt.show()

    def set_config(self, start_pos, goal_pos, priority):
        """
        Set the configuration for the problem definition with the given agent positions, goals, and priorities.

        Args:
            start_pos (list): A list of agent start positions.
            goal_pos (list): A list of agent goal positions.
            priority (list): A list of agent priorities.
        """
        # print("START POSITIONS:")
        #
        # for x, y in start_pos:
        #     print("{", end="")
        #     print(f"{x}, {y}", end="")
        #     print("},", end="\n")
        #
        # print("GOAL POSITIONS:")
        #
        # for x, y in goal_pos:
        #     print("{", end="")
        #     print(f"{x}, {y}", end="")
        #     print("},", end="\n")
        #
        # print("PRIORITIES POSITIONS:")
        # print("{", end="")
        # for p in priority:
        #     print(f"{p},", end="")
        # print("}", end="\n")
        self.problem_definition.setConfig(start_pos, goal_pos, priority)

    def get_and_set_config(self):
        """
        Retrieve and set the configuration for the problem definition using the current agent positions, goals, and priorities.
        """
        start_pos, goal_pos, priority = self.get_config()
        self.set_config(start_pos, goal_pos, priority)

    def find_paths(self):
        """
        Find paths for agents using the PIBT solver.
        """
        # Initiate config
        self.get_and_set_config()

        self.pibt_solver = PIBTSolver(self.problem_definition)

        self.pibt_solver.solve()
        if self.pibt_solver.succeed():
            solution = self.pibt_solver.getSolution()
            for agent_index, agv in enumerate(self.agv_dict.values()):
                path = solution.getPathToGoalXY(agent_index)
                # print(f"Agent {agent_index} path: {path}")
                # Path contains the start position, so we remove it
                agv.path = path[1:].tolist()
                debug = 0
        else:
            print("No solution found (in time)!")
            solution = self.pibt_solver.getSolution()
            for agent_index, agv in enumerate(self.agv_dict.values()):
                path = solution.getPathXY(agent_index)
                # print(f"Agent {agent_index} path: {path}")
                # Path contains the start position, so we remove it
                agv.path = path[1:].tolist()
                if list(agv.goal) != path[-1].tolist():
                    print(f"Agent {agent_index} goal {agv.goal} not at end of path!")
                    if list(agv.goal) not in path:
                        print(f"Agent {agent_index} goal {agv.goal} not in path!")
                debug = 0
        s, g, p = self.problem_definition.getConfigs()
        self.print(self.pibt_solver.printResult())

    def server_action(self):
        if self.replan:
            self.replan = False
            self.find_paths()

    def setup_random_scenario(self, config):
        """
        Set up a random scenario for the simulation based on the given configuration.

        Args:
            config (dict): The configuration dictionary containing parameters for the random scenario.
        """
        scenario_config = config['random_scenario']
        self.num_units = scenario_config['num_agvs']
        id_counter = 0

        # Create obstacles
        empty_spaces = [(x, y) for x in range(self._grid_width) for y in range(self._grid_height)]
        empty_spaces = self.random.shuffle(empty_spaces)
        for _ in range(scenario_config['num_obstacles']):
            o = AObstacle(id_counter, self, config["obstacle_params"])
            coord = empty_spaces[id_counter]
            self.grid.place_agent(o, coord)
            self.obstacle_map[coord.x, coord.y] = 1
            id_counter += 1

        # Create charging stations
        for _ in range(scenario_config['num_charging_stations']):
            cs = AChargingStation(id_counter, self, config['charging_station_params'])
            self.schedule.add(cs)
            coord = empty_spaces[id_counter]
            self.grid.place_agent(cs, coord)
            self.charging_stations_dict[(coord.x, coord.y)] = cs
            id_counter += 1

        # Create infeed stations
        for _ in range(scenario_config['num_infeed']):
            infeed = AInfeedStation(id_counter, self, config["infeed_params"])
            self.schedule.add(infeed)
            coord = empty_spaces[id_counter]
            self.grid.place_agent(infeed, coord)
            self.infeed_dict[(coord.x, coord.y)] = infeed
            id_counter += 1

        # Create chutes
        for _ in range(scenario_config['num_chutes']):
            c = AChute(id_counter, self, config["chute_params"])
            self.schedule.add(c)
            coord = empty_spaces[id_counter]
            self.grid.place_agent(c, coord)
            self.chutes_dict[(coord.x, coord.y)] = c
            id_counter += 1

        # Create AGV agents
        for _ in range(scenario_config['num_agvs']):
            agv = AAGV(id_counter, self, config['agv_params'])
            self.schedule.add(agv)
            coord = empty_spaces[id_counter]
            self.grid.place_agent(agv, coord)
            self.agv_dict[id_counter] = agv
            id_counter += 1

    def setup_scenario_from_image(self, config):
        """
        Set up a scenario for the simulation using images as input for the configuration.

        Args:
            config (dict): The configuration dictionary containing parameters for the scenario.
        """
        id_counter = 0
        # Load image
        scenario_config = config['image_scenario']
        self.num_units = scenario_config['num_agvs']

        # 1. First check for images that exists and insert in map
        ###################### OBSTACLES FROM IMAGE ###########################
        empty_space_map = np.ones((self._grid_width, self._grid_height), dtype=np.uint8)
        if Path(scenario_config['img_dir_path'], scenario_config['obstacle_img_name']).exists():
            # Code to process obstacles from image
            self.print('Inserting obstacles into map')
            obstacle_img = load_image_to_np(Path(scenario_config['img_dir_path'], scenario_config['obstacle_img_name']),
                                            convert=None, remove_alpha=True, rotate=3)
            if obstacle_img.shape[:2] != (self._grid_width, self._grid_height):
                raise ValueError(f'Obstacle map does not have the same size as the grid.\n'
                                 f'Please go into the config and change the grid size to match the obstacle map size.\n'
                                 f'current grid size: h:{self._grid_height} x w:{self._grid_width}\n'
                                 f'obstacle map size: h:{obstacle_img.shape[0]} x w:{obstacle_img.shape[1]}')

            empty_space_map[np.where(
                np.all(obstacle_img[:, :] != scenario_config["floor_color"],
                       2))] = 0  # Remove every space that is not floor

            obstacles = np.where(np.all(obstacle_img[:, :] == scenario_config["obstacle_color"], 2))
            self.obstacle_map[obstacles] = 1  # Find all obstacles
            outside = np.where(np.all(obstacle_img[:, :] == scenario_config["outside_color"], 2))
            self.obstacle_map[outside] = 1  # Find all obstacles
            obstacles = np.transpose(obstacles)

            # Code to place an obstacle in the grid
            for x, y in obstacles:
                o = AObstacle(id_counter, self, config["obstacle_params"])
                self.grid.place_agent(o, (x, y))  # Sets the position of the agent
                id_counter += 1

        ###################### CHARGING STATIONS FROM IMAGE ###########################
        if Path(scenario_config['img_dir_path'], scenario_config['charging_station_img_name']).exists():
            # Code to process charging stations from image
            self.print('Inserting charging stations into map')
            charging_station_img = load_image_to_np(
                Path(scenario_config['img_dir_path'], scenario_config['charging_station_img_name']),
                convert=None, remove_alpha=True, rotate=3)
            if charging_station_img.shape[:2] != (self._grid_width, self._grid_height):
                raise ValueError(f'Charging station map does not have the same size as the grid.\n'
                                 f'Please go into the config and change the grid size to match the charging station map size.\n'
                                 f'current grid size: h:{self._grid_height} x w:{self._grid_width}\n'
                                 f'charging station map size: h:{charging_station_img.shape[0]} x w:{charging_station_img.shape[1]}')

            charging_stations = np.where(
                np.all(charging_station_img[:, :] == scenario_config["charging_station_color"], 2))
            charging_stations = np.transpose(charging_stations)

            # Code to place a charging station in the grid
            for x, y in charging_stations:
                cs = AChargingStation(id_counter, self, config['charging_station_params'])
                self.schedule.add(cs)
                self.charging_stations_dict[(x, y)] = cs
                self.grid.place_agent(cs, (x, y))
                id_counter += 1

        ###################### INFEED FROM IMAGE ###########################
        if Path(scenario_config['img_dir_path'], scenario_config['infeed_img_name']).exists():
            # Code to process infeeds from image
            self.print('Inserting infeeds into map')
            infeed_img = load_image_to_np(Path(scenario_config['img_dir_path'], scenario_config['infeed_img_name']),
                                          convert=None, remove_alpha=True, rotate=3)
            if infeed_img.shape[:2] != (self._grid_width, self._grid_height):
                raise ValueError(f'Infeed map does not have the same size as the grid.\n'
                                 f'Please go into the config and change the grid size to match the infeed map size.\n'
                                 f'current grid size: h:{self._grid_height} x w:{self._grid_width}\n'
                                 f'infeed map size: h:{infeed_img.shape[0]} x w:{infeed_img.shape[1]}')

            infeeds = np.where(np.all(infeed_img[:, :] == scenario_config["infeed_color"], 2))
            infeeds = np.transpose(infeeds)
            # Code to place an infeed in the grid
            for x, y in infeeds:
                i = AInfeedStation(id_counter, self, config["infeed_params"])
                self.infeed_dict[(x, y)] = i
                self.grid.place_agent(i, (x, y))
                id_counter += 1

        ###################### CHUTE FROM IMAGE ###########################
        if Path(scenario_config['img_dir_path'], scenario_config['chute_img_name']).exists():
            # Code to process chutes from image
            self.print('Inserting chutes into map')
            chute_img = load_image_to_np(Path(scenario_config['img_dir_path'], scenario_config['chute_img_name']),
                                         convert=None, remove_alpha=True, rotate=3)
            if chute_img.shape[:2] != (self._grid_width, self._grid_height):
                raise ValueError(f'Chute map does not have the same size as the grid.\n'
                                 f'Please go into the config and change the grid size to match the chute map size.\n'
                                 f'current grid size: h:{self._grid_height} x w:{self._grid_width}\n'
                                 f'chute map size: h:{chute_img.shape[0]} x w:{chute_img.shape[1]}')

            chutes = np.where(np.all(chute_img[:, :] == scenario_config["chute_color"], 2))
            chutes = np.transpose(chutes)
            # Code to place a chute in the grid
            for x, y in chutes:
                c = AChute(id_counter, self, config["chute_params"])
                self.schedule.add(c)
                self.chutes_dict[(x, y)] = c
                self.grid.place_agent(c, (x, y))
                id_counter += 1

        ###################### AGVS FROM IMAGE ###########################
        if Path(scenario_config['img_dir_path'], scenario_config['agv_img_name']).exists():
            self.print('Inserting agvs into map')
            agv_img = load_image_to_np(Path(scenario_config['img_dir_path'], scenario_config['agv_img_name']),
                                       convert=None, remove_alpha=True, rotate=3)
            if agv_img.shape[:2] != (self._grid_width, self._grid_height):
                raise ValueError(f'AGV map does not have the same size as the grid.\n'
                                 f'Please go into the config and change the grid size to match the AGV map size.\n'
                                 f'current grid size: h:{self._grid_height} x w:{self._grid_width}\n'
                                 f'AGV map size: h:{agv_img.shape[0]} x w:{agv_img.shape[1]}')

            agvs = np.where(np.all(agv_img[:, :] == scenario_config["agv_color"], 2))
            agvs = np.transpose(agvs)
            for x, y in agvs:
                a = AAGV(id_counter, self, config['agv_params'])
                self.schedule.add(a)
                self.grid.place_agent(a, (x, y))
                self.agv_dict[id_counter] = a
                id_counter += 1

        # 2. Then check for images that does not exists and insert random in map
        list_of_empty_coord = np.transpose(np.where(empty_space_map == 1))
        np.random.shuffle(list_of_empty_coord)
        self.empty_coords = list_of_empty_coord
        index = 0
        ###################### OBSTACLES RANDOM ###########################
        if not Path(scenario_config['img_dir_path'], scenario_config['obstacle_img_name']).exists():
            self.print(
                f'Obstacle map not found.\nInserting {scenario_config["num_obstacles"]} random obstacles into map')
            # Create obstacles
            for _ in range(scenario_config['num_obstacles']):
                x, y = list_of_empty_coord[index]
                o = AObstacle(id_counter, self, config["obstacle_params"])
                self.grid.place_agent(o, (x, y))
                self.obstacle_map[x, y] = 1
                id_counter += 1
                index += 1

        ###################### CHARGING STATION RANDOM ###########################
        if not Path(scenario_config['img_dir_path'], scenario_config['charging_station_img_name']).exists():
            self.print(
                f'Charging station map not found.\nInserting {scenario_config["num_charging_stations"]} random charging stations into map')
            for _ in range(scenario_config['num_charging_stations']):
                x, y = list_of_empty_coord[index]
                cs = AChargingStation(id_counter, self, config['charging_station_params'])
                self.charging_stations_dict[(x, y)] = cs
                self.schedule.add(cs)
                self.grid.place_agent(cs, (x, y))
                id_counter += 1
                index += 1

        ###################### INFEED RANDOM ###########################
        if not Path(scenario_config['img_dir_path'], scenario_config['infeed_img_name']).exists():
            self.print(f'Infeed map not found.\nInserting {scenario_config["num_infeeds"]} random infeeds into map')
            for _ in range(scenario_config['num_infeeds']):
                x, y = list_of_empty_coord[index]
                i = AInfeedStation(id_counter, self, config["infeed_params"])
                self.infeed_dict[(x, y)] = i
                self.schedule.add(i)
                self.grid.place_agent(i, (x, y))
                id_counter += 1
                index += 1

        ###################### CHUTE RANDOM ###########################
        if not Path(scenario_config['img_dir_path'], scenario_config['chute_img_name']).exists():
            self.print(f'Chute map not found.\nInserting {scenario_config["num_chutes"]} random chutes into map')
            for _ in range(scenario_config['num_chutes']):
                x, y = list_of_empty_coord[index]
                c = AChute(id_counter, self, config["chute_params"])
                self.chutes_dict[(x, y)] = c
                self.schedule.add(c)
                self.grid.place_agent(c, (x, y))
                id_counter += 1
                index += 1

        ###################### AGVS RANDOM ###########################
        if not Path(scenario_config['img_dir_path'], scenario_config['agv_img_name']).exists():
            self.print(f'AGV map not found.\nInserting {scenario_config["num_agvs"]} random AGVs into map')
            for _ in range(scenario_config['num_agvs']):
                x, y = list_of_empty_coord[index]
                a = AAGV(id_counter, self, config['agv_params'])
                self.schedule.add(a)
                self.grid.place_agent(a, (x, y))
                self.agv_dict[id_counter] = a
                id_counter += 1
                index += 1


##################################################
#
# Python Solar Panel Characterization
# -- analyzing Arduino .csv data
#
# by Joshua Hrisko | Maker Portal LLC (c) 2021
#
##################################################
#
#
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import csv

##################################################
# Data parsing from .csv
##################################################
#
filename = "SOLARLOG.CSV" # csv filename

data_array = [] # for saving variables
with open(filename,"r") as csvfile:
    csvreader = csv.reader(csvfile,delimiter=",")
    header = next(csvreader) # print header
    for row in csvreader:
        data_array.append(row) # save data to variable array

start_pt = 0 # start point
end_pt   = int(0.8*(np.shape(data_array)[0])) # end point
data_array = data_array[start_pt:end_pt] # select the portion of data with valid data
time_vector = np.array([float(ii[0]) for ii in data_array]) # time in [ms] since start of program
voltage_V   = np.array([float(ii[1]) for ii in data_array]) # voltage in [V]
current_mA  = np.array([float(ii[2]) for ii in data_array]) # current in [mA]

##################################################
# Data Plotting of Voltage [V] vs Current [mA]
##################################################
#
plt.style.use('ggplot') # style plot
fig = plt.figure(figsize=(15,10))
ax = fig.add_subplot(211)
ax.tick_params(labelsize=14)

ax.plot(voltage_V,current_mA,linewidth=3.5,color=plt.cm.tab20c(0),
        label='54mm x 54mm Solar Panel') # plot V vs I
ax.set_xlabel(r'$V$, Voltage [V]',fontsize=16)
ax.set_ylabel(r'$I$, Current [mA]',fontsize=16)

ax2 = fig.add_subplot(212)
ax2.tick_params(labelsize=14)

power_W = voltage_V*(current_mA/1000.0) # power output [W]
max_P_volt = voltage_V[np.argmax(power_W)] # max power Voltage [V]
maX_P_curr = current_mA[np.argmax(power_W)] # max power current [mA]
max_power_W = np.max(power_W) # max power [W]

ax.plot(np.repeat(np.max(voltage_V),len(current_mA)),current_mA,
        linestyle='dotted',color=plt.cm.tab20c(4)) # open-circuit voltage
ax.plot(voltage_V,np.repeat(np.max(current_mA),len(voltage_V)),
        linestyle='--',color=plt.cm.tab20c(1)) # short-circuit current

ax2.plot(voltage_V,power_W,linewidth=3.5,color=plt.cm.tab20c(8),
         label='54mm x 54mm Solar Panel') # plot V vs I

ax2.plot(np.repeat(max_P_volt,len(power_W)),power_W,
         linestyle='dotted',color=plt.cm.tab20c(12)) # plot max voltage
ax2.plot(voltage_V,np.repeat(max_power_W,len(voltage_V)),
         color=plt.cm.tab20c(9),linestyle='--') # plot max current

ax2.set_xlabel(r'$V$, Voltage [V]',fontsize=16)
ax2.set_ylabel(r'$P$, Power [W]',fontsize=16)

ax.legend(fontsize=14)
ax2.legend(fontsize=14)

print("Max Power: {0:2.2f} W\nMax Power Voltage: {1:2.2f} V\nMax Power Current: {2:2.2f} mA".format(
    max_power_W,max_P_volt,maX_P_curr)+"\nOpen-Circuit Voltage: {0:2.2f} V".format(np.max(voltage_V))+
      "\nShort-Circuit Current: {0:2.2f} mA".format(np.max(current_mA)))

plt.savefig('solar_panel_char_output.png',dpi=300,bbox_inches='tight',facecolor='#FFFFFF')
plt.show() # show plot


# -*- coding=utf-8 -*-
# author: zhihuaye@gmail.com

"""
1. try to get flags
https://www.countries-ofthe-world.com/flags-of-the-world.html
2. find the sample url
https://www.countries-ofthe-world.com/flags-normal/flag-of-China.png
3. add UA to get file
"""

import sys
import os
from pyquery import PyQuery as pq
import requests
import time
import functools
import errno
from timeit import default_timer as timer
import shutil
from concurrent import futures

FLAG_BASEURL='https://www.countries-ofthe-world.com/'
INDEX_URI='flags-of-asia.html'
FLAGPAGE='./flags.html'
FLAGDIR='./flags'
Headers = {
    'user-agent': "Mozilla"
}
PY3k = sys.version_info >= (3,)

def benchmark(func):
    """
    decorator to print the time a function executes.
    :param func:
    :return:
    """
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        start = timer()
        res = func(*args, **kwargs)
        print("{0} takes {1:.7f} seconds".format(func.__name__, timer() - start))
        return res
    return wrapper

@benchmark
def hello():
    print("start to run")

@benchmark
def mkdirp(dirname):
    try:
        os.makedirs(dirname)
    except OSError as error:
        if error.errno == errno.EEXIST and os.path.isdir(dirname):
            pass
        else:
            raise

@benchmark
def getflagpage():
    rsp = requests.get(FLAG_BASEURL + INDEX_URI, headers=Headers)
    rsp.raise_for_status()
    if PY3k:
        with open(FLAGPAGE, 'w+', encoding='utf-8') as handle:
            handle.write(rsp.text)   #works for py 3.x
            return rsp.text
    else:
        with open(FLAGPAGE, 'w+') as handle:
            htmlstr = rsp.text.encode('utf-8')
            handle.write(htmlstr) # works for py 2.7
            return htmlstr

@benchmark
def getoneflag(imgelement):
    imgurl = FLAG_BASEURL + imgelement.get('src')
    #flags-normal/flag-of-Afghanistan.png
    imgname = FLAGDIR + '/' + imgelement.get('src').split('/')[1]
    rsp = requests.get(imgurl, headers=Headers)
    rsp.raise_for_status()
    # open as binary and get content
    with open(imgname, 'wb') as handle:
        handle.write(rsp.content)

@benchmark
def getcountryflags(htmlstr):
    #avoid file open in pq, unicode is not well supported.
    page = pq(htmlstr)
    #listcomp, lxml element
    #imglist = [ i.get('src') for i in page('table').find('img')]
    #print(imglist)
    #use items() is better, so imgele will be PyQuery obj instead of lxml obj.
    for imgelement in page('table').find('img'):
        getoneflag(imgelement)

@benchmark
def futuresgetflags(htmlstr):
    page = pq(htmlstr)
    workers = 20
    imgelementlist = [element for element in page('table').find('img')]
    with futures.ThreadPoolExecutor(workers) as executor:
        executor.map(getoneflag, imgelementlist)

if __name__ == '__main__':
    if os.path.exists(FLAGDIR):
        shutil.rmtree(FLAGDIR)
    mkdirp(FLAGDIR)
    htmlstr=getflagpage()
    getcountryflags(htmlstr)
    #futuresgetflags(htmlstr)


import pushmanager.core.util
import tornado.gen
import tornado.web
from pushmanager.core.requesthandler import RequestHandler

class PushByRequestServlet(RequestHandler):

    @tornado.web.asynchronous
    @tornado.gen.engine
    def get(self):
        requestid = pushmanager.core.util.get_int_arg(self.request, 'id')
        response = yield tornado.gen.Task(
                        self.async_api_call,
                        "pushbyrequest",
                        {"id": requestid}
                    )

        push = self.get_api_results(response)
        if push:
            self.redirect('/push?id=%s' % push['id'])


from django.urls import path
from .views import FasesList, FasesUpdate, FasesDelete, FasesCreate

urlpatterns = [
    path('list', FasesList.as_view(), name='list_fases'),
    path('update/<int:pk>/', FasesUpdate.as_view(), name='update_fases'),
    path('delete/<int:pk>/', FasesDelete.as_view(), name='delete_fases'),
    path('create', FasesCreate.as_view(), name='create_fases'),
]


import torch

from ..optimizers.mirror_descent_optimizer import MirrorDescentOptimizer

###############################################################################
#
# This file implements logging of certain metrics associated to well-specified
# linear regression datasets. In particular, the ones generated using the
# rprml.data.synthetic_data.generate_synthetic_regression_dataset function.
# The model class is assumed to be of type rprml.models.LinearModel.
#
###############################################################################

def log_linear_model_complexity_parameters(simulation):
    """ Registers logging various complexity parameters, such as in-sample
    prediction-error, model parameters at iteration t,
    Bregman divergence to the true minimizer (if applied to a mirror descent
    simulation), etc. """

    X = simulation.train_dataset.X
    y = simulation.train_dataset.y
    w_star = simulation.train_dataset.w_star.view(-1, 1)
    loss_w_star = torch.sum((X.matmul(w_star) - y)**2) / X.shape[0]
    covariance_matrix = None

    if hasattr(simulation.train_dataset, 'covariance_matrix'):
        covariance_matrix = simulation.train_dataset.covariance_matrix
    if covariance_matrix is None:
        # Set covariance_matrix to identity
        covariance_matrix = torch.diag(
            torch.ones(X.shape[1], device=X.device, dtype=X.dtype))
    else:
        if not isinstance(covariance_matrix, torch.Tensor):
            covariance_matrix = torch.tensor(
                covariance_matrix, device=X.device, dtype=X.dtype)

    def _log_wt(engine, executor):
        return simulation.model.get_w().detach().cpu().numpy().flatten()

    def _log_bregman_divergence(engine, executor):
        with torch.no_grad():
            if not isinstance(simulation.optimizer, MirrorDescentOptimizer):
                return -1  # This is not a mirror descent simulation.
            w_t = simulation.model.get_w()
            mirror_map = simulation.optimizer.mirror_map
            return mirror_map.compute_bregman_divergence(w_star, w_t)

    def _log_true_excess_risk(engine, executor):
        with torch.no_grad():
            w_t = simulation.model.get_w()
            delta_t = w_t - w_star
            excess_risk_t = delta_t.t() @ covariance_matrix @ delta_t
            return excess_risk_t.item()

    def _log_rt(engine, executor):
        # Logs in sample prediction.
        with torch.no_grad():
            w_t = simulation.model.get_w()
            r_t = torch.sum(X.matmul(w_t - w_star)**2) / X.shape[0]
            return r_t.item()

    def _log_delta_t(engine, executor):
        with torch.no_grad():
            w_t = simulation.model.get_w()
            loss_w_t = torch.sum((X.matmul(w_t) - y)**2) / X.shape[0]
            return (loss_w_t - loss_w_star).item()

    simulation.executor.register_not_printable_metric(
        _log_wt, 'w_t')
    # The excess risk metric should be ignored for fixed-design simulations.
    # In such a case, covariance is assumed to be identity and excess risk
    # outputs the l2 squared parameter estimation error.
    simulation.executor.register_not_printable_metric(
        _log_true_excess_risk, 'excess_risk_t')
    simulation.executor.register_not_printable_metric(
        _log_rt, 'r_t')
    simulation.executor.register_not_printable_metric(
        _log_delta_t, 'delta_t')
    simulation.executor.register_not_printable_metric(
        _log_bregman_divergence, 'bregman_t')


from gensim.test.utils import common_corpus, common_dictionary, datapath, get_tmpfile, common_texts
from gensim.models import HdpModel
from gensim.corpora import MalletCorpus
from gensim.corpora import Dictionary
from gensim import corpora
from six import iteritems # collect statistics about all tokens
import numpy as np
import pandas as pd
import sys

# import sets of words to remove
from gensim.parsing.preprocessing import remove_stopwords, STOPWORDS
from OSTK_STOPWORDS import DESCRIPTIVE_WORDS

def create_dict(corpus_file_path):
    dictionary = corpora.Dictionary(line.lower().split() for line in open(corpus_file_path))
    return dictionary

def get_filter_ids(filter_word_set, dictionary):
    filter_set_ids = [dictionary.token2id[filter_word] for filter_word in filter_word_set
                      if filter_word in dictionary.token2id]
    return filter_set_ids

class hHdpModel(object):
    def __init__(self, corpus_file_path):
        pass

class MyCorpus(object):
    def __init__(self, corpus_file_path):
        self.corpusFilepPath = corpus_file_path
        self.dict = create_dict(corpus_file_path)
        self.filteredTokens = set({}) # This can be used later when creating a sub-hpd
        self.hdpModels = [] # This is where each model will live for the current corpus

    def __iter__(self):
        for line in open(self.corpusFilepPath):
            yield self.dict.doc2bow(line.lower().split()) # assume there's one document per line, tokens separated by whitespace

    def __len__(self):
        return len(list(iter(self)))

    def remove_words(self, *filter_word_sets):
        for filter_word_set in filter_word_sets:
            if not isinstance(filter_word_set, (set, frozenset)): # type check
                sys.exit("Filter word set must by of type set or frozenset")
            filter_set_ids = get_filter_ids(filter_word_set, dictionary = self.dict)
            self.filteredTokens.add(filter_word_set) # Record which tokens were removed from the current corpus instance
            self.dict.filter_tokens(filter_set_ids) # remove the tokens by the id
            self.dict.compactify() # remove the gaps in the dictionary left from the filter step

    def get_new_doc_id(self, doc):
        doc_id = self.dict.doc2bow(doc.lower().split())
        return doc_id

    def run_hdp(self, modelId, **kwargs):
        print(kwargs)
        hdpModel = HdpModel(self, self.dict, kwargs)
        hdpData = {modelId:
                   {'model':hdpModel,
                    'args':kwargs}}

        self.hdpModels.append(hdpData)

    def get_new_doc_topics(self, doc, *modelIds):
        for modelId in modelIds:
            doc_ids = []
            doc_ids.append(self.hdpModels[modelid]['model'][self.get_new_doc_id(doc)])
            return doc_ids

def main():
    # Create the corpus
    hundoGrandoCorpus = MyCorpus("../data/corpus_clean100000.txt")
    # Remove words
    hundoGrandoCorpus.remove_words(STOPWORDS, DESCRIPTIVE_WORDS)
    hundoGrandoCorpus.dict.filter_extremes(no_below=1)
    hundoGrandoCorpus.dict.compactify()

    hundoGrandoCorpus.run_hdp(modelId='testMod',chunksize=1000, max_time=100, K = 4, T = 10000, alpha=10, gamma=1, scale=1, eta=0.5)

if __name__ == '__main__':
    main()


import string
from collections import defaultdict, Counter

# Average letter frequencies found in English (from Wikipedia)
letter_frequencies = [
  ('A', 0.08167),
  ('B', 0.01492),
  ('C', 0.02782),
  ('D', 0.04253),
  ('E', 0.12702),
  ('F', 0.02228),
  ('G', 0.02015),
  ('H', 0.06094),
  ('I', 0.06966),
  ('J', 0.00153),
  ('K', 0.00772),
  ('L', 0.04025),
  ('M', 0.02406),
  ('N', 0.06749),
  ('O', 0.07507),
  ('P', 0.01929),
  ('Q', 0.00095),
  ('R', 0.05987),
  ('S', 0.06327),
  ('T', 0.09056),
  ('U', 0.02758),
  ('V', 0.00978),
  ('W', 0.02360),
  ('X', 0.00150),
  ('Y', 0.01974),
  ('Z', 0.00074),
]

def all_substrings(text, size):
  '''Returns a list of all substrings with length of size in text'''
  length = len(text)
  return [text[i:i+size] for i in range(length) if i+size<=length]

def rotate_letter(letter, shift):
  '''Rotates the letter clockwise by the given shift value.'''
  rotated_letter = ord(letter) + shift
  if rotated_letter < ord('A'):
    rotated_letter = ord('Z') - (ord('A') - rotated_letter) + 1
  if rotated_letter > ord('Z'):
    rotated_letter = ord('A') + (rotated_letter - ord('Z')) - 1
  return chr(rotated_letter)

def calculate_distances(string_list):
  '''
  Takes in a list of strings and tells how far away matching elements are from
  one another. ['a','a'] has a distance of 1.
  Returns list((string, distance)).
  '''
  distances = []
  length = len(string_list)
  for i in range(length):
    for x in range(length):
      if string_list[i] == string_list[x] and i != x and i-x > 0:
        distances.append((string_list[i], i-x))
  return distances

def mod_counts(numbers, max_mod):
  '''
  Takes in a list(int). Calculates how many of the int%x==0 for 1<x<max_mod .
  Returns a defaultdict{mod, count} showing how many of the int%mod==0 .
  '''
  counts = defaultdict(int)
  for i in range(2, max_mod):
    for num in numbers:
      if num%i == 0:
        counts[i] += 1
  return counts

def find_shift_value(shift_column):
  '''
  Takes in a list of letters. Finds the most common occurrances.
  Uses these common occurrances to estimate how much the text was shifted.
  Returns a probable integer shift value.
  '''
  count = Counter(shift_column)
  # Ensure counts of 0 appear
  unfound_letters = [l for l in string.uppercase if l not in count]
  for l in unfound_letters: count[l] = 0
  total_letters = 0.0
  for l,c in count.most_common(): total_letters += c
  # Try to find the smallest difference between actual and expected frequencies
  differences = defaultdict(float)
  # Try shifting through every combination
  for r in range(len(letter_frequencies)):
    for l,f in letter_frequencies:
      rotated = rotate_letter(l, r)
      differences[r] += abs(f - count[rotated]/total_letters)
  # The smallest difference is most likely the shift value
  smallest = 0
  for s,d in differences.iteritems():
    if differences[s] < differences[smallest]:
      smallest = s
  return smallest

def vigenere_shift(text, shift_values):
  '''Rotates text by the shift_values given. Returns shifted value.'''
  key_length = len(shift_values)
  text = text.upper()
  shifted_letters = []
  for i in range(len(text)):
    rotated_letter = rotate_letter(text[i], -shift_values[i%key_length])
    shifted_letters.append(rotated_letter)
  return ''.join(shifted_letters)

def main():
  '''Main method'''
  cipher_text = raw_input('Enter the cipher text to decrypt: \n')
  print('Calculating...\n')
  substrings = all_substrings(cipher_text, 3)
  distances = calculate_distances(substrings)
  counts = mod_counts(zip(*distances)[1], 20)
  counts = [x for x in counts.iteritems()]
  counts.sort(key=lambda x: -x[1])
  # counts[x][0] should now contain key sizes from most to least probable
  key_size = counts[0][0]
  # Split letters by the key length and find the most common occurrences
  shift_values = []
  for i in range(key_size):
    shift_column = [cipher_text[x] for x in range(len(cipher_text)) if x%key_size==i]
    shift_values.append(find_shift_value(shift_column))
  decrypted_text = vigenere_shift(cipher_text, shift_values)
  print('\nExpected key: ')
  print(''.join([rotate_letter('A', c) for c in shift_values]))
  print('\nAlternative key (different starting shift sometimes encountered): ')
  print(''.join([rotate_letter('Z', c) for c in shift_values]))
  print('\nDecrypted text: ')
  print(decrypted_text)

# Call main method
if __name__ == '__main__':
  main()