import connect_db as conn
import time
import os
import send_email as se
import pandas as pd
import numpy as np
import json
import re
from openpyxl.utils import get_column_letter, column_index_from_string
from openpyxl import load_workbook
from xlrd import open_workbook
from itertools import groupby

upload_task = {}
os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.UTF8'

# None to ''
def xstr(s):
    return '' if s is None else str(s).strip()

# Upload progress
def get_progress(user_key):
    global upload_task

    if user_key in upload_task:
        return upload_task[user_key]
    return 0

# Check username and password
def check_account(username, password):
    if not (username and password):
        print('用户名或密码为空')
        return False

    sql = "select 用户号 from erpbase..tblOperatorData where 用户号 = '%s' and 密码= '%s' " % (
        username, password)
    results = conn.MssConn.query(sql)
    if not results:
        print('查询不到数据')
        return False
    return True

# Get customer list
def get_cust_code_list():
    json_data = []

    sql = "SELECT DISTINCT CUSTOMERSHORTNAME FROM TBLTSVNPIPRODUCT ORDER BY CUSTOMERSHORTNAME "
    results = conn.OracleConn.query(sql)
    for row in results:
        result = {}
        result['value'] = xstr(row[0])
        result['label'] = xstr(row[0])

        json_data.append(result)
    return json_data

# Get customer po template
def get_po_template(cust_code):
    if not cust_code:
        print('客户代码不可为空')
        return []

    json_data = []
    sql = "SELECT CUST_CODE,TEMPLATE_FILE ,TEMPLATE_PIC ,KEY_LIST ,FILE_LEVEL,FILE_URL,ACCEPT,TEMPLATE_ID FROM CMP_CUST_PO_TEMPLATE WHERE CUST_CODE  = '%s'" % (
        cust_code)
    results = conn.OracleConn.query(sql)
    for row in results:
        result = {}
        result['file_name'] = xstr(row[1])
        result['img_url'] = xstr(row[2])
        result['file_key'] = xstr(row[3])
        result['level'] = xstr(row[4])
        result['file_url'] = xstr(row[5])
        result['accept'] = xstr(row[6])
        result['file_id'] = xstr(row[7])
        result['show_progress_flag'] = False
        result['show_filelist_flag'] = False
        result['load_progress'] = 0

        json_data.append(result)
    return json_data

# Upload po file
def upload_po_file(f, po_header):
    global upload_task
    upload_task[po_header['file_id']] = 0

    if not f:
        print('文件不存在')
        po_header['err_desc'] = '上传的文件不存在'
        return False

    file_dir = os.path.join(os.getcwd(), 'uploads/po/' +
                            po_header['po_type']+'/'+po_header['cust_code'])

    if not os.path.exists(file_dir):
        os.makedirs(file_dir)

    file_path = os.path.join(file_dir, f.filename)
    f.save(file_path)

    sql = "select PO_ITEM_SEQ.nextval from dual"
    po_header['upload_id'] = conn.OracleConn.query(sql)[0][0]

    if not parse_po_file(file_path, po_header):
        return False

    ret = get_upload_data(po_header['upload_id'])
    mail_attachment = []
    mail_attachment.append(file_path)
    mail_attachment.append(os.path.join(os.getcwd(), '已上传订单.xlsx'))
    send_mail(ret, po_header, mail_attachment)

    upload_task[po_header['file_id']] = 100

    return ret

# Sent mail
def send_mail(ret_data, po_header, mail_attachment):
    mail_keyid = po_header['upload_id']

    mail_body = get_mail_body(
        po_header['user_name'], mail_keyid, po_header['mail_tip'], ret_data)

    if po_header['user_name'] == '07885':
        sql = "select recv_user_to from erp_email_recv where email_type = 'WO_UPLOAD_RECV_MY' "
    else:
        sql = "select recv_user_to from erp_email_recv where email_type = 'WO_UPLOAD_RECV' "

    mail_recv = conn.OracleConn.query(sql)[0][0].split(',')

    sql = "select recv_user_cc from erp_email_recv where email_type = 'WO_UPLOAD_RECV' "
    mail_recv_cc = conn.OracleConn.query(sql)[0][0].split(',')

    mail_title = '已上传客户' + po_header['cust_code'] + '的订单, 明细请查看附件'

    se.send_email(mail_title, mail_body, mail_attachment,
                  mail_recv, mail_recv_cc)

# Get mail body
def get_mail_body(createBy, uploadid, addContent, json_data):
    # Get total data
    content = f'''

内勤人员：{createBy} 上传WO

{addContent}

以下为上传WO的汇总信息

                  序号	保税	客户代码	客户PO	客户机种	客户Fab机种	厂内机种	晶圆料号
                  客户LOTID	片数	DIES	上传人员工号	上传时间

          '''

    for row in json_data['total_data']:
        content = content + f'''
{row['id']}	{row['banded']}	{row['cust_code']}	{row['po_id']}
        {row['cust_device']}	{row['fab_device']}	{row['ht_pn']}	{row['wafer_pn']}
        {row['lot_id']}	{row['wafer_qty']}	{row['die_qty']}	{row['upload_by']}	{row['upload_date']}

        '''

    content = content + '
'

    # Get detail data
    content = content + f'''
以下为明细数据

                  序号	客户PO	客户机种	客户Fab机种	厂内机种
                  客户LOTID	waferID	grossdies	打标码

    '''

    for row in json_data['detail_data']:
        content = content + f'''
{row['id']}	{row['po_id']}
        {row['cust_device']}	{row['fab_device']}	{row['ht_pn']}
        {row['lot_id']}	{row['wafer_id']}	{row['gross_dies']}	{row['mark_code']}

        '''

    content = content + '
'

    return content

# Get upload data
def get_upload_data(upload_id):
    dict_data = {}
    json_data_total = []
    json_data_detail = []

    # Total data
    sql = f'''
    select row_number() over(ORDER BY bb.lotid) as 序号,case bb.substratetype when 'A' THEN '保税' when 'B' THEN '非保税' else '未知' end as 是否保税,
    bb.customershortname as 客户代码,aa.po_num,aa.mpn_desc as 客户机种名,aa.Fab_conv_id as 客户Fab机种,aa.mtrl_num as 厂内机种,cc.MARKETLASTUPDATE_BY AS 晶圆料号,
    bb.lotid AS LOTID,count(DISTINCT bb.wafer_id) AS 片,sum(bb.passbincount + bb.failbincount) Dies,bb.qtech_created_by, to_char(bb.qtech_created_date,'yyyy-MM-dd'),cc.residual
    from customeroitbl_test aa inner join mappingdatatest bb on to_char(aa.id) = bb.filename INNER JOIN TBLTSVNPIPRODUCT cc ON aa.CUSTOMERSHORTNAME  = cc.CUSTOMERSHORTNAME
    AND aa.MPN_DESC  = cc.CUSTOMERPTNO1 AND aa.FAB_CONV_ID  = cc.CUSTOMERPTNO2  where aa.wafer_visual_inspect = '{upload_id}'
    group by bb.customershortname,aa.Fab_conv_id,aa.mpn_desc,bb.lotid,aa.mtrl_num,bb.passbincount,bb.failbincount,aa.po_num,bb.qtech_created_by,
    to_char(bb.qtech_created_date,'yyyy-MM-dd'),bb.substratetype,cc.MARKETLASTUPDATE_BY,cc.residual
    '''
    results = conn.OracleConn.query(sql)
    for row in results:
        result = {}
        result['id'] = xstr(row[0])
        result['banded'] = xstr(row[1])
        result['cust_code'] = xstr(row[2])
        result['po_id'] = xstr(row[3])
        result['cust_device'] = xstr(row[4])
        result['fab_device'] = xstr(row[5])
        result['ht_pn'] = xstr(row[6])
        result['wafer_pn'] = xstr(row[7])
        result['lot_id'] = xstr(row[8])
        result['wafer_qty'] = row[9]
        result['die_qty'] = row[10]
        result['upload_by'] = xstr(row[11])
        result['upload_date'] = xstr(row[12])
        result['npi_owner'] = xstr(row[13])

        json_data_total.append(result)

    dict_data['total_data'] = json_data_total

    # Detail data
    sql = f'''
    select row_number() over(ORDER BY  bb.lotid,bb.substrateid) as 序号,case bb.substratetype when 'A' then '保税' when 'B' then '非保税' else '未知' end as 是否保税, bb.customershortname as 客户代码,
          aa.Fab_conv_id as FAB机种,aa.mpn_desc as 客户机种,cc.residual as NPI负责人员,
          aa.mtrl_num as 厂内机种,
          aa.po_num as PO_NUM,
          bb.lotid as LOT_ID,
          bb.wafer_id as WAFER_NO,
          bb.substrateid as WAFERID,
          bb.passbincount as GOOD_DIES,
          bb.failbincount as NG_DIES,
          bb.passbincount + bb.failbincount as GROSS_DIES,
          bb.productid as 打标码,
          aa.Imager_Customer_Rev as 二级代码, bb.qtech_created_by as 上传人员,bb.qtech_created_date as 上传时间,  bb.qtech_lastupdate_by as 更新人员, bb.qtech_lastupdate_date as 更新时间
     from customeroitbl_test aa
     left join tbltsvnpiproduct cc on cc.customerptno1 = aa.mpn_desc  and  cc.qtechptno = aa.mtrl_num  and cc.customershortname = aa.customershortname and cc.residual is not null
    inner join mappingdatatest bb
       on to_char(aa.id) = bb.filename
      and aa.wafer_visual_inspect = '{upload_id}'
      group by  bb.customershortname,cc.residual,aa.mtrl_num,aa.Fab_conv_id, aa.mpn_desc,aa.po_num,bb.lotid,bb.wafer_id,bb.substrateid,bb.passbincount,bb.failbincount,bb.productid,aa.Imager_Customer_Rev ,bb.substratetype,bb.qtech_created_by,bb.qtech_created_date,bb.qtech_lastupdate_by,bb.qtech_lastupdate_date
    '''

    results = conn.OracleConn.query(sql)
    for row in results:
        result = {}
        result['id'] = xstr(row[0])
        result['banded'] = xstr(row[1])
        result['cust_code'] = xstr(row[2])
        result['fab_device'] = xstr(row[3])
        result['cust_device'] = xstr(row[4])
        result['npi_owner'] = xstr(row[5])
        result['ht_pn'] = xstr(row[6])
        result['po_id'] = xstr(row[7])
        result['lot_id'] = xstr(row[8])
        result['wafer_no'] = xstr(row[9])
        result['wafer_id'] = xstr(row[10])
        result['good_dies'] = row[11]
        result['ng_dies'] = row[12]
        result['gross_dies'] = row[13]
        result['mark_code'] = xstr(row[14])
        result['second_code'] = xstr(row[15])
        result['upload_by'] = xstr(row[16])
        result['upload_date'] = xstr(row[17])
        result['update_by'] = xstr(row[18])
        result['update_date'] = xstr(row[19])

        json_data_detail.append(result)

    dict_data['detail_data'] = json_data_detail

    set_xl(json_data_total, json_data_detail)
    return dict_data

# Get cell value
def get_cell_val(row, col, data, header):
    if col > len(header) or (row-2) > len(data):
        val = ""
    else:
        val = data[row-3][header[col-1]]
    return val

# Set new xl
def set_xl(total_data, detail_data):
    wb = load_workbook('export_xl_template/template.xlsx')
    # Total
    header_list = ["id", "po_id", "cust_code",  "cust_device", "fab_device",
                   "ht_pn", "lot_id", "wafer_qty", "die_qty", "banded", "upload_by", "upload_date", "npi_owner"]

    ws = wb.get_sheet_by_name(wb.sheetnames[0])
    for row in range(3, len(total_data)+3):
        for col in range(1, len(header_list)):
            ws.cell(column=col, row=row, value=get_cell_val(
                row, col, total_data, header_list))

    # Detail
    header_list = ["id", "po_id", "cust_code",  "cust_device", "fab_device", "ht_pn", "lot_id",
                   "wafer_no", "wafer_id", "gross_dies", "good_dies", "ng_dies", "mark_code"]

    ws = wb.get_sheet_by_name(wb.sheetnames[1])

    for row in range(3, len(detail_data)+3):
        for col in range(1, 30):
            ws.cell(column=col, row=row, value=get_cell_val(
                row, col, detail_data, header_list))

    wb.save("已上传订单.xlsx")

# Parse po file
def parse_po_file(file_name, po_header):
    po_dict = get_po_config(po_header)
    if not po_dict:
        po_header['err_desc'] = '查询不到配置文件无法解析'
        return False

    file_type = po_dict['file_type']
    if not file_name.upper().endswith(file_type.upper()):
        po_header['err_desc'] = '您上传的文件和模板设定的文件类型不一致'
        return False

    if file_type == 'xlsx' or file_type == 'xls':
        if not parse_xlsx_file(file_name, po_header, po_dict):
            return False
    else:
        po_header['err_desc'] = '文件类型不支持解析'
        return False

    return True

# Get Json config
def get_po_config(po_header):
    sql = "SELECT TEMPLATE_CONFIG FROM CMP_CUST_PO_TEMPLATE WHERE TEMPLATE_ID  = %s" % (
        po_header['file_id'])
    results = conn.OracleConn.query(sql)
    if not results:
        print("无法获取配置文件")
        return False

    template_config = results[0][0]
    file_dir = os.path.join(os.getcwd(), template_config)

    f = open(file_dir, 'r', encoding="utf-8")
    po_dict = json.load(f)
    return po_dict

# Parse xlsx file
def parse_xlsx_file(file_name, po_header, po_dict):
    file_cfg = ['file_key', 'file_index', 'file_header', 'file_max_cols']
    for item in file_cfg:
        if not item in po_dict:
            po_header['err_desc'] = '配置文件错误，请联系IT处理'
            return False

    file_key = po_dict['file_key']
    file_index = po_dict['file_index']
    file_header = po_dict['file_header']
    file_max_cols = po_dict['file_max_cols']

    po_data = []
    df = pd.DataFrame(pd.read_excel(
        file_name, sheet_name=file_index, header=file_header, keep_default_na=False))
    for index, row in df.iterrows():
        po_row_data = {}
        for key in file_key:
            col_name = file_key[key]['position']['col_name']
            if col_name != "":
                if col_name in row:
                    po_row_data[key] = row[col_name]
            elif 'other_key' in po_dict:
                if key in po_dict['other_key']:
                    po_row_data[key] = get_cell_val_by_openpyxl(
                        file_name, po_dict['other_key'][key]['position'])
            else:
                po_row_data[key] = ''

        po_data.append(po_row_data)

    print(po_data)
    if not check_po_data(po_header, po_dict, po_data):
        return False

    if not save_po_data(po_header, po_dict, po_data):
        return False
    return True

# Get cell value by openpyxl/xlrd
def get_cell_val_by_openpyxl(file_name, key_postion):
    if file_name.endswith('.xls'):
        dict1 = thans_col_row_from_string(key_postion)
        cell_val = open_workbook(file_name).sheets()[0].cell_value(
            dict1['row'], dict1['col'])

    elif file_name.endswith('.xlsx'):
        wb = load_workbook(file_name)
        ws = wb.get_sheet_by_name(wb.sheetnames[0])
        cell_val = ws[key_postion].value

    print(cell_val)
    return cell_val

def thans_col_row_from_string(s):
    dict = {}
    ss = [''.join(list(g)) for k, g in groupby(s, key=lambda x: x.isdigit())]

    dict['col'] = column_index_from_string(ss[0]) - 1
    dict['row'] = int(ss[1]) - 1
    return dict

# Get list
def get_wafer_list(wafer_str):
    if wafer_str == "":
        return []
    wafer_str = str(wafer_str)
    wafer_str_new = re.sub(r'[_~-]', ' _ ', wafer_str)
    pattern = re.compile(r'[A-Za-z0-9_]+')
    result1 = pattern.findall(wafer_str_new)

    # extend
    for i in range(1, len(result1)-1):
        if result1[i] == '_':
            if result1[i-1].isdigit() and result1[i+1].isdigit():
                bt = []
                if int(result1[i-1]) < int(result1[i+1]):
                    for j in range(int(result1[i-1])+1, int(result1[i+1])):
                        bt.append(f'{j}')
                else:
                    for j in range(int(result1[i-1])-1, int(result1[i+1]), -1):
                        bt.append(f'{j}')
                result1.extend(bt)

    # remove '_'
    result2 = sorted(set(result1), key=result1.index)
    if '_' in result2:
        result2.remove('_')

    return result2

# Check po data
def check_po_data(po_header, po_dict, po_data):

    for item in po_data:
        # Check 0:necessary key
        necessary_key_list = ['po_id', 'customer_device', 'lot_id', 'wafer_id']
        for key in necessary_key_list:
            if not key in item:
                po_header['err_desc'] = '配置文件必要KEY不存在错误，或选错上传文件，无法解析上传文件，请联系IT处理'
                return False

        wafer_id_list = get_wafer_list(item['wafer_id'])
        if len(wafer_id_list) == 0:
            continue

        # Check 1:Wafer qty
        if 'wafer_qty' in item:
            wafer_qty = item['wafer_qty']
            if len(wafer_id_list) != int(wafer_qty):
                print('wafer qty和wafer list明细不一致')
                po_header['err_desc'] = 'wafer qty和wafer list明细不一致'
                return False

        # Check 2:customer code => customer device
        if 'cust_fab_device' in item:
            sql = f"SELECT count(1) FROM TBLTSVNPIPRODUCT t  WHERE t.CUSTOMERSHORTNAME  = '{po_header['cust_code']}' AND instr( '{item['cust_fab_device']}',t.CUSTOMERPTNO1) > 0  "
        else:
            sql = f"SELECT count(1) FROM TBLTSVNPIPRODUCT t  WHERE t.CUSTOMERSHORTNAME  = '{po_header['cust_code']}' AND t.CUSTOMERPTNO1  = '{item['customer_device']}'  "

        if conn.OracleConn.query(sql)[0][0] == 0:
            print('客户代码和客户机种无法对应，或者NPI对照表未维护对应关系')
            po_header['err_desc'] = '客户代码和客户机种无法对应，或者NPI对照表未维护对应关系'
            return False

    return True

# Save po data
def save_po_data(po_header, po_dict, po_data):
    global upload_task

    num = 0
    for item in po_data:
        wafer_id_list = get_wafer_list(item['wafer_id'])
        num = num + len(wafer_id_list)

    for item in po_data:
        wafer_id_list = get_wafer_list(item['wafer_id'])
        if len(wafer_id_list) == 0:
            continue

        for i in range(len(wafer_id_list)):
            insert_po_data(wafer_id_list[i], po_header, item)
            upload_task[po_header['file_id']
                        ] = upload_task[po_header['file_id']] + 100 / float(num)
            if upload_task[po_header['file_id']] >= 100:
                upload_task[po_header['file_id']] = 99

    return True

# Insert to DB
def insert_po_data(wafer_id, po_header, po_data):
    # Get data
    if wafer_id.isdigit() and (len(wafer_id) == 1):
        wafer_id = ('00' + wafer_id)[-2:]
    sql = "select CustomerBCtbl_SEQ.nextval ID from dual"
    max_id = conn.OracleConn.query(sql)[0][0]
    upload_id = po_header['upload_id']
    bonded = 'A' if po_header['bonded_type'] == '保税' else 'B'
    create_by = po_header['user_name']
    lot_id = po_data['lot_id']
    cust_code = po_header['cust_code']
    po_id = po_data['po_id']
    cust_device = po_data['customer_device']
    fab_device = po_data['fab_device']
    cust_fab_device = po_data['cust_fab_device'] if 'cust_fab_device' in po_data else ''
    product_code = po_data['add_0']
    ret = get_cust_pn_info(cust_code, cust_device,
                           fab_device, cust_fab_device, product_code)
    if not ret:
        po_header['err_desc'] = '无法对应NPI对照表唯一厂内机种'

    fab_device = ret['fab_device'] if ret else ''
    cust_device = ret['cust_device'] if ret else ''
    ht_pn = ret['ht_pn'] if ret else ''
    passbin_count = ret['gross_dies'] if ret else '0'
    failbin_count = '0'
    product_id = ret['product_id'] if ret else ''
    wafer_vesion = po_data['add_0']
    ship_comment = po_data['add_1']
    probe_ship_part_type = po_data['add_2']
    shipping_mst_level = po_data['add_2']
    reticle_level_71 = po_data['add_3']
    shipping_mst_260 = po_data['add_3']
    reticle_level_72 = po_data['add_4']
    reticle_level_73 = po_data['add_5']
    assembly_facility = po_data['add_6']
    batch_comment_assy = po_data['add_7']
    mark_code = po_data['mark_code']

    # Delete old wafer data
    delete_po_data('1', str(lot_id)+str(wafer_id))
    # Oracle insert
    sql = ''' insert into mappingDataTest(id,substrateid,substratetype,productid,micronlotid,lotid,Wafer_ID,passbincount,
              failbincount,CustomerShortName,flag,Qtech_Created_By,Qtech_Created_Date,filename)
              values( mappingData_SEQ.Nextval,'%s','%s','%s','%s',
                     '%s','%s','%s','%s','%s','Y','%s',sysdate,'%s')
          ''' % (str(lot_id)+str(wafer_id), bonded, mark_code, 'std_new', lot_id, wafer_id, passbin_count, failbin_count, cust_code, create_by, max_id)

    conn.OracleConn.exec(sql)

    sql = f''' insert into CustomerOItbl_test(id,po_num,wafer_visual_inspect,source_batch_id,SHIP_SITE,Test_site,FAB_CONV_ID,mpn_desc,
            Imager_Customer_Rev,Created_Date,mtrl_num,CustomerShortName,flag,Qtech_Created_By,Qtech_Created_Date,
            probe_ship_part_type,RETICLE_LEVEL_71,RETICLE_LEVEL_72,RETICLE_LEVEL_73,ASSEMBLY_FACILITY,BATCH_COMMENT_ASSY,
            jobno,date_code,shipping_mst_level,shipping_mst_260,TARGET_WAF_THICKNESS,COMP_CODE,SHIP_COMMENT)
            values( {max_id},'{po_id}','{upload_id}','{lot_id}','{cust_code}','HTKS','{fab_device}','{cust_device}','{wafer_vesion}','','{ht_pn}','{cust_code}',
            'Y','{create_by}',sysdate,'{probe_ship_part_type}','{reticle_level_71}','{reticle_level_72}','{reticle_level_73}','{assembly_facility}',
            '{batch_comment_assy}','{bonded}','','{shipping_mst_level}','{shipping_mst_260}','{lot_id}','HTKS','{ship_comment}')
          '''

    conn.OracleConn.exec(sql)

    # Sqlserver insert
    sql = ''' insert into [ERPBASE].[dbo].[tblmappingData](substrateid,substratetype,productid,lotid,Wafer_ID,passbincount,
              failbincount,CustomerShortName,flag,Qtech_Created_By,Qtech_Created_Date,filename)
              values('%s','%s','%s','%s','%s','%s','0','%s','Y','%s',getdate(),'%s')
          ''' % (str(lot_id)+str(wafer_id), bonded, mark_code, lot_id, wafer_id, passbin_count, cust_code, create_by, max_id)

    conn.MssConn.exec(sql)

    sql = f''' insert into [ERPBASE].[dbo].[tblCustomerOI](id,po_num,wafer_visual_inspect,source_batch_id,SHIP_SITE,Test_site,FAB_CONV_ID,mpn_desc,
            Imager_Customer_Rev,Created_Date,mtrl_num,CustomerShortName,flag,Qtech_Created_By,Qtech_Created_Date,
            probe_ship_part_type,RETICLE_LEVEL_71,RETICLE_LEVEL_72,RETICLE_LEVEL_73,ASSEMBLY_FACILITY,BATCH_COMMENT_ASSY,
            jobno,date_code,shipping_mst_level,shipping_mst_260,TARGET_WAF_THICKNESS,COMP_CODE,SHIP_COMMENT)
            values( {max_id},'{po_id}','{upload_id}','{lot_id}','{cust_code}','HTKS','{fab_device}','{cust_device}','{wafer_vesion}','','{ht_pn}','{cust_code}',
            'Y','{create_by}',getdate(),'{probe_ship_part_type}','{reticle_level_71}','{reticle_level_72}','{reticle_level_73}','{assembly_facility}',
            '{batch_comment_assy}','{bonded}','','{shipping_mst_level}','{shipping_mst_260}','{lot_id}','HTKS','{ship_comment}')
          '''
    conn.MssConn.exec(sql)

def get_cust_pn_info(cust_code, cust_device, fab_device, cust_fab_device, product_code):
    if cust_fab_device != '':
        sql = f''' SELECT QTECHPTNO,CUSTOMERDIEQTY,QTECHPTNO2,CUSTOMERPTNO2,CUSTOMERPTNO1 FROM TBLTSVNPIPRODUCT
            WHERE CUSTOMERSHORTNAME = '{cust_code}' and instr('{cust_fab_device}',CUSTOMERPTNO1) > 0
            AND   instr('{cust_fab_device}',CUSTOMERPTNO2) > 0 AND STRUCKSTR3 = '{product_code}'
            and substr(QTECHPTNO2 ,-3,2) <> 'WB' and QTECHPTNO NOT LIKE '%FT'
            AND MARKETLASTUPDATE_BY IS NOT null
            '''
    else:
        if fab_device != '':
            sql = f''' SELECT QTECHPTNO,CUSTOMERDIEQTY,QTECHPTNO2,CUSTOMERPTNO2,CUSTOMERPTNO1 FROM TBLTSVNPIPRODUCT
                WHERE CUSTOMERSHORTNAME = '{cust_code}' and CUSTOMERPTNO1  = '{cust_device}'
                and CUSTOMERPTNO2 = '{fab_device}'
                and substr(QTECHPTNO2 ,-3,2) <> 'WB' and QTECHPTNO NOT LIKE '%FT'
                AND MARKETLASTUPDATE_BY IS NOT null
                '''
        else:
            sql = f''' SELECT QTECHPTNO,CUSTOMERDIEQTY,QTECHPTNO2,CUSTOMERPTNO2,CUSTOMERPTNO1 FROM TBLTSVNPIPRODUCT
                WHERE CUSTOMERSHORTNAME = '{cust_code}' and CUSTOMERPTNO1  = '{cust_device}'
                and substr(QTECHPTNO2 ,-3,2) <> 'WB' and QTECHPTNO NOT LIKE '%FT'
                AND MARKETLASTUPDATE_BY IS NOT null
                '''

    results = conn.OracleConn.query(sql)

    if not results:
        print('无法获取NPI对照表对应机种信息')
        return False

    if len(results) > 1:
        print('无法获取唯一的NPI对照表对应机种信息')
        return False

    ret = {}
    ret['ht_pn'] = results[0][0]
    ret['gross_dies'] = results[0][1]
    ret['product_id'] = results[0][2]
    ret['fab_device'] = results[0][3]
    ret['cust_device'] = results[0][4]
    return ret

# Delete po data
def delete_po_data(flag_, del_id):
    if flag_ == '0':
        # Delete by lot
        sql = f"delete from mappingdatatest where lotid = '{del_id}' "
        conn.OracleConn.exec(sql)
        sql = f"delete from CustomerOItbl_test where source_batch_id = '{del_id}' "
        conn.OracleConn.exec(sql)
        sql = f"delete from [ERPBASE].[dbo].[tblmappingData] where lotid = '{del_id} "
        conn.MssConn.exec(sql)
        sql = f"delete from [ERPBASE].[dbo].[tblCustomerOI] where source_batch_id = '{del_id}' "
        conn.MssConn.exec(sql)
    elif flag_ == '1':
        # Delete by wafer
        sql = f"delete from CustomerOItbl_test where to_char(id) in (select filename from mappingdatatest where substrateid = '{del_id}') "
        conn.OracleConn.exec(sql)
        sql = f"delete from mappingdatatest where substrateid = '{del_id}' "
        conn.OracleConn.exec(sql)
        sql = f"delete from [ERPBASE].[dbo].[tblCustomerOI] where CONVERT(char(20),id) in (select filename from [ERPBASE].[dbo].[tblmappingData] where substrateid = '{del_id}') "
        conn.MssConn.exec(sql)
        sql = f"delete from [ERPBASE].[dbo].[tblmappingData] where substrateid = '{del_id}' "
        conn.MssConn.exec(sql)
    elif flag_ == '2':
        # Delete by uploadid
        sql = f"delete from mappingdatatest where filename in (select to_char(id) from CustomerOItbl_test where wafer_visual_inspect = '{del_id}') "
        conn.OracleConn.exec(sql)
        sql = f"delete from CustomerOItbl_test where wafer_visual_inspect = '{del_id}' "
        conn.OracleConn.exec(sql)
        sql = f"delete from [ERPBASE].[dbo].[tblmappingData] where filename in (select CONVERT(char(20),id)  from [ERPBASE].[dbo].[tblCustomerOI] where wafer_visual_inspect = '{del_id}') "
        conn.MssConn.exec(sql)
        sql = f"delete from [ERPBASE].[dbo].[tblCustomerOI] where wafer_visual_inspect = '{del_id}' "
        conn.MssConn.exec(sql)


#!/usr/bin/python3
import unittest
from console import HBNBCommand
from models import storage
from io import StringIO
from unittest.mock import patch

class TestDoCreate(unittest.TestCase):
    """Test the do_create function"""

    def setUp(self):
        """Set up for the tests"""
        self.console = HBNBCommand()

    def test_do_create(self):
        """Test creating an instance without parameters"""
        with patch('sys.stdout', new=StringIO()) as f:
            self.console.onecmd("create State")
            output = f.getvalue().strip()
            self.assertTrue(len(output) > 0)
            self.assertTrue("State" in storage.all())

    def test_do_create_with_params(self):
        """Test creating an instance with parameters"""
        with patch('sys.stdout', new=StringIO()) as f:
            self.console.onecmd('create State name="California"')
            state = list(storage.all("State").values())[-1]
            self.assertEqual(state.name, "California")

    def test_do_create_with_invalid_params(self):
        """Test creating an instance with invalid parameters"""
        with patch('sys.stdout', new=StringIO()) as f:
            self.console.onecmd('create State invalid_param')
            output = f.getvalue().strip()
            self.assertTrue("State" in storage.all())

if __name__ == '__main__':
    unittest.main()


import datetime

from MFramework.database.alchemy.mixins import Cooldown, Snowflake
from mlib.database import Base, Default
from sqlalchemy import (
    TIMESTAMP,
    Column,
    Enum,
    Float,
    ForeignKey,
    Integer,
    String,
    UnicodeText,
)
from sqlalchemy.orm import relationship

from . import types
from .mixins import EventID, ItemID, LocationID, UserID

# https://github.com/sqlalchemy/sqlalchemy/wiki/UniqueObject
# https://github.com/sqlalchemy/sqlalchemy/wiki/UniqueObjectValidatedOnPending
# Might be useful for Items
class Location(Default, Cooldown, Base):
    pass

class Event(Default, Base):
    start: datetime.datetime = Column(TIMESTAMP(True))
    end: datetime.datetime = Column(TIMESTAMP(True))

    def __init__(self, name, start, end):
        self.name = name
        self.start = start
        self.end = end

check_type = type

class Item(Default, Cooldown, Base):
    type: types.Item = Column(Enum(types.Item), nullable=False)
    description: str = Column(UnicodeText)
    global_limit: int = Column(Integer)
    rarity: types.Rarity = Column(Enum(types.Rarity))
    worth: int = Column(Integer, default=0)
    durability: int = Column(Integer)
    repairs: int = Column(Integer)
    damage: int = Column(Integer)
    flags: int = Column(Integer, default=0)
    req_skill: int = Column(ForeignKey("Skill.id", ondelete="SET NULL", onupdate="Cascade"))

    icon: str = Column(String, nullable=True)
    emoji: str = Column(String, nullable=True)

    def __init__(self, name: str, type: types.Item) -> None:
        super().__init__(name)
        self.type = type if check_type(type) is not str else types.Item.get(type)

class Inventory(ItemID, UserID, Base):
    user_id: Snowflake = Column(
        ForeignKey("User.id", ondelete="Cascade", onupdate="Cascade"), primary_key=True, nullable=False
    )
    quantity: int = Column(Integer, default=0)
    item: Item = relationship("Item")

    def __init__(self, Item: Item = None, quantity: int = 1):
        self.item = Item
        self.quantity = quantity

class Drop(ItemID, LocationID, EventID, Base):
    location_id: int = Column(ForeignKey("Location.id", ondelete="Cascade", onupdate="Cascade"), primary_key=True)
    weight: float = Column(Float)
    chance: float = Column(Float)
    region_limit: int = Column(Integer)
    quantity_min: int = Column(Integer, default=0)
    quantity_max: int = Column(Integer, default=1)

    item: Item = relationship("Item", uselist=False)
    location: Location = relationship("Location", uselist=False)
    event: Event = relationship("Event", uselist=False)

from . import types

class Items:
    Coin = types.Item.Currency
    Crypto = types.Item.Currency

    Material = types.Item.Resource
    Scrap = types.Item.Resource
    Junk = types.Item.Resource
    Trash = types.Item.Resource
    Metal = types.Item.Resource
    Ore = types.Item.Resource
    Ingot = types.Item.Resource

    Liquid = types.Item.Fluid

    Food = types.Item.Resource
    Drink = types.Item.Resource
    Cookie = types.Item.Resource
    Beverage = types.Item.Resource

    Sword = types.Item.Weapon
    Bow = types.Item.Weapon

    Advent = types.Item.Event
    EasterEgg = types.Item.Event

    Upgrade = types.Item.Modification
    Enemy = types.Item.Entity
    NPC = types.Item.Entity
    Ally = types.Item.Entity


from utils import read_file, timer
from itertools import chain
from math import sqrt
from itertools import count

def divisors(n):
    return set(chain.from_iterable((i,n//i) for i in range(1,int(sqrt(n))+1) if n%i == 0))

def count_presents(n):
    div = list(divisors(n))
    tot = 0
    for d in div:
        tot += d * 10
    return tot

@timer
def solve():
    input = int(read_file("20"))
    for house in count(1):
        if count_presents(house) >= input:
            return house

result = solve()
print(f'Solution: {result}')


"""
Usage: python seam_carving.py <rs> <r/c> <scale> <image_in> <image_out> <energy_map_type> <y/n>
Or:python seam_carving.py <rm> <image_in> <image_out> <image_mask>
<rm/rs> rm represents remove rs represents resize
<r/c> 删除列还是删除行
<scale> 缩小为原来的多少(0,1)之间的浮点数
<energy_map_type> c:calc_energy b:backward_energy f:forward_energy
<y/n> y:使用grabCut n:不使用grabCut
"""
import sys
from scipy.ndimage import convolve
import cv2
import numba
import numpy as np
from tqdm import trange
from scipy import ndimage as ndi

ENERGY_MASK_CONST = 100000.0              # large energy value for protective masking
MASK_THRESHOLD = 10                       # minimum pixel intensity for binary mask
SEAM_COLOR = np.array([255, 200, 200])

def visualize(im, seams, rotate=False):
    vis = im.astype(np.uint8)
    r, c, _ = vis.shape
    if seams is not None:
        for i in range(r):
            vis[i][seams[i]] = SEAM_COLOR
    if rotate:
        vis = np.rot90(vis, 3, (0, 1))
    #cv2.namedWindow('visualization', cv2.WINDOW_NORMAL)
    #cv2.resizeWindow("visualization", 500, 500)
    cv2.imshow("visualization", vis)
    cv2.waitKey(30)
    return vis

@numba.jit()
def cumulative_energy(energy):
    r, c = energy.shape
    backtrack = np.zeros((r, c), dtype=np.int64)
    energy_map = energy.copy()
    energy_map[0] = energy[0]

    for i in range(1, r):
        for j in range(0, c):
            # 处理左侧边缘
            if j == 0:
                idx = np.argmin(energy_map[i - 1, j : j + 2])
                backtrack[i, j] = idx
                min_energy = energy_map[i - 1, idx + j]
            # 处理右侧边缘
            elif j == c - 1:
                idx = np.argmin(energy_map[i - 1, j - 1 : j + 1])
                backtrack[i, j] = idx - 1  # idx + j - 1
                min_energy = energy_map[i - 1, idx + j - 1]
            else:
                idx = np.argmin(energy_map[i - 1, j - 1 : j + 2])
                backtrack[i, j] = idx - 1  # idx+j-1
                min_energy = energy_map[i - 1, idx + j - 1]
            energy_map[i, j] += min_energy

    return backtrack, energy_map

def calc_energy(img):
    # np.array函数创建了一个类似矩阵的东西
    # 一个二维数组
    filter_x = np.array(
        [
            [1.0, 2.0, 1.0],
            [0.0, 0.0, 0.0],
            [-1.0, -2.0, -1.0],
        ]
    )
    # stack函数,我们要对红绿蓝都计算
    filter_x = np.stack([filter_x] * 3, axis=2)

    filter_y = np.array(
        [
            [1.0, 0.0, -1.0],
            [2.0, 0.0, -2.0],
            [1.0, 0.0, -1.0],
        ]
    )
    filter_y = np.stack([filter_y] * 3, axis=2)

    # 类型转化，转化为float
    img = img.astype("float32")

    # 计算卷积和
    convolved = np.absolute(convolve(img, filter_x)) + np.absolute(
        convolve(img, filter_y)
    )
    # 将红绿蓝通道中的能量相加
    energy_map = convolved.sum(axis=2)

    return energy_map

def backward_energy(im):
    xgrad = ndi.convolve1d(im, np.array([1, 0, -1]), axis=1, mode="wrap")
    ygrad = ndi.convolve1d(im, np.array([1, 0, -1]), axis=0, mode="wrap")
    grad_mag = np.sqrt(np.sum(xgrad**2, axis=2) + np.sum(ygrad**2, axis=2))

    return grad_mag

def forward_energy(im):
    """
    Forward energy algorithm as described in "Improved Seam Carving for Video Retargeting"
    by Rubinstein, Shamir, Avidan.
    Vectorized code adapted from
    https://github.com/axu2/improved-seam-carving.
    """
    h, w = im.shape[:2]
    im = cv2.cvtColor(im.astype(np.uint8), cv2.COLOR_BGR2GRAY).astype(np.float64)

    energy = np.zeros((h, w))
    m = np.zeros((h, w))

    U = np.roll(im, 1, axis=0)
    L = np.roll(im, 1, axis=1)
    R = np.roll(im, -1, axis=1)

    cU = np.abs(R - L)
    cL = np.abs(U - L) + cU
    cR = np.abs(U - R) + cU

    for i in range(1, h):
        mU = m[i - 1]
        mL = np.roll(mU, 1)
        mR = np.roll(mU, -1)
        mULR = np.array([mU, mL, mR])
        cULR = np.array([cU[i], cL[i], cR[i]])
        mULR += cULR
        argmins = np.argmin(mULR, axis=0)
        m[i] = np.choose(argmins, mULR)
        energy[i] = np.choose(argmins, cULR)

    return energy

def crop_c(img, scale_c, energy_map_type, if_grabCut, rotated):
    r, c, _ = img.shape
    new_c = int((1 - scale_c) * c)
    img = resize_image(img, new_c, energy_map_type, if_grabCut, rotated)
    return img

def crop_r(img, scale_r, energy_map_type, if_grabCut):
    img = np.rot90(img, 1, (0, 1))
    img = crop_c(img, scale_r, energy_map_type, if_grabCut, True)
    img = np.rot90(img, 3, (0, 1))
    return img

def add_c(img, scale_c, energy_map_type, if_grabCut, rotated):
    r, c, _ = img.shape
    new_c = int(scale_c * c)
    img = add_column(img, new_c - c, energy_map_type, if_grabCut, rotated)
    return img

def add_r(img, scale_r, energy_map_type, if_grabCut):
    img = np.rot90(img, 1, (0, 1))
    img = add_c(img, scale_r, energy_map_type, if_grabCut, True)
    img = np.rot90(img, 3, (0, 1))
    return img

def find_seam(paths, end_x):
    r, c = paths.shape[:2]
    seam = [end_x]
    for i in range(r - 1, 0, -1):
        cur_x = seam[-1]
        offset_of_prev_x = paths[i][cur_x]
        seam.append(cur_x + offset_of_prev_x)
    seam.reverse()

    return seam

def remove_seam(img, seam):
    r, c = img.shape[:2]
    img_removed = np.zeros((r, c - 1, 3))
    for r2 in range(r):
        img_removed[r2] = np.delete(img[r2], seam[r2], axis=0)
    return img_removed

def remove_seam_grayscale(img ,seam):
    r, c = img.shape[:2]
    img_removed = np.zeros((r, c - 1))
    for i in range(r):
        img_removed[i]=np.delete(img[i], seam[i], axis=0)
    return img_removed

def add_seam(im, seam_idx, rotated=False):
    h, w = im.shape[:2]
    visualize(im, seam_idx, rotated)
    output = np.zeros((h, w + 1, 3))

    for row in range(h):
        col = seam_idx[row]
        for ch in range(3):
            if col == 0:
                p = np.average(im[row, col : col + 2, ch])
                output[row, col, ch] = im[row, col, ch]
                output[row, col + 1, ch] = p
                output[row, col + 1 :, ch] = im[row, col:, ch]
            else:
                p = np.average(im[row, col - 1 : col + 1, ch])
                output[row, :col, ch] = im[row, :col, ch]
                output[row, col, ch] = p
                output[row, col + 1 :, ch] = im[row, col:, ch]

    return output

def add_column(img, add_c, energy_map_type, if_grabCut, rotated=False):
    img_original = img.copy()
    r, c, channels = img.shape
    seams_record = np.ones((add_c, r), dtype=int)

    if(if_grabCut):
        energy_grabCut=improve_seam(img,rotated)

    for i in trange(add_c, desc="cropping image by {0} pixels".format(add_c)):
        if energy_map_type == "c":
            energy_map = calc_energy(img)
        elif energy_map_type == "b":
            energy_map = backward_energy(img)
        else:
            energy_map = forward_energy(img)

        if(if_grabCut):
            energy_map+=energy_grabCut

        backtrack, energy_totals = cumulative_energy(energy_map)
        energy_minc = list(energy_totals[-1]).index(min(energy_totals[-1]))
        seam = find_seam(backtrack, energy_minc)
        visualize(img, seam, rotated)
        seams_record[i] = seam
        img = remove_seam(img, seam)
        if(if_grabCut):
            energy_grabCut=remove_seam_grayscale(energy_grabCut,seam)

    for i in trange(add_c, desc="expanding image by {0} pixels".format(add_c)):
        img_original = add_seam(img_original, seams_record[i], rotated)
        for i2 in range(i + 1, add_c):
            seams_record[i2][np.where(seams_record[i2] >= seams_record[i])] += 2

    return img_original

def resize_image(
    full_img,
    cropped_c,
    energy_map_type,
    if_grabCut,
    rotated=False,
):
    img = full_img.copy()
    if(if_grabCut):
        energy_grabCut=improve_seam(img,rotated)

    for _ in trange(cropped_c, desc="cropping image by {0} pixels".format(cropped_c)):

        if energy_map_type == "c":
            energy_map = calc_energy(img)
        elif energy_map_type == "b":
            energy_map = backward_energy(img)
        else:
            energy_map = forward_energy(img)

        if(if_grabCut):
            energy_map+=energy_grabCut

        backtrack, energy_totals = cumulative_energy(energy_map)
        energy_minc = list(energy_totals[-1]).index(min(energy_totals[-1]))
        seam = find_seam(backtrack, energy_minc)
        visualize(img, seam, rotated)
        img = remove_seam(img, seam)
        if(if_grabCut):
            energy_grabCut=remove_seam_grayscale(energy_grabCut,seam)

    return img

def improve_seam(im_rgb, rotated):
    im_cut = im_rgb.copy().astype('uint8')
    if rotated:
        im_cut_rot = np.rot90(im_cut, 3, (0, 1))
    else:
        im_cut_rot = im_cut.copy()
    r0, c0 = im_cut.shape[:2]
    r = cv2.selectROI('input', im_cut_rot, False)
    rect = (int(r[1]), int(r0-r[0]-r[2]), int(r[3]), int(r[2]))
    rows, cols = im_cut.shape[0], im_cut.shape[1]
    mask = np.zeros(im_cut.shape[:2],np.uint8)
    bgdModel = np.zeros((1,65),np.float64)
    fgdModel = np.zeros((1,65),np.float64)
    cv2.grabCut(im_cut,mask,rect,bgdModel,fgdModel,10,cv2.GC_INIT_WITH_RECT)
    mask2 = np.where((mask==2)|(mask==0),0,1).astype('uint8')#0和2做背景
    energy_map =calc_energy(im_rgb)
    for i in range(rows):
            for j in range(cols):
                if (mask2[i][j] == 1):
                    energy_map[i][j] = 10000
                elif ( mask2[i][j]==3):
                    energy_map[i][j] = 10000
                else:
                    energy_map[i][j] = 0
    return energy_map

def object_removal(img,mask):
    h, w = img.shape[:2]
    output = img

    while len(np.where(mask>MASK_THRESHOLD)[0])> 0 :
        energy_map=calc_energy(img)
        #在能量图中导入MASK的影响
        energy_map[np.where(mask>MASK_THRESHOLD)]= -ENERGY_MASK_CONST
        backtrack, energy_totals = cumulative_energy(energy_map)
        energy_minc = list(energy_totals[-1]).index(min(energy_totals[-1]))
        seam = find_seam(backtrack, energy_minc)
        visualize(img, seam)
        img = remove_seam(img, seam)
        mask=remove_seam_grayscale(mask,seam)
    num_add=int(w-img.shape[1])
    output=add_column(img,num_add,'c')
    return output

def main():
    if len(sys.argv) == 5:
        in_filename = sys.argv[2]
        out_filename = sys.argv[3]
        mask_filename = sys.argv[4]
        img_in = cv2.imread(in_filename).astype(np.float64)
        #read in protect mask in gray scale
        img_mask = cv2.imread(mask_filename,0).astype(np.float64)
        img_out=object_removal(img_in,img_mask)
        cv2.imwrite(out_filename, img_out.astype(np.float64))
    elif len(sys.argv) == 8:

        which_axis = sys.argv[2]
        scale = float(sys.argv[3])
        in_filename = sys.argv[4]
        out_filename = sys.argv[5]
        energy_map_type = sys.argv[6]
        if_grabCut =sys.argv[7]

        img = cv2.imread(in_filename).astype(np.float64)
        if(if_grabCut)=='y':
            have_grabCut=True
        else:
            have_grabCut=False

        if scale < 1:
            if which_axis == "r":
                out = crop_r(img, scale, energy_map_type, have_grabCut)
            elif which_axis == "c":
                out = crop_c(img, scale, energy_map_type, have_grabCut, False)
            else:
                print(
                    "Usage: python seam_carving.py <rm/rs> <r/c> <scale> <image_in> <image_out> <energy_map_type> <y/n>",
                    file=sys.stderr,
                )
                sys.exit(1)

        elif scale > 1:
            if which_axis == "r":
                out = add_r(img, scale, energy_map_type, have_grabCut)
            elif which_axis == "c":
                out = add_c(img, scale, energy_map_type, have_grabCut, False)
            else:
                print(
                    "Usage: python seam_carving.py <rm/rs> <r/c> <scale> <image_in> <image_out> <energy_map_type> <y/n>",
                    file=sys.stderr,
                )
                sys.exit(1)

        out = out.astype(np.float64)
        cv2.imwrite(out_filename, out.astype(np.float64))
    else:
        print(
            "Usage: python seam_carving.py <rm/rs> <r/c> <scale> <image_in> <image_out> <energy_map_type> <y/n>",
            file=sys.stderr,
        )
        sys.exit(1)

if __name__ == "__main__":
    main()


"""
Demo showing how km_dict and insegtannotator may be used together for
interactive segmentation.

@author: vand and abda
"""

import sys
import insegtprobannotator
import skimage.io
import skimage.data
import km_dict
import feat_seg
import numpy as np
import glob

#%% EXAMPLE: nerve fibres
# /Users/abda/Documents/Projects/Code/TextureGUI/InSegt/data/nerve
in_dir = '../data/nerve/'
file_names = sorted(glob.glob(in_dir + '*.png'))
n_im = len(file_names)
n = int(n_im/2)
print(n)
#%%
## loading image
print('Loading image')
image = skimage.io.imread(file_names[n])

#%% COMMON PART

int_patch_size = 9
branching_factor = 5
number_layers = 5
number_training_patches = 200000
normalization = False

patch_size_feat = 5
n_train = 50000
n_keep = 10
order_keep = (True, True, True)

image_float = image.astype(np.float)/255

# Compute feature image
feat_im, vec, mean_patch, norm_fac = feat_seg.get_pca_feat(image_float, patch_size_feat, n_train, n_keep, order_keep, sigma = -1)
feat_im = np.asarray(feat_im.transpose(2,0,1), order='C')

# Build tree
T = km_dict.build_km_tree(feat_im, 1, branching_factor, number_training_patches, number_layers, normalization)

# Search km-tree and get assignment
A, number_nodes = km_dict.search_km_tree(feat_im, T, branching_factor, normalization)
# number of repetitions for updating the segmentation
number_repetitions = 2

def processing_function(labels):
    r,c = labels.shape
    l = np.max(labels)+1
    if(l>1):
        label_image = np.zeros((r,c,l))
        for k in range(number_repetitions):
            for i in range(1,l):
                label_image[:,:,i] = (labels == i).astype(float)
            D = km_dict.improb_to_dictprob(A, label_image[:,:,1:], number_nodes, int_patch_size) # Dictionary
            P = km_dict.dictprob_to_improb(A, D, int_patch_size) # Probability map
            labels = (np.argmax(P,axis=2) + 1)*(np.sum(P,axis=2)>0)# Segmentation
    else:
        P = np.empty((r,c,0))
    return labels, P.transpose(2,0,1), D

pf = lambda labels: processing_function(labels)[:2]

print('Showtime')

# showtime
app = insegtprobannotator.PyQt5.QtWidgets.QApplication(sys.argv)
ex = insegtprobannotator.InSegtProbAnnotator(image, pf)
sys.exit(app.exec_())

#%% Run the model on a batch of images
import matplotlib.pyplot as plt
import time
# Get the labels
labels = ex.rgbaToLabels(ex.pixmapToArray(ex.annotationPix))
# Compute dictionary
segmentation, P_out, D = processing_function(labels)
n_im = 50
r,c = image.shape
V = np.empty((r,c,n_im))
for i in range(0,n_im):
    t = time.time()
    im_in = skimage.io.imread(file_names[i]).astype(np.float)/255
    feat_im_b = feat_seg.get_pca_feat(im_in, vec = vec, mean_patch = mean_patch, sigma = -1, norm_fac = norm_fac)[0].transpose(2,0,1)# Compute feature image
    A_b = km_dict.search_km_tree(feat_im_b, T, branching_factor, normalization)[0]
    V[:,:,i] = km_dict.dictprob_to_improb(A_b, D, int_patch_size)[:,:,0]
    print(f'Iteration: {i:03.0f} of {n_im:03.0f} Time {time.time()-t:0.3} sec')

fig, ax = plt.subplots(1,2)
ax[0].imshow(V[:,:,10]<0.5)
ax[1].imshow(V[:,10,:]<0.5)

#%%

import vizVTK
vizVTK.visgray((V*255).astype(np.uint8), scalarOpacity = {0: 1.0, 100: 0.001},
               gradientOpacity=None, colorTransfer = {0: (0.0, 1.0, 1.0), 255: (1.0, 0.0, 0.0)},
               windowSize = (1600,1600))

#%%
labels = ex.rgbaToLabels(ex.pixmapToArray(ex.annotationPix))
labels1 = np.argmax(ex.probabilities.transpose(1,2,0), axis = 2)
import matplotlib.pyplot as plt
fig, ax = plt.subplots(1,2)
ax[0].imshow(labels)
ax[1].imshow(labels1)

#%%
import matplotlib.colors

labels = ex.rgbaToLabels(ex.pixmapToArray(ex.annotationPix))

segmentation, P_out, D = processing_function(labels)

colors = np.asarray(ex.colors)/255
cm = matplotlib.colors.LinearSegmentedColormap.from_list('insegt_colors', colors, N=len(ex.colors))

l = int(D.shape[1]/(int_patch_size**2))
fig,ax = plt.subplots(1, 3, sharex=True, sharey=True, figsize=(9,3))
ax[0].imshow(image, cmap = 'gray')
ax[1].imshow(labels, vmin = 0, vmax = len(ex.colors), cmap = cm)
ax[2].imshow(segmentation, vmin = 0, vmax = len(ex.colors), cmap = cm)
plt.show()
fig,ax = plt.subplots(1, figsize=(5,3))
ax.imshow(D, aspect='auto', cmap = 'jet')
plt.show()

# ax.imshow(ex.probabilities.transpose(1,2,0)[:,:,0])

# def processing_function(labels):
#     r,c = labels.shape
#     l = np.max(labels)+1
#     label_image = np.zeros((r,c,l))
#     for k in range(number_repetitions):
#         for i in range(1,l):
#             label_image[:,:,i] = (labels == i).astype(float)
#         D = km_dict.improb_to_dictprob(A, label_image, number_nodes, int_patch_size) # Dictionary
#         P = km_dict.dictprob_to_improb(A, D, int_patch_size) # Probability map
#         labels = np.argmax(P,axis=2) # Segmentation
#     return labels


# Databricks notebook source
# MAGIC %run ./imports_variables

# COMMAND ----------

dbutils.widgets.text('count', '')
count = int(dbutils.widgets.get('count'))

# COMMAND ----------

import csv
import uuid
from random import *
import time
from pathlib import Path
from datetime import datetime

# count = 0
Path(event_source_path).mkdir(parents=True, exist_ok=True)

row_list = [ ["id", "amount", "ts"],
               [randint(1,7), randint(1,20), datetime.now()],
               [randint(8,20), randint(1,20), datetime.now()],
               [randint(1,7), randint(1,20), datetime.now()],
               [randint(1,7), randint(1,20), 999],
               [randint(1,7), randint(1,20)]
             ]
file_location = f'{event_source_path}/file_{count}.csv'

with open(file_location, 'w', newline='') as file:
    writer = csv.writer(file)
    writer.writerows(row_list)
    file.close()

count += 1
dbutils.fs.mv(f'file:{file_location}', f'dbfs:{file_location}')
time.sleep(5)
print(f'New CSV file created at dbfs:{file_location}. Contents:')

with open(f'/dbfs{file_location}', 'r') as file:
    reader = csv.reader(file, delimiter=' ')
    for row in reader:
      print(', '.join(row))
    file.close()


from flask import Flask, render_template, url_for, redirect, request
from datetime import datetime
from auth import *
from vk_api import VkApi
from vk_api.bot_longpoll import VkBotLongPoll, VkBotEventType
app = Flask(__name__)

vk_session = VkApi(token=TOKEN)
@app.route('/vk_stat/<int:group_id>')
def vk_stat(group_id):
    vk = vk_session.get_api()
    activity = {}
    ages = {}
    cities = []
    for i in vk.stats.get(group_id=group_id, app_id=7403595, extended=True, intervals_count=10):
        if 'activity' in i:
            for j in i['activity']:
                if j in activity:
                    activity[j] += 1
                else:
                    activity[j] = 1
        if i['reach']['reach'] != 0:
            for j in i['reach']['age']:
                if j['value'] not in ages:
                    ages[j['value']] = j['count']
                else:
                    ages[j['value']] += j['count']
            for j in i['reach']['cities']:
                if j['name'] not in cities:
                    cities.append(j['name'])
    return render_template('vk_stat.html', title='Vk Statistics', activity=activity, ages=ages, cities=cities)

if __name__ == '__main__':
    app.run(port=8080, host='127.0.0.1')


import re,random
from modules.images import FilenameGenerator
from PIL import Image
from modules import processing,shared,generation_parameters_copypaste
from modules.shared import opts,state
from modules.sd_samplers import samplers,samplers_for_img2img
import logging

def wh_chg_n(p):
    return
def wh_chg_w(p):
    if p.height>p.width:
        (p.width,p.height)=(p.height,p.width)
def wh_chg_h(p):
    if p.width>p.height:
        (p.width,p.height)=(p.height,p.width)
def wh_chg_r(p):
    if random.random()>0.5:
        (p.width,p.height)=(p.height,p.width)

def create_infotext(p):

    clip_skip = getattr(p,'clip_skip',opts.CLIP_stop_at_last_layers)

    generation_params = {
        #"Steps": p.steps,
        #"Sampler": processing.get_correct_sampler(p)[p.sampler_index].name,
        #"CFG scale": p.cfg_scale,
        "Face restoration": (opts.face_restoration_model if p.restore_faces else None),
        "Size": f"{p.width}x{p.height}",
        "Model hash": getattr(p,'sd_model_hash',None if not opts.add_model_hash_to_info or not shared.sd_model.sd_model_hash else shared.sd_model.sd_model_hash),
        "Model": (None if not opts.add_model_name_to_info or not shared.sd_model.sd_checkpoint_info.model_name else shared.sd_model.sd_checkpoint_info.model_name.replace(',','').replace(':','')),
        "Hypernet": (None if shared.loaded_hypernetwork is None else shared.loaded_hypernetwork.name),
        "Seed resize from": (None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}"),
        "Denoising strength": getattr(p,'denoising_strength',None),
        "Eta": (None if p.sampler is None or p.sampler.eta == p.sampler.default_eta else p.sampler.eta),
        "Clip skip": None if clip_skip <= 1 else clip_skip,
        "ENSD": None if opts.eta_noise_seed_delta == 0 else opts.eta_noise_seed_delta,
    }

    generation_params.update(p.extra_generation_params)

    generation_params_text = ",".join([k if k == v else f'{k}: {generation_parameters_copypaste.quote(v)}' for k,v in generation_params.items() if v is not None])

    negative_prompt_text = "\nNegative prompt: \n" + p.negative_prompt if p.negative_prompt else ""

    return f"{p.prompt[0] if type(p.prompt) == list else p.prompt}{negative_prompt_text}\n{generation_params_text}".strip()


import json
import requests
import botogram
from datetime import datetime

TOKEN = "enter your bot token here.. "
# get the token ...
bot = botogram.create( TOKEN )

type_of_loads = {
    "SEA": "Seats Available",
    "SDA": "Standing Available",
    "LSD":"Limited Standing"
    }
type_of_bus = {
    "SD" : "SD.png",
    "DD":"DD.png",
    "BD":"BD.png"
    }

@bot.command("hello")
def hello_command(chat, message, args):
    """Say hello to the world!"""
    btns = botogram.Buttons()
    chat.send("Hello Buddy..ðŸ˜„")
    chat.send("I am Your SG Bus Assistant. You can ask for timings of Buses.")
    chat.send("Try sending, </getbusses 17159> </getbusses bustopcode> to see the list of available busses at your bus stop..")
    # chat.send_photo('logo.png', caption="Hello ....",)
    chat.send_sticker('logo.png')

@bot.callback("_get_bus_")
def delete_callback(query, chat, message,data):
    datalist = data.split(":")
    bus_no = datalist[0]
    bus_stop_code = datalist[1]
    status, next_bus_ = nextBus(bus_stop_code,bus_no)
    sendchathandler (status,next_bus_, chat,bus_no )
    query.notify("you clicked "+data)

@bot.command("getbusses")
def getbuses_command(chat, message, args):
    """I will list you the busses in this busstop """
    try:
        if len(args)==1:
            print(len(args))
            bstop_code = args[0]
            bus_list = getBusses(bstop_code)
            btns = botogram.Buttons()

            if len(bus_list )>0:
                msg='Here you go. Choose the Bus Number You are looking for.'
                for index,bus in enumerate(bus_list):
                    btns[index].callback(bus , "_get_bus_",bus+":"+bstop_code)
                    # msg = msg + bus +" , "
                chat.send(msg, attach=btns)
            else:
                chat.send(f"Invalid bus-stop-code : {bstop_code} â˜¹")
        else:
            chat.send("Bus stop code required..â˜¹ Try calling '/getbusses 17159'")
    except Exception as e:
        print (e.__doc__)
        chat.send( "Something went wrong..â˜¹ Try calling '/getbusses 17159'")

@bot.command("getnextbus")
def getnextbus_command(chat, message, args):
    """I will provide you with the next available bus timing"""
    try:
        print(len(args))
        if len(args)==0:
            chat.send("Bus stop code and bus_number required.â˜¹ Try calling '/getnextbus 17159 183'")
        elif len(args)==1:
            chat.send("Bus_number required.â˜¹ Try calling '/getnextbus 17159 183'")
        elif len(args)==2:
            bus_no = args[1]
            bus_stop_code = args[0]
            status, next_bus_ = nextBus(bus_stop_code,bus_no)
            btns=botogram.Buttons()
            sendchathandler (status,next_bus_, chat , bus_no)
        else:
            chat.send("Bus stop code followed by Bus_Number is required..â˜¹ Try calling '/getnextbus 17159 183'")
    except Exception as e:
        print (e.__doc__)
        chat.send ("Something went wrong...â˜¹ Bus stop code followed by Bus_Number is required..Try calling '/getnextbus 17159 183'")

# **************************************************************************************************

def getresult(bus_stop):
    acc_key ='DATAMALL API KEY'
    header = { 'AccountKey' : acc_key,
     'accept' : 'application/json'}

    path=f'http://datamall2.mytransport.sg/ltaodataservice/BusArrivalv2?BusStopCode={bus_stop}'
    resp = requests.get(path,headers=header)
    if resp.ok and len(resp.json().get("Services")) > 0:
        print("valid bus stop code ...")
        result = resp.json()
        return True,result
    else:
        print("invalid bus stop code ...")
        return False, ''

def getBusses(bus_stop):
    status,result = getresult(bus_stop)
    if status:
        bus_list =[]
        for item in result["Services"]:
            bus_list.append(item["ServiceNo"])
        return bus_list
    else:
        return []

def nextBus(bus_stop_code, bus_no):
    status,result = getresult(bus_stop_code)
    dic={}
    if status:
        for item in result["Services"]:
            listtemp = []
            for key, value in item.items():
                if isinstance(value, dict):
                    _item = item[key]
                    if _item.get("EstimatedArrival", False):
                        listtemp.append( {
                            "EstimatedArrival":_item["EstimatedArrival"],
                            "Load":_item["Load"],
                            "Type":_item["Type"],
                            } )
            dic[item["ServiceNo"]]= listtemp
        return True, dic.get(str(bus_no),False)
    else:
        return False , f"â˜¹ Invalid bustop code : {bus_stop_code}"

def message_based_on_time_difference(time):
    time_utc = datetime.fromisoformat(time)
    minutes , seconds = divmod((time_utc - datetime.now(time_utc.tzinfo)).total_seconds(), 60)
    print(minutes,",", seconds)
    if minutes <= 1:
        return ("Arriving Now")
    else:
        return (f"Will be arriving in {int(minutes)} minutes")

def sendchathandler (status,next_bus_, chat,bus_no ):
    if status:
        if next_bus_:
            chat.send(f"Bus Number : {bus_no}")
            for index,item in enumerate(next_bus_):
                message = message_based_on_time_difference(item["EstimatedArrival"])
                busload = item["Load"]
                bustype = item["Type"]
                chat.send(f"{index+1} : {message} ðŸ˜„")
                # chat.send(f'{type_of_loads.get(busload,"")}')
                chat.send_sticker(type_of_bus.get(bustype,"logo.png"))
        else:
            chat.send(f"â˜¹ Bus '{bus_no}' is not available now. ")
    else:
        chat.send(next_bus_)

# **************************************************************************************************

if __name__ == "__main__":
    bot.run()


import tensorflow as tf
from sklearn.metrics import confusion_matrix
import numpy as np
from sklearn.decomposition import PCA
from sklearn.cross_validation import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score
from skimage.feature import hog
import json

import h5py

def hog_dataset(Xtrain, Xtest):
    Xtrain_hog = []
    Xdev_hog = []

    size = 56

    for idx, x in enumerate(Xtrain):
        x = x.reshape(size, size)
        xg = hog(x, orientations=8, pixels_per_cell=(8, 8),
                 cells_per_block=(1, 1), feature_vector=True)
        Xtrain_hog.append(xg)
        del xg
        if idx % 10000 == 0:
            print(idx)
    Xtrain_hog = np.array(Xtrain_hog).astype('float32')
    print(Xtrain_hog.data.shape)

    for idx, x in enumerate(Xtest):
        x = x.reshape(size, size)
        xg = hog(x, orientations=8, pixels_per_cell=(8, 8),
                 cells_per_block=(1, 1), feature_vector=True)
        Xdev_hog.append(xg)
        del xg
        if idx % 1000 == 0:
            print(idx)
    Xdev_hog = np.array(Xdev_hog).astype('float32')
    print(Xdev_hog.data.shape)

    return Xtrain_hog, Xdev_hog

def predict_test_set(Xtrain_hog, ytrain, Xtest_hog, n_components, n_neighbors):
    pca = PCA(n_components=n_components)
    pca.fit(Xtrain_hog)
    Xtrain_pca = pca.transform(Xtrain_hog)
    Xdev_pca = pca.transform(Xtest_hog)
    print("n_components", n_components)
    print("Captured variance", np.cumsum(pca.explained_variance_ratio_[:n_components])[-1])

    model = KNeighborsClassifier(n_neighbors=n_neighbors, algorithm='kd_tree')
    model.fit(Xtrain_pca, ytrain)

    # compute accuracy
    preds = model.predict(Xdev_pca)

    image_ids = np.arange(len(preds)) + 1

    submission_format = (np.column_stack((image_ids, preds))).astype(int)
    np.savetxt("knn_56_sub.csv", submission_format, delimiter=',', fmt='%i', header="id,predicted", comments="")
    print("Predictions saved to file!")

def acc_on_dev_set(Xtrain_hog, ytrain, Xval_hog, yval, n_components, n_neighbors):
    pca = PCA(n_components=n_components)
    pca.fit(Xtrain_hog)
    Xtrain_pca = pca.transform(Xtrain_hog)
    Xdev_pca = pca.transform(Xval_hog)
    print("n_components", n_components)
    print("Captured variance", np.cumsum(pca.explained_variance_ratio_[:n_components])[-1])

    model = KNeighborsClassifier(n_neighbors=n_neighbors, algorithm='kd_tree')
    model.fit(Xtrain_pca, ytrain)

    # compute accuracy
    preds = model.predict(Xdev_pca)
    acc = accuracy_score(yval, preds)
    print("Final accuracy: {}".format(acc), "n_neighbors", n_neighbors)

def pca_grid_search(Xtrain, ytrain, n_components):
    Xtrain_split, Xdev, ytrain_split, ydev = train_test_split(Xtrain, ytrain, test_size=0.03, random_state=42)

    Xtrain_hog, Xdev_hog = hog_dataset(Xtrain_split, Xdev)

    for k in range(15):
        pca = PCA(n_components=n_components)
        pca.fit(Xtrain_hog)
        Xtrain_pca = pca.transform(Xtrain_hog)
        Xdev_pca = pca.transform(Xdev_hog)
        print("n_components", n_components)
        print("Captured variance", np.cumsum(pca.explained_variance_ratio_[:n_components])[-1])
        n_neighbors = 3
        for i in range(2):
            model = KNeighborsClassifier(n_neighbors=n_neighbors, algorithm='kd_tree')
            model.fit(Xtrain_pca, ytrain_split)

            # compute accuracy
            preds = model.predict(Xdev_pca)
            acc = accuracy_score(ydev, preds)
            print("Final accuracy: {}".format(acc), "n_neighbors", n_neighbors)

            n_neighbors = n_neighbors + 2

        n_components = n_components + 1

if __name__ == "__main__":
    d_train = h5py.File('train_56.h5', 'r')
    d_test = h5py.File('test_56.h5', 'r')
    # d_val = h5py.File('validation_56.h5', 'r')

    Xtrain = np.array(d_train['train']['images'])
    ytrain = np.array(d_train['train']['labels'])

    # Xval = d_val['validation']['images']
    # yval = d_val['validation']['labels']

    Xtest = np.array(d_test['test']['images'])

    n_components = 90
    n_neighbors = 5

    # pca_grid_search(Xtrain, ytrain, n_components)

    # Xtrain_hog, Xval_hog = hog_dataset(Xtrain, Xval)
    # acc_on_dev_set(Xtrain_hog, ytrain, Xval_hog, yval, n_components, n_neighbors)

    Xtrain_hog, Xtest_hog = hog_dataset(Xtrain, Xtest)
    predict_test_set(Xtrain_hog, ytrain, Xtest_hog, n_components, n_neighbors)


#!/usr/bin/env python
import pickle
import os

import numpy as np
from keras.backend.tensorflow_backend import set_session
import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config))
from keras.preprocessing.sequence import pad_sequences
from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.utils import plot_model

from src import get_model

def pad(sentences, mapping, pad_length):
    return pad_sequences([[mapping.get(word, 0) for word in sentence] for sentence in sentences],
                         pad_length, padding='post', truncating='post')

def main(args):
    with open(args.training_path, 'rb') as fp:
        train_q, train_a, train_y, valid_q, valid_a, valid_y = pickle.load(fp)
    embedding = np.load('models/char_base_100_embedding.npy')
    model = get_model(args.model, embedding)
    os.makedirs(f'models/{args.model}', exist_ok=True)
    early_stopping = EarlyStopping(monitor='val_loss', patience=3)
    checkpoint = ModelCheckpoint(f'models/{args.model}/{args.model}' + '_{epoch:03d}_{val_loss:.4f}')
    plot_model(model, f'models/{args.model}/structure.png')
    model.fit([train_q, train_a], train_y, validation_data=([valid_q, valid_a], valid_y),
              batch_size=256, epochs=100, callbacks=[early_stopping, checkpoint])

def parse_args():
    import argparse
    parser = argparse.ArgumentParser('hw5')
    parser.add_argument('-train', '--training-path', default='./raw_data/train_padded.pkl')
    parser.add_argument('-test', '--testing-path', default='./raw_data/testing_data.txt')
    parser.add_argument('-o', '--output-path', default='prediction.csv')
    parser.add_argument('-m', '--model', default='stacked_gru_2_dot')
    parser.add_argument('--mapping', default='./models/mapping.pkl')
    return parser.parse_args()

if __name__ == '__main__':
    main(parse_args())


__author__ = 'achmed'

import zmq
from zmq.devices import Device

from ...config.configwrapper import ConfigWrapper

import logging
logger = logging.getLogger(__name__)

class ZMQSampleBroker(object):
    def __init__(self, cfg):
        self.cfg = ConfigWrapper(other=cfg.get('zmq'))
        self.pulladdr = cfg.get('pulladdr', '0.0.0.0')
        self.pubaddr = cfg.get('pubaddr', '0.0.0.0')
        self.pullport = cfg.get('pullport', 59000)
        self.pubport = cfg.get('pubport', 59001)

    def initialize(self):
        self.forwarddevice = Device(zmq.QUEUE, zmq.PULL, zmq.PUB)
        self.forwarddevice.bind_in('tcp://%s:%d' % (self.pulladdr, self.pullport))
        self.forwarddevice.bind_out('tcp://%s:%d' % (self.pubaddr, self.pubport))
        logger.info('created broker in: PULL@%s:%d out: PUB@%s:%d' % (self.pulladdr, self.pullport, self.pubaddr, self.pubport))

    def start(self):
        logger.info('starting broker')
        self.forwarddevice.start()


from os import getenv

from dotenv import load_dotenv

load_dotenv()

API_ID = int(getenv("API_ID"))
API_HASH = getenv("API_HASH")

TOKEN = getenv("TOKEN")
MONGO_DB_URL = getenv("MONGO_DB_URL", None)

OWNER_ID = int(getenv("OWNER_ID", 5938660179))
SUPPORT_CHAT = getenv("SUPPORT_CHAT", "https://t.me/FallenAssociation")

LOGGER_ID = getenv("LOGGER_ID", "-1001819078701")
DEV_USERS = set(int(x) for x in getenv("DEV_USERS", "").split())


import os
import time
from tqdm import tqdm
import torch
import pickle
import numpy as np
from torch import nn
import torch.nn.functional as F
import matplotlib.pyplot as plt
from torch.utils.data import TensorDataset,DataLoader

print(torch.__version__) # This code has been updated for PyTorch 1.0.0

# trainFeats, trainLabel = None, None

savePath = 'lecture5_output/'
# if not os.path.isdir(savePath):
#     os.makedirs(savePath)
#
#     # Loading the saved features
#     with open("trainFeats.pckl", "rb") as f:
#         trainFeats = pickle.load(f)
#         print(trainFeats)
#     with open("trainLabel.pckl", "rb") as f:
#         trainLabel = pickle.load(f)
#
#     with open("testFeats.pckl", "rb") as f:
#         testFeats = pickle.load(f)
#     with open("testLabel.pckl", "rb") as f:
#         testLabel = pickle.load(f)
#
#     # Generating 1-hot label vectors
#     trainLabel2 = np.zeros((50000, 10))
#     testLabel2 = np.zeros((10000, 10))
#     for d1 in range(trainLabel.shape[0]):
#         trainLabel2[d1, trainLabel[d1]] = 1
#     for d2 in range(testLabel.shape[0]):
#         testLabel2[d2, testLabel[d2]] = 1
#
#     print('Finished loading saved feature matrices from the disk!')

with open("trainFeats.pckl", "rb") as f:
    trainFeats = pickle.load(f)
    print(trainFeats)
with open("trainLabel.pckl", "rb") as f:
    trainLabel = pickle.load(f)

with open("testFeats.pckl", "rb") as f:
    testFeats = pickle.load(f)
with open("testLabel.pckl", "rb") as f:
    testLabel = pickle.load(f)

# Generating 1-hot label vectors
trainLabel2 = np.zeros((50000, 10))
testLabel2 = np.zeros((10000, 10))
for d1 in range(trainLabel.shape[0]):
    trainLabel2[d1, trainLabel[d1]] = 1
for d2 in range(testLabel.shape[0]):
    testLabel2[d2, testLabel[d2]] = 1

device = "cpu"
pinMem = False

# Defining the perceptron
class perceptron(nn.Module):
    def __init__(self,n_channels): #n_channels => length of feature vector
        super(perceptron, self).__init__()
        self.L = nn.Linear(n_channels,10) #Mapping from input to output
    def forward(self,x): #x => Input
        x = self.L(x) #Feed-forward
        x = F.softmax(x,dim=1) #Softmax non-linearity, dim=1 corresponds to labels
        return x

# Checking availability of GPU
# use_gpu = torch.cuda.is_available()
# if use_gpu:
#     print('GPU is available!')
#     device = "cuda"
#     pinMem = True
# else:
#     print('GPU is not available!')

# Creating pytorch dataset from the feature matices
trainDataset = TensorDataset(torch.from_numpy(trainFeats), torch.from_numpy(trainLabel2))
testDataset = TensorDataset(torch.from_numpy(testFeats), torch.from_numpy(testLabel2))
# Creating dataloader
trainLoader = DataLoader(trainDataset, batch_size=1, shuffle=True,num_workers=4, pin_memory=pinMem)
testLoader = DataLoader(testDataset, batch_size=1, shuffle=False,num_workers=4, pin_memory=pinMem)

# Definining the training routine
def train_model(model, criterion, num_epochs, learning_rate):
    start = time.time()
    train_loss = []  # List for saving the loss per epoch
    train_acc = []  # List for saving the accuracy per epoch
    tempLabels = []  # List for saving shuffled labels as fed into the network
    for epoch in range(num_epochs):
        epochStartTime = time.time()
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))

        running_loss = 0.0
        # Loading data in batches
        batch = 0
        for data in tqdm(trainLoader):
            inputs, labels = data

            inputs, labels = inputs.float().to(device), labels.float().to(device)

            # Initializing model gradients to zero
            model.zero_grad()
            # Data feed-forward through the network
            outputs = model(inputs)
            # Predicted class is the one with maximum probability
            _, preds = outputs.data.max(1)
            # Finding the MSE
            loss = criterion(outputs, labels)
            # Accumulating the loss for each batch
            running_loss += loss.item()

            # Backpropaging the error
            if batch == 0:
                totalLoss = loss
                totalPreds = preds
                tempLabels = labels.data.cpu()
                batch += 1
            else:
                totalLoss += loss
                totalPreds = torch.cat((totalPreds, preds), 0)
                tempLabels = torch.cat((tempLabels, labels.data.cpu()), 0)
                batch += 1

        totalLoss = totalLoss / batch
        totalLoss.backward()

        # Updating the model parameters
        for f in model.parameters():
            f.data.sub_(f.grad.data * learning_rate)

        epoch_loss = running_loss / 50000  # Total loss for one epoch
        train_loss.append(epoch_loss)  # Saving the loss over epochs for plotting the graph

        # Accuracy per epoch
        tempLabels = tempLabels.numpy()
        _, totalLabels = np.where(tempLabels == 1)
        epoch_acc = np.sum(np.equal(totalPreds.cpu().numpy(), np.array(totalLabels))) / 50000.0
        train_acc.append(epoch_acc * 100)  # Saving the accuracy over epochs for plotting the graph

        epochTimeEnd = time.time() - epochStartTime
        print('Average epoch loss: {:.6f}'.format(epoch_loss))
        print('Average epoch accuracy: {:.4f} %'.format(epoch_acc * 100))
        print('-' * 25)
        # Plotting Loss vs Epochs
        fig1 = plt.figure(1)
        plt.plot(range(epoch + 1), train_loss, 'r--', label='train')
        if epoch == 0:
            plt.legend(loc='upper right')
            plt.xlabel('Epochs')
            plt.ylabel('Loss')
            plt.title('Plot of training loss vs epochs')
        fig1.savefig(savePath + 'lossPlot.png')
        # Plotting Accuracy vs Epochs
        fig2 = plt.figure(2)
        plt.plot(range(epoch + 1), train_acc, 'g--', label='train')
        if epoch == 0:
            plt.legend(loc='upper left')
            plt.xlabel('Epochs')
            plt.ylabel('Accuracy')
            plt.title('Plot of training accuracy vs epochs')
        fig2.savefig(savePath + 'accPlot.png')

    time_elapsed = time.time() - start
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    return model

featLength = 2+5+2
# Initilaizing the model
model = perceptron(featLength).to(device)
criterion = nn.MSELoss()
model = train_model(model,criterion,num_epochs=100,learning_rate=1) # Training the model

# Finding testing accuracy
test_running_corr = 0
# Loading data in batches
batches = 0
testLabels = []

model.eval()  # Testing the model in evaluation mode

for tsData in tqdm(testLoader):
    inputs, labels = tsData

    inputs, labels = inputs.float().to(device), labels.float()

    with torch.no_grad():  # No back-propagation during testing; gradient computation is not required

        # Feedforward train data batch through model
        output = model(inputs)
        # Predicted class is the one with maximum probability
        _, preds = output.data.max(1)
        if batches == 0:
            totalPreds = preds
            testLabels = torch.argmax(labels, dim=1)  # Converting 1-hot vector labels to integer labels
            batches = 1
        else:
            totalPreds = torch.cat((totalPreds, preds), 0)
            testLabels = torch.cat((testLabels, torch.argmax(labels, dim=1)), 0)

        # Finding total number of correct predictions
ts_corr = np.sum(np.equal(totalPreds.cpu().numpy(), testLabels.numpy()))
# Calculating accuracy
ts_acc = ts_corr / testLabels.shape[0]
print('Accuracy on test set = ' + str(ts_acc * 100) + '%')


# -*- coding: utf-8  -*-

import pytest

import collections
import itertools
import sys

from lxml import etree
import numpy as np
import xmlserdes as X
import xmlserdes.utils as XU
from xmlserdes.errors import XMLSerDesError, XMLSerDesWrongChildrenError

import re

make_TD = X.TypeDescriptor.from_terse

def list_product(*args):
    return list(itertools.product(*args))

class TestAtomicTypes(object):
    @pytest.mark.parametrize('td_func',
                             [X.Atomic, make_TD],
                             ids=['verbose', 'terse'])
    #
    def test_type_descriptor(self, td_func):
        for tp, val, exp_txt in [(int, 42, '42'),
                                 (float, 3.5, '3.5'),
                                 (str, '3 < 5', '3 < 5'),
                                 (np.uint8, np.uint8(42), '42')]:
            td = td_func(tp)
            elt = td.xml_element(val, 'foo')
            assert XU.str_from_xml_elt(elt) == '<foo>%s</foo>' % exp_txt
            val_round_trip = td.extract_from(elt, 'foo')
            assert type(val_round_trip) is tp
            assert val_round_trip == val

    @pytest.mark.parametrize(
        'dtype_str',
        ['i1', 'i2', 'i4', 'u1', 'u2', 'u4'])
    #
    def test_terse_string_descriptors(self, dtype_str):
        td = make_TD(dtype_str)
        tp = np.dtype(dtype_str).type

        elt = td.xml_element(19, 'foo')
        assert '<foo>19</foo>' == XU.str_from_xml_elt(elt)
        val_round_trip = td.extract_from(elt, 'foo')
        assert tp is type(val_round_trip)
        assert val_round_trip == 19

    def test_bad_terse_string_descriptors(self):
        with pytest.raises(TypeError, match='data type .* not understood'):
            make_TD('not-a-real-dtype-code')

    def test_bool(self):
        td = make_TD(bool)

        elt = td.xml_element(True, 'foo')
        assert XU.str_from_xml_elt(elt) == '<foo>true</foo>'
        assert td.extract_from(elt, 'foo') is True

        elt = td.xml_element(False, 'foo')
        assert XU.str_from_xml_elt(elt) == '<foo>false</foo>'
        assert td.extract_from(elt, 'foo') is False

    def test_bool_bad_serialize_values(self):
        td = make_TD(bool)

        with pytest.raises(XMLSerDesError,
                           match='expected True or False but got "42"') as exc_info:
            td.xml_element(42, 'foo')
        assert exc_info.value.xpath == []

    def test_bool_bad_deserialize_values(self):
        td = make_TD(bool)

        with pytest.raises(XMLSerDesError,
                           match='expected "true" or "false" but got "banana"') as exc_info:
            #
            bad_xml = etree.fromstring('<foo>banana</foo>')
            td.extract_from(bad_xml, 'foo')
        assert exc_info.value.xpath == ['foo']

    def test_int_bad_deserialize_values(self):
        td = make_TD(int)
        bad_xml = etree.fromstring('<foo>banana</foo>')
        with pytest.raises(XMLSerDesError,
                           match='could not parse "banana" as "int"') as exc_info:
            td.extract_from(bad_xml, 'foo')
        assert exc_info.value.xpath == ['foo']

@pytest.mark.skipif(sys.version_info < (3, 4),
                    reason='requires Python 3.4 or higher')
class TestAtomicEnum(object):
    def test_bad_enum(self):
        with pytest.raises(TypeError, match='expected Enum-derived type'):
            X.AtomicEnum(int)

    def test_bad_enum_value(self):
        from enum import Enum
        Animal = Enum('Animal', 'Cat Dog Rabbit')
        td = X.AtomicEnum(Animal)
        with pytest.raises(ValueError, match="expected instance of <enum 'Animal'>"):
            td.xml_element(42, 'bad-animal')

class BareRectangle(collections.namedtuple('BareRectangle', 'width height')):
    pass

class TestListTypes(object):
    @pytest.mark.parametrize('td',
                             [X.List(X.Atomic(int), 'wd'),
                              make_TD([int, 'wd'])],
                             ids=['verbose', 'terse'])
    #
    def test_type_descriptor(self, td):
        val = [42, 100, 99, 123]
        elt = td.xml_element(val, 'widths')
        assert (XU.str_from_xml_elt(elt) ==
                '<widths>%s</widths>' % ''.join('<wd>%d</wd>' % x for x in val))
        val_round_trip = td.extract_from(elt, 'widths')
        assert val_round_trip == val

    @pytest.mark.parametrize('list_descr,exc_re',
                             [([1, 2, 3], 'expected 1 or 2 elements'),
                              ([], 'expected 1 or 2 elements'),
                              ([BareRectangle], '1-elt list: .* has no "xml_default_tag"')],
                             ids=['three-elt', 'empty', 'no-default-tag'])
    #
    def test_bad_construction(self, list_descr, exc_re):
        with pytest.raises(ValueError, match=exc_re):
            make_TD(list_descr)

class TestNestedListTypes(object):
    @pytest.mark.parametrize('td',
                             [X.List(X.List(X.Atomic(int), 'wd'), 'stripe-group'),
                              make_TD([[int, 'wd'], 'stripe-group'])],
                             ids=['verbose', 'terse'])
    #
    def test_type_descriptor(self, td):
        groups = [[1, 2], [3, 4, 5]]

        elt = td.xml_element(groups, 'stripe-groups')

        exp_txt = ('<stripe-groups>%s</stripe-groups>'
                   % ''.join(('<stripe-group>%s</stripe-group>'
                              % ''.join('<wd>%d</wd>' % x for x in grp))
                             for grp in groups))

        assert exp_txt == XU.str_from_xml_elt(elt)

        groups_round_trip = td.extract_from(elt, 'stripe-groups')
        assert groups == groups_round_trip

class Rectangle(collections.namedtuple('BareRectangle', 'width height')):
    xml_descriptor = X.SerDesDescriptor([('width', X.Atomic(int)),
                                         ('height', X.Atomic(int))])

class TestRectangleEquality(object):
    def test_equality(self):
        r0 = Rectangle(42, 100)
        r1 = Rectangle(42, 100)
        assert r1 is not r0
        assert r1 == r0

class TestInstanceTypes(object):
    def test_bad_construction_verbose(self):
        with pytest.raises(ValueError, match='has no xml_descriptor'):
            X.Instance(BareRectangle)

    def test_bad_construction_terse(self):
        with pytest.raises(ValueError, match='no "xml_descriptor" attribute'):
            make_TD(BareRectangle)
        with pytest.raises(ValueError, match='unhandled terse descriptor'):
            make_TD(lambda x: x)

    @pytest.mark.parametrize('td',
                             [X.Instance(Rectangle),
                              make_TD(Rectangle)],
                             ids=['verbose', 'terse'])
    #
    def test_type_descriptor(self, td):
        rect = Rectangle(42, 100)
        elt = td.xml_element(rect, 'rect')
        assert XU.str_from_xml_elt(elt) == expected_rect_xml(42, 100)

        rect_round_trip = td.extract_from(elt, 'rect')
        assert rect_round_trip == rect

    def test_bad_xml_wrong_n_children(self):
        td = X.Instance(Rectangle)
        bad_xml = etree.fromstring('<rect>42100<c>123</c></rect>')
        with pytest.raises(XMLSerDesWrongChildrenError,
                           match=('mismatched children: '
                                  r'\[missing: width, missing: height, '
                                  r'unexpected: a, '
                                  r'unexpected: b, '
                                  r'unexpected: c\]')) as exc_info:
            td.extract_from(bad_xml, 'rect')
        assert exc_info.value.xpath == ['rect']

    def test_bad_xml_wrong_tag(self):
        td = X.Instance(Rectangle)
        bad_xml = etree.fromstring('<rect>42100</rect>')
        with pytest.raises(XMLSerDesError,
                           match='.*missing: width.*unexpected: a') as exc_info:
            td.extract_from(bad_xml, 'rect')
        assert exc_info.value.xpath == ['rect']

class _TestNumpyBase(object):
    @pytest.mark.parametrize(
        'x,regexp',
        [('hello', 'not ndarray'),
         (np.array([[1, 2], [3, 4]]), 'not 1-dimensional'),
         (np.zeros((12,), dtype=np.float32), 'expected dtype .* but got "float32"')],
        ids=['string', 'multi-dml', 'wrong-dtype'])
    #
    def test_bad_value(self, x, regexp):
        with pytest.raises(XMLSerDesError, match=regexp) as exc_info:
            self.td.xml_element(x, 'values')
        assert exc_info.value.xpath == []

class TestNumpyAtomic(_TestNumpyBase):
    def setup_method(self, method):
        self.td = X.NumpyAtomicVector(np.int32)

    def round_trip_1(self, xs, td, go_via_string):
        written_elt = td.xml_element(xs, 'values')
        read_elt = (etree.fromstring(XU.str_from_xml_elt(written_elt))
                    if go_via_string
                    else written_elt)
        xs_round_trip = td.extract_from(read_elt, 'values')
        assert xs_round_trip.shape == xs.shape
        assert np.all(xs_round_trip == xs)

    # flake8 doesn't realise this is defined when we use in below,
    # hence 'noqa' at that point.
    dtypes_to_test = [np.uint8, np.uint16, np.uint32, np.uint64,
                      np.int8, np.int16, np.int32, np.int64,
                      np.float32, np.float64]

    @pytest.mark.parametrize(
        'dtype,td_func,use_empty_xs,go_via_string',
        list_product(dtypes_to_test,
                     [X.NumpyAtomicVector, lambda dt: make_TD((np.ndarray, dt))],
                     [False, True],
                     [False, True]),
        ids=['-'.join(flds)
             for flds in list_product([dt.__name__ for dt in dtypes_to_test],  # noqa
                                      ['verbose', 'terse'],
                                      ['nonempty', 'empty'],
                                      ['via-xml-elt', 'via-string'])])
    #
    def test_round_trips(self, dtype, td_func, use_empty_xs, go_via_string):
        xs = np.array([] if use_empty_xs
                      else [-1.23, -9.99, 0.234, 42, 99, 100.11],
                      dtype=dtype)
        self.round_trip_1(xs, td_func(dtype), go_via_string)

    def test_content(self):
        xs = np.array([32, 42, 100, 99, -100], dtype=np.int32)
        elt = self.td.xml_element(xs, 'values')
        assert XU.str_from_xml_elt(elt) == '<values>32,42,100,99,-100</values>'

    def test_empty_content(self):
        xs = np.array([], dtype=np.int32)
        elt = self.td.xml_element(xs, 'values')
        assert XU.str_from_xml_elt(elt) == '<values></values>'

class TestNumpyAtomicConvenience(TestNumpyAtomic):
    def setup_method(self, method):
        self.td = X.NumpyVector(np.int32)

class TestNumpyScalarDType(object):
    def test_content(self):
        colour_dtype = np.dtype([('red', np.uint8),
                                 ('green', np.uint8),
                                 ('blue', np.uint8)])
        # The '[()]' indexing extracts scalar from 0-dim array:
        colour = np.array((25, 50, 75), dtype=colour_dtype)[()]
        td = make_TD(colour_dtype)
        elt = td.xml_element(colour, 'colour')
        assert XU.str_from_xml_elt(elt) == ('<colour>'
                                            '<red>25</red>'
                                            '<green>50</green>'
                                            '<blue>75</blue></colour>')

RectangleDType = np.dtype([('width', np.int32), ('height', np.int32)])

RectanglePairDType = np.dtype([('big', RectangleDType), ('small', RectangleDType)])

def remove_whitespace(s):
    return re.sub(r'\s+', '', s)

class TestNumpyRecordStructured(_TestNumpyBase):
    type_descr = X.NumpyRecordVectorStructured(RectangleDType, 'rect')

    def setup_method(self, method):
        self.td = TestNumpyRecordStructured.type_descr
        self.vals = np.array([(42, 100), (99, 12)], dtype=RectangleDType)

    @pytest.mark.parametrize('td',
                             [type_descr, make_TD((np.ndarray, RectangleDType, 'rect'))],
                             ids=['verbose', 'terse'])
    #
    def test_type_descriptor(self, td):
        xml_elt = td.xml_element(self.vals, 'rects')
        expected_xml = remove_whitespace(
            """<rects>
                 <rect><width>42</width><height>100</height></rect>
                 <rect><width>99</width><height>12</height></rect>
               </rects>""")
        assert XU.str_from_xml_elt(xml_elt) == expected_xml
        vals_rt = td.extract_from(xml_elt, 'rects')
        assert vals_rt.dtype == self.vals.dtype
        assert vals_rt.shape == self.vals.shape
        assert np.all(vals_rt == self.vals)

    @pytest.mark.parametrize(
        'bad_inner_str,exc_re,exp_xpath',
        [('<rect><width>42</width><height>100</height><depth>99</depth></rect>',
          'mismatched children',
          ['rectangles', 'rect[1]']),
         ('<rect><width>42</width><height>100</height></rect>'
          '<rect><width>42</width><height>100</height></rect>'
          '<rect><width>42</width><height>100</height><depth>99</depth></rect>',
          'mismatched children',
          ['rectangles', 'rect[3]']),
         ('<rect><wd>42</wd><ht>100</ht></rect>',
          'missing: width.*unexpected: wd',
          ['rectangles', 'rect[1]'])],
        ids=['wrong-n-elts', 'wrong-n-elts-third-child', 'wrong-child-tag'])
    #
    def test_bad_xml(self, bad_inner_str, exc_re, exp_xpath):
        bad_str = '<rectangles>%s</rectangles>' % bad_inner_str
        bad_xml = etree.fromstring(bad_str)
        with pytest.raises(XMLSerDesError, match=exc_re) as exc_info:
            self.td.extract_from(bad_xml, 'rectangles')
        assert exc_info.value.xpath == exp_xpath

class TestNumpyDTypeScalar(object):
    atomics_td = X.DTypeScalar(RectangleDType)
    atomics_val = np.array((42, 100), dtype=RectangleDType)[()]
    atomics_exp = '<rect><width>42</width><height>100</height></rect>'

    nested_td = X.DTypeScalar(RectanglePairDType)
    nested_val = np.array(((42, 100), (4, 10)), dtype=RectanglePairDType)[()]
    nested_exp = remove_whitespace(
        """<rect-pair>
               <big><width>42</width><height>100</height></big>
               <width>4</width><height>10</height>
           </rect-pair>""")

    @pytest.mark.parametrize(
        'type_descr,val,exp_xml,tag',
        [(atomics_td, atomics_val, atomics_exp, 'rect'),
         (nested_td, nested_val, nested_exp, 'rect-pair')],
        ids=['atomics', 'nested'])
    #
    def test_round_trip(self, type_descr, val, exp_xml, tag):
        assert val.shape == ()

        xml_elt = type_descr.xml_element(val, tag)
        assert XU.str_from_xml_elt(xml_elt) == exp_xml

        val_rt = type_descr.extract_from(xml_elt, tag)
        assert type(val_rt) == np.ndarray
        assert val_rt.dtype == val.dtype
        assert val_rt.shape == val.shape
        assert val_rt == val

    ColourDType = np.dtype([('r', np.uint8), ('g', np.uint8), ('b', np.uint8)])

    @pytest.mark.parametrize(
        'x,regexp',
        [('hello', 'not numpy scalar'),
         (np.array([1, 2]), 'not numpy scalar'),
         (np.array([[1, 2], [3, 4]]), 'not numpy scalar'),
         (np.array((12,), dtype=np.float32)[()], 'not numpy scalar'),
         (np.array((10, 20, 30), dtype=ColourDType)[()], 'expected dtype')],
        ids=['string', 'vector', 'multi-dml', 'wrong-dtype-atomic', 'wrong-dtype-structured'])
    #
    def test_bad_value(self, x, regexp):
        with pytest.raises(XMLSerDesError, match=regexp) as exc_info:
            self.atomics_td.xml_element(x, 'values')
        assert exc_info.value.xpath == []

class TestNumpyRecordStructuredNested(object):
    type_descr = X.NumpyRecordVectorStructured(RectanglePairDType, 'rect-pair')
    vals = np.array([((420, 100), (42, 10)), ((430, 110), (43, 11))],
                    dtype=RectanglePairDType)

    def test_round_trip(self):
        xml_elt = self.type_descr.xml_element(self.vals, 'rect-pairs')
        expected_xml = remove_whitespace(
            """<rect-pairs>
                 <rect-pair>
                     <big><width>420</width><height>100</height></big>
                     <width>42</width><height>10</height>
                 </rect-pair>
                 <rect-pair>
                     <big><width>430</width><height>110</height></big>
                     <width>43</width><height>11</height>
                 </rect-pair>
               </rect-pairs>""")
        assert expected_xml == XU.str_from_xml_elt(xml_elt)
        vals_rt = self.type_descr.extract_from(xml_elt, 'rect-pairs')
        assert vals_rt.dtype == self.vals.dtype
        assert vals_rt.shape == self.vals.shape
        assert np.all(vals_rt == self.vals)

class TestNumpyRecordStructuredConvenience(TestNumpyRecordStructured):
    def setup_method(self, method):
        TestNumpyRecordStructured.setup_method(self, method)
        self.td = X.NumpyVector(RectangleDType, 'rect')

class TestDescriptors(object):
    def setup_method(self, method):
        self.rect = BareRectangle(42, 100)

    @pytest.mark.parametrize(
        'descriptor_tup,exp_tag,exp_vslot',
        [(('width', X.Atomic(int)), 'width', 'width'),
         (('wd', 'width', X.Atomic(int)), 'wd', 'width'),
         (('wd', (lambda x: x.width), X.Atomic(int)), 'wd', None)],
        ids=['pair', 'triple-attr-name', 'triple-function'])
    #
    def test_tuple_construction(self, descriptor_tup, exp_tag, exp_vslot):
        elt_descriptor = X.ElementDescriptor.new_from_tuple(descriptor_tup)
        val = 42
        assert elt_descriptor.tag == exp_tag
        assert elt_descriptor.value_from(self.rect) == val
        assert elt_descriptor.value_slot == exp_vslot
        xml_elt = elt_descriptor.xml_element(self.rect)
        assert XU.str_from_xml_elt(xml_elt) == '<%s>%d</%s>' % (exp_tag, val, exp_tag)
        round_trip_val = elt_descriptor.extract_from(xml_elt)
        assert round_trip_val == val

    @pytest.mark.parametrize(
        'bad_arg,exc_tp,exc_re',
        [((1, 2, 3, 4), ValueError, 'length'),
         (42, TypeError, 'object .* has no len')],
        ids=['wrong-length', 'wrong-type'])
    #
    def test_bad_construction(self, bad_arg, exc_tp, exc_re):
        with pytest.raises(exc_tp, match=exc_re):
            X.ElementDescriptor.new_from_tuple(bad_arg)

def expected_rect_xml(w, h):
    return '<rect><width>%d</width><height>%d</height></rect>' % (w, h)

class TestObject(object):
    def setup_method(self, method):
        self.rect = Rectangle(42, 123)

    def test_1(self):
        serialized_xml = X.serialize(self.rect, 'rect')
        assert XU.str_from_xml_elt(serialized_xml) == expected_rect_xml(42, 123)

        rect_round_trip = X.deserialize(Rectangle, serialized_xml, 'rect')
        assert rect_round_trip == self.rect

    @pytest.mark.parametrize(
        'xml_str,des_tag,exc_re,exp_xpath',
        [('<rect><width>99</width></rect>', 'rect', 'mismatched children', ['rect']),
         (expected_rect_xml(42, 100), 'rectangle', 'expected tag .* but got', [])],
        ids=['wrong-n-children', 'wrong-tag'])
    #
    def test_bad_input(self, xml_str, des_tag, exc_re, exp_xpath):
        bad_xml = etree.fromstring(xml_str)
        with pytest.raises(XMLSerDesError, match=exc_re) as exc_info:
            X.deserialize(Rectangle, bad_xml, des_tag)
        assert exc_info.value.xpath == exp_xpath

class Layout(collections.namedtuple('Layout_', 'colour cornerprops stripes ids shape components')):
    xml_descriptor = X.SerDesDescriptor(
        [('colour', X.Atomic(str)),
         ('corner-properties', 'cornerprops', X.List(X.List(X.Atomic(str), 'prop'), 'corner')),
         ('stripes', X.List(X.Atomic(str), 'stripe-colour')),
         ('product-id-codes', 'ids', X.NumpyAtomicVector(np.uint32)),
         ('shape', X.Instance(Rectangle)),
         ('components', X.NumpyRecordVectorStructured(RectangleDType, 'rect'))])

class TestComplexObject(object):
    def test_1(self):
        layout = Layout('dark-blue',
                        [['rounded', 'red'],
                         ['chamfered', 'matt'],
                         ['pointy', 'anodized', 'black'],
                         ['bevelled', 'twisted', 'plastic-coated']],
                        ['red', 'burnt-ochre', 'orange'],
                        np.array([99, 42, 123], dtype=np.uint32),
                        Rectangle(210, 297),
                        np.array([(20, 30), (40, 50)], dtype=RectangleDType))
        xml = X.serialize(layout, 'layout')
        xml_str = XU.str_from_xml_elt(xml)
        expected_str = remove_whitespace(
            """<layout>
                 <colour>dark-blue</colour>
                 <corner-properties>
                   <corner>
                     <prop>rounded</prop>
                     <prop>red</prop>
                   </corner>
                   <corner>
                     <prop>chamfered</prop>
                     <prop>matt</prop>
                   </corner>
                   <corner>
                     <prop>pointy</prop>
                     <prop>anodized</prop>
                     <prop>black</prop>
                   </corner>
                   <corner>
                     <prop>bevelled</prop>
                     <prop>twisted</prop>
                     <prop>plastic-coated</prop>
                   </corner>
                 </corner-properties>
                 <stripes>
                   <stripe-colour>red</stripe-colour>
                   <stripe-colour>burnt-ochre</stripe-colour>
                   <stripe-colour>orange</stripe-colour>
                 </stripes>
                 <product-id-codes>99,42,123</product-id-codes>
                 <shape>
                   <width>210</width>
                   <height>297</height>
                 </shape>
                 <components>
                   <rect>
                     <width>20</width>
                     <height>30</height>
                   </rect>
                   <rect>
                     <width>40</width>
                     <height>50</height>
                   </rect>
                 </components>
               </layout>""")
        assert xml_str == expected_str
        X.deserialize(Layout, xml, 'layout')

class TestTerseErrorInputs(object):
    @pytest.mark.parametrize(
        'bad_tup,exc_re',
        [((), 'empty tuple'),
         ((42, 'hello', 'world'), 'numpy.ndarray as first'),
         ((np.ndarray, BareRectangle), 'atomic numpy type as second'),
         ((np.ndarray, 'foo', 'bar'), 'numpy dtype as second'),
         ((np.ndarray, 2, 3, 4), 'expected 2 or 3 elements but got 4')],
        ids=['empty', 'wrong-first-elt', '2-elt-not-dtype', '3-elt-not-dtype', 'wrong-length'])
    #
    def test_numpy_descriptor(self, bad_tup, exc_re):
        with pytest.raises(ValueError, match=exc_re):
            make_TD(bad_tup)


from pyspark.sql import SparkSession
import pandas
import sys

user_declared_state = sys.argv[1]
print(user_declared_state, 'state:::::::::::::')

df_mapping = pandas.read_csv('/home/raja/Desktop/Ek_Step/Up text book/State_district.csv')
df_mapping = df_mapping.loc[df_mapping['state'] == user_declared_state]
city_list = list(df_mapping['district'])

spark = SparkSession.builder.appName('UpTextbook').getOrCreate()

df1 = spark.read.csv("/home/raja/Desktop/Ek_Step/Up text book/oct-2022/oct 18/18th Oct UP Textbook - QR Code Scans "
                     "18th Oct 2022.csv", header=True)

df4 = df1.filter(df1["derived_loc_district"].isin(city_list)).select("state_slug", "derived_loc_district",
                                                                     "SUM(total_count)")
df4.repartition(1).write.mode('overwrite').format('csv').option('header', True)\
    .save("/home/raja/Desktop/Ek_Step/Up text book/oct-2022/oct 18/QR")


from django.test import TestCase
from django.core.urlresolvers import reverse

from .models import Category
# Create your tests here.

class CategoryTest(TestCase):
	def create_category(self, category_id = "113", resource_uri = "https://www.eventbriteapi.com/v3/categories",
						name = "Arts", name_localized = "Arts", short_name = "Arts",
						short_name_localized = "Arts"):
		return Category.objects.create(category_id=category_id, resource_uri=resource_uri, name=name, name_localized=name_localized,
									   short_name=short_name, short_name_localized=short_name_localized)

	def test_category_creation(self):
		a = self.create_category()
		self.assertTrue(isinstance(a, Category))

	def test_category_list_view(self):
		a = self.create_category()
		url = reverse('show_cats')
		resp = self.client.get(url)

		self.assertEqual(resp.status_code, 200)
		self.assertIn(a.category_id, resp.content)
		self.assertIn(a.name, resp.content)