# -*- coding: utf-8 -*-
"""
    This spider is a ZPJobs spider created on top of the ATSSpider
    scrapy crawl 100zpjobs -a mining_job_id=9999 -a iteration=1 -a extract=1 -a url="http://www.100zp.com/job/joblist.aspx"

    sample job url:
        http://www.100zp.com/job/company_job.aspx?id=24389
"""

from re import compile
from scrapy.http import Request
from scrapy.http import FormRequest
from scrapy.selector import Selector
from urlparse import urljoin

from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader
from brightcorp.processors import Prefix, ConvertDateString, Replace

Ref_Num = compile(r"id=(\d+)")
Event_Argument = compile(r",'(\S+)'\)")

class ZPJobs(ATSSpider):

    name = "100zpjobs"

    def parse(self, response):
        selector = Selector(response)
        jobs = selector.xpath('//div[@class="job_info1"]/div[@class="job_info1_l"]/a/@href').extract()
        for job in jobs:
            yield Request(
                callback=self.parse_job_callback(),
                url=urljoin(response.url, job)
            )

        next_page = Event_Argument.search(''.join(selector.xpath(
            '//tr[@class="pagerclass2"]//span/../following-sibling::td[1]/a/@href'
        ).extract()))
        if next_page and next_page.group(1):
            yield FormRequest(
                callback=self.parse,
                formdata={
                    '__EVENTTARGET': 'GridView1',
                    '__LASTFOCUS': ''.join(selector.xpath('//input[@name="__LASTFOCUS"]/@value').extract()),
                    '__VIEWSTATE': ''.join(selector.xpath('//input[@name="__VIEWSTATE"]/@value').extract()),
                    '__EVENTVALIDATION': ''.join(selector.xpath('//input[@name="__EVENTVALIDATION"]/@value').extract()),
                    '__EVENTARGUMENT': next_page.group(1),
                    'hka': '',
                    'job': '',
                    'jobtypes': '',
                    'jtzw': unicode('请输入岗位关键字', 'utf-8'),
                    'rtime': '',
                    'hka_': '',
                    'hka2_': '',
                    'jobtypes_': '',
                    'job_': '',
                    'jtzw_': '',
                    'rtime_': '',
                },
                url=response.url,
            )

    def parse_job(self, response):
        selector = Selector(response)
        loader = BrightcorpItemLoader(selector=selector)

        loader.add_xpath('company', '//span[@id="name"]//text()')
        loader.add_xpath('date', '//span[@id="retime"]/text()', Replace(r"\D"), ConvertDateString('%Y%m%d%H%M'))
        loader.add_xpath('baseSalary', '//span[@id="moneys"]/text()')
        loader.add_xpath('description', '//span[@id="zptext"]')
        loader.add_xpath('educationrequirements', '//span[@id="edus"]/text()')
        loader.add_xpath('experiencerequirements', '//span[@id="zyes"]/text()')
        loader.add_xpath('jobcategory', '//span[@id="job"]/text()')
        loader.add_xpath('jobtype', '//span[@id="jobtypes"]/text()')
        loader.add_xpath('language', '//span[@id="languages"]/text()')
        loader.add_xpath('title', '//h2/span[@id="jtzw"]/text()')
        loader.add_xpath('zip_code', '//span[@id="zip"]/text()')

        # Considering city first, if null take province, if null take address
        loader.add_xpath('location', '//span[@id="city"]/text()')
        if not loader.get_output_value('location'):
            loader.add_xpath('location', '//span[@id="address"]/text()')
        if not loader.get_output_value('location'):
            loader.add_xpath('location', '//span[@id="hka"]/text()')

        loader.add_value('apply_url', response.url)
        loader.add_value('referencenumber', response.url, Prefix('%s-' % self.name), re=Ref_Num)
        loader.add_value('url', response.url)

        yield loader.load_item()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Apr 24 15:38:51 2021

@author: seimit
"""
import os
import tempfile
import gzip
import bz2
import lzma
import zipfile
import time
from Semi_ATE import STDF

def test_records_from_file():

    print("\n")

#   Make 2 records and put them into one temporary file

    record = STDF.FAR()
    data = record.__repr__()

    exp_atdf_far = 'FAR:A|4|2|U'
    exp_stdf_far = 'FAR (0,10) @ V4\n\
      REC_LEN = \'2\' [U*2] (Bytes of data following header)\n\
      REC_TYP = \'0\' [U*1] (Record type)\n\
      REC_SUB = \'10\' [U*1] (Record sub-type)\n\
      CPU_TYPE = \'2\' [U*1] (CPU type that wrote this file)\n\
      STDF_VER = \'4\' [U*1] (STDF version number)\n'

    record = STDF.WIR()

    rec_len = 0;

    head_num = 1
    record.set_value('HEAD_NUM', head_num)
    rec_len += 1;

    site_grp = 1
    record.set_value('SITE_GRP', site_grp)
    rec_len += 1;

    start_t = 1609462861
    record.set_value('START_T', start_t)
    rec_len += 4;

    t = ""
    if os.name == "nt":
        t = time.strftime("%#H:%#M:%#S %#d-%b-%Y", time.gmtime(start_t))
    else:
        t = time.strftime("%-H:%-M:%-S %-d-%b-%Y", time.gmtime(start_t))
    t = t.upper()

    t1 = ""
    if os.name == "nt":
        t1 = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(start_t))
    else:
        t1 = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(start_t))
    t1 = t1.upper()

    data += record.__repr__()

    exp_atdf_wir = 'WIR:1|'+t+'|1|'

    exp_atdf = [exp_atdf_far, exp_atdf_wir]

    exp_stdf_wir = '   WIR (2,10) @ V4\n\
      REC_LEN = \'7\' [U*2] (Bytes of data following header)\n\
      REC_TYP = \'2\' [U*1] (Record type)\n\
      REC_SUB = \'10\' [U*1] (Record sub-type)\n\
      HEAD_NUM = \'1\' [U*1] (Test head number)\n\
      SITE_GRP = \'1\' [U*1] (Site group number)\n\
      START_T = \'1609462861\' [U*4] (Date and time first part tested) = '+t1+'\n\
      WAFER_ID = \'\' [C*n] (Wafer ID)\n'

    exp_stdf = [exp_stdf_far, exp_stdf_wir]

    f = tempfile.NamedTemporaryFile(mode="w+b", delete = False)
    file_name = f.name
    f.write(data)
    f.close()

    f = open(file_name)

    print("\nTest standard printing of STDF record\n")

    i=0
    for REC in STDF.records_from_file(file_name):
        print(REC)
        assert exp_stdf[i].replace(" ", "") == REC.__str__().replace(" ", "")
        i+=1

    print("\nTest ATDF printing of STDF record\n")

    i=0
    for REC in STDF.records_from_file(file_name):
        print(REC.to_atdf())
        assert exp_atdf[i] == REC.to_atdf()
        i+=1

    f.close()
    os.remove(file_name)

    print("\nTest printing from gzip compressed STDF record\n")

    gz = tempfile.NamedTemporaryFile(mode="w+b", delete=False)

    with gzip.open(gz, 'wb') as f_gz:
        f_gz.write(data)
    gz.close()

    i=0
    for REC in STDF.records_from_file(gz.name):
        print(REC)
        assert exp_stdf[i].replace(" ", "") == REC.__str__().replace(" ", "")
        i+=1

    os.remove(gz.name)

    print("\nTest printing from bz2 compressed STDF record\n")

    bz = tempfile.NamedTemporaryFile(mode="w+b", delete=False)

    with bz2.open(bz.name, "wb") as f_bz:
        f_bz.write(data)
    bz.close()

    i=0
    for REC in STDF.records_from_file(bz.name):
        print(REC)
        assert exp_stdf[i].replace(" ", "") == REC.__str__().replace(" ", "")
        i+=1

    os.remove(bz.name)

    print("\nTest printing from lzma compressed STDF record\n")

    lz = tempfile.NamedTemporaryFile(mode="w+b", delete=False)

    with lzma.open(lz.name, "wb") as f_lz:
        f_lz.write(data)
    lz.close()

    i=0
    for REC in STDF.records_from_file(lz.name):
        print(REC)
        assert exp_stdf[i].replace(" ", "") == REC.__str__().replace(" ", "")
        i+=1

    os.remove(lz.name)

    print("\nTest printing from zip compressed STDF record\n")

    zo = tempfile.NamedTemporaryFile(mode="w+b", delete=False)
    zo.write(data)
    zo.close()

    z = tempfile.NamedTemporaryFile(mode="w+b", delete=False)

    with zipfile.ZipFile(z.name, "w") as f_z:
        f_z.write(zo.name)
    z.close()

    i=0
    for REC in STDF.records_from_file(z.name):
        print(REC)
        assert exp_stdf[i].replace(" ", "") == REC.__str__().replace(" ", "")
        i+=1

    os.remove(z.name)
    os.remove(zo.name)

def test_dict_to_rec():

    record = STDF.WIR()

    rec_len = 0;

    head_num = 1
    record.set_value('HEAD_NUM', head_num)
    rec_len += 1;

    site_grp = 1
    record.set_value('SITE_GRP', site_grp)
    rec_len += 1;

    start_t = 1609462861
    record.set_value('START_T', start_t)
    rec_len += 4;

    waf_id = 'NAS12345'
    record.set_value('WAFER_ID', waf_id)
    rec_len += 9;

    rec_dict = record.to_dict();
    rec = STDF.utils.dict_to_rec(rec_dict, '<')

    assert rec.get_value('REC_LEN') == rec_len
    assert rec.get_value('HEAD_NUM') == head_num
    assert rec.get_value('SITE_GRP') == site_grp
    assert rec.get_value('START_T') == start_t
    assert rec.get_value('WAFER_ID') == waf_id


"""empty message

Revision ID: 411e0ea55c07
Revises: None
Create Date: 2015-04-20 14:14:55.853461

"""

# revision identifiers, used by Alembic.
revision = '411e0ea55c07'
down_revision = None

from alembic import op
import sqlalchemy as sa
import geoalchemy

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('sidewalk_elevation',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('sidewalk_objectid', sa.Integer(), nullable=True),
    sa.Column('coordinates', geoalchemy.geometry.LineString(), nullable=True),
    sa.Column('grade', sa.Float(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('curb',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('sidewalk_objectid', sa.Integer(), nullable=True),
    sa.Column('coordinates', geoalchemy.geometry.Point(), nullable=True),
    sa.Column('angle', sa.Float(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('curb')
    op.drop_table('sidewalk_elevation')
    ### end Alembic commands ###


import numpy as np
import matplotlib.pyplot as plt
import csv

# Usage: python plot.py
# Specify the csv file names in data_files

# data_files = ['s1', 's2', 's5', 's10', 's25', 's50']
data_files = ['s=1', 's=2', 's=5', 's=10', 's=25', 'svr_adaptive', 'cli_adaptive']
data = {}

for file in data_files:
    acc, time = [], []

    with open(f'{file}.csv') as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        i = 0
        for row in csv_reader:
            if i != 0:
                acc.append(float(row[2]))
                time.append(float(row[3]))

            i += 1

    data[file] = np.array(acc), np.array(time)

for series in data:
    acc, time = data[series]
    if series == 'svr_adaptive':
        plt.plot(time, acc, 'o-', label=series, color='#e377c2')
    elif series == 'cli_adaptive':
        plt.plot(time, acc, '^-', label=series, color='#8c564b')
    else:
        plt.plot(time, acc, '-.', label=series)

plt.xlabel("Time (sec)")
plt.ylabel("Accuracy")
plt.legend(loc='lower right')
plt.savefig('lenet_adaptive.png', dpi=300)


from datetime import datetime
from datetime import timedelta
import importlib
import sys

__author__ = 'vfourrier'

def run(module_path, global_path_utilities, compute_date, result_date):
    module_launcher = module_path
    class_module = module_path.split('.')[-1]
    importlib.import_module(module_launcher)
    run_module = sys.modules[module_launcher]
    run_class = getattr(run_module, class_module)
    run_class(module_path=module_path + class_module, global_path_utilities=global_path_utilities, compute_date=compute_date,
              result_date=result_date)

def main():
    module_path = 'QuantPack.Equity.Products.Options.EQDPriceable'
    compute_date = datetime(2018, 1, 1)
    result_date = compute_date - timedelta(days=1)
    global_path_utilities = r'/Volumes/SECURITY 1/Vincent/BSCM/ATM/Utilities/GetParams/Museum_Wrapper.xlsx'

    run(module_path=module_path, global_path_utilities=global_path_utilities, compute_date=compute_date,
        result_date=result_date)

if __name__ == '__main__':
    main()


from typing import Optional

from mypy_boto3_builder.service_name import ServiceName
from mypy_boto3_builder.constants import MODULE_NAME
from mypy_boto3_builder.import_helpers.import_record import ImportRecord
from mypy_boto3_builder.type_annotations.fake_annotation import FakeAnnotation

class InternalImport(FakeAnnotation):
    is_internal = True

    def __init__(
        self,
        name: str,
        service_name: ServiceName,
        module_name: str = "service_resource",
    ) -> None:
        self.name = name
        self.service_name = service_name
        self.module_name = module_name
        self.real_service_name: Optional[ServiceName] = None

    def render(self) -> str:
        return f"{self.scope}.{self.name}"

    @property
    def scope(self) -> str:
        return f"{self.service_name.module_name}_scope"

    def localize(self, service_name: ServiceName) -> None:
        self.real_service_name = service_name

    def get_import_record(self) -> ImportRecord:
        if self.real_service_name is None:
            raise ValueError("Non-localized ImportString")

        return ImportRecord(
            source=f"{MODULE_NAME}_{self.real_service_name.name}.{self.module_name}",
            alias=self.scope,
        )


from rdflib import Graph, Literal
from rdflib.namespace import Namespace, FOAF
from PracticaECSDI.Constants import Ontologies, FIPAACLPerformatives, Constants

class FlightRequestMessage:
    def __init__(self, uuid, maxprice, initDate, finalDate, departureAirport, arrivalAirport):
        self.uuid = uuid
        self.maxPrice = maxprice
        self.firstDay = initDate
        self.lastDay = finalDate
        self.departureAirport = departureAirport
        self.arrivalAirport = arrivalAirport

    def to_graph(self):
        graph = Graph()
        namespace = Namespace(Constants.ONTOLOGY_NAME)
        flight = namespace.__getattr__('#RequestFlight#' + str(self.uuid))
        graph.add((flight, FOAF.Uuid, Literal(self.uuid)))
        graph.add((flight, FOAF.MaxPrice, Literal(self.maxPrice)))
        graph.add((flight, FOAF.FirstDay, Literal(self.firstDay)))
        graph.add((flight, FOAF.LastDay, Literal(self.lastDay)))
        graph.add((flight, FOAF.DepartureAirport, Literal(self.departureAirport)))
        graph.add((flight, FOAF.ArrivalAirport, Literal(self.arrivalAirport)))
        return graph

    @classmethod
    def from_graph(cls, graph):
        query = """SELECT ?x ?uuid ?maxPrice ?firstDay ?lastDay ?arrivalAirport ?departureAirport
           WHERE {
                ?x ns1:Uuid ?uuid.
                ?x ns1:MaxPrice ?maxPrice.
                ?x ns1:FirstDay ?firstDay.
                ?x ns1:LastDay ?lastDay.
                ?x ns1:DepartureAirport ?departureAirport.
                ?x ns1:ArrivalAirport ?arrivalAirport
           }
        """
        qres = graph.query(query)
        search_res = []
        for f, uuid, maxprice, firstDay, lastDay, arrivalAirport, departureAirport in qres:
            """search_res.append(FlightMessage(
                uuid.toPython(),
                maxprice.toPython(),
                firstDay.toPython(),
                lastDay.toPython(),
                departureAirport.toPython(),
                arrivalAirport.toPython()
            ))"""
            return FlightRequestMessage(uuid.toPython(), maxprice.toPython(), firstDay.toPython(), lastDay.toPython(),
                                        departureAirport.toPython(),arrivalAirport.toPython())
        #return search_res


import numpy as np
import matplotlib.pyplot as plt

class Descriptor:
    @classmethod
    def create(cls, setting = None):
        return Descriptor()
    def __init__(self):
        self.clear_history()
    def clear_history(self):
        self.loss_history = []
        self.accuracy_history = []
    def add_loss_history(self, loss):
        self.loss_history.append(np.mean(loss))
    def add_accuracy(self, accuracy):
        self.accuracy_history.append(accuracy)
    def descript(self, model):
        losses = self.loss_history
        print('last loss', losses[-1])
        print('last accuracy', self.accuracy_history[-1])
        x = range(len(losses))
        plt.figure()
        plt.plot(x, losses)
        plt.xlabel('iteration count')
        plt.ylabel('loss rate')
        plt.show()
        plt.figure()
        plt.plot(x, self.accuracy_history)
        plt.xlabel('iteration count')
        plt.ylabel('accuracy')
        plt.show()


import argparse
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
from skimage import measure
from scipy import linalg, special

# define functions for use in the program

def lam(i):
    alpha = 0.001
    beta = 1.9
    return alpha*beta**i

def norm(i,L):
    """normalisation factor"""
    return np.sqrt( 2*(2*lam(i))**(L+1.5) / special.gamma(L + 1.5) )

def overlap(i, j, L):
    """overlap matrix elements"""
    return norm(i,L)*norm(j,L)*special.gamma(L+1.5) / ( 2*(lam(i)+lam(j))**(L+1.5) )

def kinetic(i, j, L):
    """kinetic matrix elements"""
    return norm(i,L)*norm(j,L)*special.gamma(L+2.5)*lam(i)*lam(j) / (lam(i)+lam(j))**(L+2.5)

def potential(i, j, L, Z):
    """potential matric elements"""
    return -Z*norm(i,L)*norm(j,L)*special.factorial(L) / (2*(lam(i)+lam(j))**(L+1))

def build_matrices(dim, charge, L):
    """ builds the overlap and hamiltonian matrices """
    S = np.zeros((dim, dim))
    T = np.zeros((dim, dim))
    V = np.zeros((dim, dim))
    for i in range(dim):
        for j in range(i,dim):
            S[i, j] = overlap(i, j, L)
            S[j, i] = S[i, j]
            T[i, j] = kinetic(i, j, L)
            T[j, i] = T[i, j]
            V[i, j] = potential(i, j, L, charge)
            V[j, i] = V[i, j]
    return S, T, V

def basis_func(i, L, M, r, theta, phi):
    """ S type gaussian basis function """
    N = norm(i,L)
    return N*r**(L)*np.exp(-lam(i)*r**2)*special.sph_harm(M,L,phi,theta)

def wavefunction(eigenvecs, n, L, M, r, theta, phi):
    """ returns nth excited state wavefunction for corresponding atom at x,y,z """
    total = 0
    for i, coeff in enumerate(eigenvecs[:, n]):
        total += coeff * basis_func(i,L,M,r,theta,phi)
    return total

def probability_density(eigenvecs, n, L, M, x, y, z,real):
    """returns probability density for the atom in the n, L, M state at x,y,z"""
    r = np.sqrt(x**2+y**2+z**2)
    theta = np.arccos(z/r)
    phi = np.arctan2(y,x)
    if real:
        return np.absolute(np.real(wavefunction(eigenvecs, n, L, M, r, theta, phi)))
    else:
        return np.absolute(wavefunction(eigenvecs, n, L, M, r, theta, phi))

def plot_orbital(orbital, size, iso, charge=1, dim=20, real=False, points = 50j, title=True):

    # naming dictionaries
    atom = {1:'Hydrogen',2:'Helium',3:'Lithium',4:'Beryllium'}
    L_val = {'s':0,'p':1,'d':2,'f':3,'g':4,'h':5}

    # identify quantum numbers of given orbital
    n,L,M = int(orbital[0]), L_val[orbital[1]], int(orbital[2])
    level = n-(L+1)
    if level<0 or np.abs(M)>L:
        raise ValueError("The orbital you are trying to plot does not exist")

    # solve the eigenvalue equation
    S, T, V = build_matrices(dim, charge, L)
    eigenvals, eigenvecs = linalg.eigh(T+V, S)

    # plot
    X,Y,Z = np.mgrid[-size:size:points, -size:size:points, -size:size:points]
    surf_vals = probability_density(eigenvecs, level, L, M, X, Y, Z, real)
    verts, faces, _, _ = measure.marching_cubes_lewiner(surf_vals, iso*np.amax(surf_vals))
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    if title: plt.title(f"{atom[charge]} {orbital} orbital")
    ax.plot_trisurf(verts[:, 0], verts[:,1], faces, verts[:, 2], cmap='plasma', lw=0.05, edgecolors='black')
    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_zticks([])
    plt.axis('off')
    plt.show()

# main program
if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("orbital_name")
    parser.add_argument("range")
    args = parser.parse_args()

    print('\n===============================')
    print(f'PLOTTING {args.orbital_name} HYDROGENIC ORBITAL')
    print('===============================\n')

    plot_orbital(args.orbital_name, float(args.range), 0.1, points=50j, real=True, title=False)


# 0501.py
import cv2
import numpy as np
src = cv2.imread('./heart10.jpg', cv2.IMREAD_GRAYSCALE)
cv2.imshow('src',  src)

#ret, dst = cv2.threshold(src, 120, 255, cv2.THRESH_BINARY) # thresholding 값: 120, 기준이 올라가면 올라갈 수록, 검은색 픽셀이 많아진다
ret, dst = cv2.threshold(src, 200, 255, cv2.THRESH_BINARY_INV)   # THRESH_BINARY_INV : 색 반전

print('ret=', ret)
cv2.imshow('dst',  dst)

ret2, dst2 = cv2.threshold(src, 200, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
#THRESH_OTSU : 제시된 thresholding 값을 쓰지 않고, 자동으로 계산한 뒤 Binary에서 이진화 계산

print('ret2=', ret2)
cv2.imshow('dst2',  dst2)

cv2.waitKey()
cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
"""
Created on Sun Aug  6 12:43:47 2017

@author: gansc
"""

import numpy as np
import csv
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
import cv2

class ImageGenerator(object):
    ''' Fetches all images from the driving similar in path + set
        and returns them as batch for training after preprocessing.
        Images from left and right camera are returned with a
        corrected steering angle
    '''
    def __init__(self, path, sets, batch_size=32, train_val_split=0.2):
        ''' Initialize samples with all files from .csv.
        '''
        self._samples = []
        self._correction = 0.09
        self._batch_size = batch_size
        self._cameras = ['center']
        for set in sets:
            with open(path + '/' + set + '/driving_log.csv') as csvfile:
                reader = csv.reader(csvfile)
                for line in reader:
                    self._samples.append(line)

        self._train_samples, self._valid_samples = train_test_split(self._samples, test_size=train_val_split)

    def train_data_gen(self):
        yield from self._data_gen(self._train_samples)

    def valid_data_gen(self, batch_size=32):
        yield from self._data_gen(self._valid_samples)

    def _read_image(self, file):
        name = './data/' + '/'.join(file.replace('\\', '/').split('/')[-3:])
        image = cv2.imread(name)
        return cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    def _data_gen(self, samples):
        ''' Generator to return batches of training / validation data
        '''
        batch_size = self._batch_size
        num_samples = len(samples)
        while 1: # Loop forever so the generator never terminates
            shuffle(samples)
            images = []
            cmds = []
            for i in range(batch_size):
                image_center = self._read_image(samples[i][0])
                angle_center, throttle, brake = [float(s) for s in samples[i][3:6]]
                images.append(image_center)
                cmds.append([angle_center, throttle, brake])

                image_center_flipped = np.fliplr(image_center)
                angle_center_flipped = -angle_center
                images.append(image_center_flipped)
                cmds.append([angle_center_flipped, throttle, brake])

                image_left = self._read_image(samples[i][1])
                angle_left = angle_center + self._correction
                images.append(image_left)
                cmds.append([angle_left, throttle, brake])

                image_right = self._read_image(samples[i][2])
                angle_right = angle_center - self._correction
                images.append(image_right)
                cmds.append([angle_right, throttle, brake])

            X_train = np.array(images)
            y_train = np.array(cmds)

            yield shuffle(X_train, y_train)

    def train_data_len(self):
        return len(self._train_samples) / self._batch_size

    def valid_data_len(self):
        return len(self._valid_samples) / self._batch_size

#############################

if __name__ == '__main__':
    import matplotlib.pyplot as plt

    img_gen = ImageGenerator(r'.\data', ['set1'], batch_size=1)
    train_gen = img_gen.train_data_gen()
    img,angle = next(train_gen)

    for i in img:
        plt.imshow(i)
        plt.show()


import scipy as sp
from scipy.sparse import spdiags
from scipy.sparse import lil_matrix
from scipy.sparse import csr_matrix
from scipy.sparse.linalg import spsolve

def makeStructureMatrix( n ):
    e = sp.ones(n)
    A = spdiags([e, -4*e, 6*e,-4*e, e],[-2,-1,0,1,2],n,n)
    A = lil_matrix(A)
    B = csr_matrix([[16.0,-9.0,8.0/3.0,1.0/4.0],
                    [16.0/17.0,-60.0/17.0,72.0/17.0,-28.0/17],
                    [-12.0/17.0,96.0/17,-156.0/17.0,72.0/17.0]])

    A[0, 0:4] = B[0, :]
    A[n-2, n-4:n] = B[1, :]
    A[n-1, n-4:n] = B[2, :]
    return csr_matrix(A)

def getB(n):
    g = -9.81 #gravity constant earth (negative)
    L = 2.0 #length
    w = 0.30 #width
    d = 0.03 #depth
    density = 480.0 #density of material

    E = 1.3*10.0**10 #Young's modulus
    I = (w*d**3.0)/12.0 #inertia around the center of mass
    m = density*w*d #mass per meter of beam
    f = m*g #downward force per meter of beam
    h = L/n

    print(f)

    b = [(h**4/(E*I))*f] *n

    return b

n = 10
A = makeStructureMatrix(n)
b = getB(n)
print(A.toarray())
print(b)
#b =
y = spsolve(A, b)
print(y)

for i in range(11):
    n = 20 * 2**i

    A = makeStructureMatrix(n)
    b = getB(n)
    print(A.toarray())
    print(b)
    #b =
    y = spsolve(A, b)


#!/usr/bin/env python
"""Python 2.5 script. Creates a Notification pop-up bubble"""
import dbus
item              = "org.freedesktop.Notifications"
path              = "/org/freedesktop/Notifications"
interface         = "org.freedesktop.Notifications"
app_name          = "Test Application"
id_num_to_replace = 0
icon              = "/usr/share/icons/Tango/32x32/status/sunny.png"
title             = "Notification Title"
text              = "This is the body"
actions_list      = ''
hint              = ''
time              = 5000   # Use seconds x 1000

bus = dbus.SessionBus()
notif = bus.get_object(item, path)
notify = dbus.Interface(notif, interface)
notify.Notify(app_name, id_num_to_replace, icon, title, text, actions_list, hint, time)


from sikuli import *
import logging
import csv
import myTools

#---------------------------------------------------#
def fSetup_TemplateRefs():
#---------------------------------------------------#

    logging.debug('- fSetup_TemplateRefs')

# switch to template clients
    rightClick("02351_nn2.png")
    type("t")
    time.sleep(1)

    refDataFile = Settings.dataFolder + "\\templateRefs.csv"
    allTemplateRefs = csv.DictReader(open(refDataFile))

    for tempRef in allTemplateRefs:

        logging.debug('-- create reference: ' + tempRef["nn1"])
        type("n",KeyModifier.CTRL)

        type(tempRef["nn1"])
        type(Key.TAB)

        type(tempRef["nn2"])
        type(Key.ENTER)
        time.sleep(1)

#---------------------------------------------------#
def fExport_TemplateSetting():
#---------------------------------------------------#

    logging.debug('- fExport_TemplateSetting')

# Also use Default template
    logging.debug('-- open Client Reference Setup')
    type(Key.HOME)
    click(Pattern("reference_setup.png").targetOffset(6,10))

# Also use template
    logging.debug('-- choose Default template')
    type("t", KeyModifier.ALT)
    type("d")
    time.sleep(1)

# Export
    logging.debug('-- export settings')
    type("e", KeyModifier.ALT)
    time.sleep(1)

# choose all clients

    if int(Settings.tsVersion) > 2014:
        myTools.pressTAB(1)
        type(Key.INSERT)
    else:
        click(Pattern("acton_client.png").targetOffset(-7,-6))
        type(Key.END, KeyModifier.SHIFT)
        type(Key.SPACE)
    time.sleep(1)

# click Export
    type(Key.TAB)
    type(Key.SPACE)

# click YES
    time.sleep(1)
    type(Key.ENTER)

# click OK
    logging.debug('-- export successful')
    wait("export_successful.png", FOREVER)
    type(Key.ENTER)

# click OK
    logging.debug('-- close setup')
    type("d",KeyModifier.ALT)
    type(Key.ENTER)

#---------------------------------------------------#
def fCreate_Refs():
#---------------------------------------------------#

    myTools.sectionStartTimeStamp("create refs")
    logging.debug('Create_Refs')

    # make sure timeslips has focus
    myTools.getFocus()

    logging.debug('- open reference list')
    type("r", KeyModifier.CTRL + KeyModifier.SHIFT)
    time.sleep(1)

    fExport_TemplateSetting()
    fSetup_TemplateRefs()

    type(Key.F4,KeyModifier.CTRL)

    myTools.sectionEndTimeStamp()


# -*- coding: utf-8 -*-

from datetime import datetime

from alascrapy.spiders.base_spiders.ala_spider import AlaSpider
import alascrapy.lib.dao.incremental_scraping as incremental_utils

class Whathifi_comSpider(AlaSpider):
    name = 'whathifi_com'
    allowed_domains = ['www.whathifi.com']
    start_urls = ['https://www.whathifi.com/products/headphones',
                  'https://www.whathifi.com/products/tvs',
                  'https://www.whathifi.com/products/tablets-and-smartphones',
                  'https://www.whathifi.com/products/home-cinema',
                  'https://www.whathifi.com/products/portable',
                  'https://www.whathifi.com/products/digital-tv-boxes',
                  'https://www.whathifi.com/products/all-in-one-systems',
                  'https://www.whathifi.com/products/games-consoles',
                  'https://www.whathifi.com/products/streaming',
                  'https://www.whathifi.com/products/hi-fi', ]

    def __init__(self, *args, **kwargs):
        super(Whathifi_comSpider, self).__init__(self, *args, **kwargs)
        self.stored_last_date = incremental_utils.get_latest_pro_review_date(
            self.mysql_manager, self.spider_conf["source_id"])

        if not self.stored_last_date:
            self.stored_last_date = datetime(1970, 1, 1)

    def parse(self, response):
        ocn_xpath = '//h1/text()'
        ocn = self.extract(response.xpath(ocn_xpath))

        category = ocn.split(' reviews')[0]

        urls_xpath = '//div[@data-page="1"]/a/@href'
        urls = self.extract_list(response.xpath(urls_xpath))

        for url in urls:
            yield response.follow(url, callback=self.parse_page,
                                  meta={'category': category})

    def parse_page(self, response):
        product_xpaths = {
            'PicURL': '//meta[@property="og:image"]/@content',
            'TestUrl': '//meta[@property="og:url"]/@content'
        }

        review_xpaths = {
            'TestSummary': '//meta[@property="og:description"]/@content',

            'Author': '//span[@class="no-wrap by-author"]//text()',

            'TestDateText': 'substring-before(//time/@datetime, "T")',

            'TestUrl': '//meta[@property="og:url"]/@content',
            'TestVerdict': '//div[@class="sub-box full"]/p/text()',
            'TestPros': '//div[@class="sub-box"][1]/ul/li/text()',
            'TestCons': '//div[@class="sub-box"][2]/ul/li/text()',
        }

        product = self.init_item_by_xpaths(response, 'product', product_xpaths)
        review = self.init_item_by_xpaths(response, 'review', review_xpaths)

        title_xpath = '//h1/text()'
        title = self.extract(response.xpath(title_xpath))
        productname = title.split(' review')[0]
        if 'Hands on:' in productname:
            productname = productname.split(': ')[1]
        else:
            productname = productname

        category = response.meta.get('category')
        product['OriginalCategoryName'] = category

        review['TestTitle'] = title
        review['ProductName'] = productname
        product['ProductName'] = productname

        rating_xpath = '//meta[@itemprop="ratingValue"]/@content'
        rating = self.extract(response.xpath(rating_xpath))

        SCALE = 5

        if rating:
            review['SourceTestRating'] = rating
            review['SourceTestScale'] = SCALE

        review['DBaseCategoryName'] = 'PRO'
        source_int_id = response.url
        source_internal_id = source_int_id.split('/')[-1]

        review['source_internal_id'] = source_internal_id
        product['source_internal_id'] = source_internal_id

        review_date = datetime.strptime(review['TestDateText'], "%Y-%m-%d")

        if review_date > self.stored_last_date:
            yield review
            yield product


"""Test module for data_interface"""

import os
from dataclasses import dataclass
import unittest
import pandas as pd
from torch.utils.data import Dataset
from pymarlin.core.data_interface import DataInterface, DataProcessor

@dataclass
class MyArgs:
    filepath_train: str = os.path.join("outputs", "file1.csv")
    filepath_test: str = os.path.join("outputs", "file2.csv")
    text_field: str = "text"
    label_field: str = "label"

class MyDataset(Dataset):
    def __init__(self, df, text_field, label_field):
        self.df = df
        self.text_field = text_field
        self.label_field = label_field

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        row = self.df.iloc[idx]
        return (row[self.text_field], row[self.label_field])

class MyData(DataInterface):

    def __init__(self):
        self._train_ds = None
        self._val_ds = None

    def setup_datasets(self, train_ds, val_ds):
        self._train_ds = train_ds
        self._val_ds = val_ds

    def get_train_dataset(self):
        return self._train_ds

    def get_val_dataset(self):
        return self._val_ds

class MyDataProcessor(DataProcessor):

    def __init__(self, args):
        super().__init__()
        self.args = args

    def process(self):
        df = pd.read_csv(self.args.filepath_train)
        return MyDataset(df, self.args.text_field,
                         self.args.label_field)

class MyDataMultiProcessor(DataProcessor):

    def __init__(self, args):
        super().__init__()
        self.args = args

    def process(self, filename):
        df = pd.read_csv(filename)
        return MyDataset(df, self.args.text_field,
                         self.args.label_field)

class TestDataInterface(unittest.TestCase):

    def setUp(self):

        self.args = MyArgs()
        self.data_interface = MyData()
        self.data_processor = MyDataProcessor(self.args)
        self.data_multiprocessor = MyDataMultiProcessor(self.args)

        if not os.path.exists("outputs"):
            os.makedirs("outputs")

    def tearDown(self):
        for f in os.listdir("outputs"):
            if f.endswith(".csv"):
                os.remove(os.path.join("outputs", f))

    def test_process_data(self):
        df = pd.DataFrame({self.args.text_field: ['one', 'two'],
                           self.args.label_field: [1, 2]})
        df.to_csv(self.args.filepath_train)

        train_ds = self.data_processor.process_data()
        assert train_ds[0] == ('one', 1)
        assert len(train_ds) == 2

    def test_multi_process_data(self):
        df1 = pd.DataFrame({self.args.text_field: ['one', 'two'],
                            self.args.label_field: [1, 2]})
        df2 = pd.DataFrame({self.args.text_field: ['three', 'four', 'five'],
                            self.args.label_field: [3, 4, 5]})
        df1.to_csv(self.args.filepath_train)
        df2.to_csv(self.args.filepath_test)

        train_ds_list = self.data_multiprocessor.multi_process_data(
            [self.args.filepath_train, self.args.filepath_test],
            process_count=2)
        assert train_ds_list[0] == ('one', 1)
        assert train_ds_list[2] == ('three', 3)
        assert len(train_ds_list) == 5

    def test_collect_params(self):
        a_number = 1
        single_argument = 'single_argument'
        list_to_split = ['first', 'second', 'third']
        a_dict = {'test': 'dict'}
        list_not_to_split = [['this', 'lists', 'elements', 'dont', 'split']]
        self.data_processor._set_ranks()
        list_params = self.data_processor._collect_params(
            a_number,
            single_argument,
            list_to_split,
            a_dict,
            list_not_to_split
        )
        assert len(list_params) == 5
        assert list_params[2][0] == list_to_split[0]
        assert list_params[2][1] == list_to_split[1]
        assert list_params[2][2] == list_to_split[2]

        for param in list_params[1]:
            assert param == single_argument

        for param in list_params:
            assert len(param) == 3


"""Plugin for plex media server (www.plexapp.com)."""
from xml.dom.minidom import parse, parseString
import re
import logging
from flexget.utils import requests
from flexget.plugin import register_plugin, PluginError
from flexget.entry import Entry
from socket import gethostbyname
from os.path import basename

log = logging.getLogger('plex')

class InputPlex(object):
    """
    Uses a plex media server (www.plexapp.com) tv section as an input.

    'section'           Required parameter, numerical (/library/sections/<num>) or section name.
    'selection'         Can be set to different keys:
        - all
        - unwatched
        - recentlyAdded
        - recentlyViewed
        - recentlyViewedShows
      'all' and 'recentlyViewedShows' will only produce a list of show names while the other three will produce
      filename and download url.
    'username'          Myplex (http://my.plexapp.com) username, used to connect to shared PMS'.
    'password'          Myplex (http://my.plexapp.com) password, used to connect to shared PMS'.
    'server'            Host/IP of PMS to connect to.
    'lowercase_title'   Convert filename (title) to lower case.
    'strip_year'        Remove year from title, ex: Show Name (2012) 01x01 => Show Name 01x01
    'original_filename' Use filename stored in PMS instead of transformed name. lowercase_title and strip_year
                        will be ignored.

    Default paramaters:
      server           : localhost
      port             : 32400
      selection        : all
      lowercase_title  : no
      strip_year       : yes
      original_filename: no

    Example:

      plex:
        server: 192.168.1.23
        section: 3
        selection: recentlyAdded
    """

    def validator(self):
        from flexget import validator
        config = validator.factory('dict')
        config.accept('text', key='server')
        config.accept('text', key='selection')
        config.accept('integer', key='port')
        config.accept('text', key='section', required=True)
        config.accept('integer', key='section', required=True)
        config.accept('text', key='username')
        config.accept('text', key='password')
        config.accept('boolean', key='lowercase_title')
        config.accept('boolean', key='strip_year')
        config.accept('boolean', key='original_filename')
        return config

    def prepare_config(self, config):
        config.setdefault('server', '127.0.0.1')
        config.setdefault('port', 32400)
        config.setdefault('selection', 'all')
        config.setdefault('username', '')
        config.setdefault('password', '')
        config.setdefault('lowercase_title', False)
        config.setdefault('strip_year', True)
        config.setdefault('original_filename', False)
        return config

    def on_task_input(self, task, config):
        config = self.prepare_config(config)
        accesstoken = ""
        plexserver = config['server']
        if gethostbyname(config['server']) != config['server']:
            config['server'] = gethostbyname(config['server'])
        if config['username'] and config['password'] and config['server'] != '127.0.0.1':
            header = {'X-Plex-Client-Identifier': 'flexget'}
            log.debug("Trying to to connect to myplex.")
            try:
                r = requests.post('https://my.plexapp.com/users/sign_in.xml',
                    auth=(config['username'], config['password']), headers=header)
            except requests.RequestException as e:
                raise PluginError('Could not login to my.plexapp.com: %s. Username: %s Password: %s'
                    % (e, config['username'], config['password']))
            log.debug("Managed to connect to myplex.")
            if 'Invalid email' in r.text:
                raise PluginError('Could not login to my.plexapp.com: invalid username and/or password!')
            log.debug("Managed to login to myplex.")
            dom = parseString(r.text)
            plextoken = dom.getElementsByTagName('authentication-token')[0].firstChild.nodeValue
            log.debug("Got plextoken: %s" % plextoken)
            try:
                r = requests.get("https://my.plexapp.com/pms/servers?X-Plex-Token=%s" % plextoken)
            except requests.RequestException as e:
                raise PluginError('Could not get servers from my.plexapp.com using authentication-token: %s.'
                    % plextoken)
            dom = parseString(r.text)
            for node in dom.getElementsByTagName('Server'):
                if node.getAttribute('address') == config['server']:
                    accesstoken = node.getAttribute('accessToken')
                    log.debug("Got accesstoken: %s" % plextoken)
                    accesstoken = "?X-Plex-Token=%s" % accesstoken
            if accesstoken == "":
                raise PluginError('Could not retrieve accesstoken for %s.' % config['server'])
        if not isinstance(config['section'], int):
            try:
                r = requests.get("http://%s:%d/library/sections/%s" %
                    (config['server'], config['port'], accesstoken))
            except requests.RequestException as e:
                raise PluginError('Error retrieving source: %s' % e)
            dom = parseString(r.text.encode("utf-8"))
            for node in dom.getElementsByTagName('Directory'):
                if node.getAttribute('title') == config['section']:
                    config['section'] = int(node.getAttribute('key'))
        if not isinstance(config['section'], int):
            raise PluginError('Could not find section \'%s\'' % config['section'])
        try:
            r = requests.get("http://%s:%d/library/sections/%s/%s%s" %
                (config['server'], config['port'], config['section'], config['selection'], accesstoken))
        except requests.RequestException as e:
            raise PluginError('Error retrieving source: %s' % e)
        dom = parseString(r.text.encode("utf-8"))
        entries = []
        plexsectionname = dom.getElementsByTagName('MediaContainer')[0].getAttribute('title1')
        if dom.getElementsByTagName('MediaContainer')[0].getAttribute('viewGroup') == "show":
            for node in dom.getElementsByTagName('Directory'):
                title = node.getAttribute('title')
                if config['strip_year']:
                    title = re.sub(r'^(.*)\(\d+\)$', r'\1', title)
                title = re.sub(r'[\(\)]', r'', title)
                title = re.sub(r'\&', r'And', title)
                title = re.sub(r'[^A-Za-z0-9- ]', r'', title)
                if config['lowercase_title']:
                    title = title.lower()
                e = Entry()
                e['title'] = title
                e['url'] = "NULL"
                e['plexserver'] = plexserver
                e['plexport'] = config['port']
                e['plexsection'] = config['section']
                e['plexsectionname'] = plexsectionname
                entries.append(e)
        elif dom.getElementsByTagName('MediaContainer')[0].getAttribute('viewGroup') == "episode":
            for node in dom.getElementsByTagName('Video'):
                title = node.getAttribute('grandparentTitle')
                season = int(node.getAttribute('parentIndex'))
                if node.getAttribute('parentIndex') == node.getAttribute('year'):
                    season = node.getAttribute('originallyAvailableAt')
                    filenamemap = "%s_%s%s_%s_%s_%s.%s"
                    episode = ""
                elif node.getAttribute('index'):
                    episode = int(node.getAttribute('index'))
                    filenamemap = "%s_%02dx%02d_%s_%s_%s.%s"
                else:
                    log.debug("Could not get episode number for '%s' (Hint, ratingKey: %s)"
                        % (title, node.getAttribute('ratingKey')))
                    break
                for media in node.getElementsByTagName('Media'):
                    vcodec = media.getAttribute('videoCodec')
                    acodec = media.getAttribute('audioCodec')
                    container = media.getAttribute('container')
                    resolution = media.getAttribute('videoResolution') + "p"
                    for part in media.getElementsByTagName('Part'):
                        key = part.getAttribute('key')
                        e = Entry()
                        if config['original_filename']:
                            e['title'] = basename(part.getAttribute('file'))
                        else:
                            if config['strip_year']:
                                title = re.sub(r'^(.*)\(\d+\)$', r'\1', title)
                            title = re.sub(r'[\(\)]', r'', title)
                            title = re.sub(r'\&', r'And', title).strip()
                            title = re.sub(r'[^A-Za-z0-9- ]', r'', title).replace(" ", ".")
                            if config['lowercase_title']:
                                title = title.lower()
                            e['title'] = filenamemap % (title, season, episode, resolution, vcodec, acodec, container)
                        e['url'] = "http://%s:%d%s%s" % (config['server'], config['port'], key, accesstoken)
                        e['plex_server'] = plexserver
                        e['plex_server_ip'] = config['server']
                        e['plex_port'] = config['port']
                        e['plex_section'] = config['section']
                        e['plex_section_name'] = plexsectionname
                        e['plex_path'] = key
                        entries.append(e)
        else:
            raise PluginError('Selected section is not a TV section.')
        return entries

register_plugin(InputPlex, 'plex', api_ver=2)


import cv2
import os
import numpy as np
import utility
import matplotlib.pyplot as plt

img_dir = 'C:/Users/xiangpu/Downloads/Test/'
pos_dir = 'pos_imgs_Test/'
neg_dir = 'neg_imgs_Test/'
if not os.path.exists(pos_dir):
    os.mkdir(pos_dir)
if not os.path.exists(neg_dir):
    os.mkdir(neg_dir)

def on_click(event, x, y, flags, param):
    if event == cv2.EVENT_LBUTTONDOWN:
        xy = "x: %d, y: %d, value: %d" % (x, y, id_imgs[1][y, x])
        print(xy)

for root, dirs, files in os.walk(img_dir):
    print("there are {0} imgs in dir".format(len(files)))
    for img_idx in range(len(files)):
        print("process {0} img".format(img_idx))
        img = cv2.imread(img_dir + files[img_idx], cv2.IMREAD_GRAYSCALE)  # type:np.ndarray
        hist1 = cv2.calcHist([img], [0], None, [256], [0, 256])
        img_equHist = cv2.equalizeHist(img)
        hist2 = cv2.calcHist([img_equHist], [0], None, [256], [0, 256])
        x = cv2.Sobel(img_equHist, cv2.CV_16S, 1, 0)
        y = cv2.Sobel(img_equHist, cv2.CV_16S, 0, 1)
        absX = cv2.convertScaleAbs(x)
        absY = cv2.convertScaleAbs(y)
        img_sobel = cv2.addWeighted(absX, 1, absY, 1, 0)
        thresh, img_binary = cv2.threshold(img_sobel, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
        contours, hierarchy = cv2.findContours(img_binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
        # print("there are {0} contours".format(len(contours)))
        # i = 0
        # while i < hierarchy.shape[1]:
        #     if hierarchy[0][i][3] != -1:
        #         hierarchy = np.delete(hierarchy, i, axis=1)
        #         del contours[i]
        #         i -= 1
        #     i += 1
        contour_size = []
        for contour in contours:
            contour_size.append(contour.shape[0])
            # print(contour.shape, end=' ')
        # print('')
        sorted_size = sorted(enumerate(contour_size), key=lambda x: x[1])
        id_contours = []
        for i in range(1, len(sorted_size)):
            ratio = sorted_size[i - 1][1] / sorted_size[i][1]
            if 0.8 < ratio < 1.2:
                id_contours.append(contours[sorted_size[i - 1][0]])
                id_contours.append(contours[sorted_size[i][0]])
                break
        if len(id_contours) < 2:
            print("img {} failed".format(img_idx))
            continue
        rects = [cv2.minAreaRect(id_contours[i]) for i in range(2)]
        boxes = [cv2.boxPoints(rects[i]) for i in range(2)]
        for i in range(2):
            rects[i] = list(rects[i])
            rects[i][1] = list(rects[i][1])
            if rects[i][1][0] < rects[i][1][1]:
                rects[i][1][0], rects[i][1][1] = rects[i][1][1], rects[i][1][0]
                rects[i][2] += 90
        # print("size: {0}".format(rects[0][1]))
        show_contour = np.ones(img_binary.shape, np.uint8) * 255
        show_box = np.ones(img_binary.shape, np.uint8) * 255
        for i in range(2):
            for j in range(4):
                cv2.line(show_box, tuple(boxes[i][j]), tuple(boxes[i][(j + 1) % 4]), (0, 0, 0))
        img_clone = img.copy()
        cv2.drawContours(img_clone, id_contours, -1, 0, 1)

        rotation_img = [cv2.warpAffine(img, cv2.getRotationMatrix2D(rects[i][0], rects[i][2], 1), img_equHist.shape)
                        for i in range(2)]
        id_imgs = [rotation_img[i][int(rects[i][0][1] - rects[i][1][1] / 2):int(rects[i][0][1] + rects[i][1][1] / 2),
                   int(rects[i][0][0] - rects[i][1][0] / 2):int(rects[i][0][0] + rects[i][1][0] / 2)] for i in range(2)]
        # cv2.namedWindow("img1")
        # cv2.setMouseCallback("img1", on_click)
        id_filter = [cv2.medianBlur(id_imgs[i], 81)[40:-40, 40:-40] for i in range(2)]
        if id_filter[0].min() < id_filter[1].min():
            id_filter[0], id_filter[1] = id_filter[1], id_filter[0]
            id_imgs[0], id_imgs[1] = id_imgs[1], id_imgs[0]
        # print(id_filter[0].min())
        # print(id_filter[1].min())
        logo_index = np.where(id_filter[1] == id_filter[1].min())
        logo_index = [logo_index[i][len(logo_index[i]) // 2] for i in range(2)]
        # print("logo pos {0}".format(logo_index))
        if logo_index[1] > id_imgs[1].shape[1] / 2:
            id_imgs[1] = cv2.warpAffine(id_imgs[1],
                                        cv2.getRotationMatrix2D((id_imgs[1].shape[1] // 2, id_imgs[1].shape[0] // 2),
                                                                180, 1), id_imgs[1].shape[::-1])
        mser = cv2.MSER_create(_delta=5, _min_area=10, _max_variation=0.7, _max_area=200)
        regions, boxes = mser.detectRegions(id_imgs[0])
        hulls = [cv2.convexHull(p.reshape(-1, 1, 2)) for p in regions]
        # cv2.polylines(id_imgs[0], hulls, 1, (0, 255, 0), 1)
        keep = []
        for hull in hulls:
            x, y, w, h = cv2.boundingRect(hull)
            keep.append([x, y, x + w, y + h])
        pick = utility.non_max_suppression_fast(np.array(keep), overlapThresh=0.4)
        mser_sum = 0
        for (startX, startY, endX, endY) in pick:
            # cv2.rectangle(id_imgs[0], (startX, startY), (endX, endY), (0, 0, 255), 1)
            center = ((startX + endX) // 2, (startY + endY) // 2)
            if center[0] > id_imgs[0].shape[1] - 150 and id_imgs[0].shape[0] - 25 < center[1] < id_imgs[0].shape[0] - 10:
                # cv2.circle(id_imgs[0], center, 2, (255, 0, 0))
                mser_sum += 1
        if mser_sum > 5:
            id_imgs[0] = cv2.warpAffine(id_imgs[0],
                                        cv2.getRotationMatrix2D((id_imgs[0].shape[1] // 2, id_imgs[0].shape[0] // 2),
                                                                180, 1), id_imgs[0].shape[::-1])
        # cv2.imshow("img1", id_imgs[0])
        # cv2.imshow("img2", id_imgs[1])
        # cv2.imshow("boxs", show_box)
        # cv2.imshow("img_equhist", img_equHist)
        # cv2.imshow("img_sobel", img_sobel)
        # cv2.imshow("img_binary", img_binary)
        # cv2.imshow("contour", img_clone)
        # cv2.waitKey(0)
        cv2.imwrite(pos_dir + "pos_" + files[img_idx], id_imgs[0])
        cv2.imwrite(neg_dir + "neg_" + files[img_idx], id_imgs[1])
        # if img_idx > 500:
        #     break
    break


#!/usr/bin/env python
#!-*-coding:utf-8 -*-
"""
@version: python3.7
@author: v-enshi
@license: Apache Licence
@contact: 123@qq.com
@site:
@software: PyCharm
@file: model_traditional_attn.py
@time: 2019/4/26 17:51
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.nn.utils.rnn as rnn_utils

import random
import numpy as np

torch.manual_seed(1)
use_gpu = False
#use_gpu = True

##1. parameters setting
if use_gpu:
    device = torch.device("cuda")
else:
    device = torch.device("cpu")

## 3.1 LSTM component
class LSTM_component(nn.Module):

    def __init__(self, vocab_value_size, value_dim, vocab_type_size, type_dim,
                 hidden_dim, batch_size, context_window, dropout_p=0.5):
        super(LSTM_component, self).__init__()
        self.hidden_dim = hidden_dim
        self.batch_size = batch_size
        self.context_window = context_window
        self.value_embeddings = nn.Embedding(vocab_value_size, value_dim, padding_idx=0)

        self.type_embeddings = nn.Embedding(vocab_type_size, type_dim, padding_idx=0)
        self.dropout = nn.Dropout(dropout_p)
        self.lstm = nn.LSTM(value_dim + type_dim, hidden_dim, batch_first=True)

    def forward(self, sentence, setence_len, hc):
        embeds_type = self.type_embeddings(sentence[0])

        embeds_value = self.value_embeddings(sentence[1])
        embeds = torch.cat([embeds_value, embeds_type], 2)
        h0 = hc[0]
        c0 = hc[1]

        embeded = rnn_utils.pack_padded_sequence(embeds, setence_len, batch_first=True)

        lstm_out, (lstm_h, lstm_c) = self.lstm(embeded, (h0, c0))
        lstm_out, _ = torch.nn.utils.rnn.pad_packed_sequence(lstm_out, batch_first=True)

        return lstm_out, lstm_h, lstm_c

    def initHidden(self):
        return torch.randn(1, self.batch_size, self.hidden_dim, device=device), torch.randn(1, self.batch_size,
                                                                                self.hidden_dim,
                                                                                device=device)

'''

#model_LSTM = LSTM_component(20,EMBEDDING_value,10,EMBEDDING_type,HIDDEN_SIZE, BATCH_SIZE, CONTEXT_WINDOW)

model_LSTM = LSTM_component(20,2,20,3,5, 2, 3)
with torch.no_grad():
    inputs = [torch.tensor([[6, 6, 5, 3], [7, 4, 0, 0]]), torch.tensor([[3, 3, 2, 1], [4, 1, 0, 0]])]
    output, hn,cn = model_LSTM(inputs,[4,2],model_LSTM.initHidden())
    #print("tag_scorces", output)
'''

## 3.2 attention component
class Context_atten(nn.Module):
    def __init__(self, hidden_dim, context_window, dropout_p=0.25):
        super(Context_atten, self).__init__()
        self.hidden_dim = hidden_dim
        self.context_window = context_window

        self.attn = nn.Linear(self.hidden_dim * 2, hidden_dim)
        self.v = nn.Parameter(torch.rand(hidden_dim))
        self.linear1 = nn.Linear(hidden_dim, hidden_dim, bias=False)

    def forward(self, inputs, setence_len, hc, parent):
        batch_size = inputs.size(0)
        timestep = self.context_window

        Mt = inputs[0, setence_len[0] - timestep: setence_len[0], :].unsqueeze(0)
        par = inputs[0, setence_len[0] - parent[0] - 1, :].unsqueeze(0)

        for i in range(batch_size - 1):
            Mt = torch.cat([Mt, inputs[i + 1, setence_len[i + 1] - timestep: setence_len[i + 1], :].unsqueeze(0)],
                           0)  # [B,T,H]
            par = torch.cat([par, inputs[i + 1, setence_len[i + 1] - parent[i + 1] - 1, :].unsqueeze(0)], 0)  # [B,H]

        h = hc.repeat(timestep, 1, 1).transpose(0, 1)  # hc[1,B,H] -> [B,T,H]
        attn_weights = self.score(h, Mt).unsqueeze(1)  # [B*T*1]
        context = attn_weights.bmm(Mt)  # (B,1,H)
        return attn_weights, context, par

    def score(self, hidden, attn_outputs):
        energy = torch.tanh(self.attn(torch.cat([hidden, attn_outputs], 2)))  # [B*T*2H]->[B*T*H]
        energy = energy.transpose(1, 2)  # [B*H*T]
        v = self.v.repeat(attn_outputs.size(0), 1).unsqueeze(1)  # [B*1*H]
        energy = torch.bmm(v, energy)  # [B*1*T] a
        energy = F.softmax(energy, dim=2)
        return energy.squeeze(1)  # [B*T]

'''
    def score(self, hidden, attn_outputs):
        # [B*T*2H]->[B*T*H]
        energy = F.softmax(self.attn(torch.cat([hidden, attn_outputs], 2)),dim = 2)
        energy = energy.transpose(1, 2)  # [B*H*T]
        v = self.v.repeat(attn_outputs.size(0), 1).unsqueeze(1)  # [B*1*H]
        energy = torch.bmm(v, energy)  # [B*1*T] a
        return energy.squeeze(1)  # [B*T]

'''

'''
HIDDEN_SIZE = 5
CONTEXT_WINDOW = 2
model_catten = Context_atten(HIDDEN_SIZE, CONTEXT_WINDOW)

with torch.no_grad():
    alpha , out_ct,parent = model_catten (output,[3,2], hn,[0,1])
    #print(out_ct)

'''

## 3.3parent attention
class Parent_atten(nn.Module):
    def __init__(self, hidden_dim, context_window):
        super(Parent_atten, self).__init__()
        self.hidden_dim = hidden_dim
        self.context_window = context_window
        self.Wg_linear = nn.Linear(hidden_dim * 3, hidden_dim, bias=False)
        self.Wv_linear = nn.Linear(hidden_dim, self.context_window)

    def forward(self, ht, ct, pt):
        # ht[1,B,H] ct[B,1,H] pt[B,H]
        ct = ct.transpose(0, 1)  # (1,B,H)
        pt = pt.unsqueeze(0)

        Gt = torch.tanh(self.Wg_linear(torch.cat([ht, ct, pt], 2)))
        yt = F.log_softmax(self.Wv_linear(Gt), dim=1)
        return yt

'''
model_par_atten_type = Parent_atten(HIDDEN_SIZE, CONTEXT_WINDOW)
with torch.no_grad():
    Yt_type = model_par_atten_type  ( hn,out_ct,parent)
    print(Yt_type)

'''

## 3 main model
class main_model(nn.Module):
    def __init__(self, vocab_value_size, value_dim, vocab_type_size, type_dim,
                 hidden_dim, batch_size, context_window, dropout_p=0.25):
        super(main_model, self).__init__()
        self.hidden_dim = hidden_dim
        self.batch_size = batch_size
        self.context_window = context_window

        self.model_LSTM = LSTM_component(vocab_value_size, value_dim, vocab_type_size, type_dim, hidden_dim, batch_size,
                                         context_window).to(device)
        self.model_catten = Context_atten(hidden_dim, context_window).to(device)
        self.model_par_atten_value = Parent_atten(hidden_dim, vocab_value_size).to(device)

    def forward(self, inputs, input_len, hc, parent):
        output, hn, cn = self.model_LSTM(inputs, input_len, hc)
        alpha, out_ct, par = self.model_catten(output, input_len, hn, parent)
        Yt = self.model_par_atten_value(hn, out_ct, par)
        return Yt

    def initHidden(self):
        # return torch.zeros(1, self.batch_size, self.hidden_dim, device=device), torch.zeros(1, self.batch_size,self.hidden_dim, device=device)
        return torch.randn(1, self.batch_size, self.hidden_dim, device=device), torch.randn(1, self.batch_size,
                                                                                self.hidden_dim,
                                                                                device=device)


from packages.CRITICAL.CLIENT import client
import discord
from commands.data import FOOTER, BOT_CHANNELS

@client.command(pass_context=True, aliases=["Ping"])
async def ping(ctx):
	if ctx.channel.name in BOT_CHANNELS.channels:
		pong1 = (round(client.latency * 1000))

		if pong1 >= 500:
			update_embed = discord.Embed(
				colour=discord.Colour.red(),
				title="",
				description=f"Pong!: {pong1}ms\nSpeeds are not looking good, Please ask a staff member or dev to restart the bot.")
			await ctx.send(embed = update_embed)

		elif pong1 >= 300:
			update_embed = discord.Embed(
				colour=discord.Colour.gold(),
				title="",
				description=f"Pong!: {pong1}ms\nSpeeds are looking slow...")
			await ctx.send(embed = update_embed)

		else:
			update_embed = discord.Embed(
				colour=discord.Colour.green(),
				title="",
				description=f"Pong!: {pong1}ms\nSpeeds are looking fast!")
			await ctx.send(embed = update_embed)


'''
Data Mining Project

Print topics found in Yelp business reviews

If training is specified, dataset will be read from disk and model will be trained.
Otherwise, pickled model will be read from disk.
'''

import argparse
from collections import defaultdict
import json
import pickle
import os
import sys
from time import time

import gensim
from gensim.utils import simple_preprocess
from gensim.parsing.preprocessing import STOPWORDS
import nltk
from nltk.stem import WordNetLemmatizer, SnowballStemmer
from nltk.stem.porter import *
import numpy as np
np.random.seed(400)

PICKLED_MODEL_FILENAME = 'lda_model.p'
PICKLED_RESTAURANT_REVIEW_DATA_FILENAME = 'restaurants.p'

default_dataset_path = os.path.join(
	os.path.dirname(__file__),
	'dataset',
)

parser = argparse.ArgumentParser()

parser.add_argument(
	'-t', '--train',
	action='store_true',
	help='train new model (Yelp dataset must also be present on disk))',
)

parser.add_argument(
	'-d', '--dataset',
	default=default_dataset_path,
	help='specify absolute path to dataset',
)

parser.add_argument(
	'-n', '--num_topics',
	default=14,
	help='number of topics to train LDA on',
)

# Write functions to perform the preprocessing
stemmer = SnowballStemmer('english')
def lemmatize_stemming(text):
    return stemmer.stem(WordNetLemmatizer().lemmatize(text, pos='v'))

# Tokenize and lemmatize
def preprocess(text):
    result=[]
    for token in simple_preprocess(text, deacc=True, min_len=4):
        if token not in STOPWORDS:
            result.append(lemmatize_stemming(token))
    return result

if __name__ == '__main__':
	args = parser.parse_args()
	arg_dict = vars(args)

	if arg_dict['train']:

		# try to grab pickled version of dataset, if possible
		if not os.path.exists(PICKLED_RESTAURANT_REVIEW_DATA_FILENAME):
			print('importing dataset...')

			dataset_path = arg_dict['dataset']

			try:
				datasets = {
					f.split('.')[0]: dataset_path + os.sep + f for f in filter(lambda x: x.endswith('.json'), os.listdir(dataset_path))
				}
			except:
				print('Could not find dataset at {}'.format(dataset_path))
				sys.exit(0)

			# organize data to be used in training model
			restaurants = defaultdict(dict)

			with open(datasets['business']) as f:
				for line in f:
					contents = json.loads(line)
					if contents['categories'] and 'Restaurants' in contents['categories']:
						restaurants[contents['business_id']]['categories'] = contents['categories']
						restaurants[contents['business_id']]['reviews'] = []

			with open(datasets['review']) as f:
				for line in f:
					contents = json.loads(line)
					if contents['business_id'] and contents['business_id'] in restaurants:
						restaurants[contents['business_id']]['reviews'].append(contents['text'])

			all_the_reviews = []
			for v in restaurants.values():
				all_the_reviews += v['reviews']

			print('Saving pickled dataset to disk as {}'.format(PICKLED_RESTAURANT_REVIEW_DATA_FILENAME))
			pickle.dump(all_the_reviews, open(PICKLED_RESTAURANT_REVIEW_DATA_FILENAME, 'wb'))

		else:
			print('Using pickled dataset file found on disk as {}'.format(PICKLED_RESTAURANT_REVIEW_DATA_FILENAME))
			all_the_reviews = pickle.load(open(PICKLED_RESTAURANT_REVIEW_DATA_FILENAME, 'rb'))

		# grab wordnet if not present for preprocessing data
		nltk.download('wordnet')

		print('Preview of document after preprocessing')

		doc_sample = all_the_reviews[0]

		print("Original document: ")
		words = [word for word in doc_sample.split(' ')]
		print("\nBefore preprocessing: ")
		print(words)
		print("\nTokenized and lemmatized document: ")
		print(preprocess(doc_sample))

		# Need to split reviews into training and testing
		training_reviews = all_the_reviews[:50000]
		processed_reviews = [preprocess(review) for review in training_reviews]

		'''
		Bag of words on the dataset
		Create a dictionary from 'processed_docs' containing the number of times a word appears
		in the training set. To do that, we pass processed_docs to gensim.corpora.Dictionary()
		'''
		dictionary = gensim.corpora.Dictionary(processed_reviews)

		'''
		Filter out tokens that appear only a few times or in more than 10% of documents
		'''
		dictionary.filter_extremes(
			no_below=0.001*len(processed_reviews),
			no_above=0.1,
		)

		'''
		Create the Bag-of-words model for each document i.e for each document we create a dictionary reporting how many
		words and how many times those words appear. Save this to 'bow_corpus'
		'''
		bow_corpus = [dictionary.doc2bow(review) for review in processed_reviews]

		'''
		Preview BOW for our sample preprocessed document
		'''
		print('\nBag of Words Preview on example document:')
		print(training_reviews[25])

		for doc in bow_corpus[25]:
		    print("Word {} (\"{}\") appears {} time.".format(doc[0], dictionary[doc[0]], doc[1]))

		'''
		Running LDA using Bag of Words
		We are going for 10 topics in the document corpus.

		We will be running LDA using 8 CPU cores to parallelize and speed up model training.
		'''
		num_topics = int(arg_dict['num_topics'])
		print('\nBuilding LDA model with {} topics...'.format(num_topics))
		lda_model =  gensim.models.LdaMulticore(
			bow_corpus,
			num_topics=num_topics,
			id2word=dictionary,
			passes=2,
			workers=8,
		)

		# serialize and save LDA model object to disk
		if not os.path.exists(PICKLED_MODEL_FILENAME):
			print('\nsaving pickled model to disk {}'.format(PICKLED_MODEL_FILENAME))
			pickle.dump(lda_model, open(PICKLED_MODEL_FILENAME, 'wb'))

	else:
		try:
			lda_model = pickle.load(open(PICKLED_MODEL_FILENAME, 'rb'))
		except:
			print('\nPickled LDA model not found on disk as {}... try running program with -t option to train new model!'.format(PICKLED_MODEL_FILENAME))
			sys.exit(0)

	# print discovered topics to screen
	for i, topic in lda_model.print_topics(
		num_topics=-1,
		num_words=20,
	):
	    print("Topic: {} \nWords: {}".format(i, topic))


# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Response
from Jobbole.items import JobboleItem

class JobboleSpider(scrapy.Spider):
    #爬虫文件的名称
    name = 'jobbole'
    #设置爬虫允许爬取的域，可以设置多个 （是一个list）
    allowed_domains = ['jobbole.com']
    #设置起始的url地址，可以设置多个（是一个list）
    start_urls = ['http://blog.jobbole.com/all-posts/']

    #parse,获取响应结果的回调方法
    def parse(self, response):
        """
        :param response: 请求的响应结果
        :return:
        """
        print('请求成功，获取到了页面源码')
        #获取响应的状态码
        code = response.status
        # # 获取响应的页面源码
        # html = response.text
        # 获取当前请求的url地址
        url = response.url
        #获取响应的二进制数据
        # b_html = response.body
        # #获取响应头
        # response_headers = response.headers
        # #获取当前请求的request对想
        # request = response.request
        # #获请求的请求头
        # request_headers = response.request.headers

        print(code, url)
        # 提取目标数据
        # 获取文章列表
        article_divs = response.xpath('//div[@class="post floated-thumb"]')

        print(len(article_divs))

        for article_div in article_divs:
            articleItem = JobboleItem()
            # title(标题)
            # title = article_div.xpath('.//a[@class="archive-title"]/text()').extract()[0]
            #extract_first():优势，如果没有取到返回 None，也可以自己设置一个默认值
            articleItem['title'] = article_div.xpath('.//a[@class="archive-title"]/text()').extract_first('')
            # # 封面图
            articleItem['coverImage'] = article_div.xpath('.//div[@class="post-thumb"]/a/img/@src').extract_first('暂无')
            print(articleItem)

            #获取详情页面的url地址
            detailUrl = article_div.xpath('.//a[@class="archive-title"]/@href').extract_first()
            if detailUrl:
                #存在则发起请求
                """
                 url, :要请求的url地址
                 callback=None, 请求成功的回调函数
                 method='GET', 设置请求的方式
                 headers=None, 设置请求头,dict类型
                 cookies=None, 设置用户的cookies信息 dict类型
                 meta=None, 传递参数 dict类型
                 encoding='utf-8', （编码类型，一般不需要改变）
                 priority=0, （设置优先级，不需要改变）
                 dont_filter=False, （是否过滤url,False表示过滤，True：表示不过滤）
                 errback=None ,设置请求错误的回调函数
                """
                yield scrapy.Request(
                    detailUrl,
                    callback=self.parse_article_data,
                    meta={'item':articleItem}
                )

        #获取下一页url地址，继续发起请求
        # next_url = response.xpath('//a[@class="next page-numbers"]/@href').extract_first()
        # if next_url:
        #     yield scrapy.Request(next_url,callback=self.parse)

    def parse_article_data(self,response):
        """
        文章详情，请求成功够的回调
        :param response:
        :return:
        """
        print(response.status,'文章详情获取成功')
        #取出meta传递过来的值
        articleItem = response.meta['item']

        # # source（来源）
        articleItem['source'] = response.xpath('//div[@class="copyright-area"]/a[1]/text()').extract_first('暂无出处')
        # # 内容
        articleItem['content'] = ';'.join(response.xpath('//div[@class="entry"]//p/text()').extract())
        # # 点赞量
        articleItem['zanNum'] = int(response.xpath('//span[@class=" btn-bluet-bigger href-style vote-post-up   register-user-only "]/h10/text()').extract_first('0'))
        # # 评论量
        # commentNum = scrapy.Field()
        # # 收藏
        # markNum = scrapy.Field()
        # # 发布时间
        publishTime = ','.join(response.xpath('//p[@class="entry-meta-hide-on-mobile"]/text()').extract()).replace(' ','').replace('\n','').replace('\r','')

        #注意这里使用的是yield
        #激活管道，只有激活了，管道文件才能接收到item
        yield articleItem
        # print(articleItem)


from collections import namedtuple
from text import symbols

def convert_to_objects(**kwargs):
    return namedtuple("hparams", kwargs.keys())(*kwargs.values())

def create_hparams(hparams_string=None, verbose=False):
    """Create model hyperparameters. Parse nondefault from given string."""

    hparams = convert_to_objects(
        ################################
        # Experiment Parameters        #
        ################################
        epochs=1500,
        iters_per_checkpoint=1000,
        seed=1234,
        dynamic_loss_scaling=True,
        fp16_run=False,
        distributed_run=False,
        dist_backend="nccl",
        dist_url="tcp://localhost:54321",
        cudnn_enabled=True,
        cudnn_benchmark=False,
        ignore_layers=['embedding.weight'],
        num_workers=6,

        ################################
        # Data Parameters             #
        ################################
        load_mel_from_disk=False,
        training_files='filelists/ljs_audio_text_train_filelist.txt',
        validation_files='filelists/ljs_audio_text_val_filelist.txt',
        text_cleaners=['english_cleaners'],

        ################################
        # Audio Parameters             #
        ################################
        max_wav_value=32768.0,
        sampling_rate=22050,
        filter_length=1024,
        hop_length=256,
        win_length=1024,
        n_mel_channels=80,
        mel_fmin=0.0,
        mel_fmax=8000.0,

        ################################
        # Model Parameters             #
        ################################
        # n_symbols + 1 for stop token
        n_symbols=(len(symbols)+2),
        symbols_embedding_dim=512,

        # Encoder parameters
        encoder_kernel_size=5,
        encoder_n_convolutions=3,
        encoder_embedding_dim=512,
        encoder_output_dim=512,

        # GST parameters
        gst_embedding_dim=128,
        gst_heads=4,
        gst_output_channels=[32, 32, 64, 64, 128, 128],
        gst_kernel_size=[3,3],
        gst_stride=[2,2],
        gst_padding=[1,1],

        # TPSE parameters
        tpse_rnn_size=256,

        # Duration parameters
        duration_rnn_dim=512,
        range_rnn_dim=512,
        duration_lambda=2.0,
        positional_embedding_dim=32,
        timestep_denominator=10000.0,
        lambda_duration=2.0,

        # Decoder parameters
        n_frames_per_step=1,  # currently only 1 is supported
        decoder_rnn_dim=1024,
        prenet_dim=256,
        max_decoder_steps=1000,
        gate_threshold=0.5,
        p_attention_dropout=0.1,
        p_decoder_dropout=0.1,

        # Attention parameters
        attention_rnn_dim=1024,
        attention_dim=128,

        # Location Layer parameters
        attention_location_n_filters=32,
        attention_location_kernel_size=31,

        # Mel-post processing network parameters
        postnet_embedding_dims=[512,512,512,512,128],
        postnet_kernel_size=5,
        postnet_n_convolutions=5,

        ################################
        # Optimization Hyperparameters #
        ################################
        use_saved_learning_rate=False,
        learning_rate=1e-3,
        weight_decay=1e-6,
        grad_clip_thresh=10.0,
        batch_size=128,
        mask_padding=True  # set model's padded outputs to padded values
    )

    return hparams


from django.contrib import admin
from .models import Users
# Register your models here.

@admin.register(Users)

class UserAdmin(admin.ModelAdmin):
    list_display = (
        'user_id',
        'user_pw',
        'user_name',
        'user_email',
    )


"""Implements a browser for claims."""

import logging

from pyramid.view import view_config

from arachcurator.models.model_utils import (
    get_term_label,
    get_citation,
    get_property_label,
    make_participant_choices,
    get_element_label,
    get_claim_participants,
    NOSPEC
)
from arachcurator.models.model_wrappers import Claim, Participant, ParticipantElement
from arachcurator.models.models import ParticipantLink, PropertyRow
from arachcurator.views.form_set import FormSet
from arachcurator.views.forms.ElementForm import TypeElementForm, InactiveElementEdit
from arachcurator.views.forms.ParticipantForm import ParticipantForm, InactiveParticipantEdit

@view_config(route_name='claim_browse',
             renderer='../templates/claim_browse.jinja2')
class Forms(FormSet):
    """Define an object that collects the schema and forms for the page."""

    def __init__(self, request):
        FormSet.__init__(self, request)

    def __call__(self):
        """Show a read-only view of a claim and associated participants.

        Includes some logic to handle choosing which associated
        participant to display.
        """
        ref_id = self.ref_id
        session = self.session
        request = self.request

        claim = Claim.load(ref_id, session)
        self.set_claim_form(claim, session)
        parts, part_links, active = get_participants(session, ref_id)
        if request.POST:
            post = request.POST
            if 'choose_element' in request.POST:
                element = ParticipantElement.load(session=session, ref_id=post.get('element_choice'))
                participant = element.participant
                self.set_participant_form(participant, session, parts)
                self.update_element_form(element, post, session, participant)
            elif 'choose_participant' in request.POST:
                participant = Participant.load(ref_id=post.get('participant_choice'), session=session)
                head_element = get_head_element(session, participant)
                self.update_participant_form(participant, post, session, parts)
                if head_element:
                    self.set_element_form(head_element, session, participant)
                else:
                    self.element_form = InactiveElementEdit(None)
            else:
                raise RuntimeError("Bad Post: {0}".format(request.POST))
        else:
            parts, part_links, active = get_participants(session, ref_id)
            if active:
                active_element = get_head_element(session, active)
                self.set_participant_form(active, session, parts)
                if active_element:
                    self.set_element_form(active_element, session, active)
                else:
                    self.element_form = InactiveElementEdit(None)
            else:
                self.participant_form = InactiveParticipantEdit(None)
                self.element_form = InactiveElementEdit(None)
        self.update_graph(claim, parts, part_links)
        return self.render()

    def set_claim_form(self, claim: Claim, session):
        """Initialize the form that displays the claim itself."""
        from arachcurator.views.form_set import NOTFOUND
        self.claim_form = claim.edit_self()
        citation = get_citation(session, claim.publication)
        if claim.behavior_term:
            blabel = get_term_label(session, claim.behavior_term)
        else:
            blabel = NOTFOUND
        if claim.evidence:
            evidence_label = get_term_label(session, claim.evidence)
        else:
            evidence_label = NOSPEC
        self.claim_form.behavior_term.data = blabel
        self.claim_form.publication.data = citation
        self.claim_form.evidence.data = evidence_label
        self.claim_form.ref_id = claim.ref_id

    def set_participant_form(self, participant, session, parts):
        """Initialize the form that displays the participant."""
        spf = participant.browse()
        spf.ref_id = participant.ref_id
        choices = make_participant_choices(session, parts)
        logging.info("Choices = %s", repr(choices))
        spf.participant_choice.choices = choices
        self.participant_form = self.finish_participant_form(spf, participant, session)

    def update_participant_form(self, participant, post, session, parts):
        """Initialize the form that displays the participant."""
        spf = ParticipantForm(post, participant)
        spf.ref_id = participant.ref_id
        choices = make_participant_choices(session, parts)
        logging.info("Choices = %s", repr(choices))
        spf.participant_choice.choices = choices
        self.participant_form = self.finish_participant_form(spf, participant, session)

    @staticmethod
    def finish_participant_form(spf, participant, session):
        """Initialize fields common to set and update."""
        spf.type.data = participant.type
        spf.label.data = participant.label
        spf.publication_taxon.data = participant.publication_taxon
        spf.publication_anatomy.data = participant.publication_anatomy
        spf.publication_substrate.data = participant.publication_substrate
        spf.publication_text.data = participant.publication_text
        plabel = get_property_label(session,
                                    participant.participation_property) or NOSPEC
        spf.participation_property.data = plabel
        spf.ref_id = participant.ref_id  # no data - not a form field
        return spf

    def set_element_form(self, element, session, participant):
        """Initialize the form that displays participant elements."""
        sef = TypeElementForm(None, element)
        self.element_form = finish_element_form(sef, element, session, participant)

    def update_element_form(self, element, post, session, participant):
        """Initialize the form that displays participant elements."""
        sef = TypeElementForm(post, element)
        self.element_form = finish_element_form(sef, element, session, participant)

    def render(self):
        """Return the 3 forms + graph on this page for template generation."""
        return {'c_form': self.claim_form,
                'p_form': self.participant_form,
                'e_form': self.element_form,
                'graph': self.graph.render()}

def finish_element_form(sef, element, session, participant):
    sef.element_choice.choices = participant.element_choices(session)
    sef.label.data = get_element_label(session,
                                       element.element_id)
    sef.parent_link_property.data = parent_prop_label(session,
                                                      element.element_id)
    return sef

ACTIVELY_PARTICIPATES_IN = 'http://purl.obolibrary.org/obo/RO_0002217'

def get_participants(session, claim_id):
    """Return a tuple listing all participants and an active participant.

    The active participant is determined as last participant encountered
    that is linked with the ACTIVE_PARTICIPANT property.
    :param session: database session
    :param claim_id: ref_id of the current claim object
    :return: tuple of dict of participants by ref_id, links by participants,
    ref_id of (an) active participant
    """
    parts = {}
    links = {}
    active = None
    for part_id, in get_claim_participants(session, claim_id):
        part = Participant.load(ref_id=part_id, session=session)
        link = session.query(PropertyRow).get(part.participation_property)
        assert isinstance(link, PropertyRow), "Property {0} should resolve".format(part.participation_property)
        parts[part.ref_id] = part
        links[part.ref_id] = link.ref_id
        assert (isinstance(part, Participant))
        if part.participation_property == ACTIVELY_PARTICIPATES_IN:
            active = part
    if active is None:  # probably a curator error, but try to recover
        for p in parts.keys():
            active = parts[p]
            break  # choose something (most likely choose 1 of 1)
    return parts, links, active

def get_head_element(session, part) -> ParticipantElement:
    """Return the head element of a participant."""
    if part.head_element:
        return ParticipantElement.load(ref_id=part.head_element, session=session)

def parent_prop_label(session, element_id):
    """Return the label of an element's parent property or None."""
    prop = parent_property(session, element_id)
    if prop:
        return get_property_label(session, prop)

def parent_property(session, element_id):
    """Return the ref_if of an element's parent property or None."""
    plink = session.query(ParticipantLink).filter_by(child=element_id).first()
    if plink:
        return plink.property


"""test_examples.py - tests for examples.py
"""
import pytest

from examples import power2
from examples import fibo

def test_power2():
    assert power2(0) == 1
    assert power2(1) == 2
    assert power2(2) == 4

"""
@pytest.mark.parametrize('n,expected', [
    (1,1),
    (2,1),
    (3,2),
    (4,3),
    (5,5),
    (6,8)
])

def test_fibo(n, expected):
    assert fibo(n) == expected
"""

@pytest.mark.parametrize('n,expected', [
    (1,1),
    (2,1),
    (7,13),
    ...
    (8,21),
    (0, RuntimeError),
    (-1, RuntimeError)
])
def test_fibo(n,expected):
    if isinstance(expected,type) and issubclass(expected,Exception):
        with pytest.raises(expected):
            fibo(n)
    else:
        assert fibo(n) == expected


import json
from prettyprinter import cpprint
from app.businesslogic.MeasurementUnit import Concentration
from app.businesslogic.MeasurementUnit import Halflife
from app.businesslogic.MeasurementUnit import Time
from app.businesslogic.Nuclide import Nuclide
from app.businesslogic.DecayChain import Generator

class Request:
    def __init__(self, request_dict):
        self.measurements = request_dict.get('measurements')
        self.target_time_value = request_dict.get('target_time_value')
        self.target_time_unit = request_dict.get('target_time_unit')

class Result:
    def __init__(self, nuclide: Nuclide, concentration: Concentration, time: Time):
        self.nuclide = nuclide
        self.concentration = concentration
        self.time = time

    def __json__(self, request):
        return {
            'nuclide': self.nuclide,
            'calculation': {
                'concentration': self.concentration,
                'time': self.time
            }
        }

class Calculator:
    def __init__(self):
        self.chain_generator = Generator()

    def calculate(self, decay_calculation_request: Request) -> [Result]:
        result = []

        target_time = Time(
            value=decay_calculation_request.target_time_value,
            unit=decay_calculation_request.target_time_unit
        )

        for measurement in decay_calculation_request.measurements:
            nuclide_name = measurement.get('nuclide_name')
            chains = self.chain_generator.get_for_nuclide_name(nuclide_name)
            chains_names_only = [
                chain.get_nuclide_names()
                for chain in chains
            ]

            print(json.dumps(chains_names_only, indent=4))

            initial_concentration = Concentration(
                value=measurement.get('concentration_value'),
                unit=measurement.get('concentration_unit')
            )

            # calculated_concentration = nuclide.calculate_concentration_at_time(initial_concentration, target_time)

            # result.append(
            #     Result(
            #         nuclide=nuclide,
            #         concentration=calculated_concentration,
            #         time=target_time
            #     )
            # )

        return result


try:
    from PIL import Image
except ImportError:
    import Image
import pytesseract
import cv2

file = 'sample.jpg'

img = cv2.imread(file, cv2.IMREAD_GRAYSCALE)
img = cv2.resize(img, None, fx=10, fy=10, interpolation=cv2.INTER_LINEAR)
img = cv2.medianBlur(img, 9)
th, img = cv2.threshold(img, 185, 255, cv2.THRESH_BINARY)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (4,8))
img = cv2.morphologyEx(img, cv2.MORPH_CLOSE, kernel)
cv2.imwrite("sample2.jpg", img)

file = 'sample2.jpg'
text = pytesseract.image_to_string(file)
print(''.join(x for x in text if x.isdigit()))


# -*- coding: utf-8 -*-
import base64
from django.conf import settings
from cryptography.fernet import Fernet

cipher = Fernet(base64.urlsafe_b64encode(settings.SECRET_KEY.encode()[0:32]).decode())

def encrypt(text, b16=False):
    s = cipher.encrypt(str(text).encode('utf-8'))
    if b16:
        s = base64.b16encode(s)
    return s.decode('utf-8')

def decrypt(text, b16=False):
    if b16:
        text = base64.b16decode(text.encode('utf-8'))
    else:
        text = text.encode('utf-8')
    s = cipher.decrypt(text)
    return s.decode('utf-8')


import pygame
from pygame.locals import *

from EventManager import *
from KeyboardMouseRef import *
from XboxContRef import *

class KeyboardMouseGameContController:
	"""KeyboardController takes Pygame events and notifies the control mapper
	"""
	def __init__(self, evManager):
		self.evManager = evManager
		#List every event for which this object listens
		self.evManager.registerListener(self,[TickEvent])

		pygame.joystick.init()
		#print("JOYSTICK COUNT ", pygame.joystick.get_count())
		gamepads = []
		for gamepadNumber in range (pygame.joystick.get_count()):
			newGamepad = pygame.joystick.Joystick(gamepadNumber)
			# try:
				# newGamepad = pygame.joystick.Joystick(gamepadNumber)
			# except pygame.error as message:
				# raise ErrorType(message)
			newGamepad.init()
			gamepads += [newGamepad]

		# if gamepads:
			# evManager.post()

		# joystickNo = 0
		# joy = pygame.joystick.Joystick(joystickNo)
		# joy.init()

	#----------------------------------------------------------------------
	def notify(self, event):
		if event.is_a(TickEvent ):
			#Handle Input Events
			for event in pygame.event.get():
				ev = None
				if event.type == QUIT:
					ev = QuitEvent()
				elif event.type == KEYDOWN and event.key == K_ESCAPE:
					ev = QuitEvent()
				elif event.type == KEYDOWN:
					ev = KeyboardInputEvent(event.key, 1)
				elif event.type == KEYUP:
					ev = KeyboardInputEvent(event.key, 0)
				elif event.type == MOUSEBUTTONDOWN:
					ev = MouseInputEvent(event.button, event.pos)
					#TOTRY pygame.mouse.get_pos()
				elif event.type == MOUSEBUTTONUP:
					ev = MouseInputEvent(event.button, event.pos)
				elif event.type == JOYAXISMOTION:
					#event.joy tells us which player
					if event.axis in xboxContRef.pygameAxisInt_xboxInt:
						ev = GameContInputEvent(
							event.joy,
							xboxContRef.pygameAxisInt_xboxInt[event.axis],
							event.value)
				elif event.type == JOYHATMOTION:
					ev = GameContInputEvent(event.joy, 16, event.value)
					#TOTRY event.hat I think this should always be synonomous with event.joy. Or it's for joypads with multiple dpads
				elif event.type == JOYBUTTONUP:
					ev = GameContInputEvent(event.joy, xboxContRef.pygameButtonInt_xboxInt[event.button], 0)
				elif event.type == JOYBUTTONDOWN:
					ev = GameContInputEvent(event.joy, xboxContRef.pygameButtonInt_xboxInt[event.button], 1)

				if ev:
					self.evManager.post(ev)


#######################################################################
# Декорирование функции input
oldinput = input
def input(prompt):
    print(prompt)
    return oldinput('>>> ')

#######################################################################

import os
import ctypes
from ctypes import POINTER, Structure, c_wchar, c_int, sizeof, byref
from ctypes.wintypes import BYTE, WORD, DWORD, LPWSTR, LPSTR
try:
    import win32api
except :
    print('Библиотека win32api не найдена. Вы можете её установить следующей коммандой в консоли с правами администратора: ')
    print('pip install pywin32')
    input('Нажмите Enter для завершения\n')
    exit()
HICON = c_int
LPTSTR = LPWSTR
TCHAR = c_wchar
MAX_PATH = 260
FCSM_ICONFILE = 0x00000010
FCS_FORCEWRITE = 0x00000002
SHGFI_ICONLOCATION = 0x000001000

class GUID(Structure):
    _fields_ = [
        ('Data1', DWORD),
        ('Data2', WORD),
        ('Data3', WORD),
        ('Data4', BYTE * 8)]

class SHFOLDERCUSTOMSETTINGS(Structure):
    _fields_ = [
        ('dwSize', DWORD),
        ('dwMask', DWORD),
        ('pvid', POINTER(GUID)),
        ('pszWebViewTemplate', LPTSTR),
        ('cchWebViewTemplate', DWORD),
        ('pszWebViewTemplateVersion', LPTSTR),
        ('pszInfoTip', LPTSTR),
        ('cchInfoTip', DWORD),
        ('pclsid', POINTER(GUID)),
        ('dwFlags', DWORD),
        ('pszIconFile', LPTSTR),
        ('cchIconFile', DWORD),
        ('iIconIndex', c_int),
        ('pszLogo', LPTSTR),
        ('cchLogo', DWORD)]

class SHFILEINFO(Structure):
    _fields_ = [
        ('hIcon', HICON),
        ('iIcon', c_int),
        ('dwAttributes', DWORD),
        ('szDisplayName', TCHAR * MAX_PATH),
        ('szTypeName', TCHAR * 80)]

def seticon(folderpath, iconpath, iconindex):
    """Set folder icon.

    >>> seticon(".", "C:\\Windows\\system32\\SHELL32.dll", 10)

    """
    shell32 = ctypes.windll.shell32

    folderpath = os.path.abspath(folderpath)
    iconpath = os.path.abspath(iconpath)

    fcs = SHFOLDERCUSTOMSETTINGS()
    fcs.dwSize = sizeof(fcs)
    fcs.dwMask = FCSM_ICONFILE
    fcs.pszIconFile = iconpath
    fcs.cchIconFile = 0
    fcs.iIconIndex = iconindex

    hr = shell32.SHGetSetFolderCustomSettings(byref(fcs), folderpath,
                                              FCS_FORCEWRITE)
    if hr:
        raise WindowsError(win32api.FormatMessage(hr))

    sfi = SHFILEINFO()
    hr = shell32.SHGetFileInfoW(folderpath, 0, byref(sfi), sizeof(sfi), SHGFI_ICONLOCATION)

    index = shell32.Shell_GetCachedImageIndexW(sfi.szDisplayName, sfi.iIcon, 0)

    shell32.SHUpdateImageW(sfi.szDisplayName, sfi.iIcon, 0, index)

#######################################################################

def NOD(A,B):
    # Находим наибольший общий делитель
    a,b = A,B
    while a != 0 and b != 0:
        if a > b:
            a %= b
        else:
            b %= a
    return a + b

def createListOfPrimeFactors(number):
    # Разбиваем на все простые множители
    num = number
    listOfPrimeFactors = []
    d = 2
    while d * d <= num:
        if num % d == 0:
            listOfPrimeFactors.append(d)
            num //= d
        else:
            d += 1
    if num > 1:
        listOfPrimeFactors.append(num)
    return listOfPrimeFactors

def createListOfAllMultipliers(inputNum):
    # Находим список всех делителей числа
    ls = createListOfPrimeFactors(inputNum)
    from collections import Counter
    kkk = dict(Counter(ls)).items()

    d, m = zip(*kkk)
    k = [0 for _ in range(len(set(ls)))]
    ln = range(len(m))
    arr = set()
    try:
        while True:
            r = 1
            for i1, i2 in zip(d, k):
                r *= i1 ** i2
            arr.add(r)
            k[0] += 1
            for i in ln:
                if k[i] > m[i]:
                    k[i] = 0
                    k[i+1] += 1  # IndexError
    except IndexError:
        pass
    arr = list(arr)
    arr.sort()
    return arr

#######################################################################

def mkdir(path, alert=True):
    # Обёртка над функцией создания папки
    try:
        os.makedirs(path)
    except OSError:
        print('Создать директорию %s не удалось' % path)
        oldinput('Завершите программу, иначе дальнейшая корректная работа не будет гарантирована\n')
    else:
        alert and print('Успешно создана директория %s \n' % path)

#######################################################################

# Подключение библиотеки pillow
try:
    from PIL import Image
except :
    print('Библиотека PIL не найдена. Вы можете её установить следующей коммандой в консоли с правами администратора: ')
    print('pip install pillow')
    input('Нажмите Enter для завершения\n')
    exit()

dir_for_icons = 'C:\\PleaseDontDeleteMe\\'

# Чистка предыдущих временных файлов
if os.path.isdir(dir_for_icons):
    print('Обнаружены временные файлы с предыдущих запусков. Очистка...\n')
    from shutil import rmtree
    rmtree(dir_for_icons, ignore_errors = True)

# Создание временной папки для иконок
from random import random
dir_for_icons += f'dir_for_icons{random()}\\'
mkdir(dir_for_icons)

# Ввод, проверка имени, открытие исходной картинки
isExists = False
while not isExists:
    imgName = input('Введите имя файла картинки, которая лежит в одной папке с программой:')
    if imgName and os.path.isfile(imgName):
        isExists = True
    elif os.path.isfile('input.jpg'):
        print('По заданному пути файл не найден, кроме заданного по умолчанию: input.jpg\n')
        isExists = True
        imgName = 'input.jpg'
    else:
        print('Файл с картинкой по указанному пути не найден! Попробуйте снова...\n')

    if isExists:
        try:
            img = Image.open(imgName).convert('RGBA')
        except:
            print(f'Файл {imgName} скорее всего не является картинкой. Попробуйте другую картинку.\n')
            isExists = False
        else:
            imgWidth = img.size[0]
            imgHeight = img.size[1]
            print(f'Картинка с разрешением {imgWidth} * {imgHeight} успешно открыта.\n')

# Обнаружение рабочего стола пользователя
try:
    env = dict(os.environ)
    deskPath = f"{env['HOMEDRIVE'] + env['HOMEPATH']}\\Desktop\\"
except:
    user = input('Введите имя вашего пользователя: ')
    deskPath = f'C:\\Users\\{user}\\Desktop\\'
    if not user and os.path.isdir(deskPath):
        deskPath = 'C:\\Users\\User\\Desktop\\'

# Создание списка доступных разрешний мозаик
listOfAllMultipliers = createListOfAllMultipliers(NOD(imgWidth,imgHeight))
listOfAllMultipliers.sort()
stringForChoose = ', '.join(map(str,listOfAllMultipliers))

# Выбор размера мозайки
mode = input('''Выберете режим ввода:
Вы задаёте разрешение иконки в пикселях[1](по умолчанию)
Вы задаёте разрешение мозаики в ячейках[2]''')

if not mode in ['1','2']:
    print('Вы ввели недопустимое значение! Будет установлен режим по умолчанию: 1')
    mode = '1'

if mode == '1':
    print('\n', stringForChoose, '\n')
    maxStep = min(imgWidth, imgHeight)

oldImgWidth = imgWidth
oldImgHeight = imgHeight

changeChoice = True
while changeChoice:
    try:
        if mode == '1':
            step = int(input('Введите размер иконки из предложенного выше списка:'))
            if step < 1 or step > maxStep:
                raise
            cellX = imgWidth // step
            cellY = imgHeight // step
        else:
            cellX = int(input('Введите ширину в ячейках:'))
            cellY = int(input('Введите высоту в ячейках:'))
            if cellX < 1 or cellX > imgWidth or cellY < 1 or cellY > imgHeight:
                raise
            step = min(imgWidth // cellX, imgHeight // cellY)
    except:
        print('Вы ввели недопустимые значения!')
        continue
    allCells = cellX * cellY
    print(f'У вас получится мозайка состоящая из {allCells} ячеек: {cellX} в ширину и {cellY} в высоту, при размере иконки {step}')
    imgWidth = cellX * step
    imgHeight = cellY * step
    (imgWidth, imgHeight) != (oldImgWidth, oldImgHeight) and print(f'Картинка обрежется до: {imgWidth} в ширину и {imgHeight} в высоту')

    if input('Изменить выбор? [д\Н]') in ['да','ДА','Да','Д','д','Yes','yes','YES','Y','y','l','L']:
        imgWidth = oldImgWidth
        imgHeight = oldImgHeight
        changeChoice = True
    else:
        changeChoice = False
print()

# Генерация папок, нарезка иконок, привязка иконок к папкам
imgWidth -= step - 1
imgHeight -= step - 1
numNow = 0

for numx, x in enumerate(range(0, imgWidth, step)):
    for numy, y in enumerate(range(0, imgHeight, step)):
        numNow += 1
        canvasDir = deskPath + ' ' * numNow + '\\'
        if os.path.isdir(canvasDir):
            print('Обнаружена ячейка с прошлой игры. Замена...')
        else:
            mkdir(canvasDir, False)

        print(f'Создаётся ячейка {numNow} из {allCells}.')

        newImg = img.crop( (x, y, x + step, y + step) )
        iconDir = f'{dir_for_icons}x{numx + 1}_y{numy + 1}.ico'
        newImg.save(iconDir)

        seticon(canvasDir, iconDir,0)

input('Нажмите Enter для завершения')