# -*- coding: utf-8 -*-
"""
C2017-38
"""
import scrapy
from carbuisness.items import fangshopitem
import time
from scrapy.conf import settings
from scrapy.mail import MailSender
import logging
import json
import re
import random
import hashlib
from hashlib import md5

website='fang_shop'

class CarSpider(scrapy.Spider):

    name=website

    def __init__(self,**kwargs):
        super(CarSpider,self).__init__(**kwargs)
        self.mailer=MailSender.from_settings(settings)
        self.counts=0
        self.carnum=800000

        settings.set('CrawlCar_Num',self.carnum,priority='cmdline')
        settings.set('MONGODB_DB','carbusiness',priority='cmdline')
        settings.set('MONGODB_COLLECTION',website,priority='cmdline')

    def start_requests(self):
        city_list=['','.sh','.gz','.tj','.cq','.cd','.suzhou','.wuhan','.xian','.dg','.km','.hz','.jn','.wuxi','.zz','.nc','.qd','.sjz','.nanjing','.dl','.hkproperty']
        for temp in city_list:
            url="http://shop"+temp+".fang.com/zu/house/"
            yield scrapy.Request(url,callback=self.parse)

    def parse(self, response):
        x = response.xpath('//div[@class="qxName"]/a')
        for i in range(1,len(x)-1):
            urlbase=x[i].xpath('@href').extract_first()
            url=response.urljoin(urlbase)
            yield scrapy.Request(url,callback=self.middle1_parse, dont_filter=True)

    def middle1_parse(self,response):
        x = response.xpath('//div[@class="houseList"]/dl')
        for temp in x:
            urlbase = temp.xpath('dd/p[@class="title"]/a/@href').extract_first()
            url=response.urljoin(urlbase)
            yield scrapy.Request(url,callback=self.parse_info, dont_filter=True)
        next_page=response.xpath(u'//a[contains(text(),"下一页")]/@href').extract_first()
        if next_page:
            url=response.urljoin(next_page)
            yield scrapy.Request(url,callback=self.middle1_parse, dont_filter=True)

    def parse_info(self,response):
        item=fangshopitem()
        item['url'] = response.url
        item['website'] = website
        item['grabtime'] = time.strftime('%Y-%m-%d %X', time.localtime())
        item['status'] = response.url
        item['shortdesc']=response.xpath('//div[@class="title"]/h1/text()').extract_first().strip() \
            if response.xpath('//div[@class="title"]/h1/text()').extract_first() else "-"                   # 标题描述
        item['houseid']=response.xpath(u'//span[contains(text(),"房源编号")]/text()').re('\d+')[0] \
            if response.xpath(u'//span[contains(text(),"房源编号")]/text()').re('\d+') else "-"             # 房源编号
        item['posttime']=response.xpath('//p[@class="gray9"]/text()[3]').re(u'发布时间：(.*)\(')[0] \
            if response.xpath('//p[@class="gray9"]/text()[3]').re(u'发布时间：(.*)\(') else "-"              # 发布时间
        item['price']=response.xpath('//span[@class="red20b"]/text()').extract_first() \
            if response.xpath('//span[@class="red20b"]/text()').extract_first() else "-"                        # 价格
        item['paystyle']=response.xpath('//dt[@class="gray6 zongjia1"]/text()[3]').re(u'支付方式：(.*)\)')[0] \
            if response.xpath('//dt[@class="gray6 zongjia1"]/text()[3]').re(u'支付方式：(.*)\)') else "-"    # 支付方式
        item['area']=response.xpath(u'//dt[contains(text(),"建筑面积：")]/strong/text()').extract_first() \
            if response.xpath(u'//dt[contains(text(),"建筑面积：")]/strong/text()').extract_first() else "-"     # 建筑面积
        item['phone']=response.xpath('//div[@class="phone_top"]/span/label/text()').extract_first() \
            if response.xpath('//div[@class="phone_top"]/span/label/text()').extract_first() else "-"               # 电话
        item['shopname']=response.xpath(u'//span[contains(text(),"楼盘名称：")]/../text()[2]').extract_first().strip() \
            if response.xpath(u'//span[contains(text(),"楼盘名称：")]/../text()[2]').extract_first() else "-"        # 楼盘名称
        item['address']=response.xpath(u'//span[contains(text(),"楼盘地址：")]/../text()[1]').extract_first() \
            if response.xpath(u'//span[contains(text(),"楼盘地址：")]/../text()[1]').extract_first() else "-"        # 楼盘地址
        item['propertycost']=response.xpath(u'//span[contains(text(),"物业费：")]/../text()[1]').extract_first() \
            if response.xpath(u'//span[contains(text(),"物业费：")]/../text()[1]').extract_first() else "-"             # 物业费
        item['sfb']=response.xpath(u'//span[contains(text(),"适合经营：")]/../span[2]/text()').extract_first() \
            if response.xpath(u'//span[contains(text(),"适合经营：")]/../span[2]/text()').extract_first() else "-"       # 适合经营
        item['type']=response.xpath('//div[@class="inforTxt"]/dl[2]/dd[3]/text()[2]').extract_first() \
            if response.xpath('//div[@class="inforTxt"]/dl[2]/dd[3]/text()[2]').extract_first() else "-"                    # 房屋类型
        item['uniprice']=response.xpath(u'//span[contains(text(),"元/平米·天")]/../span/text()').extract_first().strip() \
            if response.xpath(u'//span[contains(text(),"元/平米·天")]/../span/text()').extract_first() else "-"         # 租金
        yield item


# This script pulls from a job website and stores positions into a database. If there is a new posting it notifies the user.
# CNA 330
# Zachary Rubin, zrubin@rtc.edu
import mysql.connector
import sys
import requests
import os
import time
from bs4 import BeautifulSoup

# Connect to database
# You should not need to edit anything in this function
def connect_to_sql():
    conn = mysql.connector.connect(user='root', password='',
                                  host='127.0.0.1',
                                  database='cna330')
    return conn

# Create the table structure
def create_tables(cursor, table):
    ## Add your code here. Starter code below
    cursor.execute('''CREATE TABLE IF NOT EXISTS Jobs (
        ID int NOT NULL PRIMARY KEY AUTO_INCREMENT,
        PostDate TEXT,
        Title TEXT,
        Location TEXT,
        Description TEXT,
        Company TEXT,
        Apply_info TEXT,
        Salary FLOAT,
        RawMessage TEXT,
        Craig_ID VARCHAR(12) UNIQUE); ''')
    return

# Add a new job
insert_stmt = (
    "INSERT IGNORE INTO jobs (PostDate, Title, Location, Description, Apply_info) "
    "VALUES (%(created_at)s, %(title)s, %(location)s, %(description)s, %(how_to_apply)s)"
)
# Check if new job
check_stmt = """SELECT EXISTS(SELECT * FROM Jobs WHERE %(title)s)"""
# Delete given job
delete_stmt = """DELETE FROM Jobs WHERE Title = %(title)s"""

# Grab new jobs from a website
def fetch_new_job(arg_dict, job, jobdetails):
    jobdetails['title'] = str(job.p.a.string) #Get title of job
    jobdetails['created_at'] = str(job.time['datetime']) #Get date posted of job

    #Get location of job
    try:
        job_location = job.p.a.next_sibling.next_sibling.find('span', class_='result-hood').text
        jobdetails['location'] = str(job_location.replace('(', '').replace(')', '')) #Clean output
    except:
        jobdetails['location'] = "No location given"

    joburl = job.a['href'] #Get url for job
    jobdetails['how_to_apply'] = str(joburl)

    #Delve into each job link and get compensation and description
    jobr = requests.get(joburl)
    jobsoup = BeautifulSoup(jobr.text, 'html.parser')
    jobdetails['salary'] = str(jobsoup.find('p', class_='attrgroup').span.text) #Get compensation

    #Get description of job and clean output
    jobdesc = jobsoup.find('section', id='postingbody')
    jobdetails['description'] = str(jobdesc.text.replace('\n', ' ').replace('\t', ' ')[30:]) #[30:] cleans the first 30 characters referencing a QR code

    return jobdetails

# Load a text-based configuration file
def load_config_file():
    argument_dictionary = []
    file = 0
    # Code from https://github.com/RTCedu/CNA336/blob/master/Spring2018/FileIO.py
    try:
        filename = sys.argv[1]
        file = open(filename, "r")
    except (FileNotFoundError, IndexError):
        print("File not found, it will be created.")
        filename = os.path.join(os.path.dirname(__file__), 'cntconfig.txt')
        file = open(filename, "w+")

    #Read each line of file and add it to a list
    for aline in file:
        aline = aline.strip()
        argument_dictionary.append(aline)

    file.close()
    return argument_dictionary

# Main area of the code.
def jobhunt(cursor, arg_dict):
    # Fetch jobs from website
    ## Add your code here to parse the job page

    url = "https://jobs.github.com/positions.json?location=seattle"
    r = requests.get(url) #Get page
    soup = BeautifulSoup(r.text, 'html.parser')
    job_links = soup.find_all('li', class_='result-row') #Find all job listings
    for job in job_links: #Get single job
        jobdetails = {}
        jobdetails['id'] = job['data-pid'] #Get ID of job
        if not cursor.execute(check_stmt, jobdetails): #Check if ID of job already exists in DB
            jobdetails = fetch_new_job(arg_dict, job, jobdetails)
            cursor.execute(insert_stmt, jobdetails)
            print("Job " + jobdetails['title'] + " added.")

    ## Add in your code here to check if the job already exists in the DB

    ## Add in your code here to notify the user of a new posting

    ## EXTRA CREDIT: Add your code to delete old entries

# Setup portion of the program. Take arguments and set up the script
# You should not need to edit anything here.
def main():
    # Connect to SQL and get cursor
    conn = connect_to_sql()
    cursor = conn.cursor(buffered=True)
    create_tables(cursor, "table")
    # Load text file and store arguments into dictionary
    arg_dict = load_config_file()
    while(1):
        jobhunt(cursor, arg_dict)
        conn.commit()
        time.sleep(3600) # Sleep for 1h

if __name__ == '__main__':
    main()


from flask import request, render_template
from app import app, db
from app.database import Message

@app.post("/messages")
def create_message():
    msg_data = request.json
    title = msg_data.get("title")
    body = msg_data.get("body")
    if not title or not body:
        return "
Invalid syntax
", 400
    message = Message(title=title, body=body)
    db.session.add(message)
    db.session.commit
    return "
Created
", 201

@app.get("/messages")
def read_all_messages():
    messages = Message.query.all()
    return render_template("message_list.html", message_list=messages)

@app.route("/messages/<int:pk>")
def read_message_by_id(pk):
    message=Message.query.filter_by(id=pk).first()
    return render_template("message_detail.html",message=message )

@app.get("/greeting/<name>")
def greet(name):
    return render_template("home.html", username=name)


from sklearn.decomposition import PCA
import pandas as pd

def assert_None(args,
                raise_error=True):
    '''Make sure args are not None
    '''
    if any(x is None for x in args):
        num_none = sum(x is None for x in args)
        if raise_error:
            string='Nones detected : {}'.format(str(num_none))
            raise ValueError(string)
        else:
            return num_none

def assert_array_dimension(array, dimensions):
    '''Make sure arrays are the right dimensions
    '''
    if len(array.shape) != dimensions:
        raise ValueError('You must pass in a {}-D array!'.format(dimensions))

def embed_to_pca(EFILE, OUTFILE):
    """build pca model with embed file
    """
    Ef=pd.read_csv(EFILE,sep=' ',header=None).dropna(axis=1).transpose()
    Ef.columns=['x'+str(i) for i in Ef.columns]
    xf=Ef#.assign(IF=dx.ido)

    pca = PCA(n_components=2).fit(xf)
    ef=pca.fit_transform(xf)
    pd.DataFrame(ef).to_csv(OUTFILE,header=None,index=None)
    return ef

def replace_nan(df):
    """replace nans in a df and fill with empty string
    """
    df = df.replace('nan',np.nan).fillna('')
    return df


# coding: utf-8
# Copyright (c) Max-Planck-Institut für Eisenforschung GmbH - Computational Materials Design (CM) Department
# Distributed under the terms of "New BSD License", see the LICENSE file.

from ctypes import c_double, c_int
from mpi4py import MPI
import numpy as np
import pickle
import sys
from lammps import lammps

__author__ = "Sarath Menon, Jan Janssen"
__copyright__ = (
    "Copyright 2020, Max-Planck-Institut für Eisenforschung GmbH - "
    "Computational Materials Design (CM) Department"
)
__version__ = "1.0"
__maintainer__ = "Jan Janssen"
__email__ = "janssen@mpie.de"
__status__ = "production"
__date__ = "Feb 28, 2020"

# dict for extract atom methods
atom_properties = {
    "x": {"type": 3, "gtype": 1, "dim": 3},
    "mass": {"type": 2, "gtype": 1, "dim": 1},
    "id": {"type": 0, "gtype": 0, "dim": 1},
    "type": {"type": 0, "gtype": 0, "dim": 1},
    "mask": {"type": 0, "gtype": 0, "dim": 1},
    "v": {"type": 3, "gtype": 1, "dim": 3},
    "f": {"type": 3, "gtype": 1, "dim": 3},
    "molecule": {"type": 0, "gtype": 0, "dim": 1},
    "q": {"type": 2, "gtype": 1, "dim": 1},
    "mu": {"type": 3, "gtype": 1, "dim": 3},
    "omega": {"type": 3, "gtype": 1, "dim": 3},
    "angmom": {"type": 3, "gtype": 1, "dim": 3},
    "torque": {"type": 3, "gtype": 1, "dim": 3},
    "radius": {"type": 2, "gtype": 1, "dim": 1},
    "image": {"type": 2, "gtype": 0, "dim": 3},
    # we can add more quantities as needed
    # taken directly from atom.cpp -> extract()
}

# Lammps executable
args = ["-screen", "none"]
if len(sys.argv) > 1:
    args.extend(sys.argv[1:])
job = lammps(cmdargs=args)

def extract_compute(funct_args):
    # if MPI.COMM_WORLD.rank == 0:
    id = funct_args[0]
    style = funct_args[1]
    type = funct_args[2]
    length = funct_args[3]
    width = funct_args[4]
    filtered_args = [id, style, type]

    val = job.extract_compute(*filtered_args)

    # now process
    # length should be set
    data = []
    if style == 1:
        length = job.get_natoms()
    if type == 2:
        for i in range(length):
            dummy = []
            for j in range(width):
                dummy.append(val[i][j])
            data.append(np.array(dummy))
        data = np.array(data)
    elif type == 1:
        for i in range(length):
            data.append(val[i])
        data = np.array(data)
    else:
        data = val
    return data

def get_version(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.version()

def get_file(funct_args):
    job.file(*funct_args)
    return 1

def commands_list(funct_args):
    job.commands_list(*funct_args)
    return 1

def commands_string(funct_args):
    job.commands_string(*funct_args)
    return 1

def extract_setting(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.extract_setting(*funct_args)

def extract_global(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.extract_global(*funct_args)

def extract_box(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.extract_box(*funct_args)

def extract_atom(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        # extract atoms return an internal data type
        # this has to be reformatted
        name = str(funct_args[0])
        if not name in atom_properties.keys():
            return []

        # this block prevents error when trying to access values
        # that do not exist
        try:
            val = job.extract_atom(name, atom_properties[name]["type"])
        except ValueError:
            return []
        # this is per atom quantity - so get
        # number of atoms - first dimension
        natoms = job.get_natoms()
        # second dim is from dict
        dim = atom_properties[name]["dim"]
        data = []
        if dim > 1:
            for i in range(int(natoms)):
                dummy = [val[i][x] for x in range(dim)]
                data.append(dummy)
        else:
            data = [val[x] for x in range(int(natoms))]

        return np.array(data)

def extract_fix(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.extract_fix(*funct_args)

def extract_variable(funct_args):
    # in the args - if the third one,
    # which is the type is 1 - a lammps array is returned
    if MPI.COMM_WORLD.rank == 0:
        # if type is 1 - reformat file
        try:
            data = job.extract_variable(*funct_args)
        except ValueError:
            return []
        if funct_args[2] == 1:
            data = np.array(data)
        return data

def get_natoms(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.get_natoms()

def set_variable(funct_args):
    return job.set_variable(*funct_args)

def reset_box(funct_args):
    job.reset_box(*funct_args)
    return 1

def gather_atoms(funct_args):
    # extract atoms return an internal data type
    # this has to be reformatted
    name = str(funct_args[0])
    if name not in atom_properties.keys():
        return []

    # this block prevents error when trying to access values
    # that do not exist
    try:
        val = job.gather_atoms(
            name, atom_properties[name]["gtype"], atom_properties[name]["dim"]
        )
    except ValueError:
        return []
    # this is per atom quantity - so get
    # number of atoms - first dimension
    val = list(val)
    dim = atom_properties[name]["dim"]
    if dim > 1:
        data = [val[x : x + dim] for x in range(0, len(val), dim)]
    else:
        data = list(val)
    return np.array(data)

def gather_atoms_concat(funct_args):
    # extract atoms return an internal data type
    # this has to be reformatted
    name = str(funct_args[0])
    if not name in atom_properties.keys():
        return []

    # this block prevents error when trying to access values
    # that do not exist
    try:
        val = job.gather_atoms_concat(
            name, atom_properties[name]["gtype"], atom_properties[name]["dim"]
        )
    except ValueError:
        return []
    # this is per atom quantity - so get
    # number of atoms - first dimension
    val = list(val)
    dim = atom_properties[name]["dim"]
    if dim > 1:
        data = [val[x : x + dim] for x in range(0, len(val), dim)]
    else:
        data = list(val)
    return np.array(data)

def gather_atoms_subset(funct_args):
    # convert to ctypes
    name = str(funct_args[0])
    lenids = int(funct_args[1])
    ids = funct_args[2]

    # prep ids
    cids = (lenids * c_int)()
    for i in range(lenids):
        cids[i] = ids[i]

    if not name in atom_properties.keys():
        return []

    # this block prevents error when trying to access values
    # that do not exist
    try:
        val = job.gather_atoms_subset(
            name,
            atom_properties[name]["gtype"],
            atom_properties[name]["dim"],
            lenids,
            cids,
        )
    except ValueError:
        return []
    # this is per atom quantity - so get
    # number of atoms - first dimension
    val = list(val)
    dim = atom_properties[name]["dim"]
    if dim > 1:
        data = [val[x : x + dim] for x in range(0, len(val), dim)]
    else:
        data = list(val)
    return np.array(data)

def create_atoms(funct_args):
    # we have to process the input items
    # args are natoms, ids, type, x, v, image, shrinkexceed
    natoms = funct_args[0]
    ids = funct_args[1]
    type = funct_args[2]
    x = funct_args[3]
    v = funct_args[4]
    image = funct_args[5]
    shrinkexceed = funct_args[6]

    id_lmp = (c_int * natoms)()
    id_lmp[:] = ids

    type_lmp = (c_int * natoms)()
    type_lmp[:] = type

    image_lmp = (c_int * natoms)()
    image_lmp[:] = image

    args = [natoms, id_lmp, type_lmp, x, v, image_lmp, shrinkexceed]
    job.create_atoms(*args)
    return 1

def has_exceptions(funct_args):
    return job.has_exceptions

def has_gzip_support(funct_args):
    return job.has_gzip_support

def has_png_support(funct_args):
    return job.has_png_support

def has_jpeg_support(funct_args):
    return job.has_jpeg_support

def has_ffmpeg_support(funct_args):
    return job.has_ffmpeg_support

def installed_packages(funct_args):
    return job.installed_packages

def set_fix_external_callback(funct_args):
    job.set_fix_external_callback(*funct_args)

def get_neighlist(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.get_neighlist(*funct_args)

def find_pair_neighlist(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.find_pair_neighlist(*funct_args)

def find_fix_neighlist(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.find_fix_neighlist(*funct_args)

def find_compute_neighlist(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.find_compute_neighlist(*funct_args)

def get_neighlist_size(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.get_neighlist_size(*funct_args)

def get_neighlist_element_neighbors(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return job.get_neighlist_element_neighbors(*funct_args)

def get_thermo(funct_args):
    if MPI.COMM_WORLD.rank == 0:
        return np.array(job.get_thermo(*funct_args))

def scatter_atoms(funct_args):
    name = str(funct_args[0])
    py_vector = funct_args[1]
    # now see if its an integer or double type- but before flatten
    py_vector = np.array(py_vector).flatten()

    if atom_properties[name]["gtype"] == 0:
        c_vector = (len(py_vector) * c_int)(*py_vector)
    else:
        c_vector = (len(py_vector) * c_double)(*py_vector)

    job.scatter_atoms(
        name, atom_properties[name]["gtype"], atom_properties[name]["dim"], c_vector
    )
    return 1

def scatter_atoms_subset(funct_args):
    name = str(funct_args[0])
    lenids = int(funct_args[2])
    ids = funct_args[3]

    # prep ids
    cids = (lenids * c_int)()
    for i in range(lenids):
        cids[i] = ids[i]

    py_vector = funct_args[1]
    # now see if its an integer or double type- but before flatten
    py_vector = np.array(py_vector).flatten()

    if atom_properties[name]["gtype"] == 0:
        c_vector = (len(py_vector) * c_int)(*py_vector)
    else:
        c_vector = (len(py_vector) * c_double)(*py_vector)

    job.scatter_atoms_subset(
        name,
        atom_properties[name]["gtype"],
        atom_properties[name]["dim"],
        lenids,
        cids,
        c_vector,
    )
    return 1

def command(funct_args):
    job.command(funct_args)
    return 1

def select_cmd(argument):
    """
    Select a lammps command

    Args:
        argument (str): [close, extract_compute, get_thermo, scatter_atoms, command, gather_atoms]

    Returns:
        function: the selected function
    """
    switcher = {
        f.__name__: f
        for f in [
            extract_compute,
            get_version,
            get_file,
            commands_list,
            commands_string,
            extract_setting,
            extract_global,
            extract_box,
            extract_atom,
            extract_fix,
            extract_variable,
            get_natoms,
            set_variable,
            reset_box,
            gather_atoms_concat,
            gather_atoms_subset,
            scatter_atoms_subset,
            create_atoms,
            has_exceptions,
            has_gzip_support,
            has_png_support,
            has_jpeg_support,
            has_ffmpeg_support,
            installed_packages,
            set_fix_external_callback,
            get_neighlist,
            find_pair_neighlist,
            find_fix_neighlist,
            find_compute_neighlist,
            get_neighlist_size,
            get_neighlist_element_neighbors,
            get_thermo,
            scatter_atoms,
            command,
            gather_atoms,
        ]
    }
    return switcher.get(argument)

if __name__ == "__main__":
    while True:
        if MPI.COMM_WORLD.rank == 0:
            input_dict = pickle.load(sys.stdin.buffer)
            # with open('process.txt', 'a') as file:
            #     print('Input:', input_dict, file=file)
        else:
            input_dict = None
        input_dict = MPI.COMM_WORLD.bcast(input_dict, root=0)
        if input_dict["c"] == "close":
            job.close()
            break
        output = select_cmd(input_dict["c"])(input_dict["d"])
        if MPI.COMM_WORLD.rank == 0 and output is not None:
            # with open('process.txt', 'a') as file:
            #     print('Output:', output, file=file)
            pickle.dump(output, sys.stdout.buffer)
            sys.stdout.flush()


import cv2
import numpy as np

image = cv2.imread('images/indian_flag.png')

#reading pixel value as (0,0)
(b, g, r) = image[0, 0]
print("Pixel at (0, 0) - Red: {}, Green: {}, Blue: {}".format(r, g, b))

#replacing pixel value as (0,0) with 255
image[0, 0] = (0, 0, 255)
(b, g, r) = image[0, 0]
print("Pixel at (0, 0) - Red: {}, Green: {}, Blue: {}".format(r, g, b))

#cropping corner part from image
corner = image[0:100, 0:100]
cv2.imshow("Corner", corner)

#replacing corner pixles
image[0:100, 0:100] = (2255, 0, 255)

cv2.imshow('Indian Flag', image)
cv2.waitKey(0)


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
Uso: preprocesar.py

Aplica los procesadores definidos en preprocesadores a cada
una de las páginas, para producir la prioridad con la que página
será (o no) incluída en la compilación.

"""

from __future__ import with_statement, unicode_literals, print_function

import os
import codecs
import config
import logging
import operator
import sys
import time

from collections import Counter
from os.path import join, abspath, dirname

from src.armado import to3dirs
from src.preproceso import preprocesadores
from src import utiles

logger = logging.getLogger(__name__)

LOG_SCORES_ACCUM = os.path.join(config.DIR_TEMP, 'page_scores_accum.txt')
LOG_SCORES_FINAL = os.path.join(config.DIR_TEMP, 'page_scores_final.txt')

class WikiArchivo(object):
    def __init__(self, cwd, last3dirs, nombre_archivo):
        self.ruta_relativa = join(last3dirs, nombre_archivo)
        self.url = nombre_archivo
        self._filename = join(cwd, nombre_archivo)
        self._html = None

    def get_html(self):
        """Devuelve el contenido del archivo, lo carga si no lo tenía."""
        if self._html is None:
            with open(self._filename, 'rb') as fh:
                self._html = fh.read()

        return self._html

    def set_html(self, data):
        """Setea el html."""
        self._html = data

    html = property(get_html, set_html)

    def guardar(self):
        """Guarda el archivo, crea los directorios si no están."""
        destino = join(config.DIR_PREPROCESADO, self.ruta_relativa)
        try:
            os.makedirs(dirname(destino))
        except os.error:
            # ya estaba
            pass

        with open(destino, 'wb') as fh:
            fh.write(self._html)

    def __str__(self):
        return "<WikiArchivo: %s>" % self.url.encode("utf8")

class WikiSitio(object):

    def __init__(self, root_dir):
        self.origen = unicode(abspath(root_dir))
        self.preprocesadores = [proc() for proc in preprocesadores.TODOS]
        self.prof_quant = Counter()
        self.prof_times = Counter()

    def process(self):
        """Process all pages under a root directory."""
        # let's see what was processed from before, and open the log file to keep adding
        if os.path.exists(config.LOG_PREPROCESADO):
            with codecs.open(config.LOG_PREPROCESADO, "rt", "utf8") as fh:
                processed_before_set = set(x.strip() for x in fh)
        else:
            processed_before_set = set()
        processed_before_log = codecs.open(config.LOG_PREPROCESADO, "at", "utf8")

        # get the total of directories to parse
        logger.info("Getting how many pages under root dir")
        total_pages = sum(len(filenames) for _, _, filenames in os.walk(self.origen))
        logger.info("Quantity of pages to process: %d", total_pages)

        # open the scores file to keep adding
        scores_log = codecs.open(LOG_SCORES_ACCUM, "at", "utf8")

        count_processed = count_new_ok = count_new_discarded = count_old_before = 0
        tl = utiles.TimingLogger(30, logger.debug)
        for cwd, _, filenames in os.walk(self.origen):
            parts_dir = cwd.split(os.path.sep)
            last3dirs = join(*parts_dir[-3:])

            if len(last3dirs) != 5:  # ej: u"M/a/n"
                # we're not in a leaf, we shouldn't have any files
                if filenames:
                    logger.warning("We have content in a non-leaf directory: %s %s",
                                   last3dirs, filenames)
                continue

            for page_path in filenames:
                count_processed += 1
                tl.log("Processing %s (%d/%d)", last3dirs, count_processed, total_pages)

                if " " in page_path:
                    logger.warning("Have names with spaces! %s %s", last3dirs, page_path)

                # check if the page was processed or discarded before
                if page_path in processed_before_set:
                    count_old_before += 1
                    continue

                wikipage = WikiArchivo(cwd, last3dirs, page_path)

                this_total_score = 0
                other_pages_scores = []
                for procesador in self.preprocesadores:
                    tini = time.time()
                    try:
                        (this_score, other_scores) = procesador(wikipage)
                    except:
                        logger.error("Processor %s crashed on page %r", procesador, page_path)
                        raise
                    self.prof_times[procesador] += time.time() - tini
                    self.prof_quant[procesador] += 1

                    # keep the score for other pages (check before to avoid a bogus function call)
                    if other_scores:
                        other_pages_scores.extend(other_scores)

                    if this_score is None:
                        # the processor indicated to discard this page
                        count_new_discarded += 1
                        break

                    # keep the score for this page
                    this_total_score += this_score

                else:
                    # all processors done, page not discarded
                    count_new_ok += 1
                    wikipage.guardar()

                    # save the real page score
                    scores_log.write("{}|R|{:d}\n".format(
                        to3dirs.to_pagina(page_path), this_total_score))

                    # save the extra pages score (that may exist or not in the dump)
                    for extra_page, extra_score in other_pages_scores:
                        scores_log.write("{}|E|{:d}\n".format(extra_page, extra_score))

                # with score or discarded, log it as processed
                processed_before_log.write(page_path + "\n")

        # all processing done for all the pages
        logger.info("Processed pages: %d new ok, %d discarded, %d already processed before",
                    count_new_ok, count_new_discarded, count_old_before)
        scores_log.close()
        processed_before_log.close()
        for procesador in self.preprocesadores:
            procesador.close()
            logger.debug("Preprocessor %17s usage stats: %s", procesador.nombre, procesador.stats)

    def commit(self):
        """Commit all the processing done, adjusting some logs."""
        colsep = config.SEPARADOR_COLUMNAS

        # load the score files and compress it
        all_scores = Counter()
        real_pages = set()
        with codecs.open(LOG_SCORES_ACCUM, "rt", "utf8") as fh:
            for line in fh:
                page, status, score = line.strip().split(colsep)
                all_scores[page] += int(score)
                if status == 'R':
                    real_pages.add(page)

        # load the redirects
        redirects = {}
        with codecs.open(config.LOG_REDIRECTS, "r", "utf-8") as fh:
            for linea in fh:
                r_from, r_to = linea.strip().split(colsep)
                redirects[r_from] = r_to

        # transfer score
        transferred_scores = Counter()
        for page, score in all_scores.items():
            if page not in redirects:
                transferred_scores[page] += score
                continue

            # dereference the redirect, avoiding a possible loop; note that intermediate
            # pages have not score (see processor)
            loop_guard = []
            while page in redirects:
                page = redirects[page]
                if page in loop_guard:
                    logger.warning("Redirect loop found: %s", loop_guard, page)
                    break
                loop_guard.append(page)

            # add the original score to the dereferenced page
            transferred_scores[page] += score

        # store the scores again, but only for the real pages (there is no point in storing
        # scores for pages that were not included in the dump)
        with codecs.open(LOG_SCORES_FINAL, "wt", "utf8") as fh:
            for page in real_pages:
                fh.write("{}|{:d}\n".format(page, transferred_scores[page]))

class PagesSelector(object):
    """Select the htmls that will be included in this version."""

    def __init__(self):
        self._calculated = False
        self._top_pages = None

        # used from outside to decided regenerate index, blocks, etc
        self._same_info_through_runs = None

    @property
    def top_pages(self):
        """The list of top pages."""
        if not self._calculated:
            raise ValueError("You need to first 'calculate' everything.")
        return self._top_pages

    @property
    def same_info_through_runs(self):
        """The top HTMLs for this run is the same of the previous one."""
        if not self._calculated:
            raise ValueError("You need to first 'calculate' everything.")
        return self._same_info_through_runs

    def calculate(self):
        """Calculate the HTMLs with more score and store both lists."""
        self._calculated = True

        # read the preprocessed file
        all_pages = []
        colsep = config.SEPARADOR_COLUMNAS
        with codecs.open(LOG_SCORES_FINAL, 'rt', encoding='utf8') as fh:
            for line in fh:
                page, score = line.strip().split(colsep)
                dir3, fname = to3dirs.get_path_file(page)
                all_pages.append((dir3, fname, int(score)))

        # order by score, and get top N
        all_pages.sort(key=operator.itemgetter(2), reverse=True)
        page_limit = config.imageconf['page_limit']
        self._top_pages = all_pages[:page_limit]

        # get all items after N that still has the same score that last one
        last_score = self._top_pages[-1][2]
        for more_info in all_pages[page_limit:]:
            if more_info[2] == last_score:
                self._top_pages.append(more_info)

        separator = config.SEPARADOR_COLUMNAS
        if os.path.exists(config.PAG_ELEGIDAS):
            # previous run for this info! same content?
            with codecs.open(config.PAG_ELEGIDAS, "rt", "utf8") as fh:
                old_stuff = []
                for linea in fh:
                    dir3, arch, score = linea.strip().split(separator)
                    old_stuff.append((dir3, arch, int(score)))
                if sorted(old_stuff) == sorted(self._top_pages):
                    self._same_info_through_runs = True

        if not self._same_info_through_runs:
            # previous info not there, or different: write to disk
            with codecs.open(config.PAG_ELEGIDAS, "wt", "utf8") as fh:
                for dir3, fname, score in self._top_pages:
                    info = (dir3, fname, str(score))
                    fh.write(separator.join(info) + "\n")

pages_selector = PagesSelector()

def run(root_dir):
    if os.path.exists(LOG_SCORES_FINAL):
        logger.info("Skipping the whole processing stage as the final scores log was found.")
        return

    wikisitio = WikiSitio(root_dir)
    wikisitio.process()
    wikisitio.commit()

def profiled_run(root_dir):
    # import cProfile

    tini = time.time()
    wikisitio = WikiSitio(root_dir)

    # uncomment the following if you want to profile just ONE preprocessor (fix which one)
    wikisitio.preprocesadores = [preprocesadores.HTMLCleaner()]

    # select here to run the profiled process or not
    # cProfile.runctx("wikisitio.process()", globals(), locals(), "/tmp/procesar.stat")
    wikisitio.process()

    wikisitio.commit()
    tend = time.time()
    print("Whole process", tend - tini)
    print("In processors", sum(wikisitio.prof_times.values()))
    for proc in wikisitio.prof_times:
        quant = wikisitio.prof_quant[proc]
        total = wikisitio.prof_times[proc]
        print("         proc ", proc, quant, total, total / quant)
    print("Full stats (if profile run) saved in /tmp/procesar.stat")

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: preprocesar.py root_dir")
        exit()
    if os.path.exists(config.LOG_PREPROCESADO):
        print("ERROR: The PREPROCESADO file is there, it will make some articles to be skipped:",
              config.LOG_PREPROCESADO)
        exit()

    # fix config for a needed variable
    config.langconf = dict(include=[])

    profiled_run(sys.argv[1])


#! /usr/bin/env python3
# -*- coding: utf-8 -*-
'''
This crawler is used to crawl a
web site and change it to pdf
'''

__author__ = 'ly'

from urllib import request
from urllib import parse
from urllib import error
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import pdfkit
import time, os, re, logging

class Crawler(object):
    '''
    爬虫基类，所有爬虫都应该继承此类
    '''
    name = None

    def __init__(self, name, start_url):
        '''
        初始化
        ：param name: 保存PDF文件名，不需要后缀名
        : param start_url: 爬虫入口url
        '''
        self.name = name
        self.start_url = start_url
	# 爬取网站的域名，但是这里没有使用这个
        self.domain = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(self.start_url))

    def crawl(self, url):
        '''
        return url
        '''
        return request.urlopen(url)

    def parse_menu(self, response):
        '''
        解析目录结构
        '''
        raise NotImplementedError

    def parse_body(self, response):
        '''
        解析正文
        '''
        raise NotImplementedError
    def parse_css(self, response):
        '''
        解析css(注： 可以有多个css文件，但是所有网页的css是一样的)
        '''
        return None

    def run(self):
        '''
        run method
        '''
        start = time.time()
        css_list = self.parse_css(self.crawl(self.start_url))
        menu_list = self.parse_menu(self.crawl(self.start_url))
        if not menu_list:
            logging.error('在起始url没有找到目录，程序返回', exc_info=True)
            return None
        # 对html、css等文件的出来
        # try except finally 方法
        html_template = '''
        <!DOCTYPE html>

            <meta charset="UTF-8">
        '''
        if css_list:
            for css in css_list:
                html_template = html_template + '\n <link rel="stylesheet" href="{}">'.format(css)
            html_template = html_template + '\n\n'
        with open('final.html', 'wb') as f_html:
            f_html.write(html_template.encode('utf-8'))
            for url in menu_list:
                cur_html = self.parse_body(self.crawl(url))
                print('抓取 "{}" 网页完成， 正在写入本地文件'.format(cur_html.h1.get_text()))
                f_html.write(cur_html.encode('utf-8'))
            f_html.write('\n\n'.encode('utf-8'))

        print('translate to "{}.pdf"!!'.format(self.name))
        try:
            pdfkit.from_file('final.html', self.name + ".pdf")
            # 调试的时候保留了html源文件，实际使用的时候可以删掉
            #os.remove('final.html')
        except Exception as e:
            logging.error('转化为pdf失败！\n', exc_info=True)
        total_time = time.time() - start
        print(u'总计耗时： %f 秒' % total_time)

class WebCrawler(Crawler):
    '''
    廖雪峰python3教程
    '''

    def parse_menu(self, response):
        '''
        解析目录结构
        '''
        bs_obj = BeautifulSoup(response, 'lxml')
        nav_div = bs_obj.find('div', class_='x-sidebar-left-content')
        link_list = nav_div.findAll('a', {'href' : re.compile('/wiki/')})
        for alist in link_list:
            src = alist['href']
            if not src.startswith('http'):
                src = urljoin(self.start_url, src)
            yield src

    def parse_css(self, response):
        '''
        解析css文件
        '''
        bs_obj = BeautifulSoup(response, 'lxml')
        css_label = bs_obj.head.findAll('link', {'href' : re.compile(r'.*?\.css')})
        for alabel in css_label:
            yield urljoin(self.start_url, alabel['href'])

    def parse_body(self, response):
        '''
        解析正文
        '''
        try:
            bs_obj = BeautifulSoup(response, 'lxml')
            title = bs_obj.h4.get_text()
            main_page = bs_obj.find('div', class_='x-wiki-content')

            new_center = bs_obj.new_tag('center')
            new_title = bs_obj.new_tag('h1')
            new_title.string = title
            new_center.insert(1, new_title)
            main_page.insert(1, new_center)

            for img in main_page.findAll('img', {'src' : re.compile('^(?!http)')}):
                img['src'] = urljoin(self.start_url, img['src'])

            return main_page
        except Exception as e:
            logging.error('解析错误!\n', exc_info=True)

if __name__ == '__main__':
    start_url = 'http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000'
    crawler = WebCrawler('python3', start_url)
    crawler.run()


import os, sys
import json
import requests
from bs4 import BeautifulSoup
import urllib
import time
import selenium
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from datetime import datetime
import pymongo
from pymongo import MongoClient
import geocoder
# 0 : test
def test():
    g = geocoder.google('Mountain View, CA')
    print(g.lat)

#test()
# 1: crawling from otodom and save in the otodom_body.txt
def crawling_otodom():
    try:

        url = "https://otodom.pl/sprzedaz/mieszkanie/warszawa/?search%5Bdist%5D=0"

        # FireFox 를 이용하여 크롤링 시작 #

        driver = webdriver.Firefox()

        driver.implicitly_wait(30)
        driver.get(url)
        for index in range(0,100): # 100 page * 25 posts = 2,500 posts crawling
            try:
                for i in range(4,34):
                    time.sleep(1)  # 10초간 데이터 받기를 기다립니다
                    depart_element = driver.find_elements_by_xpath("//article")
                    driver.execute_script("window.scrollTo(0, " + str(i*10) + ");")
                    depart_element[i].click()

                    elem = driver.find_element_by_xpath("//body")
                    source_code = elem.get_attribute("outerHTML")
                    #print(source_code)
                    f = open("./templates/log/response/otodom_body.txt", 'w')
                    enc_response = urllib.parse.unquote(source_code)
                    f.write(enc_response)
                    f.close()
                    data_processing_otodom()
                    driver.back()
            except Exception as E:
                print("otom_crawling fail internal iteration : " + str(E))
                pass
            depart_element = driver.find_elements_by_xpath("//a[@data-dir='next']")
            depart_element[0].click()
            time.sleep(5)
        driver.quit()
        print("crwaling finished")
        print(datetime.now().strftime("%H:%M:%S"))
        # 크롤링 종료
    except Exception as E:
        print("otom_crawling fail overall : " + str(E))
        print(datetime.now().strftime("%H:%M:%S"))
        pass

#2 : load the otodom_body.txt and save in the mongoDB

def data_processing_otodom():
    try:
        url = "./templates/log/response/otodom_body.txt"
        page = open(url)
        soup = BeautifulSoup(page.read(), "lxml")

        otodom_json_catched = str(soup).split("GPT.targeting = ")[1].split(";")[0]
        print(otodom_json_catched)
        items = json.loads(otodom_json_catched)
        items['crawling_website'] = "otodom"

        address_catched = soup.find_all("p", class_="address-links")[0]
        address = address_catched.find_all("a")[-2].text.strip()
        items['Address'] = address
        items['Address_nospace'] = address.replace(" ","")
        g = geocoder.google(items['Address'])
        items['lat'] = g.lat
        items['lng'] = g.lng

        thumbnail_catched = soup.find_all("a", class_="gallery-box-image-item atlas-slider-item")[0]
        thumbnail = thumbnail_catched.find_all("img")[0]["src"]
        items['Thumbnail'] = thumbnail

        postname_catched = soup.find_all("header", class_="col-md-offer-content")[0]
        postname = postname_catched.find_all("h1")[0].text.replace(">","").strip()
        items['Postname'] = postname

        crwaling_date = datetime.now().strftime('%Y/%m/%d')
        items['Crawling_date'] = crwaling_date

        client = MongoClient()
        client = MongoClient('localhost', 27017)
        db = client["Crawling_Info_For_Flats_In_Poland"]
        collection = db["test_collection"]
        posts = db.posts
        print("insert to Mongo DB")
        post_id = posts.insert(items)

    except Exception as E:
        print("otodom_crawling fail : " + str(E))
        pass

crawling_otodom()
#data_processing_otodom()


#!/usr/bin/env python3
import numpy as np
from stemming.porter2 import stem

vocab = []
vocab_index = {}

f = open('data/vocab_full.txt')
a = 0
for i, line in enumerate(f):
    print("line {}".format(i))
    strip = line.strip().split(" ")
    if len(strip) == 2:
        stemming = stem(strip[1])
        if vocab_index.get(stemming) is None:
            vocab.append(stemming)
            vocab_index[stemming] = a
            a += 1

np.save('vocab_full', vocab)
np.save('vocab_index_full', vocab_index)

neg = np.zeros((len(vocab),1))
pos = np.zeros((len(vocab),1))

f_pos = open('data/train_pos_full.txt')
for i, line in enumerate(f_pos):
    print("line {}", i)
    words = [t for t in line.strip().split()]
    for w in words:
        w = stem(w)
        index = vocab_index.get(w)
        if vocab_index.get(w) is not None:
            pos[index] += 1

f_neg = open('data/train_neg_full.txt')
for i, line in enumerate(f_neg):
    print("line {}", i)
    words = [t for t in line.strip().split()]
    for w in words:
        w = stem(w)
        index = vocab_index.get(w)
        if index is not None:
            neg[index] += 1

np.save('pos_full', pos)
np.save('neg_full', neg)


from django.conf.urls import url
from django.views.generic.base import RedirectView
from .views import user_views, db_views, report_views, vps_views, task_views, log_views, views

urlpatterns = {
    #  ====================================================================================
    # 前端界面
    url(r'^index$', views.index, name='index'),
    url(r'^favicon\.ico$', RedirectView.as_view(url='/static/isadmin/images/favicon.ico')),
    # 用户相关
    url(r'^$', user_views.login, name='login'),
    url(r'^login$', user_views.login, name='login'),
    url(r'^register$', user_views.register, name='register'),
    url(r'^retrieve$', user_views.retrieve, name='retrieve'),
    # db管理
    url(r'^pubsoc_list$', db_views.pubsoc_list, name='pubsoc_list'),
    url(r'^prisoc_list$', db_views.prisoc_list, name='prisoc_list'),
    url(r'^malicious_domains_list$', db_views.malicious_domains_list, name='malicious_domains_list'),
    url(r'^snapshot_list$', db_views.snapshot_list, name='snapshot_list'),
    url(r'^suspicious_records_list$', db_views.suspicious_records_list, name='suspicious_record_list'),
    url(r'^pocr_list$', db_views.pocr_list, name='pocr_list'),
    url(r'^dcbp_list$', db_views.dcbp_list, name='dcbp_list'),
    # 快照展示
    url(r'^show_snapshot$', views.show_snapshot, name='show_snapshot'),
    url(r'^show_snapshot/(?P<id>[0-9]+)$', views.show_snapshot, name='show_snapshot'),
    # vps监控
    url(r'^vps_monitor$', vps_views.vps_monitor, name='vps_monitor'),
    url(r'^vps_monitor_reload$', vps_views.vps_monitor_reload, name='vps_monitor_reload'),
    url(r'^vps_detail$', vps_views.vps_detail, name='vps_detail'),
    url(r'^cpu_chart$', vps_views.cpu_chart, name='cpu_chart'),
    url(r'^memory_chart$', vps_views.memory_chart, name='memory_chart'),
    url(r'^disks_chart$', vps_views.disks_chart, name='disks_chart'),
    # 数据报表
    url(r'^report/redirect_records$', report_views.redirect_records, name='redirect_records'),
    url(r'^report/compare_unique$', report_views.compare_unique, name='compare_unique'),
    url(r'^report/filted_suspicious$', report_views.filted_suspicious, name='filted_suspicious'),
    # 一键功能
    url(r'^one_button$', views.one_button, name='one_button'),
    # 快照任务
    url(r'^add_task$', task_views.add_task, name='add_task'),
    url(r'^get_tasks$', task_views.get_tasks, name='get_tasks'),
    url(r'^edit_task$', task_views.edit_task, name='edit_task'),
    # 日志模块
    url(r'^real_time_log$', log_views.real_time_log, name='real_time_log'),

    # ======================================================================================
    # 后端接口
    # 用户相关
    url('^login_action$', user_views.login_action, name='login_action'),
    url('^logout$', user_views.logout, name='logout'),
    url('^register_action$', user_views.register_action, name='register_action'),
    url('^invite_code_action$', user_views.invite_code_action, name='invite_code_action'),
    url('^retrieve_email_action$', user_views.retrieve_email_action, name='retrieve_email_action'),
    url('^retrieve_action$', user_views.retrieve_action, name='retrieve_action'),
    # CURD
    url(r'^pubsocs$', db_views.pubsocs, name='pubsocs'),
    url(r'^pubsocs/(?P<id>[0-9]+)$', db_views.pubsocs, name='pubsocs'),
    url(r'^pubsocs_jqgrid$', db_views.pubsocs_jqgrid, name='pubsocs_jqgrid'),
    url(r'^prisocs$', db_views.prisocs, name='prisocs'),
    url(r'^prisocs/(?P<id>[0-9]+)$', db_views.pubsocs, name='prisocs'),
    url(r'^prisocs_jqgrid$', db_views.prisocs_jqgrid, name='prisocs_jqgrid'),
    url(r'^malicious_domains/(?P<id>[0-9]+)$', db_views.malicious_domains, name='malicious_domains'),
    url(r'^malicious_domains_jqgrid$', db_views.malicious_domians_jqgrid, name='malicious_domains_jqgrid'),
    url(r'^snapshots$', db_views.snapshots, name='snapshots'),
    url(r'^snapshots/(?P<id>[0-9]+)$', db_views.snapshots, name='snapshots'),
    url(r'^snapshots_jqgrid$', db_views.snapshots_jqgrid, name='snapshots_jqgrid'),
    url(r'^suspicious_records$', db_views.suspicious_records, name='suspicious_records'),
    url(r'^suspicious_records/(?P<id>[0-9]+)$', db_views.suspicious_records, name='suspicious_records'),
    url(r'^suspicious_records_jqgrid$', db_views.suspicious_records_jqgrid, name='suspicious_records_jqgrid'),
    url(r'^pocr_records$', db_views.pocr_records, name='pocr_records'),
    url(r'^pocr_records/(?P<id>[0-9]+)$', db_views.pocr_records, name='pocr_records'),
    url(r'^pocr_jqgrid$', db_views.pocr_jqgrid, name='pocr_jqgrid'),
    url(r'^dcbps/(?P<id>[0-9]+)$', db_views.dcbps, name='dcbps'),
    url(r'^dcbps_jqgrid$', db_views.dcbps_jqgrid, name='dcbps_jqgrid'),
    # 数据报表
    url('^redirect_records_datas$', report_views.redirect_records_datas, name="redirect_records_datas"),
    url('^compare_unique_datas$', report_views.compare_unique_datas, name="compare_unique_datas"),
    url('^filted_suspicious_datas$', report_views.filted_suspicious_datas, name="filted_suspicious_datas"),
    # 人工后处理
    url(r'^check_compare_unique$', report_views.check_compare_unique, name="check_compare_unique"),
    url(r'^check_suspicious$', report_views.check_suspicious, name="check_suspicious"),
    # ssh
    url('^beat_start$', views.beat_start, name="beat_start"),
    url('^beat_stop$', views.beat_stop, name="beat_stop"),
    url('^beat_restart$', views.beat_restart, name="beat_restart"),
    # 快照任务
    url(r'^tasks$', task_views.tasks, name="tasks"),
    # 日志模块
    url(r'^get_log_files$', log_views.get_log_files, name="get_log_files")
}


import pygame
import sys
import math
import random
import copy
import profile
pygame.init()
chess_number = 20  # 设置棋盘大小
LIVEDENSITY = 1 # 生命密度
TICK = 5 # 帧数
BG = (255, 255, 255)
# BG = (20, 20, 20)  # 背景色
LINECOLOR = (52, 53, 46)  # 网格色
LIFECOLOR = (31, 97, 189)  # 活细胞的颜色
black = (0, 0, 0)
CELL_LENGTH = int(600 / chess_number)  # 每个格子的像素大小
LINE_WIDTH = 2  # 线的宽度
START_POSX = 50
START_POSY = 50

gameDisplay = pygame.display.set_mode( (chess_number ,chess_number ) )

# 设置背景框大小
size = width, height = 2 * START_POSX + chess_number * CELL_LENGTH, 2 * START_POSY + chess_number * CELL_LENGTH
# 设置帧率，返回clock 类
clock = pygame.time.Clock()
screen = pygame.display.set_mode(size)
pygame.display.set_caption("Accelerator made")

# 画
def draw(livcell):
    '''
    进行一次绘画操作，可以理解为进行一帧操作所更新的画面
    param:操作前活细胞图
    return:操作后活细胞图
    '''
    for i in range(chess_number + 1):
        pygame.draw.line(screen, LINECOLOR, (START_POSX, START_POSY + i * CELL_LENGTH),
                         (START_POSX + chess_number * CELL_LENGTH, START_POSY + i * CELL_LENGTH), LINE_WIDTH)  # 横线
        pygame.draw.line(screen, LINECOLOR, (START_POSX + i * CELL_LENGTH, START_POSY),
                         (START_POSX + i * CELL_LENGTH, START_POSY + chess_number * CELL_LENGTH), LINE_WIDTH)  # 竖线#
    # 画活细胞
    livcell = rule(livcell)
    print('drawnew', livcell)
    return livcell

def drawcell(i, j, cellkind):
    '''
    画出一个具体的方块
    param:行，列，方块颜色种类
    '''
    pygame.draw.rect(screen, cellkind,
                     [START_POSX + CELL_LENGTH * j + (LINE_WIDTH - 1), START_POSY + CELL_LENGTH * i + (LINE_WIDTH - 1),
                      CELL_LENGTH - LINE_WIDTH, CELL_LENGTH - LINE_WIDTH], 0)  # 终点.  Rect(left,top,width,height)

def creatlife(density):
    '''
    在初始状态下创造生命
    param:所要求生成生命细胞的密度
    return:初始生命细胞的位置图
    '''
    livcell = [[0] * chess_number for i in range(chess_number)]
    for i in range(chess_number):
        for j in range(chess_number):
            pwall = random.random()
            if pwall < density:
                livcell[i][j] = 1
    return livcell

def neighborcell(pos):
    '''
    获得一个细胞周围的细胞位置，并且存入数组
    param:细胞的位置
    return:这个细胞所有的邻居细胞
    '''
    neighborList = []
    x = pos[0]
    y = pos[1]
    neighborList = [[x - 1, y - 1], [x - 1, y], [x - 1, y + 1], [x, y - 1], [x, y + 1], [x + 1, y - 1], [x + 1, y],
                    [x + 1, y + 1]]

    realnList = copy.deepcopy(neighborList)
    for i in neighborList:
        if i[0] < 0 or i[0] > chess_number - 1 or i[1] < 0 or i[1] > chess_number - 1:
            realnList.remove(i)

    return realnList

def rule(livcell):
    '''
    制定生命游戏的游戏规则
    param:一次操作前所有活着细胞的位置
    return:一次操作后所有活着细胞的位置
    '''
    newlivcell = copy.deepcopy(livcell)
    print('livcell', livcell)
    for i in range(chess_number):
        for j in range(chess_number):
            if livcell[i][j] == 1:
                drawcell(i, j, LIFECOLOR)
                alive = 0
                for cell in neighborcell([i, j]):
                    if livcell[cell[0]][cell[1]] == 1:
                        alive += 1
                if alive == 0 or alive == 1:
                    newlivcell[i][j] = 0
                elif alive == 2 or alive == 3:
                    newlivcell[i][j] = 1
                else:
                    newlivcell[i][j] = 0
            else:
                alive = 0
                for cell in neighborcell([i, j]):
                    if livcell[cell[0]][cell[1]] == 1:
                        alive += 1
                if alive == 3:
                    newlivcell[i][j] = 1
    return newlivcell

def text_objects(text, font):
    textSurface = font.render(text, True, black)
    return textSurface, textSurface.get_rect()

def main():
    livcell = creatlife(LIVEDENSITY)
    while True:
        for event in pygame.event.get():
            # 查找关闭窗口事件
            if event.type == pygame.QUIT:
                sys.exit()
        # 填充背景色

        screen.fill(BG)
        livcell = draw(livcell)
        # 刷新图s
        pygame.display.flip()
        clock.tick(TICK)

if __name__ == "__main__":
    profile.run("main()")


# -*- encoding: utf-8 -*-
# ##########################################################################
#    Module Writen to OpenERP, Open Source Management Solution
#
#    Authors: Openpyme (<http://openpyme.mx>)
#
#
#    Coded by: Salvador Martínez (chavamm.83@gmail.com)
#              Miguel Angel Villafuerte Ojeda (mikeshot@gmail.com)
#              Luis Felipe Lores Caignet (luisfqba@gmail.com)
#              Agustín Cruz Lozano (agustin.cruz@openpyme.mx)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import logging
import xml.etree.ElementTree as ET
from openerp.osv import orm
from openerp.report import report_sxw
from .report_to_file import ReportToFile
from openerp.tools.translate import _
from collections import defaultdict

logger = logging.getLogger(__name__)

class journal_entries_parser(report_sxw.rml_parse):
    """
    Parser class to build the journal entries report in format xml.
    """
    def __init__(self, cr, uid, name, context=None):
        super(journal_entries_parser, self).__init__(
            cr, uid, name, context=context
        )
        self.localcontext.update({
            "cr": cr,
            "uid": uid,
            "actual_context": context,
        })

class JournalEntriesXml(ReportToFile):
    """
    Class to build the journal entries report in format xml based on company,
    period and target_move.
    """
    def __init__(
        self, name, table, rml=False, parser=False, header=True, store=False
    ):
        """
        Set default data for report
        """
        super(JournalEntriesXml, self).__init__(
            name, table, rml, parser, header, store
        )

    def _get_lst_account(self, cr, uid, account_id, context):
        account_obj = self.pool['account.account']
        actual_account = account_obj.browse(
            cr, uid, account_id, context=context
        )
        lst_account = []
        self._fill_list_account_with_child(lst_account, actual_account)
        return tuple(lst_account)

    def _fill_list_account_with_child(self, lst_account, account):
        # no more child
        lst_account.append(account.id)
        if not account.child_id:
            return
        for child in account.child_id:
            self._fill_list_account_with_child(lst_account, child)

    def get_trans_cta(self, cr, uid, account_id, context=None):
        """
        Get the account number of  transaction based on the account id
        account number of  transaction is the account code.
        """
        context = context or {}
        account_obj = self.pool.get('account.account').browse(
            cr, uid, account_id
        )
        if account_obj.sat_group_id:
            result = account_obj
        else:
            if not account_obj.parent_id:
                raise orm.except_orm(
                    _('Error'),
                    _('Not found Sat Code on account %s-%s. \n'
                      'Verify that the configuration of Account Sat_Group is'
                      ' correct!' % (account_obj.code, account_obj.name))
                )
            result = self.get_trans_cta(
                cr, uid, account_obj.parent_id.id, context=context
            )
        return result

    def get_journal_entry_type(self, cr, uid, ids, context, journal_id):
        """
        Get the type of account journal entry based on the account journal id
        """
        journal_obj = self.pool.get('account.journal')
        journal_entry_type = journal_obj.browse(
            cr, uid, journal_id, context=context
        ).type
        return journal_entry_type

    def check_journal_default_credit_debit_account_id(
        self, cr, uid, ids, context, journal_id, account_id
    ):
        """
        Checks if account_id is the default credit or debit account_id of
        account journal based on the account journal_id and account_id of
        account journal entry
        """
        journal_model = self.pool.get('account.journal')
        journal = journal_model.browse(
            cr, uid, [journal_id], context=context
        )[0]

        if (
            account_id == journal.default_debit_account_id.id or
            account_id == journal.default_credit_account_id.id
        ):
            return True
        return False

    def get_journal_entrie_type_xml(self, parser, entrie):
        """
        Get the SAT type of account journal entry based on journal_type and
        # the default account for journal_id
        """
        journal_entry_type_xml = 3
        for item_line in entrie:

            if self.check_journal_default_credit_debit_account_id(
                    parser.cr, parser.uid, parser.active_ids,
                    parser.actual_context, item_line.get('journal_id'),
                    item_line.get('account_id')
            ):
                journal_entry_type = self.get_journal_entry_type(
                    parser.cr, parser.uid, parser.active_ids,
                    parser.actual_context,
                    item_line.get('journal_id')
                )

                if (journal_entry_type in ['cash', 'bank']) and item_line.get(
                    'credit'
                ):
                    return 2
                elif (
                    journal_entry_type in ['cash', 'bank']) and item_line.get(
                    'debit'
                ):
                    return 1

        return journal_entry_type_xml

    def get_journal_entry_concept(self, cr, uid, ids, context, move_id):
        """
        Get the concept of account journal entry based on the account move id
        concept of account journal entry is the account.move ref.
        """
        move_obj = self.pool.get('account.move')
        concept = move_obj.browse(cr, uid, move_id).ref or "N/A"
        return concept

    def get_trans_currency(self, cr, uid, ids, context, currency_id):
        """
        Get the currency name of transaction based on the currency id
        """
        currency_obj = self.pool.get('res.currency')
        currency_name = currency_obj.browse(cr, uid, currency_id).name
        return currency_name

    def get_check_data(
        self, cr, uid, ids, context, move_id, journal_id, account_id,
        transaccion
    ):
        """
        Get the info of the check based on the move_id of journal entry
        """
        check_obj = self.pool.get('account.voucher')
        # Gets the default credit account ID for the journal
        journal = self.pool.get('account.journal').browse(
            cr, uid, journal_id
        )

        # If Payment Type Not is Check, then exit
        if journal.payment_type_id.code not in ['02']:
            return

        credit_acc_id = journal.default_credit_account_id.id

        # If the used account is not the default credit account
        # don't continue
        if account_id != credit_acc_id:
            return

        # Get the bank data asociated to the journal used
        # in the journal entry
        partner_bank_data = self.get_partner_bank_data(
            cr, uid, ids, context, journal_id
        )
        # Verify that exists bank account asociated to the journal
        if partner_bank_data:
            # Gets the info of the check data
            check_id = check_obj.search(cr, uid, [('move_id', '=', move_id)])
            check_data = check_obj.browse(cr, uid, check_id)

            # Process each check if any
            for check in check_data:
                check_number = check.check_number
                # Not all vouchers are checks, skip function if not check_done
                if not check.check_done:
                    continue

                check_date = check.date
                check_amount = check.amount
                partner_name = check.partner_id.name
                partner_vat = check.partner_id.vat_split

                # Adds the check info to the xml report
                bank_acc = partner_bank_data[0].acc_number
                bank_sat_code = partner_bank_data[0].bank.sat_code

                cheque = ET.SubElement(transaccion, 'PLZ:Cheque')
                cheque.set("Num", str(check_number))
                cheque.set("BanEmisNal", (bank_sat_code).encode(
                    'utf-8', 'ignore').decode('utf-8')
                )
                cheque.set("CtaOri", (bank_acc).encode(
                    'utf-8', 'ignore').decode('utf-8')
                )
                cheque.set("Fecha", str(check_date))
                cheque.set("Monto", str(check_amount))
                cheque.set("Benef", (partner_name).encode(
                    'utf-8', 'ignore').decode('utf-8')
                )
                cheque.set("RFC", (partner_vat).encode(
                    'utf-8', 'ignore').decode('utf-8')
                )
        return True

    def get_partner_bank_data(self, cr, uid, ids, context, journal_id):
        """
        Get the info of the bank account based on the journal_id of
        journal entry
        """
        bank_obj = self.pool.get('res.partner.bank')
        bank_id = bank_obj.search(cr, uid, [('journal_id', '=', journal_id)])
        bank_data = bank_obj.browse(cr, uid, bank_id, context=context)
        return bank_data

    def get_invoice_data(
        self, cr, uid, ids, context, acc_move_line, journal_entry_type_xml,
        transaccion
    ):
        """
        Get the info of the Invoice based on the move_id of journal entry
        """
        invoice_obj = self.pool.get('account.invoice')

        filters = []
        if (
            journal_entry_type_xml in [1, 2] and
            (acc_move_line.get('reconcile_id') or
             acc_move_line.get('reconcile_partial_id'))
        ):
            # if journal entry in Ingreso|Egreso
            reconcile_field = acc_move_line.get(
                'reconcile_id') and 'reconcile_id' or 'reconcile_partial_id'
            reconcile_value = acc_move_line.get(
                'reconcile_id') or acc_move_line.get('reconcile_partial_id')

            filters = [(reconcile_field, '=', reconcile_value),
                       ('journal_id', '!=', acc_move_line.get('journal_id'))]

            move_obj = self.pool.get('account.move.line')
            move_id = move_obj.search(cr, uid, filters)
            if move_id:
                move_data = move_obj.browse(cr, uid, move_id)

                for move in move_data:
                    filters = [
                        ('move_id', '=', move.move_id.id),
                        ('account_id', '=', acc_move_line.get('account_id'))
                    ]
            else:
                return False

        elif journal_entry_type_xml in [3]:
            filters = [('move_id', '=', acc_move_line.get('move_id')),
                       ('account_id', '=', acc_move_line.get('account_id'))]
        else:
            return False

        invoice_id = invoice_obj.search(cr, uid, filters)
        invoice_data = invoice_obj.browse(cr, uid, invoice_id)

        # Process each invoice if any
        for invoice in invoice_data:
            # Only add information about Comprobante
            # if there is an UUID at current invoice
            if invoice.cfdi_folio_fiscal:
                if not invoice.partner_id.vat_split:
                    raise orm.except_orm(
                        _('Error'),
                        _('Partner %s do not have a VAT number assigned')
                        % invoice.partner_id.name
                    )
                comprobante = ET.SubElement(transaccion, 'PLZ:CompNal')
                comprobante.set('MontoTotal', str(invoice.amount_total))
                comprobante.set('RFC', invoice.partner_id.vat_split)
                comprobante.set('UUID_CFDI', invoice.cfdi_folio_fiscal)
                if invoice.currency_id.name != 'MXN':
                    comprobante.set('Moneda', invoice.currency_id.name)
                    comprobante.set('TipCamb', str(invoice.rate))
        return True

    def get_transferencia_data(
        self, cr, uid, acc_move_line, journal_entry_type_xml,
        transaccion, context=None
    ):
        """
        Get the info of the Transferencia based on the move_id of journal entry
        """
        context = context or {}
        payment_obj = self.pool.get('account.voucher')
        # Gets the default credit account ID for the journal
        journal = self.pool.get('account.journal').browse(
            cr, uid, acc_move_line.get('journal_id'), context=context
        )

        # If Payment Type Not is Transfer, then exit
        if journal.payment_type_id.code not in ['03']:
            return

        credit_acc_id = journal.default_credit_account_id.id

        # If the used account is not the default credit account
        # don't continue
        if acc_move_line.get('account_id') != credit_acc_id:
            return

        payment_line_ids = payment_obj.search(
            cr, uid,
            [('move_id', '=', acc_move_line.get('move_id'))],
            context=context
        )
        payment_line_data = payment_obj.browse(
            cr, uid, payment_line_ids, context=context
        )

        # Process each payment line if any
        for payment_line in payment_line_data:
            transferencia = ET.SubElement(transaccion, 'PLZ:Transferencia')
            partnerBank = payment_line.partner_bank_id
            companyBank = payment_line.journal_id.res_partner_bank_id[0]
            partner = payment_line.partner_id

            # If dealing with income money
            rfc = partner.vat_split or 'N/A'
            if journal_entry_type_xml == 1:
                cta_orig = partnerBank.acc_number
                banco_orig = partnerBank.bank.sat_code
                cta_dest = companyBank.acc_number
                banco_dest = companyBank.bank.sat_code
                benef = companyBank.partner_id.name
            # If dealing with out money
            elif journal_entry_type_xml == 2:
                cta_orig = companyBank.acc_number
                banco_orig = companyBank.bank.sat_code
                cta_dest = partnerBank.acc_number
                banco_dest = partnerBank.bank.sat_code
                benef = partnerBank.partner_id.name

            transferencia.set('CtaOri', cta_orig)
            transferencia.set('BancoOriNal', banco_orig)
            transferencia.set('Monto', str(payment_line.amount))
            transferencia.set('CtaDest', cta_dest)
            transferencia.set('BancoDestNal', banco_dest)
            transferencia.set('Benef', benef)
            transferencia.set('RFC', rfc)
            transferencia.set('Fecha', payment_line.date)
        return

    def get_otr_metodo_pago(
        self, cr, uid, acc_move_line, journal_entry_type_xml,
        transaccion, context=None
    ):
        """
        Get the info of the otrMetodoPago based on the move_id of journal entry
        """
        context = context or {}
        payment_obj = self.pool.get('account.voucher')
        # Gets the default credit account ID for the journal
        journal = self.pool.get('account.journal').browse(
            cr, uid, acc_move_line.get('journal_id'), context=context
        )

        credit_acc_id = journal.default_credit_account_id.id

        # If the used account is not the default credit account
        # don't continue
        if acc_move_line.get('account_id') != credit_acc_id:
            return

        if not journal.payment_type_id:
            raise orm.except_orm(
                    _('Error'),
                    _('Journal %s do not have set a Payment Type')
                    % journal.name
                )

        # If Payment Type is Check or Transfer, then exit
        if journal.payment_type_id.code in ['02', '03']:
            return

        payment_line_ids = payment_obj.search(
            cr, uid,
            [('move_id', '=', acc_move_line.get('move_id'))],
            context=context
        )
        payment_line_data = payment_obj.browse(
            cr, uid, payment_line_ids, context=context
        )

        # Process each payment line if any
        for payment_line in payment_line_data:
            otrmetodopago = ET.SubElement(transaccion, 'PLZ:OtrMetodoPago')

            # MetPagoPol
            # Fecha
            # Benef
            # RFC
            # Monto
            # Moneda
            # TipCamb

            partnerBank = payment_line.partner_bank_id
            partner = payment_line.partner_id

            # If dealing with income money
            rfc = partner.vat_split or 'N/A'
            benef = partner.name

            otrmetodopago.set('MetPagoPol', str(journal.payment_type_id.code))
            otrmetodopago.set('Fecha', payment_line.date)
            otrmetodopago.set('Benef', benef)
            otrmetodopago.set('RFC', rfc)
            otrmetodopago.set('Monto', str(payment_line.amount))

            if payment_line.is_multi_currency:
                otrmetodopago.set('Moneda', str(payment_line.currency_id.name))
                otrmetodopago.set('TipCamb', str(payment_line.payment_rate))

        return

    def generate_report(self, parser, data, objects):
        """
        Build journal entries report in format xml based on company,
        period and target_move.
        """
        context = parser.actual_context
        import tempfile

        account_move_model = self.pool.get('account.move')

        Etree = ET.ElementTree()
        # #Begin of Sql query Section
        period_id = data["form"]["period_id"]
        target_move = data["form"]["target_move"]
        period_obj = self.pool['account.period']
        period = period_obj.browse(
            parser.cr, parser.uid, period_id,
            context=parser.actual_context
        )

        sql_select = """
        SELECT al.debit,al.credit,al.amount_currency,al.date,al.journal_id,
        al.ref,al.move_id,al.name,al.currency_id,al.account_id,
        al.reconcile_id, al.reconcile_partial_id
        FROM account_move_line al """
        sql_where = """
        WHERE al.period_id =  %(period_id)s
        AND al.state = 'valid'
        AND al.account_id in %(account_id)s """
        search_params = {
            'period_id': period_id,
            'account_id': self._get_lst_account(
                parser.cr, parser.uid, data['form']['account_id'],
                context=context
            )
        }
        sql_joins = ''
        sql_orderby = 'ORDER BY al.move_id'
        if target_move == 'posted':
            sql_joins = 'LEFT JOIN account_move am ON move_id = am.id '
            sql_where += """ AND am.state = %(target_move)s"""
            search_params.update({'target_move': target_move})
        query_sql = ' '.join((sql_select, sql_joins, sql_where, sql_orderby))
        parser.cr.execute(query_sql, search_params)
        lines = parser.cr.dictfetchall()
        # #End of Sql query Section
        if not lines:
            raise orm.except_orm(
                _('Error'),
                _('Not found Journal Entries for this Period.\n'
                  'Choose another Period!')
            )
        # if Journal Entries have lines will create the xml report
        # Create main journal entries node
        type_request = data["form"]["type_request"]
        order_num = data["form"]["order_num"]
        pro_num = data["form"]["pro_num"]
        from datetime import datetime
        time_period = datetime.strptime(period.date_start, "%Y-%m-%d")

        polizas = ET.Element('PLZ:Polizas')
        # namespace
        polizas.set("xsi:schemaLocation",
                    "www.sat.gob.mx/esquemas/ContabilidadE/1_1/PolizasPeriodo http://www.sat.gob.mx/esquemas/ContabilidadE/1_1/PolizasPeriodo/PolizasPeriodo_1_1.xsd")
        polizas.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance")
        polizas.set("xmlns:PLZ", "www.sat.gob.mx/esquemas/ContabilidadE/1_1/PolizasPeriodo")

        polizas.set('Version', '1.1')
        polizas.set('RFC', parser.company.partner_id.vat_split)
        polizas.set('Mes', str(time_period.month).rjust(2, '0'))
        polizas.set('Anio', str(time_period.year))
        polizas.set('TipoSolicitud', type_request)
        if order_num:
            polizas.set('NumOrden', order_num)
        if pro_num:
            polizas.set('NumTramite', pro_num)
        Etree._setroot(polizas)

        # Groups of account move based on move id,
        # each group is one journal entries
        groups = defaultdict(list)
        for line in lines:
            groups[line.get('move_id')].append(line)
        for move_id in groups.keys():
            # Load account.move object for current move lines
            move = account_move_model.browse(
                parser.cr, parser.uid, move_id, context=context
            )
            cumul_debit = 0.0
            cumul_credit = 0.0
            poliza = ET.SubElement(polizas, 'PLZ:Poliza')

            # Check for SAT journal type using
            journal_entry_type_xml = self.get_journal_entrie_type_xml(
                parser, groups[move_id]
            )

            for item_line in groups[move_id]:
                # required fields on journal entry node
                cumul_debit += item_line.get('debit') or 0.0
                cumul_credit += item_line.get('credit') or 0.0
                journal_id = item_line.get('journal_id')
                # #Begin on transaction node
                trans_concept = item_line.get('name')
                acccta = self.get_trans_cta(
                    parser.cr, parser.uid,
                    item_line.get('account_id'), context=context
                )
                transaccion = ET.SubElement(poliza, 'PLZ:Transaccion')
                transaccion.set("NumCta", acccta.code)
                transaccion.set("DesCta", acccta.name)
                transaccion.set("Concepto", (trans_concept).encode(
                    'utf-8', 'ignore').decode('utf-8')
                )
                transaccion.set("Debe", str(item_line.get('debit')))
                transaccion.set("Haber", str(item_line.get('credit')))

                # Begin on journal entry node
                if journal_entry_type_xml in [2]:
                    # Get the check data related to the journal entry move
                    self.get_check_data(
                        parser.cr, parser.uid, parser.active_ids,
                        context, move_id, journal_id,
                        acccta.id, transaccion
                    )
                    # #End of Check section
                # # Begin for Invoice CFDI Section
                self.get_invoice_data(
                    parser.cr, parser.uid, parser.active_ids,
                    context, item_line, journal_entry_type_xml,
                    transaccion
                )
                # # End for Invoice CFDI Section

                # Get the Transferencia or otrMetodoPago data related for the Journal Entry move
                if journal_entry_type_xml in [1, 2]:
                    self.get_transferencia_data(
                        parser.cr, parser.uid,
                        item_line, journal_entry_type_xml,
                        transaccion, context=context,
                    )

                    self.get_otr_metodo_pago(
                        parser.cr, parser.uid,
                        item_line, journal_entry_type_xml,
                        transaccion, context=context,
                    )
                # # End of Transferencia Section
                # #End on transaction node
            journal_entry_concept = self.get_journal_entry_concept(
                parser.cr, parser.uid, parser.active_ids,
                context, move_id
            )

            concept_description = ("%s | $%s | $%s") % (
                (journal_entry_concept).encode(
                    'utf-8', 'ignore').decode(
                    'utf-8'), cumul_debit, cumul_credit
            )
            poliza.set("NumUnIdenPol", move.name)
            poliza.set("Concepto", concept_description)
            poliza.set("Fecha", move.date)
            # #End on journal entry node
        # Write data into temporal file
        with tempfile.NamedTemporaryFile(delete=False) as report:
            Etree.write(report.name, encoding='UTF-8')
            self.fname = report.name

        return

JournalEntriesXml(
    'report.journal.entries.xml',
    'account.account',
    parser=journal_entries_parser
)

{'tempfile': 'tempfile', 'datetime': 'datetime.datetime'}


from django.core.management.base import BaseCommand, CommandError
from django.conf import settings

from subprocess import check_output, call
from optparse import make_option
import re,os

from dom4gameserver.models import Game
from dom4gameserver.helpers import findgameprocess

class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('-g', '--game',
            dest='gamename',
            action='store',
            default='all',
            metavar='GAME',
            help='Game name to check'),
        )

    def handle(self, *args, **options):
        if options['gamename'] != 'all':
            self.stdout.write('Unimplemented Feature.')
        else:
            for gg in Game.objects.filter(status__in = ['PRETENDER','RUNNING']):
                if not findgameprocess(gg):
                    self.stdout.write("I didn't find game: %s.   Attempting to Restart..." % gg.name)
                    call(["sudo", '-u',settings.DOM4_USER, os.path.join(settings.DOM4GAME_DIR,gg.servername)],cwd=settings.DOM4GAME_DIR)


from individu import Individu
from collections import deque
import random
import sys

class Pool():

    """Population de (n) snakes a tester et a croiser

    Attributes:
        n (int): Taille de la population
        population (deque): ensemble des N individus de la populations
    """

    def __init__(self, n):
        self.population = deque()
        for i in range(n):
            self.population.append(Individu())
        self.mutationcoeff = 1/self.getFitnessMax()[0]
        self.trained = 0
        self.min = 0
        self.moy = 0
        self.max = 0
        self.n = n

    def breeding(self):
        #remplissage initial de la pool par entraînement des N premiers snakes
        self.trained+=1
        if self.trained < self.n+1:
            return self.population[self.trained-1]
        else:
            # Creation du tableau de croisement
            tab = []
            fitnessMax = self.getFitnessMax()[0]
            self.mutationcoeff = 1/self.getFitnessMax()[0]
            for individu in self.population:
                nb_apparition = int((individu.getFitness() * 100) / fitnessMax)
                for i in range(nb_apparition):
                    tab.append(individu)

            # Choix des deux parents
            index1 = random.randint(0,len(tab)-1)
            index2 = random.randint(0,len(tab)-1)

            while(index1 == index2): # eviter d'avoir le meme index
                index2 = random.randint(0,len(tab)-1)

            #print("index1 " + str(index1))
            #print("index2 " + str(index2))

            parent1 = tab[index1]
            parent2 = tab[index2]

            # Remplacer l'individu avec le fitness le plus faible par un enfant
            index_pire_indiv = self.getFitnessMin()[1]
            dna_enfant = parent1.dna.crossover(parent2.dna, self.mutationcoeff)
            enfant = Individu(dna_enfant)
            self.population[index_pire_indiv] = enfant

            return enfant

    def updateStatistics(self):
        self.min = self.getFitnessMin()[0]
        self.moy = self.getFitnessMoy()
        self.max = self.getFitnessMax()[0]

    def getFitnessMax(self):
        fitnessMax = 0
        indexMax = 0
        for i,individu in enumerate(self.population):
            if individu.getFitness() > fitnessMax:
                fitnessMax = individu.getFitness()
                indexMax = i
        return [fitnessMax,indexMax]

    def getFitnessMin(self):
        fitnessMin = sys.maxsize
        indexMin = 0
        for i,individu in enumerate(self.population):
            if individu.getFitness() < fitnessMin:
                fitnessMin = individu.getFitness()
                indexMin = i

        return [fitnessMin,indexMin]

    def getFitnessMoy(self):
        somme = 0
        for individu in self.population:
            somme += individu.getFitness()
        return somme/self.n

    def __str__(self):
        string = ""
        for i in range(self.n):
            string+=(str(self.trained)+" : "+str(self.population[i])+"\n")
        return string

"""
#Tests

# Test de la creation du tableau dans breeding
a = Pool(2)
a.population[0].size = 11
print(a.getFitnessMax())
print(a.breeding())
"""


from astropy.io import fits
from astropy.wcs import WCS
import numpy as np
import matplotlib.pyplot as plt

# Finish the new Observer class!
class Observer():
    '''
    This class creates an artificial night sky observer.
    '''

    # This function will get called automatically
    # when a new "observer" is created
    def __init__(self,im1_filename,im2_filename):
        '''
        When initializing the observer, the "red" image should be given
        as the first input argument and the "ir" image should be the second input
        '''
        self.im1_filename = im1_filename
        self.im2_filename = im2_filename
        self.load_images(im1_filename,im2_filename)

    def load_images(self,im1_filename,im2_filename):
        image_data_red = fits.getdata(self.im1_filename)
        image_data_ir = fits.getdata(self.im2_filename)
        image = {}
        image[self.im1_filename] = image_data_red
        image[self.im2_filename] = image_data_ir
        return image

    def make_composite(self,im_dict):
        '''
        This function is incomplete! Make sure to finish it and
        then update this docstring to explain what the function does!
        '''
        ir_hud_list = fits.open(self.im2_filename)[0]
        wcs = WCS(ir_hud_list.header)
        # Define the array for storing RGB values
        rgb = np.zeros((im_dict[self.im1_filename].shape[0],im_dict[self.im1_filename].shape[1],3))

        # Define a normalization factor for our denominator using the R filter image
        norm_factor = im_dict[self.im1_filename].astype("float").max()

        # Compute the red channel values and then clip them to ensure nothing is > 1.0
        rgb[:,:,0] = 1.5 * (im_dict[self.im2_filename].astype("float")/norm_factor)
        rgb[:,:,0][rgb[:,:,0] > 1.0] = 1.0
            # Define the array for storing RGB values
        rgb = np.zeros((im_dict[self.im1_filename].shape[0],im_dict[self.im1_filename].shape[1],3))

    # Define a normalization factor for our denominator using the R filter image
        norm_factor = im_dict[self.im1_filename].astype("float").max()

    # Compute the red channel values and then clip them to ensure nothing is > 1.0
        rgb[:,:,0] = (im_dict[self.im2_filename].astype("float")/norm_factor) * 1.5
        rgb[:,:,0][rgb[:,:,0] > 1.0] = 1.0

        rgb[:,:,1] = ((im_dict[self.im2_filename].astype("float")+ im_dict[self.im1_filename].astype("float"))/2)/norm_factor

        rgb[:,:,1][rgb[:,:,1] > 1.0] = 1.0

        rgb[:,:,2] = (im_dict[self.im2_filename].astype("float"))/norm_factor

        rgb[:,:,2][rgb[:,:,2] > 1.0] = 1.0

        plt.subplot(projection = wcs)
        plt.xlabel('Right Acension')
        plt.ylabel('Declination')
        plt.grid(color='white')
        plt.imshow(rgb)

    def calc_stats(self):
        image_dict = np.array(list(self.load_images(self.im1_filename,self.im2_filename)[self.im1_filename]))
        image_dict = np.append(image_dict,np.array(list(self.load_images(self.im1_filename,self.im2_filename)[self.im2_filename])))
        mean = image_dict.mean()
        std = image_dict.std()
        return mean, std


import unittest
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.chrome.options import Options

class NavigationTest(unittest.TestCase):
    def setUp(self):
        # create new chrome session
        chrome_option = Options()
        chrome_option.add_argument("--disable-extensions")
        self.driver = webdriver.Chrome(chrome_options=chrome_option)
        self.driver.implicitly_wait(3)

        # navigate to the application home gage
        self.driver.get('http://google.com')
        self.driver.maximize_window()
        go_en = self.driver.find_element_by_link_text('Перейти на Google.com')
        go_en.click()

    def test_browser_navigation(self):
        driver = self.driver
        # get the search text box
        # go_en = driver.find_element_by_link_text('Перейти на Google.com')
        # go_en.click()
        search_field = driver.find_element_by_name('q')
        search_field.clear()

        # enter search keyword and submit
        search_field.send_keys('selenium webdriver')
        search_field.submit()

        se_wd_link = driver.find_element_by_link_text('Selenium WebDriver')
        se_wd_link.click()
        self.assertEqual('Selenium WebDriver', driver.title)

        driver.back()
        self.assertTrue(WebDriverWait(self.driver, 10).
                        until(expected_conditions.
                              title_is('selenium webdriver - Google Search')))

        driver.forward()
        self.assertTrue(WebDriverWait(self.driver, 10).
                        until(expected_conditions.
                              title_is('Selenium WebDriver')))

        driver.refresh()
        self.assertTrue(WebDriverWait(self.driver, 10).
                        until(expected_conditions.
                              title_is('Selenium WebDriver')))

    def tearDown(self):
        # close the browser window
        self.driver.quit()

if __name__ == '__main__':
    unittest.main(verbosity=2)


from multiprocessing import  Process , Pipe
import  time ,os

fd1,fd2 = Pipe(True)
def fun(name):
    time.sleep(1)
    #管道中写入内容
    fd1.send("hello"+ str(name))
    # print(name)
if __name__ == '__main__':
    jobs = []
    for i in range(5):
        P = Process(target=fun,args=(i,))
        jobs.append(P)
        P.start()
    for i in range(5):
        #读取管道内容
        data = fd2.recv()
        print(data)
    for i in jobs:
        i.join()


from PIL import Image
from wordcloud import WordCloud, ImageColorGenerator, STOPWORDS
import matplotlib.pyplot as plt
import numpy as np
import jieba
import jieba.analyse
from collections import Counter

# jieba.enable_parallel(4)  # 开启并行分词模式，参数为并行进程数 windows不支持

jieba.load_userdict("./timo.txt")

# 创建停用词list
def stopwordslist(filepath):
    stopwords = [line.strip() for line in open(filepath, 'r', encoding='utf-8').readlines()]
    return stopwords

# 对句子去除停用词
def movestopwords(sentence):
    stopwords = stopwordslist('./timo.txt')  # 这里加载停用词的路径
    santi_words = [x for x in sentence if len(x) > 1 and x not in stopwords]
    return santi_words

def main():
    text = open('frxxz.txt', 'r', encoding='UTF-8').read()
    text_cut = jieba.cut(text)
    text_cut_list = movestopwords(text_cut)
    words_split = " ".join(text_cut_list)  # 列表解析为字符串
    abel_mask = np.array(Image.open("psz.png"))
    font_path = 'simhei.ttf'
    my_wordcloud = WordCloud(background_color='white',
                             mask=abel_mask,
                             max_words=200,
                             width=1000, height=600,
                             stopwords=STOPWORDS,
                             font_path=font_path,
                             max_font_size=100,
                             random_state=42,  # 设置多少种配色方案
                             # scale=.5
                             ).generate(words_split)
    print(words_split)
    # 词频统计
    mycount = Counter(text_cut_list)

    print("韩立", mycount.get('韩立'))
    print("南宫婉", mycount.get("南宫"))
    print("元瑶", mycount.get("元瑶"))
    print("银月", mycount.get("银月"))

    for key, val in mycount.most_common(100):  # 有序（返回前10个）
        print(key, val)
    # 根据图片生成词云颜色
    image_colors = ImageColorGenerator(abel_mask)
    # my_wordcloud.recolor(color_func=image_colors)

    # 以下代码显示图片'
    my_wordcloud.to_file('ss.png')
    plt.imshow(my_wordcloud)
    plt.axis("off")
    fig = plt.gcf()
    plt.show()
    fig.savefig("result.png", dpi=100)

if __name__ == '__main__':
    main()


import torch
import numpy as np
import sys
import pandas as pd

# OPTIONAL: if you want to have more information on what's happening, activate the logger as follows
import logging
logging.basicConfig(level=logging.INFO)

use_cuda = torch.cuda.is_available()
device = torch.device('cuda:0' if use_cuda else 'cpu')

path_to_wsc = '../../data/wsc_data/enhanced.tense.random.role.syn.voice.scramble.freqnoun.tsv'
wsc_datapoints = pd.read_csv(path_to_wsc, sep='\t')

#wsc_file = open(path_to_wsc, 'r')
#wsc_datapoints = wsc_file.readlines()

def insert_highlight(text_tokenized, pronoun_index):
    text_tokenized.insert(pronoun_index, " ")
    text_tokenized.insert(pronoun_index + 2, " ")

    return text_tokenized

def find_sub_list(sl,l):
    results=[]
    sll=len(sl)
    for ind in (i for i,e in enumerate(l) if e.lower() == sl[0].lower()):
        item = str(l[ind:ind+sll]).lower()
        sub_item = str(sl).lower()
        if  item == sub_item:
            results.append((ind,ind+sll))
    return results

def replace_pronoun(tokenized_text, pronoun_index, tokenized_option):
    tokenized_text = tokenized_text[:pronoun_index] + tokenized_option + tokenized_text[pronoun_index:]
    new_pronoun_index = pronoun_index + len(tokenized_option)
    tokenized_text.pop(new_pronoun_index)
    return tokenized_text

counter = 0
limit_of_dps = 60
split_counter = 0
filename = sys.argv[1]

for q_index, dp_split in wsc_datapoints.iterrows():
        if counter % limit_of_dps == 0 or counter == 0:

            split_counter = split_counter + 1
            fsplitname = str(filename) + "." + str(split_counter)
            if counter != 0:
                f.close()
            f = open(fsplitname, 'w')

        correct_answer = dp_split['correct_answer'].strip()
        pronoun = dp_split['pron'].strip()
        pronoun_index_orig =  dp_split['pron_index_scrambled']

        answer_A = dp_split['answer_a'].strip()
        answer_B = dp_split['answer_b'].strip()
        text = dp_split['text_scrambled'].strip()

        if  pronoun_index_orig != '-' and  pronoun_index_orig:
            counter = counter + 1
            text_tokenized = text.split()
            pronoun_index_orig = int(pronoun_index_orig)

            #match pron
            matched_pronouns = find_sub_list([pronoun], text_tokenized)
            first_indices = np.array([mp[0] for mp in  matched_pronouns])
            correct_idx = (np.abs(first_indices - pronoun_index_orig)).argmin()
            pronoun_index_text = matched_pronouns[correct_idx][0]

            text_tokenized_highlighted = insert_highlight(text_tokenized, pronoun_index_text)

            label = "l: " + str(counter)
            type = "t: radio"
            question = "q: " + ' '.join(text_tokenized_highlighted)
            answera = "- " + answer_A
            answerb = "- " + answer_B

            f.write(label)
            f.write('\n')
            f.write(type)
            f.write('\n')
            f.write(question)
            f.write('\n')
            f.write(answera)
            f.write('\n')
            f.write(answerb)
            f.write('\n')
            f.write('\n')

            #print(label)
            #print(type)
            #print(question)
            #print(answera)
            #print(answerb)
            #print('\n')