"""
model.py: bert ner 模型
by: qliu
update date: 2021-12-17
"""
import os, torch
import torch.nn as nn
from transformers import BertForTokenClassification

class BertNer(nn.Module):
    def __init__(self, common_config, model_config):
        self.common_config = common_config
        self.model_config = model_config
        self.start_epoch = 0
        self.prev_acc_score = 0
        self.prev_f1_score = 0

    def get_model(self):
        """
        通过读取配置参数,加载 bert ner 模型
        """
        checkpoint_path = os.path.join(self.common_config.checkpoint_dir, self.model_config.bert_model_name)
        if self.model_config.load_checkpoint and os.path.exists(checkpoint_path):
            checkpoint = torch.load(checkpoint_path, map_location="cpu")
            self.start_epoch = checkpoint["epoch"] + 1
            self.prev_acc_score = checkpoint["acc_score"]
            self.prev_f1_score = checkpoint["f1_score"]
            model = BertForTokenClassification.from_pretrained(self.common_config.bert_model_dir,state_dict=checkpoint["model_state"], num_labels=self.common_config.labels_num)
            print("Loaded pretrain bert ner model, previous acc: {}, previous f1_score : {}".format(self.prev_acc_score, self.prev_f1_score))
        else:
            model = BertForTokenClassification.from_pretrained(self.common_config.bert_model_dir, num_labels=self.common_config.labels_num)
        return model


import pprint

from PyPDF2 import PdfFileReader, PdfFileMerger
import PyPDF2
from os import walk

if __name__ == '__main__':
    filenames = next(walk("."), (None, None, []))[2]
    infiles = []
    print("searching...")
    for name in filenames:
        #print(name[-4:])
        if(name[-4:]==".pdf"):
            print(name);
            infiles.append(name)
    print("processing...")
    for name in infiles:
        print(name)
        file_in = open(name, 'rb')
        pdf_reader = PdfFileReader(file_in)
        try:
            metadata = pdf_reader.getDocumentInfo()
            pprint.pprint(metadata)

            pdf_merger = PdfFileMerger()
            pdf_merger.append(file_in)
            pdf_merger.addMetadata({
                '/Title': name[:-4]
            })

            file_out = open(name, 'wb')
            pdf_merger.write(file_out)
            file_in.close()
            file_out.close()
        except PyPDF2.errors.PdfReadError:
            print("File encrypted! Skipping")


import cv2,os
from pywidgets.kivy.Img.__origin import *
from pycv2.img.utils import *
from pycv2.img.drawing.box import *
def donothing(*args,**kwrgs):pass
FILE_PATH=os.path.dirname(__file__)
CRITERIA = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
REMOVEDBACKGROUND=cv2.imread(os.path.join(FILE_PATH,"icons\\removedbackground.png"))
if REMOVEDBACKGROUND is None:
    raise "where is the image"
class __Croped_img(Img_with_scrollbar):
    def crop_Img(self,imgcv,b=None):
        if b is None:
            Img_with_scrollbar.define_image(self,imgcv)
        else:
            Img_with_scrollbar.define_image(self,imgcv[b[1]:b[1]+b[3], b[0]:b[0]+b[2]])
class DrawedImg(Img_with_scrollbar):
    def draw_box_image(self,imgcv,circles=None):
        super().define_image(draw_box_moving(imgcv,circles))
class Added_background_Img(Img_with_scrollbar):
    def define_image(self,imgcv,imported_backgrounds,mask=None):
        finalimg=imgcv.copy()
        for background,pos in imported_backgrounds:
            h,w=background["background"].shape[:2]
            _backmask=cv2.resize(background["mask"],(w,h)).astype(np.bool)
            finalimg[pos[1]:pos[1]+h,pos[0]:pos[0]+w][_backmask]=background["background"][_backmask]
        finalimg=add_back_ground(imgcv.copy(),mask,finalimg)
        super().define_image(finalimg)
class Colored_Img(DrawedImg):
    def define_image(self,imgcv,mask=None,coloerbackground=None,circles=None):
        if mask is None:mask=np.zeros(imgcv.shape[:2],"uint8")
        if coloerbackground is None:coloerbackground=np.zeros(imgcv.shape[:2],"uint8")
        self.draw_box_image(add_back_ground(imgcv.copy(),mask, coloerbackground), circles)
class Bluerd_img(DrawedImg):
    def define_image(self,imgcv,mask=None,circles=None,blured=0):
        mask=mask if not mask is None else np.ones(imgcv.shape[:2],"uint8")*255
        if blured!=0:
            self.draw_box_image(bluring(imgcv.copy(), mask, blured), circles)
        else:
            self.draw_box_image(imgcv, circles)
class Cutted_brush(DrawedImg):
    def define_image(self, imgcv,mask=None,radiusarea=2,circles=None):
        if mask is None:mask=np.zeros(self.shape[:2],"uint8")
        resultimg=cv2.inpaint(imgcv, mask, radiusarea, cv2.INPAINT_TELEA)
        self.draw_box_image(resultimg,circles)
class Removed_Background(DrawedImg):
     def define_image(self, imgcv,mask=None,circles=None):
        if mask is None:mask=np.zeros(imgcv.shape[:2],"uint8")
        background=duplicate_image(REMOVEDBACKGROUND,imgcv.shape[:2])

        self.draw_box_image(add_back_ground(imgcv.copy(),255-mask,background),circles)
class Croped_Img(__Croped_img):
    def define_image(self,imgcv,mask=None,box=None):
        if mask is None:
            coloredimg=imgcv
        else:
            coloredimg=color_back_ground(imgcv,mask,(255,255,255))
        self.crop_Img(coloredimg,box)
class Croped_mask(__Croped_img):
    def define_image(self,mask,box=None,):
        self.crop_Img(mask,box)
class Mask(DrawedImg):
    def define_image(self,mask,circles=None):
        self.draw_box_image(mask,circles)
class Image_viewer(DrawedImg):
    def define_image(self,imgcv,colorimgstate=False,mask=None,circles=None):
        if mask is None or not colorimgstate:
            colored_imgcv=imgcv
        else:
            colored_imgcv=color_back_ground(imgcv.copy(),mask,(255,255,255))
        self.draw_box_image(colored_imgcv,circles)
class ClusteringtheImage(DrawedImg):
    def __init__(self,**kwargs):
        super().__init__(**kwargs)
        #for saving processor effort
        self.last_kmeans=None
        self.last_img=None
        self.last_clusterd_img=None
    def define_image(self, imgcv,mask=None,kmeans=3,circles=None):
        mask=mask if not mask is None else np.zeros(imgcv.shape,"uint8")
        if kmeans!=self.last_kmeans or not ARE_EQUALE(self.last_img,imgcv):
            self.last_kmeans=kmeans
            self.last_img=imgcv.copy()
            Z = imgcv.reshape((-1,3))
            # convert to np.float32
            Z = np.float32(Z)
            # define criteria, number of clusters(K) and apply kmeans()
            _,label,center=cv2.kmeans(Z,kmeans,None,CRITERIA,10,cv2.KMEANS_RANDOM_CENTERS)
            # Now convert back into uint8, and make original image
            center = np.uint8(center)
            res = center[label.flatten()]
            self.last_clusterd_img=res.reshape((imgcv.shape))
        _background=add_back_ground(self.last_img.copy(),mask,self.last_clusterd_img)
        self.draw_box_image(_background,circles)
class Gray_Image_viewer(DrawedImg):
    def define_image(self, imgcv,mask=None,circles=None):
        mask=mask if not mask is None else np.zeros(imgcv.shape,"uint8")
        #conver BGR 2 Gray and add 2 channels by cv2
        grayimg=cv2.cvtColor(cv2.cvtColor(imgcv,cv2.COLOR_BGR2GRAY),cv2.COLOR_GRAY2BGR)
        _background=add_back_ground(imgcv.copy(),mask,grayimg)
        self.draw_box_image(_background,circles)


import functools

def cache(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        cache_key = args + tuple(kwargs.values())
        if cache_key not in wrapper.internal_cache:
            wrapper.internal_cache[cache_key] = func(*args, **kwargs)
        return wrapper.internal_cache[cache_key]

    wrapper.internal_cache = {}
    return wrapper

@cache
def fibonacci(n):
    print(f'Calculating F({n})')
    if n == 0:
        return 0
    elif n == 1:
        return 1
    return fibonacci(n - 2) + fibonacci(n - 1)

@cache
def test(n):
    print(f'T({n})')
    return 5

print(fibonacci(111))
print(fibonacci(10))
print(test(100))
print(test(100))
print(test(101))
print(fibonacci(10))
print(test.internal_cache)
print(fibonacci.internal_cache)


#!/usr/bin/python

import sys
import time
import boto.ec2

#if len(sys.argv) < 3:
#    print('Given the region and id that an AMI images is currently in, this script will copy it to other regions.')
#    exit('Usage: {0} region image_id'.format(sys.argv[0]))

#REGION = sys.argv[1]
#IMAGE_ID = sys.argv[2]
new_images = {}

def get_image_state(region, ec2Connection):
    image = ec2Connection.get_all_images(image_ids=[new_images[region]], owners=['self'])[0]
    if image.state == 'failed':
        exit('Image ({0}) could not be copied.'.format(image.id))
    if image.state == 'pending':
        return False
    if image.state == 'available':
        return True

def main(REGION, IMAGE_ID, SERVICE_NAME):
    global new_images
    regions = ('us-east-1', 'us-west-1', 'us-west-2')
    for region in regions:
        if region == REGION:
            continue

        # Connect to the region the image will be copied to
        ec2Connection = boto.ec2.connect_to_region(region)
        # Copy the image from the source region to the current region
        new_images[region] = ec2Connection.copy_image(
            source_region=REGION,
            source_image_id=IMAGE_ID,
            name=SERVICE_NAME,
            description='Copied from {0} in the {1} region.'.format(IMAGE_ID, REGION)
        ).image_id

    print('Copying new image to other regions: {0}'.format(new_images))

    # Now we watch and wait for the new images to be available
    for region in new_images:
        ec2Connection = boto.ec2.connect_to_region(region)
        while not get_image_state(region, ec2Connection):
            time.sleep(5)

    print('All new images are available.')
    return new_images


import awkward as ak
import numpy as np
import onnxruntime as ort
import vector
from numba import jit

vector.register_awkward()

import torch
from torch import nn
from torch_geometric.nn.conv import DynamicEdgeConv
from torch_geometric.nn.pool import avg_pool_x

import workflows.SUEP_utils as SUEP_utils

def SSDMethod(self, indices, events, out_label=""):
    #####################################################################################
    # ---- ML Analysis
    # Each event is converted into an input for the ML models. Using ONNX, we run
    # inference on each event to obtain a prediction of the class (SUEP or QCD).
    # N.B.: Conversion is done elsewhere.
    # N.B.: The inference skips the lost tracks for now.
    #####################################################################################

    if self.do_inf:
        pred_dict = {}
        ort_infs = {}
        options = ort.SessionOptions()
        options.inter_op_num_threads = 1  # number of threads used to parallelize the execution of the graph (across nodes). Default is 0 to let onnxruntime choose.
        for model in self.ssd_models:
            ort_infs.update(
                {
                    model: ort.InferenceSession(
                        f"data/onnx_models/resnet_{model}_{self.era}.onnx"
                    )
                }
            )
        # In order to avoid memory issues convert events to images and run inference in batches
        # also exploits the numba-compiled convert_to_images function
        for i in range(0, len(inf_cands), self.batch_size):
            if i + self.batch_size > len(inf_cands):
                self.batch_size = len(inf_cands) - i
            batch = inf_cands[i : i + self.batch_size]
            imgs = convert_to_images(self, batch)
            for model in self.ssd_models:
                batch_resnet_jets = run_inference_SSD(self, imgs, ort_infs[model])
                if i == 0:
                    resnet_jets = batch_resnet_jets
                else:
                    resnet_jets = np.concatenate((resnet_jets, batch_resnet_jets))
                pred_dict.update(
                    {model: resnet_jets[:, 1]}
                )  # highest SUEP prediction per event

        for model in self.ssd_models:
            self.out_vars.loc[indices, model + "_ssd" + out_label] = pred_dict[model]

def DGNNMethod(
    self,
    indices,
    SUEP_tracks,
    SUEP_cand,
    ISR_tracks=None,
    ISR_cand=None,
    out_label="",
    do_inverted=False,
):
    #####################################################################################
    # ---- ML Analysis
    # Each event is converted into an input for the ML models. Using ONNX, we run
    # inference on each event to obtain a prediction of the class (SUEP or QCD).
    # N.B.: Conversion is done elsewhere.
    #####################################################################################

    if self.do_inf:
        import yaml

        device = torch.device("cpu")

        modelDir = "data/GNN/"

        # consistency check
        assert len(self.dgnn_model_names) == len(self.configs)

        for model_name, config in zip(self.dgnn_model_names, self.configs):
            model_path = modelDir + model_name + ".pt"

            # initialize model with original configurations and import the weights
            config = yaml.safe_load(open(modelDir + config))
            suep = SUEPNet(
                out_dim=config["model_pref"]["out_dim"],
                hidden_dim=config["model_pref"]["hidden_dim"],
            ).to(device)
            suep.load_state_dict(
                torch.load(model_path, map_location=torch.device("cpu"))["model"]
            )
            suep = suep.float()
            suep.eval()

            # run GNN inference on the SUEP tracks
            results = run_inference_GNN(self, suep, SUEP_tracks, SUEP_cand)
            self.out_vars.loc[
                indices, "SUEP_" + model_name + "_GNN" + out_label
            ] = results

            # calculate other obserables to store
            boost_SUEP = ak.zip(
                {
                    "px": SUEP_cand.px * -1,
                    "py": SUEP_cand.py * -1,
                    "pz": SUEP_cand.pz * -1,
                    "mass": SUEP_cand.mass,
                },
                with_name="Momentum4D",
            )
            SUEP_tracks_b = SUEP_tracks.boost_p4(boost_SUEP)
            eigs = SUEP_utils.sphericity(SUEP_tracks_b, 1.0)  # Set r=1.0 for IRC safe
            self.out_vars.loc[indices, "SUEP_nconst_GNN" + out_label] = ak.num(
                SUEP_tracks
            )
            self.out_vars.loc[indices, "SUEP_S1_GNN" + out_label] = 1.5 * (
                eigs[:, 1] + eigs[:, 0]
            )

            if do_inverted:
                # run GNN inference on the SUEP tracks
                results = run_inference_GNN(self, suep, ISR_tracks, ISR_cand)
                self.out_vars.loc[
                    indices, "ISR_" + model_name + "_GNN" + out_label
                ] = results

                # calculate other obserables to store
                boost_ISR = ak.zip(
                    {
                        "px": ISR_cand.px * -1,
                        "py": ISR_cand.py * -1,
                        "pz": ISR_cand.pz * -1,
                        "mass": ISR_cand.mass,
                    },
                    with_name="Momentum4D",
                )
                ISR_tracks_b = ISR_tracks.boost_p4(boost_ISR)
                eigs = SUEP_utils.sphericity(
                    ISR_tracks_b, 1.0
                )  # Set r=1.0 for IRC safe
                self.out_vars.loc[indices, "ISR_nconst_GNN" + out_label] = ak.num(
                    ISR_tracks
                )
                self.out_vars.loc[indices, "ISR_S1_GNN" + out_label] = 1.5 * (
                    eigs[:, 1] + eigs[:, 0]
                )

def run_inference_GNN(self, model, tracks, SUEP_cand):
    results = np.array([])
    for i in range(0, len(tracks), self.batch_size):
        # define batch and convert objects in a coordinate frame
        batch = tracks[i : i + self.batch_size]
        this_batch_size = len(batch)
        batch_SUEP_cand = SUEP_cand[i : i + self.batch_size]
        batch = GNN_convertEvents(self, batch, batch_SUEP_cand)

        sigmoid = torch.nn.Sigmoid()
        with torch.no_grad():
            # count how many tracks per event are nonzero: dimensions of (events, tracks)
            Nlc = np.count_nonzero(batch[:, :, 0], axis=1)

            # reshape the batch for correct DGNN format
            x_pf = None  # dims: (events times tracks, 4)
            x_pf_batch = None  # dim: (events times tracks)
            for idx in range(this_batch_size):
                # flatten the batch
                if idx == 0:
                    x_pf = batch[idx, : Nlc[idx], :]
                else:
                    x_pf = np.vstack((x_pf, batch[idx, : Nlc[idx], :]))
                # event indices
                if idx == 0:
                    x_pf_batch = np.ones(Nlc[idx]) * idx
                else:
                    x_pf_batch = np.concatenate((x_pf_batch, np.ones(Nlc[idx]) * idx))

            # convert to torch
            x_pf = torch.from_numpy(x_pf).float()
            x_pf_batch = torch.from_numpy(x_pf_batch).float()

            # batch predictions
            out = model(x_pf, x_pf_batch)

            # normalize the outputs
            nn1 = out[0][:, 0]
            nn1 = sigmoid(nn1)
            results = np.concatenate((results, nn1.cpu().numpy()))

    return results

class SUEPNet(nn.Module):
    def __init__(self, out_dim=1, hidden_dim=16):
        super().__init__()

        # hidden_dim = 16
        # out_dim = 2

        self.pf_encode = nn.Sequential(
            nn.Linear(4, hidden_dim),
            nn.ELU(),
            nn.Linear(hidden_dim, hidden_dim),
            nn.ELU(),
        )

        self.conv = DynamicEdgeConv(
            nn=nn.Sequential(nn.Linear(2 * hidden_dim, hidden_dim), nn.ELU()), k=8
        )

        self.output = nn.Sequential(
            nn.Linear(hidden_dim, 8),
            nn.ELU(),
            nn.Linear(8, 4),
            nn.ELU(),
            nn.Linear(4, out_dim)
            # nn.Sigmoid()
        )

    def forward(self, x_pf, batch_pf):
        x_pf_enc = self.pf_encode(x_pf)

        # create a representation of PFs to PFs
        feats1 = self.conv(x=(x_pf_enc, x_pf_enc), batch=(batch_pf, batch_pf))
        feats2 = self.conv(x=(feats1, feats1), batch=(batch_pf, batch_pf))

        batch = batch_pf
        out, batch = avg_pool_x(batch, feats2, batch)

        out = self.output(out)

        return out, batch

@jit(forceobj=True)
def convert_to_images(self, events):
    # Turn the PFcand info into indexes on the image map
    idx_eta = ak.values_astype(
        np.floor((events.eta - self.eta_span[0]) * self.eta_scale), "int64"
    )
    idx_phi = ak.values_astype(
        np.floor((events.phi - self.phi_span[0]) * self.phi_scale), "int64"
    )
    idx_eta = ak.where(idx_eta == self.eta_pix, self.eta_pix - 1, idx_eta)
    idx_phi = ak.where(idx_phi == self.phi_pix, self.phi_pix - 1, idx_phi)
    pt = events.pt

    to_infer = np.zeros((len(events), 1, self.eta_pix, self.phi_pix))
    for event_i in range(len(events)):
        # form image
        to_infer[event_i, 0, idx_eta[event_i], idx_phi[event_i]] = pt[event_i]

        # normalize pt
        m = np.mean(to_infer[event_i, 0, :, :])
        s = np.std(to_infer[event_i, 0, :, :])
        if s != 0:
            to_infer[event_i, 0, :, :] = (to_infer[event_i, 0, :, :] - m) / s

    return to_infer

def run_inference_SSD(self, imgs, ort_sess):
    # Running the inference in batch mode
    input_name = ort_sess.get_inputs()[0].name
    cl_outputs = np.array([])

    for i, img in enumerate(imgs):
        # SSD: grab classification outputs (0 - loc, 1 - classifation, 2 - regression)
        # resnet: only classification as output
        cl_output = ort_sess.run(None, {input_name: np.array([img.astype(np.float32)])})
        cl_output_softmax = softmax(cl_output)[0]
        if i == 0:
            cl_outputs = cl_output_softmax
        else:
            cl_outputs = np.concatenate((cl_outputs, cl_output_softmax))

    return cl_outputs

def softmax(data):
    # some numpy magic
    return np.exp(data) / (np.exp(data).sum(axis=-1)[:, :, None])

def GNN_convertEvents(self, events, SUEP_cand, max_objects=1000):
    allowed_objects = ["pfcand", "bpfcand"]
    if self.obj.lower() not in allowed_objects:
        raise Exception(self.obj + " is not supported in GNN_convertEvents.")

    if self.obj.lower() == "pfcand":
        events = events

    elif self.obj.lower() == "bpfcand":
        boost_SUEP = ak.zip(
            {
                "px": SUEP_cand.px * -1,
                "py": SUEP_cand.py * -1,
                "pz": SUEP_cand.pz * -1,
                "mass": SUEP_cand.mass,
            },
            with_name="Momentum4D",
        )
        events = events.boost_p4(boost_SUEP)

    else:
        raise Exception()

    new_events = SUEP_utils.convert_coords(self.coords, events, max_objects)

    return new_events


import sys  # sys нужен для передачи argv в QApplication
from PyQt5 import QtWidgets
import design
import requests
import time
import re

url_get_info = 'https://callapi.gravitel.ru/api/v1/getdialoutstat'
url_list = 'https://callapi.gravitel.ru/api/v1/listdialouts'
headers = {'Content-Type': 'application/json'}

def check_token():
    forCheckToken = requests.post(url_list, headers=headers, json={'token': token}).json()
    if forCheckToken['status'] == 'error':
        print("May be invalid token!")
        exit()

class ExampleApp(QtWidgets.QMainWindow, design.Ui_CounterCallAPI):
    def __init__(self):
        # Это здесь нужно для доступа к переменным, методам
        # и т.д. в файле design.py
        super().__init__()
        self.setupUi(self)  # Это нужно для инициализации нашего дизайна
        self.found_dialouts.pressed.connect(self.connectUI)
        self.get_info_dialout.pressed.connect(self.get_info)

    def output_message(self, message_to_ui):
        self.output_info.setText(message_to_ui)

    def connectUI(self):
        token = self.token_enter.text()
        try:
            all_dialouts = requests.post(url_list, headers=headers, json={'token': token}).json()
            for_output = """

                    ID автообзвона
                    Дата начала автообзвона
                    Дата окончания автообзвона
                    Статус автообзвона

"""
            for id_num in all_dialouts['result']:
                tsstart = time.ctime(id_num['tsstart'])
                tsend = time.ctime(id_num['tsend'])
                if id_num['status'] == 1:
                    status = 'Новый'
                elif id_num['status'] == 2:
                    status = 'Обработан'
                elif id_num['status'] == 3:
                    status = 'Запущен'
                elif id_num['status'] == 4:
                    status = 'Выполнен'
                elif id_num['status'] == 5:
                    status = 'Остановлен'
                else:
                    status = 'Ошибка'
                for_output = for_output + "
" + str(id_num['id']) + "	" + tsstart + \
                             "	" + tsend + "	" + status + "
"
            self.output_message(for_output)
        except:
            message_to_ui = "Не правильно введен токен"
            self.output_message(message_to_ui)

    def get_info(self):
        token = self.token_enter.text()
        id_dialout = self.dialout.text()
        try:
            all_dialouts = requests.post(url_get_info, headers=headers,
                                         json={'token': token, 'dialoutid': id_dialout}).json()
            for_output = """

                    Номер
                    Время начала звонка
                    Статус звонка
                    Статус вызова
                    Длительность вызова

"""
            for info_num in all_dialouts['result']['data']:
                if (info_num['duration'] // 60) == 0:
                    duration = str(info_num['duration'] % 60) + ' сек'
                else:
                    duration = str((info_num['duration'] // 60)) + ' мин ' + str(info_num['duration'] % 60) + ' сек'

                if info_num['callstatus'] == 1:
                    callstatus = 'Новый'
                elif info_num['callstatus'] == 2:
                    callstatus = 'Активный'
                else:
                    callstatus = 'Завершен'

                if info_num['intstatus'] == 1:
                    intstatus = 'Новый'
                elif info_num['intstatus'] == 2:
                    intstatus = 'Получен ответ'
                elif info_num['intstatus'] == 3:
                    intstatus = 'Занято'
                elif info_num['intstatus'] == 4:
                    intstatus = 'Не сняли трубку'
                else:
                    intstatus = 'Ошибка'

                for_output = for_output + "
" + str(info_num['number']) + "	" + info_num['tscall'] + \
                             "	" + callstatus + "	" + intstatus + "	" + duration + "
"
            self.output_message(for_output)
        except:
            message_to_ui = "Что-то пошло не так"
            self.output_message(message_to_ui)

def main():
    app = QtWidgets.QApplication(sys.argv)  # Новый экземпляр QApplication
    window = ExampleApp()  # Создаём объект класса ExampleApp
    window.show()  # Показываем окно
    app.exec_()  # и запускаем приложение

if __name__ == '__main__':  # Если мы запускаем файл напрямую, а не импортируем
    main()  # то запускаем функцию main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from itertools import chain, combinations, product
from pprint import pprint
import pickle
import csv

########################
#  Simple hierarchies  #
########################

number_hrcs = {'sp': {('s', 'p')},
               'ps': {('p', 's')}}

person_hrcs = {'123': {('1', '2'), ('2', '3'), ('1', '3')},
               '12|3': {('1', '2'), ('1', '3')},
               '1|23': {('1', '3'), ('2', '3')}}

########################
#  Build Crossproduct  #
########################

def extract_elements(hierarchy: set) -> set:
    return set([element for ranking in hierarchy for element in ranking])

def add_ranking(node1: str, node2: str, person: set, number: set) -> tuple:
    person1, number1 = node1
    person2, number2 = node2
    if node1 != node2 and\
       (person1 == person2 or (person1, person2) in person) and\
       (number1 == number2 or (number1, number2) in number):
        return (node1, node2)

def crossalgebra(person: set, number: set) -> set:
    persons = extract_elements(person)
    numbers = extract_elements(number)
    nodes = set([person+number
                 for person in persons
                 for number in numbers])

    output = set(add_ranking(node, other_node, person, number)
                 for node in nodes
                 for other_node in nodes)
    output.remove(None)
    return output

######################################
#  Compute Algebras with Added Arcs  #
######################################

def reachability_closure(algebra: set) -> set:
    new_algebra = algebra.copy()
    for ranking in algebra:
        for other_ranking in algebra:
            if ranking[1] == other_ranking[0] and\
               ranking[0] != other_ranking[1]:
                new_algebra.add((ranking[0], other_ranking[1]))
    if new_algebra == algebra:
        return new_algebra
    else:
        return reachability_closure(new_algebra)

def algebra_expansions(algebra: set, possible_additions: iter) -> iter:
    for additions in possible_additions:
        # print("adding", additions)
        yield algebra.copy().union(additions)

def arc_addition(algebra: set, person: set, number: set) -> set:
    # compute elements
    persons = extract_elements(person)
    numbers = extract_elements(number)

    # set of all arcs
    nodes = set(p+n for p in persons for n in numbers)
    all_arcs = set((n1, n2) for n1 in nodes for n2 in nodes
                   if n1 != n2)

    # remove arcs that are already in the algebra
    all_arcs -= algebra

    # the powerset of all remaining arcs,
    # i.e. the set of all possible algebra extensions;
    # this is an iterator, so memory usage should be manageable
    possible_additions = chain.from_iterable(combinations(all_arcs, r)
                                             for r in range(len(all_arcs)+1))

    # compute iterator of all algebras with 0 or more added arcs
    algebras = algebra_expansions(algebra, possible_additions)

    # close each algebra under reachability
    closed_algebras = set(frozenset(reachability_closure(algebra))
                          for algebra in algebras)

    # return set of all remaining algebras
    return closed_algebras

######################################
#  Compute Syncretisms from Algebra  #
######################################

def algebra_interpretation(algebra: set) -> tuple:
    presentation = ['1s', '2s', '3s', '1p', '2p', '3p']
    syncretism = [0]
    for cell in presentation[1:]:
        threshold = len(syncretism)
        for pos in range(threshold):
            other_cell = presentation[pos]
            if (other_cell, cell) in algebra and\
               (cell, other_cell) in algebra:
                value = syncretism[pos]
                syncretism.append(value)
                break
        if len(syncretism) == threshold:
            syncretism.append(max(syncretism)+1)
    return tuple(syncretism)

def number_to_letter(syncretism: tuple) -> tuple:
    return tuple([chr(65 + number) for number in syncretism])

def syn_patterns(algebras: set) -> set:
    return set(number_to_letter(algebra_interpretation(algebra))
               for algebra in algebras)

def all_syncretisms() -> dict:
    syncretisms = {}
    for num_label, num_hier in number_hrcs.items():
        for pers_label, pers_hier in person_hrcs.items():
            algebra = crossalgebra(pers_hier, num_hier)
            new_algebras = arc_addition(algebra, pers_hier, num_hier)
            syncretisms[(pers_label, num_label)] = \
                syn_patterns(new_algebras)
    syncretisms['total'] = set().union(*syncretisms.values())
    return syncretisms

try:
    syncretisms = pickle.load(open("syncretisms.p", "rb"))
except:
    syncretisms = all_syncretisms()
    pickle.dump(syncretisms, open("syncretisms.p", "wb"))

##################
#  Analyze Data  #
##################

def read_data(filename: str='data.csv') -> list:
    patterns = []
    with open('data.csv') as datafile:
        data = csv.reader(datafile)
        for row in data:
            patterns.append(tuple(row[2:8]))
    return patterns

def overgeneration(algebra: set, data: list) -> list:
    return [pattern for pattern in algebra if pattern not in data]

def undergeneration(algebra: set, data: list) -> list:
    return [pattern for pattern in data if pattern not in algebra]

def all_generation(function=overgeneration) -> dict:
    data = list(set(read_data()))
    return {key: set(function(syncretisms[key], data))
            for key in syncretisms.keys()}

def show_generation(function=overgeneration):
    generated = all_generation(function=function)

    for key, val in generated.items():
        print(key, ":", len(val))
        pprint(sorted(list(val)))
        print("--------------")

def all_overgeneration() -> dict:
    return all_generation(function=overgeneration)

def all_undergeneration() -> dict:
    return all_generation(function=undergeneration)

def show_overgeneration():
    return show_generation(function=overgeneration)

def show_undergeneration():
    return show_generation(function=undergeneration)

def show_generators(algebras: dict, patterns: list) -> dict:
    overview = {}
    for pattern in patterns:
        overview[pattern] = set()
        for algebra, generated in algebras.items():
            if pattern in generated and algebra != 'total':
                overview[pattern].add(algebra)
    return overview

#############################
#  Monotonicity Calculator  #
#############################

base_algebras = {(plab, nlab): reachability_closure(crossalgebra(p, n))
                 for plab, p in person_hrcs.items()
                 for nlab, n in number_hrcs.items()}

def relabel_algebra(algebra, syncretism):
    presentation = ['1s', '2s', '3s', '1p', '2p', '3p']
    labeling = dict(zip(presentation, syncretism))
    return {(labeling[a], labeling[b])
            for (a, b) in algebra}

def test_monotonicity(algebra, syncretism):
    algebra = relabel_algebra(algebra, syncretism)
    for a, b in algebra:
        if a != b:
            for c, d in algebra:
                if (a, b) == (d, c):
                    return False
    return True

def is_monotonic(algebras, syncretism):
    return [key for key, val in algebras.items()
            if test_monotonicity(val, syncretism)]

def show_monotonicity(algebras=base_algebras, data=read_data()):
    return {syncretism: is_monotonic(algebras, syncretism)
            for syncretism in data}

def all_patterns(cells=6):
    letters = [[chr(65 + number) for number in range(cells)]
               for _ in range(cells)]
    return [i for i in product(*letters)]

try:
    patterns = pickle.load(open("patterns.p", "rb"))
except:
    patterns = all_patterns()
    pickle.dump(patterns, open("patterns.p", "wb"))

def all_monotonicity():
    return show_monotonicity(base_algebras, patterns)

def letter_to_number(xs):
    if not xs:
        return xs

    ys = [1]
    for pos in range(1, len(xs)):
        x = xs[pos]
        for left in range(pos):
            if x == xs[left]:
                ys.append(ys[left])
                break
        else:
            ys.append(max(ys) + 1)
    return ys

def isomorphic(xs, ys):
    return letter_to_number(xs) == letter_to_number(ys)

def monotonicity_overgeneration():
    all_monotonic = {key: val
                     for key, val in all_monotonicity().items()
                     if val}

    unique_monotonic = {}
    for key, val in all_monotonic.items():
        for key2, val2 in unique_monotonic.items():
            if isomorphic(key, key2):
                break
        else:
            unique_monotonic[key] = val

    attested_monotonic = show_monotonicity()

    return {key: val
            for key, val in unique_monotonic.items()
            for key2 in attested_monotonic
            if not isomorphic(key, key2)}

try:
    monotonic_overgen = pickle.load(open("monotonic_overgen.p", "rb"))
except:
    monotonic_overgen = monotonicity_overgeneration()
    pickle.dump(monotonic_overgen, open("monotonic_overgen.p", "wb"))


import sys
# basic libraries
import pandas as pd
#SQL library
from sqlalchemy import create_engine
# NLP
import re
import nltk
from nltk.tokenize import word_tokenize
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.stem.porter import PorterStemmer
nltk.download(['punkt', 'wordnet'])
# Machine Learning
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.multioutput import MultiOutputClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV
# pickle to store model
import pickle

def load_data(database_filepath):
    """
    Load disaster dateset from SQLite & split dataset into messages and categories

    Input: database_filepath: path of SQLite database file of processed dataset
    Return: X: Message part of disaster_message dataframe
        y: category part of disaster_message dataframe
        categories: names of categories, columns of y
    """
    # Load SQLite dataset
    engine = create_engine('sqlite:///' + database_filepath)
    df = pd.read_sql_table('disaster_messages', engine)

    # split dataset and return message, categories, and column names
    X = df['message']
    y = df.iloc[:, 4:]
    category_names = y.columns.tolist()

    return X, y, category_names

def tokenize(text):
    """
    Clean and tokenize the message

    Input: text: the message for tokenization(X from load_data)
    Return: clean_tokens: token list of message
    """
    # remove non-numerical non-character types and tokenize text
    tokens = word_tokenize(re.sub(r"[^a-zA-Z0-9]", " ", text.lower()))

    # Instantiate lemmatizer and stemmer
    wnl = WordNetLemmatizer()
    ps = PorterStemmer()

    # lemmatize tokens of nouns and verbs and stemming
    clean_tokens = []
    for tok in tokens:
        clean_tok = wnl.lemmatize(tok)
        clean_tok = wnl.lemmatize(clean_tok, 'v')
        clean_tok = ps.stem(clean_tok)
        clean_tokens.append(clean_tok.strip())

    return clean_tokens

def build_model():
    """
    Builds the pipeline of NLP + Classification, Tf-Idf as message
    transformation and AdaBoost+MultiOutputClassifier as
    classification model, along with GridSearchCV strategy

    Return: model: machine learning model described above
    """
    # parameter set for GridSearchCV
    parameters = {
        'tfidf__max_df': (0.9, 1.0),
        'tfidf__ngram_range': ((1, 1), (1, 3))
    }

    # pipeline containing Tf-Idf and AdaBoost
    pipeline = Pipeline([
        ('tfidf', TfidfVectorizer(tokenizer=tokenize)),
        ('clf', MultiOutputClassifier(AdaBoostClassifier(n_estimators=100, random_state=42)))
    ])

    # Instantiate GridSearchCV
    model = GridSearchCV(pipeline, param_grid=parameters)

    return model

def evaluate_model(model, X_test, Y_test, category_names):
    """
    Evaluate model performance and print metrics of validation

    Input:model: classification model from build_model()
        X_test: test set of X(messages)
        Y_test: test set of y(categories)
        category_names: names of message categories(categories column names)
    """
    # print metrics of classification for each column
    Y_pred = pd.DataFrame(model.predict(X_test), columns=category_names)
    print("Classification Accuracy")
    for i in range(36):
        print("Category Name: ",category_names[i])
        print(classification_report(Y_test.iloc[:, i], Y_pred.iloc[:, i]))

def save_model(model, model_filepath):
    """
    Save classification model to pickle file

    Input:model: validated classification model
        model_filepath: specified storage path
    """
    pickle.dump(model, open(model_filepath, 'wb'))

def main():
    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

        print('Building model...')
        model = build_model()

        print('Training model...')
        model.fit(X_train, Y_train)

        print('Evaluating model...')
        evaluate_model(model, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(model, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database ' \
              'as the first argument and the filepath of the pickle file to ' \
              'save the model to as the second argument. \n\nExample: python ' \
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
    main()


"""
Demo the model at test time
"""
import os
import pickle
import yaml
import numpy as np

from utils import load_data, get_wide_deep_model, process_data, parse_cli_args

def main():
    """Main block of code. Loads the data, model and vectoriser and shows a demo"""
    args = parse_cli_args()
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # Supress TF warnings

    print("Loading data...")
    X, y = load_data(
        dataset_path=args.data_path,
        feature_field=args.features_field,
        target_field=args.target_field,
    )
    # Choose five random examples to show
    random_idx = (np.random.rand(5)*len(X)).astype(int)
    X = [X[idx] for idx in random_idx]
    y = [y[idx] for idx in random_idx]

    X_wide_deep = process_data(
        text_feature=X,
        count_vec=pickle.load(open(args.vectoriser_path, "rb")),
    )

    print("Constructing Keras model")
    prediction_model = get_wide_deep_model(
        num_wide_features=X_wide_deep[0].shape[1],
        num_deep_features=X_wide_deep[1].shape[1],
        **yaml.safe_load(open(args.model_config, "r")),
    )
    prediction_model.load_weights(args.model_path)
    print("Predicting...")
    predictions = prediction_model.predict([X_wide_deep[0], X_wide_deep[1]], verbose=0)

    for prediction, text, target in zip(predictions, X, y):
        print("="*100)
        print("Text:\n", text)
        print("Target:", target)
        print("Model's prediction:", prediction)

if __name__ == "__main__":
    main()


import os
from flask import Flask
from flask import render_template
app = Flask(__name__, template_folder='./templates')

@app.route('/')
def hello_world():
	name = os.environ.get("MAIN_TEXT")
	color = os.environ.get("COLOR")
	print(color)
	print(name)
	return render_template('index.html', name=name, color=color)


# show the learned policy

import gym
import pickle
import numpy as np
from copy import deepcopy

path_q_table = 'q_table.pickle'

with open(path_q_table, 'rb') as f:
    q_table = pickle.load(f)

NUM_OBS = q_table.shape[0]

env = gym.make('MountainCar-v0', render_mode='human')

start = env.reset()

obs, _ = start

done = False

# we need to round the obs space to the defined number of values
obs_min = env.observation_space.low
obs_max = env.observation_space.high
n_obs = len(obs_min)

# create concrete bins for the observations
obs_bins = []

# do this for all observations
for i in range(n_obs):
    obs_bin = np.arange(obs_min[i], obs_max[i] + 0.0001, (obs_max[i] - obs_min[i]) / NUM_OBS)
    obs_bins.append(obs_bin)

total_reward = 0
episodes = 0

while not done:

    # make the observation discrete
    obs_cat = []

    for bins, o in zip(obs_bins, obs):

        # subtract the obs from the bins and make each number positive to get the closest bin using argmin
        bin = deepcopy(bins) - o
        bin = [abs(b) for b in bin]
        bin = np.argmin(bin)  # this is the current obs rounded

        obs_cat.append(bin)

    best_action = np.argmax(q_table[obs_cat[0], obs_cat[1], :])

    obs, reward, done, _, _ = env.step(best_action)

    total_reward += reward
    episodes += 1

print(f'Finished after {episodes} with {total_reward} reward')


import logging
import magic
import os

from cms.medias.utils import get_file_type_size
from django.conf import settings
from django.core.files.uploadedfile import InMemoryUploadedFile

from . import settings as app_settings
from . utils import to_webp

logger = logging.getLogger(__name__)
FILETYPE_IMAGE = getattr(settings, 'FILETYPE_IMAGE',
                         app_settings.FILETYPE_IMAGE)

def set_file_meta(media_object):
    data = get_file_type_size(media_object)
    media_object.file_size = data['file_size']
    media_object.file_type = data['mime_type']

def webp_image_optimizer(media_object):

    for field_name in ('file', 'image'):
        field = getattr(media_object, field_name, None)
        if field:
            break

    if not getattr(field, '_file', None): # pragma: no cover
        return

    mimetype = magic.Magic(mime=True).from_buffer(field._file.file.read())
    if mimetype in FILETYPE_IMAGE:
        field._file.seek(0)
        byte_io = to_webp(field._file)
        byte_io.seek(0, os.SEEK_END)
        content_size = byte_io.tell()
        byte_io.seek(0)

        fname = '.'.join(field.name.split('.')[:-1]) + '.webp'
        field._file = InMemoryUploadedFile(file = byte_io,
                                           name = fname,
                                           content_type = 'image/webp',
                                           size = content_size,
                                           charset='utf-8',
                                           field_name = field_name)
        field._file._name = fname
        field.name = fname

        field._file.size = content_size
        field._file.content_type = 'image/webp'

        # if they are valuable ... otherwise nothins happens to model
        media_object.file_size = content_size
        media_object.file_type = 'image/webp'
        logger.info(f'Image {fname} converted from {mimetype} to {media_object.file_type}')
        return True

def remove_file(media_object):
    fpath = media_object.file.path
    try:
        os.remove(fpath)
    except Exception as e: # pragma: no cover
        _msg = 'Media Hook remove_file: {} cannot be removed: {}'
        logger.error(_msg.format(fpath, e))


from module import HandDetector
import cv2
import math
import time
import pygame
import serial

Arduino = serial.Serial(port='COM3  ', baudrate=9600)

class Timer:
    def _init_(self, time_between=5):
        self.start_time = time.time()
        self.time_between = time_between

    def can_send(self):
        if time.time() > (self.start_time + self.time_between):
            self.start_time = time.time()
            return True
        return False

def cvimage_to_pygame(image):
    return pygame.image.frombuffer(image.tostring(), image.shape[1::-1], "RGB")

cap = cv2.VideoCapture(0)
detector = HandDetector(maxHands=2, minTrackCon=0.6, detectionCon=0.6)

width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)

SCREEN = pygame.display.set_mode((int(width), int(height)))

wheel = pygame.image.load('wheel.png')

REVERSE = False
orientation = "Forward"

timer  = Timer(time_between=5)

Running = True
while Running:
    flag = False
    success, img = cap.read()
    hands = detector.findHands(img, draw=False)

    for event in pygame.event.get():
            if event.type == pygame.QUIT:
                Running = False

    if len(hands) == 1:
        hand = hands[0]
        fingers = detector.fingersUp(hand)
        print(fingers)
        if fingers == [1, 1, 1, 1, 1]:
            if timer.can_send():
                REVERSE = not REVERSE

    elif len(hands) == 2:
        flag = True
        hand1 = hands[0]
        hand2 = hands[1]
        centerPoint1 = hand1['center']
        centerPoint2 = hand2['center']

        x1, y1 = centerPoint1
        x2, y2 = centerPoint2

        stearing_center = (int((x1 + x2)/2), int((y2 + y1)/2))
        distance = int(math.hypot(x2 - x1, y2 - y1)/2)

        if REVERSE == True:
            orientation = "REVERSE"
        else:
            orientation = "FORWARD"
        command = orientation
        try:
            slope = (y2 - y1) / (x2 - x1)
            if slope > 1:

                # print("Go LEFT")
                command += " GO LEFT\n"
                # Arduino.write(b"Left\n")

            elif slope < -1:
                # print("GO RIGHT")
                command += " GO RIGHT\n"
                # Arduino.write(b"Right\n")
            else:
                # print("Forward")
                command += " GO STRAIGHT\n"
                # Arduino.write(b"Forward\n")
        except:
            pass
    else:
        # print("Stop")
        command = "STOP\n"
        # Arduino.write(b"Stop\n")

    Arduino.write(bytes(command, 'utf-8'))
    print(command.strip())

    img = cv2.flip(img, 1)
    cv2.putText(img=img, text=str(orientation), org=(10, 20), fontFace=cv2.FONT_HERSHEY_TRIPLEX, fontScale=1, color=(0, 255, 0),thickness=3)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img = cvimage_to_pygame(img)
    SCREEN.blit(img, (0, 0))
    if flag:
        wheel1 =  pygame.transform.scale(wheel, (distance*2, distance*2))
        x, y = stearing_center
        x -= distance
        y -= distance
        x = width - x - (distance*2)
        wheel1 = pygame.transform.rotate(wheel1, math.degrees(math.atan(slope)))
        SCREEN.blit(wheel1, (x, y))
    pygame.display.update()

pygame.quit()
cap.release()


import copy
import json
import logging
import optparse
import os
import random
import sys
import threading

import customtabs_benchmark

_SRC_PATH = os.path.abspath(os.path.join(
    os.path.dirname(__file__), '..', '..', '..', '..'))

sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil'))
from devil.android import device_utils

sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
import devil_chromium

_KEYS = ['url', 'warmup', 'no_prerendering', 'delay_to_may_launch_url',
         'delay_to_launch_url', 'cold']

def _ParseConfiguration(filename):
  """Reads a JSON file and returns a list of configurations.

  Each valid value in the JSON file can be either a scalar or a list of
  values. This function expands the scalar values to be lists. All list must
  have the same length.

  Sample configuration:
  {
    "url": "https://www.android.com",
    "warmup": [false, true],
    "no_prerendering": false,
    "delay_to_may_launch_url": [-1, 1000],
    "delay_to_launch_url": [-1, 1000],
    "cold": true
  }

  Args:
    filename: (str) Point to a file containins a JSON dictionnary of config
              values.

  Returns:
    A list of configurations, where each value is specified.
  """
  config = json.load(open(filename, 'r'))
  has_all_values = all(k in config for k in _KEYS)
  assert has_all_values
  config['url'] = str(config['url']) # Intents don't like unicode.
  has_list = any(isinstance(config[k], list) for k in _KEYS)
  if not has_list:
    return [config]
  list_keys = [k for k in _KEYS if isinstance(config[k], list)]
  list_length = len(config[list_keys[0]])
  assert all(len(config[k]) == list_length for k in list_keys)
  result = []
  for i in range(list_length):
    result.append(copy.deepcopy(config))
    for k in list_keys:
      result[-1][k] = result[-1][k][i]
  return result

def _CreateOptionParser():
  parser = optparse.OptionParser(description='Loops tests on all attached '
                                 'devices, with randomly selected '
                                 'configurations, and outputs the results in '
                                 'CSV files.')
  parser.add_option('--config', help='JSON configuration file. Required.')
  parser.add_option('--output_file_prefix', help='Output file prefix. Actual '
                    'output file is prefix_<device ID>.csv', default='result')
  return parser

def _RunOnDevice(device, output_filename, configs, should_stop):
  """Loops the tests described by configs on a device.

  Args:
    device: (DeviceUtils) device to run the tests on.
    output_filename: (str) Output file name.
    configs: (list of dict) List of configurations.
    should_stop: (Event) When set, this function should return.
  """
  with open(output_filename, 'a') as f:
    while not should_stop.is_set():
      config = configs[random.randint(0, len(configs) - 1)]
      result = customtabs_benchmark.RunOnce(
          device, config['url'], config['warmup'], config['no_prerendering'],
          config['delay_to_may_launch_url'], config['delay_to_launch_url'],
          config['cold'])
      if result is not None:
        f.write(result + '\n')
        f.flush()
      should_stop.wait(10.)

def _Run(output_file_prefix, configs):
  """Loops the tests described by the configs on connected devices.

  Args:
    output_file_prefix: (str) Prefix for the output file name.
    configs: (list of dict) List of configurations.
  """
  devices = device_utils.DeviceUtils.HealthyDevices()
  should_stop = threading.Event()
  threads = []
  for device in devices:
    output_filename = '%s_%s.csv' % (output_file_prefix, str(device))
    thread = threading.Thread(
        target=_RunOnDevice,
        args=(device, output_filename, configs, should_stop))
    thread.start()
    threads.append(thread)
  for thread in threads:
    try:
      thread.join()
    except KeyboardInterrupt as e:
      should_stop.set()

def main():
  parser = _CreateOptionParser()
  options, _ = parser.parse_args()
  if options.config is None:
    logging.error('A configuration file must be provided.')
    sys.exit(0)
  devil_chromium.Initialize()
  configs = _ParseConfiguration(options.config)
  _Run(options.output_file_prefix, configs)

if __name__ == '__main__':
  main()


# -*- coding: utf-8 -*-

from processr.processr import (
    rename_keys,
    project_dict,
    transform_values,
    transform_dict,
    transform_values_strict)

import pytest

##############################################################
#                        rename_keys                         #
##############################################################

def test_rename_keys():
    d = {'the_answer': 42}
    opts = {'the_answer': 'the_universe_and_everything'}
    expected_output = {'the_universe_and_everything': 42}

    output = rename_keys(d, opts)
    assert output == expected_output

def test_rename_keys_extra_fields():
    d = {'the_answer': 42, 'not_the_answer': 43}
    opts = {'the_answer': 'the_universe_and_everything'}
    expected_output = {'the_universe_and_everything': 42, 'not_the_answer': 43}

    output = rename_keys(d, opts)
    assert output == expected_output

##############################################################
#                          project                           #
##############################################################

def test_project_dict():
    d = {'the_answer': 42, 'not_the_answer': 43}
    opts = ('the_answer', )
    expected_output = {'the_answer': 42}

    output = project_dict(d, opts)
    assert output == expected_output

def test_project_dict_extra_fields():
    d = {'the_answer': 42, 'not_the_answer': 43}
    opts = ('the_answer', 'the_universe')

    with pytest.raises(KeyError):
        project_dict(d, opts)

##############################################################
#                      transform_values                      #
##############################################################

def test_transform_values():
    d = {'not_the_answer': [9, 9, 9, 9, 9, 9]}
    opts = {'not_the_answer': [sum, str]}

    # I always thought something was fundamentally wrong with the universe
    expected_output = {'not_the_answer': '54'}
    output = transform_values(d, opts)
    assert output == expected_output

def test_transform_values_extra_fields():
    d = {'not_the_answer': [9, 9, 9, 9, 9, 9], 'the_answer': 42}
    opts = {'not_the_answer': [sum, str]}

    expected_output = {'not_the_answer': '54', 'the_answer': 42}
    output = transform_values(d, opts)
    assert output == expected_output

def test_transform_values_non_strictness():
    d = {'not_the_answer': [9, 9, 9, 9, 9, 9]}
    opts = {'the_answer': [str]}

    expected_output = {'not_the_answer': [9, 9, 9, 9, 9, 9]}
    output = transform_values(d, opts)
    assert output == expected_output

def test_transform_values_strict():
    d = {'not_the_answer': [9, 9, 9, 9, 9, 9]}
    opts = {'not_the_answer': [sum, str]}

    expected_output = {'not_the_answer': '54'}
    output = transform_values_strict(d, opts)
    assert output == expected_output

def test_transform_values_strict_ko():
    d = {'not_the_answer': [9, 9, 9, 9, 9, 9]}
    opts = {'the_answer': [str]}

    with pytest.raises(KeyError):
        transform_values_strict(d, opts)

##############################################################
#                       transform_dict                       #
##############################################################

def test_dict_transform():
    d = {'the_answer': 42, 'not_the_answer': 43}
    ops = [
        lambda d: dict((v, k) for k, v in d.items())  # Invert key and values
    ]

    expected_output = {42: 'the_answer', 43: 'not_the_answer'}
    output = transform_dict(d, ops)
    assert output == expected_output


from django import forms

topic_choices = (
    ('general', 'General enquiry'),
    ('bug' 'Report a bug'),
    ('suggestion', 'Suggestion'),
)

class ContactForm(forms.Form):
    topic = forms.ChoiceField(choices=topic_choices)
    message = forms.CharField(widget=forms.Textarea())
    sender = forms.EmailField(required=False)

    # def clean_message(self):
    #     message = self.clean_data.get('message', '')
    #     num_words = len(message.spit())
    #     if num_words < 4:
    #         raise forms.ValidationError('Max. 4 chars!')
    #     return message


import torch
import torch.nn as nn
from torch import Tensor

from nugi_rl.model.components.Transformer import EncoderLayer, DecoderLayer

class GlobalExtractor(nn.Module):
    def __init__(self, dim: int, num_layers = 2) -> None:
        super().__init__()

        object_queries = torch.ones(1, 1, dim)
        self.register_buffer('object_queries', object_queries)

        self.encoder    = nn.ModuleList([EncoderLayer(dim, heads = 4, b = 8) for _ in range(num_layers)])
        self.decoder    = nn.ModuleList([DecoderLayer(dim, heads = 4, b = 8) for _ in range(num_layers)])

    def forward(self, inputs: Tensor, mask: Tensor) -> Tensor:
        src_mask        = self.transform_mask(mask)
        src_embeddings  = inputs
        tgt_embeddings  = self.object_queries

        for layer in self.encoder:
            src_embeddings = layer(src_embeddings, src_mask)

        for layer in self.decoder:
            tgt_embeddings = layer(tgt_embeddings, src_embeddings)

        return tgt_embeddings

    def transform_mask(self, mask: Tensor) -> Tensor:
        mask = mask != -1
        return mask.unsqueeze(1).unsqueeze(2)

class Policy_Model(nn.Module):
    def __init__(self, state_dim: int, action_dim: int):
        super(Policy_Model, self).__init__()

        self.feedforward_1 = nn.Sequential(
            nn.Linear(state_dim, 256),
            nn.GELU(),
            nn.Linear(256, 64),
            nn.GELU(),
        )

        self.feedforward_3 = nn.Sequential(
            nn.Linear(8, 64),
            nn.GELU(),
        )

        self.extractors = GlobalExtractor(64)

        self.feedforward_2 = nn.Sequential(
            nn.Linear(128, 16),
            nn.GELU(),
            nn.Linear(16, action_dim),
            nn.Softmax(-1)
        )

    def forward(self, states: Tensor, detach: bool = False) -> tuple:
        masks = states[0][:, :, -1]

        x = self.feedforward_1(states[0])
        x = self.extractors(x, masks).squeeze(1)

        x2 = self.feedforward_3(states[1])
        x = torch.cat([x, x2], dim = 1)

        action = self.feedforward_2(x)

        if detach:
            return (action.detach(), )
        else:
            return (action, )

class Value_Model(nn.Module):
    def __init__(self, state_dim: int):
        super(Value_Model, self).__init__()

        self.feedforward_1 = nn.Sequential(
            nn.Linear(state_dim, 256),
            nn.GELU(),
            nn.Linear(256, 64),
            nn.GELU(),
        )

        self.feedforward_3 = nn.Sequential(
            nn.Linear(8, 64),
            nn.GELU(),
        )

        self.extractors = GlobalExtractor(64)

        self.feedforward_2 = nn.Sequential(
            nn.Linear(128, 16),
            nn.GELU(),
            nn.Linear(16, 1)
        )

    def forward(self, states: Tensor, detach: bool = False) -> Tensor:
        masks = states[0][:, :, -1]

        x = self.feedforward_1(states[0])
        x = self.extractors(x, masks).squeeze(1)

        x2 = self.feedforward_3(states[1])
        x = torch.cat([x, x2], dim = 1)

        value = self.feedforward_2(x)

        if detach:
            return value.detach()
        else:
            return value


# -*- coding: utf-8 -*-

from django import forms
from django.core.exceptions import ImproperlyConfigured
from django.forms.models import ModelForm
from django.utils.encoding import smart_text
from django.utils.translation import ugettext as _

from cms.api import create_page
from cms.constants import TEMPLATE_INHERITANCE_MAGIC
from cms.models import Page, UserSettings
from cms.test_utils.testcases import CMSTestCase, TransactionCMSTestCase
from cms.wizards.wizard_base import Wizard
from cms.wizards.wizard_pool import wizard_pool, AlreadyRegisteredException

class WizardForm(forms.Form):
    pass

class ModelWizardForm(ModelForm):
    class Meta:
        model = UserSettings
        exclude = []

class BadModelForm(ModelForm):
    class Meta:
        pass

class WizardTestMixin(object):
    page_wizard = None
    title_wizard = None

    def assertSequencesEqual(self, seq_a, seq_b):
        seq_a = list(seq_a)
        seq_b = list(seq_b)
        zipped = list(zip(seq_a, seq_b))
        if len(zipped) < len(seq_a) or len(zipped) < len(seq_b):
            self.fail("Sequence lengths are not the same.")
        for idx, (a, b) in enumerate(zipped):
            if a != b:
                self.fail("Sequences differ at index {0}".format(idx))

    @classmethod
    def setUpClass(cls):
        super(WizardTestMixin, cls).setUpClass()
        # This prevents auto-discovery, which would otherwise occur as soon as
        # tests start, creating unexpected starting conditions.
        wizard_pool._discovered = True

        class PageWizard(Wizard):
            pass

        # This is a basic Wizard
        cls.page_wizard = PageWizard(
            title=_(u"Page"),
            weight=100,
            form=WizardForm,
            model=Page,
            template_name='my_template.html',  # This doesn't exist anywhere
        )

        class SettingsWizard(Wizard):
            pass

        # This is a Wizard that uses a ModelForm to define the model
        cls.user_settings_wizard = SettingsWizard(
            title=_(u"UserSettings"),
            weight=200,
            form=ModelWizardForm,
        )

        class TitleWizard(Wizard):
            pass

        # This is a bad wizard definition as it neither defines a model, nor
        # uses a ModelForm that has model defined in Meta
        cls.title_wizard = TitleWizard(
            title=_(u"Page"),
            weight=100,
            form=BadModelForm,
            template_name='my_template.html',  # This doesn't exist anywhere
        )

class TestWizardBase(WizardTestMixin, TransactionCMSTestCase):

    def test_str(self):
        self.assertEqual(str(self.page_wizard), self.page_wizard.title)

    def test_repr(self):
        self.assertEqual(self.page_wizard.__repr__(), 'Wizard: "Page"')

    def test_user_has_add_permission(self):
        # Test does not have permission
        user = self.get_staff_user_with_no_permissions()
        self.assertFalse(self.page_wizard.user_has_add_permission(user))

        # Test has permission
        user = self.get_superuser()
        self.assertTrue(self.page_wizard.user_has_add_permission(user))

    def test_get_success_url(self):
        user = self.get_superuser()
        page = create_page(
            title="Sample Page",
            template=TEMPLATE_INHERITANCE_MAGIC,
            language="en",
            created_by=smart_text(user),
            parent=None,
            in_navigation=True,
            published=False
        )
        url = "{0}?edit".format(page.get_absolute_url(language="en"))
        self.assertEqual(self.page_wizard.get_success_url(
            page, language="en"), url)

        # Now again without a language code
        url = "{0}?edit".format(page.get_absolute_url())
        self.assertEqual(self.page_wizard.get_success_url(page), url)

    def test_get_model(self):
        self.assertEqual(self.page_wizard.get_model(), Page)
        self.assertEqual(self.user_settings_wizard.get_model(), UserSettings)
        with self.assertRaises(ImproperlyConfigured):
            self.title_wizard.get_model()

class TestWizardPool(WizardTestMixin, CMSTestCase):

    def test_discover(self):
        wizard_pool._reset()
        self.assertFalse(wizard_pool._discovered)
        self.assertEqual(len(wizard_pool._entries), 0)
        wizard_pool._discover()
        self.assertTrue(wizard_pool._discovered)

    def test_register_unregister_isregistered(self):
        wizard_pool._clear()
        self.assertEqual(len(wizard_pool._entries), 0)
        wizard_pool.register(self.page_wizard)
        # Now, try to register the same thing
        with self.assertRaises(AlreadyRegisteredException):
            wizard_pool.register(self.page_wizard)

        self.assertEqual(len(wizard_pool._entries), 1)
        self.assertTrue(wizard_pool.is_registered(self.page_wizard))
        self.assertTrue(wizard_pool.unregister(self.page_wizard))
        self.assertEqual(len(wizard_pool._entries), 0)

        # Now, try to unregister something that is not registered
        self.assertFalse(wizard_pool.unregister(self.user_settings_wizard))

    def test_get_entry(self):
        wizard_pool._clear()
        wizard_pool.register(self.page_wizard)
        entry = wizard_pool.get_entry(self.page_wizard)
        self.assertEqual(entry, self.page_wizard)

    def test_get_entries(self):
        """
        Test that the registered entries are returned in weight-order, no matter
        which order they were added.
        """
        wizard_pool._clear()
        wizard_pool.register(self.page_wizard)
        wizard_pool.register(self.user_settings_wizard)
        wizards = [self.page_wizard, self.user_settings_wizard]
        wizards = sorted(wizards, key=lambda e: getattr(e, 'weight'))
        entries = wizard_pool.get_entries()
        self.assertSequencesEqual(entries, wizards)

        wizard_pool._clear()
        wizard_pool.register(self.user_settings_wizard)
        wizard_pool.register(self.page_wizard)
        wizards = [self.page_wizard, self.user_settings_wizard]
        wizards = sorted(wizards, key=lambda e: getattr(e, 'weight'))
        entries = wizard_pool.get_entries()
        self.assertSequencesEqual(entries, wizards)


from .helpers import build_dataset
import nussl
import gin
import torch
import os
import logging
from torch import multiprocessing
from ignite.contrib.handlers import ProgressBar
from .handlers import add_train_handlers
from datetime import datetime

@gin.configurable
def build_model_optimizer_scheduler(model_config, optimizer_class,
                                    scheduler_class=None, device='cuda',
                                    verbose=False):
    model = nussl.ml.SeparationModel(
        model_config, verbose=verbose).to(device)
    # the rest of optimizer params comes from gin
    optimizer = optimizer_class(model.parameters())
    scheduler = scheduler_class(optimizer)
    return model, optimizer, scheduler

@gin.configurable
def train(batch_size, loss_dictionary, num_data_workers, seed,
          output_folder, num_epochs, val_loss_dictionary=None,
          val_batch_size=None, combination_approach='combine_by_sum',
          device='cuda', epoch_length=None, resume=False,
          save_by_epoch=None):
    nussl.utils.seed(seed)

    with gin.config_scope('train'):
        train_dataset = build_dataset()
    with gin.config_scope('val'):
        val_dataset = build_dataset()

    model, optimizer, scheduler = build_model_optimizer_scheduler(device=device)
    logging.info(model)

    if not torch.cuda.is_available():
        device = 'cpu'

    os.makedirs(output_folder, exist_ok=True)
    logging.info(f'Saving to {output_folder}')

    num_data_workers = min(num_data_workers, multiprocessing.cpu_count())
    val_batch_size = batch_size if val_batch_size is None else val_batch_size

    # Set up dataloader
    train_dataloader = torch.utils.data.DataLoader(
        train_dataset, num_workers=num_data_workers,
        batch_size=batch_size, shuffle=True)
    val_dataloader = torch.utils.data.DataLoader(
        val_dataset, num_workers=num_data_workers,
        batch_size=val_batch_size, shuffle=True)

    val_loss_dictionary = (
        loss_dictionary
        if val_loss_dictionary is None
        else val_loss_dictionary
    )

    # Build the closures for each loop
    train_closure = nussl.ml.train.closures.TrainClosure(
        loss_dictionary, optimizer, model,
        combination_approach=combination_approach)
    val_closure = nussl.ml.train.closures.ValidationClosure(
        val_loss_dictionary, model,
        combination_approach=combination_approach)

    # Build the engine and add handlers
    train_engine, val_engine = nussl.ml.train.create_train_and_validation_engines(
        train_closure, val_closure, device=device)
    nussl.ml.train.add_validate_and_checkpoint(
        output_folder, model, optimizer, train_dataset, train_engine,
        val_data=val_dataloader, validator=val_engine, save_by_epoch=save_by_epoch)
    nussl.ml.train.add_stdout_handler(train_engine, val_engine)

    now = datetime.now()
    tensorboard_folder = os.path.join(
        output_folder, 'logs', now.strftime("%Y.%m.%d-%H.%M.%S"))

    logging.info(f'Logging to {tensorboard_folder}')

    nussl.ml.train.add_tensorboard_handler(
        tensorboard_folder, train_engine)
    nussl.ml.train.add_progress_bar_handler(
        train_engine, val_engine)

    # handlers and their config come from gin config
    add_train_handlers(
        train_engine, model, scheduler, optimizer,
        train_closure, device
    )

    # print what we are using
    logging.info(gin.operative_config_str())

    if resume:
        model_path = os.path.join(
            output_folder, 'checkpoints', 'latest.model.pth')
        logging.info(f"Model resuming from {model_path}")
        model_state = torch.load(
            model_path, map_location=lambda storage, loc: storage)
        model.load_state_dict(model_state['state_dict'])

        optimizer_path = os.path.join(
            output_folder, 'checkpoints', 'latest.optimizer.pth')
        logging.info(f"Optimizer resuming from {optimizer_path}")
        optimizer_state = torch.load(
            optimizer_path, map_location=lambda storage, loc: storage)
        optimizer.load_state_dict(optimizer_state)

        logging.info(f"Trainer updating to {model_state['metadata']['trainer.state_dict']}")
        train_engine.load_state_dict(model_state['metadata']['trainer.state_dict'])
        train_engine.state.epoch_history = model_state['metadata']['trainer.state.epoch_history']

    # run the engine
    train_engine.run(
        train_dataloader, max_epochs=num_epochs,
        epoch_length=epoch_length)

@gin.configurable
def cache(num_cache_workers, batch_size, scopes=['train', 'val']):
    num_cache_workers = min(
        num_cache_workers, multiprocessing.cpu_count())
    for scope in scopes:
        with gin.config_scope(scope):
            dataset = build_dataset()
            dataset.cache_populated = False
            cache_dataloader = torch.utils.data.DataLoader(
                dataset, num_workers=num_cache_workers,
                batch_size=batch_size)
            nussl.ml.train.cache_dataset(cache_dataloader)

    alert = "Make sure to change cache_populated = True in your gin config!"
    border = ''.join(['=' for _ in alert])

    logging.info(
        f'\n\n{border}\n'
        f'{alert}\n'
        f'{border}\n'
    )


# coding: utf-8

import asyncio
import requests

from retrying_async import retry

def request_api_sync():
    print('正在获取')

    response = requests.get(url="https://www.google.com")
    print(response.status_code, response.content)
    raise Exception("异常")

@retry(attempts=3, delay=3)
async def request_api_async():
    print('正在获取')

    response = requests.get(url="https://www.google.com")
    print(response.status_code, response.content)
    raise Exception("异常")

if __name__ == '__main__':

    loop = asyncio.get_event_loop()
    loop.run_until_complete(request_api_async())


# -*- coding: utf-8 -*-
from jowent.bannerviewlet import MessageFactory as _
from zope import schema
from zope.interface import Interface
from zope.schema.vocabulary import SimpleVocabulary, SimpleTerm

class IBannerViewletInstalled(Interface):
    """ A layer specific for this add-on product.

    This interface is referred in browserlayer.xml.

    views and viewlets registered against this layer will appear on
    a Plone site only when the add-on installer has been run.
    """

validation_policy_options = SimpleVocabulary(
    [
        SimpleTerm(value='exact', title=_(u'Exact - The supplied images must exactly match the Banner Height & Width specified below')),
        SimpleTerm(value='minimum', title=_(u'Minimum - An image with larger dimensions than below may be supplied, but it will be shrunk to the Banner Height & Width specified below')),
    ])

undersized_banner_behavior_options = SimpleVocabulary(
    [
        SimpleTerm(value='centred', title=_(u'Centred horizontally within container')),
        SimpleTerm(value='stretched', title=_(u'Stretch width-ways to fill parent (maintaining aspect ratio)')),
    ])

class IBannerViewletSettings(Interface):
    """Bannerviewlet settings. This describes records stored in the
    configuration registry and obtainable via plone.registry.
    """

    validation_policy = schema.Choice(
        title=_(u"Banner Validation Policy"),
        description=_(u"How strictly the dimensions of a Banner Slide will be validated"),
        vocabulary=validation_policy_options,
        default='exact',
        required=True,
    )

    required_height = schema.Int(title=_(u"Banner Viewlet Height"),
                                 description=_(u"The height of the banner viewlet"),
                                 required=True,
                                 default=320)

    required_width = schema.Int(title=_(u"Banner Viewlet Width"),
                                description=_(u"The width of the banner viewlet"),
                                required=True,
                                default=2000)

    max_filesize = schema.Int(title=_(u"Maximum Filesize for a Banner Slide (in KB)"),
                              description=_(u"Leave this empty to stop filesize being checked"),
                              required=False,
                              default=200)

    undersized_banner_behavior = schema.Choice(
        title=_(u"Undersized Banner Viewlet Behavior"),
        description=_(u"How the banner viewlet behaves when it's dimensions (specified above) are smaller than it's containing html tag"),
        vocabulary=undersized_banner_behavior_options,
        default='centred',
        required=True,
    )


from bs4 import BeautifulSoup
import requests
from models import Listing
import pandas as pd
from serializer import Serializer
import re
import os

class PreviewScraper:

    def __init__(self, search_query, url="", soup=None):
        self.search_query = search_query
        self.url = url
        self.soup = soup
        self.get_url()
        self.generate_soup()

    def get_url(self):
        self.url = self.search_query.url
        return self.url

    def generate_soup(self):
        self.get_url()
        r = requests.get(self.url)
        soup = BeautifulSoup(r.text, 'html.parser')
        self.soup = soup

    def get_listings_urls(self, first_page_url):
        urls = []
        r = requests.get(first_page_url)
        soup = BeautifulSoup(r.text, 'html.parser')
        main = soup.find("div", {"id": "listing-index"})
        listings_thumbnails = main.find_all("div", {"class": "advert"})
        for l in listings_thumbnails:
            ad_img_link = l.find("a", {"class": "advert__image-link"})
            urls.append(ad_img_link["href"])

        return urls

    def check_last_page(self, url):
        r = requests.get(url)
        soup = BeautifulSoup(r.text, 'html.parser')
        pagination = soup.find("ul", {"class": "pagination"})
        if pagination.find("li", {"class": ["last", "paging-next"]}):
            return False
        else:
            return True

    def get_all_pages_urls(self):
        pages_urls = []

        # LIMITED for testing
        while len(pages_urls) < 50:
            url = self.search_query.url
            pages_urls.append(url)
            print("~~> added page: ", self.search_query.page)
            self.search_query.next_page()

            try:
                if self.check_last_page(url):
                    break
            except:
                continue

        return pages_urls

class ListingScraper:

    def __init__(self, url, soup=None):
        self.url = url
        self.soup = soup
        self.generate_soup()

    def generate_soup(self):
        r = requests.get(self.url)
        soup = BeautifulSoup(r.text, 'html.parser')
        self.soup = soup

    def scrape_tables(self):
        rows = []
        tables = self.soup.find_all("table", class_=["profile-table", "profile__text-data"])
        try:
            for table in tables:
                for sibling in table.tr.next_siblings:
                    row = []
                    print(sibling)
                    for td in sibling:
                        print("td: ", td)
                        try:
                            if td != ' ':
                                row.append(td.text.replace('\n', ''))
                        except:
                            # print("error adding value to row")
                            continue
                    print(row)
                    if len(row) > 0:
                        print(len(row))
                        rows.append(row)
        except:
            print("Error adding rows to the table")
            pass
        return rows

    def scrape_details(self):
      find = self.soup.find
      rows = []
      apt_features = find("p", class_="profile-info-panels__text")
      if apt_features:
          rows.append([apt_features.b.text.replace('\n', ''),
                       apt_features.b.next_sibling.text.replace('\n', '')])

      try:
          description = find("div", class_="profile__text")
          rows.append(["description", description.text.replace('\n', '')])
      except:
          print("couldn't add the description")

      try:
          title = find("div", class_="profile__header-container")
          rows.append(["title", title.text.replace('\n', '')])
      except:
          print("couldn't add the title")

      images_urls = self.scrape_img_urls()
      if images_urls:
          rows.append(["images", images_urls])

      return rows

    def scrape_listing(self):
        # returns a dict with all the values to save
        table = []
        table_rows = self.scrape_tables()
        details_rows = self.scrape_details()
        url_row = [["url", self.url]]
        table += table_rows + details_rows + url_row

        df = pd.DataFrame(table)
        df.dropna()
        print(df)
        return df

    # def scrape_map(self):
    #     # not completed - experimental feature
    #     url = self.url
    #     driver = webdriver.Chrome()
    #     driver.get(url)
    #     time.sleep(5)
    #     driver.set_window_size(100,1000)
    #     # accept cookies
    #     driver.find_element_by_css_selector('.cc_btn.cc_btn_accept_all').click()
    #     driver.find_element_by_class_name("ol-zoom-out").click()
    #     report_button = driver.find_element_by_class_name("report")
    #     map_ = driver.find_element_by_class_name('ol-unselectable')
    #     actions = ActionChains(driver)
    #     actions.move_to_element(report_button).perform()
    #     time.sleep(4)
    #     # save map_
    #     # Upload to dropbox
    #     file_name = re.findall('\d{6}', url)[0]
    #     map_.screenshot(f'{file_name}.png')
    #     dbx.files_upload(open(f'{file_name}.png',"rb").read(),f'/{file_name}.png')
    #     return file_url

    def scrape_img_urls(self):
        images_divs = self.soup.find_all("div", class_="owl-gallery__content")
        images_urls = []
        if images_divs:
            for div in images_divs:
                try:
                    img = div.find("img")["src"]
                    print(img)
                    type(img)
                    images_urls.append(img)
                except:
                    pass

        formatted = []
        for img in set(images_urls):
            formatted.append({"url": img})
        return formatted

    def download_images_urls(self):
        images_urls = self.scrape_img_urls()

# pc = PreviewScraper()
# pc.scrape(listings_thumbnails)


# -*- coding: utf-8 -*-

# Import Python libs
import os
import sys
import shutil
import tempfile
import textwrap
import copy
from cStringIO import StringIO

# Import Salt Testing libs
from salttesting import TestCase
from salttesting.helpers import ensure_in_syspath

ensure_in_syspath('../')

# Import Salt libs
import integration
import salt.loader
import salt.config
import salt.utils
from salt.state import HighState
from salt.utils.pydsl import PyDslError

REQUISITES = ['require', 'require_in', 'use', 'use_in', 'watch', 'watch_in']

OPTS = salt.config.minion_config(None)
OPTS['state_events'] = False
OPTS['id'] = 'whatever'
OPTS['file_client'] = 'local'
OPTS['file_roots'] = dict(base=['/tmp'])
OPTS['cachedir'] = 'cachedir'
OPTS['test'] = False
OPTS['grains'] = salt.loader.grains(OPTS)

class PyDSLRendererTestCase(TestCase):

    def setUp(self):
        self.HIGHSTATE = HighState(OPTS)
        self.HIGHSTATE.push_active()

    def tearDown(self):
        self.HIGHSTATE.pop_active()

    def render_sls(self, content, sls='', env='base', **kws):
        return self.HIGHSTATE.state.rend['pydsl'](
            StringIO(content), env=env, sls=sls, **kws
        )

    def test_state_declarations(self):
        result = self.render_sls(textwrap.dedent('''
            state('A').cmd.run('ls -la', cwd='/var/tmp')
            state().file.managed('myfile.txt', source='salt://path/to/file')
            state('X').cmd('run', 'echo hello world', cwd='/')

            a_cmd = state('A').cmd
            a_cmd.run(shell='/bin/bash')
            state('A').service.running(name='apache')
        '''))
        self.assertTrue('A' in result and 'X' in result)
        A_cmd = result['A']['cmd']
        self.assertEqual(A_cmd[0], 'run')
        self.assertEqual(A_cmd[1]['name'], 'ls -la')
        self.assertEqual(A_cmd[2]['cwd'], '/var/tmp')
        self.assertEqual(A_cmd[3]['shell'], '/bin/bash')

        A_service = result['A']['service']
        self.assertEqual(A_service[0], 'running')
        self.assertEqual(A_service[1]['name'], 'apache')

        X_cmd = result['X']['cmd']
        self.assertEqual(X_cmd[0], 'run')
        self.assertEqual(X_cmd[1]['name'], 'echo hello world')
        self.assertEqual(X_cmd[2]['cwd'], '/')

        del result['A']
        del result['X']
        self.assertEqual(len(result), 2)
        # 2 rather than 1 because pydsl adds an extra no-op state
        # declaration.

        s_iter = result.itervalues()
        try:
            s = s_iter.next()['file']
        except KeyError:
            s = s_iter.next()['file']
        self.assertEqual(s[0], 'managed')
        self.assertEqual(s[1]['name'], 'myfile.txt')
        self.assertEqual(s[2]['source'], 'salt://path/to/file')

    def test_requisite_declarations(self):
        result = self.render_sls(textwrap.dedent('''
            state('X').cmd.run('echo hello')
            state('A').cmd.run('mkdir tmp', cwd='/var')
            state('B').cmd.run('ls -la', cwd='/var/tmp') \
                        .require(state('X').cmd) \
                        .require(cmd='A') \
                        .watch(service='G')
            state('G').service.running(name='collectd')
            state('G').service.watch_in(state('A').cmd)

            state('H').cmd.require_in(cmd='echo hello')
            state('H').cmd.run('echo world')
        '''))
        self.assertTrue(len(result), 6)
        self.assertTrue(set("X A B G H".split()).issubset(set(result.keys())))
        b = result['B']['cmd']
        self.assertEqual(b[0], 'run')
        self.assertEqual(b[1]['name'], 'ls -la')
        self.assertEqual(b[2]['cwd'], '/var/tmp')
        self.assertEqual(b[3]['require'][0]['cmd'], 'X')
        self.assertEqual(b[4]['require'][0]['cmd'], 'A')
        self.assertEqual(b[5]['watch'][0]['service'], 'G')
        self.assertEqual(result['G']['service'][2]['watch_in'][0]['cmd'], 'A')
        self.assertEqual(
            result['H']['cmd'][1]['require_in'][0]['cmd'], 'echo hello'
        )

    def test_include_extend(self):
        result = self.render_sls(textwrap.dedent('''
            include(
                'some.sls.file',
                'another.sls.file',
                'more.sls.file',
                delayed=True
            )
            A = state('A').cmd.run('echo hoho', cwd='/')
            state('B').cmd.run('echo hehe', cwd='/')
            extend(
                A,
                state('X').cmd.run(cwd='/a/b/c'),
                state('Y').file('managed', name='a_file.txt'),
                state('Z').service.watch(file='A')
            )
        '''))
        self.assertEqual(len(result), 4)
        self.assertEqual(
            result['include'],
            [{'base': sls} for sls in
             ('some.sls.file', 'another.sls.file', 'more.sls.file')]
        )
        extend = result['extend']
        self.assertEqual(extend['X']['cmd'][0], 'run')
        self.assertEqual(extend['X']['cmd'][1]['cwd'], '/a/b/c')
        self.assertEqual(extend['Y']['file'][0], 'managed')
        self.assertEqual(extend['Y']['file'][1]['name'], 'a_file.txt')
        self.assertEqual(len(extend['Z']['service']), 1)
        self.assertEqual(extend['Z']['service'][0]['watch'][0]['file'], 'A')

        self.assertEqual(result['B']['cmd'][0], 'run')
        self.assertTrue('A' not in result)
        self.assertEqual(extend['A']['cmd'][0], 'run')

    def test_cmd_call(self):
        result = self.HIGHSTATE.state.call_template_str(textwrap.dedent('''\
            #!pydsl
            state('A').cmd.run('echo this is state A', cwd='/')

            some_var = 12345
            def do_something(a, b, *args, **kws):
                return dict(result=True, changes={'a': a, 'b': b, 'args': args, 'kws': kws, 'some_var': some_var})

            state('C').cmd.call(do_something, 1, 2, 3, x=1, y=2) \
                          .require(state('A').cmd)

            state('G').cmd.wait('echo this is state G', cwd='/') \
                          .watch(state('C').cmd)
        '''))
        ret = (result[k] for k in result.keys() if 'do_something' in k).next()
        changes = ret['changes']
        self.assertEqual(
            changes,
            dict(a=1, b=2, args=(3,), kws=dict(x=1, y=2), some_var=12345)
        )

        ret = (result[k] for k in result.keys() if '-G_' in k).next()
        self.assertEqual(ret['changes']['stdout'], 'this is state G')

    def test_multiple_state_func_in_state_mod(self):
        with self.assertRaisesRegexp(PyDslError, 'Multiple state functions'):
            self.render_sls(textwrap.dedent('''
                state('A').cmd.run('echo hoho')
                state('A').cmd.wait('echo hehe')
            '''))

    def test_no_state_func_in_state_mod(self):
        with self.assertRaisesRegexp(PyDslError, 'No state function specified'):
            self.render_sls(textwrap.dedent('''
                state('B').cmd.require(cmd='hoho')
            '''))

    def test_load_highstate(self):
        result = self.render_sls(textwrap.dedent('''
            import yaml
            __pydsl__.load_highstate(yaml.load("""
            A:
              cmd.run:
                - name: echo hello
                - cwd: /
            B:
              pkg:
                - installed
              service:
                - running
                - require:
                  - pkg: B
                - watch:
                  - cmd: A
            """))

            state('A').cmd.run(name='echo hello world')
            '''))
        self.assertEqual(len(result), 3)
        self.assertEqual(result['A']['cmd'][0], 'run')
        self.assertEqual(result['A']['cmd'][1]['name'], 'echo hello')
        self.assertEqual(result['A']['cmd'][2]['cwd'], '/')
        self.assertEqual(result['A']['cmd'][3]['name'], 'echo hello world')

        self.assertEqual(len(result['B']['pkg']), 1)
        self.assertEqual(result['B']['pkg'][0], 'installed')

        self.assertEqual(result['B']['service'][0], 'running')
        self.assertEqual(result['B']['service'][1]['require'][0]['pkg'], 'B')
        self.assertEqual(result['B']['service'][2]['watch'][0]['cmd'], 'A')

    def test_ordered_states(self):
        result = self.render_sls(textwrap.dedent('''
            __pydsl__.set(ordered=True)
            A = state('A')
            state('B').cmd.run('echo bbbb')
            A.cmd.run('echo aaa')
            state('B').cmd.run(cwd='/')
            state('C').cmd.run('echo ccc')
            state('B').file.managed(source='/a/b/c')
            '''))
        self.assertEqual(len(result['B']['cmd']), 3)
        self.assertEqual(result['A']['cmd'][1]['require'][0]['cmd'], 'B')
        self.assertEqual(result['C']['cmd'][1]['require'][0]['cmd'], 'A')
        self.assertEqual(result['B']['file'][1]['require'][0]['cmd'], 'C')

    def test_pipe_through_stateconf(self):
        dirpath = tempfile.mkdtemp(dir=integration.SYS_TMP_DIR)
        if not os.path.isdir(dirpath):
            self.skipTest(
                'The temporary directory {0!r} was not created'.format(
                    dirpath
                )
            )
        output = os.path.join(dirpath, 'output')
        try:
            write_to(os.path.join(dirpath, 'xxx.sls'), textwrap.dedent(
                '''#!stateconf -os yaml . jinja
                .X:
                  cmd.run:
                    - name: echo X >> {0}
                    - cwd: /
                .Y:
                  cmd.run:
                    - name: echo Y >> {1}
                    - cwd: /
                .Z:
                  cmd.run:
                    - name: echo Z >> {2}
                    - cwd: /
                '''.format(output, output, output)))
            write_to(os.path.join(dirpath, 'yyy.sls'), textwrap.dedent('''\
                #!pydsl|stateconf -ps

                __pydsl__.set(ordered=True)
                state('.D').cmd.run('echo D >> {0}', cwd='/')
                state('.E').cmd.run('echo E >> {1}', cwd='/')
                state('.F').cmd.run('echo F >> {2}', cwd='/')
                '''.format(output, output, output)))

            write_to(os.path.join(dirpath, 'aaa.sls'), textwrap.dedent('''\
                #!pydsl|stateconf -ps

                include('xxx', 'yyy')

                # make all states in xxx run BEFORE states in this sls.
                extend(state('.start').stateconf.require(stateconf='xxx::goal'))

                # make all states in yyy run AFTER this sls.
                extend(state('.goal').stateconf.require_in(stateconf='yyy::start'))

                __pydsl__.set(ordered=True)

                state('.A').cmd.run('echo A >> {0}', cwd='/')
                state('.B').cmd.run('echo B >> {1}', cwd='/')
                state('.C').cmd.run('echo C >> {2}', cwd='/')
                '''.format(output, output, output)))

            state_highstate({'base': ['aaa']}, dirpath)
            with salt.utils.fopen(output, 'r') as f:
                self.assertEqual(''.join(f.read().split()), "XYZABCDEF")

        finally:
            shutil.rmtree(dirpath, ignore_errors=True)

    def test_rendering_includes(self):
        dirpath = tempfile.mkdtemp(dir=integration.SYS_TMP_DIR)
        if not os.path.isdir(dirpath):
            self.skipTest(
                'The temporary directory {0!r} was not created'.format(
                    dirpath
                )
            )
        output = os.path.join(dirpath, 'output')
        try:
            write_to(os.path.join(dirpath, 'aaa.sls'), textwrap.dedent('''\
                #!pydsl|stateconf -ps

                include('xxx')
                yyy = include('yyy')

                # ensure states in xxx are run first, then those in yyy and then those in aaa last.
                extend(state('yyy::start').stateconf.require(stateconf='xxx::goal'))
                extend(state('.start').stateconf.require(stateconf='yyy::goal'))

                extend(state('yyy::Y2').cmd.run('echo Y2 extended >> {0}'))

                __pydsl__.set(ordered=True)

                yyy.hello('red', 1)
                yyy.hello('green', 2)
                yyy.hello('blue', 3)
                '''.format(output)))

            write_to(os.path.join(dirpath, 'xxx.sls'), textwrap.dedent('''\
                #!stateconf -os yaml . jinja

                include:
                  - yyy

                extend:
                  yyy::start:
                    stateconf.set:
                      - require:
                        - stateconf: .goal

                  yyy::Y1:
                    cmd.run:
                      - name: 'echo Y1 extended >> {0}'

                .X1:
                  cmd.run:
                    - name: echo X1 >> {1}
                    - cwd: /
                .X2:
                  cmd.run:
                    - name: echo X2 >> {2}
                    - cwd: /
                .X3:
                  cmd.run:
                    - name: echo X3 >> {3}
                    - cwd: /

                '''.format(output, output, output, output)))

            write_to(os.path.join(dirpath, 'yyy.sls'), textwrap.dedent('''\
                #!pydsl|stateconf -ps

                include('xxx')
                __pydsl__.set(ordered=True)

                state('.Y1').cmd.run('echo Y1 >> {0}', cwd='/')
                state('.Y2').cmd.run('echo Y2 >> {1}', cwd='/')
                state('.Y3').cmd.run('echo Y3 >> {2}', cwd='/')

                def hello(color, number):
                    state(color).cmd.run('echo hello '+color+' '+str(number)+' >> {3}', cwd='/')
                '''.format(output, output, output, output)))

            state_highstate({'base': ['aaa']}, dirpath)
            expected = textwrap.dedent('''\
                X1
                X2
                X3
                Y1 extended
                Y2 extended
                Y3
                hello red 1
                hello green 2
                hello blue 3
                ''')

            with salt.utils.fopen(output, 'r') as f:
                self.assertEqual(sorted(f.read()), sorted(expected))

        finally:
            shutil.rmtree(dirpath, ignore_errors=True)

    def test_compile_time_state_execution(self):
        if not sys.stdin.isatty():
            self.skipTest('Not attached to a TTY')
        dirpath = tempfile.mkdtemp(dir=integration.SYS_TMP_DIR)
        if not os.path.isdir(dirpath):
            self.skipTest(
                'The temporary directory {0!r} was not created'.format(
                    dirpath
                )
            )
        try:
            write_to(os.path.join(dirpath, 'aaa.sls'), textwrap.dedent('''\
                #!pydsl

                __pydsl__.set(ordered=True)
                A = state('A')
                A.cmd.run('echo hehe > {0}/zzz.txt', cwd='/')
                A.file.managed('{1}/yyy.txt', source='salt://zzz.txt')
                A()
                A()

                state().cmd.run('echo hoho >> {2}/yyy.txt', cwd='/')

                A.file.managed('{3}/xxx.txt', source='salt://zzz.txt')
                A()
                '''.format(dirpath, dirpath, dirpath, dirpath)))
            state_highstate({'base': ['aaa']}, dirpath)
            with salt.utils.fopen(os.path.join(dirpath, 'yyy.txt'), 'r') as f:

                self.assertEqual(f.read(), 'hehe\nhoho\n')
            with salt.utils.fopen(os.path.join(dirpath, 'xxx.txt'), 'r') as f:
                self.assertEqual(f.read(), 'hehe\n')
        finally:
            shutil.rmtree(dirpath, ignore_errors=True)

    def test_nested_high_state_execution(self):
        dirpath = tempfile.mkdtemp(dir=integration.SYS_TMP_DIR)
        if not os.path.isdir(dirpath):
            self.skipTest(
                'The temporary directory {0!r} was not created'.format(
                    dirpath
                )
            )
        output = os.path.join(dirpath, 'output')
        try:
            write_to(os.path.join(dirpath, 'aaa.sls'), textwrap.dedent('''\
                #!pydsl
                __salt__['state.sls']('bbb')
                state().cmd.run('echo bbbbbb', cwd='/')
                '''))
            write_to(os.path.join(dirpath, 'bbb.sls'), textwrap.dedent(
                '''
                # {{ salt['state.sls']('ccc')
                test:
                  cmd.run:
                    - name: echo bbbbbbb
                    - cwd: /
                '''))
            write_to(os.path.join(dirpath, 'ccc.sls'), textwrap.dedent(
                '''
                #!pydsl
                state().cmd.run('echo ccccc', cwd='/')
                '''))
            state_highstate({'base': ['aaa']}, dirpath)
        finally:
            shutil.rmtree(dirpath, ignore_errors=True)

    def test_repeat_includes(self):
        dirpath = tempfile.mkdtemp(dir=integration.SYS_TMP_DIR)
        if not os.path.isdir(dirpath):
            self.skipTest(
                'The temporary directory {0!r} was not created'.format(
                    dirpath
                )
            )
        output = os.path.join(dirpath, 'output')
        try:
            write_to(os.path.join(dirpath, 'b.sls'), textwrap.dedent('''\
                #!pydsl
                include('c')
                include('d')
                '''))
            write_to(os.path.join(dirpath, 'c.sls'), textwrap.dedent('''\
                #!pydsl
                modtest = include('e')
                modtest.success
                '''))
            write_to(os.path.join(dirpath, 'd.sls'), textwrap.dedent('''\
                #!pydsl
                modtest = include('e')
                modtest.success
                '''))
            write_to(os.path.join(dirpath, 'e.sls'), textwrap.dedent('''\
                #!pydsl
                success = True
                '''))
            state_highstate({'base': ['b']}, dirpath)
            state_highstate({'base': ['c', 'd']}, dirpath)
        finally:
            shutil.rmtree(dirpath, ignore_errors=True)

def write_to(fpath, content):
    with salt.utils.fopen(fpath, 'w') as f:
        f.write(content)

def state_highstate(state, dirpath):
    opts = copy.copy(OPTS)
    opts['file_roots'] = dict(base=[dirpath])
    HIGHSTATE = HighState(opts)
    HIGHSTATE.push_active()
    try:
        high, errors = HIGHSTATE.render_highstate(state)
        if errors:
            import pprint
            pprint.pprint('\n'.join(errors))
            pprint.pprint(high)

        out = HIGHSTATE.state.call_high(high)
        # pprint.pprint(out)
    finally:
        HIGHSTATE.pop_active()

if __name__ == '__main__':
    from integration import run_tests
    run_tests(PyDSLRendererTestCase, needs_daemon=False)