# -*- coding: utf-8 -*-

from selenium import webdriver
import requests
import re

from settings import CHROMEDRIVER_EXECUTABLE_PATH, SELENIUM_IMPLICITLY_WAIT
from settings import REQUESTS_TIMEOUT

# # selenium
# CHROMEDRIVER_EXECUTABLE_PATH = 'F:/chromedriver_win32/chromedriver.exe'
# SELENIUM_IMPLICITLY_WAIT = 10
#
# # requests
# REQUESTS_TIMEOUT = 10

def requests_init():
    params = {}

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
        # 'User-Agent': get_ua()  # 获取随机 UA头
    }

    # ip, port = get_proxy_ip()  # 获取随机 代理IP
    # ip, port = ('116.196.81.58', '3128')
    # proxies = {
    #     'http': 'http://{}:{}'.format(ip, port),
    #     'https': 'http://{}:{}'.format(ip, port),
    # }
    proxies = {}  # 不使用代理

    timeout = REQUESTS_TIMEOUT

    return params, headers, proxies, timeout

def requests_get_url(url):
    # ================================ 生成 requests 的 get 方法的参数 ================================
    params, headers, proxies, timeout = requests_init()  # 可在 requests_init 方法中写 获取随机UA头，代理IP
    # =================================================================================================

    # ================================ 请求网页 ================================
    response = requests.get(url=url, params=params, headers=headers, proxies=proxies, timeout=timeout)
    html_str = response.text
    # ==========================================================================

    # ================================ 设置网页编码 ================================
    charset = requests.utils.get_encodings_from_content(html_str)  # 从html的meta中抽取
    '''
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
    '''
    if not charset:
        charset = [response.apparent_encoding]  # apparent_encoding 返回真实编码  # 由程序分析出编码
    response.encoding = charset[0]
    html_str = response.text
    # ==============================================================================

    return html_str

def selenium_init():
    # 获取随机 UA头
    user_agent = None

    # 获取随机 代理IP
    # ip, port = ('114.55.236.62', '3128')
    ip, port = (None, None)

    # ================================ 浏览器 ================================
    # 设置浏览器
    chrome_options = webdriver.ChromeOptions()

    # 设置无界面浏览
    # chrome_options.set_headless(True)
    chrome_options.add_argument('--headless')

    # 设置 user_agent
    if user_agent:
        chrome_options.add_argument('user-agent={}'.format(user_agent))

    # 设置代理 IP
    if ip and port:
        chrome_options.add_argument("--proxy-server=http://{0}:{1}".format(ip, port))
        # 一定要注意，=两边不能有空格，不能是这样--proxy-server = http://202.20.16.82:10152

    # 浏览器
    browser = webdriver.Chrome(executable_path=CHROMEDRIVER_EXECUTABLE_PATH,
                               chrome_options=chrome_options)  # chromedriver.exe 的路径
    browser.maximize_window()  # 窗口最大化
    browser.implicitly_wait(SELENIUM_IMPLICITLY_WAIT)  # 隐式等待。如果找到，就继续执行，如果没找到就等待10s
    # ========================================================================

    return browser

def selenium_get_url(url, browser=None):
    use_def_browser = False  # 是否使用方法内创建的浏览器
    # 传参时，没有 browser 的话，将使用该方法内部创建的浏览器，将在该方法最后关闭浏览器，不返回 browser
    # 传参时，传入 browser 的话，将使用外部传入的浏览器，该方法将不会关闭传入的浏览器，返回 browser
    # 若使用外部浏览器  # 注意在外部关闭浏览器
    # 若要使用不同随机UA头，随机IP 请求网页的话建议使用方法的内部浏览器（只传入 url）
    # 若要使用同一个UA头，同一个IP 请求网页的话建议使用外部浏览器（传入 url 和 browser）
    # 在外部创建浏览器时，可以使用 selenium_init 方法
    try:
        if not browser:  # 创建浏览器
            use_def_browser = True  # 更改旗标
            browser = selenium_init()  # 可在 selenium_init 方法中写 获取随机UA头，代理IP

        # 请求网页
        browser.get(url)

        # # 切换到新打开的页面
        # current_window = browser.current_window_handle
        # all_handles = browser.window_handles
        # for handle in all_handles:
        #     if handle != current_window:
        #         browser.switch_to.window(handle)

        html_str = browser.page_source

        if not use_def_browser:
            return html_str, browser
        else:
            return html_str
    except Exception as e:
        print(str(e))
    finally:
        if browser and use_def_browser:
            browser.quit()

def html_str_noise_reduce(html_str):
    re_comment = re.compile('<!--[^>]*-->')  # HTML注释
    html_str = re_comment.sub('', html_str)  # 去掉HTML注释  # 去除单行的注释，且注释中的内容不含>

    # re_script = re.compile('<s*script[^>]*>[^<]*<s*/s*scripts*>', re.I)  # 去除 Script
    # html_str = re_script.sub('', html_str)

    re_style = re.compile('<s*style[^>]*>[^<]*<s*/s*styles*>', re.I)  # style
    html_str = re_style.sub('', html_str)  # 去掉 style

    return html_str

if __name__ == '__main__':
    # browser = selenium_init()

    url = 'https://news.sina.com.cn/roll/'
    # html_str = requests_get_url(url)  # requests
    html_str = selenium_get_url(url)  # selenium 使用内部浏览器
    # html_str = selenium_get_url(url, browser)  # selenium 使用外部浏览器
    # browser.quit()
    print(html_str)


from django.shortcuts import render, get_object_or_404, redirect
from .models import Writing
from django.core.paginator import Paginator
from django.utils import timezone
from .forms import SongLyrics

def new(request):
    return render(request, 'new.html')

def main(request):
    songs = Writing.objects
    songs_list = Writing.objects.all()
    paginator = Paginator(songs_list, 3)
    page = request.GET.get('page')
    posts = paginator.get_page(page)
    return render(request,'main.html',{'writings': songs, 'posts':posts, } )
# Create your views here.

def more(request,song_id):
    song_lyrics = get_object_or_404(Writing,pk = song_id)

    return render(request, 'lyrics.html',{'song':song_lyrics})

def create(request):
    song = Writing()
    song.title = request.GET['title']
    song.body = request.GET['lyrics']
    song.pub_date = timezone.datetime.now()
    song.save()
    return redirect('/lyrics/'+str(song.id))

def portfolio(request):
    return render(request, 'portfolio.html')

def songpost(request):
    if request.method == 'POST':
        form = SongLyrics(request.POST)
        if form.is_valid():
            post = form.save(commit=False)
            post.pub_date = timezone.now()
            post.save()
            return redirect('home')
    else:
        form = SongLyrics()
        return render(request, 'new.html', {'form':form})


import logging
import torch
import torch.distributed as dist
import torch.multiprocessing as mp

from utils import comm

__all__ = ['launch']

def _find_free_port():
    import socket
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.bind(("", 0))
    port = sock.getsockname()[1]
    sock.close()
    return port

def launch(main_func, num_gpus_per_machine, num_machines=-1, machine_rank=0, dist_url=None, args=()):
    """
    :param main_func: a funtion that will be called bt 'main_func(*args)'
    :param num_machines:  the total number of machines
    :param machine_rank:  the rank of this machine
    :param dist_url: url to connect to for distributed training, including protocol
                            e.g. "tcp://127.0.0.1:8686"
    :param args:  arguments passed to main_func
    :return:
    """
    world_size = num_machines * num_gpus_per_machine
    if world_size > 1:
        if dist_url == "auto":
            assert num_machines == 1, "dist_url=auto cannot work with distributed training."
            port = _find_free_port()
            dist_url = f"tcp://127.0.0.1:{port}"
        mp.spawn(
            _distributed_worker,
            nprocs=num_gpus_per_machine,
            args=(main_func, world_size, num_gpus_per_machine, machine_rank, dist_url, args),
            daemon=False
        )
    else:
        main_func(*args)

def _distributed_worker(local_rank, main_func, world_size, num_gpus_per_machine, machine_rank, dist_url, args):
    assert torch.cuda.is_available(), "CUDA is not available. Please check your installation."
    global_rank = machine_rank * num_gpus_per_machine + local_rank
    try:
        dist.init_process_group(
            backend="NCCL", init_method=dist_url, world_size=world_size, rank=global_rank
        )
    except Exception as e:
        logger = logging.getLogger(__name__)
        logger.error("Process group URL: {}".format(dist_url))
        raise e
    # synchronize is needed here to prevent a possible timeout after calling init_process_group
    comm.synchronize()

    assert num_gpus_per_machine <= torch.cuda.device_count()
    torch.cuda.set_device(local_rank)

    assert comm._LOCAL_PROCESS_GROUP is None
    num_machines = world_size // num_gpus_per_machine
    for i in range(num_machines):
        ranks_on_i = list(range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine))
        pg = dist.new_group(ranks_on_i)
        if i == machine_rank:
            comm._LOCAL_PROCESS_GROUP = pg

    main_func(*args)


from collections import deque

visited = []

class Avnode:
    def __init__(self, data=0):
        self.data = data
        self.next = None

class Graph(object):
    """docstring for Graph"""

    def __init__(self, nv, ne=0):
        super(Graph, self).__init__()
        self.nv = nv
        self.ne = ne
        self.ah = []
        for i in range(self.nv + 1):
            tmp = Avnode(i)
            self.ah.append(tmp)

def main():
    firstline = input()
    v, e = [int(i) for i in firstline.split(' ')]
    G = BuildG(v, e)
    OutputE(G)

def Init_Graph(v):
    for i in range(v + 1):
        visited.append(0)
    G = Graph(v)

    return G

def OutputE(G):
    for i in range(1, G.nv + 1):
        cnt = BFS(G, i)

        for j in range(1, G.nv + 1):
            visited[j] = 0
        print('{0}: {1:.2f}%'.format(i, cnt * 100 / G.nv))

def BuildG(v, e):
    G = Init_Graph(v)
    for i in range(e):
        edges = input()
        v1, v2 = [int(k) for k in edges.split(' ')]
        Insert_E(G, v1, v2)
    return G

def Insert_E(G, v1, v2):

    av = Avnode(v2)
    av.next = G.ah[v1].next  # 将点插入到头部
    G.ah[v1].next = av

    av2 = Avnode(v1)
    av2.next = G.ah[v2].next  # 无向图两边都要插入；
    G.ah[v2].next = av2

def BFS(G, v):
    visited[v] = 1
    cnt = 1
    queue = deque()
    last = v
    queue.append(v)

    tail = 0
    level = 0
    while queue != deque([]):
        v = queue.popleft()
        w = G.ah[v].next
        while w:
            if not visited[w.data]:
                visited[w.data] = 1
                cnt += 1
                tail = w.data
                queue.append(w.data)
            w = w.next

        if v == last:
            level += 1
            last = tail
        if level == 6:
            break
    return cnt

if __name__ == '__main__':
    main()


import glob, os
import time
start_time = time.time()
from pathlib import Path

#import tensorflow as tf
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
print(tf.__version__)
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.applications.imagenet_utils import decode_predictions

physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "Not enough GPU hardware devices available"
config = tf.config.experimental.set_memory_growth(physical_devices[0], True)

import numpy as np
# import the models for further classification experiments
from tensorflow.keras.applications import (
        vgg16,
        resnet50,
        mobilenet,
        inception_v3
    )

# init the models
vgg_model = vgg16.VGG16(weights='imagenet')
inception_model = inception_v3.InceptionV3(weights='imagenet')
resnet_model = resnet50.ResNet50(weights='imagenet')
mobilenet_model = mobilenet.MobileNet(weights='imagenet')

def run_load_prediction(filename):
    # assign the image path for the classification experiments
    # load an image in PIL format
    original = load_img(filename, target_size=(224, 224))
    print('PIL image size',original.size)
    plt.imshow(original)
    plt.show()

    # convert the PIL image to a numpy array
    # IN PIL - image is in (width, height, channel)
    # In Numpy - image is in (height, width, channel)
    numpy_image = img_to_array(original)
    plt.imshow(np.uint8(numpy_image))
    plt.show()
    print('numpy array size',numpy_image.shape)

    # Convert the image / images into batch format
    # expand_dims will add an extra dimension to the data at a particular axis
    # We want the input matrix to the network to be of the form (batchsize, height, width, channels)
    # Thus we add the extra dimension to the axis 0.
    image_batch = np.expand_dims(numpy_image, axis=0)
    print('image batch size', image_batch.shape)
    plt.imshow(np.uint8(image_batch[0]))

    # prepare the image for the VGG model
    processed_image = resnet50.preprocess_input(image_batch.copy())

    # get the predicted probabilities for each class
    predictions = resnet_model.predict(processed_image)
    # print predictions
    # convert the probabilities to class labels
    # we will get top 5 predictions which is the default
    label = decode_predictions(predictions)
    # print VGG16 predictions
    for prediction_id in range(len(label[0])):
        print(label[0][prediction_id])

data_dir = Path('dsc650/assignments/assignment06/').joinpath('Data')
os.chdir(data_dir)
for filename in glob.glob("*.png"):
    print(filename)
    run_load_prediction(filename)


#!/usr/bin/python
# -*- coding:utf-8 -*-

"""
===============================================================================
author: 赵明星
desc:   第三个模型，使用GBDT。
===============================================================================
"""

from sklearn import preprocessing
from sklearn.ensemble import GradientBoostingClassifier
import dataset.generate_train_and_test_set_for_day_x as FEATURE

import sys
reload(sys)
sys.setdefaultencoding('utf-8')

def train_and_predict(train_feature, train_label, test_feature, day_x, mode):
    scale_train_feature = preprocessing.scale(train_feature)
    clf = GradientBoostingClassifier(n_estimators=200)
    clf.fit(scale_train_feature, train_label)
    scale_test_feature = preprocessing.scale(test_feature)
    predict_labels = clf.predict(scale_test_feature)
    if mode == "dev":
        dev_feature, dev_label = FEATURE.get_day_x_week2_dev_set(
                day_x,
                predict_labels)
        scale_week2_dev = preprocessing.scale(dev_feature)
        week_2_labels = clf.predict(scale_week2_dev)
        return predict_labels, week_2_labels, dev_label
    else:
        week_2_test = FEATURE.get_day_x_week2_test_set(day_x, predict_labels)
        scale_week2_test = preprocessing.scale(week_2_test)
        week_2_labels = clf.predict(scale_week2_test)
        return predict_labels, week_2_labels

if __name__ == "__main__":
    pass


# -*- coding: utf-8 -*-

from flask import g
from flask import abort
from flask import jsonify
from flask import request
from flask import redirect
from flask import Blueprint
from flask import make_response
from flask import render_template

from .shop import Shop

s = Shop()
bp = Blueprint('st16', __name__, static_folder='static')

@bp.before_request
def before_request():
    g._selfurl = f"/{request.url_rule.rule.split('/')[1]}"
    _connect = getattr(g, '_database', None)
    if _connect is None:
        s.get_connect()
        g._database = s.connect

@bp.route("/delete", methods=['GET'])
def delete():
    s.remove_from_list(int(request.args.get("id", "")))
    return redirect(f"{g._selfurl}/")

@bp.route("/clear")
def clear_list():
    s.clear_current_list()
    return redirect(f"{g._selfurl}/")

@bp.route("/edit", methods=['GET', 'POST'])
def edit():
    if request.method == "GET":
        id = int(request.args.get("id", ""))
        t = request.args.get("type", "")
        return render_template(
            "asm1904/st16/form.html",
            type=t, data=s.get_phone(id),
            id=id, selfurl=g._selfurl)
    elif request.method == "POST":
        s.edit_list_entry(request.form)
        return redirect(f"{g._selfurl}/")

@bp.route("/add", methods=['GET', 'POST'])
def add():
    if request.method == "GET":
        t = request.args.get("type", "")
        return render_template(
            "asm1904/st16/form.html",
            type=t, selfurl=g._selfurl)
    elif request.method == "POST":
        s.add_list_entry(request.form)
        return redirect(f"{g._selfurl}/")

@bp.route("/import")
def import_list():
    s.import_list()
    return redirect(f"{g._selfurl}/")

@bp.route("/")
def show_list():
    return render_template(
        "asm1904/st16/list.html",
        mobile_list=s.get_list(),
        selfurl=g._selfurl)

@bp.errorhandler(403)
def api_ad(error):
    return make_response(jsonify({'status': 'Invalid method'}), 403)

@bp.errorhandler(400)
def api_br(error):
    return make_response(jsonify({'status': 'Invalid request'}), 400)

@bp.route("/api/entry/delete/<int:id>", methods=['DELETE'])
def api_delete(id):
    if request.method != 'DELETE':
        abort(403)
    s.remove_from_list(id)
    return jsonify({'status': 'Ok'})

@bp.route("/api/clear", methods=['DELETE'])
def api_clear_list():
    if request.method != 'DELETE':
        abort(403)
    s.clear_current_list()
    return jsonify({'status': 'Ok'})

@bp.route("/api/entry/edit/<int:id>", methods=['PUT'])
def api_edit(id):
    if request.method != 'PUT':
        abort(403)
    s.edit_list_entry(request.json)
    return jsonify({'status': 'Ok'})

@bp.route("/api/entry/add", methods=['POST'])
def api_add():
    if not request.json:
        abort(400)
    s.add_list_entry(request.json)
    return jsonify({'status': 'Ok'})

@bp.route("/api/get-list", methods=['GET'])
def api_get_list():
    mobile_shop = s.get_list()
    return jsonify({'list': mobile_shop})

@bp.route("/api/entry/get-type/<int:phone_id>", methods=['GET'])
def api_get_phone_type(phone_id):
    phone_type = s.get_phone_type(phone_id)
    return jsonify({'type': phone_type})

if __name__ == '__main__':
    pass


import json
import logging
import math
import os
import time
from concurrent.futures import ThreadPoolExecutor

import requests
import urllib3

from modules.proxyprovider import SequenceProxyProvider

# 屏蔽requests的SSL告警
urllib3.disable_warnings()
logging.getLogger("requests").setLevel(logging.WARNING)

class ProjectCrawler:
    def __init__(self):
        self.proxy_provider = SequenceProxyProvider()
        self.stats = {"total": 0, "not_found": 0}  # 统计信息
        self.total = 14209647  # 最大的项目ID编号
        self.NAME = "Project"
        self.MAX_WORKERS = 200  # 根据实际情况进行调整

    def get_url(self, id):
        return 'https://www.freelancer.com/api/projects/0.1/projects/%s/?compact&full_description=true&upgrade_details=true&job_details=true&attachment_details=true&file_details=true&selected_bids=true&qualification_details=true&user_details=true&hireme_details=true&invited_freelancer_details=true&recommended_freelancer_details=true' % id

    def parse_result(self, id, result, stats):
        try:
            j = json.loads(result)
            if j["status"] == "success":
                # 成功后存储返回值及计数
                is_valid = True
                self.save(id, result)
                stats["total"] += 1
            elif j['error_code'] == "RestExceptionCodes.NOT_FOUND":
                # 没有找到
                is_valid = True
                stats['not_found'] += 1
            else:
                # 出错打印result的值以便进一步分析问题所在
                print(result)
                is_valid = False
        except:
            # 出错打印result的值以便进一步分析问题所在
            print(result)
            is_valid = False

        return is_valid

    def get(self, id, stats):
        retry_count = 0
        while retry_count < 10:
            proxy = self.proxy_provider.pick()

            try:
                url = self.get_url(id)

                headers = {
                    'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36",
                    'accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
                    'accept-encoding': "gzip, deflate",
                    'accept-language': "en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,ja;q=0.2",
                    'cache-control': "no-cache",
                }

                print("Crawling %s" % (id))

                response = requests.request("GET", url, headers=headers,
                                            proxies={"https": proxy.url},
                                            timeout=5, verify=False)

                is_valid = self.parse_result(id, response.text, stats)

                # 计算速度
                elapsed = time.time() - self.start
                print("Total got %s, not found: %s, elasped: %s, speed: %s"
                      % (stats["total"], stats["not_found"], elapsed, stats["total"] / elapsed * 60))

                if not is_valid:
                    # 如果出错则认为代理有问题，减分
                    proxy.error()
                else:
                    proxy.success()
                    return
            except:
                print("Retrying", id)
                proxy.error()

            retry_count += 1

    def start(self):
        self.start = time.time()

        # 由于需要爬取的数量太多，如果所有的项目都submit到ThreadPoolExecutor中，会造成内部队列过大而导致性能问题
        # 这里将大队列分为多个片段依次进行抓取
        segment_length = 100000

        crawled = self.redis.smembers(self.NAME)

        executor = ThreadPoolExecutor(max_workers=self.MAX_WORKERS)
        added_jobs = 0
        for id in range(0, self.total):
            if str(id) in crawled:
                continue

            executor.submit(self.get, id, self.stats)

            added_jobs += 1
            if added_jobs > segment_length:
                added_jobs = 0
                executor.shutdown()
                executor = ThreadPoolExecutor(max_workers=self.MAX_WORKERS)

        print("Done")

    def save(self, id, result):
        # 一个文件夹中存储一万个
        subdir = int(id / 10000)
        dirname = "./out/%s/%s" % (self.NAME, subdir)
        os.makedirs(dirname, exist_ok=True)

        filename = "%s/%s-%s.json" % (dirname, self.NAME, id)

        with open(filename, "w", encoding="utf-8") as data:
            data.write(result)

ProjectCrawler().start()


import clr
clr.AddReference('RevitAPI')
from Autodesk.Revit.DB import *

def HasVariableLayer(item):
	if hasattr(item, "GetCompoundStructure"):
		compstruc = item.GetCompoundStructure()
		if hasattr(compstruc, "VariableLayerIndex"):
			if compstruc.VariableLayerIndex == -1: return False
			else: return True
		else: return False
	else: return False

items = UnwrapElement(IN[0])

if isinstance(IN[0], list): OUT = [HasVariableLayer(x) for x in items]
else: OUT = HasVariableLayer(items)


from tkinter import *
from tkinter import scrolledtext
import logging

import options.sequential, options.nonsequential
from options.utils import TextHandler
import profile_manager as profileManager

class GUI:
    def __init__(self, master):
        self.master = master
        master.title('Simple Clicker')
        self.setWindowIcon(master)

        self.status = Label(master, text='Inactive', fg='#ff0000')
        self.status.grid(row=0, column=0)

        self.uptime = Label(master, text='0', fg='#ff0000', width=20)
        self.uptime.grid(row=0, column=1)

        self.config = LabelFrame(master, text='Configuration')
        self.config.grid(row=1, column=0, columnspan=2, sticky=W, padx=5, pady=5)

        self.hotkeyFrame = Frame(self.config)
        self.hotkeyFrame.grid(row=0, column=0, sticky=W)

        self.hotkeyLabel = Label(self.hotkeyFrame, text='Hotkey')
        vcmd = (master.register(self.limitChar), '%i')
        self.hotkey = Entry(self.hotkeyFrame, validate='key', validatecommand=vcmd, width=2)
        self.hotkeyLabel.grid(row=0, column=0, sticky=E)
        self.hotkey.grid(row=0, column=1, sticky=W)

        self.profiles = LabelFrame(self.config, text='Profile')
        self.profiles.grid(row=2, column=0, columnspan=2, sticky=W, padx=5, pady=5)

        self.profileLabel = Label(self.profiles, text='Set Profile')
        self.profileLabel.grid(row=1, column=0)

        self.profile = StringVar(master)
        self.setProfile = OptionMenu(self.profiles, self.profile, *self.profileList(), command=self.handleSetProfile)
        self.setProfile.grid(row=1, column=1)

        self.saveProfile = Button(self.profiles, text='Save', command=self.handleSaveProfile)
        self.saveProfile.grid(row=1, column=2)

        self.addProfile = Button(self.profiles, text='➕', command=self.handleAddProfile)
        self.addProfile.grid(row=1, column=3)

        self.delProfile = Button(self.profiles, text='❌', command=self.handleConfirmDelProfile)
        self.delProfile.grid(row=1, column=4)

        self.refreshButton = Button(self.config, text='Refresh Configuration', command=self.readSetting)
        self.refreshButton.grid(row=3, column=0)

        self.saveButton = Button(self.config, text='Save Configuration', command=self.writeSetting)
        self.saveButton.grid(row=3, column=1)

        self.addOptionFrame = Frame(master)
        self.addOptionFrame.grid(row=2, column=0, sticky=W)

        self.scrollFrame = ScrollFrame(master)
        self.scrollFrame.grid(row=3, column=0, columnspan=2, sticky=W)

        self.options = LabelFrame(self.scrollFrame.viewPort, text='Options')
        self.options.grid(row=0, column=0)

        self.logFrame = LabelFrame(master, text='Log')
        self.logFrame.grid(row=4, column=0, columnspan=2, sticky=W, padx=5, pady=5)

        self.level = StringVar(master)
        self.setLevel = OptionMenu(self.logFrame, self.level, *list(logging._levelToName.values()), command=self.changeLevel)
        self.setLevel.grid(row=0, column=0)

        self.clearLogButton = Button(self.logFrame, text='Clear Log', command=self.clearLog)
        self.clearLogButton.grid(row=0, column=1)

        self.log = scrolledtext.ScrolledText(self.logFrame, width=50, height=10, state='disabled')
        self.log.grid(row=1, column=0, columnspan=2)

        self.textHandler = TextHandler(self.log)
        self.textHandler.setLevel(logging.DEBUG)
        self.fileHandler = logging.FileHandler('log.txt')
        self.fileHandler.setLevel(logging.DEBUG)
        self.consoleHandler = logging.StreamHandler()
        self.consoleHandler.setLevel(logging.ERROR)
        self.formatter = logging.Formatter('[%(asctime)s][%(levelname)s] %(message)s')
        self.textHandler.setFormatter(self.formatter)
        self.fileHandler.setFormatter(self.formatter)
        self.consoleHandler.setFormatter(self.formatter)
        self.logger = logging.getLogger()
        self.logger.setLevel(logging.DEBUG)
        self.logger.addHandler(self.textHandler)
        self.logger.addHandler(self.fileHandler)
        self.logger.addHandler(self.consoleHandler)

        self.optionManager = OptionManager(self.options, options.nonsequential.optList, self.logger)

        self.readSetting()
        self.changeLevel(self.level.get())

    def setWindowIcon(self, window):
        try:
            window.iconbitmap('icon.ico')
        except TclError:
            # Linux compatibility
            window.iconphoto(False, PhotoImage(file='icon.png'))

    def limitChar(self, i):
        if i == '1': #if the index is 1 it means the string will be 2 characters long
            return False
        else:
            return True

    def profileList(self):
        profiles = list(profileManager.read().keys())
        if profiles == []:
            profiles = [None]
        return profiles

    def handleSaveProfile(self):
        self.newProfileName = Entry(self.master)
        self.newProfileName.delete(0, END)
        self.newProfileName.insert(0, self.profile.get())
        self.handleCreateProfile()

    def handleAddProfile(self):
        self.childWindow = Toplevel(self.master)
        self.childWindow.title('Name Profile')
        self.setWindowIcon(self.childWindow)
        self.childWindow.geometry('250x50')
        self.newProfileName = Entry(self.childWindow)
        self.newProfileName.pack(fill=X, expand=YES)
        createButton = Button(self.childWindow, text="Create", command=self.handleCreateProfile)
        createButton.pack(fill=X, expand=YES)

    def handleCreateProfile(self):
        profile = self.optionManager.getProfile()
        profileManager.write(self.newProfileName.get(), profile)
        self.refreshProfiles()
        self.profile.set(self.newProfileName.get())
        try:
            self.childWindow.destroy()
        except AttributeError:
            pass
        self.handleSetProfile()

    def handleSetProfile(self, *args):
        self.optionManager.destroyOptions()
        self.optionManager.setProfile(profileManager.read()[self.profile.get()])

    def menuCommand(self, value):
        self.profile.set(value)
        self.handleSetProfile()

    def refreshProfiles(self):
        profiles = self.profileList()

        menu = self.setProfile['menu']
        menu.delete(0, END)
        for string in profiles:
            menu.add_command(label=string,
                             command=lambda value=string: self.menuCommand(value))

    def handleConfirmDelProfile(self):
        self.childWindow = Toplevel(self.master)
        self.childWindow.title('Confirm Delete Profile')
        self.setWindowIcon(self.childWindow)
        self.childWindow.geometry('300x50')
        label = Label(self.childWindow, text='Delete Profile "{}"?'.format(self.profile.get()))
        label.pack(fill=X, expand=YES)
        createButton = Button(self.childWindow, text="Delete", command=self.handleDelProfile)
        createButton.pack(fill=X, expand=YES)

    def handleDelProfile(self):
        self.childWindow.destroy()
        profileManager.remove(self.profile.get())
        self.refreshProfiles()
        self.profile.set('')

    def readSetting(self):
        f = open('config.ini', 'r')
        self.hotkey.delete(0, END)
        self.hotkey.insert(0, f.readline().strip())
        self.profile.set(f.readline().strip())
        if self.profile.get() in self.profileList():
            self.handleSetProfile()
        self.level.set(f.readline().strip())
        f.close
        self.refreshProfiles()

    def writeSetting(self):
        f = open('config.ini', 'w')
        f.write(self.hotkey.get()+'\n')
        f.write(self.profile.get()+'\n')
        f.write(self.level.get())
        f.close()

    def changeLevel(self, level):
        self.logger.setLevel({v: k for k, v in logging._levelToName.items()}[level])

    def clearLog(self):
        f = open('log.txt', 'w').close()
        self.log.configure(state='normal')
        self.log.delete('1.0', END)
        self.log.configure(state='disabled')

class ScrollFrame(Frame):
    # from https://gist.github.com/mp035/9f2027c3ef9172264532fcd6262f3b01
    def __init__(self, parent):
        super().__init__(parent)

        self.canvas = Canvas(self, borderwidth=0, background='#F0F0F0')
        self.viewPort = Frame(self.canvas, background='#F0F0F0')
        self.vsb = Scrollbar(self, orient='vertical', command=self.canvas.yview)
        self.canvas.configure(yscrollcommand=self.vsb.set)

        self.vsb.pack(side='right', fill='y')
        self.canvas.pack(side='left', fill='both', expand=True)
        self.canvas.create_window((4,4), window=self.viewPort, anchor='nw',
                                  tags='self.viewPort')

        self.viewPort.bind('<Configure>', self.onFrameConfigure)

    def onFrameConfigure(self, event):
        '''Reset the scroll region to encompass the inner frame'''
        self.canvas.configure(scrollregion=self.canvas.bbox('all'))

class OptionWrapper:
    def __init__(self, master, sequential, option, widgets, logger):
        self.widgets = widgets

        self.frame = LabelFrame(master, text=option)

        self.deleteButton = Button(self.frame, text='❌', command=self.findIdAndDestroy)
        self.deleteButton.grid(row=0, column=0)

        self.name = option

        if sequential:
            optionObject = options.sequential.optDict.get(option)
        else:
            optionObject = options.nonsequential.optDict.get(option)
        self.widget = optionObject.Widget(self.frame, 1, logger)

        self.frame.pack(anchor=W, padx=5, pady=0)

    def findIdAndDestroy(self):
        for i in self.widgets:
            if id(i) == id(self):
                self.widgets.remove(i)
        self.frame.destroy()

class OptionManager:
    def __init__(self, parent, availableOptions, logger, sequential=False,  maxOptions=None):
        self.parent = parent
        self.max = maxOptions
        self.sequential = sequential

        self.addOptionFrame = Frame(parent)
        self.addOptionFrame.pack()

        self.addOptionLabel = Label(self.addOptionFrame, text='Add Option')
        self.addOptionLabel.grid(row=0, column=0, sticky=E)

        self.selectedOption = StringVar(parent)
        self.selectedOption.set('➕')
        self.addOptions = OptionMenu(self.addOptionFrame, self.selectedOption, *availableOptions, command=self.handleAddOption)
        self.addOptions.grid(row=0, column=1, sticky=W)

        self.wrappers = []

        self.logger = logger

    def handleAddOption(self, *args):
        if self.max == None:
            self.addOption(self.selectedOption.get())
        elif len(self.wrappers) < self.max:
            self.addOption(self.selectedOption.get())
        self.selectedOption.set('➕')

    def addOption(self, option):
        self.wrappers.append(OptionWrapper(self.parent, self.sequential, option, self.wrappers, self.logger))

    def startOptions(self):
        for o in self.wrappers:
            o.widget.start()

    def stopOptions(self):
        for o in self.wrappers:
            o.widget.stop()

    def updateOptions(self):
        for o in self.wrappers:
            o.widget.update()

    def runOptions(self):
        for o in self.wrappers:
            o.widget.run()

    def destroyOptions(self):
        while self.wrappers != []:
            for o in self.wrappers:
                o.findIdAndDestroy()

    def getProfile(self):
        profile = {}
        occurrences = {}

        for o in self.wrappers:
            if not o.name in occurrences:
                occurrences[o.name] = 0
            else:
                occurrences[o.name] += 1
            optionName = '{}-{}'.format(o.name, occurrences[o.name])
            profile[optionName] = o.widget.returnSettings()

        return profile

    def setProfile(self, profile):
        for option, attributes in profile.items():
            self.addOption(option.split('-')[0])
            self.wrappers[-1].widget.addSettings(attributes)


#!/usr/bin/env python3
# -------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# -------------------------------------------------------------

import itertools
from os.path import join
from utils_misc import split_rowcol, config_writer, mat_type_check

# This file contains configuration settings for data generation
DATA_FORMAT = 'binary'

MATRIX_TYPE_DICT = {'dense': '0.9',
                    'sparse': '0.01'}

FAMILY_NO_MATRIX_TYPE = ['clustering', 'stats1', 'stats2']

def multinomial_datagen(matrix_dim, matrix_type, datagen_dir, config_dir):
    path_name = '.'.join(['multinomial', matrix_type, str(matrix_dim)])
    datagen_write = join(datagen_dir, path_name)
    save_path = join(config_dir, path_name)

    row, col = split_rowcol(matrix_dim)

    numSamples = row
    numFeatures = col
    sparsity = MATRIX_TYPE_DICT[matrix_type]
    num_categories = '150'
    intercept = '0'
    X = join(datagen_write, 'X.data')
    Y = join(datagen_write, 'Y.data')
    fmt = DATA_FORMAT

    config = [numSamples, numFeatures, sparsity, num_categories, intercept,
              X, Y, fmt, '1']

    config_writer(save_path + '.json', config)

    return save_path

def binomial_datagen(matrix_dim, matrix_type, datagen_dir, config_dir):
    path_name = '.'.join(['binomial', matrix_type, str(matrix_dim)])
    datagen_write = join(datagen_dir, path_name)
    save_path = join(config_dir, path_name)

    row, col = split_rowcol(matrix_dim)

    numSamples = row
    numFeatures = col
    maxFeatureValue = '5'
    maxWeight = '5'
    loc_weights = join(datagen_write, 'weight.data')
    loc_data = join(datagen_write, 'X.data')
    loc_labels = join(datagen_write, 'Y.data')
    noise = '1'
    intercept = '0'
    sparsity = MATRIX_TYPE_DICT[matrix_type]
    tranform_labels = '1'
    fmt = DATA_FORMAT

    config = [numSamples, numFeatures, maxFeatureValue, maxWeight, loc_weights, loc_data,
              loc_labels, noise, intercept, sparsity, fmt, tranform_labels]
    config_writer(save_path + '.json', config)

    return save_path

def regression1_datagen(matrix_dim, matrix_type, datagen_dir, config_dir):
    path_name = '.'.join(['regression1', matrix_type, str(matrix_dim)])
    datagen_write = join(datagen_dir, path_name)
    save_path = join(config_dir, path_name)

    row, col = split_rowcol(matrix_dim)

    numSamples = row
    numFeatures = col
    maxFeatureValue = '5'
    maxWeight = '5'
    loc_weights = join(datagen_write, 'weight.data')
    loc_data = join(datagen_write, 'X.data')
    loc_labels = join(datagen_write, 'Y.data')
    noise = '1'
    intercept = '0'
    sparsity = MATRIX_TYPE_DICT[matrix_type]
    tranform_labels = '1'
    fmt = DATA_FORMAT

    config = [numSamples, numFeatures, maxFeatureValue, maxWeight, loc_weights, loc_data,
              loc_labels, noise, intercept, sparsity, fmt, tranform_labels]
    config_writer(save_path + '.json', config)

    return save_path

def regression2_datagen(matrix_dim, matrix_type, datagen_dir, config_dir):
    path_name = '.'.join(['regression2', matrix_type, str(matrix_dim)])
    datagen_write = join(datagen_dir, path_name)
    save_path = join(config_dir, path_name)

    row, col = split_rowcol(matrix_dim)

    numSamples = row
    numFeatures = col
    maxFeatureValue = '5'
    maxWeight = '5'
    loc_weights = join(datagen_write, 'weight.data')
    loc_data = join(datagen_write, 'X.data')
    loc_labels = join(datagen_write, 'Y.data')
    noise = '1'
    intercept = '0'
    sparsity = MATRIX_TYPE_DICT[matrix_type]
    tranform_labels = '1'
    fmt = DATA_FORMAT

    config = [numSamples, numFeatures, maxFeatureValue, maxWeight, loc_weights, loc_data,
              loc_labels, noise, intercept, sparsity, fmt, tranform_labels]
    config_writer(save_path + '.json', config)

    return save_path

def clustering_datagen(matrix_dim, matrix_type, datagen_dir, config_dir):

    path_name = '.'.join(['clustering', matrix_type, str(matrix_dim)])
    datagen_write = join(datagen_dir, path_name)
    save_path = join(config_dir, path_name)
    row, col = split_rowcol(matrix_dim)

    X = join(datagen_write, 'X.data')
    Y = join(datagen_write, 'Y.data')
    YbyC = join(datagen_write, 'YbyC.data')
    C = join(datagen_write, 'C.data')
    nc = '50'
    dc = '10.0'
    dr = '1.0'
    fbf = '100.0'
    cbf = '100.0'

    config = dict(nr=row, nf=col, nc=nc, dc=dc, dr=dr, fbf=fbf, cbf=cbf, X=X, C=C, Y=Y,
                  YbyC=YbyC, fmt=DATA_FORMAT)

    config_writer(save_path + '.json', config)
    return save_path

def stats1_datagen(matrix_dim, matrix_type, datagen_dir, config_dir):

    path_name = '.'.join(['stats1', matrix_type, str(matrix_dim)])
    datagen_write = join(datagen_dir, path_name)
    save_path = join(config_dir, path_name)

    row, col = split_rowcol(matrix_dim)

    DATA = join(datagen_write, 'X.data')
    TYPES = join(datagen_write, 'types')
    TYPES1 = join(datagen_write, 'set1.types')
    TYPES2 = join(datagen_write, 'set2.types')
    INDEX1 = join(datagen_write, 'set1.indices')
    INDEX2 = join(datagen_write, 'set2.indices')
    MAXDOMAIN = '1100'
    SETSIZE = '20'
    LABELSETSIZE = '10'

    # NC should be less than C and more than num0
    # NC = 10 (old value)
    # num0 = NC/2
    # num0 < NC < C
    # NC = C/2
    NC = int(int(col)/2)

    config = dict(R=row, C=col, NC=NC, MAXDOMAIN=MAXDOMAIN, DATA=DATA, TYPES=TYPES, SETSIZE=SETSIZE,
                  LABELSETSIZE=LABELSETSIZE, TYPES1=TYPES1, TYPES2=TYPES2, INDEX1=INDEX1,
                  INDEX2=INDEX2, fmt=DATA_FORMAT)

    config_writer(save_path + '.json', config)

    return save_path

def stats2_datagen(matrix_dim, matrix_type, datagen_dir, config_dir):

    path_name = '.'.join(['stats2', matrix_type, str(matrix_dim)])
    datagen_write = join(datagen_dir, path_name)
    save_path = join(config_dir, path_name)
    row, col = split_rowcol(matrix_dim)

    D = join(datagen_write, 'X.data')
    Xcid = join(datagen_write, 'Xcid.data')
    Ycid = join(datagen_write, 'Ycid.data')
    A = join(datagen_write, 'A.data')

    config = dict(nr=row, nf=col, D=D, Xcid=Xcid, Ycid=Ycid,
                  A=A, fmt=DATA_FORMAT)

    config_writer(save_path + '.json', config)
    return save_path

def dimreduction_datagen(matrix_dim, matrix_type, datagen_dir, config_dir):

    path_name = '.'.join(['dimreduction', matrix_type, str(matrix_dim)])
    datagen_write = join(datagen_dir, path_name)
    save_path = join(config_dir, path_name)
    row, col = split_rowcol(matrix_dim)

    R = row
    C = col
    OUT = join(datagen_write, 'X.data')

    config = dict(R=R, C=C, OUT=OUT, FMT=DATA_FORMAT)

    config_writer(save_path + '.json', config)
    return save_path

def config_packets_datagen(algo_payload, matrix_type, matrix_shape, datagen_dir, dense_algos, config_dir):
    """
    This function has two responsibilities. Generate the configuration files for
    datagen algorithms and return a dictionary that will be used for execution.

    algo_payload : List of tuples
    The first tuple index contains algorithm name and the second index contains
    family type.

    matrix_type: String
    Type of matrix to generate e.g dense, sparse, all

    matrix_shape: String
    Shape of matrix to generate e.g 100k_10

    datagen_dir: String
    Path of the data generation directory

    dense_algos: List
    Algorithms that support only dense matrix type

    config_dir: String
    Location to store

    return: Dictionary {string: list}
    This dictionary contains algorithms to be executed as keys and the path of configuration
    json files to be executed list of values.
    """

    config_bundle = {}
    distinct_families = set(map(lambda x: x[1], algo_payload))

    # Cross Product of all configurations
    for current_family in distinct_families:
        current_matrix_type = mat_type_check(current_family, matrix_type, dense_algos)
        config = list(itertools.product(matrix_shape, current_matrix_type))
        # clustering : [[10k_1, dense], [10k_2, dense], ...]
        config_bundle[current_family] = config

    config_packets = {}
    for current_family, configs in config_bundle.items():
        config_packets[current_family] = []
        for size, type in configs:
            family_func = current_family.lower() + '_datagen'
            conf_path = globals()[family_func](size, type, datagen_dir, config_dir)
            config_packets[current_family].append(conf_path)

    return config_packets


#!/usr/bin/env python3

# https://stackoverflow.com/questions/6908143/should-i-put-shebang-in-python-scripts-and-what-form-should-it-take

import sys
import argparse
import subprocess

def recibeConfig():
    parser = argparse.ArgumentParser(description='Saludar a la gente')
    parser.add_argument('--nombre',
                        help='nombre de quien quieres saludar',
                        default="Pepe"
                        )

    args = parser.parse_args()
    print(args)
    return args

def saludameSay(nombre):
    cmd = ['/bin/bash', '-c', "say \"{}\"".format(nombre)]
    print(cmd)
    subprocess.Popen(cmd)
    print("Saludado")

def main():
    # Pipeline

    print(sys.argv)

    # PASO 1 - Recibir flags y estandarizarlos en un dict
    config = recibeConfig()

    # PASO 2 - Saludar
    print("Hola {}".format(config.nombre))
    for _ in range(5):
        saludameSay(config.nombre)

    #df = getDatos(config.url)
    #df_clean = cleanDf(df)

if __name__=="__main__":
    main()


from sqlalchemy.orm import Session

from moquant.dbclient import db_client
from moquant.dbclient.mq_share_pool import MqSharePool
from moquant.dbclient.mq_strategy_trade import MqStrategyTrade
from moquant.log import get_logger
from moquant.tsclient import ts_client
from moquant.utils import date_utils, decimal_utils

log = get_logger(__name__)

__strategy = 'down_gap'
__version = '1.0'
__wiki = 'https://zhuanlan.zhihu.com/p/450095164'

def valid_gap_num(gap_dict, current_dt):
    num = 0
    for k in gap_dict:
        price = gap_dict[k]
        if price['down_limit'] and current_dt != price['trade_date']:
            continue
        num += 1
    return num

def can_buy(gap_dict, current_dt, gap_num):
    if valid_gap_num(gap_dict, current_dt) < gap_num:
        return False
    for k in gap_dict:
        if gap_dict[k]['trade_date'] == current_dt:
            return True
    return False

def delete_old(dt: str):
    s: Session = db_client.get_session()
    s.query(MqStrategyTrade).filter(MqStrategyTrade.strategy == __strategy,
                                    MqStrategyTrade.strategy_version == __version,
                                    MqStrategyTrade.buy_date == dt).delete()
    s.query(MqStrategyTrade).filter(MqStrategyTrade.strategy == __strategy,
                                    MqStrategyTrade.strategy_version == __version,
                                    MqStrategyTrade.sell_date == dt).delete()
    s.query(MqSharePool).filter(MqSharePool.dt == dt).delete()
    s.close()

def run(dt: str, min_mv=5e10, gap_num=3):
    """
    2022年开始市值 > 500亿
    一个月内出现三个以上向下缺口时买入，只能在新缺口当天买入
    一字跌停缺口不算 少于1%的缺口不算
    买入日回补缺口则次日开盘价卖出
    买入日没回补缺口则次日挂补缺价卖出 最后收盘价无脑卖出

    :param dt: 对应日期
    :param min_mv: 最低市值要求
    :param gap_num: 需要达到多少个缺口
    :return:
    """
    delete_old(dt)
    ts = ts_client.get_ts()
    pro = ts_client.get_pro()
    trade_df = pro.trade_cal(exchange='SSE', start_date=dt, end_date=dt, is_open=1)
    if len(trade_df) == 0:
        log.info('[Down Gap] Not trade date')
        return

    trade_df = pro.trade_cal(exchange='SSE', start_date='20220101', end_date=dt, is_open=1)
    first_trade_date = trade_df.iloc[0]['cal_date']

    basic = pro.daily_basic(trade_date=first_trade_date, fields=['ts_code', 'total_mv'])
    start = date_utils.format_delta(dt, -30)

    info = pro.stock_basic(fields=["ts_code", "name"]).set_index('ts_code')
    candidate_num = 0

    for ts_code in basic.loc[basic['total_mv'] * 10000 > min_mv]['ts_code']:
        log.info('[Down Gap] %s' % ts_code)
        strategy_record = []
        gap_record = {}
        from_dt = start
        last_price = None
        price_df = ts.pro_bar(ts_code=ts_code, start_date=from_dt, adj='qfq').set_index('trade_date')
        current_trade = None
        while from_dt <= dt:
            can_buy_today = current_trade is None  # 如果今天持股，即使有卖出也不该再买入了
            if from_dt not in price_df.index:  # 非交易日
                from_dt = date_utils.format_delta(from_dt, 1)
                continue
            current_price = price_df.loc[from_dt]
            if last_price is not None:
                # 判断是否应该无脑卖出 按开盘价卖出
                if current_trade is not None and current_trade['should_sell']:
                    current_trade['sell_date'] = from_dt
                    current_trade['sell_price'] = current_price['open']
                    strategy_record.append(current_trade)
                    current_trade = None

                # 先判断开盘有没有下缺口
                if current_price['open'] < last_price['low'] and last_price['low'] / current_price['open'] > 1.01:
                    down_limit = current_price['open'] == current_price['close'] and current_price['open'] == \
                                 current_price['high'] and current_price['open'] == current_price['low']
                    new_gap = {'trade_date': from_dt, 'low': last_price['low'], 'high': current_price['open'],
                               'down_limit': down_limit}
                    gap_record[from_dt] = new_gap

                # 达到3个下缺口时按开盘价买入
                if can_buy_today and can_buy(gap_record, from_dt, gap_num):
                    current_trade = {'ts_code': ts_code, 'buy_date': from_dt, 'buy_price': current_price['open'],
                                     'target_price': last_price['low'], 'should_sell': False}

                # 判断最高价回补了多少个缺口
                high = current_price['high']
                for d in list(gap_record.keys()):
                    if high >= gap_record[d]['low']:  # 补缺了
                        del (gap_record[d])
                    elif from_dt >= date_utils.format_delta(d, 30):  # 超过30天了
                        del (gap_record[d])
                    elif current_price['high'] > gap_record[d]['high']:
                        gap_record[d]['high'] = current_price['high']

                # 判断今天有没有卖出
                if current_trade is not None:
                    if high >= current_trade['target_price']:  # 补缺了
                        # 当天买的，卖不了，等第二天
                        if current_trade['buy_date'] == from_dt:
                            current_trade['should_sell'] = True
                        else:
                            current_trade['sell_date'] = from_dt
                            current_trade['sell_price'] = current_trade['target_price']
                            strategy_record.append(current_trade)
                            current_trade = None
                    else:  # 没补缺 看是不是收盘卖
                        if current_trade['buy_date'] != from_dt:
                            current_trade['sell_date'] = from_dt
                            current_trade['sell_price'] = current_price['close']
                            strategy_record.append(current_trade)
                            current_trade = None

            last_price = current_price
            from_dt = date_utils.format_delta(from_dt, 1)

        if len(strategy_record) > 0:
            last_trade = strategy_record[-1]
            if last_trade['buy_date'] == dt:  # 今日买入
                strategy_buy = MqStrategyTrade(strategy=__strategy, strategy_version=__version, strategy_intro=__wiki,
                                               ts_code=ts_code, share_name=info.loc[ts_code]['name'],
                                               buy_date=last_trade['buy_price'], buy_price=last_trade['buy_date'])
                db_client.batch_insert([strategy_buy])
            elif last_trade['sell_date'] == dt:  # 今日卖出
                s: Session = db_client.get_session()
                s.query(MqStrategyTrade).filter(MqStrategyTrade.strategy == __strategy,
                                                MqStrategyTrade.strategy_version == __version,
                                                MqStrategyTrade.ts_code == ts_code,
                                                MqStrategyTrade.buy_date == last_trade['buy_date']).delete()
                s.close()
                strategy_sell = MqStrategyTrade(strategy=__strategy, strategy_version=__version, strategy_intro=__wiki,
                                                ts_code=ts_code, share_name=info.loc[ts_code]['name'],
                                                buy_date=last_trade['buy_date'], buy_price=last_trade['buy_price'],
                                                sell_date=last_trade['sell_date'], sell_price=last_trade['sell_price'])
                db_client.batch_insert([strategy_sell])

        if valid_gap_num(gap_record, dt) == gap_num - 1:
            to_insert = MqSharePool(dt=dt, strategy=__strategy, ts_code=ts_code, share_name=info.loc[ts_code]['name'],
                                    suggest_price=decimal_utils.cal_price_with_gain_percent(current_price['low'], -1),
                                    suggest_comment='次日下缺口超1%买入')
            db_client.batch_insert([to_insert])
            candidate_num += 1

    log.info('[Down Gap] Done. Dt: %s. Total: %d' % (dt, candidate_num))

if __name__ == '__main__':
    run('20220113')


import argparse
import hashlib

DEBUG = False
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("program_path", help="Program path", type=str)
    parser.add_argument('-d', '--debug', help="Debug mode", action="store_true")

    args = parser.parse_args()
    filename = args.program_path
    DEBUG = args.debug

code = open(filename, "r").read().split("\n")
processed_code = ""

for Q in code:
    number = Q.split("|")[0]
    hash_object = Q.split("|")[1]
    h = hashlib.sha256()
    h.update(bytes(hash_object, "UTF-8"))
    hex_hash = str(h.hexdigest())[-int(number) * 2:]
    if DEBUG:
        print(hash_object + " > " + hex_hash)
    hex_char = ""
    while len(hex_hash) > 0:
        hex_char = chr(int(hex_hash[0:2], 16) % 94 + 32)
        if DEBUG:
            print(" ---> " + str(int(hex_hash[0:2], 16)) + " ('" + str(hex_char) + "')")
        hex_hash = hex_hash[2:]
        processed_code += hex_char

exec(processed_code.replace("#", "\n"))


from django.db import models
from django.core.urlresolvers import reverse

# Create your models here.
class Tag(models.Model):
    """ Description for tags goes here """
    #tags = models.ForeignKey(Project)
    tag_name = models.CharField(max_length=200)
    def __unicode__(self):              # __unicode__ on Python 2
        return self.tag_name

class Project(models.Model):
    """ Description for models goes here """
    title = models.CharField(max_length=200)
    description = models.CharField(max_length = 5000)
    collaborators = models.IntegerField(default=1)
    posted = models.DateField(db_index=True, auto_now_add=True)

    CATEGORY_CHOICES = (
        ('STUDENT', 'Student Project'),
        ('UDACITY', 'Udacity Project'),
        ('ENTERPRISE', 'Enterprise Project'),
        ('OPEN SOURCE', 'Open Source'),
        ('CONTEST', 'Contest'),
    )
    category = models.CharField(max_length=15,
                                      choices=CATEGORY_CHOICES,
                                      default='STUDENT')

    articles = models.CharField(max_length = 5000)
    tags = models.ManyToManyField(Tag)

    def get_absolute_url(self):
        return reverse('project-detail', kwargs={'pk': self.pk})

    def __unicode__(self):              # __unicode__ on Python 2
        return self.title

class User(models.Model):
    """ Description goes here """

    username = models.CharField(max_length = 100)
    email = models.EmailField(max_length=254)
    NANODEGREE_CHOICES = (
        ('FULLSTACK', 'FullStack Developer'),
        ('FRONTEND', 'Frontend Developer'),
        ('ANDROID', 'Android Developer'),
        ('DATA ANALYST', 'Data Analyst'),
        ('IOS DEVELOPER', 'IOS Developer'),
    )
    nanodegree = models.CharField(max_length=15,
                                  choices=NANODEGREE_CHOICES,
                                      )

    def __unicode__(self):              # __unicode__ on Python 2
        return self.username

class UserProject (models.Model):
    """ User connected to set of projects """
    user = models.ForeignKey(User)
    project = models.ForeignKey(Project)

class TagsProject (models.Model):
    """ Tags connected to set of project """
    tag = models.ForeignKey(Tag)
    project = models.ForeignKey(Project)


"""This module contains utility functions which other modules use.
"""

import os
import csv
import unicodedata
import re
from . import models
from .download import download_file, extract_zip
from .xml import read_xml, orig_XML_to_doc_obj
from typing import Dict, Tuple

def kansuji_to_num(x: str) -> int:
    """Convert a sequence of *kansuji* to the corresponding number.

    *Kansuji* is a representation of a number, which is used in Chinese and Japanese. This function converts a sequence of *kansuji* (a *kanji* string) to the corresponding number.

    Parameters
    ----------
    x : str
      A sequence of *kansuji*.

    Returns
    -------
    int
      The corresponding number for a given *kansuji* string.

    Examples
    --------
    >>> kawasemi.util.kansuji_to_num('三十五')
    35

    >>> kawasemi.util.kansuji_to_num('百六')
    106

"""

    result = 0

    if x == '' or x is None:
        return result

    nums = [int(unicodedata.numeric(l)) for l in x[:]]
    nums2 = [nums.pop(0)]

    for n in nums:
        if n < 10:
            nums2.append(n)
        else:
            nums2[-1] *= n
    for n in nums2:
        result += n

    return result

def lawNum_to_lawID(x: str) -> str:
    """Map a law number in Japan to the corresponding law ID.

    Each law in Japan has its unique law number. While a law number is an identifier, the *number* is difficult for a computer to deal with because the string consists of numbers and Chinese (*kanji*) characters. On the other hand, each law has its unique filename in the Japanese e-Gov database. We call such filenames law IDs for convenience.
    This function maps a law number to the corresponding law ID.

    Parameters
    ----------
    x : str
      A law number in Japan.

    Returns
    -------
    str
      The corresponding law ID.

    Examples
    --------
    >>> kawasemi.util.lawNum_to_lawID('明治二十九年法律第八十九号')
    129AC0000000089

    >>> kawasemi.util.kansuji_to_num('昭和四十五年法律第四十八号')
    345AC0000000048

"""

    era2id = {'明治': 1, '大正': 2, '昭和': 3, '平成': 4, '令和': 5}
    lawtype2id = {'法律': 'AC'}

    m = re.match(r'(.{2})(.+)年(.+)第(.+)号', x)
    era = era2id[m.group(1)]
    year = kansuji_to_num(m.group(2).replace('元', '一'))
    lawtype = lawtype2id[m.group(3)]
    num = kansuji_to_num(m.group(4))

    return '{:3d}{:s}{:010d}'.format(era * 100 + year, lawtype, num)

def make_law2lawNum_and_law2lawID(filename: str =None) -> Tuple[Dict[str, str], Dict[str, str]]:
    """Create two dictionaries whose keys are law names.

    This function creates the following two dictionaries:
    1. a dictionary of law numbers for law names, and
    2. a dictionary of law IDs for law names,
    where law names as keys include abbreviated forms of official law names.

    Parameters
    ----------
    filename : str, optional
      A tsv file of (law name, law number) pairs. If it is None, the function uses the tsv file "abbr.tsv" in the data/utils directory.

    Returns
    -------
    law2lawNum : Dict[str, str]
      A dictionary of law numbers for law names.
    law2lawID : Dict[str, str]
      A dictionary of law IDs for law names.

    Examples
    --------
    >>> law2lawNum, law2lawID = kawasemi.util.make_law2lawNum_and_law2lawID()
    >>> law2lawNum['民法']
    明治二十九年法律第八十九号
    >>> law2lawID['民法']
    129AC0000000089
    >>> law2lawNum['労働基準法']
    昭和二十二年法律第四十九号
    >>> law2lawNum['労基法']
    昭和二十二年法律第四十九号

"""

    if filename == None:
        filename = os.path.join(os.path.dirname(__file__),
                                '../../data/utils/abbr.tsv')
    law2lawNum = {}
    law2lawID = {}

    with open(filename, 'r') as f:
        for line in csv.reader(f, delimiter='\t'):
            law2lawNum[line[0]] = line[1]
            law2lawID[line[0]] = lawNum_to_lawID(line[1])
    return law2lawNum, law2lawID

def load_model(lawname):
    """load model.

    under construction

    """

    law2lawNum, law2lawID = make_law2lawNum_and_law2lawID()

    try:
        lawID = law2lawID[lawname]
    except KeyError:
        print('You should input a valid law name in Japan.')
        exit()
    tree = read_xml(lawID)

    return models.Statute(lawname)


# Визуализирует фильтр VGG16 по индексу на зашумленном изображении.

from __future__ import print_function

from keras.preprocessing.image import load_img, img_to_array
from scipy.misc import imsave
import numpy as np
import time
from keras import applications
from keras.applications import vgg16
from keras.models import Model
from keras.layers import Dropout, Flatten, Dense
from keras import backend as K

img_width = 224
img_height = 224
index = 245

def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)

def deprocess_image(x):
    # normalize tensor: center on 0., ensure std is 0.1
    x -= x.mean()
    x /= (x.std() + 1e-5)
    x *= 0.1

    # clip to [0, 1]
    x += 0.5
    x = np.clip(x, 0, 1)

    # convert to RGB array
    x *= 255
    if K.image_data_format() == 'channels_first':
        x = x.transpose((1, 2, 0))
    x = np.clip(x, 0, 255).astype('uint8')
    return x

model = applications.VGG16(weights='imagenet', include_top=True)

print('Model loaded.')

input_img = model.input

loss = K.mean(model.output[:, index])
grads = K.gradients(loss, input_img)[0]
grads = normalize(grads)
iterate = K.function([input_img], [loss, grads])

step = 1.
kept_filters = []

input_img_data = np.random.random((1, img_width, img_height, 3))
input_img_data = (input_img_data - 0.5) * 20 + 128

while(True):
    loss_value, grads_value = iterate([input_img_data])
    input_img_data += grads_value * step

    print('Current loss value:', loss_value)
    if loss_value > 0.999:

        break

img = deprocess_image(input_img_data[0])
imsave('filters/%d_filter_.png' % (index), img)


import cv2
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import animation
import matplotlib.patches as patches
from scipy.interpolate import RectBivariateSpline

from LucasKanade import LucasKanade, get_warp_idx, get_interpolated_rect

# write your script here, we recommend the above libraries for making your animation

if __name__ == '__main__':
    car_video = np.load('../data/carseq.npy')
    frame_count = car_video.shape[-1]
    a,b,c,d = 59, 116, 145, 151
    p = np.zeros(2)
    It = car_video[b:d,a:c,0]
    rect = np.array([a,b,c,d])
    first_It = car_video[b:d,a:c,0]
    first_rect = np.array([a,b,c,d])
    epsilon = 2
    all_rect = []
    all_rect.append(rect)

    for t in range(1,frame_count):
        print (t)
        It1 = car_video[:,:,t]
        rows, columns = It1.shape
        mesh = RectBivariateSpline(np.linspace(0,rows-1,rows), np.linspace(0,columns-1,columns), It1)
        p = LucasKanade(It, It1, rect)
        p_star = LucasKanade(first_It, It1, first_rect)

        if np.linalg.norm(p_star - p, 1) < epsilon:
            # print ('updating template')
            warped_idx = get_warp_idx(rect, p)
            It = get_interpolated_rect(mesh, warped_idx, 0, 0)
            It = It.reshape(d-b, c-a)
        rect = np.array([rect[0]+p[1],rect[1]+p[0],rect[2]+p[1],rect[3]+p[0]])
        all_rect.append(rect)
        visualise_img = cv2.rectangle(It1.copy(), (int(rect[0]),int(rect[1])), (int(rect[2]),int(rect[3])), (0,0,0), 1)
        cv2.imshow('Template correction LK', visualise_img)
        cv2.waitKey(1)
        # if t % 100 == 0 or t==1:
        #     fig,ax = plt.subplots(1)
        #     rect_bbox = patches.Rectangle((rect[0],rect[1]),rect[2] - rect[0],rect[3] - rect[1],linewidth=1,edgecolor='r',facecolor='none')
        #     ax.add_patch(rect_bbox)
        #     ax.imshow(It1, cmap=plt.gray())
        #     # plt.show()
        #     plt.savefig('../results/corrected_'+str(t)+'.png')
        #     plt.close()
    all_rect = np.asarray(all_rect)
    np.save('../results/carseqrects-wcrt.npy', all_rect)
    cv2.destroyAllWindows()


import torch
import torch.nn as nn
import json
from pprint import pprint
import nltk
from nltk.corpus import stopwords
import numpy as np
from trash import get_all_data
import torch.nn.functional as F
from torch import optim
import random
import argparse
import copy
import pandas as pd
from sklearn.metrics import classification_report

class member_gcn(nn.Module):
    def __init__(self,  member_size, state_size,party_size):
        super(member_gcn,self).__init__()
        self.member_embedding = nn.Embedding(member_size, 16)
        self.party_embedding = nn.Embedding(party_size, 8)
        self.state_embeddding = nn.Embedding(state_size, 8)
        self.bn1 = nn.BatchNorm1d(32)
    def forward(self, member, state, party, adjacent_matrix_hat):
        member_embed = self.member_embedding(member)
        party_embed = self.party_embedding(party)
        state_embed = self.state_embeddding(state)
        legislator_embem = torch.cat([member_embed, party_embed, state_embed], dim=1)
        legislator_embem = self.bn1(legislator_embem)
        return legislator_embem

class LSTM_MLP(nn.Module):
    def __init__(self, word_embed_dim, hidden_dim, lstm_out_dim, vocab_size, mask_size, state_size,party_size):
        super(LSTM_MLP, self).__init__()
        self.lstm_out_dim = lstm_out_dim
        self.hidden_dim = hidden_dim
        self.word_embed_dim = word_embed_dim
        # self.w0 = torch.rand(self.lstm_out_dim + 32, gcn0_dim, dtype=torch.float64)
        # self.w1 = torch.rand(gcn0_dim, 3, dtype=torch.float64)
        self.legislation_embedding = nn.Embedding(vocab_size, word_embed_dim)
        self.lstm = nn.LSTM(word_embed_dim, lstm_out_dim)
        self.member_embed = member_gcn(mask_size,state_size,party_size)
        self.mlp1 = nn.Linear(64,32)
        self.mlp2 = nn.Linear(32,3)

    def forward(self, sentence, member_all, state_all, party_all, mask_size, adjacent_matrix):
        # embeding the legislation
        legislation_embed = self.legislation_embedding(sentence)
        # padding
        legislation_embed = torch.cat([legislation_embed, legislation_embed.new_zeros([32 - legislation_embed.size(0),self.word_embed_dim])], 0)
        _, out = self.lstm(legislation_embed.view(32, 1, -1))
        lstm_out = out[0][0]
        lstm_out = lstm_out.expand(mask_size, self.lstm_out_dim)

        # embeding the legislator
        legislator_embem = self.member_embed(member_all, state_all, party_all, adjacent_matrix)

        # normalize
        leg = F.normalize(lstm_out)
        votor = F.normalize(legislator_embem)
        return votor, leg

def sen2index(sen):
    words = nltk.word_tokenize(sen)
    index_ls = []
    for word in words:
        if str.lower(word) in word_dict:
            index_ls.append(word_dict[str.lower(word)])
    if len(index_ls)>32:
        index_ls = index_ls[0:32]
    return torch.tensor(index_ls, dtype=torch.long).to(cuda0)

def get_all_mem_info(member_info):
    member_size = len(member_info)
    member_all = np.array(range(member_size))
    state_all = np.zeros(member_size)
    party_all = np.zeros(member_size)
    for i in range(member_size):
        state_all[i], party_all[i] = member_info[i]
    return member_all, state_all, party_all

def member2index(vote_data, member_size, sample_num=1000):
    member_pos,member_neg = [],[]
    member_all, state_all, party_all = get_all_mem_info(member_info)
    vote_result_dict = {0:[],1:[],2:[]}
    # get the vote data
    for vote in vote_data:
        if vote == 'basic_information' or vote_data[vote]['Vote'] not in vote_dict:
            continue
        record = vote_data[vote]
        vote_result_dict[vote_dict[record['Vote']]].append(member_dict[vote])
        state_all[member_dict[vote]] = state_dict[record['State']]
        party_all[member_dict[vote]] = party_dict[record['Party']]
    # sample from postive and negtive
    vote_kind = [tmp for tmp in vote_result_dict if vote_result_dict[tmp]]
    if len(vote_kind)<2:
        return None,None,None,None,None

    for i in range(sample_num):
        pos_vote,neg_vote = np.sort(np.random.choice(vote_kind,2,replace=False))
        pos_member, neg_member = np.random.choice(vote_result_dict[pos_vote],1), np.random.choice(vote_result_dict[neg_vote],1)
        member_pos.append(pos_member)
        member_neg.append(neg_member)

    return torch.tensor(member_all, dtype=torch.long).to(cuda0), torch.tensor(state_all, dtype=torch.long).to(cuda0), \
        torch.tensor(party_all, dtype=torch.long).to(cuda0), torch.tensor(member_pos, dtype=torch.long).to(cuda0), \
        torch.tensor(member_neg, dtype=torch.long).to(cuda0)

def member2index_eval(vote_data, member_size):
    member_all, state_all, party_all = get_all_mem_info(member_info)
    vote_result = np.zeros(member_size)
    vote_result_dict = {0:[],1:[],2:[]}
    member_mask = []
    # get the vote data
    for vote in vote_data:
        if vote == 'basic_information' or vote_data[vote]['Vote'] not in vote_dict:
            continue
        record = vote_data[vote]
        vote_result_dict[vote_dict[record['Vote']]].append(member_dict[vote])
        state_all[member_dict[vote]] = state_dict[record['State']]
        party_all[member_dict[vote]] = party_dict[record['Party']]
        vote_result[member_dict[vote]] = vote_dict[record['Vote']]
        if vote in train_member:
            member_mask.append(member_dict[vote])
    vote_num = [len(vote_result_dict[tmp]) for tmp in vote_result_dict]
    return torch.tensor(member_all, dtype=torch.long).to(cuda0), torch.tensor(state_all, dtype=torch.long).to(cuda0), \
        torch.tensor(party_all, dtype=torch.long).to(cuda0), vote_result, \
        torch.tensor(member_mask, dtype=torch.long), vote_num

def Learning_rep(args):
    # define model
    lstmgcn_model = LSTM_MLP(word_embeddim, lstm_outdim, lstm_outdim,
                                len(word_dict), len(member_dict), len(state_dict), len(party_dict)).to(cuda0)
    pdist = nn.PairwiseDistance(2)
    optimizer = optim.Adam(lstmgcn_model.parameters(), lr=args.lr)

    # some set up for training and split train and val
    train_name_list = [name for name in train]
    random.shuffle(train_name_list)
    val_name_list = train_name_list[:len(train_name_list)//5]
    train_name_list = train_name_list[len(train_name_list)//5:]

    count = 0
    loss_avg = 0
    best_acc = 0

    early_stop_count = 0

    # begin training
    for epoch in range(args.epochs):
        print('epoch' + str(epoch))
        for legislation_name in train_name_list:
            count += 1

            # evaluations
            if count % args.eval == 1:
                print('Loss of Train set is %.4f' % (loss_avg/200.0))
                loss_avg = 0
                total = 0
                right = 0
                with torch.no_grad():
                    real_freq = {0: 0, 1: 0, 2: 0}
                    predict_freq = {0: 0, 1: 0, 2: 0}
                    for legislation_name1 in val_name_list:
                        if 'Official Title as Introduced' not in k[legislation_name1]['title']:
                            title_key = 'Official Titles as Introduced'
                        else:
                            title_key = 'Official Title as Introduced'
                        legislation_input = sen2index(
                            train[legislation_name1]['basic_information']['Descrption']+' '+k[legislation_name1]['title'][title_key])
                        if legislation_input.size()[0] == 0:
                            continue

                        member_input, state_input, party_input, gt_result, member_mask, vote_num  = member2index_eval(train[legislation_name1],len(member_dict))
                        if len(member_input) == 0:
                            continue

                        # get the representation
                        member_rep, leg_rep = lstmgcn_model(legislation_input, member_input, state_input, party_input,len(member_mask),adjacent_matrix)
                        member_rep = member_rep[member_mask,:]

                        # prediction
                        pred_result = np.zeros(len(member_input))
                        pair_distance = pdist(leg_rep, member_rep)
                        _,member_idx = torch.sort(pair_distance)
                        sorted_member = member_mask[member_idx]
                        pred_result[sorted_member[0:vote_num[0]]]=0
                        pred_result[sorted_member[vote_num[0]:sum(vote_num[0:2])]]=1
                        pred_result[sorted_member[sum(vote_num[0:2]):sum(vote_num[0:3])]]=2

                        # mask
                        pred_result = pred_result[member_mask]
                        gt_result = gt_result[member_mask]
                        # compute the accuracy
                        right += sum(pred_result == gt_result)
                        total += len(member_mask)
                        for i in range(len(pred_result)):
                            predict_freq[pred_result[i]] += 1
                            real_freq[gt_result[i]] += 1
                # save the best model and early stop
                if best_acc < (right / total):
                    best_acc = (right / total)
                    best_model = LSTM_MLP(word_embeddim, lstm_outdim, lstm_outdim,
                                len(word_dict), len(member_dict), len(state_dict), len(party_dict)).to(cuda0)
                    best_model.load_state_dict(copy.deepcopy(lstmgcn_model.state_dict()))
                    early_stop_count = 0
                early_stop_count += 1
                if early_stop_count >=args.early_stop:
                    break
                # the frequence of different classes
                print('----- predict_freq -----')
                print(predict_freq)
                print('----- real_freq -----')
                print(real_freq)
                print('LSTM_GCN acc:' + str(right / total) + '\t'+ '\t' + 'epoch:' + str(epoch) +
                       '\t\t' + 'Data: '+str(time_end-time_len)+'-'+str(time_end)+'\t'+'Best acc:'+str(best_acc))

            # get the data prepared
            if 'Official Title as Introduced' not in k[legislation_name]['title']:
                title_key = 'Official Titles as Introduced'
            else:
                title_key = 'Official Title as Introduced'
            legislation_input = sen2index(
                train[legislation_name]['basic_information']['Descrption']+' '+k[legislation_name]['title'][title_key])
            if legislation_input.size()[0] == 0:
                continue
            member_all, state_all, party_all, member_pos, member_neg \
                 = member2index(train[legislation_name],len(member_dict))

            if member_all is None:
                continue

            member_rep, leg_rep = lstmgcn_model(
                legislation_input, member_all, state_all, party_all, len(member_pos), adjacent_matrix)

            # get the representation
            member_rep_pos, member_rep_neg = member_rep[member_pos,:], member_rep[member_neg,:]

            # compute the loss for the representation
            per_pair_loss = pdist(leg_rep, member_rep_pos) - pdist(leg_rep, member_rep_neg)
            per_pair_loss = F.relu(per_pair_loss+10.0)
            loss = per_pair_loss.mean()
            loss_avg += loss.item()

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
        if early_stop_count >=args.early_stop:
            break
    return best_acc, best_model

def prediction_on_test(args,rep_model,leg_dict,predicted_vote_rate, gt_vote_rate_all):
    rep_model.eval()
    pdist = nn.PairwiseDistance(2)
    # prediction on test set
    total = 0
    right = 0
    right_gt = 0
    gt_result_all = []
    pred_result_all = []
    pred_result_gt_all = []
    with torch.no_grad():
        real_freq = {0: 0, 1: 0, 2: 0}
        predict_freq = {0: 0, 1: 0, 2: 0}
        for legislation_name1 in test:
            if 'Official Title as Introduced' not in k[legislation_name1]['title']:
                title_key = 'Official Titles as Introduced'
            else:
                title_key = 'Official Title as Introduced'
            legislation_input = sen2index(
                test[legislation_name1]['basic_information']['Descrption']+' '+k[legislation_name1]['title'][title_key])
            if legislation_input.size()[0] == 0:
                continue

            member_input, state_input, party_input, gt_result, member_mask, _  = member2index_eval(test[legislation_name1],len(member_dict))
            if len(member_input) == 0:
                continue

            # get the predicted vote rate
            vote_num = len(member_mask)
            vote_rate = predicted_vote_rate[leg_dict[legislation_name1],:].tolist()
            pred_vote_num = [int(tmp*vote_num) for tmp in vote_rate]
            if sum(pred_vote_num) != vote_num:
                pred_vote_num[0] += (vote_num-sum(pred_vote_num))

            # get the ground true vote rate
            gt_vote_rate = gt_vote_rate_all[leg_dict[legislation_name1],:].tolist()
            gt_vote_num = [int(tmp*vote_num) for tmp in gt_vote_rate]
            if sum(gt_vote_num) != vote_num:
                gt_vote_num[0] += (vote_num-sum(gt_vote_num))

            # get the representation
            member_rep, leg_rep = rep_model(legislation_input, member_input, state_input, party_input,len(member_mask),adjacent_matrix)
            member_rep = member_rep[member_mask,:]

            # prediction
            pred_result = np.zeros(len(member_input))
            pair_distance = pdist(leg_rep, member_rep)
            _,member_idx = torch.sort(pair_distance)
            sorted_member = member_mask[member_idx]
            pred_result[sorted_member[0:pred_vote_num[0]]]=0
            pred_result[sorted_member[pred_vote_num[0]:sum(pred_vote_num[0:2])]]=1
            pred_result[sorted_member[sum(pred_vote_num[0:2]):sum(pred_vote_num[0:3])]]=2

            # prediction with ground true vote rate
            pred_result_gt = np.zeros(len(member_input))
            pred_result_gt[sorted_member[0:gt_vote_num[0]]]=0
            pred_result_gt[sorted_member[gt_vote_num[0]:sum(gt_vote_num[0:2])]]=1
            pred_result_gt[sorted_member[sum(gt_vote_num[0:2]):sum(gt_vote_num[0:3])]]=2

            # mask
            pred_result_gt = pred_result_gt[member_mask]
            pred_result = pred_result[member_mask]
            gt_result = gt_result[member_mask]

            # compute the accuracy
            right += sum(pred_result == gt_result)
            right_gt += sum(pred_result_gt == gt_result)
            total += len(member_mask)
            for i in range(len(pred_result)):
                predict_freq[pred_result[i]] += 1
                real_freq[gt_result[i]] += 1
            gt_result_all.append(np.squeeze(gt_result))
            pred_result_all.append(np.squeeze(pred_result))
            pred_result_gt_all.append(np.squeeze(pred_result_gt))

    # compute the recall and f1 score
    gt_result_all = np.concatenate(gt_result_all)
    pred_result_all = np.concatenate(pred_result_all)
    pred_result_gt_all = np.concatenate(pred_result_gt_all)
    target_names = ['Yes','Not vote','No']
    metrics = classification_report(gt_result_all, pred_result_all, target_names=target_names,output_dict=True)
    wei_recall=metrics['weighted avg']['recall']
    micro_recall = metrics['micro avg']['recall']
    wei_f1 = metrics['weighted avg']['f1-score']
    micro_f1 = metrics['micro avg']['f1-score']

    metrics1 = classification_report(gt_result_all, pred_result_gt_all, target_names=target_names,output_dict=True)
    wei_recall_gt=metrics1['weighted avg']['recall']
    micro_recall_gt = metrics1['micro avg']['recall']
    wei_f1_gt = metrics1['weighted avg']['f1-score']
    micro_f1_gt = metrics1['micro avg']['f1-score']

    # the frequence of different classes
    print('----- predict_freq -----')
    print(predict_freq)
    print('----- real_freq -----')
    print(real_freq)
    print('Final LSTM_GCN_tri_vote_rate acc:' + str(right / total) +
            '\t\t' + 'Data: '+str(time_end-time_len)+'-'+str(time_end)+'\t'+'GT vote rate acc:'+str(right_gt/total))
    return right / total, right_gt/total, wei_recall, micro_recall, wei_f1, micro_f1, wei_recall_gt, micro_recall_gt, wei_f1_gt, micro_f1_gt

if __name__ == '__main__':
    # important parameters: semantic, result, lstm
    parser = argparse.ArgumentParser(
        description='HyperParameters for String Embedding')
    parser.add_argument('--epochs', type=int, default=100000,
                        help='number of epochs to train (default: 10)')
    parser.add_argument('--word_dim', type=int, default=32,
                        help='word dimension')
    parser.add_argument('--lr', type=float, default=0.0001,
                        help='learning rate')
    parser.add_argument('--version', type=int, default=1,
                    help='version info')
    parser.add_argument('--eval', type=int, default=100,
                    help='evaluation frequency')
    parser.add_argument('--load', type=int, default=0,
                    help='load trained model')
    parser.add_argument('--time_start', type=int, default=2005,
                    help='whether to check result')
    parser.add_argument('--time_end', type=int, default=2017,
                    help='whether to check result')
    parser.add_argument('--time_len', type=int, default=4,
                    help='whether to check result')
    parser.add_argument('--cuda', type=int, default=0,
                    help='whether to check result')
    parser.add_argument('--lstm', type=int, default=1,
                    help='whether to use lstm in vote rate prediction')
    parser.add_argument('--semantic', type=int, default=1,
                    help='whether to use semantic gcn')
    parser.add_argument('--result', type=str, default='lstm_gcn_tri_pred_vote',
                    help='define the name for result file')
    parser.add_argument('--co_ocur', type=int, default=0,
                    help='whether to use co-ocurence adj matrix to build semantic gcn')
    parser.add_argument('--cat', type=int, default=1,
                    help='whether to cat semantic GCN output and lstm output')
    parser.add_argument('--early_stop', type=int, default=15,
                    help='early stop count')
    args = parser.parse_args()
    cuda0 = torch.device('cuda:{}'.format(args.cuda))
    # cuda0 = torch.device('cpu')
    # parameters
    word_embeddim = 100
    lstm_outdim = args.word_dim
    word_dict_len = 20000
    time_end = args.time_end
    time_len = args.time_len

    # load data
    with open('word_dict.json') as f:
        word_dict_all = json.load(f)
    word_dict = {}
    all_word = list(word_dict_all)
    for i in range(word_dict_len):
        word_dict[all_word[i]] = word_dict_all[all_word[i]]
    with open('legislation.json') as k:
        k = json.load(k)
    with open('house_vote.json') as f:
        house_vote = json.load(f)
    vote_dict = {'Yea': 0, 'Aye': 0, 'Nay': 2,
                 'No': 2, 'Not Voting': 1, 'Present': 1}
    acc = []
    acc_val = []
    acc_gt_vote_rate = []
    wei_recall_all, micro_recall_all, wei_f1_all, micro_f1_all, wei_recall_gt_all, micro_recall_gt_all, wei_f1_gt_all, micro_f1_gt_all = [],[],[],[],[],[],[],[]

    for time_end in range(args.time_start,args.time_end+1):
        # get the data this year
        train, test, member_dict, state_dict, party_dict, member_info, adjacent_matrix = get_all_data(
            time_end-time_len, time_end,load=False,eliminate=True)

        # get the menber in training data
        train_member = []
        for leg in train:
            for member in train[leg]:
                if member == 'basic_information':
                    continue
                if member not in train_member:
                    train_member.append(member)

        adjacent_matrix = torch.tensor(adjacent_matrix, dtype=torch.float32).to(cuda0)

        # get the predicted vote rate
        if args.lstm:
            from semantic_gcn_lstm import main_semantic_gcn
            predicted_vote_rate, leg_dict, gt_vote_rate = main_semantic_gcn(train,test,word_dict,args)
        else:
            from semantic_gcn import main_semantic_gcn
            predicted_vote_rate, leg_dict, gt_vote_rate = main_semantic_gcn(train,test,args)

        # Representation learning model
        acc_val_, learning_model = Learning_rep(args)

        # make prediction on test set
        acc_, acc_gt_vt, wei_recall, micro_recall, wei_f1, micro_f1, wei_recall_gt, micro_recall_gt, wei_f1_gt, micro_f1_gt = prediction_on_test(args,learning_model,leg_dict,predicted_vote_rate, gt_vote_rate)
        wei_recall_all.append(wei_recall); micro_recall_all.append(micro_recall); wei_f1_all.append(wei_f1); micro_f1_all.append(micro_f1); wei_recall_gt_all.append(wei_recall_gt); micro_recall_gt_all.append(micro_recall_gt); wei_f1_gt_all.append(wei_f1_gt); micro_f1_gt_all.append(micro_f1_gt)

        acc.append(acc_) # accuracy on test set
        acc_gt_vote_rate.append(acc_gt_vt) # with telling the ground true vote rate
        acc_val.append(acc_val_) # accuracy on validation set with known vote rate
        print('Runing acc: Test acc %.6f, Val acc %.6f' % (np.mean(acc), np.mean(acc_val)))
    acc = np.array(acc)
    acc_val = np.array(acc_val)
    acc_gt_vote_rate = np.array(acc_gt_vote_rate)
    result1 = pd.DataFrame({'wei_recall':wei_recall_all,'micro_recall':micro_recall_all, 'wei_f1':wei_f1_all, 'micro_f1':micro_f1_all,'wei_recall_gt':wei_recall_gt_all,'micro_recall_gt':micro_recall_gt_all, 'wei_f1_gt':wei_f1_gt_all, 'micro_f1_gt':micro_f1_gt_all})
    result = pd.DataFrame({'test acc':acc,'val acc':acc_val,'test acc with gt vote rate':acc_gt_vote_rate})
    result.to_csv('{}.csv'.format(args.result))
    result1.to_csv('{}_recall_f1.csv'.format(args.result))
    print('{} Final test acc: %.6f, val acc: %.6f, test acc with gt vote rate: %.6f'.format(args.result) % (np.mean(acc),np.mean(acc_val),np.mean(acc_gt_vote_rate)))


# -*- coding: utf-8 -*-
# Copyright (c) 2016, French National Center for Scientific Research (CNRS)
# Distributed under the (new) BSD License. See LICENSE for more info.

from ..core import WidgetNode, register_node_type
from pyqtgraph.Qt import QtCore, QtGui

import numpy as np
import pyqtgraph as pg

class ImageViewer(WidgetNode):
    """
    A simple image viewer using pyqtgraph.
    """
    _input_specs = {'video': dict(streamtype='video',dtype='uint8',
                                                shape=(-1, -1, 3), compression ='',
                                                ),
                                }
    def __init__(self, **kargs):
        WidgetNode.__init__(self, **kargs)

        self.layout = QtGui.QHBoxLayout()
        self.setLayout(self.layout)

        self.graphicsview = pg.GraphicsView()
        self.layout.addWidget(self.graphicsview)

        self.plot = pg.PlotItem()
        self.graphicsview.setCentralItem(self.plot)
        self.plot.getViewBox().setAspectLocked(lock=True, ratio=1)
        self.plot.hideButtons()
        self.plot.showAxis('left', False)
        self.plot.showAxis('bottom', False)

        self.image = pg.ImageItem()
        self.plot.addItem(self.image)

    def _configure(self, **kargs):
        pass

    def _initialize(self):
        in_params = self.input.params
        self.timer = QtCore.QTimer(singleShot=False)
        self.timer.setInterval(int(1./in_params['sample_rate']*1000))
        self.timer.timeout.connect(self.poll_socket)

    def _start(self):
        self.timer.start()

    def _stop(self):
        self.timer.stop()

    def _close(self):
        pass

    def poll_socket(self):
        event = self.input.socket.poll(0)
        if event != 0:
            index, data = self.input.recv()
            data = data[::-1,:,:]
            data = data.swapaxes(0,1)
            self.image.setImage(data)

register_node_type(ImageViewer)

"""
import vispy
import vispy.scene

class ImageViewer(WidgetNode):
    _input_specs = {'video' : dict(streamtype = 'video',dtype = 'uint8',
                                                shape = (-1, -1, 3), compression ='',
                                                ),
                                }
    def __init__(self, **kargs):
        WidgetNode.__init__(self, **kargs)

        self.layout = QtGui.QHBoxLayout()
        self.setLayout(self.layout)

        self.canvas = vispy.scene.SceneCanvas(keys='interactive', show=True)
        self.layout.addWidget(self.canvas.native)
        self.view = self.canvas.central_widget.add_view()
        self.view.camera = vispy.scene.PanZoomCamera(aspect=1)
        #~ self.view.camera = vispy.scene.MagnifyCamera(aspect=1)

    def start(self):
        self.timer.start()
        self._running = True

    def stop(self):
        self.timer.stop()
        self._running = False

    def close(self):
        pass

    def initialize(self):
        in_params = self.input.params
        img_data = np.zeros(in_params['shape']).astype(in_params['dtype'])
        self.image = vispy.scene.visuals.Image(img_data, parent=self.view.scene)
        # please luke hepl me here I do not known how to range the image in the full canvas
        self.view.camera.rect = (0,0) + tuple(in_params['shape'][:2])

        self.timer = QtCore.QTimer(singleShot=False)
        self.timer.setInterval(int(1./in_params['sample_rate']*1000))
        self.timer.timeout.connect(self.poll_socket)

    def configure(self, **kargs):
        pass

    def poll_socket(self):
        event =  self.input.socket.poll(0)
        if event != 0:
            index, data = self.input.recv()
            # this is a vertical flip
            # this should be done in GPU
            # please help
            data = data[::-1,:,:]
            self.image.set_data(data)
            self.image.update()
"""


# -*- coding: utf-8 -*-
##

from gi.repository import GObject

from gajim.plugins import GajimPlugin
from gajim.plugins.helpers import log_calls
from gajim.common import ged
from gajim.common import app
from gajim.common import helpers
from gajim import gtkgui_helpers
import unicodedata

def paragraph_direction_mark(text):
    """
    Determine paragraph writing direction according to
    http://www.unicode.org/reports/tr9/#The_Paragraph_Level

    Returns either Unicode LTR mark or RTL mark.
    """
    for char in text:
        bidi = unicodedata.bidirectional(char)
        if bidi == 'L':
            return '\u200E'
        elif bidi == 'AL' or bidi == 'R':
            return '\u200F'

    return '\u200E'

class ChatstatePlugin(GajimPlugin):

    @log_calls('ChatstatePlugin')
    def init(self):
        self.description = _('Chat State Notifications in roster.'
'Font color of the contact varies depending on the chat state.\n'
'The plugin does not work if you use custom font color for contacts in roster.\n'
'https://dev.gajim.org/gajim/gajim/issues/3628\nhttp://xmpp.org/extensions/xep-0085.html')
        self.config_dialog = None  # ChatstatePluginConfigDialog(self)
        self.events_handlers = {'chatstate-received':
                                    (ged.GUI2, self.chatstate_received), }
        self.active = None

    def chatstate_received(self, obj):
        if not self.active:
            return

        contact = app.contacts.get_contact_from_full_jid(obj.conn.name,
            obj.fjid)
        if not contact:
            return

        chatstate = obj.chatstate
        if chatstate not in self.chatstates.keys():
            return

        self.model = app.interface.roster.model
        child_iters = app.interface.roster._get_contact_iter(obj.jid,
            obj.conn.name, contact, self.model)

        name = GObject.markup_escape_text(contact.get_shown_name())
        contact_instances = app.contacts.get_contacts(obj.conn.name,
            contact.jid)

        # Show resource counter
        nb_connected_contact = 0
        for c in contact_instances:
            if c.show not in ('error', 'offline'):
                nb_connected_contact += 1
        if nb_connected_contact > 1:
            name += paragraph_direction_mark(name)
            name += ' (%d)' % nb_connected_contact

        for child_iter in child_iters:
            if chatstate != 'gone':
                color = self.chatstates[chatstate]
                name = '%s' % (color, name)
            if contact.status and app.config.get(
            'show_status_msgs_in_roster'):
                status = contact.status.strip()
                if status != '':
                    status = helpers.reduce_chars_newlines(status,
                            max_lines=1)
                    name += '\n%s' % (self.status_color,
                            GObject.markup_escape_text(status))
            self.model[child_iter][1] = name

    @log_calls('ChatstatePlugin')
    def activate(self):
        color = gtkgui_helpers.get_fade_color(app.interface.roster.tree,
            False, False)
        self.status_color = '#%02x%02x%02x' % (int(color.red * 255),
                                               int(color.green * 255),
                                               int(color.blue * 255))
        theme = app.config.get('roster_theme')
        self.chatstates = {'active': app.config.get('inmsgcolor'),
                            'composing': app.config.get_per('themes', theme,
                                         'state_composing_color'),
                            'inactive': app.config.get_per('themes', theme,
                                        'state_inactive_color'),
                            'paused': app.config.get_per('themes', theme,
                                        'state_paused_color'),
                            'gone': None, }
        self.active = True

    @log_calls('ChatstatePlugin')
    def deactivate(self):
        self.active = False