from random import randint
from PIL import Image
import numpy as np
import chainer
from NN import NeuralNetwork

global training_inputs, temp_training_outputs, test_inputs, test_outputs

# Initialize NumberNet as an object of NeuralNetwork class
NumberNet = NeuralNetwork()

# Load data from MNIST database
data = chainer.datasets.get_mnist(withlabel=True)

def reformatData(data):
    global training_inputs, temp_training_outputs, test_inputs, test_outputs

    # Get training inputs and outputs into the correct format (numpy arrays)
    training_input = [None for i in range(60000)]
    temp_training_outputs = [None for i in range(60000)]
    test_input = [None for i in range(60000)]
    test_outputs = [None for i in range(60000)]

    for i in range(60000):
        interim = data[0][i]
        test_interim = data[0][i]
        training_input[i] = interim[0]
        temp_training_outputs[i] = interim[1]
        test_input[i] = test_interim[0]
        test_outputs[i] = test_interim[1]

    training_inputs = np.array(training_input)
    training_inputs = training_inputs.T

    test_inputs = np.array(test_input)
    test_inputs = test_inputs.T

def train(iterations, rate):
    global training_inputs, temp_training_outputs, test_inputs, test_outputs
    NeuralNetwork.train(NumberNet, training_inputs=training_inputs, training_outputs=temp_training_outputs,
                        training_iterations=iterations, learning_rate=rate)

def test(num_tests):
    global training_inputs, temp_training_outputs, test_inputs, test_outputs
    correct = 0
    incorrect = 0

    # Test NN amount of times user specifies with num_tests
    for i in range(num_tests):
        # Get random test data
        r = randint(0, 59999)
        # Get test data in correct format
        new_test_input = np.array([[0.0 for x in range(784)]])
        temp = test_inputs[:, r]
        for j in range(784):
            new_test_input[0][j] = temp[j]
        new_test_input = new_test_input.T

        # Get NN's guess for input
        nnGuess = NeuralNetwork.think(NumberNet, test_input=new_test_input)

        # Log test results
        if test_outputs[r] == nnGuess:
            correct += 1
        else:
            incorrect += 1

    # Print test results
    print("Correct: " + str(correct))
    print("Incorrect: " + str(incorrect))

def testWithInput(input_from_user):
    nnGuess = NeuralNetwork.think(NumberNet, test_input=input_from_user)
    print("My guess is... " + str(nnGuess))

def convert_image():  # Converts from image_input.png, 700x700 pixels, returns [784, 1] array
    # Open image, converting to grayscale
    img = Image.open('image_input.png').convert('L')
    WIDTH, HEIGHT = img.size
    pic = list(img.getdata())

    # Convert that to 2D list (list of lists of integers)
    pic = [pic[offset:offset + WIDTH] for offset in range(0, WIDTH * HEIGHT, WIDTH)]

    # Convert to grayscale between 0 and 1
    temp_input_from_user = np.array(pic)
    for i in range(700):
        for j in range(700):
            temp_input_from_user[i][j] = (255 - temp_input_from_user[i][j]) / 255

    # Take average of pixels in 25x25 grids, and assign value to corresponding location of input
    input_from_user = np.empty(shape=(784, 1))
    for i in range(28):
        for j in range(28):
            totalPix = 0
            for a in range(25):
                for b in range(25):
                    totalPix += temp_input_from_user[i * 25 + a][j * 25 + b]
            totalPix = totalPix / 625
            input_from_user[(28 * i) + j] = totalPix

    return input_from_user

# Let NN decide what number user enters by running testWithInput
myNum = convert_image()
testWithInput(myNum)


from app.Extensions import db
from app.Kit import GetRequestJsonData, GetRequestFormData, GetRequestArgsData, PaginatePages, PaginatePagesArgs
from app.ReturnCode import ReturnCode
from datetime import datetime
from app.Models import Article
from app.ModelSerialize import Serialize, SerializeQuerySet

def query_article(request):
    id = GetRequestArgsData(request, 'id', None)
    article = Article.query.filter(Article.id == id).first()
    if not article:
        return ReturnCode.paramete_error, '文章不存在或参数有误', ''
    serialize = Serialize(article, obj='obj', dataprocessing='getarticlelist')
    return ReturnCode.ok, '', serialize

def query_article_list(request):
    article_type = GetRequestArgsData(request, 'article_type', None)
    query_pages = PaginatePagesArgs(request,None)

    if not article_type:
        return ReturnCode.paramete_error, '获取的文章类型缺乏', ''

    querys = Article.query.filter(Article.status == 0).order_by(Article.upload_time.desc())

    # 作品
    if int(article_type) == 1:
        querys = querys.filter(Article.article_type == article_type)

        content_type = GetRequestArgsData(request, 'content_type', None)
        if content_type:
            querys = querys.filter(Article.content_type == content_type)

    # 文章
    if int(article_type) == 2:
        querys = querys.filter(Article.article_type == article_type)

    # 项目
    if int(article_type) == 3:
        querys = querys.filter(Article.article_type == article_type)

    query_count, query_dataitems, query_datapages = SerializeQuerySet(querys, query_pages, per_page=100)
    return ReturnCode.ok, '', {
        'list':Serialize(query_dataitems,obj='list', dataprocessing='getarticlelist', notreturn=['content']),
        'queryCount': query_count,
        'dataPges': query_datapages,
        'nowPage': query_pages
        }

def upload_article(request):
    userid = GetRequestJsonData(request, 'userid', None)
    title = GetRequestJsonData(request, 'title', None)
    introduce = GetRequestJsonData(request, 'introduce', None)
    content = GetRequestJsonData(request, 'content', None)
    article_type = GetRequestJsonData(request, 'article_type', None)
    content_type = GetRequestJsonData(request, 'content_type', None)
    status = GetRequestJsonData(request, 'status', 0)
    cover = GetRequestJsonData(request, 'cover', None)

    if not userid:
        return ReturnCode.paramete_error, '用户id异常', ''

    if not title:
        return ReturnCode.paramete_error, '标题不能为空', ''

    if not introduce:
        return ReturnCode.paramete_error, '介绍不能为空', ''

    if not str(content):
        return ReturnCode.paramete_error, '内容不能为空', ''

    if not article_type:
        return ReturnCode.paramete_error, '发布类型不能为空', ''

    if not cover:
        return ReturnCode.paramete_error, '封面不能为空', ''

    if int(article_type) == 1:
        if not content_type:
            return ReturnCode.paramete_error, '作品类型不能为空', ''

    new = Article()
    new.upload_userid = userid
    new.upload_time = datetime.now()
    new.article_type = int(article_type)
    new.title = str(title)
    new.introduce = str(introduce)
    new.content = str(content)
    new.content_type = int(content_type)
    new.cover = str(cover)
    new.status = status

    db.session.add(new)
    try:
        db.session.commit()
        return ReturnCode.ok, '上传成功', {
            'id':new.id
        }

    except:
        db.session().rollback()
        return ReturnCode.server_error, '系统出错', ''


import pymongo

myclient = pymongo.MongoClient("mongodb://localhost:27017/")
mydb = myclient["Demo"]
mycol = mydb["Colection"]

# myquery = {"msg" : "send"}

mydoc = mycol.find()
# queryResult = str()
# for x in mydoc:
#   queryResult += x + ":" + mydoc[x]
queryResult = str(mydoc)

print(queryResult)


from torch.autograd.grad_mode import F
from torch.nn.functional import sigmoid
from torch.nn.modules.loss import CrossEntropyLoss
from torch.optim import SGD, Adam, lr_scheduler
from tqdm import tqdm
import math
from torch.cuda import amp
import torch
from utils.loss import DBLoss
import torch.nn as nn
import yaml
from basemodel import TextDetector
from utils.db_utils import SegDetectorRepresenter, QuadMetric
import numpy as np
from datetime import datetime
from torchsummary import summary
import numexpr
import os
import shutil
os.environ['NUMEXPR_MAX_THREADS'] = str(numexpr.detect_number_of_cores())

from db_dataset import create_dataloader
from utils.general import LOGGER, Loggers, CUDA, DEVICE
import time
import random

torch.random.manual_seed(0)
random.seed(0)
np.random.seed(0)

def one_cycle(y1=0.0, y2=1.0, steps=100):
    return lambda x: ((1 - math.cos(x * math.pi / steps)) / 2) * (y2 - y1) + y1

def eval_model(model: nn.Module, val_loader, post_process, metric_cls):
    # global DEVICE
    raw_metrics = []
    total_frame = 0.0
    total_time = 0.0
    model.eval()
    for i, batch in tqdm(enumerate(val_loader), total=len(val_loader), desc='test model'):
        with torch.no_grad():
            # 数据进行转换和丢到gpu
            for key, value in batch.items():
                if value is not None:
                    if isinstance(value, torch.Tensor):
                        batch[key] = value.to(DEVICE)
            start = time.time()
            with amp.autocast():
                preds = model(batch['imgs'])
            boxes, scores = post_process(batch, preds,is_output_polygon=False)
            total_frame += batch['imgs'].size()[0]
            total_time += time.time() - start
            raw_metric = metric_cls.validate_measure(batch, (boxes, scores))
            raw_metrics.append(raw_metric)
    metrics = metric_cls.gather_measure(raw_metrics)
    LOGGER.info('FPS:{}'.format(total_frame / total_time))
    return metrics['recall'].avg, metrics['precision'].avg, metrics['fmeasure'].avg

def train(hyp):
    start_epoch = 0
    hyp_train, hyp_data, hyp_model, hyp_logger, hyp_resume = hyp['train'], hyp['data'], hyp['model'], hyp['logger'], hyp['resume']
    epochs = hyp_train['epochs']
    batch_size = hyp_train['batch_size']

    scaler = amp.GradScaler(enabled=CUDA)
    criterion = DBLoss()
    use_bce = False
    if hyp_train['loss'] == 'bce':
        use_bce = True
    shrink_with_sigmoid = not use_bce

    model = TextDetector(hyp_model['weights'], map_location='cpu', act=hyp_model['act'])
    model.initialize_db(hyp_model['unet_weights'])
    model.dbnet.shrink_with_sigmoid = shrink_with_sigmoid
    model.train_db()
    model.to(DEVICE)

    if hyp_model['db_weights'] != '':
        model.dbnet.load_state_dict(torch.load(hyp_model['db_weights'])['weights'])
    if hyp_train['optimizer'] == 'adam':
        optimizer = Adam(model.dbnet.parameters(), lr=hyp_train['lr0'], betas=(0.937, 0.999), weight_decay=0.00002)  # adjust beta1 to momentum
    else:
        optimizer = SGD(model.dbnet.parameters(), lr=hyp_train['lr0'], momentum=hyp_train['momentum'], nesterov=True, weight_decay=hyp_train['weight_decay'])

    if hyp_train['linear_lr']:
        lf = lambda x: (1 - x / (epochs - 1)) * (1.0 - hyp_train['lrf']) + hyp_train['lrf']  # linear
    else:
        lf = one_cycle(1, hyp_train['lrf'], epochs)  # cosine 1->hyp['lrf']

    if hyp_train['linear_lr']:
        lf = lambda x: (1 - x / (epochs - 1)) * (1.0 - hyp_train['lrf']) + hyp_train['lrf']  # linear
    else:
        lf = one_cycle(1, hyp_train['lrf'], epochs)  # cosine 1->hyp['lrf']
    scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)  # plot_lr_scheduler(optimizer, scheduler, epochs)

    logger = None
    if hyp_resume['resume_training']:
        LOGGER.info(f'resume traning ... ')
        ckpt = torch.load(hyp_resume['ckpt'], map_location=DEVICE)
        model.dbnet.load_state_dict(ckpt['weights'])
        optimizer.load_state_dict(ckpt['optimizer'])
        scheduler.load_state_dict(ckpt['scheduler'])
        scheduler.step()
        start_epoch = ckpt['epoch'] + 1
        hyp_logger['run_id'] = ckpt['run_id']
        logger = Loggers(hyp)

    else:
        # if hyp_logger['type'] == 'wandb':
        logger = Loggers(hyp)

    train_img_dir, train_mask_dir, imgsz, augment, aug_param = hyp_data['train_img_dir'], hyp_data['train_mask_dir'], hyp_data['imgsz'], hyp_data['augment'], hyp_data['aug_param']
    val_img_dir, val_mask_dir = hyp_data['val_img_dir'], hyp_data['val_mask_dir']
    train_dataset, train_loader = create_dataloader(train_img_dir, train_mask_dir, imgsz, batch_size, augment, aug_param, shuffle=True, workers=hyp_data['num_workers'], cache=hyp_data['cache'])
    val_dataset, val_loader = create_dataloader(val_img_dir, val_mask_dir, imgsz, batch_size, augment=False, shuffle=False, workers=hyp_data['num_workers'], cache=hyp_data['cache'], with_ann=True)
    nb = len(train_loader)
    nw = max(round(3 * nb), 700)

    LOGGER.info(f'num training imgs: {len(train_dataset)}, num val imgs: {len(val_dataset)}')

    eval_interval = hyp_train['eval_interval']
    best_f1 = best_epoch = -1
    best_val_loss = np.inf

    accumulation_steps = hyp_train['accumulation_steps']
    summary(model, (3, 640, 640), device=DEVICE)
    metric_cls = QuadMetric()
    post_process = SegDetectorRepresenter(thresh=0.5)
    best_f1 = -1
    for epoch in range(start_epoch, epochs):  # epoch ------------------------------------------------------------------
        model.train_db()
        pbar = enumerate(train_loader)
        pbar = tqdm(pbar, total=nb, bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}')  # progress bar
        m_loss = 0
        m_loss_s = 0
        m_loss_t = 0
        m_loss_b = 0
        for i, batchs in pbar:
            if (i+2) % 256 == 0:
                train_dataset.initialize()
                pbar.set_description(f' training size: {train_dataset.img_size}')
            # warm up
            if hyp_train['warm_up']:
                ni = i + nb * epoch
                if ni <= nw:
                    xi = [0, nw]  # x interp
                    for j, x in enumerate(optimizer.param_groups):
                        x['lr'] = np.interp(ni, xi, [hyp_train['warmup_bias_lr'] if j == 2 else 0.0, x['initial_lr'] * lf(epoch)])
                        if 'momentum' in x:
                            x['momentum'] = np.interp(ni, xi, [hyp_train['warmup_momentum'], hyp_train['momentum']])

            with amp.autocast():
                for key in batchs.keys():
                    batchs[key] = batchs[key].cuda()
                preds = model(batchs['imgs'])
                metric = criterion(preds, batchs, use_bce)
            loss = metric['loss'] / accumulation_steps
            scaler.scale(loss).backward()
            if (i+1) % accumulation_steps == 0:
                scaler.step(optimizer)
                scaler.update()
                optimizer.zero_grad()
            m_loss = (m_loss * i + metric['loss'].detach()) / (i + 1)
            m_loss_s = (m_loss_s * i + metric['loss_shrink_maps'].detach()) / (i + 1)
            m_loss_t = (m_loss_t * i + metric['loss_threshold_maps'].detach()) / (i + 1)
            m_loss_b = (m_loss_b * i + metric['loss_binary_maps'].detach()) / (i + 1)

        if i % eval_interval == 0:
            recall, precision, fmeasure = eval_model(model,  val_loader, post_process, metric_cls)
            log_dict = {}
            log_dict['train/lr'] = optimizer.param_groups[0]['lr']
            log_dict['train/loss'] = m_loss
            log_dict['train/loss_shrink'] = m_loss_s
            log_dict['train/loss_threshold'] = m_loss_t
            log_dict['train/loss_binary_maps'] = m_loss_b
            log_dict['eval/recall'] = recall
            log_dict['eval/precision'] = precision
            log_dict['eval/f1'] = fmeasure

            save_best = best_f1 < fmeasure
            if save_best:
                best_f1 = fmeasure
            last_ckpt = {'epoch': epoch,
                        'best_f1': best_f1,
                        'weights': model.dbnet.state_dict(),
                        'best_val_loss': best_val_loss,
                        'optimizer': optimizer.state_dict(),
                        'scheduler': scheduler.state_dict(),
                        'run_id': logger.wandb.id if logger.wandb is not None else None,
                        'date': datetime.now().isoformat(),
                        'hyp': hyp}
            torch.save(last_ckpt, 'data/db_last.ckpt')
            if save_best:
                shutil.copy('data/db_last.ckpt', 'data/db_best.ckpt')
            if logger is not None:
                logger.on_train_epoch_end(epoch, log_dict)
        scheduler.step()
        pbar.close()

if __name__ == '__main__':
    hyp_p = r'data/train_db_hyp.yaml'
    with open(hyp_p, 'r', encoding='utf8') as f:
        hyp = yaml.safe_load(f.read())

    # hyp['data']['train_img_dir'] = r'../datasets/pixanimegirls/processed'
    hyp['data']['train_img_dir'] = [r'../datasets/codat_manga_v3/images/train', r'../datasets/codat_manga_v3/images/val', r'../datasets/pixanimegirls/processed']
    hyp['data']['train_mask_dir'] = r'../datasets/TextLines'
    # hyp['data']['train_img_dir'] = r'data/dataset/db_sub'
    hyp['data']['val_img_dir'] = r'data/dataset/db_sub'
    hyp['data']['cache'] = False
    # hyp['data']['aug_param']['size_range'] = [-1]

    hyp['train']['lr0'] = 0.01
    hyp['train']['lrf'] = 0.002
    hyp['train']['weight_decay'] = 0.00002
    hyp['train']['batch_size'] = 4
    hyp['train']['epochs'] = 160
    # hyp['train']['optimizer'] = 'sgd'

    hyp['train']['loss'] = 'bce'
    hyp['logger']['type'] =  'wandb'

    # hyp['resume']['resume_training'] = True
    # hyp['resume']['ckpt'] = 'data/db_last_bk.ckpt'
    # hyp['model']['db_weights'] = r'data/db_last.ckpt'
    train(hyp)


#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import json
import time
import logging
from Queue import Queue
from threading import Thread
from datetime import datetime

from flask import Flask, request, jsonify

from proxytools import pool_utils
from proxytools.proxy_tester import check_proxies, get_local_ip
from proxytools.shared_utils import (get_country_from_ip,
                                     load_proxies,
                                     parse_bool)
from proxytools.pool_models import (init_database,
                                    db_updater,
                                    add_proxy_direct,
                                    working_proxy_count,
                                    get_all_proxies,
                                    get_working_proxies,
                                    get_filtered_proxies,
                                    purge_invalid_proxies,
                                    flaskDb)
from proxytools.proxy_scraper import (scrape_sockslist_net,
                                      scrape_vipsocks24_net,
                                      scrape_proxyserverlist24_top,
                                      scrape_socksproxylist24_top,
                                      scrape_premproxy_free)

# Reduce noise from logs
logging.getLogger("requests").setLevel(logging.CRITICAL)
logging.getLogger("urllib3").setLevel(logging.CRITICAL)
logging.basicConfig(
    format='%(asctime)s [%(threadName)15.15s][%(levelname)8.8s] %(message)s',
    level=logging.INFO)

# Silence some loggers
logging.getLogger('werkzeug').setLevel(logging.WARNING)

log = logging.getLogger('pgproxy')

stats_conditions = [
    ("Working", "working = 1 and banned = 0 and failed = 0 and invalid = 0"),
    ("Banned", "working = 0 and banned = 1 and failed = 0 and invalid = 0"),
    ("Failed", "working = 0 and banned = 0 and failed = 1 and invalid = 0"),
    ("Invalid", "invalid = 1")
]

# ---------------------------------------------------------------------------

app = Flask(__name__)

# ---------------------------------------------------------------------------
# REST Methods
# ---------------------------------------------------------------------------

@app.route('/', methods=['GET'])
def index():
    return "PGProxy running!"

@app.route('/proxy/status', methods=['GET'])
def get_status():

    lines = """"""
    lines += "DB Queue Size: {}

".format(db_updates_queue.qsize())

    lines += "
"

    lines += " 	Active	Frequency	"
    lines += "
Auto Refresh: 	"
    if args.auto_refresh:
        lines += "ON	{}	".format(args.auto_refresh_frequency)
    else:
        lines += "OFF	0	"
    lines += "
"
    lines += "
"
    lines += "
"

    lines += "
"
    for c in stats_conditions:
        lines += "{}	".format(c[0])
    lines += "TOTAL	"

    total_count = 0
    lines += "
"
    for c in stats_conditions:
        cursor = flaskDb.database.execute_sql('''
            select count(*) from proxypool
            where {}
        '''.format(c[1]))

        count = 0
        for row in cursor.fetchall():
            count = row[0]

        total_count += count
        lines += "{}	".format(count)

    lines += "{}	".format(total_count)
    lines += "
"
    lines += "
"

    return lines

@app.route('/proxy/status/full', methods=['GET'])
def get_status_full():

    lines = get_status()

    lines += "
"
    for c in stats_conditions:

        lines += "
{}
".format(c[0])

        cursor = flaskDb.database.execute_sql('''
            select url from proxypool
            where {}
            order by url desc
        '''.format(c[1]))

        lines += "<textarea name='proxies' rows='15' style='width:100%;'>"
        for row in cursor.fetchall():
            lines += "{}\r\n".format(row[0])
        lines += "</textarea>"

    return lines

@app.route('/proxy/request', methods=['GET'])
def get_proxy_batch():

    format = request.args.get('format')

    working = parse_bool(request.args.get('working'))
    banned = parse_bool(request.args.get('banned'))
    failed = parse_bool(request.args.get('failed'))
    invalid = parse_bool(request.args.get('invalid'))

    if not banned and not failed and not invalid:
        working = True

    proxies = get_filtered_proxies(working, banned, failed, invalid)

    if format == 'txt':
        page = ''
        for p in proxies:
            page += p.url + '\r\n'
        return page
    elif format == 'list':
        page = '['
        page += ', '.join(str(p.url) for p in proxies)
        page += ']'
        return page
    else:
        return jsonify([e.serialize() for e in proxies])

@app.route('/proxy/update', methods=['POST'])
def proxy_update():

    data = json.loads(request.data)
    if isinstance(data, list):
        for update in data:
            update['last_modified'] = datetime.now()
            db_updates_queue.put(update)
    else:
        data['last_modified'] = datetime.now()
        db_updates_queue.put(data)
    return 'ok'

@app.route('/proxy/add', methods=['GET', 'POST'])
def proxy_add():

    def load_proxies(s):
        proxies = []
        for line in s.splitlines():
            if line.strip() == "":
                continue
            proxies.append({
                'url': line
            })
        return proxies

    if request.method == 'POST':

        if 'proxies' in request.form:
            data = request.form
            proxies = load_proxies(data.get('proxies'))
        elif 'proxies' in request.args:
            data = request.args
            proxies = load_proxies(data.get('proxies'))
        else:
            data = request.get_json()
            if data:
                proxies = data.get('proxies', [])
            else:
                proxies = []

        if data is None or len(proxies) == 0:
            msg = "No proxies provided, or data not parseable"
            log.warning(msg)
            return msg, 503

        proxies = list(proxies)

        if isinstance(proxies, list):
            for proxy in proxies:
                update_proxy_status(proxy['url'], False, False, False)
            return "Successfully added {} proxies.".format(len(proxies))
        else:
            update_proxy_status(proxies['url'], False, False, False)
            return "Successfully added 1 proxy."
    else:
        page = """
Manually add new proxies to the DB.
                    These will not be validated until the next call for
                    auto-validate and only if they are within the
                    assigned limit.

                   <form method=POST>
                   <textarea name='proxies' rows='15'
                       placeholder='protocol://ip:port' style='width:100%;'>
                   </textarea>

<input type='submit' value='Submit'>

                   </form>
               """
        return page

# ---------------------------------------------------------------------------
# Run Methods
# ---------------------------------------------------------------------------

def run_server():
    app.run(threaded=True, host=args.host, port=args.port)

def import_proxies():

    proxies = set()

    log.info('Loading proxies from file: %s', args.proxy_file)
    proxylist = load_proxies(args.proxy_file, args.mode)

    if len(proxylist) > 0:
        proxies.update(proxylist)
    else:
        log.error('Proxy file was configured but no proxies were loaded.')
        sys.exit(1)

    proxies = list(proxies)

    check_and_import_proxies(proxies, True, False, True)

    log.info('Import completed, exiting.')

# ---------------------------------------------------------------------------
# Proxy Updating Methods
# ---------------------------------------------------------------------------

def check_and_import_proxies(proxies, import_all_types,
                             capture_failed_anon, ignore_limit):

    log.info('Found a total of %d proxies. Starting tests...', len(proxies))

    args.local_ip = None
    if not args.no_anonymous:
        local_ip = get_local_ip(args.proxy_judge)

        if not local_ip:
            log.error('Failed to identify local IP address.')
            sys.exit(1)

        log.info(
            'Using local IP address to test for anonymous: %s', local_ip)
        args.local_ip = local_ip

    if args.batch_size > 0:
        chunks = [proxies[x:x+args.batch_size]
                  for x in xrange(0, len(proxies), args.batch_size)]

        batch_count = 0
        working_count = 0
        num_chunks = len(chunks)

        log.info('Using {} chunks of {}'
                 .format(num_chunks, args.batch_size))

        for chunk in chunks:

            batch_count += 1
            log.info('Starting batch {} of {}. Working: {}'
                     .format(batch_count, num_chunks, working_count))

            working_proxies, banned_proxies, failed_proxies = check_proxies(
                args, chunk, capture_failed_anon)

            for proxy in working_proxies:

                # Now that the IP is validated, let's check the country.
                country = get_country_from_ip(args.geoip_url, proxy)
                if country in args.ignore_country:
                    log.info('Skipping proxy from country: {} for {}'
                             .format(proxy, country))
                    continue

                # If still ok, update it now (creates if new)
                update_proxy_status(proxy, True, False, False)

                # If we've reached the limit of working proxies, break out.
                working_count += 1
                if not ignore_limit and working_count >= args.limit:
                    break

            # If requested, import all other result types.
            if import_all_types:
                for proxy in banned_proxies:
                    update_proxy_status(proxy, False, True, False)

                for proxy in failed_proxies:
                    update_proxy_status(proxy, False, False, True)

            # Second check for limit to break out of chunk loop.
            if not ignore_limit and working_count >= args.limit:
                log.info(
                    'Stopping tests, Limit reached: {} >= {}'
                    .format(working_count, args.limit))
                break

    else:
        working_proxies, banned_proxies, failed_proxies = check_proxies(
            args, proxies, capture_failed_anon)

        for proxy in working_proxies:

            # Now that the IP is validated, let's check the country.
            country = get_country_from_ip(args.geoip_url, proxy)
            if country in args.ignore_country:
                log.info('Skipping proxy from country: {} for {}'
                         .format(proxy, country))
                continue

            # If still here, add the proxy.
            update_proxy_status(proxy, True, False, False)

        if import_all_types:
            for proxy in banned_proxies:
                update_proxy_status(proxy, False, True, False)
            for proxy in failed_proxies:
                update_proxy_status(proxy, False, False, True)

    # Now log a status of everything in DB.
    log_proxy_status()

def log_proxy_status():

    # Wait until the DB is finished updating.
    wait_db_updates()

    # Now pull out the status for everything.
    total_count = 0
    lines = "Status of {} proxies. "
    for c in stats_conditions:
        cursor = flaskDb.database.execute_sql('''
            select count(*) from proxypool
            where {}
        '''.format(c[1]))

        count = 0
        for row in cursor.fetchall():
            count = row[0]

        total_count += count
        lines += "{}: {}, ".format(c[0], count)

    lines = lines.rstrip(', ')
    log.info(lines.format(total_count))

def update_proxy_status(url, working, banned, failed):

    if args.proxy_file:
        add_proxy_direct(url, working, banned, failed)
        log.info(
            "add_proxy_direct for {}. working: {}, failed: {}, banned: {}"
            .format(url, working, failed, banned))
    else:
        proxy = {}
        proxy['url'] = url
        proxy['working'] = working
        proxy['banned'] = banned
        proxy['failed'] = failed
        proxy['last_modified'] = datetime.now()

        # Reset their retry counts back to 0 if currently working.
        if working:
            proxy['banned_retry_count'] = 0
            proxy['failed_retry_count'] = 0

        db_updates_queue.put(proxy)
        if args.log_proxy_updates:
            log.info(
                """DB update_proxy_status
                for {}. working: {}, failed: {}, banned: {}"""
                .format(url, working, failed, banned))

def wait_db_updates():

    # Give time for DB to update before continuing if needed.
    if db_updates_queue.qsize() > 0:
        log.info("Waiting on DB updates.")

        while db_updates_queue.qsize() > 0:
            time.sleep(1)

        log.info("DB updates completed.")

    # Sleep at least one second regardless, just in case.
    time.sleep(1)

def clone_proxy(dbproxy):

    proxy = {}
    proxy['url'] = dbproxy.url
    proxy['working'] = dbproxy.working
    proxy['invalid'] = dbproxy.invalid
    proxy['banned'] = dbproxy.banned
    proxy['banned_retry_count'] = dbproxy.banned_retry_count
    proxy['failed'] = dbproxy.failed
    proxy['failed_retry_count'] = dbproxy.failed_retry_count
    proxy['last_modified'] = dbproxy.last_modified

    return proxy

# ---------------------------------------------------------------------------
# Validation Methods
# ---------------------------------------------------------------------------

def validate_working_proxies():

    log.info('Validating existing working proxies...')

    urls = []
    proxies = get_working_proxies()
    for proxy in proxies:
        urls.append(proxy.url)

    if len(urls) > 0:
        check_and_import_proxies(urls, True, True, True)
    else:
        log.info('No proxies available for validation.')

def validate_all_proxies():

    log.info('Validating ALL existing proxies...')

    urls = []

    proxies = get_all_proxies()
    for proxy in proxies:

        update_proxy = clone_proxy(proxy)

        # If any banned/failed proxies have reached their retry limit,
        # then update them to have invalid flag and don't add to list.
        if proxy.banned == 1:

            if proxy.banned_retry_count < args.auto_refresh_banned_retries:
                # Update current banned retry count.
                # It will be set to 0 if found valid later.
                update_proxy['banned_retry_count'] += 1
                update_proxy['last_modified'] = datetime.now()
                db_updates_queue.put(update_proxy)
                if args.log_proxy_updates:
                    log.info(
                        'Incrementing banned retry for: %s',
                        update_proxy['url'])
            else:
                # Flag this proxy as invalid.
                update_proxy['working'] = False
                update_proxy['invalid'] = True
                update_proxy['last_modified'] = datetime.now()
                db_updates_queue.put(update_proxy)
                if args.log_proxy_updates:
                    log.info(
                        'Too many ban retries. Proxy is invalid: %s',
                        update_proxy['url'])

                # Don't continue any further with this proxy.
                continue

        if proxy.failed == 1:

            if proxy.failed_retry_count < args.auto_refresh_failed_retries:
                # Update current failed retry count.
                # It will be set to 0 if found valid later.
                update_proxy['failed_retry_count'] += 1
                update_proxy['last_modified'] = datetime.now()
                db_updates_queue.put(update_proxy)
                if args.log_proxy_updates:
                    log.info(
                        'Incrementing failed retry for: %s',
                        update_proxy['url'])
            else:
                # Flag this proxy as invalid.
                update_proxy['working'] = False
                update_proxy['invalid'] = True
                update_proxy['last_modified'] = datetime.now()
                db_updates_queue.put(update_proxy)
                if args.log_proxy_updates:
                    log.info(
                        'Too many failure retries. Proxy is invalid: %s',
                        update_proxy['url'])

                # Don't continue any further with this proxy.
                continue

        # If not invalid, add this proxy to the list for (re)validation.
        if not proxy.invalid:
            urls.append(proxy.url)

    if len(urls) > 0:
        check_and_import_proxies(urls, True, True, True)
    else:
        log.info('No proxies available for validation.')

# ---------------------------------------------------------------------------
# Scraping Methods
# ---------------------------------------------------------------------------

def scrape_http(proxies):
    log.info('Scraping HTTP proxies...')
    proxies.update(scrape_proxyserverlist24_top())
    proxies.update(scrape_premproxy_free(args.ignore_country))

def scrape_socks(proxies):
    log.info('Scraping SOCKS5 proxies...')
    proxies.update(scrape_sockslist_net(args.ignore_country))
    proxies.update(scrape_vipsocks24_net())
    proxies.update(scrape_socksproxylist24_top())

def scrape_proxies():

    # Check to see if we have at least the minimum proxies wanted.
    # Read DB for working proxies and compare against args.limit
    # If we already have limit, then don't scrape for more.
    working = working_proxy_count()
    if working < args.limit:
        log.info(
            'Scraping required. Working proxies NOT within limit. {} >= {}'
            .format(working, args.limit))
    else:
        log.info(
            'Scraping NOT required. Working proxies within limit. {} >= {}'
            .format(working, args.limit))
        return

    log.info('Scraping for new proxies...')

    proxies = set()

    if args.mode == 'http':
        scrape_http(proxies)
    elif args.mode == 'socks':
        scrape_socks(proxies)
    else:
        scrape_http(proxies)
        scrape_socks(proxies)

    proxies = list(proxies)

    # Validate all the obtained proxies.
    check_and_import_proxies(proxies, False, False, False)

# ---------------------------------------------------------------------------
# Daemon Threads
# ---------------------------------------------------------------------------

def auto_refresh_daemon():

    update_frequency_mins = args.auto_refresh_frequency
    refresh_time_sec = update_frequency_mins * 60

    # Sleep the initial amount of time before first run.
    time.sleep(refresh_time_sec)

    while True:
        log.info('Running auto refresh...')

        # Purge all invalid proxies if turned on
        if args.auto_refresh_purge_invalid:
            purge_invalid_proxies()

        # Validate all existing proxies in the DB.
        validate_all_proxies()

        # Check to see if we need to scrape for new proxies..
        scrape_proxies()

        # Wait x seconds before next refresh.
        log.info('Waiting %d minutes before next auto refresh.',
                 refresh_time_sec / 60)
        time.sleep(refresh_time_sec)

# ---------------------------------------------------------------------------
# Initialize
# ---------------------------------------------------------------------------

def initialize():

    t = Thread(target=db_updater, name='db-updater',
               args=(args, db_updates_queue, db))
    t.daemon = True
    t.start()

    # Check to see if we should be doing an import instead.
    if args.proxy_file:
        # Run the import.
        import_proxies()
    else:

        # Check to see if there should be any jobs run before startup.
        updated = False
        if args.initial_validate_working:
            validate_working_proxies()
            updated = True
        elif args.initial_validate_all:
            validate_all_proxies()
            updated = True

        if args.initial_scrape:
            scrape_proxies()
            updated = True

        # Log out current status.
        if updated is False:
            log_proxy_status()

        # Start the auto refresh thread if required.
        if args.auto_refresh:
            t = Thread(target=auto_refresh_daemon, name='auto-refresh')
            t.daemon = True
            t.start()
            log.info('Auto refresh is enabled.')
        else:
            log.info('Auto refresh is disabled.')

        # Start the server.
        log.info('Server initialized and running...')
        run_server()

# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

log.info("PGProxy starting up...")

# Obtain arguments from commandline or configuration file.
args = pool_utils.get_args()

# Initialize the database.
db = init_database(args, app)

# DB Updates queue.
db_updates_queue = Queue()

# Set the logging level.
log.setLevel(logging.INFO)
if args.verbose:
    log.setLevel(logging.DEBUG)
    log.debug('Running in verbose mode (-v).')

# Start up everything.
initialize()


from sqlalchemy import Column, Integer, String, TIMESTAMP

from corylus.database import Base

class Task(Base):
    __tablename__ = 'tasks'
    id = Column(Integer, primary_key=True)
    file_id = Column(String(36))
    name = Column(String(50))
    status = Column(Integer, default=0)
    target_url = Column(String(200))
    created_at = Column(Integer)

    def __init__(self, name=None, target_url=None, file_id=None, created_at=None, status=0):
        self.name = name
        self.file_id = file_id
        self.status = status
        self.target_url = target_url
        self.created_at = created_at

    def __repr__(self):
        return '<Task %r>' % (self.name or self.file_id)


#coding=utf-8
from django.shortcuts import render
from django.conf import settings
from django.core.cache import cache
from django.http import HttpResponse
import socket
from django.views.decorators.cache import cache_page
import requests
import json
from Scan.common import test
from Scan.collect.collect_urls import crawl
from Scan.Scan.Scan import Scan

# Create your views here.
def indexs(request):
    return render(request,'indexs.html')
def domains(request):
    return render(request,'domains.html')
def ips_scan(request):
    return render(request,'ips_scan.html')
def all_scan(request):
    return render(request,'all_scan.html')
def map(request):
    return render(request,'map.html')
def error(request):
    return render(request,'404.html')

#collect url函数
def collect_urls(request):
    if request.is_ajax() and request.method == 'GET':
        url=request.GET.get("url")
        #crawl(url).run()
    return HttpResponse("aaa")

#由网址得到城市名的函数
def get_city_from_domain(request):
    url=request.GET.get("url").lstrip("http://").lstrip("https://")
    ip = socket.gethostbyname(url)
    url = "http://ip.taobao.com/service/getIpInfo.php?ip=" + ip
    s = requests.get(url=url)
    print(s.text)
    city = json.loads(s.text)["data"]["city"]
    return HttpResponse(city)

#调试api函数
def test_api(request):
    if request.is_ajax() and request.method == 'GET':
        data=[]
        for key in request.GET:
            valuelist = request.GET.getlist(key)
            data.append(valuelist)
        print(data[0])
        #print(request.body)
        return HttpResponse("aaa")
        #return  render(request, "indexs.html", {"data":data[0]})

#cache的测试函数
def cache_test(request):
    if request.GET.get("url"):
        get_url=request.GET.get("url")
        print(get_url)
    if cache.has_key("url"):
        return HttpResponse(cache.get("url"))
    else:
        cache.set("url",get_url)
#cache html
def cache_html(request):
    return render(request,"cache_html.html")

#redis缓存的三个函数
def redis_caches():
    def _deco(func):
        def __deco(*args, **kwargs):
            obj, request = args
            if request.method == 'GET':  # 如果请求是已Get方式请求的，则调用get方式的方法
                data = request.GET.dict()
            elif request.method == 'POST':  # 如果是post则调用这个方式
                data = request.POST.dict()

            class_name = obj.__class__.__name__
            cache = read_from_cache()
            if cache:
                print('%s to  cache' % class_name)
                result = cache
            else:
                print('%s to  view' % class_name)
                result = obj.process(data, request)
                print('func  return  ', result)
                write_to_cache('test1_public', result)
            return HttpResponse(result)

        return __deco

    return _deco

# read cache user id
def read_from_cache(user_name='public'):
    """
    读取缓存
    :param user_name:
    :return:
    """
    key = 'test1_'+user_name
    value = cache.get(key)
    if value == None:
        data = None
    else:
        data = json.loads(value)
    return data

def write_to_cache(key, value):
    """
    写入缓存
    :param value:
    :param user_name:
    :return:
    """
    cache.set(key, json.dumps(value), settings.REDIS_TIMEOUT)

#Scan_Vuln
def Scan_Vuln(request):
    #return HttpResponse("aaaaaa")
    if request.GET.get("url"):
        url=request.GET.get("url")
        a=Scan(url)
        a.run()
    return HttpResponse("aaaa")


from __future__ import annotations

from typing import Callable

from alchemy.cards import CardModel
from alchemy.enums import Element
from alchemy.stack import CardStack

def default_search_algorithm(search_key, key):
    return search_key & key

class IngredientShelf:
    def __init__(
        self,
        search_algorithm: Callable[[int, int], bool] = default_search_algorithm,
    ) -> None:
        self._shelf: dict[int, CardStack] = dict()

        # NOTE: may be custom for Element.any in DLC's
        self.search_algorithm = search_algorithm

    def can_get_element(self, element: Element) -> bool:
        search_key = element.index
        for key in self._shelf.keys():
            if self.search_algorithm(search_key, key):
                return True
        else:
            return False

    def get_element(self, element: Element) -> CardModel:
        assert self.can_get_element(element)

        search_key = element.index
        for key, stack in self._shelf.items():
            if self.search_algorithm(search_key, key):
                card = stack.pop()
                if not stack.can_pop():
                    self._shelf.pop(key)
                return card

    def get_card(self, card: CardModel) -> None:
        assert self.can_get_element(card.drop_elements[0])

        search_key = sum([element.index for element in card.drop_elements])
        for key, stack in self._shelf.items():
            if self.search_algorithm(search_key, key):
                stack.pop()
                if not stack.can_pop():
                    self._shelf.pop(key)
                return

    def put(self, card: CardModel) -> None:
        key = sum([element.index for element in card.drop_elements])

        if self._shelf.get(key, None) is None:
            self._shelf[key] = CardStack()

        self._shelf[key].put(card)

    def put_down(self, card: CardModel) -> None:
        key = sum([element.index for element in card.drop_elements])

        if self._shelf.get(key, None) is None:
            self._shelf[key] = CardStack()

        self._shelf[key].put_down(card)

    def toplist(self) -> list[CardModel]:
        return [stack.top() for stack in self._shelf.values() if stack.can_pop()]


#! /usr/bin/env python

import collections
import socket
from threading import Lock
import struct
import json
import math

import rospy
import tf
from suii_msgs.srv import VisionScan, VisionScanResponse
from suii_msgs.msg import VisionScanObject
from std_srvs.srv import Empty
from geometry_msgs.msg import PoseStamped
from std_msgs.msg import Float32

VisionMsg = collections.namedtuple("VisionMsg", ["id", "code", "body"])
_ids = {
    "F20_20_B": 1,
    "F20_20_G": 2,
    "S40_40_B": 3,
    "S40_40_G": 4,
    "M20_100": 5,
    "M20": 6,
    "M30": 7,
    "R20": 8,
    "Bearing_Box": 9,
    "Bearing": 10,
    "Axis": 11,
    "Distance_Tube": 12,
    "Motor": 13,
    "F20_20_Mall": 101,
    "F40_40_Mall": 103,
    "R20_Mall": 108,
    "Bolt_Mall": 105,
    "M20_Mall": 106,
    "M30_Mall": 107,
    "Red_Container_Top": 14,
    "Red_Container_Front": 16,
    "Blue_Container_Top": 15,
    "Blue_Container_Front": 17
}

class VisionClient(object):
    def __init__(self):
        self._sock = None
        self._shock_lock = Lock()

    def connect(self, address):
        try:
            self._shock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._shock.setblocking(True)
            self._shock.connect((address, 9001))
            return True
        except Exception:
            self._shock_lock.acquire()
            return False

    def process_one(self, function_id, list_args):
        if not self._shock_lock.acquire(False):
            return None
        d = self._encode(function_id, list_args)
        h, b = self._sync_data(d)
        r = self._decode(h, b)
        self._shock_lock.release()
        return r

    def _encode(self, function_id, list_args):
        if len(list_args) > 0:
            db = {}
            for x in list_args:
                db[x[0]] = x[1]
            j = json.dumps(db)
        else:
            j = ""
        sig = struct.pack('BBI', 0x00, function_id, len(j))
        return sig + j

    def _decode(self, header, body):
        dj = json.loads(body)
        r = VisionMsg(id=header[1], code=header[0], body=dj)
        return r

    def _sync_data(self, data):
        try:
            self._shock.sendall(data)
            rh = self._shock.recv(8)
            h = struct.unpack("BBI", rh)
            fragment = ""
            while len(fragment) < h[2]:
                fragment += self._shock.recv(h[2])
            return (h, fragment)
        except Exception:
            pass

class Wrapper(object):
    def __init__(self):
        self.client = VisionClient()
        if not self.client.connect("127.0.0.1"):
            rospy.logerr("Can't connect to server")

        rospy.init_node("vision_control", anonymous=False)
        rospy.Service("/get_scan_all", VisionScan, handler=self.callback_all)
        self.tfb = tf.TransformBroadcaster()
        self.tfl = tf.TransformListener()
        rospy.spin()

    def callback_all(self, service_data):
        result = self.client.process_one(0x00, [])
        if (result == None) or (result.code == 0xFF):
            return []

        rr = []
        for o in result.body:
            if o['t'] == 1:
                tr = self.addTF(o['x'], o['y'], o['z'], o['n'] + "_" + str(len(rr)))
            else:
                tr = False
            rm = VisionScanObject()
            rm.id = _ids[o['n']]
            if tr:
                rm.link = o['n'] + "_" + str(len(rr))
                rm.sure = 1
            else:
                rm.link = ""
                rm.sure = 0
            rr.append(rm)
        return VisionScanResponse(rr)

    def addTF(self, x, y, z, name):
        try:
            self.tfl.waitForTransform("camera", "ur3/base", rospy.Time(0), rospy.Duration(10))
            p = PoseStamped()
            p.header.frame_id = "camera"
            p.pose.position.x = x
            p.pose.position.y = y

            if "Mall" in name:
                p.pose.position.z = -0.4
            else:
                p.pose.position.z = -0.415

            #Special angle changes
            z = -z + (math.pi/2 * (abs(z)/z))
            p.pose.orientation.z = tf.transformations.quaternion_from_euler(0, 0, z)[2]
            p.pose.orientation.w = tf.transformations.quaternion_from_euler(0, 0, z)[3]

            pt = self.tfl.transformPose("ur3/base", p)
            self.tfb.sendTransform(
                translation=[pt.pose.position.x, pt.pose.position.y, pt.pose.position.z],
                rotation=[p.pose.orientation.x, p.pose.orientation.y, p.pose.orientation.z, p.pose.orientation.w],
                time=rospy.Time.now(),
                child=name,
                parent="ur3/base"
            )
            return True
        except tf.Exception:
            return False

if __name__ == "__main__":
    Wrapper()


import torch
import torchvision
import torch.nn.functional as F
import torch.nn as nn

class Speech2Vid(nn.Module):
    def __init__(self):
        super().__init__()
        self.audio_encoder = nn.Sequential(
            nn.Conv2d(1, 64, 3, 1, 1),
            nn.ReLU(),
            nn.Conv2d(64, 128, 3, 1, 1),
            nn.ReLU(),
            nn.AvgPool2d(3, (1, 2), 1),
            nn.Conv2d(128, 256, 3, 1, 1),
            nn.ReLU(),
            nn.Conv2d(256, 256, 3, 1, 1),
            nn.ReLU(),
            nn.Conv2d(256, 512, 3, 1, 1),
            nn.ReLU(),
            nn.AvgPool2d(3, 2, 1),
            nn.Conv2d(512, 512, 3, 2, 1),
            nn.ReLU(),
            nn.Conv2d(512, 256, 3, 2, 1),
            nn.ReLU(),
            nn.Linear(3, 2),
            nn.ReLU()
        )

        self.face_conv1 = nn.Conv2d(3, 96, 7, 2, 1)
        self.face_pool1 = nn.AvgPool2d(3, 2, 1)
        self.face_conv2 = nn.Conv2d(96, 256, 3, 2, 1)
        self.face_pool2 = nn.AvgPool2d(3, 2, 1)
        self.face_conv3 = nn.Conv2d(256, 512, 3, 2, 1)
        self.face_conv4 = nn.Conv2d(512, 512, 3, 2, 1)
        self.face_conv5 = nn.Conv2d(512, 512, 3, 2, 1)
        self.face_fc6 = nn.Conv2d(512, 512, 1, 1)
        self.face_fc7 = nn.Conv2d(512, 256, 1, 1)

        self.decoder_fc1 = nn.Conv2d(512, 128, 1, 1)
        self.decoder_convt2 = nn.ConvTranspose2d(128, 512, 6, 2, 1)
        self.decoder_convt3 = nn.ConvTranspose2d(512, 256, 5, 2, 1)
        self.decoder_convt3b = nn.ConvTranspose2d(256, 256, 6, 2, 1)
        self.decoder_convt4 = nn.ConvTranspose2d(512, 96, 3, 2, 1)

        self.decoder_convt5 = nn.ConvTranspose2d(192, 96, 5, 2, 1)
        self.decoder_convt6 = nn.ConvTranspose2d(96, 64, 4, 2, 1)
        self.decoder_convt7 = nn.ConvTranspose2d(64, 3, 3, 1, 0)

    def forward(self, audio, fixed_face):
        audio_encoded = self.audio_encoder(audio)
        face_encode = torch.relu(self.face_conv1(fixed_face))
        face_encode_decon5 = torch.relu(self.face_pool1(face_encode))
        face_encode_decon4 = torch.relu(self.face_conv2(face_encode_decon5))
        face_encode = torch.relu(self.face_pool2(face_encode_decon4))
        face_encode = torch.relu(self.face_conv3(face_encode))
        face_encode = torch.relu(self.face_conv4(face_encode))
        face_encode = torch.relu(self.face_conv5(face_encode))
        face_encode = torch.relu(self.face_fc6(face_encode))
        face_encoded = torch.relu(self.face_fc7(face_encode))
        face_audio_cat = torch.relu(torch.cat([audio_encoded, face_encoded], 1))
        face_decode = torch.relu(self.decoder_fc1(face_audio_cat))
        face_decode = torch.relu(self.decoder_convt2(face_decode))
        face_decode = torch.relu(self.decoder_convt3(face_decode))
        face_decode = torch.relu(self.decoder_convt3b(face_decode))
        face_encode_decode_cat4 = torch.cat([face_decode, face_encode_decon4], 1)
        face_decode = torch.relu(self.decoder_convt4(face_encode_decode_cat4))
        face_encode_decode_cat5 = torch.cat([face_decode, face_encode_decon5], 1)
        face_decode = torch.relu(self.decoder_convt5(face_encode_decode_cat5))
        face_decode = torch.relu(self.decoder_convt6(face_decode))
        face_decoded = torch.tanh(self.decoder_convt7(face_decode))

        return face_decoded

class Deblur(nn.Module):
    def __init__(self):
        super().__init__()
        self.deblur = nn.Sequential(
            nn.Conv2d(3, 64, 3),
            nn.Conv2d(64, 64, 3),
            nn.Conv2d(64, 64, 3),
            nn.Conv2d(64, 64, 3),
            nn.Conv2d(64, 64, 3),
            nn.Conv2d(64, 64, 3),
            nn.Conv2d(64, 64, 3),
            nn.Conv2d(64, 64, 3),
            nn.Conv2d(64, 64, 3),
            nn.Conv2d(64, 3, 3)
        )

    def forward(self, inputs):
        residual = self.deblur(inputs)
        out = torch.add(inputs, residual)
        return out


import datetime
import locale
import urllib.request

def get_day(days_from_now=0):
    """
    Get name of the day
    :param days_from_now: How many days from current day; default 0 => today
    :return: Name of the day based on current date
    """
    # hard coded function to return days in english instead of
    # language within operating system's region settings
    locale.setlocale(locale.LC_ALL, 'en_US.utf8')

    today = datetime.date.today()
    day = today + datetime.timedelta(days=int(days_from_now))

    return day

def get_weather_icon(icon):
    """
    Get icon for belonging weather report
    :param icon: Pass icon name
    :return: File path for saved icon
    """
    url = f'http://openweathermap.org/img/wn/{icon}.png'
    filename = f'icons/{icon}.png'

    urllib.request.urlretrieve(url=url, filename=filename)

    return filename


'''
Created on 2015年5月17日

@author: fufei

'''
import http.client
from bs4 import BeautifulSoup

httpClient = None
colName = ['transactedDate','stockCode','stockName','transactedDetail','currentPrice',
           'transactedPrice','transactedVolume','turnover','buyer','seller']
bigDealInfo = {}

class BigDealRecord():

    def __init__(self):
        self.transactedDate = '' #交易日期
        self.stockCode = '' #股票代码
        self.stockName = '' #股票简称
        self.transactedDetail = '' #交易详情
        self.currentPrice = 0 #当前价格
        self.transactedPrice = 0 #成交价格
        self.transactedVolume = 0 #成交数量
        self.turnover = 0 #成交金额
        self.buyer = '' #买方营业部
        self.seller = '' #卖方营业部
        self.ratio = -1000000

    def calRatio(self):
        #（成交价格-当前价格）/当前价格
        if(self.currentPrice != 0 and self.transactedPrice !=0):
            self.ratio = (self.transactedPrice - self.currentPrice)/self.currentPrice
        else:
            self.ratio = -1000000 #如果没有当前价格和成交价格，ratio设为大负值

    def toString(self):
        s = '[股票代码：{0}，股票简称：{1}，当前价格：{2}，成交价格：{3}，成交量：{4}万股，成交金额：{5}万元,比率:{6}]'
        s = s.format(self.stockCode,self.stockName,self.currentPrice,self.transactedPrice,
                 self.transactedVolume,self.turnover,self.ratio)
        return s

def fetchPartOfDealingInfo(td,bdr):
    if(not td.text == '--'):
        bdr.transactedPrice = float(td.text)
    else:
        bdr.transactedPrice = 0;

    td = td.nextSibling.nextSibling
    if(not td.text == '--'):
        bdr.transactedVolume = float(td.text)
    else:
        bdr.transactedVolume = 0

    td = td.nextSibling.nextSibling
    if(not td.text == '--'):
        bdr.turnover = float(td.text)
    else:
        bdr.turnover = 0

    td = td.nextSibling.nextSibling
    bdr.buyer = td.text

    td = td.nextSibling.nextSibling
    bdr.seller = td.text

def fetchAllBigDealingInfo():
    try:
        httpClient = http.client.HTTPConnection("data.eastmoney.com",80,timeout=60)
        httpClient.request('GET', '/dzjy/default.html')#default.html
        res = httpClient.getresponse()

        if res.status == 200:
            soup = BeautifulSoup(res.readall())
            tbody = soup.find('th',text="交易日期").parent.parent
            #Get the big deal info
            transactedDate = ''
            stockCode = ''
            stockName = ''
            currentPrice = 0
            for tr in tbody.find_all('tr',class_="list_eve"):
                bdr = BigDealRecord()
                td = tr.td
                if(td.has_attr('valign')):#如果是显示交易时间的cell
                    transactedDate = td.text
#                     print(transactedDate)
                    bigDealInfo[transactedDate]=[]
                    td = td.nextSibling.nextSibling #两个nextSibling是为了跳过NavigableString对象

                if td.has_attr('rowspan'):
                    stockCode = td.text
                    bdr.stockCode = stockCode

                    td = td.nextSibling.nextSibling
                    stockName = td.text
                    bdr.stockName = stockName

                    td = td.nextSibling.nextSibling.nextSibling.nextSibling
                    if(not td.text == '--'):
                        currentPrice = float(td.text)
                    else:
                        currentPrice = 0;
                    bdr.currentPrice = currentPrice
                    td = td.nextSibling.nextSibling
                    fetchPartOfDealingInfo(td, bdr)
                else:
                    bdr.stockCode = stockCode
                    bdr.stockName = stockName
                    bdr.currentPrice = currentPrice
                    fetchPartOfDealingInfo(td, bdr)
                bdr.calRatio()
                bigDealInfo[transactedDate].append(bdr)
    except Exception as err:
        print (err)
    finally:
        if httpClient:
            httpClient.close()
    print(list(bigDealInfo.keys()))

def findTargetStock(ratio,volume):
    for transactedDate in sorted(bigDealInfo.keys(),reverse=True):
        print(transactedDate)
        for bigDealRecord in bigDealInfo[transactedDate]:
#             print(bigDealRecord.toString())
            if(bigDealRecord.ratio>=ratio and bigDealRecord.turnover>volume):
                print(bigDealRecord.toString())

fetchAllBigDealingInfo()
findTargetStock(0,1000)


import pandas as pd
import numpy as np
from bs4 import BeautifulSoup
from selenium import webdriver
import sys

driver = webdriver.PhantomJS(executable_path = r'C:\Users\benpa\Documents\chromedriver\phantomjs-2.1.1-windows\bin\phantomjs.exe')

links = []

years = [2013, 2014, 2015, 2016]

for year in years:

    url = 'https://www.baseball-reference.com/leagues/MLB/{}.shtml'.format(year)

    driver.get(url)
    soup = BeautifulSoup(driver.page_source, 'html5lib')

    table = soup.find('table', {'id':'teams_standard_batting'})
    table_body = table.find('tbody')

    for tr in table_body.find_all('tr'):
        team = tr.find('th')
        try:
            end_link = team.find('a')['href']
            team_link = 'https://www.baseball-reference.com' + end_link
            links.append(team_link)
        except:
            continue

data = []

for link in links:

    link_split = link.split('/')
    year = link_split[-1][:4]

    driver.get(link)
    soup = BeautifulSoup(driver.page_source, 'html5lib')

    batting_stats_table = soup.find('table', {'id':'team_batting'})
    table_body = batting_stats_table.find('tbody')

    for row in table_body.find_all('tr'):
        try:
            name = row.find('td', {'data-stat':'player'}).find('a').text
            name_split = name.split(' ')
            name_first = name_split[0]
            name_last = name_split[1]
        except:
            continue
        try:
            ab = row.find('td', {'data-stat':'AB'}).text
            if not ab:
                ab = '0.0'
        except:
            continue
        try:
            bb = row.find('td', {'data-stat':'BB'}).text
            if not bb:
                bb = '0.0'
        except:
            continue
        try:
            so = row.find('td', {'data-stat':'SO'}).text
            if not so:
                so = '0.0'
        except:
            continue
        try:
            obp = row.find('td', {'data-stat':'onbase_perc'}).text
            if not obp:
                obp = '0.0'
        except:
            continue
        try:
            slg = row.find('td', {'data-stat':'slugging_perc'}).text
            if not slg:
                slg = '0.0'
        except:
            continue

        row_data = [year, name_first, name_last, ab, bb, so, obp, slg]
        data.append(row_data)

df = pd.DataFrame(data, columns = ['year', 'name_first', 'name_last', 'ab', 'bb', 'so', 'obp', 'slg'])
df.to_csv('batting_data.csv', index = False)


from django.urls import path
from .views import (
    HomeView,
    ItemDetailView,
    # OrderSummaryView,
    Cart,
    DetailView,
    add_to_cart,
    remove_from_cart,
    remove_single_item_from_cart
)
from django.conf import settings
from django.contrib.staticfiles import views
from django.urls import re_path
from django.conf.urls.static import static
from django.conf.urls import include, url
from django.contrib import admin

app_name = 'proshop'

urlpatterns = [
    path('', HomeView.as_view()),
    path('product/<slug>/', ItemDetailView.as_view(), name='product'),
    # path('order-summary/', OrderSummaryView.as_view(), name='order-summary'),
    path('cart/', Cart.as_view(), name='cart'),
    path('add-to-cart/<slug>/', add_to_cart, name='add-to-cart'),
    path('remove-from-cart/<slug>/', remove_from_cart, name='remove-from-cart'),
    path('remove-item-from-cart/<slug>/', remove_single_item_from_cart,
         name='remove-single-item-from-cart'),
]

if settings.DEBUG:
    urlpatterns += [
        re_path(r'^static/(?P<path>.*)$', views.serve),
    ]
    urlpatterns += [
        url(r'^admin/', admin.site.urls),
    ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


from server import Server
from stream import *
from motion import *
import numpy as np
import time
import cv2

STREAM = CVStream            # Stream handler
WIDTH = 320                  # Frame width (None = max)
HEIGHT = 240                 # Frame height (None = max)
FPS = None                   # Stream FPS (None = max)
FLIP = None                  # Frame flip mode (0 = H | 1 = V | -1 = both | None = no flip)

MOTION = FakeMotion          # Motion handler
MOTION_PINS = {
    'ENABLE': 3,             # A4988 enable pin
    'STEP': 5,               # A4988 step pin
    'DIR': 7,                # A4988 dir pin
    'PWM': 11,               # Servo PWM pin
    'FIRE': 24,              # Fire pin
}
MOTION_PARAMS = {
    'SPEED': 30,             # Normal speed (steps per second)
    'SLOWSPEED': 5,          # Slow speed (steps per second)
    'REVERSE': False,        # Reverse direction
    'REVOLUTION': 320,       # Steps per full revolution
    'ROTATION_RANGE': 0.5,   # Rotation range limit (0 = no limits)
    'SLEEP_TIMEOUT': 5,      # Time to wait before disabling stepper
    'FIRE_PULSE': 0.1,       # Fire pulse length (seconds)
    'RECHARGE_TIME': 5,      # Aka minimal time between shots
    'MIN_PWM': 0,            # Min servo PWM
    'MAX_PWM': 50,           # Max servo PWM
}

ARMING_TIMEOUT = 5                         # Time to wait before arming
ARMING_TEXT_FONT = cv2.FONT_HERSHEY_PLAIN  # Arming status text param: font
ARMING_TEXT_FONT_SCALE = 1.5               # Arming status text param: scale
ARMING_TEXT_POS = (5, 20)                  # Arming status text param: position

FACE_HAAR_PATH = 'data/face_default.xml'   # Path to haar cascade file (for face detection)
FACE_SCALE_FACTOR = 1.1                    # Face detection param: scaleFactor
FACE_MIN_NEIGHBOURS = 5                    # Face detection param: minNeighbours
FACE_MIN_SIZE = (50, 50)                 # Face detection param: minSize
FACE_FLAGS = cv2.CASCADE_SCALE_IMAGE       # Face detection param: flags

CAMERA_MATRIX = np.array([                 # Camera matrix coefficients (calibration result)
    [544.70473098, 0.0, 177.46434358],
    [0.0, 547.27945612, 146.11552008],
    [0.0, 0.0, 1.0],
])
CAMERA_DISTORTION = np.array([             # Camera distortion coefficients (calibration result)
    5.37779339e-01,
    9.15783371e+00,
    4.65765217e-03,
    -1.17879940e-02,
    -1.22638201e+02,
])

MARKERS_DICT = cv2.aruco.DICT_6X6_50       # Aruco markers dict
MARKER_LENGTH = 33                         # Marker side length (mm)

LOGIC_HANDLERS = [
    'MarkerTrackingHandler',
    'FaceTrackingHandler',
    'BattleHandler',
]

class Core(BaseStream):
    def __init__(self):
        super().__init__()
        self.stream = STREAM(width=WIDTH, height=HEIGHT, fps=FPS)
        self.motion = MOTION(MOTION_PINS, MOTION_PARAMS)
        self.server = Server(self)
        self.event = self.stream.subscribe()

        self.handlers = {globals()[i].name: globals()[i] for i in LOGIC_HANDLERS}
        self.aruco_dict = cv2.aruco.Dictionary_get(MARKERS_DICT)
        self.face_cascade = cv2.CascadeClassifier(FACE_HAAR_PATH)
        self.draw_arming_text = lambda img, text, color: cv2.putText(
            img, text, ARMING_TEXT_POS, ARMING_TEXT_FONT, ARMING_TEXT_FONT_SCALE, color, 1
        )
        self.last_face_time = time.time()

        self.handler = None
        self.frame = None
        self.gray = None
        self.markers = []
        self.faces = []

    def set_handler(self, name):
        print('SWITCHING HANDLER TO:', name)
        if self.handler is not None: self.handler.stop()
        if name is None: self.handler = None; return
        self.handler = self.handlers[name](self)
        self.handler.start()

    def process_markers(self):
        corners, ids, _ = cv2.aruco.detectMarkers(self.gray, self.aruco_dict)
        self.markers = []
        if ids is None: return
        centers = [np.mean(c[0], axis=0).astype(int).tolist() for c in corners]
        self.markers = [{
            'id': int(id[0]),
            'corners': co.astype(int).tolist(),
            'center': ce
        } for co, id, ce in zip(corners, ids, centers)]
        # rvec, tvec, _ = cv2.aruco.estimatePoseSingleMarkers(corners, MARKER_LENGTH, CAMERA_MATRIX, CAMERA_DISTORTION)
        # for i in self.markers: i.update(rvec=rvec.tolist(), tvec=tvec.tolist())
        cv2.aruco.drawDetectedMarkers(self.frame, corners, borderColor=(0, 0, 255))

    def process_faces(self):
        faces = self.face_cascade.detectMultiScale(
            self.gray,
            scaleFactor=FACE_SCALE_FACTOR,
            minNeighbors=FACE_MIN_NEIGHBOURS,
            minSize=FACE_MIN_SIZE,
            flags=FACE_FLAGS
        )
        self.faces = [] if not len(faces) else faces.tolist()

        if self.faces:
            self.motion.armed = False
            self.last_face_time = time.time()
        elif time.time() - self.last_face_time >= ARMING_TIMEOUT:
            self.motion.armed = True

        for (x, y, w, h) in self.faces: cv2.rectangle(self.frame, (x, y), (x + w, y + h), (0, 255, 0), 1)
        if self.motion.armed: self.draw_arming_text(self.frame, 'ARMED', (0, 0, 255))
        elif self.faces: self.draw_arming_text(self.frame, 'DISARMED', (0, 255, 0))
        else: self.draw_arming_text(self.frame, 'ARMING', (0, 200, 255))

    def stop(self):
        self.stopped = True
        self.event.set()
        self.join()

    def run(self):
        self.stream.start()
        self.motion.start()
        self.server.start()

        while not self.stopped:
            self.event.wait()
            if self.stopped: break
            self.event.clear()

            frame = self.stream.frame
            if frame is None: continue
            if FLIP is not None: frame = cv2.flip(frame, FLIP)
            self.gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            self.frame = cv2.cvtColor(self.gray, cv2.COLOR_GRAY2BGR)

            self.process_markers()
            self.process_faces()
            self.notify()

        print('[CORE] STOPPING STREAM THREAD...')
        self.stream.stop()
        print('[CORE] STOPPING MOTION THREAD...')
        self.motion.stop()
        print('[CORE] STOPPING SERVER THREAD...')
        self.server.stop()
        print('[CORE] STOPPING HANDLER THREAD...')
        if self.handler: self.handler.stop()
        print('[CORE] TERMINATED')

class BaseHandler(Thread):
    name = 'Base Handler'

    def __init__(self, core):
        super().__init__()
        self.core = core
        self.motion = self.core.motion
        self.event = self.core.subscribe()
        self.stopped = False

    def stop(self):
        self.stopped = True
        self.event.set()
        self.join()

    def run(self):
        while not self.stopped:
            self.event.wait()
            if self.stopped: break
            self.event.clear()
            self.handle()

    def handle(self): pass

class MarkerTrackingHandler(BaseHandler):
    name = 'Marker Tracker'

    THRESHOLD = 10  # px
    RATIO = 15      # steps

    def handle(self):
        if not self.core.markers: return
        marker = self.core.markers[0]
        x = marker['center'][0]
        delta = x - WIDTH // 2
        if abs(delta) < self.THRESHOLD: return
        steps = (delta / (WIDTH / 2)) * self.RATIO
        self.motion.step(steps)

class FaceTrackingHandler(BaseHandler):
    name = 'Face Tracker'

    THRESHOLD = 10
    RATIO = 15

    def handle(self):
        if not self.core.faces: return
        face = self.core.faces[0]
        x = face[0] + face[2]
        delta = x - WIDTH // 2
        if abs(delta) < self.THRESHOLD: return
        steps = (delta / (WIDTH / 2)) * self.RATIO
        self.motion.step(steps)

class BattleHandler(BaseHandler):
    name = 'BATTLE HANDLER'

    OFFSET = 10
    THRESHOLD = 5
    CHARGE_TIME = 3

    def __init__(self, core):
        super().__init__(core)
        self.last_shot = 0
        self.eliminated = []
        self.direction = 1

    def handle(self):
        if self.core.markers:
            self.motion.slowmode = False
            markers = self.core.markers.copy()
            markers = list(filter(lambda x: x['id'] not in self.eliminated, markers))
            markers.sort(key=lambda x: x['id'])
            target = markers[0]
            delta = target['center'][0] - (WIDTH // 2 + self.OFFSET)
            now = time.time()
            if abs(delta) <= self.THRESHOLD:
                if now - self.last_shot < self.CHARGE_TIME: return
                self.last_shot = now
                self.eliminated.append(target['id'])
                print('FIRE!!!')
                # self.motion.fire()
                return
        else:
            self.motion.slowmode = True
            self.motion.rotate(self.direction)
            if self.motion.onborder: self.direction *= -1

if __name__ == '__main__':
    core = Core()
    core.start()
    input('[CORE] ENTER TO TERMINATE\n\n')
    core.stop()


import torch

def hamiltonian(x, v, model):
    energy = 0.5 * torch.pow(v, 2).sum(dim=1).sum(dim=1).sum(dim=1) + model(x).squeeze()
    return energy

def leapfrog_step(x, v, model, step_size, num_steps, label, sample=False):
    x = torch.log(x / (1 - x + 1e-10))
    x.requires_grad_(requires_grad=True)
    energy = model(torch.sigmoid(x))
    im_grad = torch.autograd.grad([energy.sum()], [x])[0]
    v = v - 0.5 * step_size * im_grad
    # x = x.detach()
    im_negs = []

    for i in range(num_steps):
        x.requires_grad_(requires_grad=True)
        energy = model(torch.sigmoid(x))

        im_grad = torch.autograd.grad([energy.sum()], [x])[0]
        v = v - step_size * im_grad
        x = x + step_size * v
        x = x.detach()
        v = v.detach()

        # if i % 10 == 0:
        #     print(i, hamiltonian(torch.sigmoid(x), v, model, label).mean(), torch.abs(im_grad).mean())

    x.requires_grad_(requires_grad=True)
    energy = model(torch.sigmoid(x))
    im_grad = torch.autograd.grad([energy.sum()], [x])[0]
    v = v - 0.5 * im_grad
    x = torch.sigmoid(x.detach())

    return x, v, im_grad

def gen_hmc_image(im_neg, step_size, temperature, model_fn, num_steps=10, sample=False):
    # energy = model.forward(im_neg, label)
    v = 0.1 * torch.randn_like(im_neg)

    im_neg_new, v_new, im_grad = leapfrog_step(im_neg, v, model_fn, step_size, num_steps, None)

    orig = hamiltonian(im_neg, v, model_fn)
    new = hamiltonian(im_neg_new, v_new, model_fn)

    mask = (torch.exp((orig - new)) > (torch.rand(new.size(0))).to(im_neg.device))
    im_neg_new[mask]= im_neg[mask]
    return im_neg_new


from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from functools import reduce
from time import sleep
from statistics import mean
from send import bet_siska

def creation():
    try:
        url = "https://www.handball24.com"
        browser = webdriver.Chrome()
        browser.get(url)
        resume = input("Select matches and press enter to continue(Add to favorite) ")
        browser.implicitly_wait(1)
        matches = browser.find_elements(By.CSS_SELECTOR,"[id^='g_7']")
        checklist = list()
        for i in matches:
            link = i.get_attribute("id")
            urls = f"https://www.handball24.com/match/{link[4:]}"
            checklist.append(urls)
    finally:
        browser.quit()
    return checklist

schedule = creation()

def check_link(url):
    print(url)

    caps = DesiredCapabilities().CHROME
    caps["pageLoadStrategy"] = "eager"
    options = webdriver.ChromeOptions()
    options.add_argument('headless')
    browser = webdriver.Chrome(desired_capabilities=caps, options=options)
    browser.get(url)
    browser.implicitly_wait(1)
    team_home = browser.find_elements(By.CSS_SELECTOR, "a.participant__participantName")[0].get_attribute(
        "href") + "results/"
    team_away = browser.find_elements(By.CSS_SELECTOR, "a.participant__participantName")[1].get_attribute(
        "href") + "results/"
    title = browser.find_element(By.CSS_SELECTOR, ".tournamentHeader__country").text
    print(title)

    def separator(matches):
        match_list = list()
        for i in matches:
            line = i.text
            # print(line)
            if "(" in line or "Awrd" in line or "Abn" in line or "WO" in line:
                continue
            if len([i for i in line.split() if i.isdigit()]) < 6:
                continue
            match_list.append(line.split())
        return match_list

    def get_data(browser, link):
        browser.get(link)
        dataset = browser.find_elements(By.CSS_SELECTOR, "[id^='g_7']")
        matches = separator(dataset)
        team = browser.find_element(By.CSS_SELECTOR, "div.heading__name").get_attribute("innerHTML")
        return matches, team

    def forming(browser, link1, link2):  # NEED ADD TYPE SPORT AND FIXABLE CSS SELECTOR
        match_list_home, team1 = get_data(browser, link1)
        match_list_away, team2 = get_data(browser, link2)
        return match_list_home, match_list_away, team1, team2

    games = forming(browser, team_home, team_away)

    team1_name = games[2].split()
    team2_name = games[3].split()

    def separation_home_away(team_, all_matches):
        home_matches = list()
        away_matches = list()
        waste = ["W", "U18", "U19", "U20", "U21", "U23"]  # WASTE - U20 and another juniors and woman champs//
        for i in waste:
            if i in team_:
                team_ = [j for j in team_ if j not in waste]
        print(team_)
        for k in all_matches:
            i = [j for j in k[:len(k) - 1] if j not in waste] + k[-1:]
            x = i.index(team_[len(team_) - 1])
            if i[x + 1].isdigit():
                away_matches.append(i)
            elif "(" in i[x + 1] and i[x + 2].isdigit():
                away_matches.append(i)
            else:
                home_matches.append(i)
        return home_matches, away_matches

    team1_home, team1_away = separation_home_away(team1_name, games[0])
    team2_home, team2_away = separation_home_away(team2_name, games[1])

    results_1 = games[0]
    results_2 = games[1]

    def get_scores(results):
        scorelines = []
        for match in results:
            if len([ i for i in match if i.isdigit() ]) < 6:
                continue
            if "AET" in match or 'Pen' in match:
                scoreline = match[-9:-1]
            else:
                scoreline = match[-7:-1]
            scorelines.append(list(map(int,scoreline)))
        return scorelines

    for i in get_scores(team1_home):
        print(i)

    def half_one(scores):
        dataset1, dataset2 = [], []

        for matches in scores:
            dataset1.append(matches[2])
            dataset2.append(matches[3])

        return dataset1, dataset2, list(map(sum,zip(dataset1,dataset2)))

    def half_two(scores):
        dataset1, dataset2 = [], []

        for matches in scores:
            dataset1.append(matches[4])
            dataset2.append(matches[5])

        return dataset1, dataset2, list(map(sum,zip(dataset1,dataset2)))

    ''' Team1 at home totals '''
    team1_scored_1h_home_ind = half_one(get_scores(team1_home))[0]
    team1_conceded_1h_home_ind = half_one(get_scores(team1_home))[1]
    team1_1h_home_total = half_one(get_scores(team1_home))[2]
    team1_scored_2h_home_ind = half_two(get_scores(team1_home))[0]
    team1_conceded_2h_home_ind = half_two(get_scores(team1_home))[1]
    team1_2h_home_total = half_two(get_scores(team1_home))[2]
    team1_scored_ind_home = list(map(sum,zip(team1_scored_1h_home_ind, team1_scored_2h_home_ind)))
    team1_conceded_ind_home = list(map(sum, zip(team1_conceded_1h_home_ind, team1_conceded_2h_home_ind)))
    team1_total_home = list(map(sum, zip(team1_scored_ind_home, team1_conceded_ind_home)))
    ''' Team1  away totals '''
    team1_scored_1h_away_ind = half_one(get_scores(team1_away))[1]
    team1_conceded_1h_away_ind = half_one(get_scores(team1_away))[0]
    team1_1h_away_total = half_one(get_scores(team1_away))[2]
    team1_scored_2h_away_ind = half_two(get_scores(team1_away))[1]
    team1_conceded_2h_away_ind = half_two(get_scores(team1_away))[0]
    team1_2h_away_total = half_two(get_scores(team1_away))[2]
    team1_scored_ind_away = list(map(sum,zip(team1_scored_1h_away_ind, team1_scored_2h_away_ind)))
    team1_conceded_ind_away = list(map(sum, zip(team1_conceded_1h_away_ind, team1_conceded_2h_away_ind)))
    ''' Team2 at home totals '''
    team2_scored_1h_home_ind = half_one(get_scores(team2_home))[0]
    team2_conceded_1h_home_ind = half_one(get_scores(team2_home))[1]
    team2_1h_home_total = half_one(get_scores(team2_home))[2]
    team2_scored_2h_home_ind = half_two(get_scores(team2_home))[0]
    team2_conceded_2h_home_ind = half_two(get_scores(team2_home))[1]
    team2_2h_home_total = half_two(get_scores(team2_home))[2]
    team2_scored_ind_home = list(map(sum,zip(team2_scored_1h_home_ind, team2_scored_2h_home_ind)))
    team2_conceded_ind_home = list(map(sum, zip(team2_conceded_1h_home_ind, team2_conceded_2h_home_ind)))
    ''' Team2  away totals '''
    team2_scored_1h_away_ind = half_one(get_scores(team2_away))[1]
    team2_conceded_1h_away_ind = half_one(get_scores(team2_away))[0]
    team2_1h_away_total = half_one(get_scores(team2_away))[2]
    team2_scored_2h_away_ind = half_two(get_scores(team2_away))[1]
    team2_conceded_2h_away_ind = half_two(get_scores(team2_away))[0]
    team2_2h_away_total = half_two(get_scores(team2_away))[2]
    team2_scored_ind_away = list(map(sum,zip(team2_scored_1h_away_ind, team2_scored_2h_away_ind)))
    team2_conceded_ind_away = list(map(sum, zip(team2_conceded_1h_away_ind, team2_conceded_2h_away_ind)))
    team2_total_away = list(map(sum, zip(team2_scored_ind_away, team2_conceded_ind_away)))

    def handicap(scores):
        dataset1_total, dataset2_total = [], []
        dataset1_1half, dataset2_1half = [], []
        dataset1_2half, dataset2_2half = [], []

        for match in scores:
            dataset1_total.append(match[2]+match[4] - match[3]-match[5])
            dataset2_total.append(match[3]+match[5] - match[2]-match[4])

            dataset1_1half.append(match[2] - match[3])
            dataset2_1half.append(match[3] - match[2])

            dataset1_2half.append(match[4] - match[5])
            dataset2_2half.append(match[5] - match[4])

        return dataset1_total, dataset2_total, dataset1_1half, dataset2_1half, dataset1_2half, dataset2_2half

    ''' Team1 handicaps ...'''
    team1_handicap_total_home = handicap(get_scores(team1_home))[0]
    team1_handicap_total_away = handicap(get_scores(team1_away))[1]
    team1_handicap_1half_home = handicap(get_scores(team1_home))[2]
    team1_handicap_1half_away = handicap(get_scores(team1_away))[3]
    team1_handicap_2half_home = handicap(get_scores(team1_home))[4]
    team1_handicap_2half_away = handicap(get_scores(team1_away))[5]
    ''' Team2 handicaps ...'''
    team2_handicap_total_home = handicap(get_scores(team2_home))[0]
    team2_handicap_total_away = handicap(get_scores(team2_away))[1]
    team2_handicap_1half_home = handicap(get_scores(team2_home))[2]
    team2_handicap_1half_away = handicap(get_scores(team2_away))[3]
    team2_handicap_2half_home = handicap(get_scores(team2_home))[4]
    team2_handicap_2half_away = handicap(get_scores(team2_away))[5]

    """  Mean values ... """

    """  Team1   """
    mean_t1_ind_scored_home = round(mean(team1_scored_ind_home), 1)
    mean_t1_ind_scored1H_home = round(mean(team1_scored_1h_home_ind), 1)
    mean_t1_ind_scored2H_home = round(mean(team1_scored_2h_home_ind), 1)
    mean_t1_ind_conceded_home = round(mean(team1_conceded_ind_home), 1)
    mean_t1_ind_conceded1H_home = round(mean(team1_conceded_1h_home_ind), 1)
    mean_t1_ind_conceded2H_home = round(mean(team1_conceded_2h_home_ind), 1)

    mean_t1_ind_scored_away = round(mean(team1_scored_ind_away), 1)
    mean_t1_ind_scored1H_away = round(mean(team1_scored_1h_away_ind), 1)
    mean_t1_ind_scored2H_away = round(mean(team1_scored_2h_away_ind), 1)
    mean_t1_ind_conceded_away = round(mean(team1_conceded_ind_away), 1)
    mean_t1_ind_conceded1H_away = round(mean(team1_conceded_1h_away_ind), 1)
    mean_t1_ind_conceded2H_away = round(mean(team1_conceded_2h_away_ind), 1)

    """  Team2  """
    mean_t2_ind_scored_home = round(mean(team2_scored_ind_home), 1)
    mean_t2_ind_scored1H_home = round(mean(team2_scored_1h_home_ind), 1)
    mean_t2_ind_scored2H_home = round(mean(team2_scored_2h_home_ind), 1)
    mean_t2_ind_conceded_home = round(mean(team2_conceded_ind_home), 1)
    mean_t2_ind_conceded1H_home = round(mean(team2_conceded_1h_home_ind), 1)
    mean_t2_ind_conceded2H_home = round(mean(team2_conceded_2h_home_ind), 1)

    mean_t2_ind_scored_away = round(mean(team2_scored_ind_away), 1)
    mean_t2_ind_scored1H_away = round(mean(team2_scored_1h_away_ind), 1)
    mean_t2_ind_scored2H_away = round(mean(team2_scored_2h_away_ind), 1)
    mean_t2_ind_conceded_away = round(mean(team2_conceded_ind_away), 1)
    mean_t2_ind_conceded1H_away = round(mean(team2_conceded_1h_away_ind), 1)
    mean_t2_ind_conceded2H_away = round(mean(team2_conceded_2h_away_ind), 1)

    def per_minute(data, minute):
        ''' for half minute -> 30
            for fulltime minute -> 60 '''

        return round(data/minute, 2)

    def noramal_value(*args):
        return round(sum(args) / 2, 1)

    '''   Aprox. values for main totol coef.
            to compare with...  '''

    match_mean_total = noramal_value(mean_t1_ind_scored_home, mean_t2_ind_conceded_away, mean_t1_ind_conceded_home, mean_t2_ind_scored_away)
    match_mean_1H = noramal_value(mean_t1_ind_scored1H_home, mean_t2_ind_conceded1H_away, mean_t1_ind_conceded1H_home, mean_t2_ind_scored1H_away)
    match_mean_2H = noramal_value(mean_t1_ind_scored2H_home, mean_t2_ind_conceded2H_away, mean_t1_ind_conceded2H_home,
                                  mean_t2_ind_scored2H_away)
    match_mean_t1_H1_ind = noramal_value(mean_t1_ind_scored1H_home, mean_t2_ind_conceded1H_away)
    match_mean_t2_H1_ind = noramal_value(mean_t1_ind_conceded1H_home, mean_t2_ind_scored1H_away)
    match_mean_t1_ind = noramal_value(mean_t1_ind_scored_home, mean_t2_ind_conceded_away)
    match_mean_t2_ind = noramal_value(mean_t1_ind_conceded_home, mean_t2_ind_scored_away)

    print()
    print('CALC TOTAL:: ', match_mean_total)
    print('CALC 1HALF:: ', match_mean_1H)
    print('CALC 1HALF:: ', match_mean_2H)
    print()
    print('CALC TEAM1 IND. TOTAL::', match_mean_t1_ind,f'each: {mean_t1_ind_scored_home} {mean_t2_ind_conceded_away}')
    print('CALC TEAM2 IND. TOTAL::', match_mean_t2_ind, f'each: {mean_t1_ind_conceded_home} {mean_t2_ind_scored_away}')

    def bet_string(list):
        part1 = sorted(list)[:3]
        part2 = sorted(list)[-3:]
        return f'{part1}<{round(mean(list), 1)}>{part2}'

    print('1 HALF:: ')
    print('1T IND 1H SC:',bet_string(team1_scored_1h_home_ind))
    print('2T IND 1H CD:',bet_string(team2_conceded_1h_away_ind))
    print('1T IND 1H Ha:',bet_string(team1_handicap_1half_home))
    print('2T IND 1H Ha:',bet_string(team2_handicap_1half_away))
    print('1T TOTAL  1H:',bet_string(team1_1h_home_total))
    print('2T TOTAL  1H:',bet_string(team2_1h_away_total))
    print('FULLTIME:: ')
    print('1T IND FT SC:',bet_string(team1_scored_ind_home))
    print('2T IND FT CD:',bet_string(team2_conceded_ind_away))
    print('1T IND FT Ha:',bet_string(team1_handicap_total_home))
    print('2T IND FT Ha:',bet_string(team2_handicap_total_away))
    print('1T TOTAL FT:',bet_string(team1_total_home))
    print('2T TOTAL FT:',bet_string(team2_total_away))

    bet  = (title,' '.join(map(str,team1_name)),' '.join(map(str,team2_name)),
            "1 HALF TOTAL:: ",
            '1T IND 1H SC:' + bet_string(team1_scored_1h_home_ind),
            '2T IND 1H CD:' + bet_string(team2_conceded_1h_away_ind),
            '2T IND 1H SC:' + bet_string(team2_scored_1h_away_ind),
            '1T IND 1H CD:' + bet_string(team1_conceded_1h_home_ind),
            "HANDICAP 1HALF",
            '1T IND 1H Ha:' + bet_string(team1_handicap_1half_home),
            '2T IND 1H Ha:' + bet_string(team2_handicap_1half_away),
            '1T TOTAL  1H:' + bet_string(team1_1h_home_total),
            '2T TOTAL  1H:' + bet_string(team2_1h_away_total),
            'FULLTIME:: ',
            '1T IND FT SC:' + bet_string(team1_scored_ind_home),
            '2T IND FT CD:' + bet_string(team2_conceded_ind_away),
            '2T IND FT SC:' + bet_string(team2_scored_ind_away),
            '1T IND FT CD:' + bet_string(team1_conceded_ind_home),
            "HANDICAP FULLTIME",
            '1T IND FT Ha:' + bet_string(team1_handicap_total_home) ,
            '2T IND FT Ha:' + bet_string(team2_handicap_total_away) ,
            '1T TOTAL  FT:' + bet_string(team1_total_home) ,
            '2T TOTAL  FT:' + bet_string(team2_total_away)

                )

    bet_siska(bet)

for i in schedule:
    try:
        check_link(i)
    except:
        continue


#import mpi4py
#mpi4py.profile("mpe")
from mpi4py import MPI

import unittest

import sys, os
sys.path.insert(0, os.path.dirname(__file__))
from seq import Seq
del sys.path[0]

def test():
    size = MPI.COMM_WORLD.Get_size()
    rank = MPI.COMM_WORLD.Get_rank()
    name = MPI.Get_processor_name()
    with Seq(MPI.COMM_WORLD, 1, 10):
        print(
            "Hello, World! I am process %d of %d on %s."
            % (rank, size, name))

if __name__ == "__main__":
    test()


import getpass
import typing

from typing import Any, Callable, Optional

from src.common.encoding   import b58decode
from src.common.exceptions import CriticalError
from src.common.misc       import get_terminal_width, terminal_width_check
from src.common.output     import clear_screen, m_print, print_on_previous_line, print_spacing
from src.common.statics    import (B58_LOCAL_KEY, B58_LOCAL_KEY_GUIDE, B58_PUBLIC_KEY, B58_PUBLIC_KEY_GUIDE,
                                   CURSOR_UP_ONE_LINE, ECDHE, ENCODED_B58_KDK_LENGTH, ENCODED_B58_PUB_KEY_LENGTH,
                                   NC_BYPASS_START, NC_BYPASS_STOP)

if typing.TYPE_CHECKING:
    from src.common.db_settings import Settings

Validator = Callable[..., str]

def ask_confirmation_code(source: str  # The system the confirmation code is displayed by
                          ) -> str:    # The confirmation code entered by the user
    """\
    Ask the user to input confirmation code from Source Computer to
    verify local key has been installed.
    """
    title       = f"Enter confirmation code (from {source}): "
    input_space = len(' ff ')

    upper_line = '┌' + (len(title) + input_space) * '─' + '┐'
    title_line = '│' +      title  + input_space  * ' ' + '│'
    lower_line = '└' + (len(title) + input_space) * '─' + '┘'

    terminal_w = get_terminal_width()
    upper_line = upper_line.center(terminal_w)
    title_line = title_line.center(terminal_w)
    lower_line = lower_line.center(terminal_w)

    terminal_width_check(len(upper_line))

    print(upper_line)
    print(title_line)
    print(lower_line)
    print(3 * CURSOR_UP_ONE_LINE)

    indent = title_line.find('│')
    return input(indent * ' ' + f'│ {title}')

def box_input(message:        str,                          # Input prompt message
              default:        str                 = '',     # Default return value
              head:           int                 = 0,      # Number of new lines to print before the input
              tail:           int                 = 1,      # Number of new lines to print after input
              expected_len:   int                 = 0,      # Expected length of the input
              key_type:       str                 = '',     # When specified, sets input width
              guide:          bool                = False,  # When True, prints the guide for key
              validator:      Optional[Validator] = None,   # Input validator function
              validator_args: Optional[Any]       = None    # Arguments required by the validator
              ) -> str:                                     # Input from user
    """Display boxed input prompt with a message."""
    print_spacing(head)

    terminal_width = get_terminal_width()

    if key_type:
        key_guide = {B58_LOCAL_KEY:  B58_LOCAL_KEY_GUIDE,
                     B58_PUBLIC_KEY: B58_PUBLIC_KEY_GUIDE}.get(key_type, '')
        if guide:
            inner_spc = len(key_guide) + 2
        else:
            inner_spc = ENCODED_B58_PUB_KEY_LENGTH if key_type == B58_PUBLIC_KEY else ENCODED_B58_KDK_LENGTH
            inner_spc += 2  # Spacing around input space
    else:
        key_guide = ''
        inner_spc = terminal_width - 2 if expected_len == 0 else expected_len + 2

    upper_line = '┌'  + inner_spc * '─'  +  '┐'
    guide_line = '│ ' + key_guide        + ' │'
    input_line = '│'  + inner_spc * ' '  +  '│'
    lower_line = '└'  + inner_spc * '─'  +  '┘'
    box_indent = (terminal_width - len(upper_line)) // 2 * ' '

    terminal_width_check(len(upper_line))

    print(box_indent + upper_line)
    if guide:
        print(box_indent + guide_line)
    print(box_indent + input_line)
    print(box_indent + lower_line)
    print((5 if guide else 4) * CURSOR_UP_ONE_LINE)
    print(box_indent + '┌─┤' + message + '├')
    if guide:
        print('')

    user_input = input(box_indent + '│ ')

    if user_input == '':
        print(2 * CURSOR_UP_ONE_LINE)
        print(box_indent + '│ ' + default)
        user_input = default

    if validator is not None:
        error_msg = validator(user_input, validator_args)
        if error_msg:
            m_print(error_msg, head=1)
            print_on_previous_line(reps=4, delay=1)
            return box_input(message, default, head, tail, expected_len, key_type, guide, validator, validator_args)

    print_spacing(tail)

    return user_input

def get_b58_key(key_type:      str,         # The type of Base58 key to be entered
                settings:      'Settings',  # Settings object
                short_address: str = ''     # The contact's short Onion address
                ) -> bytes:                 # The Base58 decoded key
    """Ask the user to input a Base58 encoded key."""
    if key_type == B58_PUBLIC_KEY:
        clear_screen()
        m_print(f"{ECDHE} key exchange", head=1, tail=1, bold=True)
        m_print("If needed, resend your public key to the contact by pressing <Enter>", tail=1)

        box_msg = f"Enter public key of {short_address} (from Relay)"
    elif key_type == B58_LOCAL_KEY:
        box_msg = "Enter local key decryption key (from Transmitter)"
    else:
        raise CriticalError("Invalid key type")

    while True:
        rx_pk = box_input(box_msg, key_type=key_type, guide=not (settings.local_testing_mode or settings.qubes))
        rx_pk = ''.join(rx_pk.split())

        if key_type == B58_PUBLIC_KEY and rx_pk == '':
            return rx_pk.encode()

        try:
            return b58decode(rx_pk, public_key=(key_type == B58_PUBLIC_KEY))
        except ValueError:
            m_print("Checksum error - Check that the entered key is correct.")
            print_on_previous_line(reps=(4 if settings.local_testing_mode else 5), delay=1)

            if key_type == B58_PUBLIC_KEY and len(rx_pk) == ENCODED_B58_PUB_KEY_LENGTH:
                raise ValueError(rx_pk)

def nc_bypass_msg(key: str, settings: 'Settings') -> None:
    """Print messages about bypassing Networked Computer.

    During ciphertext delivery of local key exchange, these bypass
    messages tell the user when to bypass and remove bypass of Networked
    Computer. Bypass of Networked Computer makes initial bootstrap more
    secure by denying remote attacker the access to the encrypted local
    key. Without the ciphertext, e.g. a visually collected local key
    decryption key is useless.
    """
    m = {NC_BYPASS_START: "Bypass the Networked Computer if needed. Press <Enter> to send local key.",
         NC_BYPASS_STOP:  "Remove bypass of the Networked Computer. Press <Enter> to continue."}

    if settings.nc_bypass_messages:
        m_print(m[key], manual_proceed=True, box=True, head=(1 if key == NC_BYPASS_STOP else 0))

def pwd_prompt(message: str,          # Prompt message
               repeat:  bool = False  # When True, prints corner chars for the second box
               ) -> str:              # Password from user
    """Prompt the user to enter a password.

    The getpass library ensures the password is not echoed on screen
    when it is typed.
    """
    l, r = ('├', '┤') if repeat else ('┌', '┐')

    terminal_w  = get_terminal_width()
    input_space = len(' c ')  # `c` is where the caret sits

    upper_line = ( l  + (len(message) + input_space) * '─' +  r ).center(terminal_w)
    title_line = ('│' +      message  + input_space  * ' ' + '│').center(terminal_w)
    lower_line = ('└' + (len(message) + input_space) * '─' + '┘').center(terminal_w)

    terminal_width_check(len(upper_line))

    print(upper_line)
    print(title_line)
    print(lower_line)
    print(3 * CURSOR_UP_ONE_LINE)

    indent     = title_line.find('│')
    user_input = getpass.getpass(indent * ' ' + f'│ {message}')

    return user_input

def yes(prompt: str,                    # Question to be asked
        abort:  Optional[bool] = None,  # Determines the return value of ^C and ^D
        head:   int = 0,                # Number of new lines to print before prompt
        tail:   int = 0                 # Number of new lines to print after prompt
        ) -> bool:                      # True/False depending on input
    """Prompt the user a question that is answered with yes/no."""
    print_spacing(head)

    prompt      = f"{prompt} (y/n): "
    input_space = len(' yes ')

    upper_line = '┌' + (len(prompt) + input_space) * '─' + '┐'
    title_line = '│' +      prompt  + input_space  * ' ' + '│'
    lower_line = '└' + (len(prompt) + input_space) * '─' + '┘'

    terminal_w = get_terminal_width()
    upper_line = upper_line.center(terminal_w)
    title_line = title_line.center(terminal_w)
    lower_line = lower_line.center(terminal_w)

    indent = title_line.find('│')

    terminal_width_check(len(upper_line))

    print(upper_line)
    while True:
        print(title_line)
        print(lower_line)
        print(3 * CURSOR_UP_ONE_LINE)

        try:
            user_input = input(indent * ' ' + f'│ {prompt}')
        except (EOFError, KeyboardInterrupt):
            if abort is None:
                raise
            print('')
            user_input = 'y' if abort else 'n'

        print_on_previous_line()

        if user_input == '':
            continue

        if user_input.lower() in ['y', 'yes']:
            print(indent * ' ' + f'│ {prompt}Yes │\n')
            print_spacing(tail)
            return True

        if user_input.lower() in ['n', 'no']:
            print(indent * ' ' + f'│ {prompt}No  │\n')
            print_spacing(tail)
            return False


import streamlit as st
from st_functions import st_button, load_css
from PIL import Image

st.set_page_config(
    page_title="Linkquarium",
    page_icon="./images/favicon.PNG",
)

load_css()

# st.write("[![Star](https://img.shields.io/github/stars/dev-rakeshshaw/links.svg?logo=github&style=social)](https://github.com/dev-rakeshshaw/Linkquarium)")

col1, col2, col3 = st.columns(3)
col1.image(Image.open('./images/logo1.png'), use_column_width='always')
col2.image(Image.open('./images/dp.png'))

st.header('Rakesh Shaw, Open-Source Enthusiast.')
st.info('  ðŸ‘‹Upcoming DevðŸ‘‹ | Success requires persistent practice, guided by patience.')

icon_size = 20

st_button('resume','https://drive.google.com/file/d/1aBuysfNWHlq4BqacD-DVKsfDqglh6Drl/view?usp=share_link', 'View my Resume', icon_size)
st_button('github', 'https://github.com/dev-rakeshshaw', 'Follow me on Github', icon_size)
st_button('twitter', 'https://twitter.com/dev_rakeshshaw', 'Follow me on Twitter', icon_size)
st_button('telegram', 'https://t.me/dev_rakeshshaw', 'Chat with me on Telegram', icon_size)
st_button('linkedin', 'https://www.linkedin.com/in/rakesh-shaw-937514263/', 'Connect with me on LinkedIn', icon_size)