import argparse
import json
import os
import os.path as osp
import pickle
import random
import sys
from argparse import Namespace

import numpy as np

import cv2
import scipy.misc
import torch
import torch.backends.cudnn as cudnn
import torch.nn as nn
import torch.optim as optim
from torch.utils import data

import scops_trainer
from dataset.dataset_factory import dataset_generator
from model.model_factory import model_generator

# solve potential deadlock https://github.com/pytorch/pytorch/issues/1355
cv2.setNumThreads(0)

EXP_NAME = 'SCOPS-Test'
MODEL = 'DeepLab'
BATCH_SIZE = 10
NUM_WORKERS = 4
DATASET = 'PASCAL'
DATA_DIRECTORY = ''
DATA_LIST_PATH = ''
INPUT_SIZE = '128,128'
LEARNING_RATE = 1e-5
MOMENTUM = 0.9
NUM_CLASSES = 6
NUM_STEPS = 2000
POWER = 0.9
RANDOM_SEED = 1234
RESTORE_FROM = None
SAVE_NUM_IMAGES = 2
SAVE_PRED_EVERY = 5000
SNAPSHOT_DIR = './snapshots/'
WEIGHT_DECAY = 0.0005
CLIP_GRAD_NORM = 5
VIS_TNTERVAL = 100

TPS_SIGMA = 0.01
RAND_SCALE_LOW = 0.7
RAND_SCALE_HIGH = 1.1

NUM_PARTS = 10

LAMBDA_CON = 1e-1
LAMBDA_EQV = 10.0
LAMBDA_SC = 0.1

def get_arguments():
    """Parse all the arguments provided from the CLI.

    Returns:
      A list of parsed arguments.
    """
    parser = argparse.ArgumentParser(
        description="SCOPS: Self-supervised Co-part Segmentation")

    # load args from json files
    parser.add_argument("-f", "--arg-file", type=str, default=None,
                        help="load args from json file")

    # Exp
    parser.add_argument("--exp-name", type=str, default=MODEL,
                        help="Experiment name. Default: Part-Test")
    # Model/Data description
    parser.add_argument("--model", type=str, default=MODEL,
                        help="available options : DeepLab/DeepLab_2branch")
    parser.add_argument("--num-classes", type=int, default=NUM_CLASSES,
                        help="Number of classes to predict (including background).")
    parser.add_argument("--num-workers", type=int, default=NUM_WORKERS,
                        help="number of workers for multithread dataloading.")
    parser.add_argument("--dataset", type=str, default=DATASET,
                        help="Dataset selection")
    parser.add_argument("--data-dir", type=str, default=DATA_DIRECTORY,
                        help="Path to the directory containing the PASCAL VOC dataset.")
    parser.add_argument("--data-list", type=str, default=DATA_LIST_PATH,
                        help="Path to the file listing the images in the dataset.")
    parser.add_argument("--input-size", type=str, default=INPUT_SIZE,
                        help="Comma-separated string with height and width of images.")
    parser.add_argument("--celeba-th", type=float, default=0.0,
                        help="iou_threshold for celebAWild")

    # Data Augmentation
    parser.add_argument("--random-mirror", action="store_true",
                        help="Whether to randomly mirror the inputs during the training.")
    parser.add_argument("--random-scale", action="store_true",
                        help="Whether to randomly scale the inputs during the training.")
    parser.add_argument("--random-seed", type=int, default=RANDOM_SEED,
                        help="Random seed to have reproducible results.")

    # Training hyper parameters
    parser.add_argument("--num-steps", type=int, default=NUM_STEPS,
                        help="Number of training steps.")
    parser.add_argument("--batch-size", type=int, default=BATCH_SIZE,
                        help="Number of images sent to the network in one step.")
    parser.add_argument("--learning-rate", type=float, default=LEARNING_RATE,
                        help="Base learning rate for training with polynomial decay.")
    parser.add_argument("--power", type=float, default=POWER,
                        help="Decay parameter to compute the learning rate.")
    parser.add_argument("--momentum", type=float, default=MOMENTUM,
                        help="Momentum component of the optimiser.")
    parser.add_argument("--weight-decay", type=float, default=WEIGHT_DECAY,
                        help="Regularisation parameter for L2-loss.")
    parser.add_argument("--clip-gradients", type=float, default=CLIP_GRAD_NORM,
                        help="Clip gradients norm. Default:5.")

    # constraints weighting
    parser.add_argument("--lambda-con", type=float, default=LAMBDA_CON,
                        help="weighting parameter for concentration")
    parser.add_argument("--lambda-eqv", type=float, default=LAMBDA_EQV,
                        help="weighting parameter for equivariance")
    parser.add_argument("--lambda-lmeqv", type=float, default=LAMBDA_EQV,
                        help="weighting parameter for equivariance")

    # Equivariance setting
    parser.add_argument("--tps-mode", type=str, default='affine',
                        help="tps mode: affine/projective")
    parser.add_argument("--tps-sigma", type=float, default=TPS_SIGMA,
                        help="peturbation of tps in equivariance loss")
    parser.add_argument("--eqv-random-mirror", action="store_true",
                        help="Whether to use random mirror in equvariance.")
    parser.add_argument("--eqv-border-padding", action="store_true",
                        help="Whether to use border padding in equvariance.")
    parser.add_argument("--random-scale-low", type=float, default=RAND_SCALE_LOW,
                        help="lower bound of random scaling.")
    parser.add_argument("--random-scale-high", type=float, default=RAND_SCALE_HIGH,
                        help="higher bound of random scaling.")

    # part training config
    parser.add_argument("--ignore-saliency-fg", action="store_true",
                        help="Whether to ignore saliency foreground and only enforce BG")
    parser.add_argument("--ignore-small-parts", action="store_true",
                        help="Whether to ignore small parts.")
    parser.add_argument("--center-crop", action="store_true",
                        help="Whether to crop center (MAFL).")
    parser.add_argument("--self-ref-coord", action="store_true",
                        help="Whether to use self-referenced centroid.")
    parser.add_argument("--kp-threshold", type=int, default=6,
                        help="maximum number of missing keypoints/landmarks")

    # Semantic Consistency config
    parser.add_argument("--num-parts", type=int, default=NUM_PARTS,
                        help="Number of parts")
    parser.add_argument("--lambda-sc", type=float, default=LAMBDA_SC,
                        help="weighting parameter for semantic consistency constraint")
    parser.add_argument("--learning-rate-w", type=float, default=1e-3,
                        help="learning rate for DFF basis")
    parser.add_argument("--ref-net", type=str, default='vgg19',
                        help="reference feature network. default: vgg19")
    parser.add_argument("--ref-layer", type=str, default='relu5_4',
                        help="default: vgg19")
    parser.add_argument("--ref-norm", action="store_true",
                        help="normalize reference feature map")
    parser.add_argument("--lambda-orthonamal", type=float, default=1e2,
                        help="weighting parameter for DFF orthonormal loss")

    parser.add_argument("--detach-k", action="store_true",
                        help="detach k")
    parser.add_argument("--no-sal-masking", action="store_true",
                        help="disable saliency constraint")

    parser.add_argument("--restore-part-basis", type=str, default='',
                        help="load part basis weights")

    # Save/visualization
    parser.add_argument("--restore-from", type=str, default=RESTORE_FROM,
                        help="Where restore model parameters from.")
    parser.add_argument("--save-pred-every", type=int, default=SAVE_PRED_EVERY,
                        help="Save summaries and checkpoint every often.")
    parser.add_argument("--snapshot-dir", type=str, default=SNAPSHOT_DIR,
                        help="Where to save snapshots of the model.")
    parser.add_argument("--vis-interval", type=int, default=VIS_TNTERVAL,
                        help="visualization interval.")
    parser.add_argument("--tb-dir", type=str, default='tb_logs',
                        help="tensorbard dir.")

    parser.add_argument("--gpu", type=int, default=0,
                        help="choose gpu device.")
    return parser.parse_args()

def main():

    args = get_arguments()
    args_dict = vars(args)

    if args.arg_file is not None:
        with open(args.arg_file, 'r') as f:
            arg_str = f.read()
            file_args = json.loads(arg_str)
            args_dict.update(file_args)
            args = Namespace(**args_dict)

    args_str = '{}'.format(json.dumps(vars(args), sort_keys=False, indent=4))
    print(args_str)

    if not os.path.exists(os.path.join(args.snapshot_dir, args.exp_name)):
        os.makedirs(os.path.join(args.snapshot_dir, args.exp_name))

    # save args to file
    with open(os.path.join(args.snapshot_dir, args.exp_name, 'exp_args.json'), 'w') as f:
        print(args_str, file=f)

    h, w = map(int, args.input_size.split(','))
    input_size = (h, w)
    args.input_size = input_size

    cudnn.enabled = True
    gpu = args.gpu

    # create network
    model = model_generator(args)
    model.train()
    model.cuda(args.gpu)
    cudnn.benchmark = True

    # Initialize SCOPS trainer
    trainer = scops_trainer.SCOPSTrainer(args, model)

    train_dataset = dataset_generator(args)
    trainloader = data.DataLoader(train_dataset,
                                  batch_size=args.batch_size,
                                  shuffle=True,
                                  num_workers=args.num_workers,
                                  pin_memory=True,
                                  drop_last=True)
    trainloader_iter = enumerate(trainloader)

    for i_iter in range(args.num_steps):

        try:
            _, batch = trainloader_iter.__next__()
        except:
            trainloader_iter = enumerate(trainloader)
            _, batch = trainloader_iter.__next__()

        trainer.step(batch, i_iter)

        if i_iter >= args.num_steps - 1:
            print('save model ...')
            torch.save(model.state_dict(), osp.join(args.snapshot_dir,
                                                    args.exp_name, 'model_' + str(args.num_steps) + '.pth'))
            break

        if i_iter % args.save_pred_every == 0 and i_iter != 0:
            print('taking snapshot ...')
            torch.save(model.state_dict(), osp.join(args.snapshot_dir,
                                                    args.exp_name, 'model_' + str(i_iter) + '.pth'))

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# author:wangtaihe
# datetime:2020/12/3 13:15
# software: PyCharm

import yaml, os, re
import logging
from logging.handlers import TimedRotatingFileHandler

from logging import handlers

logger = logging.getLogger()
# format = logging.Formatter('%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s')
# rotahandler = handlers.RotatingFileHandler(os.path.join('root’, 'logs/my'), maxBytes=1024 * 1024 * 100)
# rotahandler = handlers.TimedRotatingFileHandler(os.path.join('../logs.log'), when='D')
# rotahandler.setLevel(logging.DEBUG)
# rotahandler.setFormatter(format)

file_handler = TimedRotatingFileHandler(filename='../logs', when="MIDNIGHT", interval=1, backupCount=30)
# filename="mylog" suffix设置，会生成文件名为mylog.2020-02-25.log
file_handler.suffix = "%Y-%m-%d.log"
# extMatch是编译好正则表达式，用于匹配日志文件名后缀
# 需要注意的是suffix和extMatch一定要匹配的上，如果不匹配，过期日志不会被删除。
file_handler.extMatch = re.compile(r"^\d{4}-\d{2}-\d{2}.log$")
# 定义日志输出格式
file_handler.setFormatter(
    logging.Formatter(
        "[%(asctime)s] [%(process)d] [%(levelname)s] - %(module)s.%(funcName)s (%(filename)s:%(lineno)d) - %(message)s"
    )
)

console = logging.StreamHandler()
console.setLevel(logging.INFO)
console.setFormatter(format)

logger.addHandler(file_handler)
logger.addHandler(console)

def info(msg):
    logger.info(msg)

def get_test_data(test_data_path):
    case = []  # 存储测试用例名称
    data = []  # 存储请求对象
    expected = []  # 存储预期结果
    with open(test_data_path, "r", encoding='utf-8') as f:
        dat = yaml.load(f.read(), Loader=yaml.SafeLoader)
        test = dat['tests']
        for td in test:
            case.append(td.get('case', ''))
            data.append(td.get('data', {}))
            expected.append(td.get('expected', {}))
    parameters = zip(case, data, expected)
    return case, parameters

# cases, parameters = get_test_data("../dmp/data/test_report.yml")
# list_params = list(parameters)
# print(list_params)


# 公主（举）包
from bs4 import BeautifulSoup

def set_attr(options: dict, attr: list, value: any) -> str:
    i = 0
    for x in attr:
        if i == 0:
            exec_str = "options['%s']" % x
            i += 1
        else:
            exec_str += "['%s']" % x

    exec_str += " = %s" % value
    print(exec_str)
    exec(exec_str)
    # print(options)
    return exec_str

def load_html(file_path=None):
    html_temp = '<!DOCTYPE html>\
\
\
\
    <meta charset="utf-8" />\
    <meta http-equiv="X-UA-Compatible" content="IE=edge">\
    <meta name="viewport" content="width=device-width, initial-scale=1">\
    <title></title>\
\
    \
    \
\
\
\

\
\
\
\
'
    if file_path == None:
        return BeautifulSoup(html_temp, features="lxml")
    else:
        return BeautifulSoup(open(file_path),features="lxml")

def load_data(data) -> str:
    return str(data)

def load_data_from_file(path: str, args_name: list, split=","):
    pass

def js_init_echart(chart):
    global_args = chart.get_global_args()
    options = chart.get_options()
    i = 0
    for arg in global_args["init"]:
        if i == 0:
            init_js = "var %s = echarts.init(\n\
                %s=%s,   \n \
            " % (
                global_args["id"],
                arg,
                global_args["init"][arg] ,
            )
            i = 1
        else:

            init_js += "%s = %s,\n" % (
                arg,
                global_args["init"][arg]if global_args["init"][arg] != None else "null",
            )
    init_js += ");\n"
    print(init_js)
    return init_js

def js_init_chartOption(chart):
    global_args = chart.get_global_args()
    options = chart.get_options()
    options_js = "%s.setOption(%s)" % (
        global_args["id"],
        options
    )
    options_js +=";\n"
    return options_js

def echart2html(chart,log_path=None):
    # echart_container = '<div id="%s" style="float: left;">
' % chart.get_global_args()["id"]
    if log_path == None:
        soup = load_html()
    else:
        soup = load_html(file_path=log_path+"/html/main.html")
    echart_container = soup.new_tag("div", id=chart.get_global_args()["id"],style="float: left;")
    soup.div.append(echart_container)
    return soup.prettify()

# soup = load_html().prettify
# print(soup)


import re
import string
import collections
import math
import operator
import pickle
import jieba
from nltk import ngrams
from datetime import datetime, timedelta

# ---------------- Functions ----------------

# Given news, find out the keywords for them
# 	input:
# 		news_dataset,a dict with date as key and news array as value
# 		news_dataset["2016-09-01"]=["news01", "news02", "...", ......]
# 	output:
# 		keyword_array, an array with each dataset's keywords
def get_news_keyword(total_news_dataset, stock_data, input_firm, total_keyword_num):
	print("length of total_news_dataset: ",len(total_news_dataset))
	rising_news = []
	falling_news = []
	rising_news_keyword = []
	falling_news_keyword = []
	total_keywords = []
	for key in stock_data:
		current_date = key
		two_days_ago = compute_date(current_date,2)
		if total_news_dataset.__contains__(two_days_ago) == 1 and len(stock_data[current_date]) == 2:
			if stock_data[current_date][1] == 1:
				rising_news.extend(total_news_dataset[two_days_ago])

			elif stock_data[current_date][1] == 0:
				falling_news.extend(total_news_dataset[two_days_ago])
	rising_news_keyword = get_type_news_keyword(rising_news, total_keyword_num, "rising")

	# rising_keyword_filename = "./rising_keyword.p"
	# rising_keyword_fileObject = open(rising_keyword_filename,'rb')
	# rising_news_keyword = pickle.load(rising_keyword_fileObject)
	falling_news_keyword = get_type_news_keyword(falling_news, total_keyword_num, "falling")
	total_keywords = list(set(rising_news_keyword + falling_news_keyword))

	print("length of rising_news_keyword: ",len(rising_news_keyword))
	print("length of falling_news_keyword: ",len(falling_news_keyword))
	print("length of total_keywords: ",len(total_keywords))
	filename = "./total_keyword.p"
	fread = open(filename, "wb")
	pickle.dump(total_keywords, fread)
	fread.close()
	return total_keywords

# Given news, find out the keywords for them
# 	input:
# 		news_dataset,a dict with date as key and news array as value
# 		news_dataset["2016-09-01"]=["news01", "news02", "...", ......]
# 	output:
# 		keyword_array, an array with each dataset's keywords
def get_type_news_keyword(news_dataset, total_keyword_num, keyword_type):
	grams_tupples = {}
	grams = {}
	df_tupples = []
	tf_tupples = []
	final_keyword = []

	count = 1
	print(len(news_dataset))
	# for news in news_dataset:
	for news in range(0,len(news_dataset)):
		grams_tupples[news] = []
		temp_tf_tupples = []

		news_dataset[news] = re.sub("[A-Za-z0-9\[\`\~\。\，\「\」\！\：\!\@\#\$\^\&\*\(\)\=\|\{\}\'\:\;\'\,\[\]\.\<\>\/\?\~\！\@\#\\\&\*\%]", "", news_dataset[news])
		grams = jieba.cut(news_dataset[news], cut_all=False)
		for tupples in grams:
			if len(tupples) > 1 and len(tupples) <= 4:
				grams_tupples[news].append(tupples)
				temp_tf_tupples.append(tupples)
				df_tupples.append(tupples)

		# Get the terms with tf and df
		# for n in range(2,5):
		# 	grams = ngrams(news_dataset[news], n)
		# 	check_status = 1
		# 	for tupples in grams:
		# 		for word in tupples:
		# 			if word.isdigit() or isEnglish(word) or word in "[+——！：；，。？<>「」、~@#￥%……&*（）／]+":
		# 				check_status = 0
		# 		if(check_status):
		# 			grams_tupples[news].append(tupples)
		# 			temp_tf_tupples.append(tupples)
		# 			df_tupples.append(tupples)
		# 		check_status = 1

		# df means to collect the set of each terms in one news
		for tmp_tupples in set(temp_tf_tupples):
			tf_tupples.append(tmp_tupples)

		# Select terms every 5000 news
		if count%500 == 0:
			print("Start cleaning: ",count)
			temp_df_counter = collections.Counter(df_tupples)
			temp_tf_counter = collections.Counter(tf_tupples)
			# print("temp_df_counter"+str(len(temp_df_counter)))
			# print("temp_tf_counter"+str(len(temp_tf_counter)))
			temp_df_tupples_cleaner = []
			temp_tf_tupples_cleaner = []
			for collect in temp_tf_counter.most_common(1000):
				temp_df_tupples_cleaner += [collect[0] for i in range(0,temp_df_counter[collect[0]])]
				temp_tf_tupples_cleaner += [collect[0] for i in range(0,collect[1])]
			df_tupples = []
			tf_tupples = []
			df_tupples = temp_df_tupples_cleaner
			tf_tupples = temp_tf_tupples_cleaner
			print("Finish cleaning")
			# for collect in temp_df_counter.most_common()[:-5001:-1]:
			# 	df_tupples = [tupple for tupple in df_tupples if not collect[0]]
			# 	tf_tupples = [tupple for tupple in tf_tupples if not collect[0]]

		# Count for times it runs
		count += 1
		# if count == 50:
		# 	break

	# Select terms at the end, same thing as above "Select terms every 5000 news", better to make it into a function
	temp_df_counter = collections.Counter(df_tupples)
	temp_tf_counter = collections.Counter(tf_tupples)
	print("temp_df_counter"+str(len(temp_df_counter)))
	print("temp_tf_counter"+str(len(temp_tf_counter)))
	temp_df_tupples_cleaner = []
	temp_tf_tupples_cleaner = []
	for collect in temp_tf_counter.most_common(1000):
		temp_df_tupples_cleaner += [collect[0] for i in range(0,temp_df_counter[collect[0]])]
		temp_tf_tupples_cleaner += [collect[0] for i in range(0,collect[1])]
	df_tupples = temp_df_tupples_cleaner
	tf_tupples = temp_tf_tupples_cleaner

	# Print out the most common terms
	df_counter = collections.Counter(df_tupples)
	tf_counter = collections.Counter(tf_tupples)

	# Write into xls file
	merge_subterm(df_counter,tf_counter)
	final_keyword = get_final_key_word(df_counter,tf_counter,count-1,total_keyword_num,keyword_type)
	return final_keyword

def merge_subterm(df_counter,tf_counter):
	ngram_term=[[],[],[],[],[]]
	for term in df_counter:
		ngram_term[len(term)-2].append(term)

	for i in range(4):
		for subterm in ngram_term[i]:
			ckeck = False
			st = concat(subterm)
			for term in ngram_term[i+1]:
				t = concat(term)
				if st in t:
					#print(df_counter[subterm]/df_counter[term])
					if df_counter[subterm]/df_counter[term]<1.05 :
						ckeck = True
						break
			if ckeck:
				df_counter.pop(subterm)
				tf_counter.pop(subterm)
def concat(term):
	concatterm=''
	for i in term:
		concatterm+=i
	return concatterm

def get_final_key_word(df_counter,tf_counter,count,keyword_num, keyword_type):

	total_keyword_array = {}
	final_keyword_array = []
	for terms in df_counter:
		# print(terms)
		# print(tf_counter[terms])
		# print(array_merge(terms))
		total_keyword_array[array_merge(terms)] = (1+math.log10(tf_counter[terms])) * math.log10(count/df_counter[terms])

	for index in range(0,keyword_num):
		temp_max_key_word = ""
		temp_max_key_word = max(total_keyword_array.items(), key=operator.itemgetter(1))[0]
		final_keyword_array.append(temp_max_key_word)
		del total_keyword_array[temp_max_key_word]

	filename = "./"+keyword_type+"_keyword.p"
	fread = open(filename, "wb")
	pickle.dump(final_keyword_array, fread)
	fread.close()

	return final_keyword_array

def isEnglish(s):
    try:
        s.encode(encoding='utf-8').decode('ascii')
    except UnicodeDecodeError:
        return False
    else:
        return True

def array_merge(terms):
	result = ""
	for text in terms:
		result += text
	return result

# load news keyword
# 	output:	an array with each dataset's keywords
def load_news_keyword():
	total_data={}
	filename = "./total_keyword.p"
	fileObject = open(filename,'rb')
	total_data = pickle.load(fileObject)
	return total_data

# get date string in following format, 2016-09-01
def get_date(date_time):
	return datetime.strftime(date_time,"%Y-%m-%d")

# compute date n days ago
def compute_date(date,n):
	temp_date_time = datetime.strptime(date, "%Y-%m-%d")
	temp_date_time = temp_date_time - timedelta(days = n)
	return datetime.strftime(temp_date_time,"%Y-%m-%d")

#--------------------------------------------


# -*- coding: utf-8 -*-
"""
Created on Sat Dec 19 09:58:28 2020

@author: User
"""

import pygame , random, math

class Ball(pygame.sprite.Sprite):
    dx=0
    dy=0
    x=0
    y=0

    def __init__(self,speed,srx,sry,radium,color):
        pygame.sprite.Sprite.__init__(self)
        self.x=srx
        self.y=sry
        self.image= pygame.Surface([radium*2,radium*2])
        self.image.fill((255,255,25))
        pygame.draw.circle(self.image,color,(radium,radium),radium,0)
        self.rect=self.image.get_rect() #取得球體
        self.rect.center=(srx,sry)
        move=random.randint(78,87)
        radian=math.radians(move)
        self.dx=speed *math.cos(radian)
        self.dy=speed *math.cos(radian)

    def update(self):
        self.x+=self.dx #x=x+dx
        self.y+=self.dy
        self.rect.x=self.x
        self.rect.y=self.y
        if(self.rect.left<=0 or self.rect.right >=500):
            self.dx*=-1
        elif(self.rect.top<=3 or self.rect.bottom >=500):
            self.dy*=-1

    def collidebounce(self):
        self.dx*=-1

pygame.init()
size=(500,500)
screen=pygame.display.set_mode(size)
pygame.display.set_caption("Gary")

color1=(23,76,151)
color2=(35,62,187)

background= pygame.Surface(screen.get_size())
background=background.convert()
background.fill((255,255,255))

allsprite =pygame.sprite.Group()
ball_1=Ball(3, 100,100,70,color1)
allsprite.add(ball_1)

ball_2=Ball(3, 100,100,7,color2)
allsprite.add(ball_2)

running=True

while running:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running=False

    screen.blit(background,(0,0))
    ball_1.update()
    ball_2.update()

    allsprite.draw(screen)
    result = pygame.sprite.collide_rect(ball_1,ball_2)
    if result ==True:
        ball_1.collidebounce()
        ball_2.collidebounce()
    pygame.display.update()

pygame.quit


import praw
import os
import re
from collections import deque
import threading
import yaml
import time
import requests

#set globals

r=praw.Reddit('botbustNSFW_v2')

ME = r.user.me('botbust_NSFW_v2')
SUBREDDIT = r.subreddit('botbustNSFW_v2')
LOG_SUB = r.subreddit('botbustNSFW_v2_log')
LOG_TITLE = "/u/{0} banned from /r/{1}"

BAN_NOTE = "BotBusted!"
BAN_MESSAGE = ("Known comment bots are not welcome in /r/{0}"
               "\n\n*[I am a bot, and this action was performed automatically](/r/botbust/about/sticky). "
               "If you wish to dispute your status as a blacklisted comment bot, please "
               "[click here](https://www.reddit.com/message/compose?to=%2Fr%2FBotBust&subject=ban%20dispute:%20{1}).*"
                )
WELCOME_MESSAGE = ("Hello, moderators of /r/{0}"
                   "\n\nI am now helping keep your subreddit free of known worthless comment bots."
                   "\n\nFor more information, or to submit a worthless bot to my banlist, please visit /r/BotBust."
                   )
NSFW_MESSAGE = ("Hello, moderators of /r/{0}"
                "\n\nThank you for your interest in BotBust."
                "\n\nHowever, BotBust is not available for NSFW subreddits. Thank you for understanding.")

DEBARRED_INVITE=("Hello, moderators of /r/{}"
                  "\n\nThank you for using BotBust. However, BotBust will not be serving your subreddit at this time due to the moderator status of the following debarred account(s):"
                  "\n\n{}Debarred individuals are not eligible to benefit from the BotBust service.")

DEBARRED_SUBREDDIT=("/r/{} is banned from using BotBust. Thank you for your interest.")

ALREADY_BANNED='Thank you for submitting to BotBust. The account {} is already on my blacklist. To avoid cluttering the subreddit, this submission has been removed.'
MESSAGE_FORWARD='The following message was sent to {} by /u/{}:\n\n---\n\n{}'

class Bot():

    def __init__(self):

        self.moderated=[]
        self.friends=[]
        self.triggered=deque([],maxlen=100)
        self.checked=deque([],maxlen=200)

    def load_debarments(self):

        self.debarred=yaml.load(r.subreddit('captainmeta4bots').wiki['debar'].content_md)
        for category in self.debarred:
            for entry in category:
                entry=entry.lower()

    def check_all_debarments(self):
        self.load_debarments()

        for sub in r.user.moderator_subreddits(limit=None):
            leave=False
            for mod in sub.moderator():
                    if mod.name in self.debarred['users']:
                        leave=True
            if leave:
                sub.moderator.leave()
                print("Departed {}".format(sub.display_name))

    def debarment_monitoring(self):
        while True:
            print('Checking debarments...')
            self.check_all_debarments()
            print('Done checking debarments. Thread sleeping for 1 hr.')
            time.sleep(3600)

    def reload_moderated(self):

        self.moderated=[]

        for subreddit in r.user.moderator_subreddits(limit=None):
            self.moderated.append(subreddit.display_name)

    def reload_friends(self):
        self.friends=[]
        for user in r.user.friends():
            self.friends.append(user.name)

    def run(self):

        debar_thread=threading.Thread(target=self.debarment_monitoring)
        debar_thread.start()

        print('checking mod invites')
        self.check_for_mod_invites()
        print('loading moderated sub list')
        self.reload_moderated()
        print('loading "friend" list')
        self.reload_friends()

        print('beginning loop')
        while True:
            self.check_for_new_banned()
            self.patrol_r_friends()
            self.check_for_mod_invites()

            #self.update_status()

    def update_status(self):

        data={"refresh":"62129581-rY93dhrcdOV9VdcW-KF7Waeqxko"}
        requests.post("https://api.captainmeta4.me/reddit/botbust", data=data)

    def check_for_mod_invites(self):

        for message in r.inbox.unread(limit=None):
            message.mark_read()

            #filter out everything but PMs
            if not message.fullname.startswith('t4_'):
                continue

            #filter out everything not sent by a subreddit
            #forward to me
            if message.author is not None:
                #ignore pms from mod_mailer
                if message.author.name in ['mod_mailer']:
                    continue
                r.redditor('captainmeta4').message('Message Forward: {}'.format(message.subject),MESSAGE_FORWARD.format(message.dest,message.author.name,message.body))
                continue

            #filter out messages not associated with a subreddit
            if message.subreddit is None:
                continue

            try:
                if message.subreddit.over18:
                    message.reply(NSFW_MESSAGE.format(message.subreddit.display_name))
                    continue
            except:
                pass

            try:
                self.load_debarments()
                if message.subreddit.display_name in self.debarred['subreddits']:
                    message.reply(DEBARRED_SUBREDDIT.format(message.subreddit.display_name))
                    continue
            except:
                pass

            try:
                deny=False
                for mod in message.subreddit.moderator():
                    if mod.name in self.debarred['users']:
                        deny=True
                if deny:
                    message.reply("The invitation cannot be accepted with your current mod roster, which includes at least one account that is banned from using BotBust. Thank you for your interest.")
                    continue
            except:
                pass

            try:
                message.subreddit.mod.accept_invite()
                print('accepted invite to /r/'+message.subreddit.display_name)
                self.moderated.append(message.subreddit.display_name)
                self.log_add(message.subreddit.display_name)
            except:
                pass

    def patrol_r_friends(self):

        #load comments

        for comment in r.subreddit('friends').comments(limit=100):

            #All comments here will be by a banned account.
            #the only thing to check is that it's in a moderated subreddit

            if comment.subreddit.display_name not in self.moderated:
                continue

            if comment.banned_by:
                continue

            #ignore comments whose authors are botbustproof
            if comment.author_flair_css_class:
                if "botbustproof" in comment.author_flair_css_class:
                    continue

            #avoid duplicate work
            if comment.id in self.triggered:
                continue

            #at this point the ban needs to be issued

            # protect against insufficient mod perms by using try
            try:
                comment.mod.remove()
            except:
                pass

            print('banning /u/'+comment.author.name+' from /r/'+comment.subreddit.display_name)
            try:
                comment.subreddit.banned.add(comment.author, note="BotBusted!", ban_message = BAN_MESSAGE.format(comment.subreddit.display_name, comment.author.name))
            except:
                pass
            else:
                self.log_ban(comment)

            #avoid duplicate work
            self.triggered.append(comment.id)

    def log_add(self, name):
        url="https://reddit.com/r/{}".format(name)
        title="Accepted mod invite to /r/{}".format(name)
        LOG_SUB.submit(title, url=url).mod.approve()

    def log_ban(self, comment):

        user = comment.author.name
        sub = comment.subreddit.display_name
        url = "https://reddit.com{}".format(comment.permalink)

        title = LOG_TITLE.format(user, sub)

        LOG_SUB.submit(title, url=url).mod.approve()

    def check_for_new_banned(self):

        need_to_reload=False

        #process pending check-for-bans. search by flair
        for submission in SUBREDDIT.search("flair:checking", syntax="lucene", limit=None):

            #ignore any stray search results
            if submission.link_flair_css_class !="checkban":
                continue

            #ignore submissions that have been checked already
            if submission.id in self.checked:
                continue

            self.checked.append(submission.id)

            #get the username
            name = re.match("https?://(\w{1,3}\.)?reddit.com/u(ser)?/([A-Za-z0-9_-]+)/?", submission.url).group(3)

            print('checking ban state on /u/'+name)

            if name in self.friends:
                #flair the post, remove it, and add an explanatory comment
                submission.mod.flair(text="Already Banned!", css_class="banned")
                submission.mod.remove()
                if name.startswith('_'):
                    name='\\'+name
                submission.reply(ALREADY_BANNED.format(name)).mod.distinguish(sticky=True)
                print('/u/'+name+' is already banned')
            else:
                submission.mod.flair(text="For Review", css_class = "exclam")
                print('/u/'+name+' needs moderator review')

        #process pending bans/unbans. search by flair
        for submission in SUBREDDIT.search('flair:pending', syntax="lucene", limit=None):

            #ignore any stray search results
            if submission.link_flair_css_class == "banpending":

                need_to_reload = True

                #get the username
                name = re.match("https?://(\w{1,3}\.)?reddit.com/u(ser)?/([A-Za-z0-9_-]+)/?", submission.url).group(3)

                #friend the redditor
                r.redditor(name).friend()
                print(name+" banned!")
                submission.mod.flair(text="Banned!", css_class="banned")

            elif submission.link_flair_css_class == "unbanpending":

                need_to_reload = True

                #get the username
                name = re.match("https?://(\w{1,3}\.)?reddit.com/u(ser)?/([A-Za-z0-9_-]+)/?", submission.url).group(3)

                #unfriend the redditor
                r.redditor(name).unfriend()
                print(name+" unbanned")
                submission.mod.flair(text="Unbanned", css_class="unbanned")

        if need_to_reload:
            self.reload_friends()

if __name__=="__main__":
    b=Bot()
    while True:
        try:
            b.run()
        except Exception as e:
            print(str(e))


# Created for databucket2.py library
# Library rest_utils makes some ugly errors with Databucket :[
import json
import logging
import re
import socket
import traceback
from datetime import datetime
from http.client import responses

import requests
from requests import Response, Session

log = logging.getLogger()

class DatabucketRestClient:
    # running_inside_tn_network = True
    __enable_logging = False
    __log_headers = False
    __log_console = False
    __headers = None
    __proxy = None
    request = None
    response = None
    __last_request_time = datetime.now()

    def __init__(self, headers=None):
        if headers is None:
            headers = {}
        self.__headers = headers

    def set_enabled_log(self, enabled: bool):
        self.__enable_logging = enabled

    def set_enabled_log_headers(self, enabled: bool):
        self.__log_headers = enabled

    def set_enabled_log_console(self, enabled: bool):
        self.__log_console = enabled

    def set_headers(self, headers):
        self.__headers = headers

    def set_proxy(self, proxy):
        self.__proxy = proxy

    def __send_request(self, method: str, url: str, data: str = None, verify_response_status: bool = True, params: dict = None) -> Response:
        self.request = None
        self.response = None

        self.request = requests.Request(method, url, headers=self.__headers, data=data, params=params).prepare()

        session = Session()
        # if self.running_inside_tn_network and proxy:  # communication via proxy is required between machine inside TN network (GoCD agent) and Databucket REST service
        if self.__proxy:
            session.proxies = {'http': self.__proxy, 'https': self.__proxy}
        session.verify = False  # SSL Verification

        if 'OPL0000PC00361S' in socket.gethostname() and 'databucket' in url:  # Running from Pawel Kazmierski PC
            session.proxies = {'http': '126.179.0.206:9090', 'https': '126.179.0.206:9090'}
        session.verify = False  # SSL Verification

        if 'MatrixxPaymentCompanion' in url:
            assertion_prefix = 'MPC: '
        elif 'databucket' in url:
            assertion_prefix = 'Databucket: '
        else:
            assertion_prefix = 'MXX: '

        try:
            self.response = session.send(self.request, timeout=120)

        # http://docs.python-requests.org/en/master/_modules/requests/exceptions/
        except requests.exceptions.InvalidProxyURL:
            raise AssertionError(f'{assertion_prefix}InvaliProxydURL: {session.proxies["https"]} used when connecting to {url}.')
        except requests.exceptions.InvalidURL:
            raise AssertionError(f'{assertion_prefix}InvalidURL exception for {url}.')
        except requests.exceptions.ConnectTimeout:
            raise AssertionError(f'{assertion_prefix}ConnectTimeout ({120} s) exception for {url}.')
        except requests.exceptions.ReadTimeout:
            raise AssertionError(f'{assertion_prefix}Response ReadTimeout ({120} s) exception for {url}.')
        except requests.exceptions.ConnectionError as ex:
            raise AssertionError(f'{assertion_prefix}Failed to connect to {url}.\n{ex} ')

        # self.log_any_request_response()
        if self.__enable_logging or self.__log_console:
            try:
                self.log_any_request_response()
            except Exception:
                pass

        if verify_response_status:
            # http://docs.python-requests.org/en/master/api/
            if not self.response.ok:
                raise AssertionError(f'{assertion_prefix}Unexpected response status: {self.response.status_code} ({responses[self.response.status_code]}).\nRequested URL: {url}.')

        self.__last_request_time = datetime.now()

        return self.response

    def get(self, url: str, verify_response_status: bool = True) -> Response:
        return self.__send_request('GET', url=url, data=None, verify_response_status=verify_response_status)

    def post(self, url: str, data: str = None, verify_response_status: bool = True) -> Response:
        return self.__send_request('POST', url=url, data=data, verify_response_status=verify_response_status)

    def put(self, url: str, data: str = None, verify_response_status: bool = True) -> Response:
        return self.__send_request('PUT', url=url, data=data, verify_response_status=verify_response_status)

    def delete(self, url: str, data: str = None, verify_response_status: bool = True, params: dict = None) -> Response:
        return self.__send_request('DELETE', url=url, data=data, verify_response_status=verify_response_status, params=params)

    def log_any_request_response(self):
        if re.compile(".*databucket*").match(self.request.url):
            if self.request.method == "GET":
                self.log_get_databucket_request()
            elif self.request.method == "POST":
                if re.compile(".*reserve*").match(self.request.url):
                    self.log_reserve_databucket_request()
                elif re.compile(".*get*").match(self.request.url):
                    self.log_get_databucket_request()
                else:
                    self.log_post_databucket_request()
            elif self.request.method == "PUT":
                self.log_put_databucket_request()
            elif self.request.method == "DELETE":
                self.log_delete_databucket_request()
            else:
                self.log_databucket_request()
        else:
            self.log_unknown_request()

    def log_request(self):
        try:
            if self.__log_headers:
                headers = '\n'.join(f'{k}: {v}' for k, v in self.request.headers.items())
                headers = f'\n{headers}'
                report_text_info(name='Request headers', body=headers, enable_log=self.__enable_logging, log_console=self.__log_console)

            name = f'{self.request.method} {self.request.url}'
            report_json_info(name=name, body=self.request.body, enable_log=self.__enable_logging, log_console=self.__log_console)
        except:
            print(traceback.format_exc())

    def log_response(self):
        if self.__log_headers:
            headers = '\n'.join(f'{k}: {v}' for k, v in self.response.headers.items())
            headers = f'\n{headers}'
            report_text_info(name='Response headers', body=headers, enable_log=self.__enable_logging, log_console=self.__log_console)

        name = f'Response received in {"%.1f" % self.response.elapsed.total_seconds()} s with status: {self.response.status_code}'
        body = self.response.text
        report_json_info(name=name, body=body, enable_log=self.__enable_logging, log_console=self.__log_console)

    @allure.step('Get test data')
    def log_get_databucket_request(self):
        self.log_request_response()

    @allure.step('Insert test data')
    def log_post_databucket_request(self):
        self.log_request_response()

    @allure.step('Reserve test data')
    def log_reserve_databucket_request(self):
        self.log_request_response()

    @allure.step('Update test data')
    def log_put_databucket_request(self):
        self.log_request_response()

    @allure.step('Remove test data')
    def log_delete_databucket_request(self):
        self.log_request_response()

    @allure.step('Databucket request')
    def log_databucket_request(self):
        self.log_request_response()

    @allure.step('Unknown request')
    def log_unknown_request(self):
        self.log_request_response()

    def log_request_response(self):
        self.log_request()
        self.log_response()

def report_text_info(name: str, body: str = ' ', enable_log: bool = False, log_console: bool = False):
    log.info(f'\n{name}:\n{body}\n')

    if enable_log:
        try:
            allure.attach(body=body, name=name, attachment_type=allure.attachment_type.TEXT)
        except:
            pass

    if log_console:
        print(f'\n{name}:\n{body}\n')

def report_json_info(name: str, body: str, enable_log: bool = False, log_console: bool = False):
    log.info(f'\n{name}:\n{body}\n')

    if enable_log:
        try:
            if body:
                beauty_body = beautify_json(body)
                allure.attach(body=beauty_body, name=name, attachment_type=allure.attachment_type.JSON)
            else:
                allure.attach(body='-- no body --', name=name, attachment_type=allure.attachment_type.TEXT)
        except:
            pass

    if log_console:
        print(f'\n{name}:\n{body}\n')  # log into console

def beautify_json(text_or_dict):
    try:
        if isinstance(text_or_dict, dict):
            return json.dumps(text_or_dict, indent=2, sort_keys=False)
        else:
            return json.dumps(json.loads(text_or_dict), indent=2, sort_keys=False)
    except Exception as e:
        raise AssertionError(f'Bad JSON string:\n{text_or_dict}\n{e.args}')


from argparse import ArgumentParser

import obspy
from mqs_reports.catalog import Catalog
from mqs_reports.utils import autocorrelation
from obspy import UTCDateTime as utct

def define_arguments():
    helptext = 'Create Noise time evolution vs event amplitde overview plot'
    parser = ArgumentParser(description=helptext)

    helptext = 'Input QuakeML BED file'
    parser.add_argument('input_quakeml', help=helptext)

    helptext = 'Input manual distance file'
    parser.add_argument('input_dist', help=helptext)

    helptext = 'Inventory file'
    parser.add_argument('inventory', help=helptext)

    helptext = 'Path to SC3DIR'
    parser.add_argument('sc3_dir', help=helptext)

    helptext = 'Location qualities (one or more)'
    parser.add_argument('-q', '--quality', help=helptext,
                        nargs='+', default=('A', 'B'))

    helptext = 'Event types'
    parser.add_argument('-t', '--types', help=helptext,
                        default='all')
    return parser.parse_args()

args = define_arguments()

inv = obspy.read_inventory(args.inventory)
cat = Catalog(fnam_quakeml=args.input_quakeml,
              type_select=args.types, quality=args.quality)
print(cat)
cat = cat.select(name=['S0260a', 'S0218a', 'S0264e', 'S0289a', 'S0308a',
                       'S0311a', 'S0128a'])
cat.read_waveforms(inv=inv, sc3dir=args.sc3_dir)
for event in cat:
    fig = autocorrelation(st=event.waveforms_VBB.copy(),
                          starttime=utct(event.picks['Pg']) - 10.,
                          endtime=utct(event.picks['Pg']) + 110.)
    fig.savefig('AC_%s_P.png' % event.name, dpi=200)
    fig = autocorrelation(st=event.waveforms_VBB.copy(),
                          starttime=utct(event.picks['Sg']) - 10.,
                          endtime=utct(event.picks['Sg']) + 230.)
    fig.savefig('AC_%s_S.png' % event.name, dpi=200)


import os
import pprint
import re
from ast import literal_eval

from colorama import Back, Fore
from easydict import EasyDict as edict

from dl_lib.utils.config_helper import (_assert_with_logging,
                                        _check_and_coerce_cfg_value_type,
                                        diff_dict, find_key, highlight, update)

_config_dict = dict(
    MODEL=dict(
        DEVICE="cuda",
        WEIGHTS="",
        PIXEL_MEAN=[103.530, 116.280, 123.675],  # mean value from ImageNet
        # When using pre-trained models in Detectron1 or any MSRA models,
        # std has been absorbed into its conv1 weights, so the std needs to be
        # set 1. Otherwise, you can use [57.375, 57.120, 58.395] (ImageNet std)
        PIXEL_STD=[1.0, 1.0, 1.0],
    ),
    INPUT=dict(
        AUG=dict(
            TRAIN_PIPELINES=[],
            TEST_PIPELINES=[],
        ),
        CROP=dict(ENABLED=False, TYPE="relative_range", SIZE=[0.9, 0.9],),
        FORMAT="BGR",
    ),
    DATASETS=dict(
        TRAIN=(),
        TEST=(),
    ),
    DATALOADER=dict(
        NUM_WORKERS=2,
        ASPECT_RATIO_GROUPING=True,
        SAMPLER_TRAIN="TrainingSampler",
        REPEAT_THRESHOLD=0.0,
        FILTER_EMPTY_ANNOTATIONS=True,
    ),
    SOLVER=dict(
        LR_SCHEDULER=dict(
            NAME="WarmupMultiStepLR",
            MAX_ITER=40000,
            STEPS=(30000,),
            WARMUP_FACTOR=1.0 / 1000,
            WARMUP_ITERS=1000,
            WARMUP_METHOD="linear",
            GAMMA=0.1,
        ),
        OPTIMIZER=dict(
            NAME="SGD",
            BASE_LR=0.001,
            BIAS_LR_FACTOR=1.0,
            WEIGHT_DECAY=0.0001,
            WEIGHT_DECAY_NORM=0.0,
            WEIGHT_DECAY_BIAS=0.0001,
            SUBDIVISION=1,
            MOMENTUM=0.9,
        ),
        CHECKPOINT_PERIOD=5000,
        IMS_PER_BATCH=16,
    ),
    TEST=dict(
        EXPECTED_RESULTS=[],
        AUG=dict(
            ENABLED=False,
        ),
        PRECISE_BN=dict(ENABLED=False, NUM_ITER=200),
    ),
    OUTPUT_DIR="./output",
    SEED=-1,
    CUDNN_BENCHMARK=False,
    GLOBAL=dict(
        DUMP_TRAIN=True,
        DUMP_TEST=False,
    ),
)

class BaseConfig(object):
    def __init__(self):
        self._config_dict = {}
        self._register_configuration(_config_dict)

    def _register_configuration(self, config):
        """
        Register all key and values of config as BaseConfig's attributes.

        Args:
            config (dict): custom config dict
        """
        self._config_dict = update(self._config_dict, config)
        for k, v in self._config_dict.items():
            if hasattr(self, k):
                if isinstance(v, dict):
                    setattr(self, k, update(getattr(self, k), v))
                else:
                    setattr(self, k, v)
            elif isinstance(v, dict):
                setattr(self, k, edict(v))
            else:
                setattr(self, k, v)

    def merge_from_list(self, cfg_list):
        """
        Merge config (keys, values) in a list (e.g., from command line) into
        this CfgNode.

        Args:
            cfg_list (list): cfg_list must be divided exactly.
            For example, `cfg_list = ['FOO.BAR', 0.5]`.
        """
        _assert_with_logging(
            len(cfg_list) % 2 == 0,
            "Override list has odd length: {}; it must be a list of pairs".format(
                cfg_list
            ),
        )
        # root = self
        for full_key, v in zip(cfg_list[0::2], cfg_list[1::2]):
            key_list = full_key.split(".")
            d = self
            for subkey in key_list[:-1]:
                _assert_with_logging(
                    hasattr(d, subkey), "Non-existent key: {}".format(full_key)
                )
                d = getattr(d, subkey)
            subkey = key_list[-1]
            _assert_with_logging(
                hasattr(d, subkey), "Non-existent key: {}".format(full_key)
            )
            value = self._decode_cfg_value(v)
            value = _check_and_coerce_cfg_value_type(
                value, getattr(d, subkey), subkey, full_key
            )
            setattr(d, subkey, value)

    def link_log(self, link_name="log"):
        """
        create a softlink to output dir.

        Args:
            link_name(str): name of softlink
        """
        if os.path.islink(link_name) and os.readlink(link_name) != self.OUTPUT_DIR:
            os.system("rm " + link_name)
        if not os.path.exists(link_name):
            cmd = "ln -s {} {}".format(self.OUTPUT_DIR, link_name)
            os.system(cmd)

    @classmethod
    def _decode_cfg_value(cls, value):
        """
        Decodes a raw config value (e.g., from a yaml config files or command
        line argument) into a Python object.
        If the value is a dict, it will be interpreted as a new CfgNode.
        If the value is a str, it will be evaluated as literals.
        Otherwise it is returned as-is.

        Args:
            value (dict or str): value to be decoded
        """
        # Configs parsed from raw yaml will contain dictionary keys that need to be
        # converted to CfgNode objects
        if isinstance(value, dict):
            return cls(value)
        # All remaining processing is only applied to strings
        if not isinstance(value, str):
            return value
        # Try to interpret `value` as a:
        #   string, number, tuple, list, dict, boolean, or None
        try:
            value = literal_eval(value)
        # The following two excepts allow v to pass through when it represents a
        # string.
        #
        # Longer explanation:
        # The type of v is always a string (before calling literal_eval), but
        # sometimes it *represents* a string and other times a data structure, like
        # a list. In the case that v represents a string, what we got back from the
        # yaml parser is 'foo' *without quotes* (so, not '"foo"'). literal_eval is
        # ok with '"foo"', but will raise a ValueError if given 'foo'. In other
        # cases, like paths (v = 'foo/bar' and not v = '"foo/bar"'), literal_eval
        # will raise a SyntaxError.
        except ValueError:
            pass
        except SyntaxError:
            pass
        return value

    def _get_param_list(self) -> list:
        """
        get parameter(attribute) list of current config object

        Returns:
            list: attribute list
        """
        param_list = [
            name
            for name in self.__dir__()
            if not name.startswith("_") and not callable(getattr(self, name))
        ]
        return param_list

    def diff(self, config) -> dict:
        """
        diff given config with current config object

        Args:
            cfg(BaseConfig): given config, could be any subclass of BaseConfig

        Returns:
            dict: contains all diff pair
        """
        assert isinstance(config, BaseConfig), "config is not a subclass of BaseConfig"
        diff_result = {}
        self_param_list = self._get_param_list()
        conf_param_list = config._get_param_list()
        for param in self_param_list:
            if param not in conf_param_list:
                diff_result[param] = getattr(self, param)
            else:
                self_val, conf_val = getattr(self, param), getattr(config, param)
                if self_val != conf_val:
                    if isinstance(self_val, dict):
                        diff_result[param] = diff_dict(self_val, conf_val)
                    else:
                        diff_result[param] = self_val
        return diff_result

    def show_diff(self, config):
        """
        print diff between current config object and given config object
        """
        return pprint.pformat(edict(self.diff(config)))

    def find(self, key: str, show=True, color=Fore.BLACK + Back.YELLOW) -> dict:
        """
        find a given key and its value in config

        Args:
            key (str): the string you want to find
            show (bool): if show is True, print find result; or return the find result
            color (str): color of `key`, default color is black(foreground) yellow(background)

        Returns:
            dict: if  show is False, return dict that contains all find result

        Example::

            >>> from config import config        # suppose you are in your training dir
            >>> config.find("weights")
        """
        key = key.upper()
        find_result = {}
        param_list = self._get_param_list()
        for param in param_list:
            param_value = getattr(self, param)
            if re.search(key, param):
                find_result[param] = param_value
            elif isinstance(param_value, dict):
                find_res = find_key(param_value, key)
                if find_res:
                    find_result[param] = find_res
        if not show:
            return find_result
        else:
            pformat_str = pprint.pformat(edict(find_result))
            print(highlight(key, pformat_str, color))

    def __repr__(self):
        param_dict = edict(
            {param: getattr(self, param) for param in self._get_param_list()}
        )
        return pprint.pformat(param_dict)

config = BaseConfig()


# DRP原则:Don't  repeat youself
# 框架(framework):特指为解决一个开放性问题而设计的具有一定约束性的支撑结构
# 对于所有的Web应用，本质上其实就是一个socket服务端，用户的浏览器其实就是一个socket客户端。

# Python内置了一个WSGI服务器，这个模块叫wsgiref，application()函数必须由WSGI服务器来调用
# from wsgiref.simple_server import make_server
# def application(environ,start_response):
#     #通过environ封装一个包含所有请求信息的(字典)对象，start_response一个发送HTTP响应的函数(方便的设置响应头)
#     start_response("200 OK",[("contrnt-Type","text/html")])
#     return [b"
hello,web!
"]
# #封装socket对象以及准备过程（socket,bind,listen）
# httpd=make_server("localhost",8080,application)
# print("serving http on 8000...")
# #开始监听请求
# httpd.serve_forever()

from wsgiref.simple_server import make_server
import time
def current_time(request):
    current_time = time.ctime(time.time())
    f = open('current_time.html',"rb")
    data = f.read()
    data =str(data,'utf8').replace("!current_time!",current_time)
    return [data.encode("utf8")]
def routers():
    urlpatterns = (
        ('/time',current_time),
    )
    return urlpatterns
def application(environ,start_response):
    start_response("200 OK", [("contrnt-Type", "text/html")])
    path = environ["PATH_INFO"]       #获取请求路径（地址，端口，路径）
    urlpatterns = routers()
    func = None
    for item in urlpatterns:
        if item[0] == path:
            func = item[1]
            break
    if func:
        return func(environ)
    else:
        return ["
404
".encode("utf8")]
httpd = make_server("localhost", 8080, application)
print("serving http on 8000...")
httpd.serve_forever()

# MVC就是把web应用分为模型(Model),控制器(Controller),视图(View)三层
# 模型负责业务对象与数据库的对象(ORM),视图负责与用户的交互(页面)，控制器(C)接受用户的输入调用模型和视图完成用户的请求。
# Django的MTV模式本质上与MVC模式没有什么差别，也是各组件之间为了保持松耦合关系，只是定义上有些许不同，
# Django的MTV分别代表：
#        Model(模型)：负责业务对象与数据库的对象(ORM)
#        Template(模版)：负责如何把页面展示给用户
#        View(视图)：负责业务逻辑，并在适当的时候调用Model和Template
# Django还有一个url分发器，它的作用是将一个个URL的页面请求分发给不同的view处理，view再调用相应的Model和Template
# Controller(MVC)     和    View(MTV)
# View(MVC)           和    Template(MTV)


import logging
import os
from pathlib import Path
from typing import List, Tuple

import cv2
import hydra
import imutils
import numpy as np
import pandas as pd
from omegaconf import DictConfig, OmegaConf
from PIL import Image, ImageFilter
from tqdm import tqdm

from src.data.utils import (
    extract_body_part,
    extract_series_name,
    extract_study_name,
    extract_temperature,
    extract_time_stamp,
    get_dir_list,
    get_file_list,
    read_hsi,
)
from src.models.models import AblationSegmenter

log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

def filter_mask(
    mask: np.ndarray,
) -> np.ndarray:
    unique_clusters = list(np.unique(mask))
    unique_clusters.remove(0)

    mask_out = np.zeros_like(mask)
    for cluster_idx in unique_clusters:
        mask_idx = (mask == cluster_idx).astype(np.uint8)

        # Morphological smoothing
        kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
        _mask_morph = cv2.morphologyEx(mask_idx, cv2.MORPH_CLOSE, kernel, iterations=1)

        # Filtering
        _mask_smooth = Image.fromarray(_mask_morph)
        _mask_smooth = _mask_smooth.filter(ImageFilter.ModeFilter(size=3))
        _mask_smooth = np.asarray(_mask_smooth)

        # Stacking layers
        mask_out_ = _mask_smooth * cluster_idx
        for y in range(mask.shape[0]):
            for x in range(mask.shape[1]):
                if mask_out_[y][x] == cluster_idx:
                    mask_out[y][x] = mask_out_[y][x]

    return mask_out

def segment_hsi(
    hsi_path: str,
    meta: pd.DataFrame,
    save_dir: str,
    modality: str = 'abs',
    model_name: str = 'mean_shift',
    norm_type: str = 'standard',
    box_offset: Tuple[int, int] = (0, 0),
    apply_filtering: bool = True,
    src_size: Tuple[int, int] = (744, 1000),
) -> List:
    # Read HSI file
    hsi = read_hsi(
        path=hsi_path,
        modality=modality,
    )

    # Extract additional information
    study_name = extract_study_name(path=hsi_path)
    series_name = extract_series_name(path=hsi_path)
    body_part = extract_body_part(path=hsi_path)
    temperature_idx, temperature = extract_temperature(path=hsi_path)
    date, time = extract_time_stamp(path=hsi_path)

    # Initialize clustering model
    model = AblationSegmenter(model_name)

    # Segment the ablation area
    metadata = []
    box_ = meta[(meta['study'] == study_name) & (meta['series'] == series_name)]
    box_ = box_[box_[['x1', 'y1', 'x2', 'y2']].notnull().all(1)]

    if len(box_) == 0:
        print('Status...............: Skipped')
        log.info('Status...............: Skipped')
    else:
        box = [
            int(np.mean(box_.x1)),
            int(np.mean(box_.y1)),
            int(np.mean(box_.x2)),
            int(np.mean(box_.y2)),
        ]
        save_dir_img = os.path.join(
            save_dir,
            model_name,
            study_name,
            series_name,
            f'img_{modality}',
        )
        save_dir_box_src = os.path.join(
            save_dir,
            model_name,
            study_name,
            series_name,
            f'box_{modality}_src',
        )
        save_dir_box_seg = os.path.join(
            save_dir,
            model_name,
            study_name,
            series_name,
            f'box_{modality}_seg',
        )
        os.makedirs(save_dir_img, exist_ok=True)
        os.makedirs(save_dir_box_src, exist_ok=True)
        os.makedirs(save_dir_box_seg, exist_ok=True)

        for idx in tqdm(range(hsi.shape[2]), leave=True, desc='Wavelength processing'):
            img = hsi[:, :, idx]

            # Process source image
            img_size = img.shape[:-1] if len(img.shape) == 3 else img.shape
            if img_size != src_size:
                img = imutils.resize(img, height=src_size[0], inter=cv2.INTER_LINEAR)
            img = cv2.normalize(img, None, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U)
            img = cv2.equalizeHist(img)
            img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)

            # Clustering
            box_src, box_mask = model(
                img=img,
                box=box,
                box_offset=box_offset,
                norm_type=norm_type,
            )

            if apply_filtering:
                box_mask = filter_mask(box_mask)

            # Get number of clusters
            unique_clusters = list(np.unique(box_mask))
            num_clusters = len(unique_clusters)
            wavelength = 500 + 5 * idx
            wavelength_id = idx + 1
            log.info(
                f'Study = {study_name}, '
                f'Series = {series_name}, '
                f'ID = {wavelength_id:03}, '
                f'Wavelength = {wavelength}, '
                f'Clusters = {num_clusters}',
            )

            # Stack and save segmentation masks
            box_seg = model.label_to_rgb(box_mask)
            img_name = f'{wavelength_id:03}.png'
            save_path_img = os.path.join(save_dir_img, img_name)
            save_path_box_src = os.path.join(save_dir_box_src, img_name)
            save_path_box_seg = os.path.join(save_dir_box_seg, img_name)
            cv2.imwrite(save_path_img, img)
            cv2.imwrite(save_path_box_src, box_src)
            cv2.imwrite(save_path_box_seg, box_seg)

            metadata.append(
                {
                    'source_dir': str(Path(hsi_path).parent),
                    'study_name': study_name,
                    'series_name': series_name,
                    'hsi_name': str(Path(hsi_path).name),
                    'hsi_path': hsi_path,
                    'hsi_height': hsi.shape[0],
                    'hsi_width': hsi.shape[1],
                    'hsi_depth': hsi.shape[2],
                    'img_name': img_name,
                    'img_path': save_path_img,
                    'img_height': img.shape[0],
                    'img_width': img.shape[1],
                    'box_name': img_name,
                    'box_path_src': save_path_box_src,
                    'box_path_seg': save_path_box_seg,
                    'box_height': box_src.shape[0],
                    'box_width': box_src.shape[1],
                    'date': date,
                    'time': time,
                    'body_part': body_part,
                    'temperature_id': temperature_idx,
                    'temperature': temperature,
                    'wavelength_id': idx + 1,
                    'wavelength': 500 + 5 * idx,
                    'clusters': unique_clusters,
                    'num_clusters': num_clusters,
                },
            )
        print('Status...............: Processed')
        log.info('Status...............: Processed')

    return metadata

@hydra.main(
    config_path=os.path.join(os.getcwd(), 'config'),
    config_name='inference_clustering',
    version_base=None,
)
def main(cfg: DictConfig) -> None:
    log.info(f'Config:\n\n{OmegaConf.to_yaml(cfg)}')

    # Filter the list of studied directories
    study_dirs = get_dir_list(
        data_dir=cfg.src_dir,
        include_dirs=cfg.include_dirs,
        exclude_dirs=cfg.exclude_dirs,
    )

    # Get list of HSI files
    hsi_paths = get_file_list(
        src_dirs=study_dirs,
        include_template='',
        ext_list='.dat',
    )
    log.info(f'HSI found............: {len(hsi_paths)}')

    # Read metadata
    df_meta = pd.read_excel(cfg.metadata_path)

    # Processing of HSI files
    result = []
    for hsi_path in tqdm(hsi_paths, desc='HSI processing'):
        hsi_path = os.path.normpath(hsi_path)
        print(f'\nHSI to process.......: {hsi_path}')
        log.info(f'HSI to process.......: {hsi_path}')
        result_ = segment_hsi(
            hsi_path=hsi_path,
            meta=df_meta,
            modality=cfg.modality,
            model_name=cfg.model_name,
            norm_type=cfg.norm_type,
            box_offset=tuple(cfg.box_offset),
            src_size=tuple(cfg.src_size),
            apply_filtering=cfg.apply_filtering,
            save_dir=cfg.save_dir,
        )
        result.append(result_)
    result = sum(result, [])

    df = pd.DataFrame(result)
    save_path = os.path.join(cfg.save_dir, cfg.model_name, f'metadata_{cfg.modality}.xlsx')
    df.index += 1
    df.to_excel(
        save_path,
        sheet_name='Metadata',
        index=True,
        index_label='ID',
    )
    log.info('')
    log.info(f'Complete')

if __name__ == '__main__':
    main()


import json

import pytest

from factom_did.client.constants import ENTRY_SCHEMA_V100
from factom_did.client.did import DID, DIDKeyPurpose

@pytest.fixture
def did():
    return (
        DID()
        .management_key("man-key1", 0)
        .management_key("man-key2", 2)
        .did_key("did-key1", DIDKeyPurpose.AuthenticationKey, priority_requirement=2)
        .service("gmail-service", "email-service", "https://gmail.com", 2)
    )

class TestMethodSpecVersionUpgrade:
    def test_upgrade_with_no_management_key(self):
        with pytest.raises(RuntimeError):
            DID().method_spec_version_upgrade("1.0.0")

    def test_downgrade(self, did):
        with pytest.raises(ValueError):
            did.method_spec_version_upgrade("0.1.0")

    def test_successful_upgrade(self, did):
        entry = did.method_spec_version_upgrade("1.0.0").export_entry_data()
        ext_ids = entry["ext_ids"]
        content = json.loads(entry["content"])

        assert ext_ids[0] == "DIDMethodVersionUpgrade".encode("utf-8")
        assert ext_ids[1] == ENTRY_SCHEMA_V100.encode("utf-8")
        assert ext_ids[2] == "{}#{}".format(did.id, "man-key2").encode("utf-8")
        assert len(ext_ids) == 4
        assert len(content) == 1
        assert content["didMethodVersion"] == "1.0.0"


#!/usr/bin/env python

from __future__ import print_function
import numpy as np

import time
from utils.pybullet_tools.kuka_primitives3 import BodyPose, BodyConf, Register
from utils.pybullet_tools.utils import WorldSaver, connect, dump_world, get_pose, set_pose, Pose, \
    Point, set_default_camera, stable_z, disconnect, get_bodies, HideOutput, \
    create_box, \
    load_pybullet, step_simulation, Euler, get_links, get_link_info, get_movable_joints, set_joint_positions, \
    set_camera, get_center_extent, tform_from_pose, attach_viewcone, LockRenderer, load_model, set_point

from utils.pybullet_tools.body_utils import draw_frame
from utils.pybullet_tools.pr2_utils import set_arm_conf, REST_LEFT_ARM, open_arm, \
    close_arm, get_carry_conf, arm_conf, get_other_arm, set_group_conf, PR2_URDF, DRAKE_PR2_URDF, create_gripper

from copy import copy
import os

file_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

class PlanningScenario(object):
    def __init__(self):
        with HideOutput():
            with LockRenderer():
                self.arm_left = 'left'
                self.grasp_type = 'top'
                self.arm_right = get_other_arm(self.arm_left)

                self.pr2 = load_pybullet(file_dir+"/pr2_description/urdf/pr2_simplified.urdf",
                                         fixed_base=True)

                w = .22
                h = .7
                self.h = h

                mass = 1

                self.bd_body = {
                    "floor": load_pybullet(file_dir+"/scenario_description/plane.urdf", fixed_base=True),
                    "table": create_box(w, w, h, color=(.75, .75, .75, 1)),
                    "sink": create_box(w, w, h, color=(.25, .25, .75, 1)),
                    "stove": create_box(w, w, h, color=(.75, .25, .25, 1)),
                    "box1": create_box(.07, .07, .1, mass=mass, color=(1, 0, 0, 1)),
                    "box2": create_box(.07, .07, .1, mass=mass, color=(0, 1, 0, 1)),
                    "box3": create_box(.07, .07, .1, mass=mass, color=(0, 0, 1, 1)),
                    "box4": create_box(.07, .07, .1, mass=mass, color=(0, 0.7, 0.7, 1)),
                    # "box5": create_box(.07, .07, .1, mass=mass, color=(0.7, 0, 0.7, 1)),
                    # "box6": create_box(.07, .07, .1, mass=mass, color=(0.7, 0.7, 0, 1)),
                    # "box7": create_box(.07, .07, .1, mass=mass, color=(0, 0.8, 0.6, 1)),
                    # "box8": create_box(.07, .07, .1, mass=mass, color=(0.8, 0, 0.6, 1)),
                    # "box9": create_box(.07, .07, .1, mass=mass, color=(0.6, 0.8, 0, 1)),

                }
                self.bd_body.update(dict((self.bd_body[k], k) for k in self.bd_body))

                initial_conf = get_carry_conf(self.arm_left, self.grasp_type)
                set_arm_conf(self.pr2, self.arm_left, initial_conf)
                open_arm(self.pr2, self.arm_left)
                set_arm_conf(self.pr2, self.arm_right, arm_conf(self.arm_right, REST_LEFT_ARM))
                close_arm(self.pr2, self.arm_right)
                set_group_conf(self.pr2, 'torso', [0.2])

                self.pos_table = np.array([1.2, 0, h / 2])
                set_point(self.bd_body['table'], self.pos_table)
                set_point(self.bd_body['sink'], (0, 1.0, h / 2))
                set_point(self.bd_body['stove'], (0, -1., h / 2))
                set_point(self.bd_body['box1'], (self.pos_table[0] - 0.06, self.pos_table[1] - 0.06, self.h + .1 / 2))
                set_point(self.bd_body['box2'], (self.pos_table[0] - 0.06, self.pos_table[1] + 0.06, self.h + .1 / 2))
                set_point(self.bd_body['box3'], (self.pos_table[0] + 0.06, self.pos_table[1] - 0.06, self.h + .1 / 2))
                set_point(self.bd_body['box4'], (self.pos_table[0] + 0.06, self.pos_table[1] + 0.06, self.h + .1 / 2))
                # set_point(self.bd_body['box5'], (self.pos_table[0] - 0.0, self.pos_table[1] - 0.0, self.h + .1 / 2))
                # set_point(self.bd_body['box6'], (self.pos_table[0] + 0.12, self.pos_table[1] - 0.0, self.h + .1 / 2))
                # set_point(self.bd_body['box7'], (self.pos_table[0] - 0.12, self.pos_table[1] + 0.12, self.h + .1 / 2))
                # set_point(self.bd_body['box8'], (self.pos_table[0] - 0.0, self.pos_table[1] + 0.12, self.h + .1 / 2))
                # set_point(self.bd_body['box9'], (self.pos_table[0] + 0.12, self.pos_table[1] + 0.12, self.h + .1 / 2))

                set_camera(75, -90, 1.1, Point(x=0., y=-0.95))

                # set_camera(75, -70, 2.3, Point(x=0., y=-0.))

            # set_pose(box1, Pose(Point(x=0.25, y=0.80, z=stable_z(box1, region2))))

        self.movable_bodies = [self.bd_body['box1'], self.bd_body['box2'], self.bd_body['box3'],
                               self.bd_body['box4'],
                               # self.bd_body['box5'], self.bd_body['box6'],
                               # self.bd_body['box7'], self.bd_body['box8'], self.bd_body['box9'],
                               ]
        self.env_bodies = [self.bd_body['floor']]
        self.regions = [self.bd_body['table'], self.bd_body['sink'], self.bd_body['stove']]

        self.all_bodies = list(set(self.movable_bodies) | set(self.env_bodies) | set(self.regions))

        self.sensors = []

        self.robots = [self.pr2]
        self.arms = [self.arm_left, self.arm_right]
        self.controllable = [self.arm_left]

        self.gripper = None

        self.dic_body_info = {}
        for b in self.movable_bodies:
            obj_center, obj_extent = get_center_extent(b)
            body_pose = get_pose(b)
            body_frame = tform_from_pose(body_pose)
            bottom_center = copy(obj_center)
            bottom_center[2] = bottom_center[2] - obj_extent[2] / 2
            bottom_frame = tform_from_pose((bottom_center, body_pose[1]))
            relative_frame_bottom = np.dot(bottom_frame, np.linalg.inv(body_frame))  # from pose to bottom
            center_frame = tform_from_pose((obj_center, body_pose[1]))
            relative_frame_center = np.dot(center_frame, np.linalg.inv(body_frame))

            self.dic_body_info[b] = (obj_extent, relative_frame_bottom, relative_frame_center)

        self.saved_world = WorldSaver()

    def get_gripper(self, arm='left'):
        # upper = get_max_limit(problem.robot, get_gripper_joints(problem.robot, 'left')[0])
        # set_configuration(gripper, [0]*4)
        # dump_body(gripper)
        if self.gripper is None:
            self.gripper = create_gripper(self.pr2, arm=arm)
        return self.gripper

    def reset(self):
        self.saved_world.restore()

#######################################################

def display_scenario():
    connect(use_gui=True)

    scn = PlanningScenario()
    scn.reset()

    for i in range(10000):
        step_simulation()
        time.sleep(0.1)

    disconnect()
    print('Finished.')

if __name__ == '__main__':
    display_scenario()


import sqlite3

class Database:
    def __init__(self, path_to_db):
        self.path_to_db = path_to_db

    @staticmethod
    def create_db_if_not_exists(path_to_db, execute_command):
        """
        Creates database if it doesn't exist
        :param execute_command:
        :param path_to_db: path to database
        :return: None
        """

        conn = sqlite3.connect(path_to_db)
        cursor = conn.cursor()
        cursor.execute(execute_command)
        conn.commit()
        conn.close()


from PyQt5 import QtCore
from electrum.event import Event
from PyQt5.QtGui import QMovie
from electrum.util import resource_path
from PyQt5.QtWidgets import QHBoxLayout, QLabel, QWidget
from electrum.gui.qt.util import read_QImage
from threading import Timer
from PyQt5.QtCore import QSize

class DiceHeaderWidget(QWidget):

    msg_roll_dice = QtCore.pyqtSignal(QMovie, QMovie)
    msg_dice_start = QtCore.pyqtSignal(QMovie,QMovie)

    def __init__(self, parent=None):
        QWidget.__init__(self, parent=parent)
        self.parent = parent
        self.dice_event = Event.getInstance()
        self.parent = parent
        self.dice_block = None
        self.build_ui()

    def build_ui(self):
        self.header_img = read_QImage('dice_logo.jpg')
        self.lbl_dice1 = QLabel()
        self.lbl_dice2 = QLabel()
        self.lbl_dice_header_img = QLabel()
        self.lbl_dice_header_img.setPixmap(self.header_img)
        self.lbl_dice_header_img.setFixedHeight(200)
        self.header_hbox = QHBoxLayout()
        self.header_hbox.addStretch()
        self.header_hbox.addWidget(self.lbl_dice_header_img)
        self.header_hbox.addWidget(self.lbl_dice1)
        self.header_hbox.addWidget(self.lbl_dice2)
        self.header_hbox.addStretch()

        self.dice_event.register_callback(self.roll_dice,["roll_dice"])
        self.dice_event.register_callback(self.dice_start,["tx_brodcasted"])
        self.setLayout(self.header_hbox)

    def roll_dice(self, event, result):
        #only play animation once per block, because multiple result in single block is same
        if self.dice_block == result["blockHeight"]:
            return

        self.dice_block = result["blockHeight"]
        dice1 = str(result["betInfo"]["firstDice"])
        dice2 = str(result["betInfo"]["secondDice"])
        self.anim_dice1.stop()
        self.anim_dice2.stop()

        self.anim_dice1.setFileName(resource_path("gui","animations","dice","dice"+ dice1 + ".gif"))
        self.anim_dice2.setFileName(resource_path("gui","animations","dice","dice"+ dice2 + ".gif"))

        def stop_dice1(frame_no):
            if frame_no == self.anim_dice1.frameCount()-2:
                self.anim_dice1.stop()

        def stop_dice2(frame_no):
            if frame_no == self.anim_dice2.frameCount()-2:
                self.anim_dice2.stop()

        self.anim_dice1.frameChanged.connect(stop_dice1)
        self.anim_dice2.frameChanged.connect(stop_dice2)

        self.anim_dice1.setCacheMode(QMovie.CacheAll)
        self.anim_dice2.setCacheMode(QMovie.CacheAll)

        self.lbl_dice1.setMovie(self.anim_dice1)
        self.lbl_dice2.setMovie(self.anim_dice2)

        self.anim_dice1.start()
        self.anim_dice2.start()

    def dice_start(self,event, args):
        print("dice_start called")
        if not (args == None) and args["type"] == "dice":
            self.anim_dice1 = QMovie(resource_path("gui","animations","dice","rolling" + ".gif"))
            self.anim_dice2 = QMovie(resource_path("gui","animations","dice","rolling" + ".gif"))

            self.lbl_dice_header_img.setVisible(False)
            self.lbl_dice1.setVisible(True)
            self.lbl_dice2.setVisible(True)

            self.lbl_dice1.setMovie(self.anim_dice1)
            self.lbl_dice2.setMovie(self.anim_dice1)
            self.anim_dice1.start()
            self.anim_dice2.start()


"""
Autoencoders testing environment (ATE)

Related to the work:
Stable training of autoencoders for hyperspectral unmixing
Paper ID 10040

Source code for the review process of International Conference
on Computer Vision 2021
"""

if __name__ == "__main__":
    import os
    import sys
    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

import unittest
from torch.utils.data import DataLoader

from ate_tests.test_params import get_params
from ate.ate_core import _update_dataset_info
from ate.ate_autoencoders import (get_autoencoder, get_trained_network, _aa_factory)
from ate.ate_data import get_dataset
from architectures import basic

class Test(unittest.TestCase):
    def setUp(self):
        self.autoencoder_name = "basic"
        self.dataset_name = 'Custom'
        params_global, _ = get_params()
        self.dataset = get_dataset(self.dataset_name,
                                   path=params_global['path_data'],
                                   normalisation='max')

    def test__aa_factory(self):
        params_global, params_aa = get_params()
        _update_dataset_info(params_global, self.dataset)
        aut = _aa_factory(basic.Autoencoder, params_aa, params_global)
        self.assertIsInstance(aut, basic.Autoencoder)
        self.assertEqual(aut.n_bands, params_global["n_bands"])
        self.assertEqual(aut.n_endmembers, params_global["n_endmembers"])

    def test_get_autoencoder(self):
        params_global, params_aa = get_params()
        _update_dataset_info(params_global, self.dataset)
        aut = get_autoencoder(self.autoencoder_name, params_aa, params_global)
        self.assertIsInstance(aut, basic.Autoencoder)
        self.assertEqual(aut.params_grid, aut.get_params_grid())
        self.assertEqual(aut.n_bands, params_global["n_bands"])
        self.assertEqual(aut.n_endmembers, params_global["n_endmembers"])

    def test_get_trained_network(self):
        params_global, params_aa = get_params()
        _update_dataset_info(params_global, self.dataset)
        data_loader = DataLoader(self.dataset,
                                 batch_size=params_aa["batch_size"],
                                 shuffle=True)
        net = get_trained_network(autoencoder_name=self.autoencoder_name,
                                  data_loader=data_loader,
                                  params_aa=params_aa,
                                  params_global=params_global,
                                  n_runs=2)
        self.assertIsInstance(net, basic.Autoencoder)
        self.assertEqual(net.n_bands, params_global["n_bands"])
        self.assertEqual(net.n_endmembers, params_global["n_endmembers"])

if __name__ == "__main__":
    print ("Run tests from ../run_ate_tests.py")


import requests
import os
import sys
from flask import Flask, jsonify
from flask_restful import Resource, Api, reqparse
from flask import Flask, render_template, request
from flask_uploads import UploadSet, configure_uploads, IMAGES
import uuid
from flask_cors import CORS
from werkzeug.utils import secure_filename

import database.books as books
import helpers.controller as keys

url = keys.URL

class this_read_book(Resource):
	def __init__(self):
		self._database_books = books.this_books()

	def get_all_book(self):
		data = self._database_books.read_books()
		return jsonify({"data":data})

	def get_by_name(self, name):
		totaly = f"http://"+url+"/img/"+name
		data = self._database_books.read_by_name(totaly)
		return jsonify({"data":data})

	def get_book_uploaded(self, username):
		data = self._database_books.get_book_uploaded_byusername(username)
		return jsonify({"data":data})

	def get_views(self, name):
		totaly = f"http://"+url+"/img/"+name
		data = self._database_books.get_views(totaly)
		return jsonify({'views':data})

	def update_views(self, name, views):
		totaly = f"http://"+url+"/img/"+name
		update = self._database_books.update_views(totaly, views)
		if update:
			return jsonify({"update":"Success"})
		else:
			return jsonify({"update":"Failed"})

	def get(self):
		id = request.args.get('id')
		if id == 'all_books':
			return self.get_all_book()

		if id == 'get_by_name':
			name = request.args.get('name')
			return self.get_by_name(name)

		if id == 'get_book_uploaded':
			username = request.args.get('username')
			return self.get_book_uploaded(username)

		if id == 'get_views':
			name = request.args.get('name')
			return self.get_views(name)

		if id == 'update_views':
			name = request.args.get('name')
			views = request.args.get('views')
			return self.update_views(name, views)


"""
This is mostly a placeholder for more sophisticated modification but I want it in the flow
Things like syncronizing timestamps between multiple gateways (so timestamps appear in order) and
better mapping of input RSSI/SNR to output RSSI/SNR, etc
"""

import json
import time
import logging
import random
import datetime as dt

class RXMetadataModification:
    def __init__(self):
        self.min_rssi = -120
        self.max_rssi = -90  # valid to 50 miles via FSPL filter
        self.max_snr = 1.9
        self.min_snr = -9.9
        self.tmst_offset = 0
        self.logger = logging.getLogger('RXMeta')

    def modify_rxpk(self, rxpk, src_mac=None, dest_mac=None):
        """
        JSON object
        :param rxpk: per PUSH_DATA https://github.com/Lora-net/packet_forwarder/blob/master/PROTOCOL.TXT
        :return: object with metadata modified
        """

        old_snr, old_rssi, old_ts = rxpk['lsnr'], rxpk['rssi'], rxpk['tmst']
        # simple clipping low and high, could be a lot more sophisticated to add randomness or better mapping
        if src_mac == dest_mac:
            rxpk['rssi'] += 3  # boost RSSI for src gateway
        else:
            rxpk['rssi'] += random.randint(-2, 2)  # randomize rssi +/- 2dBm
        rxpk['lsnr'] = round(rxpk['lsnr'] + random.randint(-15, 10) * 0.1, 1)  # randomize snr +/- 1dB in 0.1dB increments
        # clip after adjustments to ensure result is still valid
        rxpk['rssi'] = min(self.max_rssi, max(self.min_rssi, rxpk['rssi']))
        rxpk['lsnr'] = min(self.max_snr,  max(self.min_snr,  rxpk['lsnr']))

        # modify tmst (Internal timestamp of "RX finished" event (32b unsigned)) to be aligned to uS since midnight UTC
        # this will be discontinuous once a day but that is basically same effect as a gateway reset / forwarder reboot
        # acceptable for proof-of-concept
        # also note 'time' is only available if there is a gps connected BUT time could be way off if there is a poor GPS fix
        # therefore compare to current time and only trust 'time' field if within 1.5s of now.  If 'time' is not available or
        # cannot be trusted, use the current time as assumed arrival time
        ts_dt = dt.datetime.utcnow()
        gps_valid = False
        if 'time' in rxpk:
            ts_str = rxpk['time']
            if ts_str[-1] == 'Z':
                ts_str = ts_str[:-1]
                ts_dt = dt.datetime.fromisoformat(ts_str)
            if abs((ts_dt - dt.datetime.utcnow()).total_seconds()) > 1.5:
                ts_dt = dt.datetime.utcnow()
            else:
                gps_valid = True

        ts_midnight = dt.datetime(year=ts_dt.year, month=ts_dt.month, day=ts_dt.day, hour=0, minute=0, second=0, microsecond=0)
        elapsed_us = int((ts_dt-ts_midnight).total_seconds() * 1e6)
        elapsed_us_u32 = elapsed_us % 2**32

        #print(f"elapsed us: {elapsed_us} ({elapsed_us/1e6}s), as u32 = {elapsed_us_u32}")
        if src_mac != dest_mac:
            rxpk['tmst'] = (elapsed_us_u32 + self.tmst_offset) % 2**32
        else:
            tmst_offset = (rxpk['tmst'] - elapsed_us_u32 + 2**32) % 2**32
            #  print(f"updated tmst_offset from:{self.tmst_offset} to {tmst_offset} (error: {self.tmst_offset - tmst_offset})")
            self.tmst_offset = tmst_offset
        self.logger.debug(f"modified packet from GW {src_mac[-8:]} to vGW {dest_mac[-8:]}, rssi:{old_rssi}->{rxpk['rssi']}, lsnr:{old_snr}->{rxpk['lsnr']:.1f}, tmst:{old_ts}->{rxpk['tmst']} {'GPS SYNC' if gps_valid else ''}")
        return rxpk


from xgboost import XGBClassifier
from sklearn.multiclass import OneVsRestClassifier
# Import the `svm` model
from sklearn import svm
from sklearn.model_selection import cross_validate
from sklearn.metrics import recall_score
# Import 'knn'
from sklearn.neighbors import KNeighborsClassifier
# Import 'random forest'
from sklearn.ensemble import RandomForestClassifier

from sklearn.ensemble import BaggingClassifier
from sklearn.dummy import DummyClassifier
from sklearn.linear_model import Perceptron
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import ParameterGrid
from sklearn.tree import DecisionTreeClassifier
from sklearn.utils import check_random_state
from sklearn.neural_network import MLPClassifier

import os
import numpy as np
import pandas as pd
import scipy.signal
from sklearn.preprocessing import scale
import pickle
import time
from os.path import dirname
import warnings
import sklearn.exceptions

# Global: the number of frequency bands we have
F = 228

# For each sample we use 2 spetra to improve classification. The 2nd spectrum uses -1 because we use the diff() filter
FREQ = 2 * F - 1

# define Standard Normal Variate
snv = lambda x:(x-x.mean())/x.std()

# get path of the current script
this_dir = os.path.realpath(__file__)
parent_dir = dirname(this_dir)

# create a data frame to hold all absorbance values
# need an extra column at the start to hold "type" (i.e. the thing we are trying to predict)
col=["column"+str(i) for i in range(1,FREQ+1)]
X_train = pd.DataFrame(columns=col)
y_train = pd.DataFrame(columns=['Type'])

# Loop through all the csv files and read their data.
path = os.path.join(parent_dir, 'data_scatter')
i = 0
for filename in os.listdir(path):
    with open(os.path.join(path, filename)) as f:

        # Read csv file and skip the top 19 rows that not related to Absorbance
        data = pd.read_csv(f, skiprows=19)
        # Get Absorbance values and fill NA/NaN values using the 0
        #absorbance = data['Absorbance (AU)'].fillna(0).values
        absorbance = pd.to_numeric(data['Absorbance (AU)'].fillna(0))
        #print(absorbance.shape)

        # Apply Savitzky-Golay Smoothing Filters:
        filtered = scipy.signal.savgol_filter(absorbance, window_length=25, polyorder=5)
        # Apply Standard Normal Variate Filters:
        filtered = snv(filtered)
        # Apply differentials and concate to original filtered features
        features = np.concatenate([filtered, np.diff(filtered)])

        # features = np.append(features, filename.split('O1')[0])
        X_train.loc[i] = features
        y_train.loc[i] = filename.split('O1')[0]
        i = i + 1

y_train['Type'].astype('category')
y_train = np.ravel(y_train)
#print(X_train)
#print(y_train)
print('=========================================================================')
print('1. In this part, the models will be evaluated by cross validation. Get precision macro, recall macro, fit time and score time.\n')
# --------------------------- Cross Validation -------------------------------
warnings.filterwarnings("ignore", category=sklearn.exceptions.UndefinedMetricWarning)
print("Default SVM Cross Validation:")
scoring = ['precision_macro', 'recall_macro']
clf = svm.SVC()
scores = cross_validate(clf, X_train, y_train, scoring=scoring, cv=5, return_train_score=False)
print("SVM test_precision_macro", scores['test_precision_macro'])
print("SVM test_recall_macro", scores['test_recall_macro'])
print("SVM average fit time", scores['fit_time'].mean())
print("SVM average score time", scores['score_time'].mean())
# The mean score and the 95% confidence interval of the score estimate are hence given by:
print("SVM Accuracy: %0.2f (+/- %0.2f)" % (scores['test_precision_macro'].mean(), scores['test_precision_macro'].std() * 2))
print('\n')

print("Tuned SVM Cross Validation:")
scoring = ['precision_macro', 'recall_macro']
clf = svm.SVC(C=2., gamma=0.5, kernel='poly', degree=2, probability=True)
scores = cross_validate(clf, X_train, y_train, scoring=scoring, cv=5, return_train_score=False)
print("SVM test_precision_macro", scores['test_precision_macro'])
print("SVM test_recall_macro", scores['test_recall_macro'])
print("SVM average fit time", scores['fit_time'].mean())
print("SVM average score time", scores['score_time'].mean())
# The mean score and the 95% confidence interval of the score estimate are hence given by:
print("SVM Accuracy: %0.2f (+/- %0.2f)" % (scores['test_precision_macro'].mean(), scores['test_precision_macro'].std() * 2))
print('\n')

print("Default KNN Cross Validation:")
scoring = ['precision_macro', 'recall_macro']
clf = KNeighborsClassifier()
scores = cross_validate(clf, X_train, y_train, scoring=scoring, cv=5, return_train_score=False)
print("KNN test_precision_macro", scores['test_precision_macro'])
print("KNN test_recall_macro", scores['test_recall_macro'])
print("KNN average fit time", scores['fit_time'].mean())
print("KNN average score time", scores['score_time'].mean())
# The mean score and the 95% confidence interval of the score estimate are hence given by:
print("KNN Accuracy: %0.2f (+/- %0.2f)" % (scores['test_precision_macro'].mean(), scores['test_precision_macro'].std() * 2))
print('\n')

print("Default RF Cross Validation:")
scoring = ['precision_macro', 'recall_macro']
clf = RandomForestClassifier()
scores = cross_validate(clf, X_train, y_train, scoring=scoring, cv=5, return_train_score=False)
print("RF test_precision_macro", scores['test_precision_macro'])
print("RF test_recall_macro", scores['test_recall_macro'])
print("RF average fit time", scores['fit_time'].mean())
print("RF average score time", scores['score_time'].mean())
# The mean score and the 95% confidence interval of the score estimate are hence given by:
print("RF Accuracy: %0.2f (+/- %0.2f)" % (scores['test_precision_macro'].mean(), scores['test_precision_macro'].std() * 2))
print('\n')

print("Tuned XGBoost Cross Validation:")
scoring = ['precision_macro', 'recall_macro']
clf = OneVsRestClassifier(XGBClassifier(
 learning_rate=0.2,
 random_state=3,
 n_estimators=500,
 subsample=0.8,
 colsample_bytree=0.8,
 max_depth=5,
 objective= 'multi:softmax',
 num_class=25))
scores = cross_validate(clf, X_train, y_train, scoring=scoring, cv=5, return_train_score=False)
print("XGBoost test_precision_macro", scores['test_precision_macro'])
print("XGBoost test_recall_macro", scores['test_recall_macro'])
print("XGBoost average fit time", scores['fit_time'].mean())
print("XGBoost average score time", scores['score_time'].mean())
# The mean score and the 95% confidence interval of the score estimate are hence given by:
print("XGBoost Accuracy: %0.2f (+/- %0.2f)" % (scores['test_precision_macro'].mean(), scores['test_precision_macro'].std() * 2))
print('\n')

print('========================================================================')
print('2. In this part, the models will be evaluated by a held-out test set. Get fit time, score time and accuracy.\n')

# ---------------------------- default SVM -----------------------------
# Create the SVC model
svc_dmodel = svm.SVC()
t0=time.time()
# print(t0)
# Fit the data to the SVC model
svc_dmodel.fit(X_train, y_train)
print("Default SVC training time:", time.time()-t0, "s")
# ---------------------------- tuning parameteres -----------------------------
# Create the SVC model
svc_model = svm.SVC(C=2., gamma=0.5, kernel='poly', degree=2, probability=True)
t0=time.time()
# print(t0)
# Fit the data to the SVC model
svc_model.fit(X_train, y_train)
print("Tuned SVC training time:", time.time()-t0, "s")
'''
#------------------------- SVM candidates ----------------------------
print("SVC candidates:")
# Set the parameter candidates
parameter_candidates = [
    {'C': [1, 10, 100, 1000], 'kernel': ['linear']},
    {'C': [1, 10, 100, 1000], 'gamma': [0.5,0.1,0.01, 0.001], 'kernel': ['rbf']},
    {'C': [1, 10, 100, 1000], 'gamma': [0.5,0.1,0.01, 0.001], 'kernel': ['poly']},
]

# Create a classifier with the parameter candidates
clf_grid = GridSearchCV(estimator=svm.SVC(), param_grid=parameter_candidates, n_jobs=1)

# Train the classifier on training data
clf_grid.fit(X_train, y_train)

# Print out the results
#print('Best score for training data:', clf.best_score_)
print('Best `C`:',clf_grid.best_estimator_.C)
print('Best kernel:',clf_grid.best_estimator_.kernel)
print('Best `gamma`:',clf_grid.best_estimator_.gamma)

# Apply the classifier to the test data, and view the accuracy score
print('Best score:',clf_grid.best_score_)
print('\n')
'''
# ---------------------------- KNN ---------------------------------
# Create the default KNN model
knn_model = KNeighborsClassifier()
t0=time.time()
# print(t0)
# Fit the data to the SVC model
knn_model.fit(X_train, y_train)
print("Default KNN training time:", time.time()-t0, "s")
# ---------------------------- RF ---------------------------------
# Create the default RF model
rf_model = RandomForestClassifier()
t0=time.time()
# print(t0)
# Fit the data to the SVC model
rf_model.fit(X_train, y_train)
print("Default RF training time:", time.time()-t0, "s")
#---------------------------- XGBoost ------------------------------
classif = OneVsRestClassifier(XGBClassifier(
 learning_rate=0.2,
 random_state=3,
 n_estimators=500,
 subsample=0.8,
 colsample_bytree=0.8,
 max_depth=5,
 objective= 'multi:softmax',
 num_class=25))
t1=time.time()
classif.fit(X_train,y_train)
print("Tuned XGBoost training time:", time.time()-t1, "s")
print("\n")
# --------------------------- test -----------------------------

X_test = pd.DataFrame(columns=col)
y_test = pd.DataFrame(columns=['Type'])

# Loop through all the csv files and read their data.
path = os.path.join(parent_dir, 'test')
i = 0
for filename in os.listdir(path):
    with open(os.path.join(path, filename)) as f:

        data = pd.read_table(f, sep=",", skiprows=19)
        absorbance = pd.to_numeric(data['Absorbance (AU)'].fillna(0))

        # filtered <- filter(sgolay(p=3, n=11, m=0),absorbance)
        filtered = scipy.signal.savgol_filter(absorbance, window_length=11, polyorder=3)

        # filtered <- scale(t(filtered),center=TRUE)
        filtered = scale(np.transpose(filtered))

        # features <- c(filtered,diff(filtered))
        features = np.concatenate([filtered, np.diff(filtered)])

        # features = np.append(features, filename.split('O1')[0])
        X_test.loc[i] = features
        y_test.loc[i] = filename.split('O1')[0]
        #print(svc_model.predict([X_test.loc[i]]),y_test.loc[i],round(np.amax(svc_model.predict_proba([X_test.loc[i]])), 3),'\n')
        i = i + 1

# test_X.dropna(how='any')    #to drop if any value in the row has a nan
y_test['Type'].astype('category')
y_test = np.ravel(y_test)
t1=time.time()
q = svc_dmodel.score(X_test, y_test)
print("Default SVM score time:", time.time()-t1, "s")
t1=time.time()
x = svc_model.score(X_test, y_test)
print("Ideal SVM score time:", time.time()-t1, "s")
t1=time.time()
z = knn_model.score(X_test, y_test)
print("Default KNN score time:", time.time()-t1, "s")
t1=time.time()
y = classif.score(X_test, y_test)
print("Tuned XGBoost score time:", time.time()-t1, "s")
t1=time.time()
k = rf_model.score(X_test, y_test)
print("Default RF score time:", time.time()-t1, "s")
print('\n')

print("Default SVM accuracy is ", q)
print("Ideal SVM accuracy is ", x)
#print('Best accuracy from SVM search candidates:',clf_grid.score(X_test, y_test))
print("Default KNN accuracy is", z)
print("Default RF accuracy is", k)
print("Default XGBoost accuracy is", y)

'''
#============================ Save models ==============================
# Set working directory
os.chdir(os.path.join(parent_dir, 'model'))
# save the model to disk
filename = 'finalized_model.sav'
pickle.dump(svc_model, open(filename, 'wb'))

filename = 'xgboost_model.sav'
pickle.dump(classif, open(filename, 'wb'))
'''

print('=========================================================================')
print('3. In this part, test Bagging classifier.\n')
#=========================== Bagging ===============================
def test_bagging():
    # Check classification for various parameter settings.
    rng = check_random_state(0)
    grid = ParameterGrid({"max_samples": [0.5, 1.0],
                          "max_features": [1, 2, 4],
                          "bootstrap": [True, False],
                          "bootstrap_features": [True, False]})

    for base_estimator in [None,
                           DummyClassifier(),
                           #Perceptron(),
                           MLPClassifier(),
                           KNeighborsClassifier(),
                           DecisionTreeClassifier(),
                           svm.SVC(C=2., gamma=0.5, kernel='poly', degree=2, probability=False)]:
        print(base_estimator)
        for params in grid:
            print(params)
            print(BaggingClassifier(base_estimator=base_estimator,
                              random_state=rng,
                              **params).fit(X_train, y_train).score(X_test, y_test),'\n')

test_bagging()