import re
from random import sample
import h5py
import gensim
import torch
import os
from torch.utils.data import Dataset
from constant import DATA_DIR, MIMIC_2_DIR, MIMIC_3_DIR
import sys
import pandas as pd
import numpy as np
import math
import csv
from collections import defaultdict
import warnings
import json, ujson
warnings.filterwarnings('ignore', category=FutureWarning)

maxInt = sys.maxsize

while True:
    try:
        csv.field_size_limit(maxInt)
        break
    except OverflowError:
        maxInt = int(maxInt/10)

def create_main_code(ind2c):
    mc = list(set([c.split('.')[0] for c in set(ind2c.values())]))
    mc.sort()
    ind2mc = {ind:mc for ind, mc in enumerate(mc)}
    mc2ind = {mc:ind for ind, mc in ind2mc.items()}
    return ind2mc, mc2ind

class MimicFullDataset(Dataset):
    def __init__(self, version, mode, vocab_path, truncate_length,
                 label_truncate_length=30, term_count=1, sort_method='max'):
        self.version = version
        self.mode = mode

        if version == 'mimic2':
            raise NotImplementedError
        if version in ['mimic3', 'mimic3-50']:
            self.path = os.path.join(MIMIC_3_DIR, f"{version}_{mode}.json")

        if version in ['mimic3']:
            self.train_path = os.path.join(MIMIC_3_DIR, "train_full.csv")
        if version in ['mimic3-50']:
            self.train_path = os.path.join(MIMIC_3_DIR, "train_50.csv")

        with open(self.path, "r") as f:
            self.df = ujson.load(f)

        self.vocab_path = vocab_path
        self.word2id, self.id2word = load_vocab(self.vocab_path)

        self.truncate_length = truncate_length

        self.ind2c, _ = load_full_codes(self.train_path, version=version)
        # self.part_icd_codes = list(self.ind2c.values())
        self.c2ind = {c: ind for ind, c in self.ind2c.items()}
        self.code_count = len(self.ind2c)
        if mode == "train":
            print(f'Code count: {self.code_count}')

        self.ind2mc, self.mc2ind = create_main_code(self.ind2c)
        self.main_code_count = len(self.ind2mc)
        if mode == "train":
            print(f'Main code count: {self.main_code_count}')

        from nltk.tokenize import RegexpTokenizer
        self.tokenizer = RegexpTokenizer(r'\w+')

        self.len = len(self.df)

        self.label_truncate_length = label_truncate_length
        self.term_count = term_count
        self.sort_method = sort_method
        if self.mode == "train":
            self.prepare_label_feature(self.label_truncate_length)

    def check(self, word):
        for ch in word:
            if 'a' <= ch <= 'z' or 'A' <= ch <= "Z":
                return True
        return False

    def __len__(self):
        return self.len

    def gettext(self, index):
        return self.df[index]['TEXT']

    def split(self, text):
        sp = re.sub(r'\n\n+|  +', '\t', text.strip()).replace("\n",
                                                              " ").replace("!", "\t").replace("?", "\t").replace(".", "\t")
        return [s.strip() for s in sp.split("\t") if s.strip()]

    def tokenize(self, text):
        texts = self.split(text)
        all_text = []
        split_text = []
        sentence_index = []
        word_count = 0
        for note in texts:
            now_text = [w.lower() for w in self.tokenizer.tokenize(
                note) if not w.isnumeric()]
            if now_text:
                all_text.extend(now_text)
                split_text.append(now_text)
                word_count += len(now_text)
                sentence_index.append(word_count)
        return all_text, sentence_index, split_text

    def __get_text_label__(self, index):
        text = self.gettext(index)
        label = str(self.df[index]['LABELS']).split(';')
        return text, label

    def pad(self, l, pad_token_length, pad_token_id, reverse=False):
        if len(l) > pad_token_length:
            if not reverse:
                return l[0:pad_token_length]
            return l[-pad_token_length:]
        if not reverse:
            return l + [pad_token_id] * (pad_token_length - len(l))
        return [pad_token_id] * (pad_token_length - len(l)) + l

    def text2feature(self, text, truncate_length=None):
        if truncate_length is None:
            truncate_length = self.truncate_length

        all_text, sentence_index, split_text = self.tokenize(text)

        input_word = [self.word2id.get(
            w, self.word2id["**UNK**"]) for w in all_text]
        word_mask = [1] * len(input_word)
        input_word = self.pad(
            input_word, truncate_length, self.word2id["**PAD**"])
        word_mask = self.pad(word_mask, truncate_length, 0)
        word_sent = self.pad(sentence_index, truncate_length, -1)
        return input_word, word_mask, word_sent

    def process(self, text, label):
        input_word, word_mask, word_sent = self.text2feature(text)

        binary_label = [0] * self.code_count
        for l in label:
            if l in self.c2ind:
                binary_label[self.c2ind[l]] = 1

        main_label = [0] * self.main_code_count
        for l in label:
            if l.split('.')[0] in self.mc2ind:
                main_label[self.mc2ind[l.split('.')[0]]] = 1

        return input_word, word_mask, word_sent, \
               binary_label, main_label

    def __getitem__(self, index):
        text, label = self.__get_text_label__(index)
        processed = self.process(text, label)
        return processed

    def extract_label_desc(self, ind2c):
        if not hasattr(self, 'desc_dict'):
            self.desc_dict = load_code_descriptions()

        desc_list = []
        for i in ind2c:
            code = ind2c[i]
            if not code in self.desc_dict:
                print(f'Not find desc of {code}')
            desc = self.desc_dict.get(code, code)
            desc_list.append(desc)
        return desc_list

    def process_label(self, ind2c, truncate_length, term_count=1, method='max'):
        desc_list = self.extract_label_desc(ind2c)
        if term_count == 1:
            c_desc_list = desc_list
        else:
            c_desc_list = []
            with open(f'./embedding/icd_mimic3_{method}_sort.json', 'r') as f:
                icd_syn = ujson.load(f)
            for i in ind2c:
                code = ind2c[i]
                tmp_desc = [desc_list[i]]
                new_terms = icd_syn.get(code, [])
                if len(new_terms) >= term_count - 1:
                    tmp_desc.extend(new_terms[0:term_count - 1])
                else:
                    tmp_desc.extend(new_terms)
                    repeat_count = int (term_count / len(tmp_desc)) + 1
                    tmp_desc = (tmp_desc * repeat_count)[0:term_count]
                if i < 5:
                    print(code, tmp_desc)
                c_desc_list.extend(tmp_desc)

        c_input_word = []
        c_word_mask = []
        c_word_sent = []

        for i, desc in enumerate(c_desc_list):
            input_word, word_mask, word_sent = self.text2feature(desc, truncate_length=truncate_length)
            c_input_word.append(input_word)
            c_word_mask.append(word_mask)
            c_word_sent.append(word_sent)

        return c_input_word, c_word_mask, c_word_sent

    def prepare_label_feature(self, truncate_length):
        print('Prepare Label Feature')
        if hasattr(self, 'term_count'):
            term_count = self.term_count
        else:
            term_count = 1
        if hasattr(self, 'sort_method'):
            sort_method = self.sort_method
        else:
            sort_method = 'max'
        c_input_word, c_word_mask, c_word_sent = self.process_label(self.ind2c, truncate_length,
                                                                    term_count=term_count,
                                                                    method=sort_method)
        # mc_input_word, mc_word_mask, mc_word_sent = self.process_label(self.ind2mc, truncate_length)
        self.c_input_word = torch.LongTensor(c_input_word)
        self.c_word_mask = torch.LongTensor(c_word_mask)
        self.c_word_sent = torch.LongTensor(c_word_sent)
        # self.mc_input_word = torch.LongTensor(mc_input_word)
        # self.mc_word_mask = torch.LongTensor(mc_word_mask)
        # self.mc_word_sent = torch.LongTensor(mc_word_sent)

def my_collate_fn(batch):
    type_count = len(batch[0])
    batch_size = len(batch)
    output = ()
    for i in range(type_count):
        tmp = []
        for item in batch:
            tmp.extend(item[i])
        if len(tmp) <= batch_size:
            output += (torch.LongTensor(tmp),)
        elif isinstance(tmp[0], int):
            output += (torch.LongTensor(tmp).reshape(batch_size, -1),)
        elif isinstance(tmp[0], float):
            output += (torch.FloatTensor(tmp).reshape(batch_size, -1),)
        elif isinstance(tmp[0], list):
            dim_y = len(tmp[0])
            if isinstance(tmp[0][0], int):
                output += (torch.LongTensor(tmp).reshape(batch_size, -1, dim_y),)
            elif isinstance(tmp[0][0], float):
                output += (torch.FloatTensor(tmp).reshape(batch_size, -1, dim_y),)
    return output

def load_vocab(path):
    try:
        with open(path, "r", encoding="utf-8") as f:
            lines = f.readlines()
        words = [line.strip().split()[0] for line in lines]
    except BaseException:
        if path.endswith('.model'):
            model = gensim.models.Word2Vec.load(path)
        if path.endswith('.bin'):
            model = gensim.models.KeyedVectors.load_word2vec_format(
                path, binary=True)
        words = list(model.wv.vocab)
        del model

    # hard code to trim word embedding size
    try:
        with open('./embedding/word_count_dict.json', 'r') as f:
            word_count_dict = ujson.load(f)
    except BaseException:
        with open('../embedding/word_count_dict.json', 'r') as f:
            word_count_dict = ujson.load(f)
    words = [w for w in words if w in word_count_dict]

    for w in ["**UNK**", "**PAD**", "**MASK**"]:
        if not w in words:
            words = words + [w]
    word2id = {word: idx for idx, word in enumerate(words)}
    id2word = {idx: word for idx, word in enumerate(words)}
    return word2id, id2word

def load_full_codes(train_path, version='mimic3'):
    """
        Inputs:
            train_path: path to train dataset
            version: which (MIMIC) dataset
        Outputs:
            code lookup, description lookup
    """
    # get description lookup
    desc_dict = load_code_descriptions(version=version)
    # build code lookups from appropriate datasets
    if version == 'mimic2':
        ind2c = defaultdict(str)
        codes = set()
        with open('%s/proc_dsums.csv' % MIMIC_2_DIR, 'r') as f:
            r = csv.reader(f)
            # header
            next(r)
            for row in r:
                codes.update(set(row[-1].split(';')))
        codes = set([c for c in codes if c != ''])
        ind2c = defaultdict(str, {i: c for i, c in enumerate(sorted(codes))})
    else:
        codes = set()
        for split in ['train', 'dev', 'test']:
            with open(train_path.replace('train', split), 'r') as f:
                lr = csv.reader(f)
                next(lr)
                for row in lr:
                    for code in row[3].split(';'):
                        codes.add(code)
        codes = set([c for c in codes if c != ''])
        ind2c = defaultdict(str, {i: c for i, c in enumerate(sorted(codes))})
    return ind2c, desc_dict

def reformat(code, is_diag):
    """
        Put a period in the right place because the MIMIC-3 data files exclude them.
        Generally, procedure codes have dots after the first two digits,
        while diagnosis codes have dots after the first three digits.
    """
    code = ''.join(code.split('.'))
    if is_diag:
        if code.startswith('E'):
            if len(code) > 4:
                code = code[:4] + '.' + code[4:]
        else:
            if len(code) > 3:
                code = code[:3] + '.' + code[3:]
    else:
        code = code[:2] + '.' + code[2:]
    return code

def load_code_descriptions(version='mimic3'):
    # load description lookup from the appropriate data files
    desc_dict = defaultdict(str)
    if version == 'mimic2':
        with open('%s/MIMIC_ICD9_mapping' % MIMIC_2_DIR, 'r') as f:
            r = csv.reader(f)
            # header
            next(r)
            for row in r:
                desc_dict[str(row[1])] = str(row[2])
    else:
        with open("%s/D_ICD_DIAGNOSES.csv" % (DATA_DIR), 'r') as descfile:
            r = csv.reader(descfile)
            # header
            next(r)
            for row in r:
                code = row[1]
                desc = row[-1]
                desc_dict[reformat(code, True)] = desc
        with open("%s/D_ICD_PROCEDURES.csv" % (DATA_DIR), 'r') as descfile:
            r = csv.reader(descfile)
            # header
            next(r)
            for row in r:
                code = row[1]
                desc = row[-1]
                if code not in desc_dict.keys():
                    desc_dict[reformat(code, False)] = desc
        with open('%s/ICD9_descriptions' % DATA_DIR, 'r') as labelfile:
            for _, row in enumerate(labelfile):
                row = row.rstrip().split()
                code = row[0]
                if code not in desc_dict.keys():
                    desc_dict[code] = ' '.join(row[1:])
    return desc_dict

def load_embeddings(embed_file):
    W = []
    word_list = []
    try:
        with open(embed_file) as ef:
            for line in ef:
                line = line.rstrip().split()
                word_list.append(line[0])
                vec = np.array(line[1:]).astype(np.float)
                # also normalizes the embeddings
                vec = vec / float(np.linalg.norm(vec) + 1e-6)
                W.append(vec)
        word2id, id2word = load_vocab(embed_file)
    except BaseException:
        if embed_file.endswith('.model'):
            model = gensim.models.Word2Vec.load(embed_file)
        if embed_file.endswith('.bin'):
            model = gensim.models.KeyedVectors.load_word2vec_format(
                embed_file, binary=True)
        words = list(model.wv.vocab)

        original_word_count = len(words)

        # hard code to trim word embedding size
        with open('./embedding/word_count_dict.json', 'r') as f:
            word_count_dict = ujson.load(f)
        words = [w for w in words if w in word_count_dict]

        for w in ["**UNK**", "**PAD**", "**MASK**"]:
            if not w in words:
                words = words + [w]
        word2id = {word: idx for idx, word in enumerate(words)}
        id2word = {idx: word for idx, word in enumerate(words)}
        new_W = []
        for i in range(len(id2word)):
            if not id2word[i] in ["**UNK**", "**PAD**", "**MASK**"]:
                new_W.append(model.__getitem__(id2word[i]))
            elif id2word[i] == "**UNK**":
                print("adding unk embedding")
                new_W.append(np.random.randn(len(new_W[-1])))
            elif id2word[i] == "**MASK**":
                print("adding mask embedding")
                new_W.append(np.random.randn(len(new_W[-1])))
            elif id2word[i] == "**PAD**":
                print("adding pad embedding")
                new_W.append(np.zeros_like(new_W[-1]))
        new_W = np.array(new_W)
        print(f"Word count: {len(id2word)}")
        print(f"Load embedding count: {len(new_W)}")
        print(
            f"Original word count: {original_word_count}/{len(word_count_dict)}")
        del model
        return new_W

    if not "**UNK**" in word_list:
        # UNK embedding, gaussian randomly initialized
        print("adding unk embedding")
        word_list.append("**UNK**")
        vec = np.random.randn(len(W[-1]))
        vec = vec / float(np.linalg.norm(vec) + 1e-6)
        W.append(vec)
    if not "**MASK**" in word_list:
        # UNK embedding, gaussian randomly initialized
        print("adding unk embedding")
        word_list.append("**UNK**")
        vec = np.random.randn(len(W[-1]))
        vec = vec / float(np.linalg.norm(vec) + 1e-6)
        W.append(vec)
    if not "**PAD**" in word_list:
        print("adding pad embedding")
        word_list.append("**PAD**")
        vec = np.zeros_like(W[-1])
        W.append(vec)

    print(f"Word count: {len(id2word)}")
    print(f"Load embedding count: {len(W)}")
    print(f"Original word count: {original_word_count}/{len(word_count_dict)}")
    word2newid = {w: i for i, w in enumerate(word_list)}
    new_W = []
    for i in range(len(id2word)):
        new_W.append(W[word2newid[id2word[i]]])
    new_W = np.array(new_W)
    del model
    return new_W


#!/usr/bin/python3
'''ping sweep utility to automate and capture some of the results of
a ping sweep across the network

Usage:
        ./pysweep.py <ip> <cidr>

If <cidr> is not specified, default is "/27".

Sample:
        ./pysweep.py 192.168.0.23 /27
        ./pysweep.py 192.168.0.23
        ./pysweep.py 192.168.0.23 /24
'''
import subprocess
import sys
from datetime import datetime
import ipaddress

# global variables
detected_ip_list = []
sys.tracebacklimit = 0 # comment this to see tracebacks

def create_ip_list(ip, cidr="/27"):
    '''create a list of IP addresses'''
    ipcidr = ip + cidr
    try:
        net_ips = ipaddress.ip_network(ipcidr, strict=False)
    except ValueError:
        print('\nPlease make sure that you entered valid IP or CIDR\n')
    except TypeError:
        print('\nPlease make sure that you entered valid IP or CIDR\n')
    else:
        ip_list = [str(xip) for xip in net_ips.hosts()]
        return ip_list

def process_list(begin_time, end_time):
    '''Process List: loop over the list of detected IP addresses,
    outputs to stdout and write to file'''
    global detected_ip_list
    if detected_ip_list == []:
        print("\nSaw nothing on the network.\n")
    else:
        summary = ''
        summary += "Detected Hosts:\n===============\n"
        for ip in detected_ip_list:
            summary += f"{ip}\n"
        scan_duration = end_time - begin_time
        summary += f"\nTotal time to scan took: {scan_duration}"
        print(summary)

        with open('pingsweep-results.txt', 'w') as f:
            f.write(summary)

def fping_ip(ip):
    '''This will attempt to fping each IP in the network. Only handling SubprocessError'''
    global detected_ip_list
    try:
        output = subprocess.run(["fping", "-a", "-C", "5", "-q", ip], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    except subprocess.SubprocessError:
        print("There was an error in the command invocation.")
    else:
        if output.returncode == 0:
            output_b = output.stdout
            output_str = output_b.decode("utf-8")
            ip, response = output_str.split(':')
            print(f"Host {ip.strip()} is detected online. Response time(s) were: {response.strip()}")
            detected_ip_list.append(ip.strip())

def main(ip, cidr):
    begin_time = datetime.now()
    ip_target_list = create_ip_list(ip, cidr)
    for ip in ip_target_list:
        fping_ip(ip)
    end_time = datetime.now()
    process_list(begin_time, end_time)

if __name__ == "__main__":
    # main("192.168.0.257", "/27") # for testing
    argv_len = len(sys.argv)
    if argv_len == 3:
        main(sys.argv[1], sys.argv[2])
    elif argv_len == 2:
        main(sys.argv[1], "/27")
    else:
        sys.exit('\ntoo many or not enough arguments.\nUSAGE: ./pysweep.py <ip> <cidr>\n')


import progressbar as pb
import numpy as np
from collections import deque
import torch
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import time

def train(env, agent, episodes=5000, max_t=500, print_every=50):
    widget = ['training loop: ', pb.Percentage(), ' ',  pb.Bar(), ' ', pb.ETA()]
    timer = pb.ProgressBar(widgets=widget, maxval=episodes).start()

    scores = []
    scores_deque = deque(maxlen=100)
    steps_deque = deque(maxlen=100)
    for i_episode in range(1, episodes+1):
        states = env.reset()
        agent.reset()
        score = np.zeros(env.num_agents)
        steps = 0
        start = time.time()

        for t in range(max_t):
            actions = agent.act(states, add_noise=True)
            next_states, rewards, dones = env.step(actions)
            agent.step(i_episode, states, actions, rewards, next_states, dones)
            states = next_states

            score += rewards
            steps += 1
            if np.any(dones):
                break

        scores_deque.append(np.max(score))
        scores.append(np.max(score))
        steps_deque.append(steps)

        time_spent = time.time() - start
        print(f"Episode {i_episode}/{episodes}\t",
              f"Score: {np.max(score):.2f}\t",
              f"Steps: {steps}\t",
              f"Average Score: {np.mean(scores_deque):.2f}\t",
              f"Max Score: {np.max(scores_deque):.2f}\t",
              f"Average steps: {np.mean(steps_deque):.2f}\t",
              f"Time spent: {time_spent:.4f} seconds")
        if i_episode % print_every == 0:
            print()
            timer.update(i_episode)

        #         if (scores_deque[0]>0.5) and (np.mean(scores_deque) > 0.5):
        if np.mean(scores_deque) > 0.5:
            print(f"Environment solved in {i_episode-100} episodes!\t Average Score: {np.mean(scores_deque):.2f}")
            for i, agent in enumerate(agent.agents):
                torch.save(agent.actor.state_dict(), f"checkpoint_actor_{str(i)}.pth")
                torch.save(agent.critic.state_dict(), f"checkpoint_critic_{str(i)}.pth")
            break

    timer.finish()
    return scores

def plot_scores(scores):
    sns.set(style="whitegrid")
    episodes = np.arange(start=1, stop=len(scores)+1)

    data = pd.DataFrame(data=scores, index=episodes, columns=["Score"])

    fig = sns.lineplot(data=data)
    fig.set_xlabel("Episode #")
    plt.show()


import click
import os
import hashlib
import time
import sys

from qcloud_cos import CosConfig
from qcloud_cos import CosS3Client
import logging

logging.basicConfig(level=logging.INFO, stream=sys.stdout)
# 腾讯云COS密钥
secret_id = ""
secret_key = ""
token = None               # 如果使用永久密钥不需要填入token，如果使用临时密钥需要填入，临时密钥生成和使用指引参见https://cloud.tencent.com/document/product/436/14048
# 区域
region = ''
# 存储桶名
bucket_name = ""
# 自定义腾讯存储路径名
upload_name = ""
search_path = path = os.getcwd() + os.sep

# 腾讯云COS 初始化
config = CosConfig(Region=region, SecretId=secret_id, SecretKey=secret_key, Token=token)
client = CosS3Client(config)

def get_file_md5(file):
    with open(file, 'rb') as fp:
        data = fp.read()
        return hashlib.md5(data).hexdigest()

def upload_file(key, path):
    md5 = get_file_md5(path)
    if containMd5(md5) and not ignoreMd5:
        print('忽略文件: '+path)
        return
    try:
        response = client.upload_file(
            Bucket=bucket_name,
            LocalFilePath=path,
            Key=key,
            PartSize=1,
            MAXThread=4,
            EnableMD5=False
        )
        if response['ETag']:
            print("上传成功: " + upload_name+path)
            addLog(md5,path)
            return 1
    except BaseException:
        print("上传失败: "+path)
        return 0

def addLog(md5, path):
    localTime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    list_files.append(md5 + "," + path + "," + localTime)
    list_md5.append(md5)

def createLog():
    createFile("logs.log", list_files)
    print('已上传' + str(len(list_files)) + '个文件')

def insertMd5():
    createFile("md5.txt", list_md5)

def createFile(name, list):
    dirname, filename = os.path.split(os.path.abspath(sys.argv[0]))
    file = open(dirname + os.sep + name, 'a', encoding='utf-8')
    for fileName in list:
        file.write(fileName + '\n')
    file.close()

def containMd5(md5):
    if list_md5.__contains__(md5):
        return True
    return False

def loadMd5():
    dirname, filename = os.path.split(os.path.abspath(sys.argv[0]))
    path = dirname + os.sep + "logs.log"
    if not os.path.exists(path):
        os.mknod(path)
    file = open(path, 'r')
    if file is None:
        return
    lines = file.readlines()
    for line in lines:
        md5 = line.replace("\n", "").split(",")[0]
        if not containMd5(md5):
            list_md5.append(md5)

list_md5 = []
list_files = []

def scanfile(dir):
    for path, d, filelist in os.walk(dir):
        for filename in filelist:
            full_path = os.path.join(path, filename)
            if containType(full_path):
                md5 = get_file_md5(full_path)
                if not containMd5(md5):
                    allfiles.append(full_path)

def containType(path):
    if file_types is None:
        return True
    for ftype in file_types:
        if path.endswith("." + ftype):
            return True
    return False

allfiles = []
ignoreMd5 = False
file_types = None

@click.command()
@click.option('-f', "--file", type=str, default=None, help="单个文件压缩")
@click.option('-d', "--dir", type=str, default=None, help="被压缩的文件夹")
@click.option('-i', "--ignore", type=bool, default=False, help="忽略MD5验证")
@click.option('-p', "--path", type=str, default="", help="七牛云自定义路径")
@click.option('-t', "--type", type=str, default=None, help="指定文件后缀")
def run(file, dir, ignore, path, type):
    global ignoreMd5
    ignoreMd5 = ignore

    global upload_name
    upload_name = path

    if type is not None:
        global file_types
        file_types = type.split(",")
    if file is not None:
        allfiles.append(file)
        pass
    elif dir is not None:
        scanfile(dir)
        pass
    else:
        print("未指定文件夹，自动定位当前运行目录")
        scanfile(search_path)
        pass
    print("开始上传文件：腾讯云COS路径", ("/" if upload_name == "" else upload_name))
    runUpload()
    createLog()

def runUpload():
    for file in allfiles:
        upload_path = (upload_name + file).replace(os.sep + os.sep, os.sep)
        upload_path = upload_path[1:] if upload_path[0]=='/' else upload_path
        if upload_file(upload_path, file): continue

if __name__ == '__main__':
    try:
        loadMd5()
        run()
    except SystemExit:
        pass
    except BaseException as e:
        raise e


#若该函数在home.py 中，在my_view 中from home import send_email,运行时home.py 中先import my_view
#,然后在my_view 中from home import send_email，但是在home 中 import view 还没有完成，所以app.register_blueprint(my_view.views) 导入不成功
#应该算是循环导入问题
from flask_mail import Message
from flask import render_template,current_app
from threading import  Thread
from . import mail

def send_async_email(app,msg):
    with app.app_context():
        mail.send(msg)

def send_email(to,subject,template,**kwargs):
    app = current_app._get_current_object()
    msg = Message(app.config['MAIL_SUBJECT_PREFIX']+''+subject,sender=app.config['MAIL_SENDER'],recipients=[to])
    msg.body = render_template(template + '.txt', **kwargs)
    msg.html = render_template(template + '.html', **kwargs)
    thr = Thread(target=send_async_email,args=[app,msg])
    thr.start()
    return thr


# <url>/robots.txt
# tells you what you are allowed to scrape
# may also have a crawl delay request to stop
# users overloading servers
# practice ethical program, check t&c before scraping

# ideally you can avoid having to scrape by using
# the website's API
# Scrapy is a good tool for scraping
# May want to start looking at databases for storing data

# lets get stories the first two pages from hackernews that have over 100 points

import requests
from bs4 import BeautifulSoup
import pprint

res_pg1 = requests.get('https://news.ycombinator.com/news')
res_pg2 = requests.get('https://news.ycombinator.com/news/?p=2')

# print(res.text)
# use beautifulsoup to clean up the data
soup = BeautifulSoup(res_pg1.text, 'html.parser')
soup_p2 = BeautifulSoup(res_pg2.text, 'html.parser')
soup.extend(soup_p2)
print(type(soup))

# print(soup) # it is no longer one long string
# print(soup.body)
# print(soup.body.contents)
# print(soup.find_all('a'))
# print(soup.find_all('div'))
# print(soup.title)
# print(soup.a)
# print(soup.find(id='score_35036871'))
# print(soup.select('.score'))

# Get the info we need
links = soup.select(".titleline")
votes = soup.select(".score")
subtext = soup.select('.subtext')
# use subtext instead of votes as sometimes there are no votes

# print(votes[0].get('id'))

def sort_stories_by_vote(hn_list):
    return sorted(hn_list, key=lambda k:k['votes'], reverse=True)

def create_custom_hn(links, subtext):
    hn = []
    for i, link in enumerate(links):
        vote = subtext[i].select('.score')
        if len(vote) > 0:
            points = int(vote[0].getText().replace(' points',''))
            if points > 100:
                title = link.getText()
                href = link.find('a')['href']
                hn.append({'title':title,
                        'url':href,
                        'votes':points})
    return sort_stories_by_vote(hn)

hn = create_custom_hn(links, subtext)

pprint.pprint(hn)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Tue Nov  8 13:16:48 2016

@author: park
"""
#%%
#from keras.preprocessing import image
from keras.models import Model
from keras.layers import Dense, GlobalAveragePooling2D, Flatten, AveragePooling2D
from optparse import OptionParser
from keras.preprocessing.image import ImageDataGenerator
import datetime
import os
from keras.optimizers import Nadam
#from numpy import array_str
import lvdUtil.liveUtil as lvd
from keras.callbacks import ModelCheckpoint, CSVLogger, ReduceLROnPlateau
#import tensorflow as tf
import json
#%%
# My Own layer
def vgg_top_Model(x, classNum):
    x = GlobalAveragePooling2D(name='gavg_pool')(x) # It can changed to GolbalMaxPooling
    x = Dense(4096, activation='relu', name='fc1')(x)
    x = Dense(4096, activation='relu', name='fc2')(x)
    x = Dense(classNum, activation='softmax', name='predictions')(x)
    return x

def Xception_top_Model(x, classNum):
    x = GlobalAveragePooling2D(name='gavg_pool')(x)
    x = Dense(classNum, activation='softmax', name='predictions')(x)
    return x

def ResNet50_top_Model(x, classNum):
    x = GlobalAveragePooling2D(name='agvg_pool')(x)
#    x = Flatten()(x)
    x = Dense(classNum, activation='softmax', name='predictions')(x)
    return x

def InceptionV3_top_Model(x, classNum):
#    x = GlobalAveragePooling2D(name='agvg_pool')(x)
    x = AveragePooling2D((8, 8), strides=(8, 8), name='avg_pool')(x)
    x = Flatten(name='flatten')(x)
    x = Dense(classNum, activation='softmax', name='predictions')(x)
    return x

def jsonDumper(fileName, jsonString):
    with open(fileName, 'w') as f:
        f.write(json.dumps(jsonString))
    return

# Main function
#%%
if __name__=="__main__":
#%% Option Parser
###################################################
## uncomment me ###################################
###################################################
    use = "Usage : %prog [option]"
    parser = OptionParser(usage=use)

    parser.add_option("-d","--dataHome", dest='dataHome',
                  default="data/SegM", help="Home directory of data")
    parser.add_option("-s", "--sensor", dest="sensor",
                  default="LivDet2013/CrossMatchTrain", help="Sensor folder")
    parser.add_option("-p", "--pretrain", dest="preTrain", default="res",
                      help="Pretraining Model")
    parser.add_option("-b", "--binary", dest="binary",
                      default=True, help="Binary Class?")
    parser.add_option("-r", "--resultFolder", dest="result",
                      default="models/origin", help="saving folder")
    parser.add_option("-e", "--epoch", dest="nb_epoch", default=1, help="Number of Epoch")
    parser.add_option("-a", "--augmen", dest="augmen", default = False, help="Augemntation?")
    parser.add_option("-f", "--fine", dest="fine", default = False, help="Augemntation?")
    # parser.add_option("-g", "--gpu", dest="gpu", default = 0, help= "GPU select")

    options, args = parser.parse_args()

    #%% Option store

    sensor = options.sensor
    dataHome = options.dataHome
    preTrain = options.preTrain
    binary = options.binary
    aug = options.augmen
    fine = options.fine
    resFold = options.result
    # gpu = "/gpu:%s" % options.gpu

    if preTrain not in ['vgg', 'exc', 'res', 'inc']:
        print("Pretrain option must be one of ['vgg', 'exc', 'res', 'inc']")
        raise ValueError
    if options.binary: category = 'twoClass'
    else: category = 'manyClass'
    nb_epoch = int(options.nb_epoch)

    tempFolder = os.path.join(dataHome, 'Train', sensor, category)
    trainFolder = os.path.expanduser(os.path.join('~', tempFolder))
    tempFolder = os.path.join(dataHome, 'Val', sensor, category)
    valFolder = os.path.expanduser(os.path.join('~', tempFolder))

    modelSaveFolder = os.path.expanduser(os.path.join('~', dataHome))

    if binary:
        labels = lvd.binaryLabel()
    else:
        labels = lvd.loadLabelFromYear(sensor.split('/')[0], sensor.split('/')[1])
    label = lvd.labelOut(labels)

    classNum = len(label)

    samples_per_epoch = lvd.getSamplePerEpoch(trainFolder)
    nb_epoch = int(nb_epoch)
    nb_val_samples = lvd.getSamplePerEpoch(valFolder)
#%%
    if preTrain == 'vgg':
        from keras.applications.vgg16 import VGG16
        base_model = VGG16(weights='imagenet', include_top=False)
        predictions = vgg_top_Model(base_model.output, classNum)

    elif preTrain == 'exc':
        from keras.applications.xception import Xception
        base_model = Xception(weights='imagenet', include_top=False)
        predictions = Xception_top_Model(base_model.output, classNum)

    elif preTrain == 'res':
        from keras.applications.resnet50 import ResNet50
        base_model = ResNet50(weights='imagenet', include_top=False)
        predictions = ResNet50_top_Model(base_model.output, classNum)

    elif preTrain == 'inc':
        from preModels.inceptionV3 import InceptionV3
        model = InceptionV3(weights='imagenet', include_top=True, classNum=classNum)
#        predictions = InceptionV3_top_Model(base_model.output, classNum)
    else:
        print("There is no adequate Pretrain option.")
        raise ValueError
#%%
    if preTrain != 'inc':
        model = Model(input=base_model.input, output=predictions)
        #%%
        # first: train only the top layers (which were randomly initialized)
    ############################################################################
        for layer in base_model.layers:
            layer.trainable = False
        target_size = (224, 224)
    else: # This part is only for Inception V3 (Size problem)
        for layer in model.layers[:-1]:
            layer.trainable = False
        target_size = (299, 299)
    ############################################################
        # compile the model (should be done *after* setting layers to non-trainable)

#%%
    # this is the augmentation configuration we will use for training
    prefix = ''
    if aug:
        prefix += "Aug"
        train_datagen = ImageDataGenerator(
                           rescale = 1./255,
                           horizontal_flip=True,
                           rotation_range=30,
                           width_shift_range=0.20,
                           height_shift_range=0.20,
                           shear_range=0.3,
                           zoom_range=0.3,
                           fill_mode= 'reflect'
                           )
    else:
        train_datagen = ImageDataGenerator(rescale = 1./255)

    val_datagen = ImageDataGenerator(rescale = 1./255)

    train_generator = train_datagen.flow_from_directory(
        trainFolder,
        classes = label,
        target_size=target_size,
        batch_size=32)
    val_generator = val_datagen.flow_from_directory(
        valFolder,
        classes = label,
        target_size=target_size,
        batch_size=32)
    # Training Start
    print("%s models is starting." % preTrain)

    if not fine:
            #%% Save file name setting
        nameSplit = sensor.split('/')
        year = nameSplit[0][-2:]
        sen = nameSplit[1][:3]
        now = datetime.datetime.now().strftime('%m%d%H%M')
        saveFileName = preTrain+prefix+year+'_'+sen+'_'+category+'_'+now
        modelSaveName = os.path.expanduser(os.path.join('~',dataHome,
                     resFold,saveFileName+'.hdf5'))
        modelLogName = os.path.expanduser(os.path.join('~',dataHome,
                     resFold,saveFileName+'.log'))
        modelConfJson = os.path.expanduser(os.path.join('~',dataHome,
                     resFold,saveFileName+'.json'))
        reduceLr = ReduceLROnPlateau(monitor="val_loss", factor=0.2, patience=3)
        checkpoint = ModelCheckpoint(modelSaveName, monitor='val_loss',
                 verbose=1, save_best_only=True, mode='min')
        csv_logger = CSVLogger(modelLogName, append=True)
        callbacks_list = [checkpoint, csv_logger, reduceLr]

        model.compile(optimizer=Nadam(lr=0.0005), loss='categorical_crossentropy',
                  metrics=['accuracy'])

        log = model.fit_generator(train_generator,
            samples_per_epoch = samples_per_epoch,
            nb_epoch = nb_epoch,
            validation_data=val_generator,
            nb_val_samples = nb_val_samples,
            callbacks=callbacks_list)
        json_model = model.to_json()
        jsonDumper(modelConfJson, json_model)
    else:
        if nb_epoch < 10:
            print ("You need more epochs")
            raise
        prefix+="Fine"
        nb_epoch = nb_epoch - 5
        nameSplit = sensor.split('/')
        year = nameSplit[0][-2:]
        sen = nameSplit[1][:3]
        now = datetime.datetime.now().strftime('%m%d%H%M')
        saveFileName = preTrain+prefix+year+'_'+sen+'_'+category+'_'+now
        modelSaveName = os.path.expanduser(os.path.join('~',dataHome,
                     resFold,saveFileName+'.hdf5'))
        modelLogName = os.path.expanduser(os.path.join('~',dataHome,
                     resFold,saveFileName+'.log'))
        modelConfJson = os.path.expanduser(os.path.join('~',dataHome,
                     resFold,saveFileName+'.json'))

        reduceLr = ReduceLROnPlateau(monitor="val_loss", factor=0.2, patience=5)
        checkpoint = ModelCheckpoint(modelSaveName, monitor='val_acc',
                 verbose=1, save_best_only=True, mode='max')
        csv_logger = CSVLogger(modelLogName, append=True)
        callbacks_list = [checkpoint, csv_logger, reduceLr]

        model.compile(optimizer=Nadam(lr=0.002), loss='categorical_crossentropy',
                  metrics=['accuracy'])

        log = model.fit_generator(train_generator,
            samples_per_epoch = samples_per_epoch,
            nb_epoch = 5,
            validation_data=val_generator,
            nb_val_samples = nb_val_samples)

        for layer in model.layers:
            layer.trainable = True

        model.compile(optimizer=Nadam(lr=0.0002), loss='categorical_crossentropy',
                  metrics=['accuracy'])

        log = model.fit_generator(train_generator,
            samples_per_epoch = samples_per_epoch,
            nb_epoch = nb_epoch,
            validation_data=val_generator,
            nb_val_samples = nb_val_samples,
            callbacks=callbacks_list)
        json_model = model.to_json()
        jsonDumper(modelConfJson, json_model)

    del model


from typing import Union, List, Dict

import pymongo
from pymongo.collection import Collection
from bson.son import SON

from .commands import AggregateCommand, FindCommand, CountCommand, \
    UpdateCommand, DistinctCommand, DeleteCommand, FindAndModifyCommand

Document = Union[dict, SON]

class ExplainableCollection():
    def __init__(self, collection, verbosity=None, comment=None):
        self.collection = collection
        self.last_cmd_payload = None
        self.verbosity = verbosity or "queryPlanner"
        self.comment = comment

    def _explain_command(self, command):
        command_son = command.get_SON()
        explain_command = SON([("explain", command_son)])
        explain_command["verbosity"] = self.verbosity
        if self.comment:
            explain_command["comment"] = self.comment
        self.last_cmd_payload = command_son
        return self.collection.database.command(explain_command)

    def update_one(self, filter, update, upsert=False,
                   bypass_document_validation=False,
                   collation=None, array_filters=None, hint=None,
                   session=None, **kwargs):
        command = UpdateCommand(self.collection, filter, update,
                                bypass_document_validation=
                                    bypass_document_validation,
                                array_filters=array_filters,
                                collation=collation, hint=hint,
                                upsert=upsert, multi=False)
        return self._explain_command(command)

    def update_many(self, filter: Document, update: Document, upsert=False,
                    array_filters=None, bypass_document_validation=False,
                    collation=None, hint=None, session=None, **kwargs):
        command = UpdateCommand(self.collection, filter, update, multi=True,
                                bypass_document_validation=bypass_document_validation, upsert=upsert, collation=collation, array_filters=array_filters, hint=hint)
        return self._explain_command(command)

    def distinct(self, key: str, filter: Document=None, session=None, **kwargs):
        command = DistinctCommand(self.collection, key, filter, kwargs)
        return self._explain_command(command)

    def aggregate(self, pipeline: List[Document], session=None, **kwargs):
        command = AggregateCommand(self.collection, pipeline,
                                   {}, kwargs)
        return self._explain_command(command)

    def estimated_document_count(self,
                                 **kwargs):

        command = CountCommand(self.collection, None, kwargs)
        return self._explain_command(command)

    def count_documents(self, filter: Document, session=None,
                                 **kwargs):

        command = AggregateCommand(self.collection, [{'$match': filter},
                                                     {'$group': {'n': {'$sum': 1}, '_id': 1}}]
                                   , {}, kwargs)
        return self._explain_command(command)

    def delete_one(self, filter: Document, collation=None, session=None,
                   **kwargs):
        limit = 1
        command = DeleteCommand(self.collection, filter, limit, collation,
                                kwargs)
        return self._explain_command(command)

    def delete_many(self, filter: Document, collation=None,
                    session=None, **kwargs: Dict[str, Union[int, str,
                                                            Document,
                                                      bool]]):
        limit = 0
        command = DeleteCommand(self.collection, filter, limit, collation,
                                kwargs)
        return self._explain_command(command)

    def watch(self, pipeline: Document = None, full_document: Document = None,
              resume_after= None,
              max_await_time_ms: int = None, batch_size: int = None,
              collation=None, start_at_operation_time=None, session=None,
              start_after=None):

        change_stream_options = {}
        if start_after:
            change_stream_options["startAfter"] = start_after
        if resume_after:
            change_stream_options["resumeAfter"] = resume_after
        if start_at_operation_time:
            change_stream_options["startAtOperationTime"] = start_at_operation_time
        if full_document:
            change_stream_options["fullDocument"] = full_document

        if pipeline is not None:
            pipeline = [{"$changeStream": change_stream_options}]+pipeline
        else:
            pipeline = [{"$changeStream": change_stream_options}]
        cursor_args = {}
        if batch_size:
            cursor_args["batchSize"] = batch_size
        command = AggregateCommand(self.collection, pipeline,
                                   cursor_args,
                                   {"collation":collation, "max_await_time_ms":
                                       max_await_time_ms})
        return self._explain_command(command)

    def find(self, filter: Document = None,
             **kwargs: Dict[str, Union[int, str,Document, bool]]):
        kwargs.update(locals())
        del kwargs["self"], kwargs["kwargs"]
        command = FindCommand(self.collection,
                              kwargs)

        return self._explain_command(command)

    def find_one(self, filter: Document = None, **kwargs: Dict[str,
                                                               Union[int, str,
                                                                Document, bool]]):
        kwargs.update(locals())
        del kwargs["self"], kwargs["kwargs"]
        kwargs["limit"] = 1
        command = FindCommand(self.collection, kwargs)
        return self._explain_command(command)

    def find_one_and_delete(self, filter: Document, projection: list = None,
                            sort: Document=None, session=None,
                        **kwargs):
        kwargs["query"] = filter
        kwargs["fields"] = projection
        kwargs["sort"] = sort
        kwargs["remove"] = True
        kwargs["session"] = session

        command = FindAndModifyCommand(self.collection,
                                       kwargs)
        return self._explain_command(command)

    def find_one_and_replace(self, filter: Document, replacement:
    Document={},
                            projection: list = None, sort=None,
                             return_document=pymongo.ReturnDocument.BEFORE,
                             session=None, **kwargs):
        kwargs["query"] = filter
        kwargs["fields"] = projection
        kwargs["sort"] = sort
        kwargs["new"] = False
        kwargs["update"] = replacement
        kwargs["session"] = session
        command = FindAndModifyCommand(self.collection,
                                       kwargs)
        return self._explain_command(command)

    def find_one_and_update(self, filter: Document, update: Document,
                            projection: list = None, sort=None,
                             return_document=pymongo.ReturnDocument.BEFORE,
                             session=None, **kwargs):
        kwargs["query"] = filter
        kwargs["fields"] = projection
        kwargs["sort"] = sort
        kwargs["upsert"] = False
        kwargs["update"] = update
        kwargs["session"] = session

        command = FindAndModifyCommand(self.collection,
                                       kwargs)
        return self._explain_command(command)

    def replace_one(self, filter: Document, replacement: Document,
                    upsert=False, bypass_document_validation=False,
                    collation=None, hint=None, session=None, **kwargs):
        command = UpdateCommand(self.collection, filter, update=replacement,
                                bypass_document_validation=bypass_document_validation,
                                hint=hint, collation=collation, multi=False, upsert=upsert)

        return self._explain_command(command)

# Alias
ExplainCollection = ExplainableCollection


import dash
import dash_core_components as dcc
import dash_html_components as html
import plotly.graph_objects as go
from dash.dependencies import Input, Output, State
import pandas as pd
import numpy as np
import plotly.figure_factory as ff
import plotly.express as px
from urllib.request import urlopen
import json
import dash_bootstrap_components as dbc
import plotly.io as pio

external_stylesheets = [
    'https://codepen.io/chriddyp/pen/bWLwgP.css',
   {
        'href': 'https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css',
        'rel': 'stylesheet',
#         'integrity': 'sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO',
    'crossorigin': 'anonymous'
   }
]
data=pd.read_csv('Data_Gurgen_Blbulyan.csv')
data=data.drop('Unnamed: 0',axis=1)
data.set_index('lead_id',inplace=True)

metadata=pd.read_excel('readme.xlsx')
bool_columns=metadata[metadata['type']=='bool']['Variable'].tolist()
count_columns=metadata[metadata['type']=='continuous']['Variable'].tolist()
cat_columns=metadata[metadata['type']=='categorical']['Variable'].tolist()
date_columns=metadata[metadata['type']=='datetime']['Variable'].tolist()
id_columns=metadata[metadata['type']=='id']['Variable'].tolist()
geo_columns=metadata[metadata['type']=='geospacial']['Variable'].tolist()

data[bool_columns]=data[bool_columns].astype('bool')
data[count_columns]=data[count_columns].astype('float')
data[cat_columns]=data[cat_columns].astype('category')
data[date_columns]=data[date_columns].astype('datetime64[ns]')

ddd=pd.read_csv('uszips.csv',dtype={"county_fips": str})
dd=ddd[['zip','county_name','county_fips']]
data=data.merge(dd,'left',on='zip')

data.dropna(inplace=True)
datascl=data.copy()
datascl[count_columns]=(datascl[count_columns]-datascl[count_columns].min())/(datascl[count_columns].max()-datascl[count_columns].min())

with urlopen('https://raw.githubusercontent.com/plotly/datasets/master/geojson-counties-fips.json') as response:
    counties = json.load(response)

pio.templates.default = "ggplot2"
app = dash.Dash(external_stylesheets =external_stylesheets)

app.layout = html.Div([
            html.H1("Dataviz Dashboard with Dash", style={'text-align': 'center'}),

            html.Div([
                   html.Div([

                               html.Div([dcc.Graph(figure = {},id='map1')]),
                               html.Div([
                                   html.Div([dcc.Dropdown(
                                    id = 'map1_bool',
#                                     value=bool_columns[0],
                                    options =  [{'label': i, 'value': i} for i in bool_columns],

                                    placeholder="Select a boolian variable")],className = 'four columns'),

                                   html.Div([dcc.Dropdown(
                                    id = 'map1_color',
                                    options =  [{'label': i, 'value': i} for i in cat_columns],
#                                     value=cat_columns[0],
                                    placeholder="Select a color variable")],className = 'four columns'),

                                   html.Div([dcc.Dropdown(
                                    id = 'map1_size',
                                    options =  [{'label': i, 'value': i} for i in count_columns],
#                                     value=count_columns[0],
                                    placeholder="Select a size variable")],className = 'four columns'),

                               ],className = 'row')

                   ],className = 'eight columns'),

                   html.Div([
                               html.Div([dcc.Graph(figure = {},id='box1')]),
                               html.Div([
                                   html.Div([dcc.Dropdown(
                                    id = 'box_bool',
                                    options =  [{'label': i, 'value': i} for i in bool_columns],

                                    placeholder="Select a boolian variable")],className = 'six columns'),

                                   html.Div([dcc.Dropdown(
                                    id = 'box_count',
                                    options = [{'label': i, 'value': i} for i in count_columns],

                                    placeholder="Select a continuous variable")],className = 'six columns')

                               ],className = 'row')

                  ],className = 'four columns')

            ], className = 'row'),

    html.Div([
                   html.Div([

                               html.Div([dcc.Graph(figure = {},id='scater1')]),
                               html.Div([dcc.Dropdown(id='scat_count',
    options=[{'label': i, 'value': i} for i in count_columns],
    placeholder="Select a 3 continuous variable last for size"   ,

    multi=True
)  ])

                   ],className = 'four columns'),

                   html.Div([
                               html.Div([dcc.Graph(figure = {},id='coloplath1')]),
                               html.Div([
                                   html.Div([dcc.Dropdown(
                                    id = 'coloplath_state',
                                    options =  [{'label': i, 'value': i} for i in count_columns],

                                    placeholder="Select a continuous variable")])

                               ])

                  ],className = 'eight columns')

            ], className = 'row'),

html.Div([

                               html.Div([dcc.Graph(figure = {},id='violin')]),
                               html.Div([
                                   html.Div([dcc.Dropdown(
                                    id = 'violin_bool',
                                    options =  [{'label': i, 'value': i} for i in bool_columns],

                                    placeholder="Select a boolian variable")],className = 'four columns'),

                                   html.Div([dcc.Dropdown(
                                    id = 'violin_count',
                                    options = [{'label': i, 'value': i} for i in count_columns],

                                    placeholder="Select a continuous variable")],className = 'four columns'),
                               html.Div([dcc.Dropdown(
                                    id = 'violin_cat',
                                    options = [{'label': i, 'value': i} for i in cat_columns+bool_columns],

                                    placeholder="Select a categorical variable")],className = 'four columns')

                               ],className = 'row')

                   ]),

html.Div([

                   html.Div([

                               html.Div([dcc.Graph(figure = {},id='distplot')]),
                               html.Div([
                                   html.Div([dcc.Dropdown(
                                    id = 'dist_ver',
                                    options =  [{'label': i, 'value': i} for i in count_columns],
                                    multi=True,
                                    placeholder="Select a continuous variable")],className = 'six columns'),

                                   html.Div([dcc.Slider(
        id='my_slider',
        min=0,
        max=1,
        step=0.1,
        value=0.25,
    )],className = 'six columns'),

                               ],className = 'row')

                   ],className = 'six columns'),

    html.Div([

                               html.Div([dcc.Graph(figure = {},id='countplot')]),
                               html.Div([dcc.Dropdown(id='count_cat',
    options=[{'label': i, 'value': i} for i in bool_columns+cat_columns],
    placeholder="Select a 2 categorical variable",
    multi=True
)  ])

                   ],className = 'six columns'),

            ], className = 'row'),

    html.Div([

                               html.Div([dcc.Graph(figure = {},id='map2')]),
                               html.Div([
                                   html.Div([dcc.Dropdown(
                                    id = 'map2_val',
                                    options =  [{'label': i, 'value': i} for i in count_columns],

                                    placeholder="Select a continuous variable")],className = 'six columns'),

                                   html.Div([dcc.RangeSlider(
        id='my_rangslider',
        min=0,
        max=1,
        step=0.1,
        value=[0.5,0.7],
    )],className = 'six columns')

                   ],className='row'),

            ]),

], className = 'container')

@app.callback(
    [Output(component_id='map1', component_property='figure'),
     Output(component_id='box1', component_property='figure'),
     Output(component_id='scater1', component_property='figure'),
     Output(component_id='coloplath1', component_property='figure'),
     Output(component_id='violin', component_property='figure'),
     Output(component_id='distplot', component_property='figure'),
     Output(component_id='countplot', component_property='figure'),
     Output(component_id='map2', component_property='figure')
    ],
    [Input(component_id='map1_bool', component_property='value'),
     Input(component_id='map1_color', component_property='value'),
     Input(component_id='map1_size', component_property='value'),
     Input(component_id='box_bool', component_property='value'),
     Input(component_id='box_count', component_property='value'),
     Input(component_id='scat_count', component_property='value'),
     Input(component_id='coloplath_state', component_property='value'),
     Input(component_id='violin_count', component_property='value'),
     Input(component_id='violin_bool', component_property='value'),
     Input(component_id='violin_cat', component_property='value'),
     Input(component_id='my_slider', component_property='value'),
     Input(component_id='dist_ver', component_property='value'),
     Input(component_id='count_cat', component_property='value'),
     Input(component_id='map2_val', component_property='value'),
     Input(component_id='my_rangslider', component_property='value')
    ]
)
def update_graph(map1_bool,map1_color,map1_size,box_bool,box_count,scat_count,
                 coloplath_state,violin_count,violin_bool,violin_cat,
                 my_slider,dist_ver,count_cat,map2_val,my_rangslider):

    try:
        fig = px.scatter_mapbox(data[data[map1_bool]==True].copy(), lat="latitude", lon="longitude",         hover_name="country_code",color=map1_color,size=map1_size,title='Check the boolean variables location', zoom=3,size_max=15)
        fig.update_layout(mapbox_style="open-street-map")

    except:
        fig = px.scatter_mapbox(data[data['black_ip']==True], lat="latitude", lon="longitude",         hover_name="country_code",color='gender',size='amount_request',title='Check the boolean variables location', zoom=3,size_max=15)
        fig.update_layout(mapbox_style="open-street-map")
    try:
        fig1 = px.box(data,x=box_bool,  y=box_count,title='Box plot with boolian variable')
    except:
        fig1 = px.box(data, x="target", y='amount_request',title='Box plot with boolian variable')

    try:
        fig2 = px.scatter(data, x=scat_count[0], y=scat_count[1],size=scat_count[2],title='Scatter plot with size')

    except:
        fig2 = px.scatter(data, x="emptime", y="monthlyincome",size='emptime',title='Scatter plot with size')

    try:
        fig3 = px.choropleth(data.groupby('state')[coloplath_state].mean().reset_index(),locations='state', locationmode="USA-states", color=coloplath_state, scope="usa",title='Choropleth for USA state level')

    except:
        fig3 = px.choropleth(data.groupby('state')['median_income'].mean().reset_index(),locations='state', locationmode="USA-states", color='median_income', scope="usa",title='Choropleth for USA state level')

    try:
        fig4 = go.Figure()

        fig4.add_trace(go.Violin(x=data[violin_cat][ data[violin_bool] == True],
                        y=data[violin_count][ data[violin_bool] == True ],
                        legendgroup='Yes', scalegroup='Yes', name='Yes',
                        side='negative',
                        line_color='blue')
             )
        fig4.add_trace(go.Violin(x=data[violin_cat][ data[violin_bool] == False ],
                        y=data[violin_count][ data[violin_bool] == False ],
                        legendgroup='No', scalegroup='No', name='No',
                        side='positive',
                        line_color='orange')
             )
        fig4.update_traces(meanline_visible=True)
        fig4.update_layout(violingap=0, violinmode='overlay')

    except:
        fig4 = go.Figure()

        fig4.add_trace(go.Violin(x=data['day'][ data['target'] == True],
                        y=data['monthlyincome'][ data['target'] == True ],
                        legendgroup='Yes', scalegroup='Yes', name='Yes',
                        side='negative',
                        line_color='blue')
             )
        fig4.add_trace(go.Violin(x=data['day'][ data['target'] == False ],
                        y=data['monthlyincome'][ data['target'] == False ],
                        legendgroup='No', scalegroup='No', name='No',
                        side='positive',
                        line_color='orange')
             )
        fig4.update_traces(meanline_visible=True)
        fig4.update_layout(violingap=0, violinmode='overlay',title='Violin plot')

    try:
        fig5 = ff.create_distplot([datascl[c] for c in dist_ver] , dist_ver, bin_size=my_slider,show_rug=False)
    except:
        fig5 = ff.create_distplot([datascl['monthlyincome']], ['monthlyincome'], bin_size=.25,show_rug=False)
    fig5.update_layout(title='Distribution plot')

    try:
        fig6 = px.histogram(data, x=count_cat[0], color=count_cat[1])
    except:
        fig6 = px.histogram(data, x="target", color="bypass")

    fig6.update_layout(title='Count plot')

    try:
        fig7 = px.choropleth(datascl, geojson=counties, locations='county_fips', color=map2_val,
                           color_continuous_scale="Viridis",
                           range_color=my_rangslider,
                           scope="usa",

                          )

    except:
        fig7 = px.choropleth(datascl, geojson=counties, locations='county_fips', color='score',
                           color_continuous_scale="Viridis",
                           range_color=(0, 20),
                           scope="usa",

                          )
    fig7.update_layout(title='ZIP level Choropleth')
    return fig,fig1,fig2,fig3,fig4,fig5,fig6,fig7

if __name__ == '__main__':
    app.run_server(debug=True)

#https://github.com/Coding-with-Adam/Dash-by-Plotly/blob/master/Other/Dash_Introduction/intro.py
#https://dataviz.shef.ac.uk/blog/12/06/2020/dash-tutorial-2
#https://github.com/plotly/dash-sample-apps/tree/main/apps


import os
from pymongo import MongoClient
import pandas as pd
import pickle
import json
from time import sleep
from sklearn.ensemble import GradientBoostingClassifier
import requests

def prepare_data(new_data):
    clean_data = {}
    X_col = ['US/N', 'body_length', 'Twitter_presence','Facebook_presence', 'payout_type_',
             'payout_type_ACH', 'payout_type_CHECK', 'have_previous_payouts', \
            'show_map','sensible_age','cap_name', 'maindomain_email']
    clean_data['body_length'] = new_data['body_length']
    clean_data['show_map'] = new_data['show_map']
    clean_data['payout_type_'] = 1 if new_data['payout_type']  == '' else 0
    clean_data['payout_type_ACH'] = 1 if new_data['payout_type']  == 'ACH' else 0
    clean_data['payout_type_CHECK'] = 1 if new_data['payout_type']  == 'CHECK' else 0
    clean_data['US/N'] = 1 if new_data['country'] == "US" else 0
    clean_data['Twitter_presence'] = 1 if new_data['org_twitter'] > 0 else 0
    clean_data['Facebook_presence'] = 1 if new_data['org_facebook'] > 0 else 0
    clean_data['have_previous_payouts'] = 1 if len(new_data['previous_payouts']) != 0 else 0
    clean_data['cap_name'] = 1 if new_data['name'].isupper() == True else 0
    clean_data['sensible_age'] = 1 if (new_data['user_age'] >= 15) and (new_data['user_age']<= 80) else 0
    clean_data['maindomain_email'] = 1 if ('hotmail.com' in new_data['email_domain']) or ('gmail.com' in new_data['email_domain']) or ('yahoo.com' in new_data['email_domain']) else 0
    X = np.array([])
    for col in X_col:
        X = np.append(X, clean_data[col])
    return X

def predict_fraud(new_data, model):
    X = prepare_data(new_data)
    return model.predict(X)

def request_data_point():
    """GET request to herokuapp that supplies a new
    unscored datapoint, parses as json"""
    return requests.get('http://galvanize-case-study-on-fraud.herokuapp.com/data_point').content

def check_duplicate(data, key, table):
    """Query the database to see if the observation
    has alreayd been seen and scored"""
    return table.find({key : data}).count() > 0

def run_all():
    with open('model.pkl') as f:
        model = pickle.load(f)

    client = MongoClient()
    db = client['my_db']
    hash_table = db['hashes']
    data_table = db['data']

    while True:
        sleep(10)
        new_data = request_data_point()
        data_hash = hash(new_data)
        is_duplicate = check_duplicate(data_hash, 'data_hash', hash_table)
        if is_duplicate == False:
            hash_table.insert_one({'data_hash' : data_hash})
            new_data = json.loads(new_data)
            prediction = predict_fraud(new_data, model)
            new_data['fraud'] = prediction[0]
            data_table.insert_one(new_data)

if __name__ == "__main__":
    os.system('python fraud_flask.py &')
    run_all()


from collections import ChainMap
from typing import List

import gym
import numpy as np
import torch

from searl.neuroevolution.components.utils import to_tensor, Transition

class MPEvaluation():
    """
    evaluates and population and stores transitions an a push_queue

    """

    def __init__(self, config, logger, push_queue=None):

        self.rng = np.random.RandomState(config.seed.evaluation)
        self.cfg = config
        self.log = logger
        self.push_queue = push_queue
        self.eval_episodes = config.eval.eval_episodes

    def test_individual(self, individual, epoch):
        return_dict = self._evaluate_individual(individual, self.cfg, self.cfg.eval.test_episodes, epoch, False)
        fitness = np.mean(return_dict[individual.index]["fitness_list"])
        return fitness

    @staticmethod
    def _evaluate_individual(individual, config, num_episodes, seed, exploration_noise=False, start_phase=False):

        actor_net = individual.actor

        num_frames = 0
        fitness_list = []
        transistions_list = []
        episodes = 0

        env = gym.make(config.env.name)
        env.seed(seed)
        actor_net.eval()

        with torch.no_grad():
            while episodes < num_episodes or num_frames < config.eval.min_eval_steps:
                episode_fitness = 0.0
                episode_transitions = []
                state = env.reset()
                t_state = to_tensor(state).unsqueeze(0)
                done = False
                while not done:
                    if start_phase:
                        action = env.action_space.sample()
                        action = to_tensor(action)
                    else:
                        action = actor_net(t_state)
                    action.clamp(-1, 1)
                    action = action.data.numpy()
                    if exploration_noise is not False:
                        action += config.eval.exploration_noise * np.random.randn(config.action_dim)
                        action = np.clip(action, -1, 1)
                    action = action.flatten()

                    step_action = (action + 1) / 2  # [-1, 1] => [0, 1]
                    step_action *= (env.action_space.high - env.action_space.low)
                    step_action += env.action_space.low

                    next_state, reward, done, info = env.step(step_action)  # Simulate one step in environment

                    done_bool = 0 if num_frames + 1 == env._max_episode_steps else float(done)

                    t_next_state = to_tensor(next_state).unsqueeze(0)

                    episode_fitness += reward
                    num_frames += 1

                    transition = Transition(state, action, next_state, np.array([reward]),
                                            np.array([done_bool]).astype('uint8'))
                    episode_transitions.append(transition)
                    t_state = t_next_state
                    state = next_state
                episodes += 1
                fitness_list.append(episode_fitness)
                transistions_list.append(episode_transitions)

        return {individual.index: {"fitness_list": fitness_list, "num_episodes": num_episodes, "num_frames": num_frames,
                                   "id": individual.index, "transitions": transistions_list}}

    def evaluate_population(self, population: List, exploration_noise=False, total_frames=1, pool=None):
        population_id_lookup = [ind.index for ind in population]
        new_population_mean_fitness = np.zeros(len(population))
        new_population_var_fitness = np.zeros(len(population))

        start_phase = total_frames <= self.cfg.rl.start_timesteps
        if start_phase:
            self.log("start phase", time_step=total_frames)

        args_list = [(ind, self.cfg, self.eval_episodes, self.rng.randint(0, 100000), exploration_noise, start_phase)
                     for ind in population]
        result_dicts = [pool.apply(self._evaluate_individual, args) for args in args_list]
        result_dict = dict(ChainMap(*result_dicts))

        eval_frames = 0
        for ind_id, value_dict in result_dict.items():
            pop_idx = population_id_lookup.index(ind_id)
            new_population_mean_fitness[pop_idx] = np.mean(value_dict['fitness_list'])
            new_population_var_fitness[pop_idx] = np.var(value_dict['fitness_list'])
            eval_frames += value_dict['num_frames']

            population[pop_idx].train_log["eval_eps"] = self.eval_episodes

            for transitions in value_dict['transitions']:
                if self.cfg.nevo.ind_memory:
                    population[pop_idx].replay_memory.add(transitions)
                else:
                    self.push_queue.put(transitions)

        for idx in range(len(population)):
            population[idx].train_log["post_fitness"] = new_population_mean_fitness[idx]
            population[idx].train_log["index"] = population[idx].index
            self.log.csv.log_csv(population[idx].train_log)
            population[idx].train_log.update({"pre_fitness": new_population_mean_fitness[idx], "eval_eps": 0})
            population[idx].fitness.append(new_population_mean_fitness[idx])
            if len(population[idx].fitness) > 1:
                population[idx].improvement = population[idx].fitness[-1] - population[idx].fitness[-2]
            else:
                population[idx].improvement = population[idx].fitness[-1]

        return new_population_mean_fitness, new_population_var_fitness, eval_frames


import email
import email.policy
import time
from datetime import datetime, timedelta

from mogo import Model, Field, EnumField, ReferenceField
from twisted.internet import defer

from ..common.db_common import get_db
from ..common.encoding import force_str, force_bytes

class RECIPIENT_STATUS:
    UNVERIFIED         = 'UNVERIFIED'       # Temporary state for existing recipients at satellite startup
    READY              = 'READY'
    FINISHED           = 'FINISHED'
    TIMEOUT            = 'TIMEOUT'
    GENERAL_ERROR      = 'GENERAL_ERROR'
    ERROR              = 'ERROR'
    WARNING            = 'WARNING'
    IN_PROGRESS        = 'IN PROGRESS'

recipient_status = (RECIPIENT_STATUS.UNVERIFIED,
                    RECIPIENT_STATUS.READY,
                    RECIPIENT_STATUS.IN_PROGRESS,
                    RECIPIENT_STATUS.WARNING,
                    RECIPIENT_STATUS.FINISHED,
                    RECIPIENT_STATUS.ERROR,
                    RECIPIENT_STATUS.TIMEOUT,
                    RECIPIENT_STATUS.GENERAL_ERROR,
                    )

class Mailing(Model):
    """Used to store headers and body."""
    # id              = models.IntegerField(primary_key=True)  # Should be the same as mailing_id in Master
    type            = Field()
    testing         = Field(bool, default=False)    # If True, emails are sent to a testing SMTP server instead of the real one.
    backup_customized_emails = Field(bool, default=False)  # If True, customized emails will be included in recipients reports.
    read_tracking   = Field(bool, default=True)     # If True, read tracking image are added to html bodies
    click_tracking  = Field(bool, default=False)    # If True, links found into html bodies are converted to allow clicks tracking
    body_downloaded = Field(bool, default=False)
    header          = Field()
    body            = Field()
    tracking_url    = Field()       # Base url for all tracking links
    deleted         = Field(bool, default=False)    # Mailing deletion requested. Will occur once all its recipients will be removed.
    dkim            = Field()  # dkim settings (dictionary). Fields are enabled (Default=True), selector, domain, privkey
    feedback_loop   = Field()  # Settings needed to implement the Google Feedback Loop requirements (dictionary)
                                # Fields are `campain_id`, `customer_id`, `mail_type_id`, `sender_id`
                                # and `dkim` which contains dkim settings for fbl domain.
                                # defaults are (in order): 'mailing.id', 'mailing.domain_name', 'mailing.type'.
                                # Other are mandatory.
    domain_name     = Field()   # sender domain
    return_path_domain = Field()   # domain used to fill Return-Path header. If None, header won't be added.
    url_encoding    = Field()

    created         = Field(datetime, default=datetime.utcnow)
    modified        = Field(datetime, default=datetime.utcnow)

    _id_type = int

    def __unicode__(self):
        return "Queue[%d]" % self.id

    def save(self, *args, **kwargs):
        if not self._id:
            raise ValueError("Mailing model: An id have to be provided")
            # super(Model, self).__setitem__(self._id_field, Sequence.get_next('mailing_id'))
        self.modified = datetime.utcnow()
        return super(Mailing, self).save(*args, **kwargs)

    def get_message(self):
        """
        Returns the mailing content as email message object.
        """
        mparser = email.parser.BytesFeedParser(policy=email.policy.default)
        mparser.feed(force_bytes(self.header))
        mparser.feed(force_bytes(self.body))
        return mparser.close()

class MailingRecipient(Model):
    # id              = AutoField(primary_key=True)
    # recipient_id     = models.IntegerField(db_index=True)   # ID on master
    mailing         = ReferenceField(Mailing, required=True)
    tracking_id     = Field()
    contact_data    = Field()  # Python dictionary containing all recipient fields
    email           = Field(required=True)
    mail_from       = Field(required=False)
    sender_name     = Field()
    domain_name     = Field(required=True)   # Recipient's domain name
    first_try       = Field(datetime)
    next_try        = Field(datetime)  # TODO maybe a int from EPOCH would be faster
    try_count       = Field(int)
    send_status     = EnumField(recipient_status, default=RECIPIENT_STATUS.READY)
    reply_code      = Field(int)
    reply_enhanced_code = Field()
    reply_text      = Field()
    smtp_log        = Field()
    in_progress     = Field(bool, default=False)  # added in temp queue
    created         = Field(datetime, default=datetime.utcnow)
    modified        = Field(datetime, default=datetime.utcnow)
    finished        = Field(bool, default=False)  # True if this recipient have been handled (successfully or not) and should be returned back to the master.

    def __unicode__(self):
        return self.email

    def save(self, *args, **kwargs):
        self.modified = datetime.utcnow()
        return super(MailingRecipient, self).save(*args, **kwargs)

    def set_send_mail_in_progress(self):
        self.send_status = RECIPIENT_STATUS.IN_PROGRESS
        if self.try_count is None:
            self.try_count = 0
        self.try_count += 1
        self.in_progress = True
        if not self.first_try:
            self.first_try = datetime.utcnow()
        self.next_try = datetime.utcnow()
        self.save()

    def update_send_status(self, send_status, smtp_code=None, smtp_e_code=None, smtp_message=None, in_progress=False,
                           smtp_log=None, target_ip=None):
        """
        Updates the contact status.
        """
        LiveStats.add_log(mailing_id = self.mailing.id, domain_name=self.domain_name,
                          mail_from=self.mail_from, mail_to=self.email,
                          send_status=send_status, reply_code=smtp_code, reply_enhanced_code=smtp_e_code,
                          reply_text=smtp_message, target_ip=target_ip)
        self.send_status = send_status
        if send_status == RECIPIENT_STATUS.FINISHED:
            self.next_try = datetime.utcnow()
        self.reply_code = smtp_code and smtp_code or None
        self.reply_enhanced_code = smtp_e_code and smtp_e_code or None
        self.reply_text = smtp_message and force_str(smtp_message, errors='replace') or None
        self.smtp_log = smtp_log and str(smtp_log, errors='replace') or None
        self.in_progress = in_progress
        self.save()
        #if send_status in (RECIPIENT_STATUS.ERROR, RECIPIENT_STATUS.GENERAL_ERROR):
            #self.contact.status = CONTACT_STATUS.ERROR
            #self.contact.save()
        #elif send_status == RECIPIENT_STATUS.FINISHED:
            #self.contact.status = CONTACT_STATUS.VALID
            #self.contact.save()

    def set_send_mail_next_time(self):
        """
        This implement the mailing specific strategy for retries and mark this entry as removable from queue.
        """
        self.in_progress = False
        if self.try_count < 3:
            self.next_try = datetime.utcnow() + timedelta(minutes=10)
        elif self.try_count < 10:
            self.next_try = datetime.utcnow() + timedelta(minutes=60)
        else:
            self.next_try = datetime.utcnow() + timedelta(hours=6)
        self.finished = True
        self.save()

    def mark_as_finished(self):
        self.finished = True
        self.save()

class HourlyStats(Model):
    date        = Field(datetime)
    epoch_hour  = Field(int)
    sent        = Field(int, default=0)
    failed      = Field(int, default=0)
    tries       = Field(int, default=0)  # Total tentatives count, including sent, failed and temporary failed.
    up_to_date  = Field(bool, default=False)  # If false, this entry needs to be sent to the CloudMaster.

    @staticmethod
    def __generic_update(operations):
        r = HourlyStats.update({'epoch_hour': int(time.time() / 3600)},
                               dict(operations, **{'$set': {'up_to_date': False}}),
                               upsert=True,
                               w=1)
        if r is None or not r['updatedExisting']:
            # entry = HourlyStats.search_or_create(epoch_hour=int(time.time() / 3600))
            entry = HourlyStats.grab(r['upserted'])
            entry.update(date=datetime.utcnow().replace(minute=0, second=0, microsecond=0))

    @staticmethod
    def add_sent():
        HourlyStats.__generic_update({'$inc': {'sent': 1, 'tries': 1}})

    @staticmethod
    def add_failed():
        HourlyStats.__generic_update({'$inc': {'failed': 1, 'tries': 1}})

    @staticmethod
    def add_try():
        HourlyStats.__generic_update({'$inc': {'tries': 1}})

class DomainStats(Model):
    """
    Tracks DNS resolution statistics as well as handled recipients count per domains.
    Also tracks any DNS error on domains resolution. It allows to take decision for domain's recipients if error
    is fatal (no-retry) or not (retry).
    For DNS error, there are two levels:
        - recoverable errors: these errors can be due to temporary network problems. So they should never be fatal for
         concerned recipients.
        - fatal errors: these errors are clearly identified as fatal, i.e. we CAN'T address these recipients in any way
         (for example, the DNS really answer us that the domain doesn't exist). In this case, we will try again later
         (in case in unexpected error with the DNS) but only few times (5 by default).
    """
    domain_name     = Field(required=True)  # Recipient's domain name

    # at recipient level (+1 for each recipient)
    sent        = Field(default=0)
    failed      = Field(default=0)
    tries       = Field(default=0)  # help_text="Total tentatives count, including sent, failed and temporary failed.")

    consecutive_sent = Field(default=0)
    consecutive_failed = Field(default=0)  # including softbounce and hardbounce

    # at DNS level (+1 for each DNS query)
    dns_tries           = Field(default=0)  # help_text="Total tentatives count, including fatal and temporary failed.")
    dns_temp_errors     = Field(default=0)  # help_text="Errors due to temporal circumstances. Its can be solved later.")
    dns_fatal_errors    = Field(default=0)
    dns_last_error      = Field()           # help_text="Full class name of the exception.")

    dns_cumulative_temp_errors  = Field(default=0)  # help_text="This counter is never reset")
    dns_cumulative_fatal_errors = Field(default=0)  # help_text="This counter is never reset")

    created         = Field(datetime, default=datetime.utcnow)
    modified        = Field(datetime, default=datetime.utcnow)

    def save(self, *args, **kwargs):
        self.modified = datetime.utcnow()
        return super(DomainStats, self).save(*args, **kwargs)

    @staticmethod
    def get_exception_fullname(ex):
        return "%s.%s" % (ex.__class__.__module__, ex.__class__.__name__)

    def check_error(self, ex):
        return DomainStats.get_exception_fullname(ex) == self.dns_last_error

    @staticmethod
    def __generic_update(domain, operations):
        operations.setdefault('$set', {})['modified'] = datetime.utcnow()
        r = DomainStats.update({'domain_name': domain},
                               operations,
                               upsert=True)
        # if r is None or not r['updatedExisting']:
        #     entry = DomainStats.search_or_create(domain_name=domain)
        #     kwargs = {}
        #     for key, value in operations['$inc'].items():
        #         kwargs[key] = value
        #     entry.update(date=datetime.utcnow().replace(minute=0, second=0, microsecond=0), **kwargs)

    @staticmethod
    def add_sent(domain):
        DomainStats.__generic_update(domain, {'$inc': {'sent': 1, 'tries': 1, 'consecutive_sent': 1},
                                              '$set': {'consecutive_failed': 0}})

    @staticmethod
    def add_failed(domain):
        DomainStats.__generic_update(domain, {'$inc': {'failed': 1, 'tries': 1, 'consecutive_failed': 1},
                                              '$set': {'consecutive_sent': 0}})

    @staticmethod
    def add_try(domain):
        DomainStats.__generic_update(domain, {'$inc': {'tries': 1, 'consecutive_failed': 1},
                                              '$set': {'consecutive_sent': 0}})

    @staticmethod
    def add_dns_success(domain):
        DomainStats.__generic_update(domain, {'$inc': {'dns_tries': 1},
                                              '$set': {'dns_temp_errors': 0,
                                                       'dns_fatal_errors': 0,
                                                       'dns_last_error': None}})

    @staticmethod
    def add_dns_temp_error(domain, ex):
        DomainStats.__generic_update(domain, {'$inc': {'tries': 1,
                                                       'dns_tries': 1,
                                                       'dns_temp_errors': 1,
                                                       'dns_cumulative_temp_errors': 1},
                                              '$set': {'dns_last_error': DomainStats.get_exception_fullname(ex)}})

    @staticmethod
    def add_dns_fatal_error(domain, ex):
        DomainStats.__generic_update(domain, {'$inc': {'tries': 1,
                                                       'failed': 1,
                                                       'dns_tries': 1,
                                                       'dns_fatal_errors': 1,
                                                       'dns_cumulative_fatal_errors': 1},
                                              '$set': {'dns_last_error': DomainStats.get_exception_fullname(ex)}})

    @staticmethod
    def get_domains_notation() -> dict:
        pipeline = [{
            "$match": {"$or": [{"consecutive_sent": {"$gte": 0}}, {"consecutive_failed": {"$gte": 0}}]}
        }, {
            "$addFields": {
                "age_hours": {
                    "$divide": [{
                        "$subtract": [datetime.utcnow(), "$modified"]
                    }, 3600000]
                }
            }
        }, {
            "$project": {
                "domain_name":        1,
                "age_hours":          1,
                "consecutive_sent":   1,
                "consecutive_failed": 1,
                "note":               {
                    "$divide": [{
                        "$subtract": ["$consecutive_sent",
                                      {"$subtract": [{"$exp": {"$min": ["$consecutive_failed", 5]}}, 1]}]
                    }, {
                        "$max": [0.1, "$age_hours"]
                    }]
                }
            }
        }, {
            "$sort": {
                "domain_name": 1
            }
        }]
        results = DomainStats._get_collection().aggregate(pipeline)
        return {r['domain_name']: r['note'] for r in results}

class DomainConfiguration(Model):
    domain_name = Field(required=True)
#   - 'active_relayers': list of active relayers for this domain
    mx_servers = Field()  # list of MX's IPs addresses
    mx_last_updated = Field(datetime)   # datetime of the last update of MX list
    max_relayers = Field(int)   # the maximum simultaneous relayers for this domain (taken from DB
                                # or None to use the default rule = half of the MX servers)
    cnx_per_mx = Field(int)     # the maximum simultaneous connections per MX server
    max_mx = Field(int)         # the maximum simultaneous connected MX servers

class ActiveQueue(Model):
    domain_name = Field(required=True)
    recipients = Field()
    created = Field(datetime, default=datetime.utcnow)

class LiveStats():
    """
    Register all tries to allow to compute real time statistics on errors, success, softbounces, etc...
    Statistics should be by domain, by MX and by mailing
    """
    # mailing_id      = Field(int)
    # domain_name     = Field()
    # ip              = Field()
    # mail_from       = Field()
    # recipient       = Field()
    # date            = Field()
    # send_status     = EnumField(recipient_status)
    # reply_code      = Field(int)
    # reply_enhanced_code = Field()
    # reply_text      = Field()

    @staticmethod
    def add_log(mailing_id, domain_name, mail_from, mail_to, send_status, reply_code, reply_enhanced_code, reply_text,
                target_ip):
        db = get_db()
        now = datetime.utcnow()
        l = [
            db.live_stats.insert_one({'date': now,
                                      'mailing_id': mailing_id, 'domain_name': domain_name, 'ip': target_ip,
                                      'mail_from': mail_from, 'mail_to': mail_to, 'send_status': send_status,
                                      'reply_code': reply_code, 'reply_enhanced_code': reply_enhanced_code,
                                      'reply_text': reply_text}),
            db.live_stats2.insert_one({'date': now,
                                       'mailing_id': mailing_id, 'domain_name': domain_name, 'ip': target_ip,
                                       'mail_from': mail_from, 'mail_to': mail_to, 'send_status': send_status,
                                       'reply_code': reply_code, 'reply_enhanced_code': reply_enhanced_code,
                                       'reply_text': reply_text})
        ]

        return defer.DeferredList(l)


from django.shortcuts import render, redirect
from crud_operations.forms import EmployeeForm
from crud_operations.models import Employee

# Create your views here.
def index(request):
    all_emp = Employee.objects.all()
    return render(request,'index.html',{'data':all_emp})

def create(request):
    if request.method == 'POST':
        form = EmployeeForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('index')
    else:
        form = EmployeeForm()
    context = {
        'form': form
    }
    return render(request,'create.html',context)

def update(request, id):
    emp = Employee.objects.get(id=id)
    if request.method == "POST":
        form = EmployeeForm(request.POST, instance=emp)
        if form.is_valid():
            form.save()
            return redirect('index')
        return redirect('update')
    else:
        form = EmployeeForm
    context = {
        'form': form
    }
    return render(request,'update.html',context)

def delete(request, id):
    emp = Employee.objects.get(id=id)
    emp.delete()
    return redirect('index')
    #return render(request,'delete.html')


import json
from collections import Counter

class UMLS_KB(object):

    def __init__(self, umls_version):
        self.umls_data = None
        self.umls_version = umls_version

        self.load(umls_version)

    def load(self, umls_version):
        json_path = 'data/UMLS/%s.json' % umls_version
        with open(json_path, 'r') as json_f:
            self.umls_data = json.load(json_f)

    def get_sts(self, cui):
        return self.umls_data[cui]['STY']

    def get_aliases(self, cui, include_name=True):
        aliases = self.umls_data[cui]['STR']
        if include_name:
            aliases.append(self.umls_data[cui]['Name'])

        return aliases

    def get_all_cuis(self):
        #
        return list(self.umls_data.keys())

    def get_all_stys(self):
        #
        all_stys = set()
        for cui in self.get_all_cuis():
            for sty in self.get_sts(cui):
                all_stys.add(sty)
        return list(all_stys)

    def get_sty_sizes(self):
        #
        sty_sizes = Counter()
        for cui in self.get_all_cuis():
            for sty in self.get_sts(cui):
                sty_sizes[sty] += 1
        return list(sty_sizes.most_common())

    def pprint(self, cui):
        cui_info = self.umls_data[cui]
        s = ''
        s += 'CUI: %s Name: %s\n' % (cui, cui_info['Name'])
        # s += 'Definition: %s\n' % '; '.join(cui_info['DEF'])
        s += 'Aliases (%d): %s\n' % (len(cui_info['STR']), '; '.join(cui_info['STR'][:5]))
        s += 'Types: %s\n' % '; '.join(cui_info['STY'])
        print(s)

umls_kb_st21pv = UMLS_KB('umls.2017AA.active.st21pv')
umls_kb_full = UMLS_KB('umls.2017AA.active.full')

if __name__ == '__main__':
    umls_kb_st21pv.pprint('C0001097')