# -*- coding: utf-8 -*-

import os
import time
import logging
import numpy as np
import jieba
from gensim.models import KeyedVectors
import sys

file_root = os.path.dirname(__file__)
sys.path.append(file_root)
from get_file import get_file

pwd_path = os.path.abspath(os.path.dirname(__file__))
USER_DATA_DIR = pwd_path
os.makedirs(USER_DATA_DIR, exist_ok=True)

def load_stopwords(file_path):
    stopwords = set()
    if file_path and os.path.exists(file_path):
        with open(file_path, 'r', encoding='utf-8') as f:
            for line in f:
                line = line.strip()
                stopwords.add(line)
    return stopwords

class Tokenizer(object):
    def __init__(self, dict_path='', custom_word_freq_dict=None):
        self.model = jieba
        self.model.default_logger.setLevel(logging.ERROR)
        # 初始化大词典
        if os.path.exists(dict_path):
            self.model.set_dictionary(dict_path)
        # 加载用户自定义词典
        if custom_word_freq_dict:
            for w, f in custom_word_freq_dict.items():
                self.model.add_word(w, freq=f)

    def tokenize(self, sentence, cut_all=False, HMM=True):
        """
        切词并返回切词位置
        :param sentence: 句子
        :param cut_all: 全模式，默认关闭
        :param HMM: 是否打开NER识别，默认打开
        :return:  A list of strings.
        """
        return self.model.lcut(sentence, cut_all=cut_all, HMM=HMM)

class Word2Vec(object):
    """Pre-trained word2vec embedding"""
    model_key_map = {
        # 腾讯词向量, 6.78G
        'w2v-tencent-chinese': {
            'tar_filename': 'Tencent_AILab_ChineseEmbedding.tar.gz',
            'url': 'https://ai.tencent.com/ailab/nlp/data/Tencent_AILab_ChineseEmbedding.tar.gz',
            'binary': False,
            'untar_filename': 'Tencent_AILab_ChineseEmbedding.txt'},
        # 轻量版腾讯词向量，二进制，111MB
        'w2v-light-tencent-chinese': {
            'tar_filename': 'light_Tencent_AILab_ChineseEmbedding.bin',
            'url': 'http://42.193.145.218/light_Tencent_AILab_ChineseEmbedding.bin',
            'binary': True,
            'untar_filename': 'light_Tencent_AILab_ChineseEmbedding.bin'},
    }

    def __init__(self, model_name_or_path='w2v-light-tencent-chinese',
                 w2v_kwargs={},
                 stopwords_file=os.path.join(pwd_path, 'data/stopwords.txt'),
                 cache_folder=USER_DATA_DIR):
        """
        Init word2vec model

        Args:
            model_name_or_path: word2vec file path
            w2v_kwargs: params pass to the ``load_word2vec_format()`` function of ``gensim.models.KeyedVectors`` -
                https://radimrehurek.com/gensim/models/keyedvectors.html#module-gensim.models.keyedvectors
            stopwords_file: str, stopwords
            cache_folder: str, save model
        """
        self.w2v_kwargs = w2v_kwargs
        if model_name_or_path and os.path.exists(model_name_or_path):
            model_path = model_name_or_path
        else:
            model_dict = self.model_key_map.get(model_name_or_path, self.model_key_map['w2v-light-tencent-chinese'])
            tar_filename = model_dict.get('tar_filename')
            self.w2v_kwargs.update({'binary': model_dict.get('binary')})
            url = model_dict.get('url')
            untar_filename = model_dict.get('untar_filename')
            model_path = os.path.join(cache_folder, untar_filename)
            if not os.path.exists(model_path):
                get_file(tar_filename, url, extract=True,
                         cache_dir=USER_DATA_DIR,
                         cache_subdir=USER_DATA_DIR,
                         verbose=1)
        t0 = time.time()
        w2v = KeyedVectors.load_word2vec_format(model_path, **self.w2v_kwargs)
        # w2v.init_sims(replace=True)
        logging.debug('Load w2v from {}, spend {:.2f} sec'.format(model_name_or_path, time.time() - t0))
        self.stopwords = load_stopwords(stopwords_file)
        self.w2v = w2v
        self.tokenizer = Tokenizer()
        logging.debug('Word count: {}, emb size: {}'.format(len(w2v.index2word), w2v.vector_size))
        logging.debug('Set stopwords: {}, count: {}'.format(sorted(list(self.stopwords))[:10], len(self.stopwords)))

    def encode(self, sentences, dims=200):
        """
        Encode embed sentences

        Args:
            sentences: Sentence list to embed
        Returns:
            vectorized sentence list
        """
        if self.w2v is None:
            raise ValueError('need to build model for embed sentence')

        input_is_string = False
        if isinstance(sentences, str) or not hasattr(sentences, '__len__'):
            sentences = [sentences]
            input_is_string = True

        all_embeddings = []
        for sentence in sentences:
            emb = []
            count = 0
            for word in sentence:
                # 过滤停用词
                if word in self.stopwords:
                    continue
                # 调用词向量
                if word in self.w2v.index2word:
                    emb.append(self.w2v.get_vector(word))
                    count += 1
                else:
                    if len(word) == 1:
                        continue
                    # 再切分，eg 特价机票
                    ws = self.tokenizer.tokenize(word, cut_all=True)
                    for w in ws:
                        if w in self.w2v.index2word:
                            emb.append(self.w2v.get_vector(w))
                            count += 1
            tensor_x = np.array(emb).sum(axis=0)  # 纵轴相加
            if count > 0:
                avg_tensor_x = np.divide(tensor_x, count)
            else:
                avg_tensor_x = np.zeros(dims, dtype=np.float32)
            all_embeddings.append(avg_tensor_x)
        all_embeddings = np.array(all_embeddings)
        if input_is_string:
            all_embeddings = all_embeddings[0]

        return all_embeddings

if __name__ == '__main__':
    w2v = Word2Vec(model_name_or_path=os.path.join(file_root, '../../transformers/word2vec/light_Tencent_AILab_ChineseEmbedding.bin'),
                   w2v_kwargs={"binary":True})


# -*- coding: utf-8 -*-
# @Time : 2022/3/5 12:14
# @Author: Shelly Tang
# @File: data.py
# @Function: data.py, deal with the dataset

import glob
import pandas as pd
import random
import csv
import os
import numpy as np
from sklearn.metrics import confusion_matrix

KeyContentLen = 512
SplitDataN = 2
PART_NUM = 10

class Data(object):
    def __init__(self, file_path, label_path):
        # org数据集包括空数据
        self.org_dataset = self.getContent(file_path)
        self.org_label = self.getLabel(label_path)
        # 剔除空数据后：
        self.dataset = self.processData2()
        # shuffle, splitdata
        self.shuffleData()      # shuffle
        self.writeDataset()     # output to csv

    @staticmethod
    def getContent(file_path):
        """
        获得原始对话数据
        :param file_path: 文件夹位置（文件已改名）
        :return: dataset[N, data], data[n_line, content]
        """
        dataset = []
        file_number = len(glob.glob(pathname=file_path+'/*.txt'))  # 获取filepath文件夹下的文件个数
        for i in range(file_number):
            file_name = file_path+"/"+str(i+1)+".txt"
            with open(file_name, "r", encoding='utf-8') as f:
                data = []
                for line in f.readlines():
                    line = line.strip('\n')
                    data.append(line)
                dataset.append(data)
        return dataset

    @staticmethod
    def getLabel(label_path):
        label = pd.read_csv(label_path, usecols=['label'])      # 按列读取csv
        return list(label['label'])     # pd对象转list返回

    @staticmethod
    def getKeyContent(content):
        """
        只是取合并customer和user之后的最后K个字输入
        :param content: 字符串，整个数据的所有内容（连在一起）
        :return: 最后的K个字
        """
        result = ""
        if len(content) > KeyContentLen:
            result = content[-KeyContentLen:]
        else:
            result = content
        return result

    @staticmethod
    def getKeyContent2(content):
        """
        取最后K个字作为关键内容，且分开角色
        :param content: [[content, role="user"/"customer"]]
        :return: @content_user:str, @content_customer:str, 分别是K范围内concat在一起的内容
        """
        temp = 0
        i = 1
        user_content = ""
        cus_content = ""
        while i < len(content)+1 and temp+len(content[-i][0]) <= KeyContentLen:
            if content[-i][1] == "user":
                user_content = content[-i][0] + user_content
            else:
                cus_content = content[-i][0] + cus_content
            i += 1
        return user_content, cus_content

    def processData(self):
        """
        处理数据集，如删掉前面的user, customer和时间，然后获取关键内容（如最后的K个字）
        :return: processed dataset
        """
        dataset = []
        for i in range(len(self.org_dataset)):
            data = self.org_dataset[i]
            new_data_content = ""
            if len(data) == 0:
                continue
            for j in range(len(data)):
                line = data[j]
                content = line.split(',')       # 只取具体内容，忽略角色和时间
                new_data_content = new_data_content + content[2]        # 先将全部连成一个字符串
                if len(content) > 3:        # 报错处理，看content有无英文逗号
                    print("Error: English comma exists in content.")

            key_content = self.getKeyContent(new_data_content)
            new_data = [i+1, new_data_content, str(self.org_label[i]), key_content]
            dataset.append(new_data)
        return dataset

    def processData2(self):
        """
        处理数据集，如删掉前面的user, customer和时间，然后获取关键内容（如最后的K个字），且要将user和customer分开，上限为K
        :return: processed dataset
        """
        dataset = []
        for i in range(len(self.org_dataset)):
            data = self.org_dataset[i]
            new_data = []       # [[content, role="user"/"customer"]]
            if len(data) == 0:
                continue
            for j in range(len(data)):
                line = data[j]
                content = line.split(',')       # 只取具体内容，忽略角色和时间
                if len(content) > 3:        # 报错处理，看content有无英文逗号
                    print("Error: English comma exists in content.")
                if content[0][:4] == "user":
                    new_data.append([content[2],"user"])
                else:
                    new_data.append([content[2], "customer"])

            user_key_content, cus_key_content = self.getKeyContent2(new_data)
            new_data = [i+1, str(self.org_label[i]), user_key_content, cus_key_content]
            dataset.append(new_data)
        return dataset

    def shuffleData(self):
        """
        将数据集[index, content, label]进行shuffle
        :return: shuffle后的数据集
        """
        random.shuffle(self.dataset)

    def writeDataset(self):
        with open("./dataset.csv", "w", newline="") as f:
            csvwriter = csv.writer(f)
            csvwriter.writerow(["fileName", "label", "user_content", "cus_content"])
            csvwriter.writerows(self.dataset)

    def k_fold(self, k, i):
        """
        将dataset分成train set和dev set
        :return: train set, dev set
        """
        total = len(self.dataset)
        step = total // k   # 步长
        start = i*step
        end = start + step
        train_set = self.dataset[:start] + self.dataset[end:]
        dev_set = self.dataset[start:end]
        return train_set, dev_set

class Data2(object):
    # multi-label class data
    def __init__(self, data_dir, full_names, names):
        # 构建原dataset(without label)
        # file_path = os.path.join(data_dir, "text_punc")
        # self.org_dataset = self.getContent(file_path)
        # self.dataset = self.processData()
        # self.writeDataset(self.dataset, full_names)     # output to csv

        self.data_dir = data_dir
        self.full_names = full_names
        self.names = names

    @staticmethod
    def getContent(file_path):
        """
        获得原始对话数据
        :param file_path: 文件夹位置（文件已改名）
        :return: dataset[N, data], data[n_line, content]
        """
        dataset = []
        file_number = len(glob.glob(pathname=file_path + '/*.txt'))  # 获取filepath文件夹下的文件个数
        for i in range(file_number):
            file_name = file_path + "/" + str(i + 1) + ".txt"
            with open(file_name, "r", encoding='utf-8') as f:
                data = []
                for line in f.readlines():
                    line = line.strip('\n')
                    data.append(line)
                dataset.append(data)
        return dataset

    @staticmethod
    def splitDataContent(content):
        """
        将一条数据分割成不同的数据（取偶数次的K个字作为一条新数据）
        例如：本content含有3段KeyContentLen个字对的分段（0,1,2)，则取0,2返回（后序可以调整取得段数）
        :param content: list of [content, role="user"/"customer", total_data_len], total_data_len是到该行的字数累加之和
        :return: @content_user:str, @content_customer:str, 分别是K范围内concat在一起的内容
        """
        total_data_len = content[-1][2]
        k_period_count = total_data_len // KeyContentLen
        k_period_count = k_period_count + 1 if total_data_len%KeyContentLen != 0 else k_period_count
        # print("Shelly Tang: total_data_len is %d, k_period_count is %d" %(total_data_len, k_period_count))

        role_to_index = {"user": 0, "customer": 1}
        result_data_content = []    # list of [起始行号,结束行号,user_content,customer_content]
        cur_content = ["", ""]
        cur_data_start, cur_data_end = 0, KeyContentLen     # 表示当前数据的起始/结束字数的序号
        is_start_flag = False                               # 是否是当前数据所取的第一行
        cur_line_start, cur_line_end = 0, 0                 # 表示当前数据的起始/结束行号的序号

        for line_index in range(len(content)):
            if cur_data_start >= total_data_len:
                # print("Shelly Tang:1")
                break
            line = content[line_index]
            if cur_data_start < line[2] < cur_data_end:     # if处于范围内，将其纳入当前新数据中
                role_index = role_to_index[line[1]]
                cur_content[role_index] += line[0]
                if not is_start_flag:
                    cur_line_start = line_index
                    is_start_flag = True
            elif line[2] >= cur_data_end:                   # 这种情况就是，夸(n, n+k)的后界，但不夸前界
                role_index = role_to_index[line[1]]         # PS第一次大于也纳入当前新数据（后面是会truncate的）
                cur_content[role_index] += line[0]
                cur_line_end = line_index

                result_data_content.append([cur_line_start, cur_line_end, cur_content[0], cur_content[1]])

                is_start_flag = False
                cur_content = ["", ""]
                cur_data_start += KeyContentLen*SplitDataN
                cur_data_end += KeyContentLen*SplitDataN
        if is_start_flag and (len(cur_content[0]) != 0 or len(cur_content[1]) != 0):
            result_data_content.append([cur_line_start, len(content)-1, cur_content[0], cur_content[1]])
        return result_data_content, k_period_count

    def processData(self):
        """
        处理数据集，如删掉前面的user, customer和时间，然后将其中的每K个字作为一条新数据，且要将user和customer分开，上限为K
        :return: processed dataset
        """
        dataset = []
        for file_index in range(len(self.org_dataset)):
        # for file_index in range(1):
            data = self.org_dataset[file_index]
            new_data = []       # [[content, role="user"/"customer"]]
            total_data_len = 0  # 计算文本总长度
            if len(data) == 0:
                continue
            for j in range(len(data)):
                line = data[j]
                content = line.split(',')       # 只取具体内容，忽略角色和时间
                total_data_len += len(content[2])       # 到j文本时的字数累加之和

                if len(content) > 3:        # 报错处理，看content有无英文逗号
                    print("Error: English comma exists in content.")
                if content[0][:4] == "user":
                    new_data.append([content[2], "user", total_data_len])
                else:
                    new_data.append([content[2], "customer", total_data_len])
            # 将当前data分成若干个K个字的片段
            result_data_content, k_period_count = self.splitDataContent(new_data)

            for k_period_index in range(len(result_data_content)):
                cur_k_period = result_data_content[k_period_index]
                cur_data = [file_index+1, k_period_count, 2*k_period_index+1, cur_k_period[0]+1, cur_k_period[1]+1,
                            cur_k_period[2], cur_k_period[3]]
                dataset.append(cur_data)
        return dataset

    def shuffleData(self):
        """
        将数据集[index, content, label]进行shuffle
        :return: shuffle后的数据集
        """
        random.shuffle(self.dataset)

    def writeDataset(self, filepath, dataset, names=None):
        # filepath = os.path.join(self.data_dir, filename)
        with open(filepath, "w", newline="", encoding="gbk") as f:
            csvwriter = csv.writer(f)
            if names != None:
                csvwriter.writerow(names)
            csvwriter.writerows(dataset)

    def readDataset(self, filename, names=None):
        filepath = os.path.join(self.data_dir, filename)
        if names == None:
            lines = pd.read_csv(filepath, encoding='gbk')
        else:
            lines = pd.read_csv(filepath, usecols=names, encoding="gbk")
        pd.set_option('display.max_columns', 50)

        lines = lines.values.tolist()
        for (index, line) in enumerate(lines):
            if pd.isnull(lines[index][1]):
                lines[index][1] = ""
            if pd.isnull(lines[index][2]):
                lines[index][2] = ""
        return lines

    def buildDataset(self, filename, start, fold_size=160):
        # filepath = os.path.join(self.data_dir, filename)
        lines = self.readDataset(filename, names=self.names)
        temp_lines = lines[:360]
        random.seed(42)
        random.shuffle(temp_lines)
        for (line_idx, line) in enumerate(temp_lines):
            for (label_idx, label) in enumerate(line[3:]):
                line[label_idx + 3] = str(int(float(label)))

        # # 第一次才要train, valid，且从新的一个标了数据的文件里面划分
        # valid_set = temp_lines[:200]
        # train_set = temp_lines[200:360]
        # valid_path = os.path.join(self.data_dir, "eval.csv")
        # train_path = os.path.join(self.data_dir, "train.csv")
        # self.writeDataset(valid_path, valid_set, self.names)
        # self.writeDataset(train_path, train_set, self.names)

        names = ["file_name", "user_content", "cus_content"]
        lines = self.readDataset("dataset.csv", names=names)
        test_set = lines[start:start + fold_size]
        filepath = os.path.join(self.data_dir, "test.csv")
        self.writeDataset(filepath, test_set, self.names)

    def writeTestResult(self, test_result_fn, test_fn, label_names, names):
        test_df = self.readDataset(test_fn)
        # test_result_df = self.readDataset(test_result_fn)
        test_result_file = os.path.join(self.data_dir, test_result_fn)
        label2id = {label: i+3 for (i, label) in enumerate(label_names)}

        with open(test_result_file, "r", encoding='gbk') as f:
            lines = csv.reader(f)
            # lines = lines
            for (line_index, line) in enumerate(lines):
                for label_pair in line:
                    label = label_pair.split(":")[0]
                    label_index = label2id[label]
                    value = label_pair.split(":")[1]
                    if pd.isnull(test_df[line_index][label_index]):
                        test_df[line_index][label_index] = value
                    else:
                        test_df[line_index][label_index] += "," + value

        self.writeDataset("test_2_new.csv", test_df, names=names)

    def writeTestResult2(self, test_result_arr, label_names, filepath):
        test_df = self.readDataset("test.csv")

        label2id = {label: i+3 for (i, label) in enumerate(label_names)}

        for (line_index, line) in enumerate(test_result_arr):
            for label_pair in line:
                label = label_pair.split(":")[0]
                label_index = label2id[label]
                value = label_pair.split(":")[1]
                if pd.isnull(test_df[line_index][label_index]):
                    test_df[line_index][label_index] = str(value)
                else:
                    test_df[line_index][label_index] += "," + str(value)

        self.writeDataset(filepath, test_df, names=self.names)

    def writeEvalResult(self, test_result_arr, label_names, filepath):
        # test_df = self.readDataset("eval.csv")

        label2id = {label: i for (i, label) in enumerate(label_names)}

        result = []
        for (line_index, line) in enumerate(test_result_arr):
            temp = [""]* len(label_names)
            for label_pair in line:
                label = label_pair.split(":")[0]
                label_index = label2id[label]
                value = label_pair.split(":")[1]
                if len(temp[label_index]) == 0:
                    temp[label_index] = value
                else:
                    temp[label_index] += "," + value
                # test_df[line_index][label_index] = value
            result.append(temp)
        self.writeDataset(filepath, result, names=label_names)

    def get_confusion_matrix(self, outputs, targets, labels):
        confusion_matrix1 = confusion_matrix(targets, outputs)
        # report = classification_report(targets, outputs, target_names=labels)
        return confusion_matrix1

class Data3(object):
    # active learning in standard single-label dataset
    def __init__(self, data_dir, org_data_path):
        self.data_dir = data_dir
        # self.org_data_path = org_data_path

        train_lines = self.readTxt("train.txt", 1400)
        eval_lines = self.readTxt("val.txt", 600)
        self.buildOrgDataset(train_lines, "dataset")
        self.buildOrgDataset(eval_lines, "eval")

        self.buildDataset("dataset.csv", 200, 200)

    def readDataset(self, filename, names=None):
        filepath = os.path.join(self.data_dir, filename)
        if names == None:
            lines = pd.read_csv(filepath, encoding='utf-8')
        else:
            lines = pd.read_csv(filepath, usecols=names, encoding="utf-8")
        pd.set_option('display.max_columns', 50)

        lines = lines.values.tolist()
        for (index, line) in enumerate(lines):
            if pd.isnull(lines[index][1]):
                lines[index][1] = ""
            if pd.isnull(lines[index][2]):
                lines[index][2] = ""
        return lines

    def readTxt(self, filename, line_num):
        filepath = os.path.join(self.data_dir, filename)
        print(filepath)
        with open(filepath, 'r', encoding='utf-8') as f:
            lines = f.readlines()
        random.seed(42)
        random.shuffle(lines)
        return lines[:line_num]

    def buildOrgDataset(self, lines, type):
        # train maxlen = 15271, avg len=897
        # eval maxlen = 6164, avg len = 921
        result = []
        max_len = 0
        total_len = 0
        for (index, line) in enumerate(lines):
            line = line.strip().strip('\n')
            content = line.split('\t')[1]
            label = line.split('\t')[0]
            result.append([index, content, label])
            if len(content)>max_len:
                max_len = len(content)
            total_len += len(content)
        print("max len: %d" %(max_len))
        print("avg len: %f" %(total_len/len(result)))
        self.writeDataset(result, os.path.join(self.data_dir, type+".csv"))

    def buildDataset(self, filename, start, fold_size=160):
        # filepath = os.path.join(self.data_dir, filename)
        lines = self.readDataset(filename)

        # # 第一次才要train，且从新的一个标了数据的文件里面划分
        train_set = lines[:fold_size]
        self.writeDataset(train_set, os.path.join(self.data_dir, "train.csv"))

        test_set = lines[start:start + fold_size]
        self.writeDataset(test_set, os.path.join(self.data_dir, "test.csv"))

    def writeDataset(self, dataset, file_name):
        with open(file_name, "w", newline="", encoding="utf-8") as f:
            csvwriter = csv.writer(f)
            csvwriter.writerow(["Index", "Content", "Label"])
            csvwriter.writerows(dataset)

    @staticmethod
    def compare_test(filename):
        diff_count = 0
        index_arr = []
        with open(filename, "r", encoding="utf-8") as f:
            lines = csv.reader(f)
            for (index,line) in enumerate(lines):
                if line[0]!=line[1]:
                    diff_count += 1
                    index_arr.append(index)
        print(diff_count)
        print(index_arr)

    def update_train_dataset(self, test_filename, train_filename, part_index):
        test_file = os.path.join(self.data_dir, test_filename)
        train_file = os.path.join(self.data_dir, train_filename)
        test_result = []
        with open(test_file, "r", encoding="utf-8") as f:
            lines = csv.reader(f)
            lines = list(lines)
            total_count = len(lines)
            print("test total_count=%d" %total_count)
            for (index, line) in enumerate(lines):
                if index < int(0.1*total_count):
                    test_result.append(line[1])
                else:
                    test_result.append(line[0])

        lines = []
        with open(train_file, "r", encoding="utf-8") as f:
            lines = csv.reader(f)
            # for line in lines:
            #     print(line)
            lines = list(lines)
            print(len(lines))
            part_size = int(len(lines) / PART_NUM)
            print(part_size)
            for (index, test_data) in enumerate(test_result):
                lines[part_size*(part_index-1)+index][2] = test_data
        with open(train_file, "w", encoding="utf-8", newline="") as f:
            writer = csv.writer(f)
            writer.writerows(lines)

if __name__ == '__main__':
    full_names = ["file_name", "period_count", "period_index", "start_line_index","end_line_index","user_content", "cus_content",
                  "is_fluent", "is_normal","stage", "category", "education", "pension", "health", "invest", "deposit"]

    names = ["file_name", "user_content", "cus_content", "is_fluent", "is_normal", "category"]
    label_names = ["is_fluent", "is_normal", "category"]


from __future__ import division
import os
import numpy as np
from scipy import stats
from osgeo import gdal,ogr
import configparser
import inspect
import sys
import pandas as pd

from GeoFlood_Filename_Finder import cfg_finder
from time import perf_counter
from scipy.stats import gmean,theilslopes

def river_attribute_estimation(segment_shp, segcatfn,
                               segcat_shp, burndemfn,
                               attribute_txt):
    rafile = open(attribute_txt,"w")
    driver = ogr.GetDriverByName("ESRI Shapefile")
    dataSource = driver.Open(segment_shp, 0)
    layer = dataSource.GetLayer()
    srs = layer.GetSpatialRef()
    featureCount = layer.GetFeatureCount()
    rafile.write(str(featureCount)+"\n")
    raster = gdal.Open(burndemfn)
    gt = raster.GetGeoTransform()
    rasterBand = raster.GetRasterBand(1)
    cat_raster = gdal.Open(segcatfn)
    cat_rasterBand = cat_raster.GetRasterBand(1)
##    raster_arr = np.array(cat_rasterBand.ReadAsArray())
##    nodata = -32768
##    nodatamask = raster_arr == nodata
##    mask = cat_rasterBand.GetMaskBand().ReadAsArray()
    mask = cat_rasterBand.GetMaskBand()
    ds = driver.CreateDataSource(segcat_shp)
    cat_layer = ds.CreateLayer(segcat_shp, srs)
    field = ogr.FieldDefn("HYDROID", ogr.OFTInteger)
    cat_layer.CreateField(field)
    dst_field = cat_layer.GetLayerDefn().GetFieldIndex("HYDROID")
    gdal.Polygonize(cat_rasterBand, mask, cat_layer, dst_field,
                    [], callback=None)
    ds.Destroy()
    ds = driver.Open(segcat_shp, 1)
    cat_layer = ds.GetLayer()
    schema=[]
    ldefn=cat_layer.GetLayerDefn()
    for n in range(ldefn.GetFieldCount()):
        fdefn = ldefn.GetFieldDefn(n)
        schema.append(fdefn.name)
    cat_layer.CreateField(ogr.FieldDefn('AreaSqKm', ogr.OFTReal))
    new_counter = 0
    for feat in cat_layer:
        geom = feat.GetGeometryRef()
        feat.SetField('AreaSqKm',float(geom.Area())/1000**2)
        cat_layer.SetFeature(feat)
        feat.Destroy()
        new_counter+=1

    # Initialize Counter for the for loop iterations
    ac_iter = 0

    # Initialize lists to hold each iterations z and m values. Each loop will append to the list.
    z_array_du = []
    m_array_du = []
    slope_array_du = []
    alec_int = 0
    mx_first=[]
    mx_last=[]
    temp_slope_list = []
    feature_id_list=[]
    print(f'Total Segments: {len(layer)}')
    for feature in layer:
        geom = feature.GetGeometryRef()
        feat_id = feature.GetField('HYDROID')
        length = feature.GetField('Length')/1000
        point_geom_list = geom.GetPoints()
        mx = np.array([])
        my = np.array([])
        for i in range(len(point_geom_list)):
            mx = np.append(mx, point_geom_list[i][0])
            my = np.append(my, point_geom_list[i][1])
        mx_first.append(mx[0]) # Used to check if previous stream is actually upstream from current one
        mx_last.append(mx[-1])
        px = ((mx - gt[0]) / gt[1]).astype(int)
        py = ((my - gt[3]) / gt[5]).astype(int)
        x_diff = np.diff(px)*gt[1]
        y_diff = np.diff(py)*gt[1]
        m_array = np.sqrt(x_diff**2 + y_diff**2)
        m_array = np.insert(m_array, 0, 0)
        m_array = np.cumsum(m_array)
        z_array = rasterBand.ReadAsArray()[py, px].flatten()
        # Gets rid of geometries that are empty
        if np.sum(m_array) == 0:
            print('Empty Geometry Encountered')
            continue

        L_10 = int(len(m_array)*.1)
        L_85 = int(len(m_array)*.85)
        slope = (z_array[L_10]-z_array[L_85])/((m_array[L_85]-m_array[L_10]))
        temp_slope_list.append(slope)

        # Append the current iteration to the du/container array.
        m_array_du.append(m_array)
        # Convert to a numpy array of arrays instead of a list of arrays. This allows it to be indexed numerically.
        m_array_du_np = np.asarray(m_array_du, dtype=object)

        # Same instructions used for the 'm_array_du' variable are done in the follwing two lines for z.
        z_array_du.append(z_array)
        z_array_du_np = np.asarray(z_array_du, dtype=object)

        #########################################################################################################
        # As done with the m and z values, append each new slope iteration to the 'slope_array_du' array
        # and convert it to a numpy array. This will be used for indexing slope values.

        slope_array_du.append(slope)
        slope_array_du_np = np.asarray(slope_array_du)

        ###### Correcting Negative and Hydroflattened Slopes: Cycles through previous reaches until the regression slope is positive.
        ###### A value of 0.000001 was chosen as the cut off as not all hydroflattened reaches have a slope of exactly zero, i.e. 9x10^-9.
        ###### As a result, any value below the threshold chosen will recompute the regression line with its upstream reaches until the
        ###### slope becomes positive or there are no more reaches to cycle through. If the latter occurs, a slope of 0.00001 is assigned.

        if (slope<=0.0000001) and (ac_iter==0):
           slope=.00001

        subtraction_iter = 1
        a1_count=0
        prev_check=0
        gmean_check=0
        if (slope<=0.0) and (ac_iter>=1):
            previous_reach_index=ac_iter-subtraction_iter # Just for indexing purposes
            while (slope <= 0.000001) and (mx_first[ac_iter]==mx_last[previous_reach_index]) and (subtraction_iter<=3):

                previous_reach_index = ac_iter - subtraction_iter
                prior_length=[]
                prior_array_length=[]
                for i in range(subtraction_iter):
                    prior_length.append(m_array_du_np[previous_reach_index+i][-1])
                    prior_array_length.append(np.size(m_array_du_np[previous_reach_index+i]))

                z_array_test = np.concatenate(z_array_du_np[previous_reach_index:])
                m_array_test = np.concatenate(m_array_du_np[previous_reach_index:])
                for i in range(len(prior_length)):
                    if len(prior_length)<2:
                        m_array_test[prior_array_length[i]:]=m_array_test[prior_array_length[i]:]+prior_length
                    else:
                        if (i+1)<len(prior_length):
                            m_array_test[prior_array_length[i+1]:(prior_array_length[i]+prior_array_length[i+1])]=m_array_test[prior_array_length[i+1]:(prior_array_length[i]+prior_array_length[i+1])]+prior_length[i+1]

                        else:
                            m_array_test[np.sum(prior_array_length):]=m_array_test[np.sum(prior_array_length):]+prior_length[0]
                L_negative_10 = int(np.size(m_array_test)*.1)
                L_negative_85 = int(np.size(m_array_test)*.85)
                slope = (z_array_test[L_negative_10]-z_array_test[L_negative_85])/(m_array_test[L_negative_85]-m_array_test[L_negative_10])
                if (slope>0):
                    prev_check=1
                subtraction_iter+=1
            if (slope<=.000001) and (subtraction_iter>3):
                slope = gmean(slope_array_du_np[slope_array_du_np>0])
                gmean_check=1
        if slope>0.0:
            slope_array_du.append(slope)
        else:
            slope = gmean(slope_array_du_np[slope_array_du_np>0])
            slope_array_du.append(slope)
        alec_int += 1
        cat_layer.SetAttributeFilter("HYDROID = "+str(feat_id))
        area = 0
        for feat in cat_layer:
            area += feat.GetField("AreaSqKm")
            feat.Destroy()
        feature.Destroy()
        rafile.write(str(feat_id)+" "+str(slope)+" "+str(length)+" "+str(area)+"\n")
        ac_iter += 1
        print(f'Segment: {ac_iter}')
        feature_id_list.append(feat_id)

    rafile.close()
    dataSource.Destroy()
    ds.Destroy()

def main():
    geofloodHomeDir,projectName,DEM_name,chunk_status,input_fn,output_fn,hr_status = cfg_finder()
    geofloodResultsDir = os.path.join(geofloodHomeDir, output_fn,
                                      "GIS", projectName)
    Name_path = os.path.join(geofloodResultsDir, DEM_name)
    segment_shp = Name_path + "_channelSegment.shp"
    segcatfn = Name_path + "_segmentCatchment.tif"
    segcat_shp = Name_path + "_segmentCatchment.shp"
    hydro_folder = os.path.join(geofloodHomeDir,
                                output_fn, "Hydraulics",
                                projectName)
    if not os.path.exists(hydro_folder):
        os.mkdir(hydro_folder)
    attribute_txt = os.path.join(hydro_folder,
                                 DEM_name+"_River_Attribute.txt")
    demfn = os.path.join(geofloodHomeDir, input_fn,"GIS", projectName, DEM_name+".tif")
    river_attribute_estimation(segment_shp, segcatfn,
                                   segcat_shp, demfn,
                                   attribute_txt)

if __name__ == '__main__':
    t0 = perf_counter()
    main()
    t1 = perf_counter()
    print(("time taken to estimate river attributes:", t1-t0, " seconds"))


#!/usr/bin/env python
# coding: utf-8
# %%

# %%

import os
import pickle
import sys
import pandas as pd

from RASLPhysio import Features
from datetime import datetime, timedelta, date

# %%

# %%

def nearest(items, pivot):
    return min(items, key=lambda x: abs(x - pivot))

# %%

subjects = [ "S" + str(i) for i in range(2, 18) if i != 12]
subject =  sys.argv[1]

window_size = int(sys.argv[2])
window_size_features = int(sys.argv[3])

# %%

path = 'Data/WESAD/GeneratedData/{0}_Seconds_{1}.pkl'.format(subject, window_size)
with open(path, 'rb') as file:
    data = pickle.load(file, encoding='latin1')

# %%

params = Features.Features_Params()
params['rfft'] = False

# %%

all_features = {}
date = data.index[0]
while date + timedelta(seconds=window_size_features) <= data.index[-1]:
    end = nearest(data.index, date + timedelta(seconds=window_size_features))
    if date == end:
        ind = data.index.tolist().index(date) + 1
        date = data.index[ind]
        end = nearest(data.index, date + timedelta(seconds=window_size_features))
    window = data.loc[date:end,:]
    label = window['Label'][-1]
    features = {}
    for key in window.keys():
        if key == 'Label' or key == 'EDA': continue
        feats = Features.Extract_Features(window[key].values, params)
        for key_specific in feats.keys():
            features[ "{0}_{1}".format(key, key_specific)] = feats[key_specific]
    features['Label'] = label
    all_features[date] = features
    date = end
    print(date)

# features_to_keep = [
#     "SCL_mean",
#     "SCR_mean",
#     "SCL_std",
#     "SCR_std",
#     "BVP_std",
#     "ACC_X_mean",
#     "ACC_Y_mean",
#     "ACC_Z_mean",
# ]

Features_df = pd.DataFrame(all_features).T
# Features_df = Features_df[Features_df.columns.intersection(features_to_keep)]

pickle.dump( all_features, open( "Data/WESAD/GeneratedData/features_wesad_{0}_{1}.p".format(subject, window_size_features), "wb" ) )

# %%

# %%


import time
from urlparse import urlparse

import boto
from scrapy.exceptions import NotConfigured
from twisted.internet.threads import deferToThread

class S3Pipeline(object):

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler.settings)

    def __init__(self, settings):
        self.time_str = time.strftime('%Y%m%d%H%M%S')
        self.root, self.bucket = self.get_root_and_bucket(self.uri, settings)

    def store(self, keyname, data):
        return deferToThread(self._store, keyname, data)

    def _store(self, keyname, data):
        key = self.bucket.new_key(keyname)
        key.set_contents_from_string(data)
        key.set_acl('public-read')
        key.close()
        url = key.generate_url(expires_in=0, query_auth=False)
        return url

    def get_root_and_bucket(self, uri, settings):
        u = urlparse(self.uri)
        bucketname = u.hostname
        access_key = u.username or settings['AWS_ACCESS_KEY_ID']
        secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY']
        if not access_key or not secret_key:
            raise NotConfigured('No AWS_ACCESS_KEY_ID or '
                                'AWS_SECRET_ACCESS_KEY set in settings.py')
        conn = boto.connect_s3(access_key, secret_key)
        return u.path, conn.get_bucket(bucketname, validate=False)


from django.db.models import Sum
import math

from rest_framework import serializers
from rest_framework.reverse import reverse as api_reverse

from tickets.models import Ticket, Visit
from customerorders.api.serializers import CustomerOrderSerializer
from customerprojects.api.serializers import CustomerProjectSerializer
from assignProject.api.serializers import SalesManagerSerializer, EngineerSerializer

class TicketSerializer(serializers.ModelSerializer):

    class Meta:
        model = Ticket
        fields=[
            'id',
            'project',
            'order',
            'ticket_id',
            'request_type',
            'source',
            'sales_manager',
            'engineer',
            'comments',
            'status',
            'scheduled_date',
            'start_date',
            'stop_date',
            'updated',
            'timestamp',
        ]

class TicketReadSerializer(serializers.ModelSerializer):

    project           = CustomerProjectSerializer(read_only=True)
    order             = CustomerOrderSerializer(read_only=True)
    sales_manager     = SalesManagerSerializer(read_only=True)
    engineer          = EngineerSerializer(read_only=True)
    t_expense         = serializers.SerializerMethodField(read_only=True)

    class Meta:
        model = Ticket
        fields=[
            'id',
            'project',
            'order',
            'ticket_id',
            'request_type',
            'source',
            'sales_manager',
            'engineer',
            'comments',
            'status',
            't_expense',
            'scheduled_date',
            'start_date',
            'stop_date',
            'updated',
            'timestamp',
        ]
        read_only_fields = ['project', 'order']

    def get_t_expense(self, obj):
        ticket  = obj.id
        total = 0

        if ticket is None:
            return total
        try:
            visits = Visit.objects.filter(ticket=ticket)
            print(visits)
            for vit in visits:
                total += vit.expense
            formatted_total  = format(total, '.2f')
            return formatted_total
        except Exception:
            return ticket

class VisitSerializer(serializers.ModelSerializer):

    class Meta:
        model = Visit
        fields=[
            'id',
            'ticket',
            'comments',
            'expense',
            'updated',
            'timestamp',
        ]


import numpy as np

import matplotlib.pyplot as plt
import matplotlib.patches as patches

def plot_tree(tree):
    ax = plt.gca()
    ax.set_axis_off()

    build_tree(tree, 0.25, 0.25)

    ax.set_xlim(-1, 2)
    ax.set_ylim(-2, 1)

    plt.show()

def build_tree(tree, left, bottom, depth=0, ax=plt.gca()):
    height = width = 0.5
    right = left + width
    top = bottom + height

    p = patches.Rectangle((left, bottom), width, height, fill=False)
    ax.add_artist(p)

    if isinstance(tree, dict):
        ax.text(
            0.5*(left+right),
            0.5*(bottom+top+0.1),
            'X[{0}] < {1}'.format(tree["feature"], tree["value"]),
            horizontalalignment='center',
            verticalalignment='center',
            fontsize=8
        )

        ax.text(
            0.5*(left+right),
            0.5*(bottom+top-0.1),
            '{0}'.format(tree["sample"]),
            horizontalalignment='center',
            verticalalignment='center',
            fontsize=8
        )

        xy1 = (0.5*(left+right), bottom)

        ha = 0.2
        y2 = bottom-ha

        ax.annotate("", xy=xy1, xytext=(left, y2), arrowprops=dict(arrowstyle="<-"))
        ax.annotate("", xy=xy1, xytext=(right, y2), arrowprops=dict(arrowstyle="<-"))

        bottom = bottom - height - ha
        build_tree(tree["left"], left - 0.6*width, bottom)
        build_tree(tree["right"], right - 0.4*width, bottom)
    else:
        ax.text(
            0.5*(left+right),
            0.5*(bottom+top),
            '{0}'.format(tree),
            horizontalalignment='center',
            verticalalignment='center',
            fontsize=8
        )

def plot_learning_curve(data, clf, start_size=1, end_size=None, num_sample=20):
    _data = np.concatenate((
                np.copy(data[0].astype(np.float64)),
                np.copy(data[1])[np.newaxis, :].T
                ), axis=1
            )

    accuracies = list()

    if not end_size:
        end_size = len(_data)-1

    for size in range(start_size, end_size):
        mean = 0
        for i in range(num_sample):
            np.random.shuffle(_data)
            train, train_labels = np.copy(_data[:size+1, :-1]), np.copy(_data[:size+1, -1])
            test, test_labels = np.copy(_data[size+1:, :-1]), np.copy(_data[size+1:, -1])

            clf.train(train, train_labels)

            count = 0
            for index in range(len(test_labels)):
                if clf.predict(test[index]) == test_labels[index]:
                    count += 1

            mean += count / len(test_labels)
        mean /= num_sample
        accuracies.append(mean)

    plt.plot(range(start_size, end_size), accuracies)

    plt.xlabel("Training set size")
    plt.ylabel("Proportion correct on test set")

    plt.show()

def courbeApprentissageNN(dataset, sizes_dataset, init_weights, learning_rate, nb_iter, step):
    import NeuralNet

    data_ = np.concatenate((np.copy(dataset[0].astype(np.float64)), np.copy(dataset[1])[np.newaxis, :].T), axis=1)

    accuracies = list()
    for size in range(1, len(data_)-1, step):
        print(size)
        mean = 0
        for i in range(20):
            np.random.shuffle(data_)
            train, train_labels = np.copy(data_[:size+1, :-1]), np.copy(data_[:size+1, -1])
            test, test_labels = np.copy(data_[size+1:, :-1]), np.copy(data_[size+1:, -1])

            # print(train_labels)

            NNet = NeuralNet.NeuralNet(sizes_dataset,init_weights)
            NNet.train(train, train_labels, learning_rate, nb_iter)
            # print(dT.tree)

            count = 0
            for index in range(len(test_labels)):
                if NNet.predict(test[index]) == test_labels[index]:
                    count += 1

            mean += count / len(test_labels)
        mean /= 20
        accuracies.append(mean)

    plt.plot(range(1, len(data_)-1, step), accuracies)
    plt.xlabel('Training set size')
    plt.ylabel('Proportion correct on test set')
    plt.show()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('fantaapp', '0015_auto_20150807_1532'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='incontrocalendario',
            name='allenatorecasa',
        ),
        migrations.RemoveField(
            model_name='incontrocalendario',
            name='allenatoretrasferta',
        ),
        migrations.RemoveField(
            model_name='incontrocalendario',
            name='lega',
        ),
        migrations.AddField(
            model_name='incontrolega',
            name='disputato',
            field=models.BooleanField(default=False),
        ),
        migrations.AddField(
            model_name='incontrolega',
            name='golcasa',
            field=models.PositiveSmallIntegerField(null=True, blank=True),
        ),
        migrations.AddField(
            model_name='incontrolega',
            name='goltrasferta',
            field=models.PositiveSmallIntegerField(null=True, blank=True),
        ),
        migrations.AlterUniqueTogether(
            name='allenatore',
            unique_together=set([('utente', 'lega')]),
        ),
        migrations.DeleteModel(
            name='IncontroCalendario',
        ),
    ]


from django.db import models
from datetime import datetime

class ShowManager(models.Manager):
    def basic_validator(self, postData):
        errors = {}
        if len(postData['title']) < 3:
            errors['title'] = 'Title should be at least 2 characters.'
        if Show.objects.filter(title=postData['title']):
            errors['title_exists'] = "There's already a show with that title."
        if len(postData['network']) < 3:
            errors['network'] = 'Network should be at least 3 characters.'
        if postData['date']:
            input_date = datetime.strptime(postData['date'], "%Y-%m-%d")
        if not postData['date'] or input_date.date() > datetime.now().date():
            errors['date'] = 'Please enter a valid date.'
        if len(postData['desc']) > 0 and len(postData['desc']) < 11:
            errors['desc'] = 'Description should be at least 10 characters.'
        return errors

class Show(models.Model):
    title = models.CharField(max_length=255, unique=True)
    network = models.CharField(max_length=255)
    date = models.DateField()
    desc = models.TextField()
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    objects = ShowManager()


# -*- coding: utf-8 -*-

import enum
from django.utils.translation import ugettext_lazy as _

class NotifyLevel(enum.IntEnum):
    involved = 1
    all = 2
    none = 3

NOTIFY_LEVEL_CHOICES = (
    (NotifyLevel.involved, _("Involved")),
    (NotifyLevel.all, _("All")),
    (NotifyLevel.none, _("None")),
)

class WebNotificationType(enum.IntEnum):
    assigned = 1
    mentioned = 2
    added_as_watcher = 3
    added_as_member = 4
    comment = 5
    mentioned_in_comment = 6

WEB_NOTIFICATION_TYPE_CHOICES = (
    (WebNotificationType.assigned, _("Assigned")),
    (WebNotificationType.mentioned, _("Mentioned")),
    (WebNotificationType.added_as_watcher, _("Added as watcher")),
    (WebNotificationType.added_as_member, _("Added as member")),
    (WebNotificationType.comment, _("Comment")),
    (WebNotificationType.mentioned_in_comment, _("Mentioned in comment")),
)


import os
import sys
import argparse
import datetime

import pandas as pd
import dxchange.reader as dxreader

from pathlib import Path
from tomolog import log

def show_tomolog(args):

    fname = args.h5_name
    if os.path.isfile(fname):
        all_tomolog = dxreader.read_dx_meta(fname)
        log.info('All tomolog data available in the HDF5 raw data file: start')
        log.info(all_tomolog)
        log.info('All tomolog data available in the HDF5 raw data file: end')
    tomolog, year_month, pi_name = extract_meta(args)
    if tomolog != None:
        log.info('All tomolog data extracted from the HDF5 raw data file:')
        print(tomolog)

def add_header(label):

    header = add_decorator(label) + '\n' + label + '\n' + add_decorator(label) + '\n\n'

    return header

def add_title(label):

    title = label + '\n' + add_decorator(label, decorator='-') + '\n\n'

    return title

def add_decorator(label, decorator='='):

    return decorator.replace(decorator, decorator*len(label))

def create_rst_file(args):

    tomolog, year_month, pi_name = extract_meta(args)

    decorator = '='
    if os.path.isdir(args.doc_dir):
        log_fname = os.path.join(args.doc_dir, 'log_' + year_month + '.rst')

    with open(log_fname, 'a') as f:
        if f.tell() == 0:
            # a new file or the file was empty
            f.write(add_header(year_month))
            f.write(add_title(pi_name))
        else:
            #  file existed, appending
            f.write(add_title(pi_name))
        f.write('\n')
        f.write(tomolog)
        f.write('\n\n')
    print(log_fname)

def extract_dict(fname, list_to_extract, index=0):

    tomolog = dxreader.read_dx_meta(fname)
    # print(tomolog)
    try:
        dt = datetime.datetime.strptime(tomolog['start_date'][0], "%Y-%m-%dT%H:%M:%S%z")
        year_month = str(dt.year) + '-' + '{:02d}'.format(dt.month)
    except ValueError:
        log.error("The start date information is missing from the hdf file %s. Error (2020-01)." % fname)
        year_month = '2020-01'
    except TypeError:
        log.error("The start date information is missing from the hdf file %s. Error (2020-02)." % fname)
        year_month = '2020-02'
    pi_name = tomolog['experimenter_name'][0]

    # compact full_file_name to file name only as original data collection directory may have changed
    tomolog['full_file_name'][0] = os.path.basename(tomolog['full_file_name'][0])

    # sub_dict = {k:v for k, v in tomolog.items() if k in list_to_extract}
    sub_dict = {(('%3.3d' % index) +'_' + k):v for k, v in tomolog.items() if k in list_to_extract}

    return sub_dict, year_month, pi_name

def extract_meta(args):

    fname = args.h5_name

    list_to_extract = ('experimenter_name', 'start_date', 'end_date', 'full_file_name', 'sample_in_x', 'sample_in_y', 'proposal', 'sample_name', 'sample_y', 'camera_objective', 'resolution', 'energy', 'camera_distance', 'exposure_time', 'num_angles', 'scintillator_type', 'model')
    list_to_extract = ('attenuator_name', 'user_filter_ds', 'user_filter_ds_legend', 'user_filter_us', 'user_filter_us_legend', 'attenuator_1_name', 'description',
        'camera_motor_stack_name', 'camera_distance', 'camera_objective', 'camera_tube_length', 'resolution', 'scintillating_thickness', 'scintillator_type', 'ADcore_version',
        'HDFplugin_version', 'SDK_version', 'acquire_period', 'array_counter', 'binning_x', 'binning_y', 'convert_pixel_format', 'dimension_x', 'dimension_y', 'driver_version',
        'exposure_time', 'firmware_version', 'frame_rate', 'frame_rate_enable', 'gain', 'gain_auto', 'manufacturer', 'model', 'pixel_format', 'pixel_size', 'microns', 'min_x', 'min_y',
        'size_x', 'size_y', 'serial_number', 'temperature', 'instrument_name', 'mirror_name', 'mirror_angle', 'mirror_dsy', 'mirror_usy', 'mirror_x', 'mirror_y', 'stripe', 'stripe_legend',
        'Energy_list', 'USArm_list', 'energy', 'energy_mode', 'energy_mode_legend', 'monochromator_name', 'dmm_ds_arm', 'dmm_m2_y', 'dmm_m2_z', 'dmm_table_ds_y',
        'dmm_table_usy_ib', 'dmm_table_usy_ob', 'dmm_us_arm', 'sample_motor_stack_name', 'sample_pitch', 'sample_roll', 'sample_rotary', 'sample_x', 'sample_x_cent', 'sample_y', 'sample_z_cent',
        'hslits_ds_center', 'hslits_ds_size', 'hslits_us_center', 'hslits_us_size', 'vslits_ds_center', 'vslits_ds_size', 'vslits_us_center', 'vslits_us_size', 'slits_name', 'beamline',
        'current', 'fill_mode', 'source_name', 'top_up', 'description_1', 'description_2', 'description_3', 'ESAF_number', 'proposal', 'title', 'email', 'experimenter_name', 'facility_user_id',
        'institution', 'file_name', 'file_path', 'full_file_name', 'sample_name', 'dark_field_mode', 'dark_field_value', 'num_dark_fields', 'end_date', 'flat_field_axis', 'flat_field_mode',
        'flat_field_value', 'num_flat_fields', 'sample_in_x', 'sample_in_y', 'sample_out_x', 'sample_out_y', 'num_angles', 'return_rotation', 'rotation_speed', 'rotation_start', 'rotation_step', 'start_date')

    # list_to_extract = ('experimenter_name', 'full_file_name', 'description_1', 'description_2', 'resolution', 'energy', 'start_date','sample_pitch', 'sample_roll')
    # set pandas display
    pd.options.display.max_rows = 999
    year_month = 'unknown'
    pi_name = 'unknown'
    if os.path.isfile(fname):
        tomolog_dict, year_month, pi_name = extract_dict(fname, list_to_extract)
        # print (tomolog_dict, year_month, pi_name)
    elif os.path.isdir(fname):
        # Add a trailing slash if missing
        top = os.path.join(fname, '')
        # Set the file name that will store the rotation axis positions.
        h5_file_list = list(filter(lambda x: x.endswith(('.h5', '.hdf')), os.listdir(top)))
        h5_file_list.sort()
        tomolog_dict = {}
        file_counter=0
        for fname in h5_file_list:
            h5fname = top + fname
            sub_dict, year_month, pi_name = extract_dict(h5fname, list_to_extract, index=file_counter)
            tomolog_dict.update(sub_dict)
            file_counter+=1
        if year_month == 'unknown':
            log.error('No valid HDF5 file(s) fund in the directory %s' % top)
            log.warning('Make sure to use the --h5-name H5_NAME  option to select  the hdf5 file or directory containing multiple hdf5 files')
            return None

    else:
        log.error('No valid HDF5 file(s) fund')
        return None

    df = pd.DataFrame.from_dict(tomolog_dict, orient='index', columns=('value', 'unit'))
    return df.to_markdown(tablefmt='grid'), year_month, pi_name


#!/Library/Frameworks/Python.framework/Versions/3.6/bin/python3
###	/usr/bin/python
###	/Library/Frameworks/Python.framework/Versions/3.6/bin/python3
###	#!/usr/bin/python -mtimeit

"""
	This Python script is written by Zhiyang Ong to incrementally
		test features for performing data analytics operations with
		my BibTeX database.

	Synopsis:
	Perform incremental software testing automatically for my
		script(s) that perform data analytics operations with my
		BibTeX database.

	This script can be executed as follows:
	./incremental_test.py [input BibTeX file]

	Parameters:
	[input BibTeX file]:	A BibTeX file that may have duplicate
								BibTeX entries.

	Revision History:
	January 18, 2018		Version 0.1	Script.
"""

__author__ = 'Zhiyang Ong'
__version__ = '1.0'
__date__ = 'January 18, 2018'

#	The MIT License (MIT)

#	Copyright (c) <2014-2018> <Zhiyang Ong>

#	Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

#	The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

#	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

#	Email address: echo "cukj -wb- 23wU4X5M589 TROJANS cqkH wiuz2y 0f Mw Stanford" | awk '{ sub("23wU4X5M589","F.d_c_b. ") sub("Stanford","d0mA1n"); print $5, $2, $8; for (i=1; i<=1; i++) print "6\b"; print $9, $7, $6 }' | sed y/kqcbuHwM62z/gnotrzadqmC/ | tr 'q' ' ' | tr -d [:cntrl:] | tr -d 'ir' | tr y "\n"	Che cosa significa?

###############################################################
"""
	Import modules from The Python Standard Library.
	sys			Get access to any command-line arguments.
	os			Use any operating system dependent functionality.
	os.path		For pathname manipulations.

	subprocess -> call
				To make system calls.
	time		To measure elapsed time.
	warnings	Raise warnings.
	re			Use regular expressions.

	pathlib->Path
				For mapping a string to a path.
"""

import sys
import os
import os.path
#from pathlib import Path
from subprocess import call
import time
import warnings
import re

###############################################################
#	Import Custom Python Packages and Modules

# Package and module to process input arguments to the script/program.
from utilities.queue_ip_arguments import queue_ip_args
# Package and module to perform file I/O (input/output) operations.
from utilities.file_io import file_io_operations
"""
	Package and module to process (key,frequency) pairs.

	A function of this Python module is called as part of automated
		regression testing to make sure that previously working
		functions still work, even if they are not a part of the
		current software build for BibTeX Analytics.
"""
from database.key_frequency_pairs import process_key_freq_pairs
# Package and module to validate the checking of BibTeX keys.
from database.key_check_test import check_bibtex_key_tester
# Package and module to check the validation of statistical analysis.
from statistics.test_statistics_tester import statistical_analysis_tester
"""
	Package and module to print statistics of software testing
		results.
"""
from statistics.test_statistics import statistical_analysis

###############################################################
"""
	Module with methods that incrementally test scripts that
		perform data analytics operations with my BibTeX database.
	It tests the following:
	+ Sort a set of (key,mode) pairs.
	+ Check if a given BibTeX key is valid.
	+ The class "entry" that represents a given BibTeX entry.
	+ The Python-based database of BibTeX entries.
"""
class Incremental_Test_Automation:
	# List of BibTeX keys
	set_of_BibTeX_keys = []
	num_of_bibtex_entries = 0
	# ============================================================
	#	Other methods.
	# ============================================================
	#	Method to add BibTeX keys into a list, "set_of_BibTeX_keys".
	#	O(n) method, where n is the number of BibTeX keys.
	@staticmethod
	def add_BibTeX_key(found_BibTeX_key):
		if (found_BibTeX_key in Incremental_Test_Automation.set_of_BibTeX_keys):
			temp_str = "Duplicate BibTeX key:"+found_BibTeX_key
			warnings.warn(temp_str)
			raise Exception("Multiple instances of a BibTeX key")
		Incremental_Test_Automation.set_of_BibTeX_keys.append(found_BibTeX_key)
	# ============================================================
	#	Method to sort BibTeX keys into a list, "set_of_BibTeX_keys".
	#	O(n*log(n)) method, where n is the number of BibTeX keys.
	@staticmethod
	def sort_BibTeX_keys():
		Incremental_Test_Automation.set_of_BibTeX_keys = sorted(Incremental_Test_Automation.set_of_BibTeX_keys)
	# ============================================================
	#	Method to read each line of the input BibTeX file.
	#	O(n) method, where n is the number of lines of the BibTeX file.
	@staticmethod
	def read_input_BibTeX_file(ip_file_object,input_BibTeX_file):
		#print "--------------------------------------------------------"
		println = "=	Reading input BibTeX file: "
		println += input_BibTeX_file
		print(println)
		# Read each available line in the input BibTeX file.
		for line in ip_file_object:
			# Is this line the 1st line of a BibTeX entry?
			if "@" == line[0]:
				# Yes.
#				print "...	First line of a BibTeX entry."
				# Increment number of BibTeX entries.
				Incremental_Test_Automation.num_of_bibtex_entries = Incremental_Test_Automation.num_of_bibtex_entries + 1
				tokenized_BibTeX_entry = re.split('@|{|,',line)
#				for i in tokenized_BibTeX_entry:
#					print i
				# Is the type of the BibTeX entry valid?
				if (tokenized_BibTeX_entry[1] in queue_ip_args.BibTeX_entry_types):
					# Yes. Try adding the BibTeX entry to "set_of_BibTeX_keys".
					Incremental_Test_Automation.add_BibTeX_key(tokenized_BibTeX_entry[2].lower())
				else:
					# No. Warn user that the type of BibTeX entry is invalid!
					temp_str = "Invalid type of BibTeX entry:"
					temp_str += tokenized_BibTeX_entry[1]
					print(temp_str)
					#warnings.warn("Invalid type of BibTeX entry")
					raise Exception("BibTeX entry has an invalid type!")
		if (Incremental_Test_Automation.num_of_bibtex_entries != len(Incremental_Test_Automation.set_of_BibTeX_keys)):
			raise Exception("Mismatch in number of BibTeX entries processed.")
		else:
			print("=	Number of BibTeX entries processed: {}" .format(str(Incremental_Test_Automation.num_of_bibtex_entries)))

###############################################################
# Main method for the program.

#	If this is executed as a Python script,
if __name__ == "__main__":
	# --------------------------------------------------------
	#	= Start of Preprocessing.
	queue_ip_args.preprocessing()
	# --------------------------------------------------------
	#	= End of Preprocessing.
	print("==================================================")
	print("Automating incremental software test automation of")
	print("	script(s) for performing data analytics")
	print("	operations on my BibTeX database.")
	print("")
	# Assign input arguments to "queue_ip_args" for processing.
	queue_ip_args.set_input_arguments(sys.argv,queue_ip_args.INCREMENTAL_TEST)
	# Check if user wants to read the brief user manual.
	queue_ip_args.check_if_help_wanted()
	# Process the first input argument.
	print("=	Process the first input argument.")
	ip_filename = queue_ip_args.process_1st_ip_arg()
	# Create a file object for reading.
	print("=	Create a file object for reading.")
	# Create a file object for input BibTeX file, in reading mode.
	ip_file_obj = file_io_operations.open_file_object_read(ip_filename)
	# The real stuff begins here...
	statistical_analysis_tester.test_statistical_analysis()
	#	### TO-DO
	# Insert test cases for testing the utilities package.
	print("-	-	-	-	-	-	-	-	-	-	-	-	-")
	"""
		### IMPORTANT NOTE: Calling this function does not test
				anything!

		This function is called as part of automated regression
			testing to make sure that previously working functions
			still work, even if they are not a part of the current
			software build for BibTeX Analytics.
	"""
	process_key_freq_pairs.sort_pairs()
	print("-	-	-	-	-	-	-	-	-	-	-	-	-")
	check_bibtex_key_tester.test_check_bibtex_key()
	print("-	-	-	-	-	-	-	-	-	-	-	-	-")
	Incremental_Test_Automation.read_input_BibTeX_file(ip_file_obj,ip_filename)
	print("!	!	!	!	!	!	!	!	!	!	!")
	print(">>	Get statistics of the software testing process.")
	statistical_analysis.print_statistics_of_software_testing()
	# Close the file object for reading.
	print("=	Close the file object for reading.")
	file_io_operations.close_file_object(ip_file_obj)


import os
import time
import base64
from PIL import Image
from io import BytesIO

import torch
from torch.autograd import Variable
from torchvision.utils import save_image

from models import TransformerNet
from utils import *

from flask import *
from flask_cors import CORS

app = Flask(__name__, )
# r'/*' 是通配符，让本服务器所有的URL 都允许跨域请求
CORS(app, resources=r'/*')

# 上传文件
@app.route("/nst/", methods=['POST'])
def uploadFile():
    result_text = {"statusCode": 200}

    if not request.form:  # 检测是否有数据
        result_text['rc'] = 'fail'
    else:
        table = {
            '0': '/home/www/flask/nst/model/van_gogh.pth',
            '1': '/home/www/flask/nst/model/line_geometry.pth',
            '2': '/home/www/flask/nst/model/mona_lisa.pth',
            '3': '/home/www/flask/nst/model/rain_princess.pth',
            '4': '/home/www/flask/nst/model/starry_night.pth',
            '5': '/home/www/flask/nst/model/wave.pth',
            '6': '/home/www/flask/nst/model/plaid_portrait.pth',
            '7': '/home/www/flask/nst/model/obama_hope.pth',
            '8': '/home/www/flask/nst/model/sunday_afternoon.pth',
            '9': '/home/www/flask/nst/model/the_scream.pth',
        }
        ind = request.form.get('style')
        img64 = request.form.get('img')

        checkpoint_model = table[ind]

        byte_data = base64.b64decode(img64)
        image_data = BytesIO(byte_data)
        img = Image.open(image_data)

        device = torch.device("cpu")

        transform = style_transform()

        # Define model and load model checkpoint
        transformer = TransformerNet().to(device)
        transformer.load_state_dict(torch.load(checkpoint_model, map_location=torch.device('cpu')))
        transformer.eval()

        # Prepare input
        image_tensor = Variable(transform(img)).to(device)
        image_tensor = image_tensor.unsqueeze(0)

        # Stylize image
        with torch.no_grad():
            stylized_image = denormalize(transformer(image_tensor)).cpu()

        t = f"/home/www/flask/nst/tmp/{str(time.time())}.png"

        save_image(stylized_image, t)
        with open(t, "rb") as f:
            base64_str = base64.b64encode(f.read())

        os.remove(t)

        result_text['rc'] = base64_str.decode()

    response = make_response(jsonify(result_text))
    response.headers['Access-Control-Allow-Origin'] = '*'
    response.headers['Access-Control-Allow-Methods'] = 'OPTIONS,HEAD,GET,POST'
    response.headers['Access-Control-Allow-Headers'] = 'x-requested-with'
    return response

if __name__ == '__main__':
    app.run(port=2306)


from collections import OrderedDict
from rest_framework import serializers

from products.serializers import ProductSerializer
from .models import Sales

class SalesSerializer(serializers.ModelSerializer):
	active = serializers.BooleanField(default=True)
	product = ProductSerializer()

	class Meta:
		model = Sales
		fields = ('id','product','user','number','date','userid','location','active',)

	def to_representation(self, instance):
		data = super().to_representation(instance)
		if data['active']:
			return data

		return OrderedDict()


"""
The core idea of the data interface is to separate the PRM model from the relational data soucre. The module :mod:`.datainterface` contains a collection of methods to access the relational data that are used by different algorithms (e.g. CPD learners, inference methods, EM algorithm). Another advantage of this approach is that the data is not required to be in a fixed database format.
"""

'''
The singular DataInterface instance contains a collection of methods that are used
by different algorithms (e.g. CPD learners, EM algorithm, Inference). The description
of a function should give examples of algorithms that use that method.

-At this point there is no error testing, e.g. the statements that will query
the data are derived from the names of the entities/relationships etc in the
prm. In the future one could envision some sort of mapping that is also passed
on as part of the specification of a prm

There will be different ways a prm (e.g. a learner or inference class), can
access the data. For large datasets that have to be queried multiple times
, there will be an MySQL interface. A similar interface for SQLite will be
implemented first. SQLite allows the data to be loaded into the memory.
Then it can be accessed faster in the usual way. Of course both MySQL and
SQLite use the SQL language interchangeably.
'''

'''
Analytics
    by adding the decorator @time_analysis before any
    function we collect runtime information
'''

import logging

from analytics.performance import time_analysis

name = 'UnNamed'
"""The name of the data interface
"""

DSI = None
"""List that contains all :class:`~data.datainterface.DataSetInterface` instances that connect to the data
"""

diType = 'UnSpecified'
"""The type of dataset used, e.g. `crossvalidation` or `testtraining`
"""

trainingSets = {}
"""Dictionary that maps a crossvalidation test set (one `DataSetInterface` in `datainterface.DSI`) with the corresponding training set (all other `DataSetInterfaces` in `datainterface.DSI`)"""

def configure(prm):
    '''
    A method that allows us to configure a dataset interface based on information from the instantiated PRM.
    '''

    # When dealing with aggregation, we want to create VIEWS to facilitate the sql queries.
    # If a dependency has a 1:n or m:n relationship, then aggregation is required.

    # print "\tConfiguring DataInterface '%s' with PRM '%s'"%(self.name,prm.name)

    # View currently disabled as we don't use them
    #
    # for dep in prm.dependencies.values():
    #     if dep.aggregator is not None:
    #         for dsi in self.DSI:
    #             print '\t Creating VIEW for %s'%dep.name
    #             dsi.createView(dep)

def computeTrainingSets():
    '''
    Returns a dictionary mapping a crossvalidation test set with the corresponding training set { datasetinstance : [datasetinstance1,datasetinstance2,....]}. Every :class:`!DataSetInterface` in `datainterface.DSI` is a key in `datainterface.trainingSets`, the value is a list of all other :class:`!DataSetInterface` instances in `datainterface.DSI`
    '''

    for dsi in DSI:
        ts = []
        for tdsi in DSI:
            if tdsi!=dsi:
                ts.append(tdsi)
        trainingSets[dsi]=ts

def datasetinterfaceFactory(path,ditype):
    '''
    Creates a connection to a database. There are possibly multiple dataset connections to do crossvalidation.

    :arg path: The path to the database
    :arg ditype: Type of database, e.g. `SQLite`
    :returns: A :class:`.DataSetInterface` instance
    '''
    logging.info('\tDataSet: %s (%s)'%(path.split('/')[-1],ditype))

    if ditype == 'SQLite':
        from data.sqliteinterface import SQLiteDI
        return SQLiteDI( path)
    elif ditype == 'MySQL':
        raise Exception("MySQL not yet implemented")
    elif ditype == 'XML':
        raise Exception("XML not yet implemented")
    else:
        raise Exception("unknown data interface type")

class DataSetInterface:
    '''
    An instance of the class connects a PRM with the relational data base that it models. This is an abstract class, sublclasses have to implement the required methdos for different database systems. E.g. :class:`data.sqliteinterface.SQLiteDI` is for SQLite.

    There data is queried by the parameter learning algorithm, e.g. :class:`learners.cpdlearners`, as well as by the inference methods; the inference :mod:`engine` is unrolling the Ground Bayesian Network.
    '''

    def __init__(self,dsiType):
        '''
        Shouldn't be executed since the class is abstract.
        '''
        self.dsiType = dsiType

    def __repr__(self):
        ''' String representation of the data interface'''
        raise Exception("method __repr__() is not implemented in the DataSetInterface")


# -*- coding: utf-8 -*-
import re

import scrapy

from ..custom_setting import inspect_settings
from ..items import InspectItem

class InspectSpider(scrapy.Spider):
    custom_settings = inspect_settings

    name = 'inspect'
    allowed_domains = ['jib.xywy.com']
    start_urls = []

    base_url = 'http://jib.xywy.com'

    # 生成字母a-z，并组合为链接
    char = list(map(chr, range(ord('a'), ord('z') + 1)))
    for c in char:
        start_urls.append(base_url + '/html/' + c + '.html')

    def parse(self, response):
        path = response.xpath('//ul[@class="ks-zm-list clearfix mt10"]/li/a/@href').extract()
        for p in path:

            title = response.xpath('//a[@href="' + p + '"]/text()').extract()[0].strip()
            if '...' in title:
                title = '###'
            item = dict()
            item['entity1'] = title

            p = re.sub('/il_sii_', '/il_sii/inspect/', p)
            yield scrapy.Request(self.base_url + p, meta={'item': item}, callback=self.parse_inspect)

    def parse_inspect(self, response):
        items = response.meta['item']
        if items['entity1'] == '###':
            items['entity1'] = response.xpath('//div[@class="jb-name fYaHei gre"]/text()').extract()[0]
        # print(title)
        检查项目s = response.xpath('//div[@class="more-zk pr"]//ul[@class="clearfix jib-check "]/li['
                               '@class="check-item"]/a/@href').extract()

        if len(检查项目s) != 0:
            for j in 检查项目s:

                title = response.xpath('//ul[@class="clearfix jib-check "]/li[@class="check-item"]/a[@href="' + j + '"]/text()').extract()[0].strip()
                if '...' in title:
                    title = '###'

                item = InspectItem()
                item['entity1'] = items['entity1']
                item['entity2'] = title
                item['relation'] = '相关检查'

                yield scrapy.Request(j, meta={'item': item}, callback=self.parse_next, dont_filter=True)

    def parse_next(self, response):
        item = response.meta['item']
        if item['entity2'] == '###':
            item['entity2'] = response.xpath('//div[@class="baby-weeks"]/div[@class="clearfix"]/strong/text()').extract()[0]
        return item


import tensorflow as tf
import json

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import f_classif

from tensorflow.python.keras import models
from tensorflow.python.keras.layers import Dense
from tensorflow.python.keras.layers import Dropout

# Vectorization parameters
# Range (inclusive) of n-gram sizes for tokenizing text.
NGRAM_RANGE = (1, 3)

# Limit on the number of features. We use the top 20K features.
TOP_K = 20000

# Whether text should be split into word or character n-grams.
# One of 'word', 'char'.
TOKEN_MODE = 'word'

# Minimum document/corpus frequency below which a token will be discarded.
MIN_DOCUMENT_FREQUENCY = 2

# Limit on the length of text sequences. Sequences longer than this
# will be truncated.
MAX_SEQUENCE_LENGTH = 500

def ngram_vectorize(train_texts, train_labels, val_texts):
    """Vectorizes texts as ngram vectors.
    1 text = 1 tf-idf vector the length of vocabulary of uni-grams + bi-grams.
    # Arguments
        train_texts: list, training text strings.
        train_labels: np.ndarray, training labels.
        val_texts: list, validation text strings.
    # Returns
        x_train, x_val: vectorized training and validation texts
    """
    # Create keyword arguments to pass to the 'tf-idf' vectorizer.
    kwargs = {
            'ngram_range': NGRAM_RANGE,  # Use 1-grams + 2-grams.
            'dtype': 'int32',
            'strip_accents': 'unicode',
            'decode_error': 'replace',
            'analyzer': TOKEN_MODE,  # Split text into word tokens.
            'stop_words' : 'english',
            'min_df': MIN_DOCUMENT_FREQUENCY
            #'max_df': 0.9
    }
    vectorizer = TfidfVectorizer(**kwargs)

    # Learn vocabulary from training texts and vectorize training texts.
    x_train = vectorizer.fit_transform(train_texts)
    print(x_train.shape)
    print(x_train)
    # Vectorize validation texts.
    x_val = vectorizer.transform(val_texts)
    # Select top 'k' of the vectorized features.
    selector = SelectKBest(f_classif, k=min(TOP_K, x_train.shape[1]))
    selector.fit(x_train, train_labels)
    x_train = selector.transform(x_train)
    x_val = selector.transform(x_val)

    x_train = x_train.astype('float32')
    x_val = x_val.astype('float32')
    return x_train, x_val

def get_num_classes(labels):
    """Gets the total number of classes.

    # Arguments
        labels: list, label values.
            There should be at lease one sample for values in the
            range (0, num_classes -1)

    # Returns
        int, total number of classes.

    # Raises
        ValueError: if any label value in the range(0, num_classes - 1)
            is missing or if number of classes is <= 1.
    """
    num_classes = max(labels) + 1
    missing_classes = [i for i in range(num_classes) if i not in labels]
    if len(missing_classes):
        raise ValueError('Missing samples with label value(s) '
                         '{missing_classes}. Please make sure you have '
                         'at least one sample for every label value '
                         'in the range(0, {max_class})'.format(
                            missing_classes=missing_classes,
                            max_class=num_classes - 1))

    if num_classes <= 1:
        raise ValueError('Invalid number of labels: {num_classes}.'
                         'Please make sure there are at least two classes '
                         'of samples'.format(num_classes=num_classes))
    return num_classes

def _get_last_layer_units_and_activation(num_classes):
    """Gets the # units and activation function for the last network layer.

    # Arguments
        num_classes: int, number of classes.

    # Returns
        units, activation values.
    """
    if num_classes == 2:
        activation = 'sigmoid'
        units = 1
    else:
        activation = 'softmax'
        units = num_classes
    return units, activation

def mlp_model(layers, units, dropout_rate, input_shape, num_classes):
    """Creates an instance of a multi-layer perceptron model.

    # Arguments
        layers: int, number of `Dense` layers in the model.
        units: int, output dimension of the layers.
        dropout_rate: float, percentage of input to drop at Dropout layers.
        input_shape: tuple, shape of input to the model.
        num_classes: int, number of output classes.

    # Returns
        An MLP model instance.
    """
    op_units, op_activation = _get_last_layer_units_and_activation(num_classes)
    model = models.Sequential()
    model.add(Dropout(rate=dropout_rate, input_shape=input_shape))
    for _ in range(layers-1):
        model.add(Dense(units=units, activation='relu'))
        model.add(Dropout(rate=dropout_rate))
    model.add(Dense(units=op_units, activation=op_activation))
    print(model.summary())
    return model

def train_ngram_model(data,
                      learning_rate=1e-3,
                      epochs=1000,
                      batch_size=32,
                      layers=2,
                      units=128,
                      dropout_rate=0.2):
    """Trains n-gram model on the given dataset.

    # Arguments
        data: tuples of training and test texts and labels.
        learning_rate: float, learning rate for training model.
        epochs: int, number of epochs.
        batch_size: int, number of samples per batch.
        layers: int, number of `Dense` layers in the model.
        units: int, output dimension of Dense layers in the model.
        dropout_rate: float: percentage of input to drop at Dropout layers.

    # Raises
        ValueError: If validation data has label values which were not seen
            in the training data.
    """
    # Get the data.
    (train_texts, train_labels), (val_texts, val_labels) = data

    # Verify that validation labels are in the same range as training labels.
    #num_classes = 4
    num_classes = get_num_classes(train_labels)
    unexpected_labels = [v for v in val_labels if v not in range(num_classes)]
    if len(unexpected_labels):
        raise ValueError('Unexpected label values found in the validation set:'
                         ' {unexpected_labels}. Please make sure that the '
                         'labels in the validation set are in the same range '
                         'as training labels.'.format(
                             unexpected_labels=unexpected_labels))

    # Vectorize texts.
    x_train, x_val = ngram_vectorize(train_texts, train_labels, val_texts)

    # Create model instance.
    print(x_train.shape)
    print(x_val.shape)
    model = mlp_model(  layers=layers,
                        units=units,
                        dropout_rate=dropout_rate,
                        input_shape=x_train.shape[1:],
                        num_classes=num_classes)

    # Compile model with learning parameters.
    if num_classes == 2:
        loss = 'binary_crossentropy'
    else:
        loss = 'sparse_categorical_crossentropy'
        #loss = 'categorical_crossentropy'

    optimizer = tf.keras.optimizers.Adam(lr=learning_rate)
    #optimizer = tf.keras.optimizers.SGD()
    #optimizer = tf.keras.optimizers.RMSprop()
    #optimizer = tf.keras.optimizers.Adamax()
    model.compile(optimizer=optimizer, loss=loss, metrics=['acc'])

    # Create callback for early stopping on validation loss. If the loss does
    # not decrease in two consecutive tries, stop training.
    callbacks = [tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=2)]

    # Train and validate model.
    history = model.fit(
            x_train,
            train_labels,
            epochs=epochs,
            callbacks=callbacks,
            validation_data=(x_val, val_labels),
            verbose=2,  # Logs once per epoch.
            batch_size=batch_size)

    # Print results.
    history = history.history
    print('Validation accuracy: {acc}, loss: {loss}'.format(
            acc=history['val_acc'][-1], loss=history['val_loss'][-1]))

    # Save model.
    # Creates a HDF5 file 'my_model.h5'
    model.save('MLPClassifier.h5')


from keras.models import load_model
from tkinter import *
import tkinter as tk
import win32gui
from PIL import ImageOps, ImageGrab
import numpy as np

model = load_model('mnist.h5')

def predict_digit(img):
    #resize image to 28x28 pixels
    img = img.resize((28,28))
    #convert rgb to grayscale
    img = img.convert('L')
    #issue here: Cause the image is inverted, the black is white and the white is black, in the training data the writing is white, but in testing
    #with real hand writing it is black.
    img =  ImageOps.invert(img)
    img = np.array(img)
    #reshaping to support our model input and normalizing
    img = img.reshape(1,28,28,1)
    img = img/255.0
    #predicting the class
    res = model.predict([img])[0]
    return np.argmax(res), max(res)

#Then we create the App class which is responsible for building the GUI for our app.
# We create a canvas where we can draw by capturing the mouse event and with a button,
# we trigger the predict_digit() function and display the results.
class App(tk.Tk):
    def __init__(self):
        tk.Tk.__init__(self)

        self.x = self.y = 0

        # Creating elements
        self.canvas = tk.Canvas(self, width=300, height=300, bg = "white", cursor="cross")
        self.label = tk.Label(self, text="Draw..", font=("Helvetica", 48))
        self.classify_btn = tk.Button(self, text = "Recognise", command = self.classify_handwriting)
        self.button_clear = tk.Button(self, text = "Clear", command = self.clear_all)

        # Grid structure
        self.canvas.grid(row=0, column=0, pady=2, sticky=W, )
        self.label.grid(row=0, column=1,pady=2, padx=2)
        self.classify_btn.grid(row=1, column=1, pady=2, padx=2)
        self.button_clear.grid(row=1, column=0, pady=2)

        #self.canvas.bind("<Motion>", self.start_pos)
        self.canvas.bind("<B1-Motion>", self.draw_lines)

    def clear_all(self):
        self.canvas.delete("all")

    def classify_handwriting(self):
        HWND = self.canvas.winfo_id()  # get the handle of the canvas
        rect = win32gui.GetWindowRect(HWND)  # get the coordinate of the canvas
        a,b,c,d = rect
        rect=(a+4,b+4,c-4,d-4)
        im = ImageGrab.grab(rect)

        digit, acc = predict_digit(im)
        self.label.configure(text= str(digit)+', '+ str(int(acc*100))+'%')

    def draw_lines(self, event):
        self.x = event.x
        self.y = event.y
        r=8
        self.canvas.create_oval(self.x-r, self.y-r, self.x + r, self.y + r, fill='black')

app = App()
mainloop()


import json
import math
from itertools import groupby
import pygal

# 将数据加载到一个列表中
filename = 'btc_close_2017_requests.json'
with open(filename) as f:
    btc_data = json.load(f)

# 创建5个列表，分别存储日期和收盘价
dates = []
months = []
weeks = []
weekdays = []
close = []
# 每一天信息
for btc_dict in btc_data:
    dates.append(btc_dict['date'])
    months.append(int(btc_dict['month']))
    weeks.append(int(btc_dict['week']))
    weekdays.append(btc_dict['weekday'])
    close.append(int(float(btc_dict['close'])))

# 绘制折线图
line_chart = pygal.Line(x_label_rotation=20, show_minor_x_labels=False)
line_chart.title = '收盘价（¥）'
line_chart.x_labels = dates
N = 20  # x 轴坐标每隔20天显示一次
line_chart.x_labels_major = dates[::N]
line_chart.add('收盘价', close)
line_chart.render_to_file('收盘价折线图.svg')

line_chart = pygal.Line(x_label_rotation=20, show_minor_x_labels=False)
line_chart.title = '收盘价对数变换（¥）：'
line_chart.x_labels = dates
N = 20
line_chart.x_labels_major = dates[::20]
close_log = [math.log10(_) for _ in close]
line_chart.add('log收盘价', close_log)
line_chart.render_to_file('收盘价对数变换折线图.svg')

def draw_line(x_date, y_date, title, y_legend):
    xy_map = []
    for key, group in groupby(sorted(zip(x_date, y_date)), key=lambda _: _[0]):
        y_list = [v for _, v in group]
        xy_map.append([key, sum(y_list) / len(y_list)])

    x_unique, y_mean = [*zip(*xy_map)]
    chart = pygal.Line()
    chart.title = title
    chart.x_labels = x_unique
    chart.add(y_legend, y_mean)
    chart.render_to_file(title + '.svg')
    return chart

idx_month = dates.index('2017-12-01')
draw_line(months[:idx_month], close[:idx_month], '收盘价月日均值', '月日均值')

idx_week = dates.index('2017-12-11')
line_chart_week = draw_line(
    weeks[1:idx_week], close[1:idx_week], '收盘价周日均值', '周日均值')

wd = ['Monday', 'Tuesday', 'Wednesday',
      'Thursday', 'Friday', 'Saturday', 'Sunday']
weekdays_int = [wd.index(w) + 1 for w in weekdays[1:idx_week]]
line_chart_weekday = draw_line(
    weekdays_int, close[1:idx_week], '收盘价星期均值', '星期均值')

line_chart_weekday.x_labels = ['周一', '周二', '周三', '周四', '周五', '周六', '周日']
line_chart_weekday.render_to_file('收盘价星期均值.svg')

with open('Dashboard.html', 'w', encoding='utf8') as html_file:
    html_file.write(
        '<title>Dashboard</title><meta charset="utf-8">\n')
    for svg in ['收盘价折线图.svg', '收盘价对数变换折线图.svg', '收盘价月日均值.svg', '收盘价周日均值.svg', '收盘价星期均值.svg']:
        html_file.write(
            '    <object type="image/svg+xml" data="{0}" height=500></object>\n'.format(svg))
    html_file.write('')


import requests
import os
from twilio.rest import Client
from dotenv import load_dotenv

load_dotenv()

account_sid = os.getenv("ACCOUNT_SID")
auth_token = os.getenv("AUTH_TOKEN")
twilio_phone = os.getenv("TWILIO_PHONE")
my_phone = os.getenv("MY_PHONE")

# "lat": -36.848461,
# "lon": 174.763336,

parameters = {
    "lat": 32.011261,
    "lon": 34.774811,
    "appid": os.getenv("APP_ID_API_WEATHER"),
    "exclude": "current, minutely, daily"
}

response = requests.get("https://api.openweathermap.org/data/2.8/onecall", params=parameters)
response.raise_for_status()
weather_data = response.json()
weather_slice = weather_data["hourly"][:12]

will_rain = False

for hour_data in weather_slice:
    condition_code = hour_data["weather"][0]["id"]
    if int(condition_code) < 700:
        will_rain = True

if will_rain:
    client = Client(account_sid, auth_token)
    message = client.messages.create(
        from_=f'whatsapp:{twilio_phone}',
        body="הולך לרדת גשם. לא לשכוח להביא מטריה ☂",
        to=f'whatsapp:{my_phone}'
    )

My Projects on Data Science, Web and more/Section 34 - API Weather/RainUpdatedEveryDay.py


import sys
import whois

def check_whois():
    for line in open(sys.argv[1], "r", encoding="utf8"):
        if line[0]== '#':
            pass
        else:
            try:
                w = whois.whois(line[:-1])
                print("""
                =============== Domain: {} ==============
                NameServer: {} \n
                Name: {} \n
                Organization: {} \n
                Register Date: {} \n
                Expiration Date: {} \n
                =========================================""".format(line[:-1], w.name_servers, w.name, w.org , w.creation_date, w.expiration_date))
            except:
                print("WHOIS NOT FOUND: {}".format(line[:-1]))

check_whois()


import matplotlib.pyplot as plt
from randomwalk import RandomWalk

while True:
    rw=RandomWalk(90000)
    rw.fill_walk()
    #绘制所有的点
    plt.scatter(rw.x_values,rw.y_values,c=rw.y_values,cmap=plt.cm.Blues,s=15,edgecolor='none')
    #绘制起点
    plt.scatter(0,0,c='green',edgecolor='none',s=100)
    #绘制终点
    plt.scatter(rw.x_values[-1],rw.y_values[-1],c='red',s=100,edgecolor='none')
    #隐藏坐标轴
    plt.axes().get_xaxis().set_visible(False)
    plt.axes().get_yaxis().set_visible(False)
    plt.show()

    keep_running=input("Mark another walk?(y/n):")
    if keep_running=="n":
        break


#!/usr/bin/env python
# Convert TelcoData file to easier to parse CSV

import csv
import sqlite3
import os

input_file = 'sms-email-ocn.csv'
output_file = 'database.sqlite'

os.remove(output_file)

# open datafile and create database
c = csv.reader(open(input_file, 'r'), delimiter=',', quotechar='"')
db = sqlite3.connect(output_file)
cur = db.cursor()
cur.execute("""
CREATE TABLE numbers (
npanxxy int PRIMARY KEY,
company text,
type text,
email text,
ocn text
)
""")

# insert data
for row in c:
    cur.execute(
        'INSERT INTO numbers VALUES("{}", "{}", "{}", "{}", "{}")'.format(
            row[0]+row[1]+row[2], row[3], row[4], row[5], row[6]
        )
    )

db.commit()
db.close()


import colors as c
from utils import ask

intro = c.pink + '''
Welcome to the pink fluffy
unicorns quiz game!!!!!!
'''  + c.reset

def q1():
    fur = ask(c.orange + 'What is the color of a pink fluffy unicorns fur?' + c.reset)
    if fur == 'pink':
        return True
    return False

def q2():
    dance = ask(c.green + 'What do pink fluffy unicorns dance on?' + c.reset)
    if dance.startswith("rainbow"):
        return True
    return False

def q3():
    texture = ask(c.blue + 'Please use one word to describe the texture of their magical fur?' + c.reset)
    if texture.startswith("smile"):
        return True
    return False
questions = [q1,q2,q3]


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""The setup script."""

from setuptools import setup, find_packages
import os

import versioneer

HERE = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(HERE, "README.md"), "r", encoding="utf8") as fh:
    README = fh.read()
with open(os.path.join(HERE, "requirements.txt"), "r") as fh:
    REQUIRES = [line.strip() for line in fh]

setup(
    version=versioneer.get_version(),
    cmdclass=versioneer.get_cmdclass(),
    author="Jacob Tomlinson",
    author_email="jacob@tomlinson.email",
    classifiers=[
        "Development Status :: 2 - Pre-Alpha",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: Apache Software License",
        "Natural Language :: English",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
    ],
    description="Home Assistant support for opsdroid",
    install_requires=REQUIRES,
    license="Apache Software License 2.0",
    long_description=README,
    long_description_content_type="text/markdown",
    include_package_data=True,
    keywords="Opsdroid Home Assistant",
    name="opsdroid-homeassistant",
    packages=["opsdroid_homeassistant"],
    entry_points={
        "opsdroid_connectors": ["homeassistant = opsdroid_homeassistant.connector"]
    },
    url="https://github.com/opsdroid/opsdroid-homeassistant",
    zip_safe=False,
)


#Importing Libraries

from __future__ import print_function
import pyaudio
import wave
import numpy as np
import csv
import os
import shutil
from time import sleep
import datetime
import librosa

from PyQt5.QtCore import *
class Thread(QObject):
    finished = pyqtSignal()
    def __init__(self):
        QThread.__init__(self)
        self.sourcePath = os.path.join(os.path.dirname(__file__), 'Recordings')
        self.destPath = os.path.join(os.path.dirname(__file__), 'Recordings','Backup')

        # RECORDING AUDIO SNIPPET USING PYAUDIO
        self.name=datetime.datetime.now()
        self.final=datetime.datetime.strftime(self.name, '%Y-%m-%d-%H-%M-%S-%f')

        self.CHUNK = 1024
        self.FORMAT = pyaudio.paInt16
        self.CHANNELS = 1
        self.RATE = 44100
        self.RECORD_SECONDS = 10
        self.AUDIO_OUTPUT_TYPE = ".wav"
        self.WAVE_OUTPUT_FILENAME_NO_EXTENSION = "Recordings//" + self.final
        #self.WAVE_OUTPUT_FILENAME = "Recordings//" + self.final + self.AUDIO_OUTPUT_TYPE
        self.WAVE_OUTPUT_FILENAME = os.path.abspath(os.path.join(os.path.dirname(__file__),"Recordings",self.final+self.AUDIO_OUTPUT_TYPE))

    def worker(self):
        audio = pyaudio.PyAudio()

        print ('\n*******************************************')
        print ('RHAPSODY MODULE-I INPUT')
        print ('*******************************************\n')
        print ('\n===========================================')
        print ('STARTED RECORDING')
        print ('===========================================\n')

        for i in range(1,4):
            print ('\n===========================================')
            print (str(i)+'...')
            print ('===========================================\n')
            sleep(1)

        stream = audio.open(format=self.FORMAT,
                            channels=self.CHANNELS,
                            rate=self.RATE,
                            input=True,
                            frames_per_buffer=self.CHUNK)

        f = []

        for i in range(0, int(self.RATE / self.CHUNK * self.RECORD_SECONDS)):
            data = stream.read(self.CHUNK)
            f.append(data)

        print ('\n===========================================')
        print ('DONE RECORDING')
        print ('===========================================\n')

        stream.stop_stream()
        stream.close()
        audio.terminate()

        wf = wave.open(self.WAVE_OUTPUT_FILENAME, 'wb')
        wf.setnchannels(self.CHANNELS)
        wf.setsampwidth(audio.get_sample_size(self.FORMAT))
        wf.setframerate(self.RATE)
        wf.writeframes(b''.join(f))
        wf.close()

        """""""""""""""""""""""""""""""""""""""
        1 - Loading File
        """""""""""""""""""""""""""""""""""""""
        filename = self.WAVE_OUTPUT_FILENAME
        y, sr = librosa.load(filename)

        """""""""""""""""""""""""""""""""""""""
        2 - Get Tempo == bpm
        """""""""""""""""""""""""""""""""""""""
        tempo, beat_frames = librosa.beat.beat_track(y=y, sr=sr)
        print ('\n===========================================')
        print('Estimated tempo: {:.2f} beats per minute'.format(tempo))
        print ('===========================================\n')

        # generate csv files with beat times
        #CSV_FILENAME = self.WAVE_OUTPUT_FILENAME_NO_EXTENSION + ".csv"

        beat_times = librosa.frames_to_time(beat_frames, sr=sr)
        CSV_FILENAME = os.path.abspath(os.path.join(os.path.dirname(__file__),"Recordings",self.final+".csv"))
        librosa.output.times_csv(CSV_FILENAME, beat_times)

        # WRITING A FILE WITH THE TEMPO
        #TEXT_FILENAME = self.WAVE_OUTPUT_FILENAME_NO_EXTENSION + ".txt"
        TEXT_FILENAME = os.path.abspath(os.path.join(os.path.dirname(__file__),"Recordings",self.final+".txt"))
        bpm_value = open(TEXT_FILENAME, 'w')
        tempo_text = str(tempo) + '\n'
        bpm_value.write(tempo_text)

        """""""""""""""""""""""""""""""""""""""
        3 - Get Notes
        """""""""""""""""""""""""""""""""""""""
        hz = librosa.feature.chroma_cqt(y=y, sr=sr)

        ## GET STRONGEST OCTAVE
        strongestOctave = 0
        strongestOctave_sum = 0
        for octave in range(len(hz)):
            sum = 0
            for frame in hz[octave]:
                sum = sum + frame
            if sum > strongestOctave_sum:
                strongestOctave_sum = sum
                strongestOctave = octave

        ## GET HEIGHEST HZ FOR EACH TIME FRAME
        strongestHz = []
        for i in range(len(hz[0])):
            strongestHz.append(0)

        notes = []
        for i in range(len(hz[0])):
            notes.append(0)

        for frame_i in range(len(hz[0])):
            strongest_temp = 0
            for octave_i in range(len(hz)):

                if hz[octave_i][frame_i] > strongest_temp:
                    strongest_temp = hz[octave_i][frame_i]
                    strongestHz[frame_i] = octave_i + 1
                    notes[frame_i] = librosa.hz_to_note(hz[octave_i][frame_i])

        # C C# D D# E F F# G G# A  A# B
        # 1 2  3 4  5 6 7  8 9  10 11 12
        strongestHz_sum = [0,0,0,0,0,0,0,0,0,0,0,0]
        for note in strongestHz:
            strongestHz_sum[note-1] = strongestHz_sum[note-1] + 1

        for i in range(len(strongestHz_sum)):
            strongestHz_sum[i] = float(strongestHz_sum[i]) / len(strongestHz)

        noteSorted = [0,0,0,0,0,0,0,0,0,0,0,0]
        for num in range(len(noteSorted)):
             biggest = strongestHz_sum.index(max(strongestHz_sum))
             noteSorted[num] = biggest+1
             strongestHz_sum[biggest] = strongestHz_sum[biggest] - 0.25

        for note in noteSorted:
            noteString = str(note) + '\n'
            bpm_value.write(noteString)

        bpm_value.close()

        print ('\n===========================================')
        print ('RECORDING ANALYSIS COMPLETED SUCCESSFULLY!!!')
        print ('===========================================\n')

        self.finished.emit()