#!/usr/bin/env python
"""
The file contains the class and methods for loading and aligning datasets
"""
from __future__ import print_function, division
import pickle
import numpy as np
from scipy.io import loadmat
import pandas as pd
from .utils import p2fa_phonemes
import warnings
from collections import OrderedDict
from copy import deepcopy
import sys

__author__ = "Prateek Vij"
__copyright__ = "Copyright 2017, Carnegie Mellon University"
__credits__ = ["Amir Zadeh", "Prateek Vij", "Soujanya Poria"]
__license__ = "GPL"
__version__ = "1.0.1"
__status__ = "Production"

if sys.version_info <= (3,):
    FileNotFoundError = IOError # python 2 doesn't have FileNotFoundError

class Dataset(object):
    """Primary class for loading and aligning dataset"""

    def __init__(self, dataset_file='', stored=False):
        """
        Initialise the Dataset class. Support two loading mechanism -
        from dataset files and from the pickle file, decided by the param
        stored.
        :param stored: True if loading from pickle, false if loading from
                       dataset feature files. Default False
        :param dataset_file: Filepath to the file required to load dataset
                             features. CSV or pickle file depending upon the
                             loading mechanism
        :timestamps: absolute or relative.
        """
        self.feature_dict = None
        self.timestamps = 'absolute' # this is fixed, we no longer support relative timestamps
        self.stored = stored
        self.dataset_file = dataset_file
        self.phoneme_dict = p2fa_phonemes
        self.loaded = False

    def __getitem__(self, key):
        """Adding direct access of internal data"""
        return self.feature_dict[key]

    def keys(self):
        """Wrapper for .keys() for the feature_dict"""
        return self.feature_dict.keys()

    def items(self):
        """Wrapper for .items() for the feature_dict"""
        return self.feature_dict.items()

    def load(self):
        """
        Loads feature dictionary for the input dataset
        :returns: Dictionary of features for the dataset with each modality
         as dictionary key
        """

        # Load from the pickle file if stored is True
        if self.stored:
            self.dataset_pickle = self.dataset_file
            self.feature_dict = pickle.load(open(self.dataset_pickle))
            return self.feature_dict

        # Load the feature dictionary from the dataset files
        self.dataset_csv = self.dataset_file
        self.feature_dict = self.controller()
        self.loaded = True
        return self.feature_dict

    def controller(self):
        """
        Validates the dataset csv file and loads the features for the dataset
        from its feature files
        """

        def validate_file(self):
            data = pd.read_csv(self.dataset_csv, header=None)
            data = np.asarray(data)
            #data = data[:,:7]
            self.dataset_info = {}
            modality_count = len(data[0]) - 4
            self.modalities = {}
            for i in range(modality_count):
                # key = 'modality_' + str(i)
                key = str(data[0][i + 4])
                info = {}
                info["level"] = str(data[1][i + 4])
                info["type"] = str(data[0][i + 4])
                self.modalities[key] = info

            for record in data[2:]:
                video_id = str(record[0])
                segment_id = str(record[1])
                if video_id not in self.dataset_info:
                    self.dataset_info[video_id] = {}
                if segment_id in self.dataset_info[video_id]:
                    raise NameError("Multiple instances of segment "
                                    + segment_id + " for video " + video_id)
                segment_data = {}
                segment_data["start"] = float(record[2])
                segment_data["end"] = float(record[3])
                for i in range(modality_count):
                    # key = 'modality_' + str(i)
                    key = str(data[0][i + 4])
                    segment_data[key] = str(record[i + 4])
                self.dataset_info[video_id][segment_id] = segment_data
            return

        def load_features(self):
            feat_dict = {}
            data = self.dataset_info
            modalities = self.modalities
            # timestamps = self.timestamps
            for key, value in modalities.items():
                api = value['type']
                level = value['level']
                loader_method = Dataset.__dict__["load_" + api]
                modality_feats = {}
                print("Loading features for", api)
                for video_id, video_data in data.items():
                    video_feats = {}
                    for segment_id, segment_data in video_data.items():
                        filepath = str(segment_data[key])
                        start = segment_data["start"]
                        end = segment_data["end"]
                        video_feats[segment_id] = loader_method(self,
                                                                filepath, start,
                                                                end, timestamps=self.timestamps,
                                                                level=level)
                    modality_feats[video_id] = video_feats
                    modality_feats = OrderedDict(sorted(modality_feats.items(), key=lambda x: x[0]))
                feat_dict[key] = modality_feats

            return feat_dict

        validate_file(self)
        feat_dict = load_features(self)

        return feat_dict

    def load_opensmile(self, filepath, start, end, timestamps='absolute', level='s'):
        """
        Load OpenSmile Features from the file corresponding to the param
        filepath
        :param start: Start time of the segment
        :param end: End time of the segment
        :param filepath: Path to the opensmile feature files
        :param level: 's' if the file contains features only for the segment,
                      i.e. interval (start, end), 'v' if for the entire video
        :param timestamps: relative or absolute
        :returns: List of tuples (feat_start, feat_end, feat_value)
                  corresponding to the features in the interval.
        Note: Opensmile support features for entire segment or video only and
              will return None if level is 'v' and start time is
        """
        features = []
        start_time, end_time = start, end
        if timestamps == 'relative':
            start_time = 0.0
            end_time = end - start

        if level == 's' or start == 0.0:
            feats = open(filepath).readlines()[-1].strip().split(',')[1:]
            feats = [float(feat_val) for feat_val in feats]
            feat_val = np.asarray(feats, dtype=np.float32)
            features.append((start_time, end_time, feat_val))
        else:
            print("Opensmile support features for the entire segment")
            return None
        return features

    def load_covarep(self, filepath, start, end, timestamps='absolute', level='s'):
        """
        Load COVAREP Features from the file corresponding to the param
        filepath
        :param start: Start time of the segment
        :param end: End time of the segment
        :param filepath: Path to the opensmile feature files
        :param level: 's' if the file contains features only for the segment,
                      i.e. interval (start, end), 'v' if for the entire video
        :param timestamps: relative or absolute
        :returns: List of tuples (feat_start, feat_end, feat_value)
                  corresponding to the features in the interval.
        """
        features = []
        time_period = 0.01
        try:
            f_content = loadmat(filepath)
        except (FileNotFoundError, TypeError, ValueError) as e: # depending on loadmat("XXX") or loadmat("XXX.mat"), error will be different
            print(".mat file cannot load at {}!".format(filepath))
            return [] # if no feature file present, return an empty list
        feats = f_content['features']
        start_time, end_time = start, end
        if timestamps == "relative":
            start_time, end_time = 0.0, end - start

        if level == 's':
            feat_start = start_time
            for feat in feats:
                feat_end = feat_start + time_period
                feat_val = np.asarray(feat)
                features.append((max(feat_start - start_time, 0), max(feat_end - start_time, 0), feat_val))
                feat_start += time_period
        else:
            feat_count = feats.shape[0]
            start_index = int(min((start / time_period), feat_count))
            end_index = int(min((end / time_period), feat_count))
            feat_start = start_time
            for feat in feats[start_index:end_index]:
                feat_end = feat_start + time_period
                feat_val = np.asarray(feat)
                features.append((max(feat_start - start_time, 0), max(feat_end - start_time, 0), feat_val))
                feat_start += time_period
        return features

    def load_phonemes(self, filepath, start, end, timestamps='relative', level='s'):
        """
        Load P2FA phonemes as Features from the file corresponding to the
        param filepath
        :param start: Start time of the segment
        :param end: End time of the segment
        :param filepath: Path to the opensmile feature files
        :param level: 's' if the file contains features only for the segment,
                      i.e. interval (start, end), 'v' if for the entire video
        :param timestamps: relative or absolute
        :returns: List of tuples (feat_start, feat_end, feat_value)
                  corresponding to the features in the interval.
        """
        features = []
        start_time, end_time = start, end
        if timestamps == "relative":
            start_time, end_time = 0.0, end - start

        if level == 's':
            with open(filepath, 'r') as f_handle:
                for line in f_handle.readlines():
                    line = line.strip()
                    if not line:
                        break
                    feat_start = float(line.split(",")[0])
                    feat_end = float(line.split(",")[1])
                    feat_val = [float(val) for val in line.split(",")[2:]]
                    feat_val = np.asarray(feat_val)
                    features.append((max(feat_start, 0), max(feat_end, 0), feat_val))
        else:
            with open(filepath, 'r') as f_handle:
                for line in f_handle.readlines():
                    line = line.strip()
                    if not line:
                        break
                    feat_start = float(line.split(",")[0])
                    feat_end = float(line.split(",")[1])
                    feat_time = feat_end - feat_start
                    if ((feat_start <= start and feat_end > end)
                        or (feat_start >= start and feat_end < end)
                        or (feat_start <= start
                            and start - feat_start < feat_time / 2)
                        or (feat_start >= start
                            and end - feat_start > feat_time / 2)):

                        feat_start = feat_start - start
                        feat_end = feat_end - start
                        feat_val = [float(val) for val in line.split(",")[2:]]
                        feat_val = np.asarray(feat_val)
                        features.append((max(feat_start, 0), max(feat_end, 0), feat_val))
        return features

    def load_embeddings(self, filepath, start, end, timestamps='relative', level='s'):
        """
        Load Word Embeddings from the file corresponding to the param
        filepath
        :param start: Start time of the segment
        :param end: End time of the segment
        :param filepath: Path to the opensmile feature files
        :param level: 's' if the file contains features only for the segment,
                      i.e. interval (start, end), 'v' if for the entire video
        :param timestamps: relative or absolute
        :returns: List of tuples (feat_start, feat_end, feat_value)
                  corresponding to the features in the interval.
        """
        features = []
        start_time, end_time = start, end
        if timestamps == "relative":
            start_time, end_time = 0.0, end - start

        if level == 's':
            with open(filepath, 'r') as f_handle:
                for line in f_handle.readlines():
                    line = line.strip()
                    if not line:
                        break
                    feat_start = float(line.split(",")[0])
                    feat_end = float(line.split(",")[1])
                    feat_val = [float(val) for val in line.split(",")[2:]]
                    feat_val = np.asarray(feat_val)
                    features.append((max(feat_start, 0), max(feat_end, 0), feat_val))
        else:
            with open(filepath, 'r') as f_handle:
                for line in f_handle.readlines():
                    line = line.strip()
                    if not line:
                        break
                    feat_start = float(line.split(",")[0])
                    feat_end = float(line.split(",")[1])
                    feat_time = feat_end - feat_start
                    if ((feat_start <= start and feat_end > end)
                        or (feat_start >= start and feat_end < end)
                        or (feat_start <= start
                            and start - feat_start < feat_time / 2)
                        or (feat_start >= start
                            and end - feat_start > feat_time / 2)):

                        feat_start = feat_start - start
                        feat_end = feat_end - start
                        feat_val = [float(val) for val in line.split(",")[2:]]
                        feat_val = np.asarray(feat_val)
                        features.append((max(feat_start, 0), max(feat_end, 0), feat_val))
        return features

    def load_words(self, filepath, start, end, timestamps='relative', level='s'):
        """
        Load one hot embeddings for words as features from the file
        corresponding to the param filepath
        :param start: Start time of the segment
        :param end: End time of the segment
        :param filepath: Path to the opensmile feature files
        :param level: 's' if the file contains features only for the segment,
                      i.e. interval (start, end), 'v' if for the entire video
        :param timestamps: relative or absolute
        :returns: List of tuples (feat_start, feat_end, feat_value)
                  corresponding to the features in the interval.
        """
        features = []
        start_time, end_time = start, end
        if timestamps == "relative":
            start_time, end_time = 0.0, end - start

        if level == 's':
            with open(filepath, 'r') as f_handle:
                for line in f_handle.readlines():
                    line = line.strip()
                    if not line:
                        break
                    feat_start = float(line.split(",")[0])
                    feat_end = float(line.split(",")[1])
                    try:
                        feat_val = int(line.split(",")[2])
                    except:
                        print(filepath, start, end)
                    # feat_val = [float(val) for val in line.split(",")[2:]]
                    # assert len(feat_val) == 1
                    # feat_val = np.asarray(feat_val)[0]
                    #print(feat_start, feat_end)
                    #assert False
                    features.append((max(feat_start, 0), max(feat_end, 0), feat_val))
        else:
            with open(filepath, 'r') as f_handle:
                for line in f_handle.readlines():
                    line = line.strip()
                    if not line:
                        break
                    feat_start = float(line.split(",")[0])
                    feat_end = float(line.split(",")[1])
                    feat_time = feat_end - feat_start
                    if ((feat_start <= start and feat_end > end)
                        or (feat_start >= start and feat_end < end)
                        or (feat_start <= start
                            and start - feat_start < feat_time / 2)
                        or (feat_start >= start
                            and end - feat_start > feat_time / 2)):

                        feat_start = feat_start - start
                        feat_end = feat_end - start
                        feat_val = [float(val) for val in line.split(",")[2:]]
                        feat_val = np.asarray(feat_val)
                        features.append((max(feat_start, 0), max(feat_end, 0), feat_val))
        return features

    def load_openface(self, filepath, start, end, timestamps='absolute', level='s'):
        """
        Load OpenFace features from the file corresponding to the param
        filepath
        :param start: Start time of the segment
        :param end: End time of the segment
        :param filepath: Path to the opensmile feature files
        :param level: 's' if the file contains features only for the segment,
                      i.e. interval (start, end), 'v' if for the entire video
        :param timestamps: relative or absolute
        :returns: List of tuples (feat_start, feat_end, feat_value)
                  corresponding to the features in the interval.
        """
        features = []
        time_period = 0.0333333

        start_time, end_time = start, end
        if timestamps == "relative":
            start_time, end_time = 0.0, end - start

        if level == 's':
            with open(filepath, 'r') as f_handle:
                for line in f_handle.readlines()[1:]:
                    line = line.strip()
                    if not line:
                        break
                    feat_start = float(line.split(",")[1])
                    feat_end = feat_start + time_period
                    feat_val = [float(val) for val in line.split(",")[2:]]
                    feat_val = np.asarray(feat_val, dtype=np.float32)
                    features.append((max(feat_start, 0), max(feat_end, 0), feat_val))

        else:
            with open(filepath, 'r') as f_handle:
                for line in f_handle.readlines()[1:]:
                    line = line.strip()
                    if not line:
                        break
                    feat_start = float(line.split(",")[1])

                    if (feat_start >= start and feat_start < end):
                        # To adjust the timestamps
                        feat_start = feat_start - start
                        feat_end = feat_start + time_period
                        feat_val = [float(val) for val in line.split(",")[2:]]
                        feat_val = np.asarray(feat_val, dtype=np.float32)
                        features.append((max(feat_start, 0), max(feat_end, 0), feat_val))
        return features

    # note that this is implicity new facet
    def load_facet(self, filepath, start, end, timestamps='absolute', level='v'):
        """
        Load FACET features from the file corresponding to the param
        filepath
        :param start: Start time of the segment
        :param end: End time of the segment
        :param filepath: Path to the opensmile feature files
        :param level: 's' if the file contains features only for the segment,
                      i.e. interval (start, end), 'v' if for the entire video
        :param timestamps: relative or absolute
        :returns: List of tuples (feat_start, feat_end, feat_value)
                  corresponding to the features in the interval.
        """
        features = []

        # load a subset of current segment and infer its format
        start_row = 0
        start_col = 0
        with open(filepath, 'r') as f_handle:
            splitted = []
            for line in f_handle.readlines()[0:10]:
                splitted.append(line.split(","))

            # check if the first row is a header by checking if the first field is a number
            try:
                float(splitted[start_row][start_col])
            except:
                start_row = 1

            # check if the first column is a index column by checking if it increments by 1 everytime
            for i in range(1, len(splitted) - 1):
                if (float(splitted[i+1][0]) - float(splitted[i][0])) != 1:
                    start_col = 0
                    break
                start_col = 1

            time_period = float(splitted[start_row][start_col])

        start_time, end_time = start, end
        # if timestamps == "relative":
        #     start_time, end_time = 0.0, end - start

        if level == 's':
            with open(filepath, 'r') as f_handle:
                for line in f_handle.readlines()[start_row:]:
                    line = line.strip()
                    if not line:
                        break
                    feat_start = float(line.split(",")[start_col])
                    feat_end = feat_start + time_period
                    feat_val = []
                    for val in line.split(",")[start_col + 1:-1]:
                        try:
                            feat_val.append(float(val))
                        except:
                            feat_val.append(0.0)
                    feat_val = np.asarray(feat_val, dtype=np.float32)
                    features.append((max(feat_start, 0), max(feat_end, 0), feat_val))

        else:
            with open(filepath, 'r') as f_handle:
                for line in f_handle.readlines()[start_row:]:
                    line = line.strip()
                    if not line:
                        break
                    feat_start = float(line.split(",")[start_col])

                    if (feat_start >= start and feat_start < end):
                        # To adjust the timestamps
                        feat_start = feat_start - start
                        feat_end = feat_start + time_period
                        feat_val = []
                        for val in line.split(",")[start_col + 1:-1]:
                            try:
                                feat_val.append(float(val))
                            except:
                                feat_val.append(0.0)
                        feat_val = np.asarray(feat_val, dtype=np.float32)
                        features.append((max(feat_start, 0), max(feat_end, 0), feat_val))
        return features

    def load_facet1(self, filepath, start, end, timestamps='absolute', level='v'):
        """
        Load FACET features from the file corresponding to the param
        filepath
        :param start: Start time of the segment
        :param end: End time of the segment
        :param filepath: Path to the opensmile feature files
        :param level: 's' if the file contains features only for the segment,
                      i.e. interval (start, end), 'v' if for the entire video
        :param timestamps: relative or absolute
        :returns: List of tuples (feat_start, feat_end, feat_value)
                  corresponding to the features in the interval.
        """
        return self.load_facet(filepath, start, end, timestamps=timestamps, level=level)

    def load_facet2(self, filepath, start, end, timestamps='absolute', level='v'):
        """
        Load FACET features from the file corresponding to the param
        filepath
        :param start: Start time of the segment
        :param end: End time of the segment
        :param filepath: Path to the opensmile feature files
        :param level: 's' if the file contains features only for the segment,
                      i.e. interval (start, end), 'v' if for the entire video
        :param timestamps: relative or absolute
        :returns: List of tuples (feat_start, feat_end, feat_value)
                  corresponding to the features in the interval.
        """
        return self.load_facet(filepath, start, end, timestamps=timestamps, level=level)

    def align(self, align_modality):
        aligned_feat_dict = {}
        modalities = self.modalities
        alignments = self.get_alignments(align_modality)
        for modality in modalities:
            if modality == align_modality:
                aligned_feat_dict[modality] = OrderedDict(sorted(self.feature_dict[modality].items(), key=lambda x: x[0]))
                continue
            aligned_modality = self.align_modality(modality, alignments)
            aligned_feat_dict[modality] = OrderedDict(sorted(aligned_modality.items(), key=lambda x: x[0]))
        self.aligned_feature_dict = aligned_feat_dict
        return aligned_feat_dict

    def get_alignments(self, modality):
        alignments = {}
        aligned_feat_dict = self.feature_dict[modality]

        for video_id, segments in aligned_feat_dict.items():
            segment_alignments = {}
            for segment_id, features in segments.items():
                segment_alignments[segment_id] = []
                for value in features:
                    timing = (value[0], value[1])
                    segment_alignments[segment_id].append(timing)
            alignments[video_id] = segment_alignments
        return alignments

    def align_modality(self, modality, alignments, merge_type="mean"):
        aligned_feat_dict = {}
        modality_feat_dict = self.feature_dict[modality]
        warning_hist = set() # Keep track of all the warnings

        for video_id, segments in alignments.items():
            aligned_video_feats = {}

            for segment_id, feat_intervals in segments.items():
                aligned_segment_feat = []

                for start_interval, end_interval in feat_intervals:
                    time_interval = end_interval - start_interval
                    feats = modality_feat_dict[video_id][segment_id]
                    try:
                        if modality == "words":
                            aligned_feat = feats[0][2] - feats[0][2]
                        else:
                            aligned_feat = np.zeros(len(feats[0][2]))
                    except:
                        if (video_id, segment_id) not in warning_hist:
                            print("\nModality {} for video {} segment {} is (partially) missing and is thus being replaced by zeros!\n".format(modality.split("_")[-1], video_id, segment_id))
                            warning_hist.add((video_id, segment_id))
                        # print(modality, video_id, segment_id, feats)
                        for sid, seg_data in modality_feat_dict[video_id].items():
                            if seg_data != []:
                                feats = seg_data
                                break
                    try:
                        if modality == "words":
                            aligned_feat = feats[0][2] - feats[0][2]
                        else:
                            aligned_feat = np.zeros(len(feats[0][2]))
                    except:
                        if modality == "words":
                            aligned_feat = 0
                        else:
                            aligned_feat = np.zeros(0)

                    for feat_tuple in feats:
                        feat_start = feat_tuple[0]
                        feat_end = feat_tuple[1]
                        feat_val = feat_tuple[2]
                        if (feat_start < end_interval
                                and feat_end >= start_interval):
                            feat_weight = (min(end_interval, feat_end) -
                                           max(start_interval, feat_start)) / time_interval
                            weighted_feat = np.multiply(feat_val, feat_weight)
                            if np.shape(aligned_feat) == (0,):
                                aligned_feat = weighted_feat
                            else:
                                aligned_feat = np.add(aligned_feat, weighted_feat)

                    aligned_feat_tuple = (start_interval, end_interval,
                                          aligned_feat)
                    aligned_segment_feat.append(aligned_feat_tuple)
                aligned_video_feats[segment_id] = aligned_segment_feat
            aligned_feat_dict[video_id] = aligned_video_feats

        return aligned_feat_dict

    @staticmethod
    def merge(dataset1, dataset2):
        # ensure the merged objects are indeed Datasets
        assert isinstance(dataset1, Dataset)
        assert isinstance(dataset2, Dataset)
        # merge the feature_dict and modalities attributes
        merged_modalities = Dataset.merge_dict(dataset1.modalities, dataset2.modalities)
        merged_feat_dict = Dataset.merge_dict(dataset1.feature_dict, dataset2.feature_dict)
        mergedDataset = Dataset()
        mergedDataset.feature_dict = merged_feat_dict
        mergedDataset.modalities = merged_modalities
        return mergedDataset

    @staticmethod
    def merge_dict(dict1, dict2):
        merged = deepcopy(dict1)
        merged.update(dict2)
        return merged


"""OpenAPI schema utility functions."""

from io import StringIO

_DEFAULT_EXAMPLES = {
    "string": "string",
    "integer": 1,
    "number": 1.0,
    "boolean": True,
    "array": [],
}

_DEFAULT_STRING_EXAMPLES = {
    "date": "2020-01-01",
    "date-time": "2020-01-01T01:01:01Z",
    "password": "********",
    "byte": "QG1pY2hhZWxncmFoYW1ldmFucw==",
    "ipv4": "127.0.0.1",
    "ipv6": "::1",
}

def example_from_schema(schema):
    """
    Generates an example request/response body from the provided schema.

    >>> schema = {
    ...     "type": "object",
    ...     "required": ["id", "name"],
    ...     "properties": {
    ...         "id": {
    ...             "type": "integer",
    ...             "format": "int64"
    ...         },
    ...         "name": {
    ...             "type": "string",
    ...             "example": "John Smith"
    ...         },
    ...         "tag": {
    ...             "type": "string"
    ...         }
    ...     }
    ... }
    >>> example = example_from_schema(schema)
    >>> assert example == {
    ...     "id": 1,
    ...     "name": "John Smith",
    ...     "tag": "string"
    ... }
    """
    # If an example was provided then we use that
    if "example" in schema:
        return schema["example"]

    elif "oneOf" in schema:
        return example_from_schema(schema["oneOf"][0])

    elif "anyOf" in schema:
        return example_from_schema(schema["anyOf"][0])

    elif "allOf" in schema:
        # Combine schema examples
        example = {}
        for sub_schema in schema["allOf"]:
            example.update(example_from_schema(sub_schema))
        return example

    elif "enum" in schema:
        return schema["enum"][0]

    elif "type" not in schema:
        # Any type
        return _DEFAULT_EXAMPLES["integer"]

    elif schema["type"] == "object" or "properties" in schema:
        example = {}
        for prop, prop_schema in schema.get("properties", {}).items():
            example[prop] = example_from_schema(prop_schema)
        return example

    elif schema["type"] == "array":
        items = schema["items"]
        min_length = schema.get("minItems", 0)
        max_length = schema.get("maxItems", max(min_length, 2))
        assert min_length <= max_length
        # Try generate at least 2 example array items
        gen_length = min(2, max_length) if min_length <= 2 else min_length

        example_items = []
        if items == {}:
            # Any-type arrays
            example_items.extend(_DEFAULT_EXAMPLES.values())
        elif isinstance(items, dict) and "oneOf" in items:
            # Mixed-type arrays
            example_items.append(_DEFAULT_EXAMPLES[sorted(items["oneOf"])[0]])
        else:
            example_items.append(example_from_schema(items))

        # Generate array containing example_items and satisfying min_length and max_length
        return [example_items[i % len(example_items)] for i in range(gen_length)]

    elif schema["type"] == "string":
        example_string = _DEFAULT_STRING_EXAMPLES.get(
            schema.get("format", None), _DEFAULT_EXAMPLES["string"]
        )
        min_length = schema.get("minLength", 0)
        max_length = schema.get("maxLength", max(min_length, len(example_string)))
        gen_length = (
            min(len(example_string), max_length)
            if min_length <= len(example_string)
            else min_length
        )
        assert 0 <= min_length <= max_length
        if min_length <= len(example_string) <= max_length:
            return example_string
        else:
            example_builder = StringIO()
            for i in range(gen_length):
                example_builder.write(example_string[i % len(example_string)])
            example_builder.seek(0)
            return example_builder.read()

    elif schema["type"] in ("integer", "number"):
        example = _DEFAULT_EXAMPLES[schema["type"]]
        if "minimum" in schema and "maximum" in schema:
            # Take average
            example = schema["minimum"] + (schema["maximum"] - schema["minimum"]) / 2
        elif "minimum" in schema and example <= schema["minimum"]:
            example = schema["minimum"] + 1
        elif "maximum" in schema and example >= schema["maximum"]:
            example = schema["maximum"] - 1
        return float(example) if schema["type"] == "number" else int(example)

    else:
        return _DEFAULT_EXAMPLES[schema["type"]]


import csv
import io
from nltk.tokenize import word_tokenize

import sys
reload(sys)
sys.setdefaultencoding('ISO-8859-1')

def findLowest(topWords):
	result = topWords.keys()[0]
	for word in topWords:
		if(topWords[word] < topWords[result]):
			result = word
	return result

with io.open("old_tweets.csv", encoding = "ISO-8859-1") as csvFile:
	#fieldnames = ['username', 'date', 'text', 'id']
	#reader = csv.DictReader(csvFile, fieldnames = fieldnames)
	reader = csv.reader(csvFile, delimiter = ';')
	reader.next()
	i = 0;
	counter = {}
	topWords = {}
	topsLowest = ""
	topMaxAllowed = 2000
	for row in reader:
		for word in word_tokenize(row[2]):
			counter[word] = counter.get(word, 0) + 1
		if(len(topWords) < topMaxAllowed):
			topWords[word] = counter[word]
			if((not hasattr(topWords, topsLowest)) or topWords[topsLowest] > counter[word]):
				topsLowest = word
		elif (topWords[topsLowest] < counter[word]):
			del topWords[topsLowest]
			topWords[word] = counter[word]
			topsLowest = findLowest(topWords)
	with io.open("word_count.csv", mode="w", encoding="ISO-8859-1") as outputFile:
		outputFile.write(unicode("word;frequency"))
		for word in topWords:
			outputFile.write(unicode("\n%s;%d" % (word, topWords[word])))


#!/usr/bin/env python
# -*- coding:utf-8 -*-

import os
import pandas as pd
from datetime import datetime
from txt_table import txt_table
import re

# 将txt文件转化为csv
def transfer_data(folder):
    #建立使用文件列表
    filenames=[]
    filelist=[]
    filexlsx=[]
    #遍历文件寻找txt
    files=os.walk(folder)
    for root,dirs,files in files:
        for file in files:
            # 自动生成文件名
            if "readme" in file:
                filexls=re.match("(.*)_readme",file).group(1)
                filexlsx.append(filexls)
            else:
                pass
            filename=os.path.join(root,file)
            filenames.append(filename)
    for filename in filenames:
        try:
            if ".txt" in filename and "readme" not in filename:
                fileout=filename.replace('.txt','.csv')
                data=txt_table(filename)
                data.to_csv(fileout,sep=',',encoding='utf-8')
                filelist.append(fileout)
            elif ".TXT" in filename and "readme" not in filename:
                fileout = filename.replace('.TXT','.csv')
                data=txt_table(filename)
                data.to_csv(fileout,sep=',',encoding='utf-8')
                filelist.append(fileout)
        except:
            print('error!')
            continue
    print('已将txt文件转化为csv!')
    #将数据合并
    #print(filexlsx[0])
    fileouts=os.path.join(folder,(filexlsx[0]+'.csv'))
    df=pd.DataFrame()
    for i in range(0,len(filelist)):
        df_i=pd.read_csv(filelist[i],sep=',',encoding='utf-8')
        #print(df_i)
        #df.append(df_i)
        df=pd.concat([df,df_i])
    df.to_csv(fileouts,sep=',',encoding='utf-8')
    print(df.head())
    print("已完成数据合并!")
    return fileouts

# 提取相关数据
def extract_data(filecsv,jiwens,file_station):
    # 读取气象数据,这一步要提取有用的气象数据并求平均值
    datas = pd.read_csv(filecsv, sep=',', encoding='utf-8')
    # 提取列名
    datacolumns= list(datas.columns)[2:]
    del datacolumns[2]  # 删除日期
    #print(datacolumns)
    datacolumnslist = [list(datas[i]) for i in datacolumns]
    #print(datacolumnslist[0])
    # shidu_ave = list(datas[u'V13003_701'])
    file_n = filecsv.replace(".csv", "_N.csv")
    print(file_n)
    stations_china = list(pd.read_excel(file_station, sheet_name='Sheet1')[u'区站号'])
    lng = list(pd.read_excel(file_station, sheet_name='Sheet1')[u'经度'])
    lat = list(pd.read_excel(file_station, sheet_name='Sheet1')[u'纬度'])
    # 建立站点数据列表
    qixiangday=['stations_n','timeavailas','lngs','lats','station_n_tem_ave','station_n_tem_ave_max','station_n_tem_ave_min',
                  'station_n_shuiqiya_ave','station_n_jiangshui_20','station_n_jiangshui_08','station_n_fengsu_ave']
    qixiangdaylist=[[] for i in range(len(qixiangday))]
    #print(qixiangdaylist)
    station_n_tem_sum = [[] for i in range(len(jiwens))]
    for i in range(0, len(stations_china)):
        # 临时列表
        qixiangtemp = ['station_n','timeavaila', 'lng_n', 'lat_n', 'station_tem_ave', 'station_tem_ave_max','station_tem_ave_min',
                       'station_shuiqiya_ave', 'station_jiangshui_20', 'station_jiangshui_08','station_fengsu_ave']
        qixiangtemplist=[[] for i in range(len(qixiangtemp))]
        # 符合条件则建立列表
        for j in range(0, len(datacolumnslist[0])):
            if datacolumnslist[0][j] == stations_china[i]:
                print(datacolumnslist[0][j])
                qixiangtemplist[0].append(datacolumnslist[0][j])  # 区站号
                qixiangtemplist[1].append(datacolumnslist[1][j])  # 有效时段
                qixiangtemplist[2].append(lng[i])  # 经度
                qixiangtemplist[3].append(lat[i])  # 纬度
                qixiangtemplist[4].append(datacolumnslist[2][j])  # 累年日平均气温
                qixiangtemplist[5].append(datacolumnslist[3][j])  # 累年平均日最高气温
                qixiangtemplist[6].append(datacolumnslist[4][j])  # 累年平均日最低气温
                qixiangtemplist[7].append(datacolumnslist[5][j])  # 累年日平均水汽压
                qixiangtemplist[8].append(datacolumnslist[6][j])  # 累年20-20时日降水量
                qixiangtemplist[9].append(datacolumnslist[7][j])  # 累年08-08时日降水量
                qixiangtemplist[10].append(datacolumnslist[8][j])  # 累年日平均风速
        #print(qixiangtemplist[9])
        if len(qixiangtemplist[4]) != 0:
            #print(qixiangtemplist[1])
            qixiangdaylist[0].append(qixiangtemplist[0][0])  # 区站号
            qixiangdaylist[1].append(str(qixiangtemplist[1][0]))  # 有效时段
            qixiangdaylist[2].append(qixiangtemplist[2][0])  # 经度
            qixiangdaylist[3].append(qixiangtemplist[3][0])  # 纬度
            # 求平均值
            qixiangdaylist[4].append(sum(qixiangtemplist[4]) / len(qixiangtemplist[4]))  # 累年日平均气温
            qixiangdaylist[5].append(sum(qixiangtemplist[5]) / len(qixiangtemplist[5]))  # 累年平均日最高气温
            qixiangdaylist[6].append(sum(qixiangtemplist[6]) / len(qixiangtemplist[6]))  # 累年平均日最低气温
            qixiangdaylist[7].append(sum(qixiangtemplist[7]) / len(qixiangtemplist[9]))  # 累年日平均水汽压
            qixiangdaylist[8].append(sum(qixiangtemplist[8]) / len(qixiangtemplist[8]))  # 累年20-20时日降水量
            qixiangdaylist[9].append(sum(qixiangtemplist[9]) / len(qixiangtemplist[9]))  # 累年08-08时日降水量
            qixiangdaylist[10].append(sum(qixiangtemplist[10]) / len(qixiangtemplist[10]))  # 累年日平均风速
            # 求活动积温
            for x in range(len(jiwens)):
                tem_sum = []
                for tem in qixiangtemplist[4]:
                    if tem > jiwens[x]:
                        tem_sum.append(tem)
                    else:
                        pass
                station_n_tem_sum[x].append(sum(tem_sum))
    #print(qixiangdaylist[10])
    # 建立积温更新后的列表
    for i in range(len(jiwens)):
        qixiangday.append('jiwen%s' % jiwens[i])
        qixiangdaylist.append(station_n_tem_sum[i])
    dfs=pd.DataFrame(qixiangdaylist)
    dfs=dfs.T
    dfs.columns=qixiangday
    print(dfs.head())
    dfs.to_csv(file_n, sep=',', encoding='utf-8')
    print('已完成气象数据提取!')

if __name__=='__main__':
    time_start = datetime.now()
    print('开始时间:' + str(time_start))
    '''第一步，将txt文件转化为csv'''
    folder = "D:\\Database\\02China\\04Qixiang\\510000\\"
    #folder="C:\\Users\\jli\\Desktop\\AAA"
    step1=input('是否进行文件转换:')
    if int(step1)==0:
        #filecsv=transfer_data(folder)
        filecsv= "D:\\Database\\02China\\04Qixiang\\510000\\SURF_CHN_MUL_MDAY_19812010.csv"
        '''第二步，提取每个站点的数据'''
        step2=input("是否提取站点数据:")
        if int(step2)==0:
            file_station="D:\\Database\\02China\\04Qixiang\\SURF_CHN_MUL_STATION.xlsx"
            jiwens=[0]
            extract_data(filecsv, jiwens, file_station)
    time_end = datetime.now()
    print('结束时间:' + str(time_end))
    time_last = time_end - time_start
    print('用时' + str(time_last))


import copy

import mock
import testtools

from stackalytics.processor import default_data_processor
from stackalytics.processor import normalizer
from stackalytics.tests.unit import test_data

class TestDefaultDataProcessor(testtools.TestCase):
    def setUp(self):
        super(TestDefaultDataProcessor, self).setUp()

        self.get_users = mock.Mock(return_value=[
            test_data.USERS,
        ])

        normalized_data = copy.deepcopy(test_data.DEFAULT_DATA)
        normalizer.normalize_default_data(normalized_data)

    def tearDown(self):
        super(TestDefaultDataProcessor, self).tearDown()

    def test_normalizer(self):
        data = copy.deepcopy(test_data.DEFAULT_DATA)

        normalizer.normalize_default_data(data)

        self.assertIn('releases', data['repos'][0])
        self.assertEqual([], data['repos'][0]['releases'],
                         message='Empty list of releases expected')
        self.assertEqual(0, data['users'][0]['companies'][-1]['end_date'],
                         message='The last company end date should be 0')
        self.assertIn('user_id', data['users'][0])
        self.assertEqual(test_data.USERS[0]['launchpad_id'],
                         data['users'][0]['user_id'],
                         message='User id should be set')

        # verify that *independent company is added automatically
        self.assertEqual(3, len(data['users'][1]['companies']))
        self.assertEqual(0, data['users'][1]['companies'][-1]['end_date'],
                         message='The last company end date should be 0')

    def test_update_project_list(self):
        with mock.patch('stackalytics.processor.default_data_processor.'
                        '_retrieve_project_list_from_gerrit') as retriever:
            retriever.return_value = [
                {'module': 'nova',
                 'uri': 'git://git.openstack.org/openstack/nova',
                 'organization': 'openstack'},
                {'module': 'qa', 'uri': 'git://git.openstack.org/openstack/qa',
                 'has_gerrit': True,
                 'organization': 'openstack'},
            ]
            dd = {
                'repos': [
                    {'module': 'qa',
                     'uri': 'git://git.openstack.org/openstack/qa',
                     'organization': 'openstack'},
                    {'module': 'tux',
                     'uri': 'git://git.openstack.org/stackforge/tux',
                     'organization': 'stackforge'},
                ],
                'project_sources': [{'organization': 'openstack',
                                     'uri': 'gerrit://'}],
                'module_groups': [],
            }

            default_data_processor._update_project_list(dd)

            self.assertEqual(3, len(dd['repos']))
            self.assertIn('qa', set([r['module'] for r in dd['repos']]))
            self.assertIn('nova', set([r['module'] for r in dd['repos']]))
            self.assertIn('tux', set([r['module'] for r in dd['repos']]))

            self.assertIn('has_gerrit', dd['repos'][0])
            self.assertNotIn('has_gerrit', dd['repos'][1])
            self.assertNotIn('has_gerrit', dd['repos'][2])

            self.assertEqual(2, len(dd['module_groups']))
            self.assertIn({'id': 'openstack',
                           'module_group_name': 'openstack',
                           'modules': ['qa', 'nova'],
                           'tag': 'organization'}, dd['module_groups'])
            self.assertIn({'id': 'stackforge',
                           'module_group_name': 'stackforge',
                           'modules': ['tux'],
                           'tag': 'organization'}, dd['module_groups'])

    def test_update_project_list_ext_project_source(self):
        with mock.patch('stackalytics.processor.default_data_processor.'
                        '_retrieve_project_list_from_github') as retriever:
            retriever.return_value = [
                {'module': 'kubernetes',
                 'uri': 'git://github.com/kubernetes/kubernetes',
                 'organization': 'kubernetes'},
            ]
            dd = {
                'repos': [],
                'project_sources': [
                    {'organization': 'kubernetes',
                     'uri': 'github://',
                     'module_group_id': 'kubernetes-group'},
                ],
                'module_groups': [],
            }

            default_data_processor._update_project_list(dd)

            self.assertEqual(1, len(dd['repos']))
            self.assertIn('kubernetes',
                          set([r['module'] for r in dd['repos']]))

            self.assertEqual(1, len(dd['module_groups']))
            self.assertIn({'id': 'kubernetes-group',
                           'module_group_name': 'kubernetes',
                           'modules': ['kubernetes'],
                           'tag': 'organization'}, dd['module_groups'])

    @mock.patch('stackalytics.processor.utils.read_json_from_uri')
    def test_update_with_driverlog(self, mock_read_from_json):
        default_data = {'repos': [{'module': 'cinder', }], 'users': []}
        driverlog_dd = {'drivers': [{
            'project_id': 'openstack/cinder',
            'vendor': 'VMware',
            'name': 'VMware VMDK Driver',
            'ci': {
                'id': 'vmwareminesweeper',
                'success_pattern': 'Build successful',
                'failure_pattern': 'Build failed'
            }
        }]}
        mock_read_from_json.return_value = driverlog_dd

        default_data_processor._update_with_driverlog_data(default_data, 'uri')

        expected_user = {
            'user_id': 'ci:vmware_vmdk_driver',
            'user_name': 'VMware VMDK Driver',
            'static': True,
            'companies': [
                {'company_name': 'VMware', 'end_date': None}],
        }
        self.assertIn(expected_user, default_data['users'])
        self.assertIn(driverlog_dd['drivers'][0],
                      default_data['repos'][0]['drivers'])

    @mock.patch('stackalytics.processor.utils.read_json_from_uri')
    def test_update_with_driverlog_specific_repo(self, mock_read_from_json):
        default_data = {'repos': [{'module': 'fuel-plugin-mellanox', }],
                        'users': []}
        driverlog_dd = {'drivers': [{
            'project_id': 'openstack/fuel',
            'repo': 'stackforge/fuel-plugin-mellanox',
            'vendor': 'Mellanox',
            'name': 'ConnectX-3 Pro Network Adapter Support plugin',
            'ci': {
                'id': 'mellanox',
                'success_pattern': 'SUCCESS',
                'failure_pattern': 'FAILURE'
            }
        }]}
        mock_read_from_json.return_value = driverlog_dd

        default_data_processor._update_with_driverlog_data(default_data, 'uri')

        expected_user = {
            'user_id': 'ci:connectx_3_pro_network_adapter_support_plugin',
            'user_name': 'ConnectX-3 Pro Network Adapter Support plugin',
            'static': True,
            'companies': [
                {'company_name': 'Mellanox', 'end_date': None}],
        }
        self.assertIn(expected_user, default_data['users'])
        self.assertIn(driverlog_dd['drivers'][0],
                      default_data['repos'][0]['drivers'])


#Import libraries
import numpy as np
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func
from flask import Flask, jsonify

#Database Connection
engine = create_engine("sqlite:///Resources/hawaii.sqlite")

#Map database
Base = automap_base()
Base.prepare(engine, reflect = True)

#Reference known tables
measurement = Base.classes.measurement
station = Base.classes.station

#Set up flask
app = Flask(__name__)

#Flask Routes
@app.route("/")
def home():
    return(
        f"Available Routes:
"
        f"/api/v1.0/precipitation
"
        f"/api/v1.0/stations
"
        f"/api/v1.0/tobs
"
        f"/api/v1.0/[enter start date: yyyy-mm-dd]
"
        f"/api/v1.0/[enter start date: yyyy-mm-dd]/[enter end date: yyyy-mm-dd]"
    )

@app.route("/api/v1.0/precipitation")
def precipitation():
    #Create session & query
    session = Session(engine)
    results = session.query(measurement.date, measurement.prcp).\
        filter(measurement.date > "2016-08-23").all()
    session.close()

    #Create dictionary and append to list
    prcp_list = []
    for date, prcp in results:
        prcp_dict = {}
        prcp_dict['date'] = date
        prcp_dict['prcp'] = prcp
        prcp_list.append(prcp_dict)

    #Return json
    return jsonify(prcp_list)

@app.route("/api/v1.0/stations")
def stations():
    #Create session & query
    session = Session(engine)
    stations = session.query(station.station).all()
    station_list = list(np.ravel(stations))
    session.close()

    #Return json
    return jsonify(station_list)

@app.route("/api/v1.0/tobs")
def tobs():
    #Create session & query
    session = Session(engine)
    tobs_results = session.query(measurement.station, measurement.date, measurement.tobs).\
        filter(measurement.station == 'USC00519281').\
        filter(measurement.date >'2016-08-23').all()
    session.close()

    #Create dictionary and append to list
    tobs_list = []
    for station, date, tobs in tobs_results:
        tobs_dict = {}
        tobs_dict['station'] = station
        tobs_dict['date'] = date
        tobs_dict['tobs'] = tobs
        tobs_list.append(tobs_dict)

    #Return json
    return jsonify(tobs_list)

@app.route("/api/v1.0/<start_date>")
def Start_date(start_date):
    #Create session & query
    session = Session(engine)
    start_results = session.query( func.avg(measurement.tobs), func.max(measurement.tobs), func.min(measurement.tobs)).\
        filter(measurement.date >= start_date)
    session.close()

    #Create dictionary and append to list
    tobs_start_list = []
    for avg, max, min in start_results:
        start_dict = {}
        start_dict['avg'] = avg
        start_dict['max'] = max
        start_dict['min'] = min
        tobs_start_list.append(start_dict)

    #Return json
    return jsonify(tobs_start_list)

@app.route("/api/v1.0/<start_date>/<end_date>")
def Start_end_date(start_date, end_date):
    #Create session & query
    session = Session(engine)
    start_results = session.query( func.avg(measurement.tobs), func.max(measurement.tobs), func.min(measurement.tobs)).\
        filter(measurement.date >= start_date).\
        filter(measurement.date <= end_date)
    session.close()

    #Create dictionary and append to list
    tobs_start_end_list = []
    for avg, max, min in start_results:
        start_end_dict = {}
        start_end_dict['avg'] = avg
        start_end_dict['max'] = max
        start_end_dict['min'] = min
        tobs_start_end_list.append(start_end_dict)

    #Return json
    return jsonify(tobs_start_end_list)

if __name__ == '__main__':
    app.run(debug=True)


"""added Client Favourite and product views

Revision ID: bcc08ae9bed7
Revises: 399549c08a2a
Create Date: 2020-01-24 22:55:04.098191

"""
from alembic import op
import sqlalchemy as sa
import sqlalchemy_utils

# revision identifiers, used by Alembic.
revision = 'bcc08ae9bed7'
down_revision = '399549c08a2a'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('client_favourite',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('product_id', sa.Integer(), nullable=False),
    sa.Column('client_id', sa.Integer(), nullable=False),
    sa.Column('created', sa.DateTime(), nullable=True),
    sa.Column('updated', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['client_id'], ['client.id'], ),
    sa.ForeignKeyConstraint(['product_id'], ['product.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('product_view',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('product_id', sa.Integer(), nullable=False),
    sa.Column('client_id', sa.Integer(), nullable=True),
    sa.Column('client_ip', sa.Integer(), nullable=False),
    sa.Column('created', sa.DateTime(), nullable=True),
    sa.Column('updated', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['client_id'], ['client.id'], ),
    sa.ForeignKeyConstraint(['client_ip'], ['product.id'], ),
    sa.ForeignKeyConstraint(['product_id'], ['product.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('product_view')
    op.drop_table('client_favourite')
    # ### end Alembic commands ###


# flake8: noqa

import nltk

nltk.download("brown")
nltk.download("names")
import numpy as np
import multiprocessing as mp
import string
import spacy
import os

os.system("python -m spacy download en_core_web_sm")
from sklearn.base import TransformerMixin, BaseEstimator
from normalise import normalise
import pandas as pd

class TextPreprocessor(BaseEstimator, TransformerMixin):
    def __init__(self, variety="BrE", user_abbrevs={}, n_jobs=1):
        """
        Text preprocessing transformer includes steps:
            1. Text normalization
            2. Punctuation removal
            3. Stop words removal
            4. Lemmatization

        variety - format of date (AmE - american type, BrE - british format)
        user_abbrevs - dict of user abbreviations mappings (from normalise package)
        n_jobs - parallel jobs to run
        """

        self.variety = variety
        self.user_abbrevs = user_abbrevs
        self.n_jobs = n_jobs
        self.nlp = spacy.load("en_core_web_sm")

    def fit(self, X, y=None):
        return self

    def transform(self, X, *_):
        X_copy = X.copy()

        partitions = 1
        cores = mp.cpu_count()
        if self.n_jobs <= -1:
            partitions = cores
        elif self.n_jobs <= 0:
            return X_copy.apply(self._preprocess_text)
        else:
            partitions = min(self.n_jobs, cores)

        data_split = np.array_split(X_copy, partitions)
        pool = mp.Pool(cores)
        data = pd.concat(pool.map(self._preprocess_part, data_split))
        pool.close()
        pool.join()

        return data

    def _preprocess_part(self, part):
        return part.apply(self._preprocess_text)

    def _preprocess_text(self, text):
        normalized_text = self._normalize(text)
        doc = self.nlp(normalized_text)
        removed_punct = self._remove_punct(doc)
        removed_stop_words = self._remove_stop_words(removed_punct)
        return self._lemmatize(removed_stop_words)

    def _normalize(self, text):
        # some issues in normalise package
        try:
            return " ".join(
                normalise(
                    text,
                    variety=self.variety,
                    user_abbrevs=self.user_abbrevs,
                    verbose=False,
                )
            )
        except:
            return text

    def _remove_punct(self, doc):
        return [t for t in doc if t.text not in string.punctuation]

    def _remove_stop_words(self, doc):
        return [t for t in doc if not t.is_stop]

    def _lemmatize(self, doc):
        return " ".join([t.lemma_ for t in doc])

def preprocess_text(save_path):
    df_bbc = pd.read_csv("data/bbc-text.csv")
    text = TextPreprocessor(n_jobs=-1).transform(df_bbc["text"])
    df_bbc["text"] = text
    df_bbc.to_csv(save_path, sep=",", index=False)
    return True


#!/usr/bin/env python3
import re
import sys
import linecache
from pathlib import Path

regex = re.compile('#?(.*)\s?=\s?(.*)')
data = ''

try:
    fpath = str(sys.argv[1])
    if not Path(fpath).is_file():
        raise Exception("file path is invalid or not a file")
except:
    print("Error: file not provided or invalid file.")
    sys.exit(1)

try:
  with open(fpath, 'r') as fh:
    lines = fh.readlines()
    for line in lines:
        if re.search('^\n', line):
            data += line
        elif regex.search(line):
            match = regex.search(line)

            comment = ''
            if re.search('^#.*', line):
                comment = '#'

            data += (comment + match.group(1).strip() + "={{ getenv(\"" + match.group(1).replace('.', '_').upper().strip() + "\", \"" + match.group(2).strip() + "\") }}" + "\n")
        elif re.search('^#.*', line):
            data += line
        else:
            pass

  with open(fpath + '.new', 'w') as fh:
    fh.write(data)
except Exception as err:
    exc_type, exc_obj, tb = sys.exc_info()
    f = tb.tb_frame
    lineno = tb.tb_lineno
    filename = f.f_code.co_filename
    linecache.checkcache(filename)
    line = linecache.getline(filename, lineno, f.f_globals)
    print('EXCEPTION IN ({}, LINE {} "{}"): {}'.format(filename, lineno, line.strip(), exc_obj))


from django.shortcuts import render
from admin_dashboard.models import Review, Package
from django.contrib import messages

def home(request):
    reviews = Review.objects.all()
    packages = Package.objects.all()
    return render(request, 'index.html', {
        'title': 'Home',
        'reviews': reviews,
        'packages': packages,
        'request': request,
        })

def handler404(request, exception, template_name='404.html'):
    response = render_to_response(template_name)
    response.status_code = 404
    return response

def handler500(request, exception, template_name='500.html'):
    response = render_to_response(template_name)
    response.status_code = 500
    return response


import pygame as p
from Chess import ChessEngine, SmartMoveFinder, DataToLearn, VisualizData
import time
import xml.etree.ElementTree as gfg
import os.path
from Chess.DataTree import TreeNode

WIDTH = HEIGHT = 512
DIMENSION = 8
SQ_SIZE = HEIGHT // DIMENSION
MAX_FPS = 15
IMAGES = {}
WHITE_PIECE_CAPTUED = []
BLACK_PIECE_CAPTUED = []
BLACK_EVALUATION = 0.5  # Win percentage at start
test = p.Rect(200 + HEIGHT * (1 - BLACK_EVALUATION), HEIGHT + 20 + 60,
              WIDTH * BLACK_EVALUATION, 50)  # Show win percentage

# Here we load the image ones
def loadImages():
    pieces = ['P', 'R', 'N', 'B', 'K', 'Q', 'p', 'r', 'n', 'b', 'k', 'q']
    pics_name = ['wP', 'wR', 'wN', 'wB', 'wK', 'wQ', 'p', 'r', 'n', 'b', 'k', 'q']
    for i in range(len(pieces)):
        IMAGES[pieces[i]] = p.transform.scale(p.image.load("allData/images/" + pics_name[i] + ".png"),
                                              (SQ_SIZE, SQ_SIZE))

def main():
    global MOVE_TIME_START
    global TREE
    global CURRENT_NODE
    p.init()
    # loading the history
    if os.stat('Tree_history_next.xml').st_size == 0:
        start = "<Level-0><State><FEN>rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR</FEN><ImportedEvaluation>0</ImportedEvaluation><Evaluation>0</Evaluation><Wins>0</Wins><Plays>0</Plays><Move /></State></Level-0>"
        fileer = open("Tree_history_next.xml", "wb")
        fileer.write(start)
        fileer.close()

    MOVE_TIME_START = time.time()
    tree = gfg.parse('Tree_history_next.xml').getroot()
    TREE = TreeNode([tree.getchildren()[0].getchildren()[0].text, tree.getchildren()[0].getchildren()[1].text,
                     tree.getchildren()[0].getchildren()[2].text, tree.getchildren()[0].getchildren()[3].text,
                     tree.getchildren()[0].getchildren()[4].text], "")
    if len(tree.getchildren()[0].getchildren()) > 6:
        TREE.read_tree(tree.getchildren()[0].getchildren()[6])
    print("Time to load: ", time.time() - MOVE_TIME_START)
    CURRENT_NODE = TREE
    VisualizData.visualize_data_print(CURRENT_NODE)
    loadImages()
    moveSound = p.mixer.Sound('allData/sound/moveSound.wav')
    conquerePieceSound = p.mixer.Sound('allData/sound/conquerePieceSound.mp3')
    screen = p.display.set_mode((WIDTH + 400, HEIGHT + 80 + 60))
    p.display.set_caption("K(ing) AI")
    clock = p.time.Clock()
    screen.fill(p.Color("beige"))
    gs = ChessEngine.GameState()
    gs.moveLogLong.append(CURRENT_NODE)
    first_move = True
    validMoves = gs.getValidMoves()
    moveMade = False  # flag variable for a valid move
    animate = False  # flag whether eh should animate
    running = True
    sqSelected = ()  # keep track of the last click of the user tuple (row,col)
    playerClicks = []  # keeps track of player clicks (two tuples: [(row,col),(row,col)]
    gameOver = False
    playerOne = False  # If human is playing white = true
    playerTwo = False  # If human is playing black = true

    while running:
        humanTurn = (gs.whiteToMove and playerOne) or (not gs.whiteToMove and playerTwo)
        for e in p.event.get():
            if e.type == p.QUIT:
                running = False
            # Mouse handler
            elif e.type == p.MOUSEBUTTONDOWN:
                if not gameOver and humanTurn:
                    location = p.mouse.get_pos()  # (x,y) of mouse
                    col = (location[0] - 200) // SQ_SIZE
                    row = (location[1] - 10 - 60) // SQ_SIZE
                    if location[0] > 200 and location[0] < WIDTH + 200 and location[1] > 10 + 60 and location[
                        1] < HEIGHT + 10 + 60:
                        if sqSelected == (row, col):  # check whether the user clicked the square twice
                            sqSelected = ()  # deselect
                            playerClicks = []  # clear player clicks
                        else:
                            sqSelected = (row, col)
                            playerClicks.append(sqSelected)
                        if len(playerClicks) == 2:
                            move = ChessEngine.Move(playerClicks[0], playerClicks[1], gs.board)

                            for i in range(len(validMoves)):
                                if move == validMoves[i]:
                                    if len(CURRENT_NODE.children) < 1:
                                        timer = time.time()
                                        SmartMoveFinder.addToTree(gs, CURRENT_NODE)
                                        print("time to add: ", time.time() - timer)
                                    gs.makeMove(validMoves[i])
                                    CURRENT_NODE = changeCurrentNode(gs)
                                    gs.moveLogLong.append(CURRENT_NODE)
                                    moveMade = True
                                    animate = True
                                    first_move = False
                                    print("Player-time: ", time.time() - MOVE_TIME_START)
                                    MOVE_TIME_START = time.time()
                                    if move.pieceCaptured != "--":
                                        conquerePieceSound.play()
                                    else:
                                        moveSound.play()
                                    # recalculate the win percentage
                                    BLACK_EVALUATION = 0.5 - float(CURRENT_NODE.evaluation) / 1000
                                    if BLACK_EVALUATION > 0.99:
                                        BLACK_EVALUATION = 0.99
                                    test.update(200 + HEIGHT * (1 - BLACK_EVALUATION), HEIGHT + 20 + 60,
                                                WIDTH * BLACK_EVALUATION, 50)
                                    sqSelected = ()
                                    playerClicks = []
                            if not moveMade:
                                playerClicks = [sqSelected]

            # key handers
            elif e.type == p.KEYDOWN:
                if e.key == p.K_z:  # undo if "z" is pressed
                    gs.undoMove()
                    moveMade = True
                    animate = False
                    gameOver = False
                if e.key == p.K_r:  # reset the board when pressing "r"
                    gs = ChessEngine.GameState()
                    validMoves = gs.getValidMoves()
                    sqSelected = ()
                    playerClicks = []
                    moveMade = False
                    animate = False
                    gameOver = False
                if e.key == p.K_1:  # save data
                    MOVE_TIME_START = time.time()
                    TREE.save_tree("Tree_history_next.xml")
                    print("Time to save: ", time.time() - MOVE_TIME_START)
                if e.key == p.K_2:
                    MOVE_TIME_START = time.time()
                    # updatePlayAndWins(gs) # save?
                    gs.checkMate = True
                    gameOver = True
                    print("Time to update the data: ", time.time() - MOVE_TIME_START)
                if e.key == p.K_3:
                    gs.staleMate = True
                    gameOver = True
                # white lost (from surrender)
                if e.key == p.K_4:
                    if gs.whiteToMove:
                        gs.checkMate = True
                        gameOver = True
                else:
                    gs.whiteToMove = True
                    gs.checkMate = True
                    gameOver = True
                # black lost (from surrender)
                if e.key == p.K_5:
                    if not gs.whiteToMove:
                        gs.checkMate = True
                        gameOver = True
                else:
                    gs.whiteToMove = False
                    gs.checkMate = True
                    gameOver = True

        # AI Movefinder
        if not gameOver and not humanTurn:
            if len(CURRENT_NODE.children) < 1:
                timer = time.time()
                SmartMoveFinder.addToTree(gs, CURRENT_NODE)
                print("time to add: ", time.time() - timer)
            timerrr = time.time()
            AImove = SmartMoveFinder.findBestMoveMinMax(gs, validMoves, CURRENT_NODE)
            print("find new Move: ", time.time() - timerrr)
            if AImove is None:
                AImove = SmartMoveFinder.findRandomMove(validMoves)
            gs.makeMove(AImove)
            moveMade = True
            animate = True
            CURRENT_NODE = changeCurrentNode(gs)
            gs.moveLogLong.append(CURRENT_NODE)
            print("AI: ", time.time() - MOVE_TIME_START)
            MOVE_TIME_START = time.time()
            if AImove.pieceCaptured != "--":
                conquerePieceSound.play()
            else:
                moveSound.play()
            # recalculate the win percentage
            BLACK_EVALUATION = 0.5 - float(CURRENT_NODE.evaluation) / 1000
            if BLACK_EVALUATION > 0.99:
                BLACK_EVALUATION = 0.99
            test.update(200 + HEIGHT * (1 - BLACK_EVALUATION), HEIGHT + 20 + 60,
                        WIDTH * BLACK_EVALUATION, 50)

        if moveMade:
            if animate:
                animateMove(gs.moveLog[-1], screen, gs.board, clock)
            validMoves = gs.getValidMoves()
            moveMade = False
            animate = False

        drawGameState(screen, gs, validMoves, sqSelected)
        if gs.checkMate:
            gameOver = True
            if gs.whiteToMove:
                font1 = p.font.SysFont('Black wins by checkmate', 64)
                img1 = font1.render('Black wins by checkmate', True, "dark red")
                screen.blit(img1, (210, 280))
            else:
                font1 = p.font.SysFont('White wins by checkmate', 64)
                img1 = font1.render('White wins by checkmate', True, "dark red")
                screen.blit(img1, (210, 280))
                # drawText(screen, 'White wins by checkmate')
            updatePlayAndWins(gs)
            MOVE_TIME_START = time.time()
            TREE.save_tree("Tree_history_next.xml")
            print("Time to save: ", time.time() - MOVE_TIME_START)
            running = False
            time.sleep(60)
        elif gs.staleMate:
            font1 = p.font.SysFont('Stalemate', 64)
            img1 = font1.render('Stalemate', True, "dark red")
            screen.blit(img1, (210, 280))
            updatePlayAndWins(gs)
            MOVE_TIME_START = time.time()
            TREE.save_tree("Tree_history_next.xml")
            print("Time to save: ", time.time() - MOVE_TIME_START)
            running = False
            time.sleep(60)
        clock.tick(MAX_FPS)
        p.display.flip()

# Higlicght the square selected
def highlightSqaures(screen, gs, validMoves, sqSelected):
    if sqSelected != ():
        r, c = sqSelected
        if (gs.board[r][c].islower() and not gs.whiteToMove) or (not gs.board[r][c].islower() and gs.whiteToMove):
            # if gs.board[r][c][0] == ('w' if gs.whiteToMove else 'b'):  # sqSelected a piece that can be moved
            # highlight selected squares
            s = p.Surface((SQ_SIZE, SQ_SIZE))
            s.set_alpha(100)  # transparancy value
            s.fill(p.Color('blue'))
            screen.blit(s, ((c * SQ_SIZE) + 200, (r * SQ_SIZE) + 10 + 60))
            # highlight moves from that square
            s.fill(p.Color('yellow'))
            for move in validMoves:
                if move.startRow == r and move.startCol == c:
                    screen.blit(s, ((SQ_SIZE * move.endCol) + 200, (move.endRow * SQ_SIZE) + 10 + 60))

def drawGameState(screen, gs, validMoves, sqSelected):
    drawBoard(screen)  # draw squares on the board
    highlightSqaures(screen, gs, validMoves, sqSelected)
    highlightLastMove(gs, screen)
    drawPieces(screen, gs.board)  # draw pieces on the top of those squares

def drawBoard(screen):
    global colors
    p.draw.rect(screen, "black", p.Rect(194, 64, HEIGHT + 12, WIDTH + 12))
    p.draw.rect(screen, "white", p.Rect(200 + WIDTH / 4, 10, WIDTH / 4, 40))
    p.draw.rect(screen, "grey", p.Rect(200 + WIDTH / 2, 10, WIDTH / 4, 40))
    p.draw.rect(screen, "white", p.Rect(200, HEIGHT + 20 + 60, WIDTH, 50))
    p.draw.rect(screen, "grey", test)

    colors = [p.Color("white"), p.Color("grey")]
    for r in range(DIMENSION):
        for c in range(DIMENSION):
            color = colors[((r + c) % 2)]
            p.draw.rect(screen, color, p.Rect(c * SQ_SIZE + 200, r * SQ_SIZE + 10 + 60, SQ_SIZE, SQ_SIZE))

def drawPieces(screen, board):
    for r in range(DIMENSION):
        for c in range(DIMENSION):
            piece = board[r][c]
            if piece != "--":
                screen.blit(IMAGES[piece], p.Rect(c * SQ_SIZE + 200, r * SQ_SIZE + 10 + 60, SQ_SIZE, SQ_SIZE))

def animateMove(move, screen, board, clock):
    global colors
    dR = move.endRow - move.startRow
    dC = move.endCol - move.startCol
    framesPerSquare = 4  # frames to move one square
    frameCount = (abs(dR) + abs(dC)) * framesPerSquare
    for frame in range(frameCount + 1):
        r, c = (move.startRow + dR * frame / frameCount, move.startCol + dC * frame / frameCount)
        drawBoard(screen)
        drawPieces(screen, board)
        # erase the piece moved from its ending sqaure
        color = colors[(move.endRow + move.endCol) % 2]
        endSquare = p.Rect((move.endCol * SQ_SIZE) + 200, (move.endRow * SQ_SIZE) + 10 + 60, SQ_SIZE, SQ_SIZE)
        p.draw.rect(screen, color, endSquare)
        # draw captured piece onto rectangle
        if move.pieceCaptured != '--':
            screen.blit(IMAGES[move.pieceCaptured], endSquare)
        # draw moving piece
        screen.blit(IMAGES[move.pieceMoved], p.Rect((c * SQ_SIZE) + 200, (r * SQ_SIZE) + 10 + 60, SQ_SIZE, SQ_SIZE))
        p.display.flip()
        clock.tick(60)
    if move.pieceCaptured.isupper():
        screen.blit(IMAGES[move.pieceCaptured],
                    p.Rect(((len(WHITE_PIECE_CAPTUED) % 3)) * SQ_SIZE,
                           (len(WHITE_PIECE_CAPTUED) // 3) * SQ_SIZE + 10 + 60, SQ_SIZE, SQ_SIZE))
        WHITE_PIECE_CAPTUED.append(move.pieceCaptured)
    elif move.pieceCaptured.islower():
        screen.blit(IMAGES[move.pieceCaptured],
                    p.Rect(((len(BLACK_PIECE_CAPTUED) % 3)) * SQ_SIZE + WIDTH + 210,
                           (len(BLACK_PIECE_CAPTUED) // 3) * SQ_SIZE + 10 + 60, SQ_SIZE, SQ_SIZE))
        BLACK_PIECE_CAPTUED.append(move.pieceCaptured)

# highlight last move made
def highlightLastMove(gs, screen):
    if len(gs.moveLog) > 0:
        s = p.Surface((SQ_SIZE, SQ_SIZE))
        s.set_alpha(100)  # transparancy value
        s.fill(p.Color('red'))
        screen.blit(s, ((gs.moveLog[-1].startCol * SQ_SIZE) + 200, (gs.moveLog[-1].startRow * SQ_SIZE) + 10 + 60))
        screen.blit(s, ((gs.moveLog[-1].endCol * SQ_SIZE) + 200, (gs.moveLog[-1].endRow * SQ_SIZE) + 10 + 60))

def drawText(screen, text):
    font = p.font.SysFont("Helvitca", 32, True, False)
    textObject = font.render(text, 0, p.Color('Gray'))
    textLocation = p.Rect(0, 0, WIDTH, HEIGHT).move(WIDTH / 2 - textObject.get_width() / 2,
                                                    HEIGHT / 2 - textObject.get_height() / 2)
    screen.blit(textObject, textLocation)
    textObject = font.render(text, 0, p.Color('Black'))
    screen.blit(textObject, textLocation.move(2, 2))

# change the current Node
def changeCurrentNode(gs):
    fen_now = DataToLearn.createFEN(gs)
    for states in CURRENT_NODE.children:
        if states.fEN == fen_now:
            return states

def updatePlayAndWins(gs):
    print (len(gs.moveLogLong))
    gs.checkMate = True
    TREE.plays = 1 + int(TREE.plays)
    if gs.checkMate and not gs.whiteToMove:
        TREE.wins = int(TREE.wins) + 1
    helperUpdatePlayAndWins(gs, TREE, level=1)

def helperUpdatePlayAndWins(gs, current_node, level):
    print(level)
    for state in current_node.children:
        if state == gs.moveLogLong[level]:
            state.plays = float(state.plays) + 1
            current_node = state
            if gs.checkMate and not gs.whiteToMove:
                state.wins = float(state.wins) + 1
            elif gs.staleMate:
                state.wins = float(state.wins) + 0.5
            if len(gs.moveLogLong) - 1 > level:
                helperUpdatePlayAndWins(gs, current_node, level + 1)

if __name__ == "__main__":
    main()

# todo: Add more evaluations (pieces covered)
# todo: try to get evaluation from the imported data
# todo: Add Player Specific Bot
# todo: make heurustic: which is the first move to watch (makes it faster)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import datetime

class Migration(migrations.Migration):

    dependencies = [
        ('bot', '0007_bot_logo'),
    ]

    operations = [
        migrations.CreateModel(
            name='Chat',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('date', models.DateTimeField(default=datetime.datetime(2016, 5, 15, 20, 22, 16, 862988))),
                ('log', models.TextField()),
                ('bot', models.ForeignKey(to='bot.Bot')),
            ],
        ),
    ]


from __future__ import annotations

import random
from datetime import timedelta
from typing import Type

from game.theater import FrontLine
from game.utils import Distance, Speed, feet
from .capbuilder import CapBuilder
from .invalidobjectivelocation import InvalidObjectiveLocation
from .patrolling import PatrollingFlightPlan, PatrollingLayout
from .waypointbuilder import WaypointBuilder

class BarCapFlightPlan(PatrollingFlightPlan[PatrollingLayout]):
    @staticmethod
    def builder_type() -> Type[Builder]:
        return Builder

    @property
    def patrol_duration(self) -> timedelta:
        return self.flight.coalition.doctrine.cap_duration

    @property
    def patrol_speed(self) -> Speed:
        return self.flight.unit_type.preferred_patrol_speed(
            self.layout.patrol_start.alt
        )

    @property
    def engagement_distance(self) -> Distance:
        return self.flight.coalition.doctrine.cap_engagement_range

class Builder(CapBuilder[BarCapFlightPlan, PatrollingLayout]):
    def layout(self) -> PatrollingLayout:
        location = self.package.target

        if isinstance(location, FrontLine):
            raise InvalidObjectiveLocation(self.flight.flight_type, location)

        start_pos, end_pos = self.cap_racetrack_for_objective(location, barcap=True)

        preferred_alt = self.flight.unit_type.preferred_patrol_altitude
        randomized_alt = preferred_alt + feet(random.randint(-2, 1) * 1000)
        patrol_alt = max(
            self.doctrine.min_patrol_altitude,
            min(self.doctrine.max_patrol_altitude, randomized_alt),
        )

        builder = WaypointBuilder(self.flight, self.coalition)
        start, end = builder.race_track(start_pos, end_pos, patrol_alt)

        return PatrollingLayout(
            departure=builder.takeoff(self.flight.departure),
            nav_to=builder.nav_path(
                self.flight.departure.position, start.position, patrol_alt
            ),
            nav_from=builder.nav_path(
                end.position, self.flight.arrival.position, patrol_alt
            ),
            patrol_start=start,
            patrol_end=end,
            arrival=builder.land(self.flight.arrival),
            divert=builder.divert(self.flight.divert),
            bullseye=builder.bullseye(),
        )

    def build(self, dump_debug_info: bool = False) -> BarCapFlightPlan:
        return BarCapFlightPlan(self.flight, self.layout())


# -*- coding:utf-8 -*-

"""
Evluate the performance of embedding via different methods.
"""

import math

import numpy as np
from sklearn import metrics
from sklearn import utils as sktools
from sklearn.cluster import AgglomerativeClustering
from sklearn.cluster import SpectralClustering
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, classification_report, balanced_accuracy_score, f1_score, precision_score, \
    recall_score
from sklearn.model_selection import cross_val_score
from sklearn.neighbors import KNeighborsClassifier

def cluster_evaluate(embeddings, labels, n_class, metric="euclidean"):
    """
        Unsupervised setting: We assess the ability of each method to embed close together nodes
        with the same ground-truth structural role. We use agglomerative clustering (with single linkage)
        to cluster embeddings learned by each method and evaluate the clustering quality via:
            (1) homogeneity, conditional entropy of ground-truth structural roles given the predicted clustering;
            (2) completeness, a measure of how many nodes with the same ground-truth structural role are assigned to the same cluster;
            (3) silhouette score, a measure of intra-cluster distance vs. inter-cluster distance.

        Supervised setting: We assess the performance of learned embeddings for node classifcation.
        Using 10-fold cross validation, we predict the structural role (label) of each node in the tests set
        based on its 4-nearest neighbors in the training set as determined by the embedding space.
        The reported score is then the average accuracy and F1-score over 25 trials.
    """
    clusters = AgglomerativeClustering(n_clusters=n_class, linkage='single', affinity=metric).fit_predict(embeddings)
    h, c, v = metrics.homogeneity_completeness_v_measure(labels, clusters)
    s = metrics.silhouette_score(embeddings, clusters)
    acc = accuracy_score(labels, clusters)
    macro_f1 = f1_score(labels, clusters, average="macro")
    print("cluster:", clusters, "labels:", labels)
    print("accuracy: ", acc)
    print("macro_score: ", macro_f1)
    print("homogeneity: ", h)
    print("completeness: ", c)
    print("v-score: ", v)
    print("silhouette: ", s)

    return h, c, v, s

def LR_evaluate(data, labels, cv=5):
    """
    Evaluate embedding effect using Logistic Regression. Mode = One vs Rest (OVR)
    """
    data, labels = sktools.shuffle(data, labels)
    lr = LogisticRegression(solver="lbfgs", penalty='l2', max_iter=1000, multi_class='ovr')
    test_scores = cross_val_score(lr, data, y=labels, cv=cv)
    print(f"LR: tests scores={test_scores}, mean_score={np.mean(test_scores)}\n")
    return np.mean(test_scores)

def KNN_evaluate(data, labels, metric="minkowski", cv=5, n_neighbor=10):
    """
    基于节点的相似度进行KNN分类，在嵌入之前进行，为了验证通过层次化相似度的优良特性。
    """
    data, labels = sktools.shuffle(data, labels)
    knn = KNeighborsClassifier(weights='uniform', algorithm="auto", n_neighbors=n_neighbor, metric=metric, p=2)
    test_scores = cross_val_score(knn, data, y=labels, cv=cv, scoring="accuracy")
    print(f"KNN: tests scores:{test_scores}, mean_score={np.mean(test_scores)}\n")
    return np.mean(test_scores)

def evalute_results(labels: list, preds: list):
    accuracy = accuracy_score(labels, preds)
    balanced_accuracy = balanced_accuracy_score(labels, preds)
    precision = precision_score(labels, preds, average="micro")
    recall = recall_score(labels, preds, average="micro")
    macro_f1 = f1_score(labels, preds, average="macro")
    micro_f1 = f1_score(labels, preds, average="micro")
    report = classification_report(labels, preds, digits=7)

    res = { "accuracy": accuracy,
            "balanced accuracy": balanced_accuracy,
            "micro precision": precision,
            "micro recall": recall,
            "macro f1": macro_f1,
            "micro f1": micro_f1,
            "report": report
           }

    print(res)
    return res

def spectral_cluster_evaluate(data, labels, n_cluster, affinity="rbf"):
    """

    :param data: 相似度矩阵 or 嵌入向量
    :param n_cluster:
    :param affinity: precomputed || rbf
    :return:
    """
    metric = "euclidean"
    if affinity == "precomputed":
        # sklearn指导，如果data是距离矩阵而不是相似度矩阵，则可以用下面的rbf转换一下
        distance_mat = data
        delta = math.sqrt(2)
        data = np.exp(-distance_mat ** 2 / (2. * delta ** 2))
        metric = affinity

    clustering = SpectralClustering(n_clusters=n_cluster, affinity=affinity, n_init=50, random_state=42)
    preds = clustering.fit_predict(data)
    h, c, v = metrics.homogeneity_completeness_v_measure(labels, preds)
    s1 = metrics.silhouette_score(embeddings, labels, metric=metric)
    s2 = metrics.silhouette_score(embeddings, preds, metric=metric)

    print(f"homogenetiy: {h}, completeness: {c}, v_measure: {v}, silhouette_score label: {s1}, silhouette_score pred: {s2}\n")


"""
5 element API Client
"""
from typing import Optional, Tuple

from bgd.constants import FIFTHELEMENT
from bgd.responses import GameSearchResult, Price
from bgd.services.abc import GameSearchResultFactory
from bgd.services.api_clients import GameSearcher, JsonHttpApiClient
from bgd.services.base import CurrencyExchangeRateService, GameSearchService
from bgd.services.constants import GET
from bgd.services.responses import APIResponse

class FifthElementApiClient(JsonHttpApiClient):
    """Api client for 5element.by"""

    BASE_SEARCH_URL = "https://api.multisearch.io"

    async def search(self, query: str, options: Optional[dict] = None) -> APIResponse:
        """Search query string"""
        search_app_id = options["search_app_id"]  # type: ignore
        url = f"?query={query}&id={search_app_id}&lang=ru&autocomplete=true"
        return await self.connect(GET, self.BASE_SEARCH_URL, url)

class FifthElementSearchService(GameSearchService):
    """Search service for 5element.by"""

    def __init__(
        self,
        client: GameSearcher,
        result_factory: GameSearchResultFactory,
        currency_exchange_rate_converter: CurrencyExchangeRateService,
        game_category_id: str,
        search_app_id: str,
    ) -> None:
        """Init 5th element Search Service"""
        # there are more than one category that we should check
        self._game_category_ids = game_category_id.split(",")
        super().__init__(client, result_factory, currency_exchange_rate_converter)
        self._search_app_id = search_app_id

    async def do_search(self, query: str, *args, **kwargs) -> Tuple[GameSearchResult]:
        response = await self._client.search(query, {"search_app_id": self._search_app_id})
        products = self.filter_results(
            response.response["results"]["items"], self._is_available_game
        )
        return self.build_results(products)

    def _is_available_game(self, product: dict) -> bool:
        """True if it's available board game"""
        return (
            product["is_presence"]
            and product["params_data"]["category_id"] in self._game_category_ids
        )

class FifthElementGameSearchResultFactory:
    """Builder for GameSearch results from 5element"""

    BASE_URL = "https://5element.by"

    def create(self, search_result: dict) -> GameSearchResult:
        """Build search result"""
        return GameSearchResult(
            description="",
            images=self._extract_images(search_result),
            location=None,
            owner=None,
            prices=[self._extract_price(search_result)],
            source=FIFTHELEMENT,
            subject=search_result["name"],
            url=self._extract_url(search_result),
        )

    @staticmethod
    def _extract_images(product: dict) -> list[str]:
        """Extract product images"""
        return [product["picture"]]

    @staticmethod
    def _extract_price(product: dict) -> Optional[Price]:
        """Extract price"""
        price = product["price"] * 100
        return Price(amount=price)

    def _extract_url(self, product: dict) -> str:
        """Extract product url"""
        return f"{self.BASE_URL}{product['url']}"


from bs4 import BeautifulSoup
import requests
import requests  # request img from web
import shutil  # save img locally
URL1 = 'http://localhost'
page = requests.get(URL1)
soup = BeautifulSoup(page.content, 'html.parser')
print(soup.prettify())
image_tags = soup.find_all('img')
for image_tag in image_tags:
    url = image_tag['src']
    res = requests.get(url, stream=True)
    url = url[url.rfind("/")+1:]
    print(url)
    with open("res/img/ "+url, 'wb') as f:
        shutil.copyfileobj(res.raw, f)
    print('Image sucessfully Downloaded: ', url)


# Enter your code here. Read input from STDIN. Print output to STDOUT
from collections import deque

deq = deque()

for i in range(int(input())):
    command = input().split()
    if command[0] == 'append':
        deq.append(command[1])
    elif command[0] == 'appendleft':
        deq.appendleft(command[1])
    elif command[0] == 'pop':
        deq.pop()
    elif command[0] == 'popleft':
        deq.popleft()
print(*deq)

'''EVAL is not the answer to everything.
AND it's dangerous'''