import mindspore
import mindspore.nn as nn
from mindspore import context
import mindspore.ops as ops
import network.mynn as mynn
import sys
from collections import OrderedDict
from functools import partial
import numpy as np

def bnrelu(channels):
    '''ReLu没有torch中的inpace属性，没有大问题，只是计算会变慢，占用多一点的内存'''
    return nn.SequentialCell(mynn.Norm2d(channels),
                         nn.ReLU())
    # return nn.SequentialCell(nn.BatchNorm2d(channels),
    #                      nn.ReLU())

class GlobalAvgPool2d(nn.Cell):

    def __init__(self):
        """Global average pooling over the input's spatial dimensions"""
        super(GlobalAvgPool2d, self).__init__()

    def construct(self, inputs):
        in_size = inputs.shape
        return inputs.view((in_size[0], in_size[1], -1)).mean(axis=2)

class IdentityResidualBlock(nn.Cell):

    def __init__(self,
                 in_channels,
                 channels,
                 stride=1,
                 dilation=1,
                 groups=1,
                 norm_act=bnrelu,
                 dropout=None,
                 dist_bn=False
                 ):
        """Configurable identity-mapping residual block

        Parameters
        ----------
        in_channels : int
            Number of input channels.
        channels : list of int
            Number of channels in the internal feature maps.
            Can either have two or three elements: if three construct
            a residual block with two `3 x 3` convolutions,
            otherwise construct a bottleneck block with `1 x 1`, then
            `3 x 3` then `1 x 1` convolutions.
        stride : int
            Stride of the first `3 x 3` convolution
        dilation : int
            Dilation to apply to the `3 x 3` convolutions.
        groups : int
            Number of convolution groups.
            This is used to create ResNeXt-style blocks and is only compatible with
            bottleneck blocks.
        norm_act : callable
            Function to create normalization / activation Module.
        dropout: callable
            Function to create Dropout Module.
        dist_bn: Boolean
            A variable to enable or disable use of distributed BN
        """
        super(IdentityResidualBlock, self).__init__()
        self.dist_bn = dist_bn

        # Check if we are using distributed BN and use the nn from encoding.nn
        # library rather than using standard pytorch.nn

        # Check parameters for inconsistencies
        if len(channels) != 2 and len(channels) != 3:
            raise ValueError("channels must contain either two or three values")
        if len(channels) == 2 and groups != 1:
            raise ValueError("groups > 1 are only valid if len(channels) == 3")

        is_bottleneck = len(channels) == 3
        need_proj_conv = stride != 1 or in_channels != channels[-1]

        self.bn1 = norm_act(in_channels)
        if not is_bottleneck:
            layers = [
                ("conv1", nn.Conv2d(in_channels,
                                    channels[0],
                                    3,
                                    stride=stride,
                                    weight_init="heuniform",
                                    pad_mode = "pad",
                                    padding = dilation,
                                    has_bias=False,
                                    dilation=dilation)),
                ("bn2", norm_act(channels[0])),
                ("conv2", nn.Conv2d(channels[0], channels[1],
                                    3,
                                    stride=1,
                                    pad_mode = "pad",
                                    padding = dilation,
                                    weight_init="heuniform",
                                    has_bias=False,
                                    dilation=dilation))
            ]
            if dropout is not None:
                layers = layers[0:2] + [("dropout", dropout())] + layers[2:]
        else:
            layers = [
                ("conv1",
                 nn.Conv2d(in_channels,
                           channels[0],
                           1,
                           stride=stride,
                           weight_init="heuniform",
                           pad_mode = "pad",
                           has_bias=False)),
                ("bn2", norm_act(channels[0])),
                ("conv2", nn.Conv2d(channels[0],
                                    channels[1],
                                    3, stride=1,
                                    weight_init="heuniform",
                                    pad_mode = "pad",
                                    padding = dilation,
                                    has_bias=False,
                                    group=groups,
                                    dilation=dilation)),
                ("bn3", norm_act(channels[1])),
                ("conv3", nn.Conv2d(channels[1], channels[2],
                                    1, stride=1, weight_init="heuniform", pad_mode = "pad", padding = 0,has_bias=False))
            ]
            if dropout is not None:
                layers = layers[0:4] + [("dropout", dropout())] + layers[4:]
        self.convs = nn.SequentialCell(OrderedDict(layers))
        self.proj_conv = nn.SequentialCell()
        if need_proj_conv:
            self.proj_conv = nn.SequentialCell(
                [norm_act(in_channels),
                nn.Conv2d(
                in_channels, channels[-1], 1, stride=stride, has_bias=False, pad_mode = "pad", padding = 0, weight_init="heuniform")])
            # self.proj_conv = nn.SequentialCell(
            #     [nn.Conv2d(in_channels, channels[-1], 1, stride=stride, has_bias=False, weight_init="ones")])
    def construct(self, x):
        """
        This is the standard forward function for non-distributed batch norm
        """
        identity = x

        x = self.bn1(x)
        shortcut = self.proj_conv(identity)
        #shortcut = self.proj_conv(x)
        #print("天明")
        out = self.convs(x)
        out = out + shortcut
        return out

class WiderResNetA2(nn.Cell):

    def __init__(self,
                 structure,
                 norm_act=bnrelu,
                 classes=0,
                 dilation=False,
                 dist_bn=False
                 ):
        """Wider ResNet with pre-activation (identity mapping) blocks

        This variant uses down-sampling by max-pooling in the first two blocks and \
         by strided convolution in the others.

        Parameters
        ----------
        structure : list of int
            Number of residual blocks in each of the six modules of the network.
        norm_act : callable
            Function to create normalization / activation Module.
        classes : int
            If not `0` also include global average pooling and a fully-connected layer
            \with `classes` outputs at the end
            of the network.
        dilation : bool
            If `True` apply dilation to the last three modules and change the
            \down-sampling factor from 32 to 8.
        """
        super(WiderResNetA2, self).__init__()
        self.dist_bn = dist_bn

        # If using distributed batch norm, use the encoding.nn as oppose to torch.nn

        #nn.Dropout = nn.Dropout2d
        '''bnrelu : 自己定义的norm和relu'''
        #norm_act = bnrelu
        self.structure = structure
        self.dilation = dilation

        if len(structure) != 6:
            raise ValueError("Expected a structure with six values")

        # Initial layers
        self.mod1 = nn.SequentialCell(OrderedDict([
            ("conv1", nn.Conv2d(3, 64, 3, stride=1,  has_bias=False, pad_mode = "pad", padding = 1,weight_init="heuniform"))
        ]))

        # Groups of residual blocks
        in_channels = 64
        channels = [(128, 128), (256, 256), (512, 512), (512, 1024), (512, 1024, 2048),
                    (1024, 2048, 4096)]
        mod_id = 0
        num = structure[0]
        blocks = []
        for block_id in range(num):
            if not dilation:
                dil = 1
                stride = 2 if block_id == 0 and 2 <= mod_id <= 4 else 1
            else:
                if mod_id == 3:
                    dil = 2
                elif mod_id > 3:
                    dil = 4
                else:
                    dil = 1
                stride = 2 if block_id == 0 and mod_id == 2 else 1

            if mod_id == 4:
                drop = partial(nn.Dropout, keep_prob=0.3)
            elif mod_id == 5:
                drop = partial(nn.Dropout, keep_prob=0.5)
            else:
                drop = None

            blocks.append((
                "block%d" % (block_id + 1),
                IdentityResidualBlock(in_channels,
                                      channels[mod_id], norm_act=norm_act,
                                      stride=stride, dilation=dil,
                                      dropout=drop, dist_bn=self.dist_bn)
            ))

            # Update channels and p_keep
            in_channels = channels[mod_id][-1]

        self.mod2 = (nn.SequentialCell(OrderedDict(blocks)))
        self.pool2 = nn.MaxPool2d(3, stride=2, pad_mode = "same")

        mod_id = 1
        num = structure[1]
        blocks = []
        for block_id in range(num):
            if not dilation:
                dil = 1
                stride = 2 if block_id == 0 and 2 <= mod_id <= 4 else 1
            else:
                if mod_id == 3:
                    dil = 2
                elif mod_id > 3:
                    dil = 4
                else:
                    dil = 1
                stride = 2 if block_id == 0 and mod_id == 2 else 1

            if mod_id == 4:
                drop = partial(nn.Dropout, keep_prob=0.3)
            elif mod_id == 5:
                drop = partial(nn.Dropout, keep_prob=0.5)
            else:
                drop = None

            blocks.append((
                "block%d" % (block_id + 1),
                IdentityResidualBlock(in_channels,
                                      channels[mod_id], norm_act=norm_act,
                                      stride=stride, dilation=dil,
                                      dropout=drop, dist_bn=self.dist_bn)
            ))

            # Update channels and p_keep
            in_channels = channels[mod_id][-1]

        self.mod3 = (nn.SequentialCell(OrderedDict(blocks)))
        self.pool3 = nn.MaxPool2d(3, stride=2, pad_mode = "same")

        mod_id = 2
        num = structure[2]
        blocks = []
        for block_id in range(num):
            if not dilation:
                dil = 1
                stride = 2 if block_id == 0 and 2 <= mod_id <= 4 else 1
            else:
                if mod_id == 3:
                    dil = 2
                elif mod_id > 3:
                    dil = 4
                else:
                    dil = 1
                stride = 2 if block_id == 0 and mod_id == 2 else 1

            if mod_id == 4:
                drop = partial(nn.Dropout, keep_prob=0.3)
            elif mod_id == 5:
                drop = partial(nn.Dropout, keep_prob=0.5)
            else:
                drop = None

            blocks.append((
                "block%d" % (block_id + 1),
                IdentityResidualBlock(in_channels,
                                      channels[mod_id], norm_act=norm_act,
                                      stride=stride, dilation=dil,
                                      dropout=drop, dist_bn=self.dist_bn)
            ))

            # Update channels and p_keep
            in_channels = channels[mod_id][-1]

        self.mod4 = (nn.SequentialCell(OrderedDict(blocks)))

        mod_id = 3
        num = structure[3]
        blocks = []
        for block_id in range(num):
            if not dilation:
                dil = 1
                stride = 2 if block_id == 0 and 2 <= mod_id <= 4 else 1
            else:
                if mod_id == 3:
                    dil = 2
                elif mod_id > 3:
                    dil = 4
                else:
                    dil = 1
                stride = 2 if block_id == 0 and mod_id == 2 else 1

            if mod_id == 4:
                drop = partial(nn.Dropout, keep_prob=0.3)
            elif mod_id == 5:
                drop = partial(nn.Dropout, keep_prob=0.5)
            else:
                drop = None

            blocks.append((
                "block%d" % (block_id + 1),
                IdentityResidualBlock(in_channels,
                                      channels[mod_id], norm_act=norm_act,
                                      stride=stride, dilation=dil,
                                      dropout=drop, dist_bn=self.dist_bn)
            ))

            # Update channels and p_keep
            in_channels = channels[mod_id][-1]

        self.mod5 = (nn.SequentialCell(OrderedDict(blocks)))

        mod_id = 4
        num = structure[4]
        blocks = []
        for block_id in range(num):
            if not dilation:
                dil = 1
                stride = 2 if block_id == 0 and 2 <= mod_id <= 4 else 1
            else:
                if mod_id == 3:
                    dil = 2
                elif mod_id > 3:
                    dil = 4
                else:
                    dil = 1
                stride = 2 if block_id == 0 and mod_id == 2 else 1

            if mod_id == 4:
                drop = partial(nn.Dropout, keep_prob=0.3)
            elif mod_id == 5:
                drop = partial(nn.Dropout, keep_prob=0.5)
            else:
                drop = None

            blocks.append((
                "block%d" % (block_id + 1),
                IdentityResidualBlock(in_channels,
                                      channels[mod_id], norm_act=norm_act,
                                      stride=stride, dilation=dil,
                                      dropout=drop, dist_bn=self.dist_bn)
            ))

            # Update channels and p_keep
            in_channels = channels[mod_id][-1]

        self.mod6 = (nn.SequentialCell(OrderedDict(blocks)))

        mod_id = 5
        num = structure[5]
        blocks = []
        for block_id in range(num):
            if not dilation:
                dil = 1
                stride = 2 if block_id == 0 and 2 <= mod_id <= 4 else 1
            else:
                if mod_id == 3:
                    dil = 2
                elif mod_id > 3:
                    dil = 4
                else:
                    dil = 1
                stride = 2 if block_id == 0 and mod_id == 2 else 1

            if mod_id == 4:
                drop = partial(nn.Dropout, keep_prob=0.3)
            elif mod_id == 5:
                drop = partial(nn.Dropout, keep_prob=0.5)
            else:
                drop = None

            blocks.append((
                "block%d" % (block_id + 1),
                IdentityResidualBlock(in_channels,
                                      channels[mod_id], norm_act=norm_act,
                                      stride=stride, dilation=dil,
                                      dropout=drop, dist_bn=self.dist_bn)
            ))

            # Update channels and p_keep
            in_channels = channels[mod_id][-1]

        self.mod7 = (nn.SequentialCell(OrderedDict(blocks)))
        # for mod_id, num in enumerate(structure):
        #     # Create blocks for module
        #     blocks = []
        #     for block_id in range(num):
        #         if not dilation:
        #             dil = 1
        #             stride = 2 if block_id == 0 and 2 <= mod_id <= 4 else 1
        #         else:
        #             if mod_id == 3:
        #                 dil = 2
        #             elif mod_id > 3:
        #                 dil = 4
        #             else:
        #                 dil = 1
        #             stride = 2 if block_id == 0 and mod_id == 2 else 1
        #
        #         if mod_id == 4:
        #             drop = partial(nn.Dropout, keep_prob=0.3)
        #         elif mod_id == 5:
        #             drop = partial(nn.Dropout, keep_prob=0.5)
        #         else:
        #             drop = None
        #
        #         blocks.append((
        #             "block%d" % (block_id + 1),
        #             IdentityResidualBlock(in_channels,
        #                                   channels[mod_id], norm_act=norm_act,
        #                                   stride=stride, dilation=dil,
        #                                   dropout=drop, dist_bn=self.dist_bn)
        #         ))
        #
        #         # Update channels and p_keep
        #         in_channels = channels[mod_id][-1]
        #
        #     # Create module
        #     if mod_id < 2:
        #         if mod_id == 1:
        #
        #             self.mod3=(nn.SequentialCell(OrderedDict(blocks)))
        #             self.pool3=nn.MaxPool2d(3, stride=2)
        #     self.insert_child_to_cell("mod%d" % (mod_id + 2), nn.SequentialCell(OrderedDict(blocks)))

        # Pooling and predictor
        self.bn_out = norm_act(in_channels)
        self.classifier = nn.SequentialCell()
        if classes != 0:
            self.classifier = nn.SequentialCell(OrderedDict([
                ("avg_pool", GlobalAvgPool2d()),
                ("fc", nn.Dense(in_channels, classes))
            ]))

    def construct(self, img):
        out = self.mod1(img)
        out = self.mod2(self.pool2(out))
        out = self.mod3(self.pool3(out))
        out = self.mod4(out)
        out = self.mod5(out)
        out = self.mod6(out)
        out = self.mod7(out)
        out = self.bn_out(out)

        print("out:",out)
        out = self.classifier(out)

        return out

# inputs = mindspore.Tensor(np.ones((2, 3, 24, 24)).astype("float32"))
# wide_resnet = WiderResNetA2(structure = [3, 3, 6, 3, 1, 1],classes=1000, dilation=True)
# #print(wide_resnet)
# outputs = wide_resnet(inputs)
# #print(outputs.shape)


"""defines patterns of addresses URL for app users."""
from django.urls import path, include
from . import views

app_name = 'users'
urlpatterns = [
    #Adding default adresses URL of authentication
    path('', include('django.contrib.auth.urls')),
    #Registration page
    path('register/', views.register, name='register'),
]


import json

# Some JSON:
# json_example = '{"name": "Imaad", "age": 24, "city": "Houston"}'

# parse the JSON variable:
# parse_example = json.loads(json_example)
# print(parse_example["age"])

# convert Python to JSON:
my_dict = {
    'name': 'Imaad',
    'age': 24,
    'city': 'Houston'
}

# convert to JSON:
x = json.dumps(my_dict)

# the result is a JSON string:
print(x)


from flask import render_template, request, redirect, session, flash, url_for, send_from_directory
import querys, crud_routes
from main import app

#####################
####### ROTAS #######
#####################
@app.route('/')
def mainpage():
    return render_template('login.html', title='Enter with your credentials')

@app.route('/list')
def listar():
    if 'current_user' not in session or session['current_user'] == None:
        return render_template('login.html', title='Enter with your credentials')

    games = querys.games_query()
    return render_template('list.html', title='My new title', jogos=games)

@app.route('/insert')
def insert():
    if 'current_user' not in session or session['current_user'] == None:
        return render_template('login.html', title='Enter with your credentials')

    form = crud_routes.FormJogo()
    return render_template('insert.html', title='Insert a new game', form=form)

@app.post('/auth')
def authenticate():
    users = querys.users_query(request.form['username'])

    if len(users) == 0:
        flash('Nome de usuário e/ou senha não encontrado(s). Tente novamente.')
    else:

        if users[0]['password'] == request.form['user_password']:
            session['current_user'] = users[0]['nickname']
            return redirect(url_for('insert'))

            #return render_template(url_for('listar'))  # not working idkw
        else:
            flash('Nome de usuário e/ou senha não encontrado(s). Tente novamente.')

    return render_template('login.html', title='Enter with your credentials')

@app.route('/logout')
def logout():
    session['current_user'] = None
    flash(f"Usuário deslogado.")
    return redirect(url_for("mainpage"))

@app.route('/edit/<id>')
def edit(id):
    if 'current_user' not in session or session['current_user'] == None:
        return render_template('login.html', title='Enter with your credentials')

    game_info = querys.games_info(id)
    capa_jogo = crud_routes.get_image(id)

    return render_template('edit.html', title='Edit game', name=game_info[0]['name'], category=game_info[0]['category'], platform=game_info[0]['platform'], game_id=id, capa_jogo=capa_jogo)

@app.route('/uploads/<nome_arquivo>')
def imagem(nome_arquivo):
    return send_from_directory('uploads', nome_arquivo)


__author__ = 'xiaocong'

# -*- coding:utf-8 -*-

import os
import logging
from scrapy.http import Request
from scrapy.pipelines.images import ImagesPipeline
from douban.utils.select_result import list_first_item

class FilmCoverImage(ImagesPipeline):
    """
    this is for download the film image and generate film_cover_path
    """

    def __init__(self,store_uri,download_func=None):
        self.image_store=store_uri
        super(FilmCoverImage,self).__init__(store_uri,download_func=None)

    def get_media_requests(self, item, info):
        if item.get('film_cover_url'):
            yield Request(item['film_cover_url'])

    def item_completed(self, results, item, info):
        if self.LOG_FAILED_RESULTS:
            msg='%s found errors processing %s'%(self.__class__.__name__,item)
            for ok,value in results:
                if not ok:
                    logging.error(msg,value,spider=info.spider)

        image_paths=[value['path'] for ok,value in results if ok]
        image_path=list_first_item(image_paths)
        item['film_cover_path']=os.path.join(os.path.abspath(self.image_store),image_path) if image_path else ''

        return item


import random
import copy
from collections import Counter

class Hat:

    def __init__(self, **kwargs):
        self.contents = []
        self.contents_save = []

        for color in kwargs:
            for x in range(kwargs[color]):
                self.contents.append(str(color))
                self.contents_save.append(str(color))

    def draw(self, sample_size):
        result = []
        self.contents = self.contents_save.copy()

        if sample_size >= len(self.contents):
            result = self.contents.copy()
            del self.contents[:]
        else:
            for _ in range(sample_size):
                index = random.randint(0, len(self.contents)-1)
                result.append(self.contents[index])
                del self.contents[index]
        return result

def experiment(hat, expected_balls, num_balls_drawn, num_experiments):
    prob = 0.0
    success_counter = 0

    for _ in range(num_experiments):
        sample = hat.draw(num_balls_drawn)
        dict_sample = dict(Counter(sample))
        success = True
        # print("Sample: " + str(dict_sample), end=" -> ")

        for exp_color in expected_balls:
            try:
                if expected_balls[exp_color] <= dict_sample[exp_color]:
                    success &= True
                else:
                    success &= False
                    break
            except:
                success &= False
                break
        # print(success)
        if success:
            success_counter += 1
    return success_counter / num_experiments


from __future__ import print_function

import math

from IPython import display
from matplotlib import cm
from matplotlib import gridspec
from matplotlib import pyplot as plt
import numpy as np
import pandas as pd
from sklearn import metrics
import tensorflow as tf
from tensorflow.python.data import Dataset

def preprocess_features(california_housing_dataframe):
    """Prepares input features from California housing data set.

    Args:
      california_housing_dataframe: A Pandas DataFrame expected to contain data
        from the California housing data set.
    Returns:
      A DataFrame that contains the features to be used for the model, including
      synthetic features.
    """

    selected_features = california_housing_dataframe[
        ["latitude",
         "longitude",
         "housing_median_age",
         "total_rooms",
         "total_bedrooms",
         "population",
         "households",
         "median_income"]]
    processed_features = selected_features.copy()
    # Create a synthetic feature.
    processed_features["rooms_per_person"] = (
            california_housing_dataframe["total_rooms"] /
            california_housing_dataframe["population"])
    return processed_features

def preprocess_targets(california_housing_dataframe):
    """Prepares target features (i.e., labels) from California housing data set.

    Args:
      california_housing_dataframe: A Pandas DataFrame expected to contain data
        from the California housing data set.
    Returns:
      A DataFrame that contains the target feature.
    """

    output_targets = pd.DataFrame()
    # Scale the target to be in units of thousands of dollars.
    output_targets["median_house_value"] = (
            california_housing_dataframe["median_house_value"] / 1000.0)
    return output_targets

def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):
    """Trains a neural net regression model.

    Args:
      features: pandas DataFrame of features
      targets: pandas DataFrame of targets
      batch_size: Size of batches to be passed to the model
      shuffle: True or False. Whether to shuffle the data.
      num_epochs: Number of epochs for which data should be repeated. None = repeat indefinitely
    Returns:
      Tuple of (features, labels) for next data batch
    """

    # Convert pandas data into a dict of np arrays.
    features = {key: np.array(value) for key, value in dict(features).items()}

    # Construct a dataset, and configure batching/repeating.
    ds = Dataset.from_tensor_slices((features, targets))  # warning: 2GB limit
    ds = ds.batch(batch_size).repeat(num_epochs)

    # Shuffle the data, if specified.
    if shuffle:
        ds = ds.shuffle(10000)

    # Return the next batch of data.
    features, labels = ds.make_one_shot_iterator().get_next()
    return features, labels

def construct_feature_columns(input_features):
    """Construct the TensorFlow Feature Columns.

    Args:
      input_features: The names of the numerical input features to use.
    Returns:
      A set of feature columns
    """

    return set([tf.feature_column.numeric_column(my_feature)
                for my_feature in input_features])

def get_quantile_based_buckets(feature_values, num_buckets):
    quantiles = feature_values.quantile(
        [(i + 1.) / (num_buckets + 1.) for i in range(num_buckets)])

    return [quantiles[q] for q in quantiles.keys()]

def get_quantile_based_boundaries(feature_values, num_buckets):
    boundaries = np.arange(1.0, num_buckets) / num_buckets

    quantile = feature_values.quantile(boundaries)

    return [quantile[k] for k in quantile.keys()]

def construct_feature_columns(input_features):
    """Construct the TensorFlow Feature Columns.

    Returns:
      A set of feature columns
    """
    return set([tf.feature_column.numeric_column(each) for each in input_features])

def linear_scale(serise):
    min_val = serise.min()
    max_val = serise.max()
    scale = (max_val - min_val) / 2.0
    return serise.apply(lambda x: ((x - min_val) / scale) - 1.0)

def normalize_linear_scale(examples_dataframe):
    for column in examples_dataframe.columns:
        examples_dataframe[column] = linear_scale(examples_dataframe[column])

    return examples_dataframe

def log_normalize(series):
    return series.apply(lambda x: math.log(x + 1.0))

def clip(series, clip_to_min, clip_to_max):
    return series.apply(lambda x: (
        min(max(x, clip_to_min), clip_to_max)))

def z_score_normalize(series):
    mean = series.mean()
    std_dv = series.std()
    return series.apply(lambda x: (x - mean) / std_dv)

def binary_threshold(series, threshold):
    return series.apply(lambda x: (1 if x > threshold else 0))

def normalize(examples_dataframe):
    """Returns a version of the input `DataFrame` that has all its features normalized."""
    processed_features = pd.DataFrame()

    processed_features["households"] = log_normalize(examples_dataframe["households"])
    processed_features["median_income"] = log_normalize(examples_dataframe["median_income"])
    processed_features["total_bedrooms"] = log_normalize(examples_dataframe["total_bedrooms"])

    processed_features["latitude"] = linear_scale(examples_dataframe["latitude"])
    processed_features["longitude"] = linear_scale(examples_dataframe["longitude"])
    processed_features["housing_median_age"] = linear_scale(examples_dataframe["housing_median_age"])

    processed_features["population"] = linear_scale(clip(examples_dataframe["population"], 0, 5000))
    processed_features["rooms_per_person"] = linear_scale(clip(examples_dataframe["rooms_per_person"], 0, 5))
    processed_features["total_rooms"] = linear_scale(clip(examples_dataframe["total_rooms"], 0, 10000))

    return processed_features

def train_nn_regression_model(
        my_optimizer,
        steps,
        batch_size,
        hidden_units,
        training_examples,
        training_targets,
        validation_examples,
        validation_targets):
    """Trains a neural network regression model.

    In addition to training, this function also prints training progress information,
    as well as a plot of the training and validation loss over time.

    Args:
      learning_rate: A `float`, the learning rate.
      steps: A non-zero `int`, the total number of training steps. A training step
        consists of a forward and backward pass using a single batch.
      batch_size: A non-zero `int`, the batch size.
      hidden_units: A `list` of int values, specifying the number of neurons in each layer.
      training_examples: A `DataFrame` containing one or more columns from
        `california_housing_dataframe` to use as input features for training.
      training_targets: A `DataFrame` containing exactly one column from
        `california_housing_dataframe` to use as target for training.
      validation_examples: A `DataFrame` containing one or more columns from
        `california_housing_dataframe` to use as input features for validation.
      validation_targets: A `DataFrame` containing exactly one column from
        `california_housing_dataframe` to use as target for validation.

    Returns:
      A `DNNRegressor` object trained on the training data.
    """

    periods = 10
    steps_per_period = steps / periods

    my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)
    dnn_regressor = tf.estimator.DNNRegressor(
        feature_columns=construct_feature_columns(training_examples),
        hidden_units=hidden_units,
        optimizer=my_optimizer
    )

    training_input_fn = lambda: my_input_fn(
        training_examples, training_targets['median_house_value'], batch_size=batch_size
    )
    predict_training_input_fn = lambda: my_input_fn(
        training_examples, training_targets['median_house_value'], batch_size=batch_size,
        shuffle=False, num_epochs=1
    )
    predict_validation_input_fn = lambda: my_input_fn(
        validation_examples, validation_targets['median_house_value'], batch_size=batch_size,
        shuffle=False, num_epochs=1
    )

    print("Training model...")
    print("RMSE (on training data):")
    training_rmse = []
    validation_rmse = []

    for period in range(0, periods):
        dnn_regressor.train(
            input_fn=training_input_fn,
            steps=steps_per_period
        )

        training_predictions = dnn_regressor.predict(input_fn=predict_training_input_fn)
        training_predictions = np.array([item['predictions'][0] for item in training_predictions])

        validation_predictions = dnn_regressor.predict(input_fn=predict_validation_input_fn)
        validation_predictions = np.array([item['predictions'][0] for item in validation_predictions])

        t_rmse = math.sqrt(metrics.mean_squared_error(training_predictions, training_targets))
        v_rmse = math.sqrt(metrics.mean_squared_error(validation_predictions, validation_targets))

        print("\tperiod %02d : %0.2f / %0.2f" % (period, t_rmse, v_rmse))
        training_rmse.append(t_rmse)
        validation_rmse.append(v_rmse)
    print("Model training finished")

    plt.ylabel("RMSE")
    plt.xlabel("Period")
    plt.title("Root mean squared error")

    plt.tight_layout()
    plt.plot(training_rmse, label="training")
    plt.plot(validation_rmse, label="validation")
    plt.legend()

    plt.show()

    return dnn_regressor, training_rmse, validation_rmse

def model_size(estimator):
    variables = estimator.get_variable_names()

    size = 0
    for variable in variables:
        if not any(x in variable
                   for x in ['global_step',
                             'centered_bias_weight',
                             'bias_weight',
                             'Ftrl']
                   ):
            size += np.count_nonzero(estimator.get_variable_value(variable))
    return size

def train_linear_classifier_model(
        learning_rate,
        regularization_strength,
        steps,
        batch_size,
        feature_columns,
        training_examples,
        training_targets,
        validation_examples,
        validation_targets):
    """Trains a linear regression model.

    In addition to training, this function also prints training progress information,
    as well as a plot of the training and validation loss over time.

    Args:
      learning_rate: A `float`, the learning rate.
      regularization_strength: A `float` that indicates the strength of the L1
         regularization. A value of `0.0` means no regularization.
      steps: A non-zero `int`, the total number of training steps. A training step
        consists of a forward and backward pass using a single batch.
      feature_columns: A `set` specifying the input feature columns to use.
      training_examples: A `DataFrame` containing one or more columns from
        `california_housing_dataframe` to use as input features for training.
      training_targets: A `DataFrame` containing exactly one column from
        `california_housing_dataframe` to use as target for training.
      validation_examples: A `DataFrame` containing one or more columns from
        `california_housing_dataframe` to use as input features for validation.
      validation_targets: A `DataFrame` containing exactly one column from
        `california_housing_dataframe` to use as target for validation.

    Returns:
      A `LinearClassifier` object trained on the training data.
    """

    periods = 7
    steps_per_period = steps / periods

    # Create a linear classifier object.
    my_optimizer = tf.train.FtrlOptimizer(learning_rate=learning_rate,
                                          l1_regularization_strength=regularization_strength)
    my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)
    linear_classifier = tf.estimator.LinearClassifier(
        feature_columns=feature_columns,
        optimizer=my_optimizer
    )

    # Create input functions.
    training_input_fn = lambda: my_input_fn(training_examples,
                                            training_targets["median_house_value_is_high"],
                                            batch_size=batch_size)
    predict_training_input_fn = lambda: my_input_fn(training_examples,
                                                    training_targets["median_house_value_is_high"],
                                                    num_epochs=1,
                                                    shuffle=False)
    predict_validation_input_fn = lambda: my_input_fn(validation_examples,
                                                      validation_targets["median_house_value_is_high"],
                                                      num_epochs=1,
                                                      shuffle=False)

    # Train the model, but do so inside a loop so that we can periodically assess
    # loss metrics.
    print("Training model...")
    print("LogLoss (on validation data):")
    training_log_losses = []
    validation_log_losses = []
    for period in range(0, periods):
        # Train the model, starting from the prior state.
        linear_classifier.train(
            input_fn=training_input_fn,
            steps=steps_per_period
        )
        # Take a break and compute predictions.
        training_probabilities = linear_classifier.predict(input_fn=predict_training_input_fn)
        training_probabilities = np.array([item['probabilities'] for item in training_probabilities])

        validation_probabilities = linear_classifier.predict(input_fn=predict_validation_input_fn)
        validation_probabilities = np.array([item['probabilities'] for item in validation_probabilities])

        # Compute training and validation loss.
        training_log_loss = metrics.log_loss(training_targets, training_probabilities)
        validation_log_loss = metrics.log_loss(validation_targets, validation_probabilities)
        # Occasionally print the current loss.
        print("  period %02d : %0.2f" % (period, validation_log_loss))
        # Add the loss metrics from this period to our list.
        training_log_losses.append(training_log_loss)
        validation_log_losses.append(validation_log_loss)
    print("Model training finished.")

    # Output a graph of loss metrics over periods.
    plt.ylabel("LogLoss")
    plt.xlabel("Periods")
    plt.title("LogLoss vs. Periods")
    plt.tight_layout()
    plt.plot(training_log_losses, label="training")
    plt.plot(validation_log_losses, label="validation")
    plt.legend()

    return linear_classifier

def select_and_transform_features(source_df):
    selected_features = pd.DataFrame()

    selected_features['median_income'] = source_df['median_income']
    selected_features['rooms_per_person'] = source_df['rooms_per_person']

    for r in zip(range(32, 44), range(33, 45)):
        selected_features['from_%d_to_%d' % r] = source_df['latitude'].apply(
            lambda l: 1.0 if r[0] <= l <= r[1] else 0.0)

    return selected_features

tf.logging.set_verbosity(tf.logging.ERROR)
pd.options.display.max_rows = 15
pd.options.display.max_columns = 15
pd.options.display.float_format = '{:.1f}'.format

california_housing_dataframe = pd.read_csv(
    # "https://download.mlcc.google.com/mledu-datasets/california_housing_train.csv", sep=",")
    "/Users/jack/Downloads/california_housing_train.csv", sep=",")

california_housing_dataframe = california_housing_dataframe.reindex(
    np.random.permutation(california_housing_dataframe.index))

# Choose the first 12000 (out of 17000) examples for training.
training_examples = preprocess_features(california_housing_dataframe.head(12000))
training_targets = preprocess_targets(california_housing_dataframe.head(12000))

# Choose the last 5000 (out of 17000) examples for validation.
validation_examples = preprocess_features(california_housing_dataframe.tail(5000))
validation_targets = preprocess_targets(california_housing_dataframe.tail(5000))

# Double-check that we've done the right thing.
print("Training examples summary:")
display.display(training_examples.describe())
print("Validation examples summary:")
display.display(validation_examples.describe())

print("Training targets summary:")
display.display(training_targets.describe())
print("Validation targets summary:")
display.display(validation_targets.describe())

normalized_dataframe = normalize(preprocess_features(california_housing_dataframe))
normalized_training_examples = normalized_dataframe.head(12000)
normalized_validation_examples = normalized_dataframe.tail(5000)

_ = train_nn_regression_model(
    my_optimizer=tf.train.AdagradOptimizer(learning_rate=0.15),
    steps=1000,
    batch_size=50,
    hidden_units=[10, 10],
    training_examples=normalized_training_examples,
    training_targets=training_targets,
    validation_examples=normalized_validation_examples,
    validation_targets=validation_targets)

def location_location_location(examples_dataframe):
    """Returns a version of the input `DataFrame` that keeps only the latitude and longitude."""
    processed_features = pd.DataFrame()
    processed_features["latitude"] = linear_scale(examples_dataframe["latitude"])
    processed_features["longitude"] = linear_scale(examples_dataframe["longitude"])
    return processed_features

lll_dataframe = location_location_location(preprocess_features(california_housing_dataframe))
lll_training_examples = lll_dataframe.head(12000)
lll_validation_examples = lll_dataframe.tail(5000)

_ = train_nn_regression_model(
    my_optimizer=tf.train.AdagradOptimizer(learning_rate=0.05),
    steps=500,
    batch_size=50,
    hidden_units=[10, 10, 5, 5, 5],
    training_examples=lll_training_examples,
    training_targets=training_targets,
    validation_examples=lll_validation_examples,
    validation_targets=validation_targets)


"""
date.py

This file is a Python module.  You can import this module by saying
import date
at the top of your Python script.
Then in your script, you can use class date.Date
"""

class Date(object):
    """
    Class Date demonstrates class and instance attributes, class and instance methods.
    It is a simple date class, containing year, month, and day integers.
    """

    lengths = [
        None,
        31, #January
        28, #February
        31, #March
        30, #April
        31, #May
        30, #June
        31, #July
        31, #August
        30, #September
        31, #October
        30, #November
        31  #December
    ]

    def __init__(self, month, day, year):
        if not isinstance(year, int):
            raise TypeError(f"year must be int, not {type(year)}")
        if not isinstance(month, int):
            raise TypeError(f"month must be int, not {type(month)}")
        if month < 1 or month > Date.monthsInYear():
            raise ValueError(f"month must be in range 1 to {Date.monthsInYear()} inclusive, not {month}")
        if not isinstance(day, int):
            raise TypeError(f"day must be int, not {type(day)}")
        if day < 1 or day > Date.lengths[month]:
            raise ValueError(f"day must be in range 1 to {Date.lengths[month]} inclusive, not {day}")
        self.year = year
        self.month = month
        self.day = day

    #These three methods are getters.

    def getYear(self):
        "Return my year."
        return self.year

    def getMonth(self):
        "Return the number of my month (1 to 12 inclusive)."
        return self.month

    def getDay(self):
        "Return the number of my day (1 to the length of my month, inclusive)."
        return self.day

    def __str__(self):
        "Return a string that looks like the contents of myself."
        return f"{self.month:02}/{self.day:02}/{self.year:04}"

    def dayOfYear(self):
        "Return my day of the year: a number in the range 1 to 365 inclusive."
        return sum(Date.lengths[1:self.month]) + self.day

    def nextDay(self):
        "Move myself one day into the future."
        if self.day < Date.lengths[self.month]:
            self.day += 1
        else:
            self.day = 1       #Go to the first day of the next month.
            if self.month < len(Date.lengths) - 1:
                self.month += 1
            else:
                self.month = 1 #Go to the first month of the next year.
                self.year += 1

    def nextDays(self, n):
        "Move myself n days into the future."
        if not isinstance(n, int):
            raise TypeError(f"n must be int, not {type(n)}")
        if n < 0:
            raise ValueError(f"n must be nonnegative, not {n}")
        for i in range(n):
            self.nextDay()     #Call the instance method in line 69.

    def monthsInYear():
        "Return the number of months in a year.  This function is selfless."
        return len(Date.lengths) - 1;

    #The definition of class Date ends here.

if __name__ == "__main__":
    import sys
    import datetime
    now = datetime.datetime.now()
    #Create a Date object holding today's date.
    d = Date(now.month, now.day, now.year)
    print(f"Today is {d}.")
    sys.exit(0)


import numpy as np
import matplotlib.pyplot as plt
from cartesian_explorer import CAEX_PLOT_KWG

def with_std(x, line_data, **plot_kwargs):
    line_data = line_data.astype(np.float64)
    maximums = np.nanmax(line_data, axis=-1)
    minimums = np.nanmin(line_data, axis=-1)

    std = np.nanstd(line_data, axis=-1)
    mean = np.nanmean(line_data, axis=-1)
    # --
    # call the plotting function
    fill_kwargs = dict(
        alpha=plot_kwargs.get('band_alpha', 0.05), color=plot_kwargs.get('color')
    )
    [plot_kwargs.pop(x, None) for x in CAEX_PLOT_KWG]
    plt.plot(x, mean, **plot_kwargs)
    #plt.fill_between(x, minimums, maximums, **fill_kwargs)
    #plot_func(x, minimums, alpha=0.3, **line_local_plot_kwargs)
    #plot_func(x, maximums, alpha=0.3, **line_local_plot_kwargs)
    plt.fill_between(x, mean-2*std, mean+2*std, **fill_kwargs)

def with_range(x, line_data, **plot_kwargs):
    line_data = line_data.astype(np.float64)
    maximums = np.nanmax(line_data, axis=-1)
    minimums = np.nanmin(line_data, axis=-1)

    mean = np.nanmean(line_data, axis=-1)
    # --
    # call the plotting function
    fill_kwargs = dict(
        alpha=plot_kwargs.get('band_alpha', 0.05), color=plot_kwargs.get('color')
    )
    [plot_kwargs.pop(x, None) for x in CAEX_PLOT_KWG]
    plt.plot(x, mean, **plot_kwargs)
    plt.fill_between(x, minimums, maximums, **fill_kwargs)

def log_std(x, line_data, sigma_interval=2, **plot_kwargs):
    """
    1. Plotting errors on log-plot is a bit tricky.

    In short, using log base e
        z = log(y)
        dz = d(log(y)) = dy/y

    See this for more details:

        https://faculty.washington.edu/stuve/log_error.pdf

    2. Means on log plot are not so easy as well.
    Chanses are, if you are plotting something with distribution
    on log plot, you expect the distribution to be
    _logarithmically normal_

    Hense, the mean should be the _geometric mean_

    See here, page 9:

    https://s3.amazonaws.com/cdn.graphpad.com/faq/1910/file/1487logaxes.pdf

    """

    line_data = line_data.astype(np.float64)
    plt.yscale('log')

    gmean = np.exp(np.nanmean(np.log(line_data), axis=-1))
    dz = np.nanstd(np.log(line_data), axis=-1)
    dmean = np.exp(sigma_interval*dz)
    # --
    # call the plotting function
    fill_kwargs = dict(
        alpha=plot_kwargs.get('band_alpha', 0.05), color=plot_kwargs.get('color')
    )
    [plot_kwargs.pop(x, None) for x in CAEX_PLOT_KWG]
    plt.plot(x, gmean, **plot_kwargs)
    plt.fill_between(x, gmean*dmean, gmean/dmean, **fill_kwargs)


from django.template import Node, Variable, TemplateSyntaxError, Library, Context
from ..template.loader import get_template

register = Library()

class IncludeHandlebarsNode (Node):
    def __init__(self, name, inner_context=None):
        self.name = Variable(name)
        self.inner_context = None if inner_context is None else Variable(inner_context)

    def render(self, context):
        name = self.name.resolve(context)
        template = get_template(name)
        if self.inner_context is None:
            inner_context = context
        else:
            inner_context = self.inner_context.resolve(context)
        return template.render(inner_context)

@register.tag
def include_handlebars(parser, token):
    """
    Include a Handlebars template processed with the current context.

    """
    bits = token.contents.split()
    if len(bits) not in (2, 3):
        raise TemplateSyntaxError(
            "'include_handlebars' tag takes one or two arguments: the name/id "
            "of the template to include and, optionally, a new context.")
    return IncludeHandlebarsNode(*bits[1:])


import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import tensorflow_datasets as tfds
from scipy.special import expit

#--------------------------------------------------------------------#
# region : Building a Linear Regression Model using custom train()

## Generate Data
X_train = np.arange(10).reshape((10, 1))
y_train = np.array([1.0, 1.3, 3.1,
                    2.0, 5.0, 6.3,
                    6.6, 7.4, 8.0,
                    9.0])
plt.plot(X_train, y_train, 'o', markersize=10)
plt.xlabel('x')
plt.ylabel('y')
plt.show()

## Standardize the features and create a TensorFlow dataset
X_train_norm = (X_train - np.mean(X_train))/np.std(X_train)

ds_train_orig = tf.data.Dataset.from_tensor_slices(
    (tf.cast(X_train_norm, tf.float32),
     tf.cast(y_train, tf.float32)))

## Define a new class derived from tf.keras
## (Allows us to use Keras tools for exploring a model, training , and evaluation)
class MyModel(tf.keras.Model):
    def __init__(self):
        super(MyModel, self).__init__()
        self.w = tf.Variable(0.0, name='weight')
        self.b = tf.Variable(0.0, name='bias')

    def call(self, x):
        return self.w*x + self.b

## Instantiate a new model from the MyModel() class
model = MyModel()
model.build(input_shape=(None, 1))
model.summary()

## Define the cost function we want to minimize
## (Here, we chose means quared error)
def loss_fn(y_true, y_pred):
    return tf.reduce_mean(tf.square(y_true - y_pred))

## Implement the train method
## (Here, we use stochastic gradient descent)
def train(model, inputs, outputs, learning_rate):
    with tf.GradientTape() as tape:
        current_loss = loss_fn(model(inputs), outputs)
    dW, db = tape.gradient(current_loss, [model.w, model.b])
    model.w.assign_sub(learning_rate * dW)
    model.b.assign_sub(learning_rate * db)

## Set the hyperparameters and train the model for 200 epochs
tf.random.set_seed(1)
num_epochs = 200
log_steps = 100
learning_rate = 0.001
batch_size = 1
steps_per_epoch = int(np.ceil(len(y_train) / batch_size))

ds_train = ds_train_orig.shuffle(buffer_size=len(y_train))
ds_train = ds_train.repeat(count=None)
ds_train = ds_train.batch(1)

Ws, bs = [], []

for i, batch in enumerate(ds_train):
    if i >= steps_per_epoch * num_epochs:
        break
    Ws.append(model.w.numpy())
    bs.append(model.b.numpy())

    bx, by = batch
    loss_val = loss_fn(model(bx), by)

    train(model, bx, by, learning_rate=learning_rate)
    if i%log_steps==0:
        print('Epoch {:4d} Step {:2d} Loss {:6.4f}'.format(
              int(i/steps_per_epoch), i, loss_val))

## Examine the trained model and plot it
print('Final Parameters:', model.w.numpy(), model.b.numpy())
X_test = np.linspace(0, 9, num=100).reshape(-1, 1)
X_test_norm = (X_test - np.mean(X_train)) / np.std(X_train)
y_pred = model(tf.cast(X_test_norm, dtype=tf.float32))

fig = plt.figure(figsize=(13, 5))
ax = fig.add_subplot(1, 2, 1)
plt.plot(X_train_norm, y_train, 'o', markersize=10)
plt.plot(X_test_norm, y_pred, '--', lw=3)
plt.legend(['Training examples', 'Linear Reg.'], fontsize=15)
ax.set_xlabel('x', size=15)
ax.set_ylabel('y', size=15)
ax.tick_params(axis='both', which='major', labelsize=15)

ax = fig.add_subplot(1, 2, 2)
plt.plot(Ws, lw=3)
plt.plot(bs, lw=3)
plt.legend(['Weight w', 'Bias unit b'], fontsize=15)
ax.set_xlabel('Iteration', size=15)
ax.set_ylabel('Value', size=15)
ax.tick_params(axis='both', which='major', labelsize=15)

plt.show()

# endregion

#--------------------------------------------------------------------#
# region : Model training via the .compile() and .fit() methods

tf.random.set_seed(1)
model = MyModel()

## Compile the model by selecting the optimizer, loss function, and evaluation metrics
model.compile(optimizer='sgd',
              loss=loss_fn,
              metrics=['mae', 'mse'])

## Call the fit() method to train the model
model.fit(X_train_norm, y_train,
          epochs=num_epochs, batch_size=batch_size,
          verbose=1)

## Visualize results
print(model.w.numpy(), model.b.numpy())

X_test = np.linspace(0, 9, num=100).reshape(-1, 1)
X_test_norm = (X_test - np.mean(X_train)) / np.std(X_train)
y_pred = model(tf.cast(X_test_norm, dtype=tf.float32))

fig = plt.figure(figsize=(13, 5))
ax = fig.add_subplot(1, 2, 1)
plt.plot(X_train_norm, y_train, 'o', markersize=10)
plt.plot(X_test_norm, y_pred, '--', lw=3)
plt.legend(['Training Samples', 'Linear Regression'], fontsize=15)

ax = fig.add_subplot(1, 2, 2)
plt.plot(Ws, lw=3)
plt.plot(bs, lw=3)
plt.legend(['W', 'bias'], fontsize=15)
plt.show()

# endregion

#--------------------------------------------------------------------#
# region : Building a multilayer perceptron for classifying flowers in the Iris dataset

## Load the Iris dataset
iris, iris_info = tfds.load('iris', with_info=True)
print(iris_info)

## Split the data into train / test
tf.random.set_seed(1)

ds_orig = iris['train']
ds_orig = ds_orig.shuffle(150, reshuffle_each_iteration=False)

print(next(iter(ds_orig)))

ds_train_orig = ds_orig.take(100)
ds_test = ds_orig.skip(100)

## Apply the transformation via map() to convert the dictionary to a tuple
ds_train_orig = ds_train_orig.map(
    lambda x: (x['features'], x['label']))

ds_test = ds_test.map(
    lambda x: (x['features'], x['label']))

next(iter(ds_train_orig))

## Use Keras API to build a model efficiently
## Dense layer: f(wx + b)
## Two hidden layers

iris_model = tf.keras.Sequential([
    tf.keras.layers.Dense(16, activation='sigmoid',
                          name='fc1', input_shape=(4,)),
    tf.keras.layers.Dense(3, name='fc2', activation='softmax')])
iris_model.summary()

## Compile the model to specify the loss function, optimizer, and evaluation metrics
iris_model.compile(optimizer='adam',
                   loss='sparse_categorical_crossentropy',
                   metrics=['accuracy'])

## Train the model
num_epochs = 100
training_size = 100
batch_size = 2
steps_per_epoch = np.ceil(training_size / batch_size)

ds_train = ds_train_orig.shuffle(buffer_size=training_size)
ds_train = ds_train.repeat()
ds_train = ds_train.batch(batch_size=batch_size)
ds_train = ds_train.prefetch(buffer_size=1000)

## This keeps the training loss and training accuracy after each epoch
history = iris_model.fit(ds_train, epochs=num_epochs,
                         steps_per_epoch=steps_per_epoch,
                         verbose=0)

## Plotting the learning curve
hist = history.history

fig = plt.figure(figsize=(12, 5))
ax = fig.add_subplot(1, 2, 1)
ax.plot(hist['loss'], lw=3)
ax.set_title('Training loss', size=15)
ax.set_xlabel('Epoch', size=15)
ax.tick_params(axis='both', which='major', labelsize=15)

ax = fig.add_subplot(1, 2, 2)
ax.plot(hist['accuracy'], lw=3)
ax.set_title('Training accuracy', size=15)
ax.set_xlabel('Epoch', size=15)
ax.tick_params(axis='both', which='major', labelsize=15)
plt.tight_layout()

plt.show()

## Evaluating the trained model on the test dataset
results = iris_model.evaluate(ds_test.batch(50), verbose=0)
print('Test loss: {:.4f}   Test Acc.: {:.4f}'.format(*results))

## Saving and reloading the trained model
iris_model.save('iris-classifier.h5',
                overwrite=True,
                include_optimizer=True,
                save_format='h5')

iris_model_new = tf.keras.models.load_model('iris-classifier.h5')
iris_model_new.summary()

results = iris_model_new.evaluate(ds_test.batch(50), verbose=0)
print('Test loss: {:.4f}   Test Acc.: {:.4f}'.format(*results))

labels_train = []
for i,item in enumerate(ds_train_orig):
    labels_train.append(item[1].numpy())

labels_test = []
for i,item in enumerate(ds_test):
    labels_test.append(item[1].numpy())
print('Training Set: ',len(labels_train), 'Test Set: ', len(labels_test))

iris_model_new.to_json()

# endregion


# -*- coding: utf-8 -*-
import logging
from datetime import datetime
from enum import Enum

import requests
from requests import Response, Session
from requests.exceptions import (
    ChunkedEncodingError,
    ConnectionError,
    ConnectTimeout,
    ContentDecodingError,
    HTTPError,
    InvalidHeader,
    InvalidSchema,
    InvalidURL,
    MissingSchema,
    ProxyError,
    ReadTimeout,
    RequestException,
    RetryError,
    SSLError,
    StreamConsumedError,
    Timeout,
    TooManyRedirects,
    UnrewindableBodyError,
    URLRequired,
)
from urllib3.exceptions import HTTPError as BaseHTTPError

# a list of exceptions that can be thrown by `requests` (and urllib3)
from directory_tests_shared.settings import (
    BASICAUTH_PASS,
    BASICAUTH_USER,
    USE_BASIC_AUTH,
)
from directory_tests_shared.utils import red

REQUEST_EXCEPTIONS = (
    BaseHTTPError,
    RequestException,
    HTTPError,
    ConnectionError,
    ProxyError,
    SSLError,
    Timeout,
    ConnectTimeout,
    ReadTimeout,
    URLRequired,
    TooManyRedirects,
    MissingSchema,
    InvalidSchema,
    InvalidURL,
    InvalidHeader,
    ChunkedEncodingError,
    ContentDecodingError,
    StreamConsumedError,
    RetryError,
    UnrewindableBodyError,
)

def basic_auth() -> tuple:
    return BASICAUTH_USER, BASICAUTH_PASS

class Method(Enum):
    """Lists all HTTP methods supported by `requests`."""

    DELETE = 0
    GET = 1
    HEAD = 2
    OPTIONS = 3
    PATCH = 4
    POST = 5
    PUT = 6

    def __str__(self):
        return self.name

    def __eq__(self, y):
        return self.value == y.value

def make_request(
    method: Method,
    url: str,
    *,
    session: Session = None,
    params: dict = None,
    headers: dict = None,
    data: dict = None,
    files: dict = None,
    allow_redirects: bool = True,
    auth: tuple = None,
    trim: bool = True,
    cookies: dict = None,
    connect_timeout: float = 3.05,
    read_timeout: int = 60,
    use_basic_auth: bool = USE_BASIC_AUTH,
    no_filename_in_multipart_form_data: bool = False,
) -> Response:
    """Make a desired HTTP request using optional parameters, headers and data.

    NOTE:
    If you want to send a POST/PUT/PATCH request as "multipart/form-data;"
    rather than a default "application/x-www-form-urlencoded",
    then provide `data` as `files`.

    :param method: HTTP method, e.g.: GET, POST, PUT etc
    :param url: URL that request will be made against
    :param session: (optional) an instance of requests Session
    :param params: (optional) query parameters
    :param headers: (optional) extra request headers. Will not be persisted
                    across requests, even if using a session.
    :param data: (optional) data to send
    :param files: (optional) a dict with a file.
                  For more details please refer to:
                  https://requests.kennethreitz.org//en/master/user/quickstart/#post-a-multipart-encoded-file
    :param allow_redirects: Follow or do not follow redirects
    :param auth: (optional) authentication tuple, e.g. ("username", "password")
    :param trim: (optional) trim long request body/response content if True
    :param cookies: (optional) extra request cookies. Will not be persisted
                    across requests, even if using a session.
    :param connect_timeout: (optional) the number of seconds Requests will wait for your client to establish
                            a connection to a remote machine
    :param read_timeout: (optional) the number of seconds the client will wait for the server to send a response
    :param use_basic_auth: (optional) use default basic auth credentials
    :param no_filename_in_multipart_form_data: (optional) remove filename parameter from multipart form data
    :return: Response
    """
    from tests.functional.utils.generic import assertion_msg, log_response

    with assertion_msg("Can't make a request without a valid URL!"):
        assert url is not None

    if not session:
        logging.warning("Session object not provided. Will default to Requests")
    req = session or requests

    if no_filename_in_multipart_form_data:
        # in order to remove the "filename" parameter from "multipart/form-data;" request, every item from the
        # dictionary has to have a tuple value, where the first item represents filename. If it's set to None, then
        # no filename is sent.
        # see: https://stackoverflow.com/a/23131823
        files = {key: (None, value) for key, value in files.items()}

    if use_basic_auth:
        auth = basic_auth()
    request_kwargs = dict(
        url=url,
        params=params,
        headers=headers,
        data=data,
        files=files,
        allow_redirects=allow_redirects,
        timeout=(connect_timeout, read_timeout),
        auth=auth,
        cookies=cookies,
    )

    logging.debug(f"{method} {url}")
    try:
        if method == Method.DELETE:
            res = req.delete(**request_kwargs)
        elif method == Method.GET:
            res = req.get(**request_kwargs)
        elif method == Method.HEAD:
            res = req.head(**request_kwargs)
        elif method == Method.OPTIONS:
            res = req.options(**request_kwargs)
        elif method == Method.PATCH:
            res = req.patch(**request_kwargs)
        elif method == Method.POST:
            res = req.post(**request_kwargs)
        elif method == Method.PUT:
            res = req.put(**request_kwargs)
        else:
            raise KeyError("Unrecognized Method: %s", method.name)
    except REQUEST_EXCEPTIONS as ex:
        red("Exception UTC datetime: %s" % datetime.isoformat(datetime.utcnow()))
        red("{} {}".format(method, url))
        red("Parameters: {}".format(params))
        if headers.get("Authorization"):
            headers["Authorization"] = "STRIPPED_OUT"
        red("Headers: {}".format(headers))
        red("Data: {}".format(data))
        if files:
            meta = [(k, v[0], v[2]) for k, v in files.items()]
            red("Files: {}".format(meta))
        raise ex

    blocked = "Unfortunately your IP address does not appear to come from"
    if blocked in res.content.decode("UTF-8"):
        logging.debug(f"Looks like we're blocked, will retry")
        res = make_request(Method.GET, res.url, session=session)

    log_response(res, trim=trim)
    return res

def check_url(response: Response, expected_url: str, *, startswith: bool = False):
    # avoid circular imports
    from tests.functional.utils.generic import assertion_msg

    if startswith:
        error = (
            f"Expected response URL to start with {expected_url} but got "
            f"{response.url} instead"
        )
        with assertion_msg(error):
            assert response.url.startswith(expected_url)
    else:
        error = f"Expected {expected_url} but got {response.url} instead"
        with assertion_msg(error):
            assert response.url == expected_url

def check_response(
    response: Response,
    status_code: int,
    *,
    location: str = None,
    locations: list = None,
    location_starts_with: str = None,
    body_contains: list = None,
    unexpected_strings: list = None,
):
    """Check if SUT replied with an expected response.

    :param response: Response object return by `requests`
    :param status_code: expected status code
    :param location: (optional) expected value of Location header
    :param locations: (optional) in one of the Location list
    :param location_starts_with: (optional) expected leading part of
                                the Location header
    :param body_contains: (optional) a list of strings that should be present
                    in the response content
    :param unexpected_strings: (optional) a list of strings that should NOT be
                               present in the response content
    """
    # avoid circular imports
    from tests.functional.utils.generic import assertion_msg

    with assertion_msg(
        f"Expected {status_code} from {response.url} but got " f"{response.status_code}"
    ):
        assert response.status_code == status_code

    if body_contains:
        with assertion_msg("Expected response with content, got an empty one"):
            assert response.content
        content = response.content.decode("utf-8").lower()
        for string in body_contains:
            with assertion_msg(
                f"Could not find '{string}' in the response from: {response.request.url}"
            ):
                assert string.lower() in content

    if unexpected_strings:
        with assertion_msg("Expected response with content, got an empty one"):
            assert response.content
        content = response.content.decode("utf-8")
        for string in unexpected_strings:
            with assertion_msg("Found unexpected '%s' in response", string):
                assert string not in content

    if location:
        new_location = response.headers.get("Location")
        with assertion_msg(
            "Expected Location header to be: '%s' but got '%s' " "instead.",
            location,
            new_location,
        ):
            assert new_location == location

    if locations:
        new_location = response.headers.get("Location")
        with assertion_msg(
            "Should redirect to one of these %d locations '%s' but instead"
            " was redirected to '%s'",
            len(locations),
            locations,
            new_location,
        ):
            assert new_location in locations

    if location_starts_with:
        new_location = response.headers.get("Location")
        with assertion_msg(
            "Expected Location header to start with: '%s' but got '%s' " "instead.",
            location_starts_with,
            new_location,
        ):
            assert new_location.startswith(location_starts_with)


import importlib
import os
import sys
import unittest

import dpdata
import numpy as np

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
__package__ = "generator"
from .comp_sys import CompSys
from .context import (
    _crd2frag,
    setUpModule,  # noqa: F401
    take_cluster,
)

@unittest.skipIf(importlib.util.find_spec("openbabel") is None, "requires openbabel")
class Test_take_cluster(unittest.TestCase, CompSys):
    def setUp(self):
        type_map = ["C", "H"]
        jdata = {"cluster_cutoff": 3.5}
        self.system_1 = take_cluster("cluster/14400.lammpstrj", type_map, 1125, jdata)
        self.system_2 = dpdata.System.load("cluster/cluster1.json")
        self.places = 0

@unittest.skipIf(importlib.util.find_spec("openbabel") is None, "requires openbabel")
class Test_take_cluster_minify(unittest.TestCase, CompSys):
    def setUp(self):
        type_map = ["C", "H"]
        jdata = {"cluster_cutoff": 3.5, "cluster_minify": True}
        self.system_1 = take_cluster("cluster/14400.lammpstrj", type_map, 1125, jdata)
        self.system_2 = dpdata.LabeledSystem(
            "cluster/input0_new.gaussianlog", fmt="gaussian/log"
        )
        self.system_2.data["cells"] = self.system_1["cells"]
        self.places = 0

class TestCrd2Frag(unittest.TestCase):
    def test_crd2frag_pbc(self):
        crds = np.array([[0.0, 0.0, 0.0], [19.0, 19.0, 19.0]])
        symbols = ["O", "O"]
        cell = np.diag([20.0, 20.0, 20.0])
        frag_numb, _ = _crd2frag(symbols, crds, pbc=True, cell=cell)
        self.assertEqual(frag_numb, 1)

    def test_crd2frag_nopbc(self):
        crds = np.array([[0.0, 0.0, 0.0], [19.0, 19.0, 19.0]])
        symbols = ["O", "O"]
        frag_numb, _ = _crd2frag(symbols, crds, pbc=False)
        self.assertEqual(frag_numb, 2)

if __name__ == "__main__":
    unittest.main()


"""Puzzles from https://adventofcode.com/2022/day/12"""
from collections import namedtuple, deque
from pathlib import Path
from typing import Dict, Iterable

Point = namedtuple('Point', 'x y')
Heightmap = namedtuple('Data', 'start end area')

def read_data(filepath: Path) -> Heightmap:
    start, end, area = None, None, {}
    for y, line in enumerate(open(filepath).read().splitlines()):
        for x, elevation in enumerate(line):
            if elevation == 'S':
                start = Point(x, y)
                area[Point(x, y)] = 0
            elif elevation == 'E':
                end = Point(x, y)
                area[Point(x, y)] = int(ord('z') - ord('a'))
            else:
                area[Point(x, y)] = int(ord(elevation) - ord('a'))
    return Heightmap(start, end, area)

def neighbours(point: Point) -> Iterable[Point]:
    for x_step, y_step in [(0, 1), (0, -1), (1, 0), (-1, 0)]:
        yield Point(point.x + x_step, point.y + y_step)

def distances_from_end(end: Point, area: Dict[Point, int]) -> Dict[Point, int]:
    todo, visited = deque([end]), {end: 0}
    while todo:
        act = todo.popleft()
        for ngh in neighbours(act):
            if ngh in area and ngh not in visited and area[act] - area[ngh] < 2:
                todo.append(ngh)
                visited[ngh] = visited[act] + 1
    return visited

def first_puzzle(heightmap: Heightmap) -> int:
    return distances_from_end(heightmap.end, heightmap.area)[heightmap.start]

def second_puzzle(heightmap: Heightmap) -> int:
    distances = distances_from_end(heightmap.end, heightmap.area)
    return min(d for p, d in distances.items() if heightmap.area[p] == 0)

if __name__ == '__main__':
    data = read_data(Path(__file__).parent / 'real.data')
    print(first_puzzle(data))
    print(second_puzzle(data))


import time
from collections import defaultdict
from mongoengine import connect, ValidationError, NotUniqueError
from tweepyrate import create_apps, get_tweets
import fire
import tweepy
from hatespeech_models import Tweet, APIError

class ResponseProcess:
    def __init__(self, tweet_to_replies):
        self.tweet_to_replies = tweet_to_replies
        self.new_tweets = 0
        self.new_errors = 0

    def process_tweet(self, tweet_id, response):
        try:
            tweet = Tweet(**response._json)
            tweet.save()

            self._mark_replies_as_processed(tweet_id)
            self.new_tweets += 1
        except (ValidationError, NotUniqueError) as e:
            pass

    def process_error(self, tweet_id, response):
        try:
            error = APIError(
                message=str(response),
                api_code=response.api_code,
                tweet_id=tweet_id
            )

            error.save()

            self._mark_replies_as_processed(tweet_id)
            self.new_errors += 1
        except NotUniqueError as e:
            pass
        except ValidationError as e:
            print("Error without api_code")
            print(e)

    def _mark_replies_as_processed(self, tweet_id):
        """
        Sets look_for_upstream=False for every
        """
        ret = Tweet.objects(id__in=self.tweet_to_replies[tweet_id]).update(
            look_for_upstream=False
        )

def search_for_nonprocessed_tweets(apps, db):
    # Look for tweets that needs processing

    upstream_ids = set(Tweet.objects(look_for_upstream=True).distinct("in_reply_to_status_id"))
    print(f"Tweets to look for: {len(upstream_ids)}")

    ## Beware! Do not check those already having errors
    errors = db.api_error.find({
        "tweet_id": {"$in": list(upstream_ids)}},
        {"tweet_id": 1}
    )
    error_tweet_ids = set(err["tweet_id"] for err in errors)
    upstream_ids = upstream_ids.difference(error_tweet_ids)

    print(f"Tweets without errors: {len(upstream_ids)}")

    tweet_to_replies = defaultdict(list)
    for reply in Tweet.objects(look_for_upstream=True):
        tweet_to_replies[reply.in_reply_to_status_id].append(reply.id)

    processor = ResponseProcess(tweet_to_replies)

    new_tweets, new_errors = get_tweets(
        apps,
        list(upstream_ids),
        processor.process_tweet,
        processor.process_error,
    )

    print(f"There are {processor.new_tweets} new tweets and {processor.new_errors} new errors")

def find_upstream_tweets(database, apps_file, sleep_time=300):
    """
    Look for tweets whose replies are within our database
    """
    apps = create_apps(apps_file)

    client = connect(database)
    db = client[database]

    epoch = 0

    while True:
        print(f"{'='*40}\n" * 3)
        print(f"Epoch number {epoch}\n")

        search_for_nonprocessed_tweets(apps, db)

        print(f"Sleeping for {sleep_time} seconds")
        time.sleep(sleep_time)

        epoch += 1

if __name__ == '__main__':
    fire.Fire(find_upstream_tweets)


from django.shortcuts import render, redirect
from .models import CourseTable, TopicTable
from .form import CourseTableForm, topicform, AssignedCoursesForm
from django.views.generic import DetailView, UpdateView, DeleteView
from django.contrib import auth
from django.contrib.auth.decorators import login_required
from users.models import user

@login_required
def cour(request):
    ku = CourseTable.objects.order_by('id')
    return render(request, 'cours/cours_home.html', {'ku': ku})
@login_required
def user_cours(request):
    ku = CourseTable.objects.order_by('id_course')#TODO: Как правильно?
    return render(request, 'cours/user_cours.html', {'ku': ku})
@login_required
def detail(request, pk):
    ku = list(CourseTable.objects.filter(id=pk).values())
    top = TopicTable.objects.filter(id_course=pk)
    user = auth.get_user(request)
    return render(request, 'cours/datail.html', {'post': ku[0], 'top': top, 'pk': pk, 'user':user})
@login_required
def topic_dateil(request, pk, kk):
    top = TopicTable.objects.filter(id_course=pk)
    ku = list(CourseTable.objects.filter(id=pk).values())
    return render(request, 'cours/topic.html', {'top': top[0], 'pk': pk, 'kk': kk})
@login_required
def my_cours(request): #TODO: Как сделать тут?
    ku = CourseTable.objects.order_by('id')
    return render(request, 'cours/mykurs.html', {'ku': ku})

class kursUpdateView(UpdateView):
    model = CourseTable
    template_name = 'cours/update.html'

    form_class = CourseTableForm

class topicUpdateView(UpdateView):
    model = TopicTable
    template_name = 'cours/update_topic.html'

    form_class = topicform

class kursDeleteView(DeleteView):
    model = CourseTable
    success_url = '/course/'
    template_name = 'cours/cours_delete.html'

class topicDeleteView(DeleteView):
    model = TopicTable
    success_url = '/course/'
    template_name = 'cours/delete_topic.html'

@login_required
def newtopic(request, pk: any):
    error = ''
    if request.method == 'POST':
        form = topicform(request.POST)
        if form.is_valid():
            # form['kursu'] = kursu.objects.filter(id=pk)
            form.save()
            return redirect('kurs')
        else:
            error = 'error'
    form = topicform()
    # form['kursu'] = kursu.objects.filter(id=pk)
    data = {
        'form': form,
        'error': error,
        'pk': pk
    }
    return render(request, 'cours/addTopic.html', data)
@login_required
def adk(request):
    error = ''
    if request.method == 'POST':
        form = CourseTableForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('kurs')
        else:
            error = 'error'
    form = CourseTableForm()
    data = {
        'form': form,
        'error': error
    }
    return render(request, 'cours/addKurs.html', data)
@login_required
def record(request):
    error = ''
    if request.method == 'POST':
        form = AssignedCoursesForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('kurs')
        else:
            error = 'error'
    form = AssignedCoursesForm()
    date = {
           'form': form,
           'error': error
    }
    return render(request, 'cours/RecordToCourse.html', date)


from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import pandas as pd
import collections

def get_response_tfidf_dict(vect_responses):
    sheet_name_response = "CREW final responses"
    df_responses = pd.read_excel('../data/Popravki - IMapBook - CREW and discussions dataset.xlsx',
                                 sheet_name=sheet_name_response)

    responses = df_responses['Collab Response']
    response_numbers = df_responses['Response Number']

    tfidf_responses = vect_responses.transform(responses)

    ti2 = tfidf_responses.T.A
    tfidf_responses = list(map(list, zip(*ti2)))

    response_tfidf_dict = {}
    for i in range(len(response_numbers)):
        response_tfidf_dict[response_numbers[i]] = tfidf_responses[i]
    return response_tfidf_dict

def get_tfidf_books(vect_book):
    with open('../data/ID260 and ID261 - The Lady or the Tiger.txt', 'r', encoding='utf-8') as file:
        book_260_data = file.read().replace('\n', ' ')
    # print(book_260_data)

    with open('../data/ID264 and ID265 - Just Have Less.txt', 'r', encoding='utf-8') as file:
        book_264_data = file.read().replace('\n', ' ')
    # print(book_264_data)

    with open('../data/ID266 and ID267 - Design for the Future When the Future Is Bleak.txt', 'r', encoding='utf-8') as file:
        book_266_data = file.read().replace('\n', ' ')
    # print(book_266_data)

    tfidf_books = vect_book.transform([book_260_data, book_264_data, book_266_data])
    ti2 = tfidf_books.T.A
    tfidf_books = list(map(list, zip(*ti2)))

    return tfidf_books

def get_book_dict():
    return {
        260: 0,
        261: 0,
        264: 1,
        265: 1,
        266: 2,
        267: 2
    }

if __name__ == "__main__":

    sheet_name_crew = "CREW data"
    sheet_name_response = "CREW final responses"

    # reading files
    df_data = pd.read_excel('../data/Popravki - IMapBook - CREW and discussions dataset.xlsx', sheet_name=sheet_name_crew)
    df_responses = pd.read_excel('../data/Popravki - IMapBook - CREW and discussions dataset.xlsx', sheet_name=sheet_name_response)

    messages = df_data['Message']
    classes = df_data['CodePreliminary']
    # classes = df_data['Pseudonym']
    mes_response_link = df_data['Response Number']
    book_ids = df_data['Book ID']

    # print(df_data.head())
    responses = df_responses['Collab Response']
    response_numbers = df_responses['Response Number']

    response_dict = {}
    for i in range(len(response_numbers)):
        response_dict[response_numbers[i]] = responses[i]

    # print(response_dict[1.4])
    # print(response_dict[mes_response_link[40]])

    # TF-IDF ---------------------------------------------------------------------------------------------------------------

    vect = TfidfVectorizer()  # parameters for tokenization, stopwords can be passed
    tfidf_messages = vect.fit_transform(messages)
    tfidf_responses = vect.transform(responses)

    ti2 = tfidf_messages.T.A
    tfidf_messages = list(map(list, zip(*ti2)))

    ti2 = tfidf_responses.T.A
    tfidf_responses = list(map(list, zip(*ti2)))

    # print(tfidf_messages[0])
    # print(tfidf_responses[0])
    # print(vect.get_feature_names())

    response_tfidf_dict = get_response_tfidf_dict(vect)

    print("Length messages: ", len(messages), " Length tfidf_messages: ", len(tfidf_messages))
    # print("Length responses: ", len(responses), " Length tfidf_responses: ", len(tfidf_responses))

    # print(cosine_similarity([tfidf_messages[4]], [tfidf_responses[0]]))

    # Calculate average similarity with response for each class
    class_dict = {}
    index = 0
    sim_dict = {}
    count_dict = {}
    final_response_id = {}
    for i in range(len(messages)):
        code = classes[i].lower()
        if code[-1] == " ":
            code = code[:-1]
        if code == "\xa0":
            code = "blank"
        if code not in class_dict:
            class_dict[code] = index
            index += 1
            sim_dict[code] = 0
            count_dict[code] = 0
            final_response_id[code] = mes_response_link[i]
        sim_dict[code] = sim_dict[code] + cosine_similarity([tfidf_messages[i]], [response_tfidf_dict[mes_response_link[i]]])[0]
        # print(mes_response_link[i], " - ", response_tfidf_dict[mes_response_link[i]])
        count_dict[code] = count_dict[code] + 1

    print('Classes:')
    print(class_dict)

    print('==============================')
    print("Average similarities (responses):")
    final_response_pseudonym = {}
    for key in sim_dict:
        sim_dict[key] = sim_dict[key] / count_dict[key]
        print(key, ": ", sim_dict[key])
        print(key, ": ", final_response_id[key])
        if final_response_id[key] not in final_response_pseudonym:
            final_response_pseudonym[final_response_id[key]] = [key, sim_dict[key][0]]
        else:
            # print(final_response_pseudonym[final_response_id[key]][1])
            # print(sim_dict[key][0])
            if(final_response_pseudonym[final_response_id[key]][1]<sim_dict[key][0]):
                final_response_pseudonym[final_response_id[key]] = [key, sim_dict[key][0]]
    final_response_pseudonym = collections.OrderedDict(sorted(final_response_pseudonym.items()))
    print(final_response_pseudonym)

    # print(sim_dict)

    # BOOKS ----------------------------------------------------------------------------------------------------------------
    print('============================================================================')

    tfidf_books = get_tfidf_books(vect)

    # print(cosine_similarity([tfidf_messages[1]], [tfidf_books[1]]))
    # print(tfidf_books)

    book_dict = get_book_dict()

    # print(book_ids)
    # print(tfidf_books[book_dict[book_ids[0]]])

    # Calculate average similarity with response for each class
    class_dict = {}
    index = 0
    sim_dict = {}
    count_dict = {}
    final_book_id = {}
    for i in range(len(messages)):
        code = classes[i].lower()
        if code[-1] == " ":
            code = code[:-1]
        if code == "\xa0":
            code = "blank"
        if code not in class_dict:
            class_dict[code] = index
            index += 1
            sim_dict[code] = 0
            count_dict[code] = 0
            final_book_id[code] = book_ids[i]
        sim_dict[code] = sim_dict[code] + cosine_similarity([tfidf_messages[i]], [tfidf_books[book_dict[book_ids[i]]]])[0]
        # print(mes_response_link[i], " - ", response_tfidf_dict[mes_response_link[i]])
        count_dict[code] = count_dict[code] + 1

    print("Average similarities (books):")
    book_pseudonym = {}
    for key in sim_dict:
        sim_dict[key] = sim_dict[key] / count_dict[key]
        print(key, ": ", sim_dict[key])
        print(key, ": ", final_book_id[key])
        if final_book_id[key] not in book_pseudonym:
            book_pseudonym[final_book_id[key]] = [key, sim_dict[key][0]]
        else:
            # print(book_pseudonym[final_book_id[key]][1])
            # print(sim_dict[key][0])
            if (book_pseudonym[final_book_id[key]][1] < sim_dict[key][0]):
                book_pseudonym[final_book_id[key]] = [key, sim_dict[key][0]]
    book_pseudonym = collections.OrderedDict(sorted(book_pseudonym.items()))
    print(book_pseudonym)


# -*- coding: utf-8 -*-
#!/usr/bin/env python3
###
#
# Python Monochrometer Interface
#
# Program Description : GUI master for the Landsat Buoy Calibration program
# Created By          : Benjamin Kleynhans
# Creation Date       : February 22, 2019
# Authors             : Benjamin Kleynhans
#
# Last Modified By    : Benjamin Kleynhans
# Last Modified Date  : February 22, 2019
# Filename            : save_file.py
#
###

# Imports
import json
import os
import pdb

from tkinter import *
from tkinter import filedialog

class Save_File:

    def __init__(self, root, master, data_type, data_dictionary=None):

        self.root = root
        self.master = master

        self.data = data_dictionary

        self.build_path(data_type)
        self.save()

    def build_path(self, data_type):

        returnValue = None

        project_root = self.root.preferences['project_root']

        if (data_type == 'config'):
            self.data = self.root.preferences
            self.save_location = os.path.join(project_root, 'gui', 'etc', 'ol750.cfg')
        elif (data_type == 'user'):
            self.save_location = filedialog.askopenfilename(
                initialdir = os.path.join(project_root, 'user_settings'),
                title = "Select file",
                filetypes = (("jpeg files","*.jpg"),("all files","*.*"))
            )

        return returnValue

    def save(self):

        json_data = json.dumps(self.data, sort_keys=True, indent=4)

        filehandle = open(self.save_location, 'w')
        filehandle.writelines(json_data)
        filehandle.close()


# License: GNU Affero General Public License v3 or later
# A copy of GNU AGPL v3 should have been included in this software package in LICENSE.txt.

""" Common functionality for finding and marking PFAMDomains within a record. """

from collections import defaultdict
from dataclasses import dataclass
import logging
import os
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple

from antismash.common import fasta, module_results, pfamdb, subprocessing
from antismash.common.pfamdb import ensure_database_pressed  # used by others # pylint: disable=unused-import
from antismash.common.secmet import Record, CDSFeature
from antismash.common.secmet.features import FeatureLocation, PFAMDomain
from antismash.common.secmet.locations import location_from_string

@dataclass(frozen=True)
class HmmerHit:  # pylint: disable=too-many-instance-attributes
    """ A class containing information about a HMMer hit

        Protein locations are expected to be python slice-style coordinates,
        with an inclusive start and an exclusive end.
    """
    location: str
    label: str
    locus_tag: str
    domain: str
    evalue: float
    score: float
    identifier: str
    description: str
    protein_start: int
    protein_end: int
    translation: str

    def __post_init__(self) -> None:
        if self.protein_start >= self.protein_end:
            raise ValueError(f"HMMer hit has inverted start and end: {self!r}")
        if len(self.translation) != len(self):
            raise ValueError("translation length does not match protein location size")

    def __len__(self) -> int:
        return self.protein_end - self.protein_start

    def to_json(self) -> Dict[str, Any]:
        """ Returns a JSON-ready representation of the instance """
        return dict(vars(self))

    @staticmethod
    def from_json(data: Dict[str, Any]) -> "HmmerHit":
        """ Reconstructs an instance from a JSON representation """
        return HmmerHit(**data)

class HmmerResults(module_results.ModuleResults):
    """ Results for hmmer-based detection """
    schema_version = 2

    def __init__(self, record_id: str, evalue: float, score: float,
                 database: str, tool: str, hits: List[HmmerHit]) -> None:
        super().__init__(record_id)
        self.hits = list(hits)
        self.evalue = float(evalue)
        self.score = float(score)
        self.database = str(database)
        self.tool = str(tool)

    def to_json(self) -> Dict[str, Any]:
        json = {"hits": [hit.to_json() for hit in self.hits],
                "record id": self.record_id,
                "schema": self.schema_version, "max evalue": self.evalue,
                "min score": self.score, "database": self.database,
                "tool": self.tool}
        return json

    @staticmethod
    def from_json(json: Dict[str, Any], record: Record) -> Optional["HmmerResults"]:
        """ Regenerate the results from JSON.
            If max_evalue or min_score aren't equal or narrower than those the
            results were generated with, the results will be discarded.
        """

        if record.id != json.get("record id"):
            logging.warning("Hmmer results are for different record, discarding previous results")
            return None

        if json.get("schema") != HmmerResults.schema_version:
            logging.warning("Hmmer results are for different result schema, discarding previous results")
            return None

        evalue = json.get("max evalue")
        score = json.get("min score")
        if evalue is None or score is None:
            raise ValueError("Invalid Hmmer result values")
        assert isinstance(score, float) and isinstance(evalue, float)

        hits = json.get("hits")
        if not isinstance(hits, list):
            raise TypeError("FullHmmer results contain unexpected types")
        hits = [HmmerHit(**hit) for hit in hits]

        return HmmerResults(record.id, evalue, score, json["database"], json["tool"], hits)

    def refilter(self, max_evalue: float, min_score: float) -> "HmmerResults":
        """ Trims the results to stricter thresholds for score and E-value """
        if max_evalue > self.evalue:
            raise ValueError(f"cannot refilter to a more lenient evalue: {self.evalue} -> {max_evalue}")
        if min_score < self.score:
            raise ValueError(f"cannot refilter to a more lenient score: {self.score} -> {min_score}")
        self.hits = [hit for hit in self.hits if hit.score >= min_score and hit.evalue <= max_evalue]
        self.evalue = max_evalue
        self.score = min_score
        return self

    def add_to_record(self, record: Record) -> None:
        """ Adds the hits as PFAMDomains to the given record """
        db_version = pfamdb.get_db_version_from_path(self.database)
        for i, hit in enumerate(self.hits):
            protein_location = FeatureLocation(hit.protein_start, hit.protein_end)
            pfam_feature = PFAMDomain(location_from_string(hit.location),
                                      description=hit.description, protein_location=protein_location,
                                      identifier=hit.identifier, tool=self.tool, locus_tag=hit.locus_tag)
            for key in ["label", "locus_tag", "domain", "evalue",
                        "score", "translation"]:
                setattr(pfam_feature, key, getattr(hit, key))
            pfam_feature.database = db_version
            pfam_feature.detection = "hmmscan"
            pfam_feature.domain_id = f"{self.tool}_{pfam_feature.locus_tag}_{i+1:04d}"
            record.add_pfam_domain(pfam_feature)

def build_hits(record: Record, hmmscan_results: List, min_score: float,
               max_evalue: float, database: str) -> List[HmmerHit]:
    """ Builds HmmerHits from the given hmmscan results

        Arguments:
            record: the Record being scanned
            hmmscan_results: the results of Bio.SearchIO.parse
            min_score: a minimum allowable bitscore for hits (exclusive)
            max_evalue: a maximum allowable evalue for hits (exclusive)
            database: the name of the database used to find the hits

        Returns:
            a list of HmmerHit instances
    """
    hits = []
    feature_by_id = record.get_cds_name_mapping()

    for result in hmmscan_results:
        for hsp in result.hsps:
            if hsp.bitscore <= min_score or hsp.evalue >= max_evalue:
                continue

            feature = feature_by_id[hsp.query_id]
            location = feature.get_sub_location_from_protein_coordinates(hsp.query_start, hsp.query_end)

            hit = {"location": str(location),
                   "label": result.id, "locus_tag": feature.get_name(),
                   "domain": hsp.hit_id, "evalue": hsp.evalue, "score": hsp.bitscore,
                   "translation": feature.translation[hsp.query_start:hsp.query_end],
                   "identifier": pfamdb.get_pfam_id_from_name(hsp.hit_id, database),
                   "description": hsp.hit_description, "protein_start": hsp.query_start,
                   "protein_end": hsp.query_end,
                   }
            hits.append(HmmerHit(**hit))
    return hits

def remove_overlapping(hits: List[HmmerHit], cutoffs: Dict[str, float],
                       overlap_limit: int = 10) -> List[HmmerHit]:
    """ Filters a list of HmmerHits so that no overlapping hits remain.
        If two hits overlap by more than the specified limit, the best is kept
        based on the following, in order:
        - highest bitscore (normalised by profile cutoff)
        - longest hit
        - earliest start
        - identifier of the profile hit in ascending order

        Arguments:
            hits: the list of hits to filter
            cutoffs: a dictionary mapping profile identifier to cutoff as a float
            overlap_limit: the number of overlapping aminos required to be filtered

        Returns:
            a list of HmmerHits, sorted by protein start position
    """

    if not hits:
        assert 0
        return []

    try:
        normalised = {hit: cutoffs[hit.identifier] / hit.score for hit in hits}
    except KeyError as err:
        raise ValueError("cutoff mapping does not contain all hit identifiers") from err

    def ranking_stats(hit: HmmerHit) -> Tuple[float, float, int, str]:
        """ In order of importance:
                highest normalised score, longest hit, earliest start, identifier

            Score and hit are inverted in order for all components to sort the
            same way. An ascending sort will have the highest scoring hit first.

            Hits with the same values for the above are the same hit and can
            ranked equivalently with no loss of information.
        """
        return normalised[hit], 1/len(hit), hit.protein_start, hit.identifier

    # ensure ordering
    hits = sorted(hits, key=lambda hit: hit.protein_start)

    # build sets of overlapping hits, no two sets should overlap
    groups: List[Set[HmmerHit]] = []
    current = {hits[0]}
    max_current = hits[0].protein_end  # the end point of the set for simple checking
    for hit in hits:
        if max_current - overlap_limit < hit.protein_start:
            groups.append(current)
            current = {hit}
            max_current = hit.protein_end
            continue
        current.add(hit)
        max_current = max(max_current, hit.protein_end)
    # not forgetting to add the in-progress set
    groups.append(current)

    # the filter stage
    cleaned = []
    for group in groups:
        best_of: List[HmmerHit] = []
        # start with the highest score and work down
        for hit in sorted(group, key=ranking_stats):
            for other in best_of:
                start = other.protein_start + overlap_limit
                end = other.protein_end - overlap_limit
                # abort if its overlap with a higher ranked hit
                if hit.protein_start <= end and hit.protein_end >= start:
                    break
            else:  # no overlap with existing best
                best_of.append(hit)
        cleaned.extend(best_of)
        assert best_of
    assert cleaned
    return sorted(cleaned, key=lambda hit: hit.protein_start)

def run_hmmer(record: Record, features: Iterable[CDSFeature], max_evalue: float,
              min_score: float, database: str, tool: str, filter_overlapping: bool = True,
              use_cut_tc: bool = True) -> HmmerResults:
    """ Build hmmer results for the given features

        Arguments:
            record: the Record instance to run hmmer over
            features: the list of CDSFeatures to run over specifically
            max_evalue: a maximum evalue allowed for hits (exclusive)
            min_evalue: a minimum evalue allowed for hits (exclusive)
            database: the database to search for hits within
            tool: the name of the specific tool calling into this module
            use_cut_tc: whether to use threshold cutoff as specified in profiles
    """
    if not os.path.exists(database):
        raise ValueError(f"Given database does not exist: {database}")
    query_sequence = fasta.get_fasta_from_features(features)
    opts: List[str] = []
    if use_cut_tc:
        opts.append('--cut_tc')
    hmmscan_results = subprocessing.run_hmmscan(database, query_sequence, opts=opts)
    hits = build_hits(record, hmmscan_results, min_score, max_evalue, database)
    if filter_overlapping:
        results_by_cds = defaultdict(list)
        for hit in hits:
            results_by_cds[hit.locus_tag].append(hit)
        cutoffs = pfamdb.get_pfam_cutoffs(database)
        hits = []
        for locus_hits in results_by_cds.values():
            hits.extend(remove_overlapping(locus_hits, cutoffs))
    return HmmerResults(record.id, max_evalue, min_score, database, tool, hits)