import numpy as np
from math import isnan, isinf
from scipy.interpolate import splev
from matplotlib import pyplot as plt

# Recursive De Boor algorithm problem.
def N(x, i, k, t):
    """ Computes the i'th basis function of order 'k'
    for the spline with knot vector 't' at the parameter value 'x'."""
    # This recursion involves a lot
    # of redundant calculation.
    # This is not the way this algorithm
    # should be implemented in real world
    # applications, but it is instructive.
    # Do k=0 case.
    if k <= 0:
        if t[i] <= x < t[i+1]:
            return 1.
        else:
            return 0.
    # Use recursion for other cases.
    else:
        # Compute left and right hand sides.
        left = (x - t[i]) / (t[i+k] - t[i])
        right = (t[i+k+1] - x) / (t[i+k+1] - t[i+1])
        # Account for nan and inf values.
        if isnan(left) or isinf(left):
            left = 0.
        if isnan(right) or isinf(right):
            right = 0.
        # Perform the recursive call.
        # It could be good to avoid calling
        # recursively for terms we already know
        # will be zero, but this matches
        # more closely with the formula
        # as it is usually written.
        return left * N(x, i, k-1, t) + right * N(x, i+1, k-1, t)

def circle_interp(m, k, res=401):
    """ Plots an interpolating spline of degree 'k'
    with parameters ranging from 0 to 'm'
    that approximates the unit circle.
    Uses scipy.integrate.splev."""
    # Make the knot vector.
    t = np.array([0]*(k) + range(m) + [m]*(k+1))
    # Preallocate the array 'c' of control points.
    c = np.empty((2, m + k + 1))
    c[:,-1] = 0.
    # Construct the circle.
    # Use n + k control points.
    theta = np.linspace(0, 2 * np.pi, m + k)
    np.cos(theta, out=c[0,:-1])
    np.sin(theta, out=c[1,:-1])
    # Generate the sample values to use for plotting.
    X = np.linspace(0, m, res)
    # Evaluate the B-spline at the given points.
    pts = splev(X, (t, c, k))
    # Plot the B-spline and its control points.
    plt.plot(pts[0], pts[1])
    plt.scatter(c[0], c[1])
    plt.show()

def my_circle_interp(m, k, res=401):
    """ Plots an interpolating spline of degree 'k'
    with parameters ranging from 0 to 'm'
    that approximates the unit circle.
    Uses the function 'N' defined above."""
    # Make the knot vector.
    t = np.array([0]*(k) + range(m) + [m]*(k+1))
    # Preallocate the array 'c' of control points.
    c = np.empty((2, m + k))
    # Construct the circle.
    # Use n + k control points.
    theta = np.linspace(0, 2 * np.pi, m + k)
    np.cos(theta, out=c[0])
    np.sin(theta, out=c[1])
    # Generate the sample vaues to use for plotting.
    # Offset just a little to not get to the end of the interval.
    # This makes the plots look identical instead of just similar.
    X = np.linspace(0, m - 1E-10, res)
    # Find the values of each basis function
    Ni = np.array([[N(x, i, k, t) for x in X] for i in xrange(m + k)])
    # Use the points to evaluate the spline, using the basis functions.
    pts = Ni.T.dot(c.T).T
    # Plot the B-spline and its control points.
    plt.plot(pts[0], pts[1])
    plt.scatter(c[0], c[1])
    plt.show()

if __name__=="__main__":
    circle_interp(20, 4)
    my_circle_interp(20, 4)


from flask import Flask, render_template, request, redirect, url_for
from find_min_std import get_recommend
from movie_detail import Movie
from html import unescape
import json
from ibm_watson import AssistantV2
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator, BasicAuthenticator
from search_movie_by_title import requestIBMD

app = Flask(__name__)

authenticator = IAMAuthenticator('k3AEmH-u5f0HYHSySDDGbWzMvtaA0IqLkanMUVouDmUa')

assistant = AssistantV2(
    version='2020-02-05',
    authenticator=authenticator
)

# replace url

assistant.set_service_url('https://api.us-south.assistant.watson.cloud.ibm.com')

assistant_idd = 'bf5d1798-657f-47be-925f-0d26863c731b'

# create session.
session_idd  = assistant.create_session(
        assistant_id=assistant_idd
    ).get_result()['session_id']

'''
# Set up Assistant service.
authenticator = IAMAuthenticator('{apikey}') # replace with API key
service = AssistantV2(
    version = '2019-02-28',
    authenticator = authenticator
)
assistant_id = '{assistant_id}' # replace with assistant ID

# Create session.
session_id = service.create_session(
    assistant_id = assistant_id
).get_result()['session_id']

'''

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/howto')
def howTo():
    return render_template('howto.html')

@app.route('/movie')
def movie():
    return render_template('page.html')

@app.route('/pred', methods=['GET', 'POST'])
def predict():
    data = request.args.get('data')
    if request.method == 'POST' and 'movie_search' in request.form:
        keyword = request.form.get("movie_search")
        movies= requestIBMD(keyword)
        print("from prediction page: ")
        print(movies)
        j = 0
        imdbs = []
        images = []
        titles = []
        synopsises = []
        releasedes = []
        runtimes = []
        genres= []
        languages = []
        actors = []
        creators = []
        directors = []
        while j < 5:
            if sig==1:
                imdbs.append(movies[j][0])
                images.append(movies[j][1])
                titles.append(movies[j][2])
                synopsises.append(movies[j][3])
                releasedes.append(movies[j][4])
                runtimes.append(movies[j][5])
                genres.append(movies[j][6])
                languages.append(movies[j][7])
                actors.append(movies[j][8])
                creators.append(movies[j][9])
                directors.append(movies[j][10])
            j = j+1
        print(titles)
        data = [imdbs, images,titles, synopsises,releasedes, runtimes, genres, languages, actors, creators, directors]
        return render_template('prediction.html', data=data)
    return render_template('prediction.html', data= data)

@app.route('/test')
def test():
    return render_template('test.html')

@app.route('/send_message/<message>')
def send_message(message):
    response = assistant.message(
        assistant_idd,
        session_idd,
        input={
            'message_type': 'text',
            'text': str(message)
        }
    ).get_result()['output']['generic'][0]['text']

    # print(response)
    # return "reponse str,false" if assistant are going to recommend
    # return "reponse str,true" if assistant just talk e.g : reply to 'how are you? ' ;
    if response[0] == '{':
        return get_movies(response)
    else:
        return response

def get_movies(movies):
    response = "I recommend"
    movies = json.loads(movies)
    for x in movies["courses"]:
        response = response + ", " + x["title"] + " (" + x["released"] + ")"
    return response

@app.route('/p', methods=['GET', 'POST'])
def presonality():
    if request.method == 'POST':
        sig = 0
        #get the data from personality questionaire
        movies = []
        if 'movie_search' in request.form:
            sig = 1
            print("Yes get movie_search")
            keyword = request.form.get("movie_search")
            movies= requestIBMD(keyword)
            print(movies)
        else:
            i = 1
            personality_detail = []
            while i <=50:
                q_result = request.form.get('q'+str(i))
                if q_result == "a":
                    personality_detail.append(1)
                elif q_result == "b":
                    personality_detail.append(2)
                elif q_result == "c":
                    personality_detail.append(3)
                elif q_result == "d":
                    personality_detail.append(4)
                elif q_result == "e":
                    personality_detail.append(5)
                else:
                    personality_detail.append(0)
                i= i+1
            #calculate the big fivepersonality
            j = 0
            pe = 20
            pa = 14
            pc = 14
            pn = 38
            po = 8
            while j < 10:
                pe = pe + ((-1)**(j))* personality_detail[5*j]
                pa = pa + ((-1)**(j+1))*personality_detail[1+5*j]
                pc = pc + ((-1)**(j))* personality_detail[2+5*j]
                pn = pn + ((-1)**(j+1))* personality_detail[3+5*j]
                po = po + ((-1)**(j))* personality_detail[4+5*j]
                j = j +1
            big_five = [po/40, pc/40, pe/40, pa/40, pn/40]
            print("big five are: ", str(big_five))
            movie_detail = get_recommend(big_five, 20)
            #handle movies data and store them into differrent categories
            i = 0
            while i < 20:
                movies.append(Movie(movie_detail[i][1]))
                i = i +1
        j = 0
        imdbs = []
        images = []
        titles = []
        synopsises = []
        releasedes = []
        runtimes = []
        genres= []
        languages = []
        actors = []
        creators = []
        directors = []
        while j < 6:
            if sig==1:
                imdbs.append(movies[j][0])
                images.append(movies[j][1])
                titles.append(movies[j][2])
                synopsises.append(movies[j][3])
                releasedes.append(movies[j][4])
                runtimes.append(movies[j][5])
                genres.append(movies[j][6])
                languages.append(movies[j][7])
                actors.append(movies[j][8])
                creators.append(movies[j][9])
                directors.append(movies[j][10])
            else:
                imdbs.append( "https://usa.newonnetflix.info/info/"+str(movie_detail[j][0]))
                images.append(movies[j].image())
                titles.append(unescape(movies[j].title()))
                synopsises.append(unescape(movies[j].synopsis()))
                releasedes.append(movies[j].released())
                runtimes.append(movies[j].runtime())
                genres.append(unescape(movies[j].genre()))
                languages.append(movies[j].language())
                actors.append(', '.join(map(str, unescape(movies[j].actor()))))
                creators.append(', '.join(map(str, unescape(movies[j].creator()))))
                directors.append(', '.join(map(str, unescape(movies[j].director()))))
            j = j+1
        print(titles)
        data = [imdbs, images,titles, synopsises,releasedes, runtimes, genres, languages, actors, creators, directors ]
        return render_template('prediction.html', data=data)
    return render_template('personality.html')

#def questionnaire():
#    if request.method == 'POST':
#        data = request.form.get('data')
#        return redirect(url_for('pred', data = data))
#    return render_template('temp_questionaire.html')

# def begin():
#     global session_idd
#     session_idd = assistant.create_session(
#         assistant_id=assistant_idd
#     ).get_result()['session_id']

#@app.route('//<message>')
#def send_message(message):
#    response = assistant.message(
#        assistant_idd,
#        session_idd,
#        input={
#            'message_type': 'text',
###    ).get_result()
#    return json.dumps(response)

'''
#methods to send message and get the response
@app.route('/send_message/<message>')
def send_message(message):

'''

if __name__ == '__main__':
    app.run(debug=True)


from django.urls import path
from . import views
from user.views import ListarUser
from django.contrib.auth import views as auth_views

app_name = 'livro'

urlpatterns = [
  path('', views.Index.as_view(), name='index'),
  path('cadastrar/', views.Cadastrar.as_view(), name='cadastrar'),
  path('listar/', views.LivroList.as_view(), name='listar'),
  path('deletar/<int:pk>/', views.DeleteLivro.as_view(), name='delete'),
  path('editar/<int:pk>/', views.EditLivro.as_view(), name='editar'),
  path('pesquisar/', views.pesquisar, name='pesquisar'),
  path('listaruser/<codigo>/', ListarUser.as_view(), name='listUser'),
  path('emprestar/<livro>/', views.Emprestar.as_view(), name='emprestar'),
  path('aviso/<aviso>/', views.Aviso.as_view(), name='aviso'),
  path('devolver/<livro>/', views.Devolver.as_view(), name='devolver'),

]


import pandas as pd
from pandas import read_csv
import seaborn as sns
from matplotlib import pyplot as plt
import numpy as np
import os

if __name__ == '__main__':
    filename = 'titanic_train.csv'
    path = os.path.dirname(__file__)
    filepath = os.path.join(path, filename)

    dataset = read_csv(filepath, index_col=['PassengerId'])
    #print(dataset.head())

    target = dataset['Survived']
    data = pd.DataFrame(dataset.drop(columns=['Survived', 'Name', 'Ticket']))
    #print(data.head())

    # 观察数据，判读是否有缺失值
    #print(data.info())

    data['Age'].fillna(int(data['Age'].mean()), inplace=True)
    data['Cabin'].fillna(0, inplace=True)
    data['Cabin'] = data['Cabin'].apply(lambda x: 0 if x == 0 else 1)
    #print(data.groupby(by=['Embarked']).count())
    data['Embarked'].fillna('S', inplace=True)
    #print(data.info())

    # 观察数据
    print(data.head(10))
    print(data.describe())

    # 图表展示数据
    sns.pairplot(data)
    plt.show()
    plt.close()
    print(data[['Age', 'Fare']])

    corrMatt = data.corr()
    sns.heatmap(corrMatt)
    plt.show()
    plt.close()


import codecs
import copy
import csv
import random
import sys
from functools import reduce

def vector_and(vector1, vector2):
    return [element1 & element2 for element1, element2 in zip(vector1, vector2)]

def vector_value(vector):
    return sum(vector)

class Cluster(object):

    def __init__(self, pool, vector, ):
        self.data = [copy.copy(vector), ]
        self.vector = copy.copy(vector)
        self.pool = pool

    def similarity_test(self, vector):
        return vector_value(
            vector_and(self.vector, vector)) / (self.pool.beta + vector_value(vector))\
             > vector_value(self.vector) / (self.pool.beta + len(vector))

    def similarity(self, vector):
        return round(vector_value(vector_and(self.vector, vector)) / (self.pool.beta + vector_value(vector)), 5)

    def size_test(self, vector):
        return ()

    def add(self, vector):
        """Return True if has been added into the cluster, False otherwise."""
        result_vector = vector_and(self.vector, vector)
        united_value = vector_value(result_vector) + 0.0001
        cluster_value = vector_value(self.vector) + 0.0001
        similar = united_value / (self.pool.beta + vector_value(vector)) > cluster_value / (self.pool.beta + len(vector))
        attention = united_value / cluster_value >= self.pool.rho
        # print('result_vector: %s\nunited_vector: %s\ncluster_value: %s\nsimilar: %s\nattention: %s\n' % (result_vector, united_value, cluster_value, similar, attention, ))
        if attention:
            self.data.append(vector)
            self.vector = result_vector
            return True
        return False

    def __repr__(self):
        return '%s -> %s' % (repr(self.data), repr(self.vector), )

class ClusterPool(object):

    def __init__(self, beta=2, rho=0.4):
        self.rho = rho
        self.beta = beta
        self.clusters = []

    def add(self, vector):
        """Return True if new cluster has been created, False otherwise."""
        # print("vector: %s" % (vector, ))
        # print("ALL: %s" % (list(map(lambda cluster:  (cluster.similarity(vector), cluster), self.clusters), )))
        best_cluster = reduce(lambda best, cluster_pair: best if best[0] > cluster_pair[0] else cluster_pair,
                              map(lambda cluster:  (cluster.similarity(vector), cluster), self.clusters), (-1, None))
        # print("Best cluster: ", best_cluster)
        if best_cluster[1] and best_cluster[1].add(vector):
            # print("used cluster: %s" % (repr(best_cluster, )))
            return False
        self.clusters.append(Cluster(self, vector))
        # print("created new cluster. recalculating")
        return True

    def __repr__(self):
        return "[Cluster B: %s P: %s]\n%s" % (self.beta, self.rho, '\n'.join(map(repr, self.clusters)), )

def get_cluster_from_csv(csv_file, beta, rho):
    reader = csv.reader(codecs.iterdecode(csv_file, 'utf-8'))
    raw_data = []
    def _parse_line(line):
        try:
            line = [1 if c == '1' else 0 for c in line]
            raw_data.append(line)
            return line
        except: return None

    pool = ClusterPool(beta=beta, rho=rho)
    for line in filter(lambda x: x, map(_parse_line, reader)):
        try:
            pool.add(line)
        except: continue
    return pool, raw_data

if __name__ == '__main__':
    with open('test.csv') as f:
        pool = get_cluster_from_csv(f)
        print(pool)

    sys.exit(0)
    cp = ClusterPool()
    V_SIZE = 6
    for i in range(20):
        cp.add([round(random.random()) for _ in range(V_SIZE)])
    print(cp)


# -*- coding: utf-8 -*-

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import matplotlib as mpl

from mltools.analysis.logger import Logger
from mltools.analysis.evaluation import TensorEval
from mltools.analysis import describe

from mltools.data import datatools
from mltools.data.features import CategoricalFeatures
from mltools.data.structure import DataStructure

# TODO:
# - for tensors of integers, histogram must have bin length = 1 minimum
# - combine predictions for training and test set
# - write function to convert from dict to dataframe (taking into account
#   tensors, etc.)

class Predictions:

    def __init__(self, X, y_pred=None, y_true=None, y_std=None,
                 model=None, inputs=None, outputs=None, metrics=None,
                 categories_fn=None, integers_fn=None, postprocessing_fn=None,
                 mode='dict', logger=None):
        """
        Inits Predictions class.

        The goal of this class is to first convert the ML output to meaningful
        results. Then, to compare with the real data if they are known.

        The inputs, targets and predictions as given in outputs (or deduced
        from the model) are stored in `X`, `y_true` and `y_pred`. They are
        represented as dict. Other formats can be accessed through the `get_*`
        methods. Available formats are `dict` and `dataframe`, and the default
        is defined in `mode`. The reason for not using arrays is that some
        features need more information (probability distribution...) which
        would be more difficult to represent as arrays.

        The predictions can be given from an ensemble, in which case they are
        stored in `all_y_pred`, and the mean value and standard deviation are
        computed and stored in `y_pred` and `y_std`.
        It is necessary to keep track of all predictions for computing the
        standard deviation of derived quantities (metrics, errors...).

        `metrics` is a dict mapping each feature to a default metric for each
        feature. If a feature has no default metric, this will use the
        metric defined in the correspondng evaluation class.

        `categories` and `integers` are dict mapping features to decision
        functions (to be applied to each sample). They can also be given as
        a list, in which case the default decision is applied:
        - integers: round
        - categories: 1 if p > 0.5 else 0 (or max p is multi-class)

        The `postprocessing_fn` is applied after everything else and can be
        used to perform additional processing of the predictions.

        The predictions for each feature are stored in `predictions`, which
        uses a different class for each type of data. These classes implement
        a set of common methods, but whose arguments may change. Arguments
        not common to all classes must be passed by name.

        Format indicates if data is stored as a dict or as a dataframe.
        """

        # TODO: add CategoricalFeatures as argument, convert data

        # TODO: update to take into account ensemble (keep all results
        #   in another variable)

        self.logger = logger
        self.model = model

        if mode not in ('dataframe', 'dict'):
            raise ValueError("Format `{}` is not supported. Available "
                             "data format are: dict, dataframe."
                             .format(mode))
        else:
            self.mode = mode

        if metrics is None:
            self.metrics = {}
        else:
            self.metrics = metrics
        # TODO: check that metrics is a dict

        # check inputs, outputs, and put in form the data
        self._check_io_data(inputs, outputs, X, y_pred, y_true, y_std)

        # apply processing to predictions
        self._process_predictions(categories_fn, integers_fn,
                                  postprocessing_fn)

        # evaluate predictions for each feature based on its type
        self.predictions = self._typed_predictions()

    def __getitem__(self, key):
        return self.get_feature(key)

    def _check_io_data(self, inputs, outputs, X, y_pred, y_true, y_std):

        # if inputs/outputs is not given, check if they can be deduced from
        # the model
        if inputs is None:
            if self.model is not None and self.model.inputs is not None:
                self.inputs = self.model.inputs
            else:
                self.inputs = None
        else:
            self.inputs = inputs

        if outputs is None:
            if self.model is not None and self.model.outputs is not None:
                self.outputs = self.model.outputs
            else:
                self.outputs = None
        else:
            self.outputs = outputs

        # if targets are not given, extract them from X if `outputs` is given
        if y_true is None:
            if self.outputs is not None:
                self.y_true = self.outputs(X, mode='col')
            else:
                #  TODO: for unsupervised, y_true can remain None
                raise ValueError("Targets `y_true` must be given or one of "
                                 "`outputs` or `model.outputs` must not be"
                                 "`None` to extract the targets from `X`.")
        else:
            if self.outputs is not None:
                self.y_true = self.outputs(y_true, mode='col')
            else:
                self.y_true = y_true

        if not isinstance(self.y_true, dict):
            raise TypeError("`y_true` must be a `dict`, found {}."
                            .format(type(self.y_true)))

        if y_pred is None:
            # if predictions are not given, compute them from the model
            # and input data
            self.y_pred = self.model.predict(X, return_all=True)
        else:
            self.y_pred = y_pred

        # if predictions are a list, assume it comes from an ensemble
        # store mean value in y_pred, keep all predictions in all_y_pred
        # note that the latter is a list of dict, instead of a dict of list
        # TODO: change this?
        if isinstance(self.y_pred, list):
            self.all_y_pred = self.y_pred

            if self.outputs is not None:
                self.y_pred, self.y_std = self.outputs.average(self.all_y_pred)
            else:
                self.y_pred, self.y_std = datatools.average(self.all_y_pred)
        else:
            self.y_std = None
            self.all_y_pred = None

        if self.outputs is not None:
            self.y_pred = self.outputs(self.y_pred, mode='col')

        # override y_std if given
        if y_std is not None:
            self.y_std = y_std

        if not isinstance(self.y_pred, dict):
            raise TypeError("`y_pred` must be a `dict`, found {}."
                            .format(type(self.y_pred)))

        # get list of id
        if "id" in X:
            self.idx = X["id"]
        elif isinstance(X, pd.DataFrame):
            self.idx = X.index.to_numpy()
        else:
            # if no id is defined (in a column or as an index), then use
            # the list index?
            # list(range(len(X)))
            self.idx = None

        # after having extracted targets and predictions, we can filter
        # the inputs
        if self.inputs is not None:
            self.X = self.inputs(X, mode='col')
        else:
            self.X = X

        if not isinstance(self.X, dict):
            raise TypeError("`X` must be a `dict`, found {}."
                            .format(type(self.X)))

        # TODO: sort X and y by id if defined?

        # get target feature list
        if self.outputs is not None:
            self.features = self.outputs.features
        else:
            self.features = list(self.y_pred.keys())

    def _process_predictions(self, categories_fn, integers_fn,
                             postprocessing_fn):

        # store processing functions
        if categories_fn is None:
            self.categories_fn = {}
        elif isinstance(categories_fn, list):
            # TODO: default decision function:
            #   1 if p > 0.5 else 0 (or max p is multi-class)
            raise NotImplementedError
        else:
            self.categories_fn = categories_fn

        if integers_fn is None:
            self.integers_fn = {}
        elif isinstance(integers_fn, list):
            # default decision for integers: round
            self.integers_fn = {col: np.round for col in integers_fn}
        else:
            self.integers_fn = integers_fn

        self.postprocessing_fn = postprocessing_fn

        # process data
        for col, fn in self.categories_fn.items():
            if col not in self.y_pred:
                continue

            self.y_pred[col] = fn(self.y_pred[col])

            if self.all_y_pred is not None:
                for i, y in enumerate(self.all_y_pred):
                    self.all_y_pred[i][col] = fn(y[col])

        for col, fn in self.integers_fn.items():
            if col not in self.y_pred:
                continue

            self.y_pred[col] = fn(self.y_pred[col]).astype(int)

            if self.all_y_pred is not None:
                for i, y in enumerate(self.all_y_pred):
                    self.all_y_pred[i][col] = fn(y[col]).astype(int)

        if self.postprocessing_fn is not None:
            self.y_pred = self.postprocessing_fn(self.y_pred)

    def _typed_predictions(self):
        """
        Built dict of predictions based on data type.
        """

        dic = {}

        for k in self.features:
            metric = self.metrics.get(k, None)
            eval_method = self._prediction_type(k)

            std = None if self.y_std is None else self.y_std[k]
            if self.all_y_pred is None:
                all_pred = None
            else:
                all_pred = [p[k] for p in self.all_y_pred]

            dic[k] = eval_method(k, self.y_pred[k], self.y_true[k],
                                 all_pred, std, self.idx, metric, self.logger)

        return dic

    def _prediction_type(self, feature):
        """
        Find the appropriate class to evaluate the feature.
        """

        # TODO: update to handle other types
        # check in CategoricalFeatures for classification

        return TensorPredictions

    def get_X(self, mode=""):
        mode = mode or self.mode

        if mode == 'dataframe':
            return pd.DataFrame(self.X)
        else:
            return self.X

    def get_y_pred(self, mode=""):
        mode = mode or self.mode

        if mode == 'dataframe':
            return pd.DataFrame(self.y_pred)
        else:
            return self.y_pred

    def get_y_true(self, mode=""):
        mode = mode or self.mode

        if mode == 'dataframe':
            return pd.DataFrame(self.y_true)
        else:
            return self.y_true

    def get_errors(self, mode="", relative=False, norm=False):
        # TODO: arguments signed?

        mode = mode or self.mode

        pred_items = self.predictions.items()

        errors = {f: p.get_errors(relative=relative, norm=norm)
                  for f, p in pred_items}

        if mode == 'dataframe':
            return pd.DataFrame(errors)
        else:
            return errors

    def describe_errors(self, relative=False, percentiles=None):

        if percentiles is None:
            percentiles = [0.25, 0.5, 0.75, 0.95]

        return self.get_errors(mode="dataframe", relative=relative)\
                   .abs().describe(percentiles=percentiles)

    def get_feature(self, feature, mode="", filename="", logtime=True):

        return self.predictions[feature].get_feature(mode, filename, logtime)

    def get_all_features(self, mode="", filename="", logtime=True):

        if self.idx is not None:
            dic = {"id": self.idx}
        else:
            dic = {}

        for feature in self.features:
            dic.update(datatools.affix_keys(self.get_feature(feature),
                                            prefix="%s_" % feature))

        # remove duplicate id columns
        dic = {k: v for k, v in dic.items() if not k.endswith("_id")}

        mode = mode or self.mode

        df = pd.DataFrame(dic)

        if self.idx is not None:
            df = df.set_index("id").sort_values(by=['id'])
        if self.logger is not None:
            self.logger.save_csv(df, filename=filename, logtime=logtime)

        if mode == "dataframe":
            return df

        else:
            return dic

    def feature_metric(self, feature, metric=None, mode=None):

        return self.predictions[feature].feature_metric(metric, mode)

    def feature_all_metrics(self, feature, metrics=None, mode=None):

        return self.predictions[feature].feature_all_metrics(metrics, mode)

    def all_feature_metric(self, mode=None):

        # TODO: add dataframe mode?

        if mode == "text":
            results = {f: self.feature_metric(f, mode="text")
                       for f in self.features}

            return Logger.dict_to_text(results, sep=":\n ")
        else:
            return {f: self.feature_metric(f) for f in self.features}

    def all_feature_all_metrics(self, mode=None, filename="", logtime=True):

        # TODO: add dataframe mode?
        # TODO: option to keep all results when n > 1

        # TODO: don't compute if not needed (in text mode without filename)
        results = {f: self.feature_all_metrics(f) for f in self.features}

        if self.logger is not None:
            if self.all_y_pred is not None:
                json = {}
                for f, v in results.items():
                    json[f] = [v[0], {k + "_std": m for k, m in v[1].items()}]
            else:
                json = results

            self.logger.save_json(json, filename=filename, logtime=logtime)

        if mode == "text":
            results = {f: self.feature_all_metrics(f, mode="dict_text")
                       for f in self.features}
            results = Logger.dict_to_text(results, sep="\n")

        return results

    def plot_feature(self, feature, plottype="step", sigma=1, density=True,
                     bins=None, log=False, filename="", logtime=True,
                     **kwargs):

        return self.predictions[feature]\
                   .plot_feature(plottype, density, sigma, bins, log,
                                 filename, logtime, **kwargs)

    def plot_all_features(self, plottype="step", sigma=1, density=True,
                          bins=None, log=False, filename="", logtime=True):

        figs = []

        for feature in self.features:
            figs.append(self.plot_feature(feature, plottype, sigma, density,
                                          bins, log))

        if self.logger is not None:
            self.logger.save_figs(figs, filename, logtime)

        return figs

    def plot_errors(self, feature, relative=False, signed=True,
                    density=True, bins=None, log=False,
                    filename="", logtime=True, **kwargs):
        """
        Plot feature errors.

        Non-universal arguments:
        - `norm` (valid for tensor).
        """

        return self.predictions[feature]\
                   .plot_errors(relative, signed, density, bins, log,
                                filename, logtime, **kwargs)

    def plot_all_errors(self, relative=False, signed=True,
                        density=True, bins=None, log=False,
                        filename="", logtime=True, **kwargs):

        figs = []

        for feature in self.features:
            figs.append(self.plot_errors(feature, relative, signed,
                                         density, bins, log, **kwargs))

        if self.logger is not None:
            self.logger.save_figs(figs, filename, logtime)

        return figs

    def plot_feature_errors(self, feature, sigma=1, signed_errors=True,
                            density=True, bins=None, log=False,
                            filename="", logtime=True, **kwargs):

        return self.predictions[feature]\
                   .plot_feature_errors(sigma, signed_errors, density, bins,
                                        log, filename, logtime, **kwargs)

    def summary_feature(self, feature, mode="", metrics=None, sigma=1,
                        signed_errors=True, density=True, bins=None,
                        log=False, filename="", logtime=True, **kwargs):

        return self.predictions[feature]\
                   .summary_feature(mode=mode, metrics=metrics, sigma=sigma,
                                    signed_errors=signed_errors,
                                    density=density, bins=bins, log=log,
                                    filename=filename, logtime=logtime,
                                    **kwargs)

    def summary(self, mode="", training_metrics=None, sigma=1,
                signed_errors=True, density=True, bins=None, log=False,
                include_io=False, filename="", logtime=True, show=False,
                extra_figs=None):

        # TODO: add information on inputs/outputs
        # TODO: add the number of models
        # TODO: add information on sampling

        # TODO: add computation of errors

        if filename == "":
            pdf_filename = ""
            csv_filename = ""
            metrics_filename = ""
            model_filename = ""
        else:
            pdf_filename = filename + "_summary.pdf"
            csv_filename = filename + "_predictions.csv"
            metrics_filename = filename + "_metrics.json"
            model_filename = filename + "_model"

        figs = []

        # first page of summary
        if self.logger is not None:
            intro_text = "# Summary -- {}\n\n"\
                            .format(self.logger.logtime_text())
        else:
            intro_text = "# Summary\n\n"

        intro_text += "Selected metrics:\n"
        intro_text += self.all_feature_metric(mode="text")

        intro_text += "\n\nTraining time:\n"
        # intro_text += "\n" if self.model.n_models > 1 else " "
        intro_text += self.model.get_training_time(mode="text")

        figs.append(self.logger.text_to_fig(intro_text))

        if show is True:
            print(intro_text)

        # summary of all metrics
        metrics_text = "## Metrics\n\n"
        metrics_text += self.all_feature_all_metrics(mode="text",
                                                     filename=metrics_filename,
                                                     logtime=logtime)

        figs.append(self.logger.text_to_fig(metrics_text))

        if show is True:
            print()
            print(metrics_text)

        # table of errors
        error_text = "## Absolute errors -- table\n\n"
        error_text += str(self.describe_errors(relative=False))

        rel_error_text = "## Relative errors -- table\n\n"
        rel_error_text += str(self.describe_errors(relative=True))

        figs.append(self.logger.text_to_fig(error_text))
        figs.append(self.logger.text_to_fig(rel_error_text))

        if show is True:
            print()
            print(error_text)
            print()
            print(rel_error_text)

        # TODO: table of standard deviations

        # distributions and error plots
        figs += self.plot_all_features(sigma=sigma, plottype="step",
                                       density=density, bins=bins, log=log)
        # figs += self.plot_all_features(sigma=sigma, plottype="plain",
        #                                density=density, bins=bins, log=log)
        # figs += self.plot_all_features(sigma=sigma, plottype="seaborn",
        #                                density=density, bins=bins, log=log)
        figs += self.plot_all_errors(relative=False, signed=signed_errors,
                                     density=density, bins=bins,
                                     log=log)
        figs += self.plot_all_errors(relative=True, signed=signed_errors,
                                     density=density, bins=bins,
                                     log=log)

        # training curves
        # TODO: by default, plot curves for all model metrics
        if isinstance(training_metrics, str):
            training_metrics = [training_metrics]

        training_figs = self.model.training_curve(log=True, metric=training_metrics)
        figs += [fig for fig in training_figs.values() if fig is not None]

        # page on model information
        model_text = self.model.summary(filename=model_filename,
                                        logtime=logtime, logger=self.logger)

        figs.append(self.logger.text_to_fig(model_text))

        # page on input and output data structures
        # note: useful only if different from `model`
        if include_io is True:
            io_text = ""

            if self.inputs is not None:
                io_text += "\n"
                name = self.inputs.name or "Inputs"
                io_text += self.inputs.summary(name=name)

            if self.outputs is not None:
                io_text += "\n\n"
                name = self.outputs.name or "Outputs"
                io_text += self.outputs.summary(name=name)
                io_text += "\n"

            if io_text != "":
                io_text = "## Inputs and outputs\n\n" + io_text
                figs.append(self.logger.text_to_fig(io_text))

            if show is True:
                print()
                print(io_text)

        # additional figures created outside the function
        if extra_figs is not None:
            figs += extra_figs

        if self.logger is not None:
            self.logger.save_figs(figs, pdf_filename, logtime)

        # save results
        data = self.get_all_features(mode, csv_filename, logtime)

        for fig in figs:
            plt.close(fig)

        return data, figs

class TensorPredictions:
    """
    Evaluate predictions for a tensor.

    Tensor norms are computed  with the L2-norm, which cannot be changed.
    """

    # TODO: make scatter plot true/pred for 2d data
    #   (for d > 2, use reduction)

    method = TensorEval

    def __init__(self, feature, y_pred, y_true, all_y_pred=None, y_std=None,
                 idx=None, metric=None, logger=None):

        self.logger = logger
        self.idx = idx

        # default metric
        self.metric = metric

        # adding the feature name is necessary for plot labels
        self.feature = feature

        self.y_pred = y_pred
        self.y_true = y_true

        self.all_y_pred = all_y_pred
        self.y_std = y_std

        self.is_scalar = self.method.is_scalar(self.y_pred)

        self.errors = self.method.errors(y_true, y_pred)
        self.rel_errors = self.method.errors(y_true, y_pred, relative=True)

        if self.is_scalar is False:
            # TODO: add norm arguments?
            norm = 2

            self.norm_errors = self.method.norm_errors(y_true, y_pred, norm)
            self.rel_norm_errors = self.method.norm_errors(y_true, y_pred,
                                                           relative=True,
                                                           norm=norm)
        else:
            self.norm_errors = None
            self.rel_norm_errors = None

    def get_errors(self, mode="", relative=False, norm=False, **kwgargs):
        """
        Return errors.

        This method is just another way to retrieve the errors which are
        always save as attributes to avoid recomputing.

        `**kwargs` is used to catch parameters transmitted for other feature
        types.
        """

        if self.is_scalar is False and norm is True:
            if relative is True:
                return self.rel_norm_errors
            else:
                return self.norm_errors
        else:
            if relative is True:
                return self.rel_errors
            else:
                return self.errors

    def _pretty_metric_text(self, data, std=None, logger=None):

        dic = {}

        logger = logger or Logger
        float_fmt = logger.styles["print:float"]

        for k, v in data.items():
            # improve names
            new_k = self.method._metric_names.get(k, k)
            # format number according to float format
            dic[new_k] = float_fmt.format(v)

            if std is not None:
                dic[new_k] += " ± " + float_fmt.format(std[k])

        return datatools.equal_length_names(dic)

    def get_feature(self, mode="dict", filename="", logtime=True):
        """
        Summarize feature results in one dataframe.

        Return a dataframe with the following columns: real, pred, error, rel
        for a given feature.
        """

        if self.idx is not None:
            dic = {"id": self.idx}
        else:
            dic = {}

        dic["true"] = self.y_true
        dic["pred"] = self.y_pred

        if self.y_std is not None:
            dic["std"] = self.y_std

        dic["err"] = self.errors
        dic["rel_err"] = self.rel_errors

        if self.is_scalar is False:
            dic["norm_err"] = self.norm_errors

        # TODO: this wll not work with tensor, create appropriate function
        # to convert from dict

        df = pd.DataFrame(dic)

        if self.idx is not None:
            df = df.set_index("id").sort_values(by=['id'])

        if self.logger is not None:
            self.logger.save_csv(df, filename=filename, logtime=logtime)

        if mode == "dataframe":
            return df
        else:
            return dic

    def feature_metric(self, metric=None, mode=None):

        logger = self.logger or Logger
        float_fmt = logger.styles["print:float"]

        metric = metric or self.metric or self.method.default_metric

        if self.all_y_pred is not None:
            all_results = [self.method.evaluate(y, self.y_true, method=metric)
                           for y in self.all_y_pred]

            result = np.mean(all_results)
            std = np.std(all_results)
        else:
            result = self.method.evaluate(self.y_pred, self.y_true,
                                          method=metric)
            std = None

        if mode == "text":
            text = "{} = {}".format(self.method._metric_names
                                        .get(metric, metric),
                                    float_fmt.format(result))
            if std is not None:
                text += " ± {}".format(float_fmt.format(std))

            return text
        else:
            if std is None:
                return result
            else:
                return result, std

    def feature_all_metrics(self, metrics=None, mode=None, filename="",
                            logtime=True):

        logger = self.logger or Logger

        if self.all_y_pred is not None:
            all_results = [self.method.eval_metrics(y, self.y_true,
                                                    metrics=metrics)
                           for y in self.all_y_pred]

            results, std = datatools.average(all_results)
        else:
            results = self.method.eval_metrics(self.y_pred, self.y_true,
                                               metrics=metrics)
            std = None

        if self.logger is not None:
            if std is not None:
                json = {}
                for k in results:
                    json[k] = results[k]
                    json[k + "_std"] = std[k]
            else:
                json = results

            self.logger.save_json(json, filename=filename, logtime=logtime)

        if mode == "text" or mode == "dict_text":
            results = self._pretty_metric_text(results, std, logger)

            if mode == "dict_text":
                return results
            else:
                return logger.dict_to_text(results)
        elif mode == "series":
            return pd.Series(list(results.values()),
                             index=list(results.keys()))
        else:
            if std is None:
                return results
            else:
                return results, std

    def plot_feature(self, plottype="step", density=True, sigma=1, bins=None,
                     log=False, filename="", logtime=True, norm=True):
        """
        Plot the distribution of a feature.

        To get the best from this method, the class must have a `Logger`
        object.
        """

        # TODO: option to remove standard deviation

        logger = self.logger or Logger
        styles = logger.styles

        if norm is True and self.is_scalar is False:
            # TODO: change norm in argument?
            pred = self.method.norm(self.y_pred, norm=2)
            true = self.method.norm(self.y_true, norm=2)

            if self.y_std is not None:
                std = self.method.norm(self.y_std, norm=2)
            else:
                std = None
        else:
            pred = self.y_pred.reshape(-1)
            true = self.y_true.reshape(-1)

            if self.y_std is not None:
                std = self.y_std.reshape(-1)
            else:
                std = None

        xlabel = str(self.feature)

        if self.logger is not None:
            fig = describe.distribution(pred, true, std, sigma=sigma,
                                        plottype=plottype, density=density,
                                        bins=bins, log=log, xlabel=xlabel,
                                        logger=self.logger, filename=filename,
                                        logtime=logtime)
        else:
            fig, ax = plt.subplots()

            bins = bins or logger.find_bins(pred)
            ylabel = "PDF" if density is True else "Count"
            label = [styles["label:pred"], styles["label:true"]]
            color = [styles["color:pred"], styles["color:true"]]

            values, bins, _ = ax.hist([pred, true], linewidth=styles["linewidth:hist"],
                                      histtype='step', bins=bins,
                                      density=density, log=log,
                                      label=label, color=color)
            ax.set_xlabel(xlabel)
            ax.set_ylabel(ylabel)
            ax.legend()
            fig.tight_layout()

        plt.close(fig)

        return fig

    def plot_errors(self, relative=False, signed=True,
                    density=True, bins=None, log=False,
                    filename="", logtime=True,
                    norm=False):

        logger = self.logger or Logger
        styles = logger.styles

        # errors defined without sign in [Skiena, p. 222]

        # TODO: filter out infinite values for relative errors

        if norm is True and self.is_scalar is False:
            if relative is True:
                errors = self.rel_norm_errors
                xlabel = "%s (norm relative errors)" % self.feature
            else:
                errors = self.norm_errors
                xlabel = "%s (norm errors)" % self.feature
        else:
            if relative is True:
                if signed is True:
                    errors = self.rel_errors
                    xlabel = "%s (relative errors)" % self.feature
                else:
                    errors = np.abs(self.rel_errors)
                    xlabel = "%s (unsigned relative errors)" % self.feature
            else:
                if signed is True:
                    errors = self.errors
                    xlabel = "%s (absolute errors)" % self.feature
                else:
                    errors = np.abs(self.errors)
                    xlabel = "%s (unsigned absolute errors)" % self.feature

        if density is True:
            ylabel = "PDF"
            density = True
        else:
            ylabel = "Count"
            density = False

        bins = bins or logger.find_bins(errors)

        fig, ax = plt.subplots()

        # hack to remove infinite values
        # this happens when considering relative errors in the case where
        # the original value is zero
        # TODO: find better way
        err = np.array([e for e in errors if e != np.inf])

        ax.hist(err, linewidth=styles["linewidth:hist"], histtype='step', bins=bins,
                density=density, log=log,
                color=styles["color:errors"])

        ax.set_xlabel(xlabel)
        ax.set_ylabel(ylabel)

        for tick in ax.get_xticklabels():
            tick.set_rotation(45)

        fig.tight_layout()

        if self.logger is not None:
            self.logger.save_fig(fig, filename, logtime)

        plt.close(fig)

        return fig

    def plot_feature_errors(self, sigma=1, signed_errors=True,
                            density=True, bins=None, log=False,
                            filename="", logtime=True, norm=True):

        feat1 = self.plot_feature(plottype="step", sigma=sigma,
                                  density=density, bins=bins, log=log,
                                  norm=norm)

        feat2 = self.plot_feature(plottype="seaborn", sigma=sigma,
                                  density=density, bins=bins, log=log,
                                  norm=norm)

        feat3 = self.plot_feature(plottype="line", sigma=sigma,
                                  density=density, bins=bins, log=log,
                                  norm=norm)

        feat4 = self.plot_feature(plottype="plain", sigma=sigma,
                                  density=density, bins=bins, log=log,
                                  norm=norm)

        # fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(6, 4))
        # axes[0] = fig.axes[0]
        # axes[1] = fig.axes[0]

        err1 = self.plot_errors(relative=False, signed=signed_errors,
                                density=density, bins=bins, log=log,
                                norm=norm)

        err2 = self.plot_errors(relative=True, signed=signed_errors,
                                density=density, bins=bins, log=log,
                                norm=norm)

        figs = [feat1, feat2, feat3, feat4, err1, err2]

        if self.logger is not None:
            self.logger.save_figs(figs, filename, logtime)

        return figs

    def summary_feature(self, mode="", metrics=None, sigma=1,
                        signed_errors=True, density=True, bins=None, log=False,
                        filename="", logtime=True, norm=True):

        # TODO: table of errors (percentiles, max, min)?

        # TODO: add computation of errors

        if filename == "":
            fig_filename = ""
            csv_filename = ""
            json_filename = ""
        else:
            fig_filename = filename + "_plot.pdf"
            csv_filename = filename + "_predictions.csv"
            json_filename = filename + "_metrics.json"

        metric_results = self.feature_all_metrics(metrics,
                                                  filename=json_filename,
                                                  logtime=logtime)

        figs = self.plot_feature_errors(sigma, signed_errors, density,
                                        bins, log, filename=fig_filename,
                                        logtime=logtime, norm=norm)

        data = self.get_feature(mode, filename=csv_filename,
                                logtime=logtime)

        for fig in figs:
            plt.close(fig)

        return metric_results, data, figs

class BinaryPredictions:

    # TODO: prepare metrics dict using Logger.styles (convert to dict of str)
    #   indeed, it's not possible to write a generic dict_to_text method
    #   since it requires knowing about the custom styles and feature names

    # TODO: probbility distribution
    # probability mean value, variance

    pass


import json
import requests
from time import sleep
from datetime import datetime
from kafka import KafkaProducer

# INCOMPLETE FILE - PENDING THE DECISION TO USE WEATHER DATA IN THE PROJECT

def api_to_kafka(endpoint: str):
    """Executes the API request to Open Weather"""
    response = requests.get(endpoint)
    weather_json = json.dumps(response.json(), indent=3)
    weather_dict = (json.loads(weather_json))
    parse_data(weather_dict)

def parse_data(river_dict: dict):
    """Parses out the necessary information to produce events to Kafka"""
    pass

if __name__ == "__main__":

    gallatin = {'latitude': 45.4973, 'longitude': -111.2707083}
    jefferson = {'latitude': 45.6132833, 'longitude': -112.3293972}
    lowerMadison = {'latitude': 45.49023056, 'longitude': -111.6345056}
    upperMadison = {'latitude': 44.88865556, 'longitude': -111.580886}

    weather_endpoint = "https://api.openweathermap.org/data/2.5/onecall?lat=" + str(gallatin["latitude"]) + "&lon=" + str(gallatin["longitude"]) + "&exclude=daily&appid=8d09ba3f05951c0a3af24e56b5b449d5"
    response = requests.get(weather_endpoint)
    weather_json = json.dumps(response.json(), indent=3)
    print(weather_json)
    weather_dict = (json.loads(weather_json))


import requests
from bs4 import BeautifulSoup
import re
from queue import Queue
import time

def convert_to_float(charNum='1/4'):
    x=charNum.split('/')
    return float(int(x[0])/int(x[1]))
def main():
    time1=time.time()
    count=0
    url='https://m.umei.cc/meinvtupian/meinvxiezhen/204337.htm'
    strhtml=requests.get(url)
    strhtml.encoding = 'utf-8'
    soup=BeautifulSoup(strhtml.text,'lxml')
    while count<40:
        datas=soup.select("#ArticleBox > p > a > img")
        for data in datas:
            picNet=data.get('src')
            f=open('pic/'+str(count)+'.png','wb')
            pic=requests.get(picNet)
            f.write(pic.content)
            f.close()
            count+=1
            print(count)
        datas=soup.select("div.pages > ul > li:nth-child(1) > a")
        for data in datas:
            page=str(data.get_text())
            pageNum=convert_to_float(page)
            print(pageNum)
        if pageNum==1:
            datas=soup.select("p:nth-child(3)")
            for data in datas:
                nextPage='https://m.umei.cc'
                dataStr=str(data)
                datalib=dataStr.split('"')
                nextPage+=datalib[1]
                strhtml=requests.get(nextPage)
                strhtml.encoding = 'utf-8'
                soup=BeautifulSoup(strhtml.text,'lxml')
        else:
            datas=soup.select("div.pages > ul > li:nth-child(3) > a")
            for data in datas:
                nextPic='https://m.umei.cc/meinvtupian/meinvxiezhen/'+data.get('href')
                strhtml=requests.get(nextPic)
                strhtml.encoding = 'utf-8'
                soup=BeautifulSoup(strhtml.text,'lxml')
    print(time.time()-time1)

if __name__=="__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Thu Jun 10 11:14:57 2021

@author: Fabretto

Defines functions for the Project 6 'Catégorisez automatiquement des questions'

"""

# import numpy as np
# import pandas as pd
# import seaborn as sns
# import matplotlib.pyplot as plt
# import scipy.stats as st

from collections import defaultdict

# import regex
import nltk
# from nltk import word_tokenize, regexp_tokenize

def to_text(token_chain):
    '''
    Change tokens chained with '|' to a text of tokens separated by a space
    '''
    return ' '.join(token for token in token_chain.split(sep='|'))

def to_list(token_chain):
    '''
    Change tokens chained with '|' to a list of tokens
    '''
    return token_chain.split(sep='|')

def freq_stats_corpora(ids_text, tokenizer, top_nstop=None):
    """ Tokenize and normalize a set of documents ids_text.
        Discards most common words from document set and nltk list
        of English stop words. Discards also words less than 3
        characters long.

        input:
            ids_text: dataframe(ids, text)
                ids: doc identifier
                text: text
            tokenizer:  a tokenizer instance
            top_nstop: number of most common words to add to the stop words
        output:
            freq: frequency distribution by doc
            stats: total and unique word count by doc
            corpora: tokenized and normalized corpus
            sw: stop words discarded from corpora
            freq_totale: total frequency distribution
    """
    corpora = defaultdict(list)
    porter = nltk.PorterStemmer()

    # Create a corpus of tokens by query
    for qid, text in ids_text.itertuples(index=False):
        corpora[qid] = tokenizer.tokenize(text.lower())

    stats, freq = dict(), dict()
    freq_totale = nltk.Counter()

    # Compute total frequency, including stop words
    for v in corpora.values():
        freq_totale += nltk.FreqDist(v)

    most_common = [] if top_nstop is None else list(
        zip(*freq_totale.most_common(top_nstop)))[0]

    sw = set()
    sw.update(most_common)
    sw.update(tuple(nltk.corpus.stopwords.words('english')))

    freq_totale.clear()

    for k, v in corpora.items():
        corpora[k] = tokens = [
            porter.stem(w) for w in v if not w in list(sw) and len(w) >= 3
        ]
        freq[k] = fq = nltk.FreqDist(tokens)
        freq_totale += fq
        stats[k] = {'total': fq.N(), 'unique': fq.B()}

    return (freq, stats, corpora, sw, freq_totale)

def freq_stats_wtags(ids_text, tokenizer):
    """
    Tokenize and normalize a set of Tags ids_text into words for grouping.

        input:
            ids_text: dataframe(ids, text)
                ids: doc identifier
                text: tags
                tokenizer: a tokenizer instance
        output:
            freq: frequency distribution by doc
            stats: total and unique word count by doc
            corpora: tokenized and normalized tags
            freq_totale:
     """
    corpora = defaultdict(list)

    # Create a corpus of tokens by query
    for qid, text in ids_text.itertuples(index=False):
        corpora[qid] = tokenizer.tokenize(text.lower())

    stats, freq = dict(), dict()
    freq_totale = nltk.Counter()

    # Compute total frequency, including stop words
    for v in corpora.values():
        freq_totale += nltk.FreqDist(v)

    for k, v in corpora.items():
        freq[k] = fq = nltk.FreqDist(v)
        freq_totale += fq
        stats[k] = {'total': fq.N(), 'unique': fq.B()}

    return (freq, stats, corpora, freq_totale)

def tokenize_raw(raw_text, tokenizer, vocabulary):
    """ Tokenize and normalize a raw text. Output only tokens
        out of a given vocabulary.
        Discards words from nltk list of English stop words. Discards also
        words less than 3 characters long.

        input:
            raw_text: raw text, typically a query title
            tokenizer:  a tokenizer instance
            vocabulary: output only tokens from this vocabulary
        output:
            token_string: tokenized and normalized string from given vocabulary
    """
    porter = nltk.PorterStemmer()

    sw = set()
    sw.update(tuple(nltk.corpus.stopwords.words('english')))

    tokens = [
        porter.stem(w) for w in tokenizer.tokenize(raw_text.lower())
        if not w in list(sw) and len(w) >= 3
    ]

    return ' '.join([token for token in tokens if token in vocabulary])


from bs4 import BeautifulSoup
import requests, lxml, json, os, math
from config import headers, cookies

def get_data():

    params = {
        'categoryId': '101', # Категория планшетов - 195
        'offset': '0',
        'limit': '24',
        'filterParams': [
            'WyJza2lka2EiLCIiLCJkYSJd',
            'WyJ0b2xrby12LW5hbGljaGlpIiwiIiwiZGEiXQ==',
        ],
        'doTranslit': 'true',
    }

    if not os.path.exists('data'):
        os.mkdir('DNS-shop/data')

    s = requests.session()
    response = s.get('https://www.mvideo.ru/bff/products/listing', params=params, cookies=cookies, headers=headers).json()

    total_items = response.get('body').get('total')
    if total_items is None:
        return '[!] No items :('

    pages_count = math.ceil(total_items / 24)
    print(f'[INFO] total position: {total_items} | Total pages: {pages_count}')

    products_ids = {}
    products_description = {}
    products_prices = {}

    for i in range(pages_count):
        offset = f'{i * 24}'

        params = {
        'categoryId': '101', # Категория планшетов - 195
        'offset': offset,
        'limit': '24',
        'filterParams': [
            'WyJza2lka2EiLCIiLCJkYSJd',
            'WyJ0b2xrby12LW5hbGljaGlpIiwiIiwiZGEiXQ==',
        ],
        'doTranslit': 'true',
        }

        response = s.get('https://www.mvideo.ru/bff/products/listing', params=params, cookies=cookies, headers=headers).json()

        products_ids_list = response.get('body').get('products')
        products_ids[i] = products_ids_list
        data = {
            'productIds': products_ids_list,
            'mediaTypes': [
                'images'
                ],
            'category': True,
            'status': True,
            'brand': True,
            'propertyTypes': [
                'KEY'
                ],
            'propertiesConfig': {
                'propertiesPortionSize': 5
                },
            'multioffer': False
            }

        response = s.post('https://www.mvideo.ru/bff/product-details/list', cookies=cookies, headers=headers, json=data).json()
        products_description[i] = response

        products_ids_str = ','.join(products_ids_list)
        params = {
            'productIds': products_ids_str,
            'addBonusRubles': 'true',
            'isPromoApplied': 'true',
            }

        response = s.get('https://www.mvideo.ru/bff/products/prices', params=params, cookies=cookies, headers=headers).json()
        material_prices = response.get('body').get('materialPrices')

        for item in material_prices:
            item_id = item.get('price').get("productId")
            item_base_price = item.get('price').get('basePrice')
            item_sale_price = item.get('price').get('salePrice')
            item_bonus = item.get('bonusRubles').get('total')

            products_prices[item_id] = {
                'item_basePrice': item_base_price,
                'item_salePrice': item_sale_price,
                'item_bonus': item_bonus
            }

        print(f'[+] Finished {i + 1} of the {pages_count} pages')

    with open("DNS-shop/data/1_products_ids.json", "w", encoding="utf-8", newline = '') as file:
        json.dump(products_ids, file, indent=4, ensure_ascii=False)

    with open("DNS-shop/data/2_products_descriptions.json", "w", encoding="utf-8", newline = '') as file:
        json.dump(products_description, file, indent=4, ensure_ascii=False)

    with open("DNS-shop/data/3_products_prices.json", "w", encoding="utf-8", newline = '') as file:
        json.dump(products_prices, file, indent=4, ensure_ascii=False)

def get_result():
    with open('DNS-shop/data/2_products_descriptions.json', encoding='utf-8', newline='') as file:
        products_data = json.load(file)

    with open('DNS-shop/data/3_products_prices.json', encoding='utf-8', newline='') as file:
        products_prices = json.load(file)

    for items in products_data.values():
        products = items.get('body').get('products')

        for item in products:
            product_id = item.get('productId')

            if product_id in products_prices:
                    prices = products_prices[product_id]

            item['item_basePrice'] = prices.get('item_basePrice')
            item['item_salePrice'] = prices.get('item_salePrice')
            item['item_bonus'] = prices.get('item_bonus')
            item['item_link'] = f'https://www.mvideo.ru/products/{item.get("nameTranslit")}-{item.get("productId")}'

    with open("DNS-shop/data/4_result.json", "w", encoding="utf-8", newline='') as file:
        json.dump(products_data, file, indent=4, ensure_ascii=False)

def main():
   get_data()
   get_result()

if __name__ == '__main__':
   main()


import requests

from requests.adapters import HTTPAdapter
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from lxml import etree
from constants import URL_SEARCH, XPATH_URLS

class MedParse:
    __url_search = URL_SEARCH
    _xpath_urls = XPATH_URLS

    def __init__(self, region, area, search_str):
        self.__region = region
        self.__area = area
        self._search_str = search_str

    def __str__(self):
        return 'Class for parse med'

    def get_inform(self):
        # search_str - search string
        # sreg - city (region)
        # dist - area
        data = {
            'free_str': self._search_str,
            'sreg': self.__region,
            'dist': self.__area
        }
        response = self.request_with_retry('POST', self.__url_search, data)
        soup = BeautifulSoup(response, 'lxml')
        options = soup.select('option')
        if options:
            href = options[1]['value']
            url_next = urljoin(self.__url_search, href)
            response = self.request_with_retry('GET', request_url=url_next)
        results = self.get_data_from_table(response, data)
        return results

    def get_data_from_table(self, response, data):
        items = list()
        soup = BeautifulSoup(response, 'lxml')
        divs = soup.select('div.trow')
        for div in divs:
            div = str(div)
            tree = etree.XML(div)
            res = dict()
            for key, value in self._xpath_urls.items():
                item = tree.xpath(value)
                if item:
                    if len(item) > 1:
                        item = ''.join(item).replace(',', '').strip()
                        item = [item]
                    res[key] = item[0]
            if res.keys():
                items.append(res)
        data['results'] = items
        return data

    def request_with_retry(self, method, request_url, data=None):
        session = requests.Session()

        session.mount('https://', HTTPAdapter(max_retries=5))

        if method == 'GET':
            return requests.get(
                url=request_url,
                timeout=3,
                allow_redirects=True).content.decode('utf-8')
        elif method == 'POST':
            return requests.post(
                url=request_url,
                timeout=3,
                allow_redirects=True,
                data=data).content.decode('utf-8')

# if __name__ == '__main__':
#     url = URL_SEARCH
#     sreg = 78
#     dist = 9
#     search_str = 'парацетамол'
#     med_parse = MedParse(sreg, dist, search_str)
#     print(med_parse)
#     print(med_parse.get_inform())


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.init as init
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler
import torchvision.transforms as T
#import random
import torchvision.models as models
# from torchsummary import summary
import numpy as np
from dataloader import Train

from PIL import Image
import cv2

class Model(nn.Module):
    def __init__(self):
        ###INPUT MODULES###
        #mobile net takes images of size (224x224x3) as input

        super(Model,self).__init__()
        self.mobile_net = models.mobilenet_v2(num_classes=512)
        self.image_module = nn.Sequential(
            nn.Linear(512,1024),
            nn.ReLU(True),
            nn.Linear(1024,512),
            nn.ReLU(True),

            )
        self.imu_module = nn.Sequential(
            nn.Linear(10, 256),
            nn.ReLU(True),
            nn.Linear(256,256),
            nn.ReLU(True),
            )

        self.depth_module = nn.Sequential(#input of size: 200x88)
            nn.Conv2d(1,32,4,2,1,bias=True),
            nn.Dropout(p=0.2),
            nn.BatchNorm2d(32),
            nn.ReLU(True),
            nn.Conv2d(32,32,4,2,1,bias=True),
            nn.Dropout(p=0.2),
            nn.BatchNorm2d(32),
            nn.ReLU(True),

            nn.Conv2d(32,64,4,2,1,bias=True),
            nn.Dropout(p=0.2),
            nn.BatchNorm2d(64),
            nn.ReLU(True),
            nn.Conv2d(64,64,4,2,1,bias=True),
            nn.Dropout(p=0.2),
            nn.BatchNorm2d(64),
            nn.ReLU(True),

          #  nn.Conv2d(64,128,4,2,1,bias=False),
          #  nn.Dropout(p=0.2),
          #  nn.BatchNorm2d(128),
          #  nn.ReLU(True),
          #  nn.Conv2d(128,128,4,2,1,bias=False),
          #  nn.Dropout(p=0.2),
          #  nn.BatchNorm2d(128),
          #  nn.ReLU(True),

          #  nn.Conv2d(128,256,4,2,1,bias=False),
          #  nn.Dropout(p=0.2),
          #  nn.BatchNorm2d(256),
          #  nn.ReLU(True),
          #  nn.Conv2d(256,256,4,2,1,bias=False),
          #  nn.Dropout(p=0.2),
          #  nn.BatchNorm2d(256),
          #  nn.ReLU(True),

            nn.Flatten(),

            #input to linear layer probably incorrect
           # nn.Linear(384 , 512),
           #  nn.Linear(3840,512),
            nn.Linear(3072,512),
            nn.Dropout(p=0.5),
            nn.ReLU(True),

            nn.Linear(512,512),
            nn.Dropout(p=0.5),
            nn.ReLU(True)
            )

        print("SUMMARY DEPTH MODULE")
        # print(summary(self.depth_module,(1,88,200)))
        self.speed_module = nn.Sequential(
            nn.Linear(1, 128),
            nn.ReLU(True),
            nn.Linear(128,128),
            nn.ReLU(True)
            )

        self.dense_layers = nn.Sequential(
            #512depth, 512image, 128speed, 256imu
            nn.Linear(1408,512),
            nn.ReLU(True)
            )

        ###COMMAND BRANCHEs###
        self.straight_net = nn.Sequential(
            nn.Linear(512,256),
            nn.ReLU(True),
            nn.Linear(256,256),
            nn.ReLU(True),
            nn.Linear(256,3),  #outputlayer is supposed to have no activation function
            nn.Sigmoid()
            )

        self.right_net = nn.Sequential(
            nn.Linear(512,256),
            nn.ReLU(True),
            nn.Linear(256,256),
            nn.ReLU(True),
            nn.Linear(256,3),  #outputlayer is supposed to have no activation function
            nn.Sigmoid()
            )
        self.left_net = nn.Sequential(
            nn.Linear(512,256),
            nn.ReLU(True),
            nn.Linear(256,256),
            nn.ReLU(True),
            nn.Linear(256,3),  #outputlayer is supposed to have no activation function
            nn.Sigmoid()
            )

        self.weight_init()

    def weight_init(self):
        for block in self._modules:
            print("Block:", block)
            if block=='mobile_net':
                pass
            else:
                for m in self._modules[block]:
                    print("M:", m)
                    if isinstance(m, nn.Conv2d):
                        nn.init.normal(m.weight,mean=0, std=0.01)
                    elif isinstance(m, nn.Linear):
                        nn.init.normal(m.weight,mean=0,std=0.01)
                    else:
                        pass
               # normal_init(m)

    def forward(self, input_data):
        image = input_data[0]
        image = self._image_module(image)
        imu   = input_data[2]
        imu   = self._imu_module(imu)
        depth = input_data[1]
        depth = self._depth_module(depth)
        speed = input_data[3]
        speed = self._speed_module(speed)
        command=input_data[4]

        concat = torch.cat((image,imu,depth,speed),1)
        concat = self._dense_layers(concat)

        ##########################################################
        ##1 here needs to be changed if batch size is changed!!##
        ##########################################################
        output = torch.Tensor()
        for i in range(1):
            if command[i]==1:
                if output.shape==torch.Size([0]):
                    output = self._straight_net(concat)
                else:
                    torch.stack(tensors=(output, self._straight_net(concat)),dim=1)

            elif command[i]==2:
                if output.shape==torch.Size([0]):
                    output = self._right_net(concat)
                else:
                    torch.stack(tensors=(output, self._right_net(concat)),dim=1)

            elif command[i]==0:
                if output.shape==torch.Size([0]):
                    output = self._left_net(concat)
                else:
                    torch.stack(tensors=(output, self._left_net(concat)),dim=1)

        return output

    def _image_module(self,x):
        x = self.mobile_net(x)
        return self.image_module(x)
    def _imu_module(self,x):
        return self.imu_module(x)

    def _depth_module(self,x):
        return self.depth_module(x)

    def _speed_module(self,x):
        return self.speed_module(x)

    def _dense_layers(self,x):
        return self.dense_layers(x)

    def _straight_net(self,x):
        return self.straight_net(x)

    def _right_net(self,x):
        return self.right_net(x)

    def _left_net(self,x):
        return self.left_net(x)

def normal_init(m,mean_std):
    if isinstance(m,(nn.Linear,nn.Conv2d)):
        m.weight.dtaa.normal_(mean_std)
        if m.bias.data is not None:
            m.bias.data.zero_()
        elif isinstance(m, (nn.BatchNorm2d, nn.BatchNorm1d)):
            m.weight.data.fill_(1)
            if m.bias.data != None:
                m.bias.data.zero_()

class Solver:
    def __init__(self):
        self.MSE_loss = torch.nn.MSELoss()
        self.cuda = True
        self.num_repetitions_per_step = 1
        self.num_train_iterations =  int(200/1) #int(size_dataset/batch_size)
        self.num_test_iterations = int(200/1) #int(size_testdataset/batchsize)

        self.loss_steer = []
        self.loss_throttle = []
        self.loss_brake = []

    def read_data(self,dataloader):
        data,label = next(iter(dataloader))
        # print("dataloader", np.asarray(data).shape)
        return data,label

    def train_iteration(self,model, dataloader):
        memory = []
        optimizer = optim.Adam(model.parameters(), lr=5e-4)
        model.train()
        if self.cuda==True:
            model = model.cuda()
        for step in range(self.num_repetitions_per_step):
            print("num_repetition inside train_step")
            for i in range(self.num_train_iterations):

                batch_data, batch_label  = self.read_data(dataloader)
                batch_label = torch.FloatTensor(batch_label)

                if self.cuda==True:
                    for j in range(len(batch_data)):
                        batch_data[j] = batch_data[j].cuda()
                    batch_label = batch_label.cuda()

                output = model(batch_data)
                loss = self.MSE_loss((output), batch_label)
                print(f"[Training iter:{i}] loss={loss.item()}")
                memory.append(loss.item())
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
 #               break
        return model, memory

    def test_iteration(self,model,dataloader):
        model.eval()
        list_mse = []
        for i in range(self.num_test_iterations):
            batch_data, batch_label = self.read_data(dataloader)
            batch_label = torch.FloatTensor(batch_label)
            if self.cuda==True:
                for j in range(len(batch_data)):
                    batch_data[j]  = batch_data[j].cuda()
                batch_label = batch_label.cuda()

            output      = model(batch_data)
            print(f"output of Model: {output.detach()}, Ground Truth: {batch_label}")
            with open("testing.txt", "a") as f:
                f.write(f"{output[0][1]},{batch_label[1]},{output[0][0]},{batch_label[0]},{output[0][2]},{batch_label[2]}\n")
            mse_step   = self.MSE_loss(output,batch_label)
#             print(f"[Testing iter:{i}] loss={mse_step.item()}")
            list_mse.append(mse_step.item())
        avg_mse = np.average(list_mse)
        return avg_mse, list_mse

# ------

# ---------------

class Test(Dataset):
    def __init__(self):
        super().__init__()
        self.dataset =[]
        transform = T.ToTensor()
        for i in range(150):
            image   = np.random.rand(135,278,3).astype('float32')
            image   = numpy_to_pil(image)
            image   = T.Resize((224,224),
                    interpolation=T.InterpolationMode.BICUBIC)(image)
            image   = transform(image)
           # print("____")
           # print(np.asarray(image).shape)

            depth   = np.random.rand(210,95,1).astype('float32')
            depth   = cv2.resize(depth,dsize=(200,88),
                    interpolation=cv2.INTER_CUBIC)
            depth   = transform(depth)
           # print(np.asarray(depth).shape)

            imu     = np.random.rand(10).astype('float32')
           # print(np.asarray(imu).shape)
            speed   = np.random.rand(1).astype('float32')
           # print(np.asarray(speed).shape)
            command = np.random.randint(0,3, size=(1)).astype('float32')
            label   = np.random.rand(2).astype('float32')

            data    = [image,depth,
                torch.Tensor(imu),torch.Tensor(speed),
                torch.Tensor(command)]
            datapoint   = [data,label]
            self.dataset.append(datapoint)

    def __getitem__(self,index):
        data, label = self.dataset[index]
        #print("data",np.asarray(data).shape)
        return data,label

    def __len__(self):
        return len(self.test_dataset)

def numpy_to_pil(image):
    minv = np.amin(image)
    maxv = np.amax(image)
    img  = Image.fromarray((255* (image-minv) / (maxv- minv)).astype(np.uint8))
    return img

if __name__=="__main__":
    print("STARTING")
    # Change train flag to "train" to do image augmentation for rgb images
    train_data  = Train(data_dir='./training_data/', train_eval_flag="no_train")
    # test_data   = Test()
    # Change the data_dir for test
    test_data  = Train(data_dir='./training_data/', train_eval_flag="no_train")

    print("INITIALIZED TRAIN AND TEST SET")
    train_indices = list(range(200)) #as many as train samples
    test_indices = list(range(200)) #as many as test samples
    train_dataloader = DataLoader(train_data,
            sampler=SubsetRandomSampler(train_indices),
            batch_size=1, drop_last=True)
    data_train = torch.utils.data.DataLoader(
    Train(
        data_dir='.',
        train_eval_flag="train")
    ,
    batch_size=2,
    num_workers=1,
    pin_memory=True,
    shuffle=True
    )
    test_dataloader = DataLoader(test_data,
            sampler=SubsetRandomSampler(test_indices),
            batch_size=1, drop_last=True)
  #  if True:
   #     stop
    model = Model()

    # ***** if you want to use the model we trained before, just uncomment the next line ********
    # model = torch.load("model.pt")
   # print(summary(model,(5,)))
   #  stop
    solver= Solver()
    list_test_loss = []
    list_train_loss = []
    for i in range(2):
        print("##############ITERATION NO {}###############".format(i))
        model, train_loss = solver.train_iteration(model,train_dataloader)
        list_train_loss += train_loss
        torch.save(model, "./model.pt")
        test_loss_avg, test_loss = solver.test_iteration(model, test_dataloader)
        print("Average Lost of ITER {} - {}".format(i, test_loss_avg))
        list_test_loss += test_loss

    # ***********for saving loss to txt file to read*****************
    # with open('train_loss.txt', 'w') as f:
    #     for item in list_train_loss:
    #         f.write("%s\n" % item)

    # with open('test_loss.txt', 'w') as f:
    #     for item in list_test_loss:
    #         f.write("%s\n" % item)


from typing import List, Union, Dict

from pydantic import BaseModel

class GcpsBase(BaseModel):
	image_name: str
	symbol_name: str

class Gcps(GcpsBase):
	pixel_x: float
	pixel_y: float
	wgs_lat: float
	wgs_lng: float

	class Config:
		orm_mode = True


#!/usr/bin/env python3

from __future__ import annotations

"""
Tool to help automate changes needed in commits during and after releases
"""

import argparse
import logging
import sys
from datetime import datetime
from pathlib import Path
from subprocess import PIPE, run
from typing import List

LOG = logging.getLogger(__name__)
NEW_VERSION_CHANGELOG_TEMPLATE = """\
## Unreleased

### Highlights

<!-- Include any especially major or disruptive changes here -->

### Stable style

<!-- Changes that affect Black's stable style -->

### Preview style

<!-- Changes that affect Black's preview style -->

### Configuration

<!-- Changes to how Black can be configured -->

### Packaging

<!-- Changes to how Black is packaged, such as dependency requirements -->

### Parser

<!-- Changes to the parser or to version autodetection -->

### Performance

<!-- Changes that improve Black's performance. -->

### Output

<!-- Changes to Black's terminal output and error messages -->

### _Blackd_

<!-- Changes to blackd -->

### Integrations

<!-- For example, Docker, GitHub Actions, pre-commit, editors -->

### Documentation

<!-- Major changes to documentation and policies. Small docs changes
     don't need a changelog entry. -->
"""

class NoGitTagsError(Exception): ...  # noqa: E701,E761

# TODO: Do better with alpha + beta releases
# Maybe we vendor packaging library
def get_git_tags(versions_only: bool = True) -> List[str]:
    """Pull out all tags or calvers only"""
    cp = run(["git", "tag"], stdout=PIPE, stderr=PIPE, check=True, encoding="utf8")
    if not cp.stdout:
        LOG.error(f"Returned no git tags stdout: {cp.stderr}")
        raise NoGitTagsError
    git_tags = cp.stdout.splitlines()
    if versions_only:
        return [t for t in git_tags if t[0].isdigit()]
    return git_tags

# TODO: Support sorting alhpa/beta releases correctly
def tuple_calver(calver: str) -> tuple[int, ...]:  # mypy can't notice maxsplit below
    """Convert a calver string into a tuple of ints for sorting"""
    try:
        return tuple(map(int, calver.split(".", maxsplit=2)))
    except ValueError:
        return (0, 0, 0)

class SourceFiles:
    def __init__(self, black_repo_dir: Path):
        # File path fun all pathlib to be platform agnostic
        self.black_repo_path = black_repo_dir
        self.changes_path = self.black_repo_path / "CHANGES.md"
        self.docs_path = self.black_repo_path / "docs"
        self.version_doc_paths = (
            self.docs_path / "integrations" / "source_version_control.md",
            self.docs_path / "usage_and_configuration" / "the_basics.md",
        )
        self.current_version = self.get_current_version()
        self.next_version = self.get_next_version()

    def __str__(self) -> str:
        return f"""\
> SourceFiles ENV:
  Repo path: {self.black_repo_path}
  CHANGES.md path: {self.changes_path}
  docs path: {self.docs_path}
  Current version: {self.current_version}
  Next version: {self.next_version}
"""

    def add_template_to_changes(self) -> int:
        """Add the template to CHANGES.md if it does not exist"""
        LOG.info(f"Adding template to {self.changes_path}")

        with self.changes_path.open("r") as cfp:
            changes_string = cfp.read()

        if "## Unreleased" in changes_string:
            LOG.error(f"{self.changes_path} already has unreleased template")
            return 1

        templated_changes_string = changes_string.replace(
            "# Change Log\n",
            f"# Change Log\n\n{NEW_VERSION_CHANGELOG_TEMPLATE}",
        )

        with self.changes_path.open("w") as cfp:
            cfp.write(templated_changes_string)

        LOG.info(f"Added template to {self.changes_path}")
        return 0

    def cleanup_changes_template_for_release(self) -> None:
        LOG.info(f"Cleaning up {self.changes_path}")

        with self.changes_path.open("r") as cfp:
            changes_string = cfp.read()

        # Change Unreleased to next version
        versioned_changes = changes_string.replace(
            "## Unreleased", f"## {self.next_version}"
        )

        # Remove all comments (subheadings are harder - Human required still)
        no_comments_changes = []
        for line in versioned_changes.splitlines():
            if line.startswith("<!--") or line.endswith("-->"):
                continue
            no_comments_changes.append(line)

        with self.changes_path.open("w") as cfp:
            cfp.write("\n".join(no_comments_changes) + "\n")

        LOG.debug(f"Finished Cleaning up {self.changes_path}")

    def get_current_version(self) -> str:
        """Get the latest git (version) tag as latest version"""
        return sorted(get_git_tags(), key=lambda k: tuple_calver(k))[-1]

    def get_next_version(self) -> str:
        """Workout the year and month + version number we need to move to"""
        base_calver = datetime.today().strftime("%y.%m")
        calver_parts = base_calver.split(".")
        base_calver = f"{calver_parts[0]}.{int(calver_parts[1])}"  # Remove leading 0
        git_tags = get_git_tags()
        same_month_releases = [t for t in git_tags if t.startswith(base_calver)]
        if len(same_month_releases) < 1:
            return f"{base_calver}.0"
        same_month_version = same_month_releases[-1].split(".", 2)[-1]
        return f"{base_calver}.{int(same_month_version) + 1}"

    def update_repo_for_release(self) -> int:
        """Update CHANGES.md + doc files ready for release"""
        self.cleanup_changes_template_for_release()
        self.update_version_in_docs()
        return 0  # return 0 if no exceptions hit

    def update_version_in_docs(self) -> None:
        for doc_path in self.version_doc_paths:
            LOG.info(f"Updating black version to {self.next_version} in {doc_path}")

            with doc_path.open("r") as dfp:
                doc_string = dfp.read()

            next_version_doc = doc_string.replace(
                self.current_version, self.next_version
            )

            with doc_path.open("w") as dfp:
                dfp.write(next_version_doc)

            LOG.debug(
                f"Finished updating black version to {self.next_version} in {doc_path}"
            )

def _handle_debug(debug: bool) -> None:
    """Turn on debugging if asked otherwise INFO default"""
    log_level = logging.DEBUG if debug else logging.INFO
    logging.basicConfig(
        format="[%(asctime)s] %(levelname)s: %(message)s (%(filename)s:%(lineno)d)",
        level=log_level,
    )

def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-a",
        "--add-changes-template",
        action="store_true",
        help="Add the Unreleased template to CHANGES.md",
    )
    parser.add_argument(
        "-d", "--debug", action="store_true", help="Verbose debug output"
    )
    args = parser.parse_args()
    _handle_debug(args.debug)
    return args

def main() -> int:
    args = parse_args()

    # Need parent.parent cause script is in scripts/ directory
    sf = SourceFiles(Path(__file__).parent.parent)

    if args.add_changes_template:
        return sf.add_template_to_changes()

    LOG.info(f"Current version detected to be {sf.current_version}")
    LOG.info(f"Next version will be {sf.next_version}")
    return sf.update_repo_for_release()

if __name__ == "__main__":  # pragma: no cover
    sys.exit(main())


import deform
from pyramid.view import view_config

from dace.processinstance.core import DEFAULTMAPPING_ACTIONS_VIEWS
from pontus.default_behavior import Cancel
from pontus.form import FormView
from pontus.view import BasicView
from pontus.view_operation import MultipleView

from novaideo.content.processes.user_management.behaviors import (
    Remind)
from novaideo.content.person import Preregistration
from novaideo import _

class RemindViewStudyReport(BasicView):
    title = 'Alert for remove'
    name = 'alertforremove'
    template = 'novaideo:views/user_management/templates/alert_remind_registration.pt'

    def update(self):
        result = {}
        values = {'context': self.context}
        body = self.content(args=values, template=self.template)['body']
        item = self.adapt_item(body, self.viewid)
        result['coordinates'] = {self.coordinates: [item]}
        return result

class RemindView(FormView):
    title = _('Remove')
    name = 'remindform'
    formid = 'formremind'
    behaviors = [Remind, Cancel]
    validate_behaviors = False

    def before_update(self):
        self.action = self.request.resource_url(
            self.context, 'novaideoapi',
            query={'op': 'update_action_view',
                   'node_id': Remind.node_definition.id})
        self.schema.widget = deform.widget.FormWidget(
            css_class='deform novaideo-ajax-form')

@view_config(
    name='remind',
    context=Preregistration,
    renderer='pontus:templates/views_templates/grid.pt',
    )
class RemindViewMultipleView(MultipleView):
    title = _('Remind the user')
    name = 'remind'
    viewid = 'remind'
    template = 'pontus:templates/views_templates/simple_multipleview.pt'
    views = (RemindViewStudyReport, RemindView)
    validators = [Remind.get_validator()]

DEFAULTMAPPING_ACTIONS_VIEWS.update(
    {Remind: RemindViewMultipleView})


from rest_framework import serializers
from rest_framework.relations import SlugRelatedField
from rest_framework.validators import UniqueTogetherValidator

from posts.models import Comment, Follow, Group, Post, User

class PostSerializer(serializers.ModelSerializer):
    author = SlugRelatedField(slug_field='username', read_only=True)

    class Meta:
        fields = '__all__'
        model = Post

class GroupSerializer(serializers.ModelSerializer):
    class Meta:
        fields = '__all__'
        model = Group

class CommentSerializer(serializers.ModelSerializer):
    post = serializers.ReadOnlyField(source='post.id')
    author = serializers.SlugRelatedField(
        read_only=True,
        slug_field='username',
    )

    class Meta:
        fields = '__all__'
        model = Comment

class FollowSerializer(serializers.ModelSerializer):
    user = serializers.SlugRelatedField(
        slug_field='username',
        read_only=True,
        default=serializers.CurrentUserDefault()
    )
    following = serializers.SlugRelatedField(
        slug_field='username',
        queryset=User.objects.all()
    )

    class Meta:
        fields = ('user', 'following')
        model = Follow
        validators = [
            UniqueTogetherValidator(
                queryset=Follow.objects.all(),
                fields=['user', 'following']
            )
        ]

    def validate_following(self, value):
        if self.context['request'].user != value:
            return value
        raise serializers.ValidationError("Self-follow is not allowed")


#!/usr/bin/env python
try:  # support either PyQt5 or 6
    from PyQt5 import uic
    from PyQt5.QtCore import *
    from PyQt5.QtGui import QSurfaceFormat
    from PyQt5.QtWidgets import QApplication, QMainWindow

    PyQtVersion = 5
except ImportError:
    print("trying Qt6")
    from PyQt6 import uic
    from PyQt6.QtCore import QEvent, Qt, QTimer
    from PyQt6.QtGui import QSurfaceFormat
    from PyQt6.QtWidgets import QApplication, QMainWindow

    PyQtVersion = 6

import math
import random
import sys

sys.path.insert(0, "../ByteImage")
import ByteImage

class MainWindow(QMainWindow):
    def __init__(self, parent=None):
        super(QMainWindow, self).__init__(parent)
        uic.loadUi("form.ui", self)  # Load the .ui file
        self.width.valueChanged.connect(self.transmit_size)
        self.height.valueChanged.connect(self.transmit_size)
        self.start_sim.clicked.connect(self.toggle_sim)
        self.timer = QTimer()
        self.timer.timeout.connect(self.transmit_data)

    def toggle_sim(self):
        if self.timer.isActive():
            self.timer.stop()
            self.start_sim.setText("Start")
        else:
            self.timer.start(self.timer_duration.value())
            self.start_sim.setText("Stop")

    def send_lines(self):
        rx = random.randint
        img = ByteImage.ByteImage(
            self.width.value(), self.height.value(), 255, 255, 255, 255
        )
        img.clear(255, 255, 255)
        cx = rx(1, img.width - 1)
        cy = rx(1, img.height - 1)
        for x in range(0, 1000):
            img.line(
                cx,
                cy,
                rx(1, img.width - 1),
                rx(1, img.height - 1),
                rx(0, 255),
                rx(0, 255),
                rx(0, 255),
            )
        with open(self.fifo_name.text(), "wb") as fifo:
            fifo.write(img.pixels)

    def send_check(self):
        rx = random.randint
        img = ByteImage.ByteImage(
            self.width.value(), self.height.value(), 255, 255, 255, 255
        )
        img.checker(
            rx(0, 255), rx(0, 255), rx(0, 255), 255, 255, 255, check_size=rx(5, 50)
        )
        with open(self.fifo_name.text(), "wb") as fifo:
            fifo.write(img.pixels)

    def send_pixel(self):
        img = ByteImage.ByteImage(
            self.width.value(), self.height.value(), 255, 255, 255, 255
        )
        img.random_pixels()
        with open(self.fifo_name.text(), "wb") as fifo:
            fifo.write(img.pixels)

    def transmit_data(self):
        if self.mode.currentIndex() == 0:
            self.send_lines()
        elif self.mode.currentIndex() == 1:
            self.send_check()
        elif self.mode.currentIndex() == 2:
            self.send_pixel()

    def transmit_size(self):
        with open("imagefifo", "wb") as fifo:
            fifo.write(f"resize {self.width.value()} {self.height.value()}".encode())

    if PyQtVersion == 5:

        def keyPressEvent(self, event):
            key = event.key()
            if key == Qt.Key_Escape:
                exit()
            self.update()

    else:  # Qt6 Versions

        def keyPressEvent(self, event):
            key = event.key()
            if key == Qt.Key.Key_Escape:
                exit()
            self.update()

if __name__ == "__main__":
    app = QApplication(sys.argv)
    window = MainWindow()
    window.resize(200, 200)
    window.show()
    if PyQtVersion == 5:
        sys.exit(app.exec_())
    else:
        sys.exit(app.exec())


import torch.optim as optim
import torch.nn as nn

def train(model, dataset, *, epochs=20, learning_rate=0.0001, verbose=True):
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)
    loss_func = nn.NLLLoss()
    loss_list = []

    model.train()
    for epoch in range(epochs):

        total_loss = []
        for (data, target) in dataset:
            optimizer.zero_grad()
            output = model(data)
            loss = loss_func(output, target)
            loss.backward()
            optimizer.step()
            total_loss.append(loss.item())

        loss_list.append(sum(total_loss) / len(total_loss))

        if verbose:
            progress = int(100. * (epoch + 1) / epochs)
            tloss = round(loss_list[-1], 5)
            print(f"Training: Progress [{progress}%] Loss [{tloss}]")

    return model