# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import (
    urlencode_postdata,
    ExtractorError,
)

class BigoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?bigo\.tv/(?:[a-z]{2,}/)?(?P<id>[^/]+)'
    # Note: I would like to provide some real test, but Bigo is a live streaming
    # site, and a test would require that the broadcaster is live at the moment.
    # So, I don't have a good way to provide a real test here.
    _TESTS = [{
        'url': 'https://www.bigo.tv/th/Tarlerm1304',
        'only_matching': True,
    }, {
        'url': 'https://bigo.tv/115976881',
        'only_matching': True,
    }]

    def _real_extract(self, url):
        user_id = self._match_id(url)

        INFO_URL = 'https://bigo.tv/studio/getInternalStudioInfo'
        info_raw = self._download_json(
            INFO_URL, user_id, data=urlencode_postdata({'siteId': user_id}))

        if info_raw['code'] != 0:
            if self._downloader.params.get('verbose', False):
                self.to_screen(
                    '[debug] getInternalStudioInfo returns code %i, msg "%s"'
                    % (info_raw['code'], info_raw['msg']))

            raise ExtractorError(
                "Failed to get user's data. Most likely the user doesn't exist",
                expected=True)
        info = info_raw['data']

        alive = info.get('alive')
        if alive is not None and alive == 0:
            raise ExtractorError(
                'User "%s" is not live at the moment' % user_id,
                expected=True)

        return {
            'id': info.get('roomId') or user_id,
            'title': info.get('roomTopic'),
            'url': info.get('hls_src'),
            'ext': 'mp4',
            'thumbnail': info.get('snapshot'),
            'uploader': info.get('nick_name'),
            'is_live': True,
        }


import vtk

def make_clipper(plane,normals):
    """
    Returns a clipper filter, of type vtk.vtkClipPolyData()
    takes as input:
      * the cut plane, of type vtk.vtkPlane()
      * The normals to the polydata, of type  vtk.vtkPolyDataNormals()
    """
    # We clip with an implicit function. Here we use a plane positioned near
    # the center of the cow model and oriented at an arbitrary angle.

    # vtkClipPolyData requires an implicit function to define what it is to
    # clip with. Any implicit function, including complex boolean combinations
    # can be used. Notice that we can specify the value of the implicit function
    # with the SetValue method.
    clipper = vtk.vtkClipPolyData()    #clip filter ?
    clipper.SetInputConnection(normals.GetOutputPort())
    clipper.SetClipFunction(plane)
    clipper.GenerateClipScalarsOn()
    clipper.GenerateClippedOutputOn()
    clipper.SetValue(0.5)

    return clipper

def make_actors(plane,clipper,normals,color):
    """ Creates three actors for a given volume: clipActor,cutActor,restActor
    takes as input plane and normal (same as parameters for  make_clipper)
    and clipper (generated by  make_clipper)
    """
    clipMapper = vtk.vtkPolyDataMapper()   # Mapper
    clipMapper.SetInputConnection(clipper.GetOutputPort())
    clipMapper.ScalarVisibilityOff()

    #backProp = vtk.vtkProperty()
    #backProp.SetDiffuseColor(tomato)

    clipActor = vtk.vtkActor()      # Actor
    clipActor.SetMapper(clipMapper)
    clipActor.GetProperty().SetColor(color)
    clipActor.GetProperty().EdgeVisibilityOff()
    clipActor.GetProperty().SetOpacity(0.5)
    clipActor.GetProperty().SetInterpolationToGouraud()

    #clipActor.SetBackfaceProperty(backProp)

    # Here we are cutting the cow. Cutting creates lines where the cut
    # function intersects the model. (Clipping removes a portion of the
    # model but the dimension of the data does not change.)
    #
    # The reason we are cutting is to generate a closed polygon at the
    # boundary of the clipping process. The cutter generates line
    # segments, the stripper then puts them together into polylines. We
    # then pull a trick and define polygons using the closed line
    # segments that the stripper created.
    cutEdges = vtk.vtkCutter()
    cutEdges.SetInputConnection(normals.GetOutputPort())
    cutEdges.SetCutFunction(plane)
    cutEdges.GenerateCutScalarsOn()
    cutEdges.SetValue(0, 0.5)

    cutStrips = vtk.vtkStripper()
    cutStrips.SetInputConnection(cutEdges.GetOutputPort())
    cutStrips.Update()

    cutPoly = vtk.vtkPolyData()
    cutPoly.SetPoints(cutStrips.GetOutput().GetPoints())
    cutPoly.SetPolys(cutStrips.GetOutput().GetLines())

    # Triangle filter is robust enough to ignore the duplicate point at
    # the beginning and end of the polygons and triangulate them.
    cutTriangles = vtk.vtkTriangleFilter()
    cutTriangles.SetInputData(cutPoly)
    cutMapper = vtk.vtkPolyDataMapper()
    cutMapper.SetInputData(cutPoly)
    cutMapper.SetInputConnection(cutTriangles.GetOutputPort())
    cutActor = vtk.vtkActor()
    cutActor.SetMapper(cutMapper)
    cutActor.GetProperty().SetColor(color)

    # The clipped part of the cow is rendered wireframe.
    restMapper = vtk.vtkPolyDataMapper()
    restMapper.SetInputConnection(clipper.GetClippedOutputPort())
    restMapper.ScalarVisibilityOff()
    restActor = vtk.vtkActor()
    restActor.SetMapper(restMapper)
    restActor.GetProperty().SetColor(color)
    restActor.GetProperty().EdgeVisibilityOff()
    restActor.GetProperty().SetOpacity(0.1)
    restActor.GetProperty().SetInterpolationToGouraud()

    #restActor.GetProperty().SetRepresentationToWireframe()
    return clipActor,cutActor,restActor


"""
Course model serializers
"""
from urllib.parse import urljoin

from django.conf import settings
from django.templatetags.static import static
from rest_framework import serializers

from courses import models

def _get_thumbnail_url(page):
    """
    Get the thumbnail URL or else return a default image URL.

    Args:
        page (cms.models.ProductPage): A product page

    Returns:
        str:
            A page URL
    """
    relative_url = (
        page.thumbnail_image.file.url
        if page
        and page.thumbnail_image
        and page.thumbnail_image.file
        and page.thumbnail_image.file.url
        else static("images/mit-dome.png")
    )
    return urljoin(settings.SITE_BASE_URL, relative_url)

class BaseCourseSerializer(serializers.ModelSerializer):
    """Basic course model serializer"""

    class Meta:
        model = models.Course
        fields = ["id", "title", "readable_id"]

class BaseCourseRunSerializer(serializers.ModelSerializer):
    """Minimal CourseRun model serializer"""

    class Meta:
        model = models.CourseRun
        fields = [
            "title",
            "start_date",
            "end_date",
            "enrollment_start",
            "enrollment_end",
            "expiration_date",
            "courseware_url",
            "courseware_id",
            "run_tag",
            "id",
        ]

class CourseRunSerializer(BaseCourseRunSerializer):
    """CourseRun model serializer"""

    class Meta:
        model = models.CourseRun
        fields = BaseCourseRunSerializer.Meta.fields

class CourseSerializer(serializers.ModelSerializer):
    """Course model serializer - also serializes child course runs"""

    courseruns = serializers.SerializerMethodField()
    next_run_id = serializers.SerializerMethodField()
    topics = serializers.SerializerMethodField()

    def get_next_run_id(self, instance):
        """Get next run id"""
        run = instance.first_unexpired_run
        return run.id if run is not None else None

    def get_courseruns(self, instance):
        """Unexpired and unenrolled course runs"""
        all_runs = self.context.get("all_runs", False)
        if all_runs:
            active_runs = instance.unexpired_runs
        else:
            user = self.context["request"].user if "request" in self.context else None
            active_runs = (
                instance.available_runs(user)
                if user and user.is_authenticated
                else instance.unexpired_runs
            )
        return [
            CourseRunSerializer(instance=run, context=self.context).data
            for run in active_runs
            if run.live
        ]

    def get_topics(self, instance):
        """List topics of a course"""
        return sorted(
            [{"name": topic.name} for topic in instance.topics.all()],
            key=lambda topic: topic["name"],
        )

    class Meta:
        model = models.Course
        fields = [
            "id",
            "title",
            "readable_id",
            "courseruns",
            "next_run_id",
            "topics",
        ]

class CourseRunDetailSerializer(serializers.ModelSerializer):
    """CourseRun model serializer - also serializes the parent Course"""

    course = BaseCourseSerializer(read_only=True)

    class Meta:
        model = models.CourseRun
        fields = [
            "course",
            "title",
            "start_date",
            "end_date",
            "enrollment_start",
            "enrollment_end",
            "expiration_date",
            "courseware_url",
            "courseware_id",
            "id",
        ]

class BaseProgramSerializer(serializers.ModelSerializer):
    """Basic program model serializer"""

    class Meta:
        model = models.Program
        fields = ["title", "readable_id", "id"]

class ProgramSerializer(serializers.ModelSerializer):
    """Program model serializer"""

    courses = serializers.SerializerMethodField()
    start_date = serializers.SerializerMethodField()
    end_date = serializers.SerializerMethodField()
    enrollment_start = serializers.SerializerMethodField()
    topics = serializers.SerializerMethodField()

    def get_courses(self, instance):
        """Serializer for courses"""
        return CourseSerializer(
            instance.courses.filter(live=True).order_by("position_in_program"),
            many=True,
        ).data

    def get_start_date(self, instance):
        """
        start_date is the starting date for the earliest live course run for all courses in a program

        Returns:
            datetime: The starting date
        """
        return (
            models.CourseRun.objects.filter(course__program=instance, live=True)
            .order_by("start_date")
            .values_list("start_date", flat=True)
            .first()
        )

    def get_end_date(self, instance):
        """
        end_date is the end date for the latest live course run for all courses in a program.

        Returns:
            datetime: The ending date
        """
        return (
            models.CourseRun.objects.filter(course__program=instance, live=True)
            .order_by("end_date")
            .values_list("end_date", flat=True)
            .last()
        )

    def get_enrollment_start(self, instance):
        """
        enrollment_start is first date where enrollment starts for any live course run
        """
        return (
            models.CourseRun.objects.filter(course__program=instance, live=True)
            .order_by("enrollment_start")
            .values_list("enrollment_start", flat=True)
            .first()
        )

    def get_topics(self, instance):
        """List all topics in all courses in the program"""
        topics = (
            models.CourseTopic.objects.filter(course__program=instance)
            .values("name")
            .distinct("name")
        )
        return list(topics)

    class Meta:
        model = models.Program
        fields = [
            "title",
            "readable_id",
            "id",
            "courses",
            "start_date",
            "end_date",
            "enrollment_start",
            "topics",
        ]

class CourseRunEnrollmentSerializer(serializers.ModelSerializer):
    """CourseRunEnrollment model serializer"""

    run = CourseRunDetailSerializer(read_only=True)

    class Meta:
        model = models.CourseRunEnrollment
        fields = ["run"]

class ProgramEnrollmentSerializer(serializers.ModelSerializer):
    """ProgramEnrollmentSerializer model serializer"""

    program = BaseProgramSerializer(read_only=True)
    course_run_enrollments = serializers.SerializerMethodField()

    def __init__(self, *args, **kwargs):
        assert (
            "context" in kwargs and "course_run_enrollments" in kwargs["context"]
        ), "An iterable of course run enrollments must be passed in the context (key: course_run_enrollments)"
        super().__init__(*args, **kwargs)

    def get_course_run_enrollments(self, instance):
        """Returns a serialized list of course run enrollments that belong to this program (in position order)"""
        return CourseRunEnrollmentSerializer(
            sorted(
                (
                    enrollment
                    for enrollment in self.context["course_run_enrollments"]
                    if enrollment.run.course.program_id == instance.program.id
                ),
                key=lambda enrollment: enrollment.run.course.position_in_program,
            ),
            many=True,
        ).data

    class Meta:
        model = models.ProgramEnrollment
        fields = [
            "id",
            "program",
            "course_run_enrollments",
        ]


from django.http import HttpRequest, Http404
from django.test import TestCase
from django.core.urlresolvers import resolve
from django.contrib.webdesign import lorem_ipsum
from les_classiques import views
from les_classiques.models import Author, Classique
from datetime import date
import mongoengine

class HomepageTest(TestCase):
    def test_url_resolver_to_homepage(self):
        found = resolve('/')
        self.assertEqual(found.func, views.homepage)

    def test_correct_html(self):
        request = HttpRequest()
        response = views.homepage(request)
        self.assertTrue(response.content.startswith(b'<html>'))
        self.assertIn(bytes('<title>Poésie</title>', 'utf-8'), response.content)
        self.assertTrue(response.content.endswith(b'</html>'))

class TextsTests(TestCase):
    def test_url_resolver_to_single_text(self):
        found = resolve('/les_classiques/text/bruxelles')
        self.assertEqual(found.func, views.text)

    def test_text_page(self):
        response = views.text(HttpRequest(), slug='bruxelles')
        self.assertIn(b'Plates-bandes', response.content)

    def test_text_does_not_exist(self):
        self.assertRaises(Http404, views.text, HttpRequest(), slug="qdsfjh")

class AuthorsTests(TestCase):
    def test_url_resolver_to_author(self):
        found = resolve('/les_classiques/author/joachim-du-bellay')
        self.assertEqual(found.func, views.author)

    def test_author_page(self):
        response = views.author(HttpRequest(), slug='joachim-du-bellay')
        self.assertIn(b'Au Roi', response.content)

    def test_author_does_not_exist(self):
        self.assertRaises(Http404, views.author, HttpRequest(), slug="qdsfjh")

class DatabaseTest(TestCase):
    def setUp(self):
        Author.ensure_indexes()
        Classique.ensure_indexes()
        # delete test author
        Author.objects(last_name='Tartempion', first_name='Michel-André').delete()

    def test_insert_and_delete_new_author(self):
        author = Author(last_name='Tartempion', first_name='Michel-André')
        author.save()
        db_author = Author.objects(last_name='Tartempion', first_name='Michel-André').first()
        self.assertEqual(db_author.last_name, 'Tartempion')
        self.assertEqual(author.slug, 'michel-andre-tartempion')

    def test_insert_duplicate_author_failure(self):
        author = Author(last_name='Tartempion', first_name='Michel-André')
        author.save()
        author2 = Author(last_name='Tartempion', first_name='Michel-André')
        self.assertRaises(mongoengine.NotUniqueError, author2.save)

    def test_insert_complete_author_and_destroy(self):
        author = Author(last_name='Tartempion', first_name='Michel-André', birthday=date(1821, 4, 9),
                        death_day=date(1867, 8, 31))
        author.save()
        self.assertEqual(Author.objects(last_name='Tartempion', first_name='Michel-André').count(), 1)
        author.delete()
        self.assertEqual(Author.objects(last_name='Tartempion', first_name='Michel-André').count(), 0)

    def test_insert_author_and_classique(self):
        author = Author(last_name='Tartempion', first_name='Michel-André')
        author.save()
        text = lorem_ipsum.COMMON_P
        classique = Classique(slug='les_miserables', title='Les Misérables', author=author, text=text)
        classique2 = Classique(slug='l_art_d_etre_grand_pere', title="L'art d'être grand père", author=author,
                               text=text, pub_date=date(1877, 1, 1), tags=('poem', 'romantic'), collection='Best of')
        classique.save()
        classique2.save()
        self.assertEqual(author.id, classique.author.id)
        self.assertEqual(author.id, classique2.author.id)
        classique.delete()
        classique2.delete()

    def test_insert_same_slug_classique(self):
        author = Author(last_name='Tartempion', first_name='Michel-André')
        author.save()
        text_sample = lorem_ipsum.COMMON_P
        classique = Classique(title='Les Misérables', author=author, text=text_sample, collection='Best of Hugo')
        classique2 = Classique(title="les-misérables", author=author, text=text_sample, pub_date=date(1877, 1, 1))
        classique3 = Classique(title="les miserables", author=author, text=text_sample, pub_date=date(1877, 1, 1))
        classique.save()
        classique2.save()
        classique3.save()
        self.assertEqual(classique.slug, 'les-miserables')
        self.assertEqual(classique2.slug, 'les-miserables1')
        self.assertEqual(classique3.slug, 'les-miserables2')
        classique.delete()
        classique2.delete()
        classique3.delete()


from builtins import enumerate

from django import template
from django.urls import reverse

from tapir.shifts.models import (
    Shift,
    ShiftTemplate,
    ShiftAttendanceTemplate,
    WEEKDAY_CHOICES,
)

register = template.Library()

@register.inclusion_tag("shifts/user_shifts_overview_tag.html", takes_context=True)
def user_shifts_overview(context, user):
    context["user"] = user
    return context

@register.inclusion_tag("shifts/shift_block_tag.html", takes_context=True)
def shift_block(context, shift: Shift, fill_parent=False):
    context["shift"] = shift_to_block_object(shift, fill_parent)
    return context

@register.inclusion_tag("shifts/shift_block_tag.html", takes_context=True)
def shift_template_block(context, shift_template: ShiftTemplate, fill_parent=False):
    context["shift"] = shift_template_to_block_object(shift_template, fill_parent)
    return context

def shift_to_block_object(shift: Shift, fill_parent: bool):
    attendances = ["empty" for _ in range(shift.num_slots)]
    for index, attendance in enumerate(shift.get_valid_attendances()):
        attendances[index] = "single"
        if (
            shift.shift_template is not None
            and ShiftAttendanceTemplate.objects.filter(
                shift_template=shift.shift_template, user=attendance.user
            ).count()
            > 0
        ):
            attendances[index] = "template"

    template_group = None
    if shift.shift_template is not None:
        template_group = template_group_name_to_character(
            shift.shift_template.group.name
        )

    background = "success"
    if shift.get_valid_attendances().count() == 0:
        background = "danger"
    elif shift.get_valid_attendances().count() < shift.num_slots:
        background = "warning"

    style = ""
    if fill_parent:
        style = "height:100%; width: 100%;"

    return {
        "attendances": attendances,
        "name": shift.name,
        "num_slots": shift.num_slots,
        "start_time": shift.start_time,
        "end_time": shift.end_time,
        "start_date": shift.start_time,
        "weekday": None,
        "template_group": template_group,
        "background": background,
        "style": style,
        "id": shift.id,
        "is_template": False,
    }

def shift_template_to_block_object(shift_template: ShiftTemplate, fill_parent: bool):
    attendances = ["empty" for _ in range(shift_template.num_slots)]
    attendance_templates = ShiftAttendanceTemplate.objects.filter(
        shift_template=shift_template
    )
    for index in range(attendance_templates.count()):
        attendances[index] = "template"

    background = "success"
    if attendance_templates.count() == 0:
        background = "danger"
    elif attendance_templates.count() < shift_template.num_slots:
        background = "warning"

    style = ""
    if fill_parent:
        style = "height:100%; width: 100%;"

    return {
        "attendances": attendances,
        "name": shift_template.name,
        "num_slots": shift_template.num_slots,
        "start_time": shift_template.start_time,
        "end_time": shift_template.end_time,
        "start_date": None,
        "weekday": WEEKDAY_CHOICES[shift_template.weekday][1],
        "template_group": template_group_name_to_character(shift_template.group.name),
        "background": background,
        "style": style,
        "id": shift_template.id,
        "is_template": True,
    }

def template_group_name_to_character(name: str):
    if name[-1] == "A":
        return "Ⓐ"
    if name[-1] == "B":
        return "Ⓑ"
    if name[-1] == "C":
        return "Ⓒ"
    if name[-1] == "D":
        return "Ⓓ"
    return None


##
# @brief
# """
# This tutorial introduces logistic regression using Theano and stochastic
# gradient descent.
#
# Logistic regression is a probabilistic, linear classifier. It is parametrized
# by a weight matrix :math:`W` and a bias vector :math:`b`. Classification is
# done by projecting data points onto a set of hyperplanes, the distance to
# which is used to determine a class membership probability.
#
# Mathematically, this can be written as:
#
# .. math::
#   P(Y=i|x, W,b) &= softmax_i(W x + b) \\
#                 &= \frac {e^{W_i x + b_i}} {\sum_j e^{W_j x + b_j}}
#
#
# The output of the model or prediction is then done by taking the argmax of
# the vector whose i'th element is P(Y=i|x).
#
# .. math::
#
#   y_{pred} = argmax_i P(Y=i|x,W,b)
#
#
# This tutorial presents a stochastic gradient descent optimization method
# suitable for large datasets.
#
#
# References:
#
#     - textbooks: "Pattern Recognition and Machine Learning" -
#                  Christopher M. Bishop, section 4.3.2
#
# """
__docformat__ = 'restructedtext en'

import os
import sys
import numpy
import theano
import theano.tensor as T

import LossFunctions
import ModelUtility
import Classifier
import LogisticRegression
import rbm

##
# @brief
#
# @param inputVariable
# @param n_in
# @param n_out
# @param layers
#
# @return
def getMultilayerPerceptron(inputVariable, n_in, layers, layerName = "mlp"):
    # parameters of the model, will be used later to update
    # This can be stored and reload to replicate the same result
    params = {}
    output = inputVariable

    outputs = {}
    wVariables = {}
    bVariables = {}

    # The first output is the input variable
    outputs[0] = output

    # initialize each layer and put them into params
    for i in range(len(layers)):
        # if this is first layer, use n_in as the input dimension,
        # otherwise use the previous layer number as input dimension
        if i == 0:
            inputNumber = n_in
        else:
            inputNumber = layers[i-1]
        outputNumber = layers[i]

        W = theano.shared(
            value=numpy.asarray(
                ModelUtility.getRandomNumpyMatrix(inputNumber, outputNumber),
                dtype=theano.config.floatX
            ),
            borrow=True
        )

        # initialize the baises b as a vector of n_out 0s
        b = theano.shared(
            value=numpy.zeros(
                (outputNumber,),
                dtype=theano.config.floatX
            ),
            borrow=True
        )
        params[layerName + "-W-" + str(i)] = W
        params[layerName + "-b-" + str(i)] = b

        # symbolic expression for computing the matrix of class-membership
        # probabilities
        # Where:
        # W is a matrix where column-k represent the separation hyper plain for
        # class-k
        # x is a matrix where row-j  represents input training sample-j
        # b is a vector where element-k represent the free parameter of hyper
        # plain-k
        output = T.tanh(T.dot(output, W) + b)

        # store output and parameters
        outputs[i + 1] = output
        wVariables[i + 1] = W
        bVariables[i + 1] = b

    return outputs, params, wVariables, bVariables

# Multi-Layer Perceptron Class
#
# A multilayer perceptron is a feedforward artificial neural network model
# that has one layer or more of hidden units and nonlinear activations.
# Intermediate layers usually have as activation function tanh or the
# sigmoid function (defined here by a ``HiddenLayer`` class)  while the
# top layer is a softamx layer (defined here by a ``LogisticRegression``
# class).
#
class MultilayerPerceptron(Classifier.Classifier):

    # must have property of learning model
    # this will used to upload to database
    description = "y_k = s(y_(k-1) * W + b), s is the activation function, repeat this process for k layers"

    def setPretrainLayer(self, layerNumber, batch_size, train_set_x, learning_rate = 0.1):

        self._pretrainModel =  rbm.getPretrainFunction(
                                       self.mlpOutputs[layerNumber-1], # the input variable is the previous layer's output
                                       self._x,
                                       self.wVariables[layerNumber],
                                       self.bVariables[layerNumber],
                                       batch_size,
                                       train_set_x,
                                       learning_rate = learning_rate)

    def saveParameterAsImage(self, name):
        ModelUtility.saveMatrixAsImage(self.wVariables[1].get_value(), name)
    ##
    # @brief            Initialize the parameters
    #
    # @param n_in       number of input units, the dimension of the space in
    #                       which the datapoints lie
    # @param n_out      number of output units, the dimension of the space in
    #                       which the labels lie
    # @param layers     an array of number that represent the number of neural in each layer
    #
    # @return
    def __init__(self, n_in, n_out, layers = [500], taskName = 'testTask2'):
        # initialize classifier class
        super(MultilayerPerceptron, self).__init__()

        # variables to store layer info
        self.mlpOutputs, self.mlpParams, self.wVariables, self.bVariables = getMultilayerPerceptron(self._x, n_in, layers, layerName = taskName + '-mlp')

        logisticRegressionInputNumber = layers[-1]
        self.p_y_given_x, logisticRegressionParams= LogisticRegression.getLogisticRegressionLayer(self.mlpOutputs[len(layers)], logisticRegressionInputNumber, n_out, layerName = taskName + '-lgd')

        self.params = {}
        self.params.update(self.mlpParams)
        self.params.update(logisticRegressionParams)

        # symbolic description of how to compute prediction as class whose
        # probability is maximal
        self.y_pred = T.argmax(self.p_y_given_x, axis=1)

        self.buildTrainingModel()


import copy
import urllib
import requests
import functools

from .login import Salesforce

from .helpers import errors
from .helpers import parsers

from .helpers.report_filters import set_filters, set_period, set_logic

from .report import (
    get_excel,
    get_tabular_reports,
    get_summary_reports,
    get_matrix_reports,
)

URL = "https://{}/services/data/v{}/analytics/reports/"

class Reportforce(Salesforce):
    """Class to interact with the Salesforce Analytics API.

    Attributes
    ----------
    url : str
        Salesforce instance/server URL.

    session : requests.Session
        Session used for making http requests.

    Methods
    -------
    get_report : pandas.DataFrame
        Get a report as a DataFrame.

    get_total : int, float
        Get the grand total of a report.

    get_metadata : dict
        Get a report metadata.
    """

    session = requests.Session()
    session.hooks["response"].append(errors.handle_error)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.url = URL.format(self.instance_url, self.version)

        self.session.headers.update(self.headers)

    def get_report(
        self,
        report_id,
        id_column=None,
        date_column=None,
        start=None,
        end=None,
        filters=[],
        logic=None,
        excel=None,
        **kwargs
    ):
        """Function to get a Salesforce report into a DataFrame.

        Parameters
        ----------
        report_id : str
            A report unique identifier.

        id_column : str (optional)
            Column name which has unique values for each row. This is needed as
            a workaround to the Analytics API's 2000 row limitation.

        date_column : str (optional)
            Date column name.

        start : str (optional)
            Initial date string, passed into dateutil.parser.parse. Must start with
            the day.

        end : str (optional)
            End date string, passed into dateutil.parser.parse. Must start with
            the day.

        filters : list (optional)
            List of tuples, each of which represents a filter:
            [("column", ">=", "value")].

        logic : str (optional)
            Logical filter. This is commonly needed if the report already has a
            report filter, but it can also be useful in case you add more filters.

        excel : bool, str (optional)
            Whether or not you want to save the report in a Excel file. If a
            non-empty string is passed, it will be used as the filename. If True,
            the workbook will be automatically named.

        Returns
        -------
        DataFrame
            A DataFrame contaning the records from the report.

        Raises
        ------
        ReportError
            If there is an error-like JSON string in the reponse body.
        """
        self.metadata = copy.deepcopy(self.get_metadata(report_id))

        if start or end:
            set_period(start, end, date_column, self.metadata)
        if logic:
            set_logic(logic, self.metadata)
        if filters:
            set_filters(filters, self.metadata)

        if excel:
            return get_excel(report_id, excel, self.metadata, self, **kwargs)

        report_format = self.metadata["reportMetadata"]["reportFormat"]

        args = (report_id, id_column, self.metadata, self)

        if report_format == "TABULAR":
            return get_tabular_reports(*args, **kwargs)
        elif report_format == "SUMMARY":
            return get_summary_reports(*args, **kwargs)
        else:
            return get_matrix_reports(*args, **kwargs)

    def get_total(self, report_id):
        """Get a report grand total."""
        url = urllib.parse.urljoin(self.url, report_id)
        report = self.session.get(url, params={"includeDetails": "false"}).json()

        return parsers.get_report_total(report)

    @functools.lru_cache(maxsize=8)
    def get_metadata(self, report_id):
        """Get a report metadata, used to manipulate reports."""
        url = urllib.parse.urljoin(self.url, report_id + "/describe")

        return self.session.get(url).json()


import warnings
from collections import MutableMapping

import numpy as np
import pandas as pd

from autocnet.utils import utils
from autocnet.matcher import health
from autocnet.matcher import outlier_detector as od
from autocnet.matcher import suppression_funcs as spf
from autocnet.matcher import subpixel as sp
from autocnet.matcher.feature import FlannMatcher
from autocnet.transformation.transformations import FundamentalMatrix, Homography
from autocnet.vis.graph_view import plot_edge
from autocnet.cg import cg

class Edge(dict, MutableMapping):
    """
    Attributes
    ----------
    source : hashable
             The source node

    destination : hashable
                  The destination node
    masks : set
            A list of the available masking arrays

    provenance : dict
                 With key equal to an autoincrementing integer and value
                 equal to a dict of parameters used to generate this
                 realization.

    weight : dict
             Dictionary with two keys overlap_area, and overlap_percn
             overlap_area returns the area overlaped by both images
             overlap_percn retuns the total percentage of overlap
    """

    def __init__(self, source=None, destination=None):
        self.source = source
        self.destination = destination

        self.homography = None
        self.fundamental_matrix = None
        self.matches = None
        self._subpixel_offsets = None

        self.weight = {}

        self._observers = set()

        # Subscribe the heatlh observer
        self._health = health.EdgeHealth()

    def __repr__(self):
        return """
        Source Image Index: {}
        Destination Image Index: {}
        Available Masks: {}
        """.format(self.source, self.destination, self.masks)

    @property
    def masks(self):
        mask_lookup = {'fundamental': 'fundamental_matrix',
                       'ratio': 'distance_ratio'}
        if not hasattr(self, '_masks'):
            if self.matches is not None:
                self._masks = pd.DataFrame(True, columns=['symmetry'],
                                           index=self.matches.index)
            else:
                self._masks = pd.DataFrame()
        # If the mask is coming form another object that tracks
        # state, dynamically draw the mask from the object.
        for c in self._masks.columns:
            if c in mask_lookup:
                truncated_mask = getattr(self, mask_lookup[c]).mask
                self._masks[c] = False
                self._masks[c].iloc[truncated_mask.index] = truncated_mask
        return self._masks

    @masks.setter
    def masks(self, v):
        column_name = v[0]
        boolean_mask = v[1]
        self.masks[column_name] = boolean_mask

    @property
    def health(self):
        return self._health.health

    def match(self, k=2):
        """
        Given two sets of descriptors, utilize a FLANN (Approximate Nearest
        Neighbor KDTree) matcher to find the k nearest matches.  Nearness is
        the euclidean distance between descriptors.

        The matches are then added as an attribute to the edge object.
        Parameters
        ----------
        k : int
            The number of neighbors to find

        Returns
        -------

        """
        def mono_matches(a, b):
            fl.add(a.descriptors, a.node_id)
            fl.train()
            self._add_matches(fl.query(b.descriptors, b.node_id, k))
            fl.clear()

        fl = FlannMatcher()
        mono_matches(self.source, self.destination)
        mono_matches(self.destination, self.source)

    def _add_matches(self, matches):
        """
        Given a dataframe of matches, either append to an existing
        matches edge attribute or initially populate said attribute.

        Parameters
        ----------
        matches : dataframe
                  A dataframe of matches
        """
        if self.matches is None:
            self.matches = matches
        else:
            df = self.matches
            self.matches = df.append(matches,
                                     ignore_index=True,
                                     verify_integrity=True)

    def symmetry_check(self):
        if hasattr(self, 'matches'):
            mask = od.mirroring_test(self.matches)
            self.masks = ('symmetry', mask)
        else:
            raise AttributeError('No matches have been computed for this edge.')

    def ratio_check(self, clean_keys=[], **kwargs):
        if hasattr(self, 'matches'):

            matches, mask = self._clean(clean_keys)

            self.distance_ratio = od.DistanceRatio(matches)
            self.distance_ratio.compute(mask=mask, **kwargs)

            # Setup to be notified
            self.distance_ratio._notify_subscribers(self.distance_ratio)

            self.masks = ('ratio', self.distance_ratio.mask)
        else:
            raise AttributeError('No matches have been computed for this edge.')

    def compute_fundamental_matrix(self, clean_keys=[], **kwargs):
        """
        Estimate the fundamental matrix (F) using the correspondences tagged to this
        edge.

        Parameters
        ----------
        clean_keys : list
                     Of strings used to apply masks to omit correspondences

        method : {linear, nonlinear}
                 Method to use to compute F.  Linear is significantly faster at
                 the cost of reduced accuracy.

        See Also
        --------
        autocnet.transformation.transformations.FundamentalMatrix
       :
        """
        if not hasattr(self, 'matches'):
            raise AttributeError('Matches have not been computed for this edge')
            return
        matches, mask = self._clean(clean_keys)

        # TODO: Homogeneous is horribly inefficient here, use Numpy array notation
        s_keypoints = self.source.get_keypoint_coordinates(index=matches['source_idx'],
                                                                 homogeneous=True)
        d_keypoints = self.destination.get_keypoint_coordinates(index=matches['destination_idx'],
                                                                homogeneous=True)

        # Replace the index with the matches index.
        s_keypoints.index = matches.index
        d_keypoints.index = matches.index

        self.fundamental_matrix = FundamentalMatrix(np.zeros((3,3)), index=matches.index)
        self.fundamental_matrix.compute(s_keypoints, d_keypoints, **kwargs)

        # Convert the truncated RANSAC mask back into a full length mask
        mask[mask] = self.fundamental_matrix.mask

        # Subscribe the health watcher to the fundamental matrix observable
        self.fundamental_matrix.subscribe(self._health.update)
        self.fundamental_matrix._notify_subscribers(self.fundamental_matrix)

        # Set the initial state of the fundamental mask in the masks
        self.masks = ('fundamental', mask)

    def compute_homography(self, method='ransac', clean_keys=[], pid=None, **kwargs):
        """
        For each edge in the (sub) graph, compute the homography
        Parameters
        ----------
        outlier_algorithm : object
                            An openCV outlier detections algorithm, e.g. cv2.RANSAC

        clean_keys : list
                     of string keys to masking arrays
                     (created by calling outlier detection)
        Returns
        -------
        transformation_matrix : ndarray
                                The 3x3 transformation matrix

        mask : ndarray
               Boolean array of the outliers
        """

        if hasattr(self, 'matches'):
            matches = self.matches
        else:
            raise AttributeError('Matches have not been computed for this edge')

        matches, mask = self._clean(clean_keys)

        s_keypoints = self.source.get_keypoint_coordinates(index=matches['source_idx'])
        d_keypoints = self.destination.get_keypoint_coordinates(index=matches['destination_idx'])

        self.homography = Homography(np.zeros((3,3)), index=self.masks.index)
        self.homography.compute(s_keypoints.values,
                                d_keypoints.values)

        # Convert the truncated RANSAC mask back into a full length mask
        mask[mask] = self.homography.mask
        self.masks = ('ransac', mask)

        # Finalize the array to get custom attrs to propagate
        self.homography.__array_finalize__(self.homography)

    def subpixel_register(self, clean_keys=[], threshold=0.8,
                          template_size=19, search_size=53, max_x_shift=1.0,
                          max_y_shift=1.0, tiled=False, **kwargs):
        """
        For the entire graph, compute the subpixel offsets using pattern-matching and add the result
        as an attribute to each edge of the graph.

        Parameters
        ----------
        clean_keys : list
             of string keys to masking arrays
             (created by calling outlier detection)

        threshold : float
                    On the range [-1, 1].  Values less than or equal to
                    this threshold are masked and can be considered
                    outliers

        upsampling : int
                     The multiplier to the template and search shapes to upsample
                     for subpixel accuracy

        template_size : int
                        The size of the template in pixels, must be odd

        search_size : int
                      The size of the search

        max_x_shift : float
                      The maximum (positive) value that a pixel can shift in the x direction
                      without being considered an outlier

        max_y_shift : float
                      The maximum (positive) value that a pixel can shift in the y direction
                      without being considered an outlier
        """
        matches = self.matches
        for column, default in {'x_offset': 0, 'y_offset': 0, 'correlation': 0, 'reference': -1}.items():
            if column not in self.matches.columns:
                self.matches[column] = default

        # Build up a composite mask from all of the user specified masks
        matches, mask = self._clean(clean_keys)

        # Grab the full images, or handles
        if tiled is True:
            s_img = self.source.geodata
            d_img = self.destination.geodata
        else:
            s_img = self.source.geodata.read_array()
            d_img = self.destination.geodata.read_array()

        source_image = (matches.iloc[0]['source_image'])

        # for each edge, calculate this for each keypoint pair
        for i, (idx, row) in enumerate(matches.iterrows()):
            s_idx = int(row['source_idx'])
            d_idx = int(row['destination_idx'])

            s_keypoint = self.source.get_keypoint_coordinates(s_idx)
            d_keypoint = self.destination.get_keypoint_coordinates(d_idx)

            # Get the template and search window
            s_template = sp.clip_roi(s_img, s_keypoint, template_size)
            d_search = sp.clip_roi(d_img, d_keypoint, search_size)
            try:
                x_offset, y_offset, strength = sp.subpixel_offset(s_template, d_search, **kwargs)
                self.matches.loc[idx, ('x_offset', 'y_offset',
                                       'correlation', 'reference')] = [x_offset, y_offset, strength, source_image]
            except:
                warnings.warn('Template-Search size mismatch, failing for this correspondence point.')
                continue

        # Compute the mask for correlations less than the threshold
        threshold_mask = self.matches['correlation'] >= threshold

        # Compute the mask for the point shifts that are too large
        query_string = 'x_offset <= -{0} or x_offset >= {0} or y_offset <= -{1} or y_offset >= {1}'.format(max_x_shift,
                                                                                max_y_shift)
        sp_shift_outliers = self.matches.query(query_string)
        shift_mask = pd.Series(True, index=self.matches.index)
        shift_mask.loc[sp_shift_outliers.index] = False

        # Generate the composite mask and write the masks to the mask data structure
        mask = threshold_mask & shift_mask
        self.masks = ('shift', shift_mask)
        self.masks = ('threshold', threshold_mask)
        self.masks = ('subpixel', mask)

    def suppress(self, suppression_func=spf.correlation, clean_keys=[], **kwargs):
        """
        Apply a disc based suppression algorithm to get a good spatial
        distribution of high quality points, where the user defines some
        function to be used as the quality metric.

        Parameters
        ----------
        suppression_func : object
                           A function that returns a scalar value to be used
                           as the strength of a given row in the matches data
                           frame.

        suppression_args : tuple
                           Arguments to be passed on to the suppression function

        clean_keys : list
                     of mask keys to be used to reduce the total size
                     of the matches dataframe.
        """
        if not hasattr(self, 'matches'):
            raise AttributeError('This edge does not yet have any matches computed.')

        matches, mask = self._clean(clean_keys)
        domain = self.source.geodata.raster_size

        # Massage the dataframe into the correct structure
        coords = self.source.get_keypoint_coordinates()
        merged = matches.merge(coords, left_on=['source_idx'], right_index=True)
        merged['strength'] = merged.apply(suppression_func, axis=1, args=([self]))

        if not hasattr(self, 'suppression'):
            # Instantiate the suppression object and suppress matches
            self.suppression = od.SpatialSuppression(merged, domain, **kwargs)
            self.suppression.suppress()
        else:
            for k, v in kwargs.items():
                if hasattr(self.suppression, k):
                    setattr(self.suppression, k, v)
            self.suppression.suppress()

        mask[mask] = self.suppression.mask
        self.masks = ('suppression', mask)

    def plot(self, ax=None, clean_keys=[], **kwargs):
        return plot_edge(self, ax=ax, clean_keys=clean_keys, **kwargs)

    def _clean(self, clean_keys, pid=None):
        """
        Given a list of clean keys and a provenance id compute the
        mask of valid matches

        Parameters
        ----------
        clean_keys : list
                     of columns names (clean keys)
        pid : int
              The provenance id of the parameter set to be cleaned.
              Defaults to the last run.

        Returns
        -------
        matches : dataframe
                  A masked view of the matches dataframe

        mask : series
               A boolean series to inflate back to the full match set
        """
        if clean_keys:
            mask = self.masks[clean_keys].all(axis=1)
        else:
            mask = pd.Series(True, self.matches.index)

        return self.matches[mask], mask

    def overlap(self):
        """
        Acts on an edge and returns the overlap area and percentage of overlap
        between the two images on the edge. Data is returned to the
        weight dictionary
        """
        poly1 = self.source.geodata.footprint
        poly2 = self.destination.geodata.footprint

        overlapinfo = cg.two_poly_overlap(poly1, poly2)

        self.weight['overlap_area'] = overlapinfo[1]
        self.weight['overlap_percn'] = overlapinfo[0]

    def coverage(self, clean_keys = []):
        """
        Acts on the edge given either the source node
        or the destination node and returns the percentage
        of overlap covered by the keypoints. Data for the
        overlap is gathered from the source node of the edge
        resulting in a maximum area difference of 2% when compared
        to the destination.

        Returns
        -------
        total_overlap_percentage : float
                                   returns the overlap area
                                   covered by the keypoints
        """
        if self.matches is None:
            raise AttributeError('Edge needs to have features extracted and matched')
            return
        matches, mask = self._clean(clean_keys)
        source_array = self.source.get_keypoint_coordinates(index=matches['source_idx']).values

        source_coords = self.source.geodata.latlon_corners
        destination_coords = self.destination.geodata.latlon_corners

        convex_hull = cg.convex_hull(source_array)

        convex_points = [self.source.geodata.pixel_to_latlon(row[0], row[1]) for row in convex_hull.points[convex_hull.vertices]]
        convex_coords = [(x, y) for x, y in convex_points]

        source_poly = utils.array_to_poly(source_coords)
        destination_poly = utils.array_to_poly(destination_coords)
        convex_poly = utils.array_to_poly(convex_coords)

        intersection_area = cg.get_area(source_poly, destination_poly)

        total_overlap_coverage = (convex_poly.GetArea()/intersection_area)

        return total_overlap_coverage


from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
from django.shortcuts import render,get_object_or_404
from .models import Course, Step, Category
import datetime
# Create your views here.

def category_list(request, category):
    courses = Course.objects.order_by('-pub_date').filter(category = category_id )
    paginator = Paginator(courses, 9)
    page = request.GET.get('page')
    paged_courses = paginator.get_page(page)
    category = Category.objects.all()
    context = {
       'courses':courses,
       'courses':paged_courses,
       'category':category
    }
    return render(request,'courses/course_list.html',context)

def course_list(request):
    courses = Course.objects.order_by('-pub_date').filter(is_published=True)
    paginator = Paginator(courses, 9)
    page = request.GET.get('page')
    paged_courses = paginator.get_page(page)
    category = Category.objects.all()
    context = {
       'courses':courses,
       'courses':paged_courses,
       'category':category
    }
    return render(request,'courses/course_list.html',context)

def  detail_view(request, course_id):
    course = get_object_or_404(Course,pk=course_id)
    context = {
        'course':course
    }
    return render(request, 'courses/course_detail.html',context)

def step_detail(request,course_pk, step_pk):
    step = get_object_or_404(Step,Course_id=course_pk,pk=step_pk)
    context ={
        'step':step
    }
    return render(request, 'courses/step_detail.html',context)

def search(request):
    context = {

    }
    return render(request, 'courses/search.html',context)


import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

dtst = pd.read_csv(r'vem amor bate n para com o DB na minha cara - Form Responses 1.csv')
dtst1 = dtst.iloc[:, :15]
# ---------------------
dtst1.iloc[:5, 2:9]
pers_data = pd.crosstab([dtst1["Sexo"], dtst1["Idade"]],
                        [dtst1["Orientação sexual"], dtst1["Cor pele"]])
sns.heatmap(pers_data, fmt="d", annot=True)
# ---------------------
sns.countplot(x='Cor pele', data=dtst1)
plt.show()

dtst1.loc[:, ["Sexo", "Idade", "Orientação sexual", "Cor pele"]].plot()
# ---------------------
test1 = pd.crosstab(dtst1['Sexo'], dtst1[("Quantidade que jogou "
                                          "League of Legends no último mês?")])

test2 = pd.crosstab(dtst1['Sexo'], dtst1[("Percebeu alguma opressão ou discriminação em "
                                          "qualquer tela neste último mês?")])
# ---------------------
tips = sns.load_dataset("tips")
tips.head()
# ---------------------
sns.heatmap(test1, fmt="d", annot=True)
plt.xticks(rotation=30)
plt.show()
sns.heatmap(test2, fmt="d", annot=True)
plt.show()


'''
https://report.amap.com/ajax/roadRank.do?roadType=0&timeType=0&cityCode=110000
'''
import requests
import re
import random
from lxml import etree
import xlsxwriter
# 创建文件，添加一个工作表
def header():
    agent1 = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"
    agent2 = "Mozilla/5.0 (iPad; CPU OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3"
    agent3 = "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0"
    agent4 = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727"
    list = [agent1, agent2, agent3, agent4]
    agent = random.choice(list)
    header = {"User-Agent": agent}
    return header
if __name__ == '__main__':
    header = header()
    url = "https://report.amap.com/ajax/roadRank.do?roadType=0&timeType=0&cityCode=110000"
    response = requests.get(url,headers = header).content.decode()
    pat = re.compile(r',"delayTime":(.*?),"')
    pat1 = re.compile(r',"dir":"(.*?)","')
    pat2 = re.compile(r',"id":(.*?),"')
    pat3 = re.compile(r',"index":(.*?),"')
    pat4 = re.compile(r'"length":(.*?),"')
    pat5= re.compile(r',"name":"(.*?)",')
    pat6= re.compile(r',"number":(.*?),"')
    pat7 = re.compile(r',"speed":(.*?),"')
    pat8 = re.compile(r',"travelTime":(.*?)}')
    print('延迟时间',pat.findall(response))
    print('方向',pat1.findall(response))
    print('道路号码',pat2.findall(response))
    print('拥堵延时指数',pat3.findall(response))
    print('路长',pat4.findall(response))
    print('路名',pat5.findall(response))
    print('路编号',pat6.findall(response))
    print('速度',pat7.findall(response))
    print('运行时间',pat8.findall(response))
    workbook = xlsxwriter.Workbook('demo.xlsx')
    worksheet = workbook.add_worksheet()
    for i in range(len(pat.findall(response))):
        worksheet.write("A"+str(i+1),pat.findall(response)[i])
    workbook.close()

python网络爬虫/个人做的爬虫练习/爬车速.py


# encoding: utf-8
import csv
import numpy as np
import os
import pandas as pd
from matplotlib import pylab
import matplotlib.pyplot as plt
from matplotlib import font_manager
from mpl_toolkits.mplot3d import Axes3D
from sklearn.cluster import KMeans,MeanShift,estimate_bandwidth
from sklearn.ensemble import RandomForestClassifier,RandomForestRegressor
from sklearn.neural_network import MLPClassifier, MLPRegressor
from sklearn.metrics import precision_score, recall_score, classification_report, confusion_matrix,r2_score,euclidean_distances
from sklearn.metrics.pairwise import pairwise_distances
from sklearn import cross_validation
from sklearn.linear_model import LogisticRegression, LinearRegression
from sklearn.grid_search import GridSearchCV
from sklearn.preprocessing import scale
from sklearn import mixture
from x_means import XMeans
import multiprocessing
import functools
import funcs
import datetime
from sklearn.externals import joblib

##listにuidが存在するか確認
def find_userid(datalist,uid):
    for data in datalist:
        if uid == data[0]:
            return True
    return False

##入力データと入力ラベルに基づいてMSに最適なパラメータを探索
def tune_MS(data):
    mss = []
    range_max = 20
    bandwidth_range = 0.35 + (np.array(range(0, range_max+1))-(range_max/2))*0.001
    for bandwidth in bandwidth_range:
        ms = MeanShift(bandwidth=bandwidth, n_jobs=-2)
        ms.fit(data)
        mss.append(ms)

    metrics=['cityblock', 'cosine', 'euclidean', 'l1', 'l2', 'manhattan']
    distance_max = 0
    retms = mss[0]
    for metric in metrics:
        print(metric)
        print("-------------")
        for ms in mss:
            distance = pairwise_distances(ms.cluster_centers_,metric=metric,n_jobs=-2)
            print(np.mean(distance,axis=0))
            distance = np.mean(distance)
            if distance > distance_max:
                distance_max = distance
                retms = ms
        print("-------------")
    return retms

##入力データと入力ラベルに基づいてGMMに最適なパラメータを探索
def tune_GMM(data):
    lowest_bic = np.infty
    bic = []
    n_components_range = range(1, 7)
    cv_types = ['spherical', 'tied', 'diag', 'full']
    for cv_type in cv_types:
        for n_components in n_components_range:
            # Fit a mixture of Gaussians with EM
            gmm = mixture.GMM(n_components=n_components, covariance_type=cv_type, random_state=1)
            gmm.fit(data)
            bic.append(gmm.bic(data))
            if bic[-1] < lowest_bic:
                lowest_bic = bic[-1]
                best_gmm = gmm
    return best_gmm

##各列毎の平均値をとる
def get_average(tup):
    uid = tup[0]
    data = tup[1]
    speed_array = np.array(data['data']).astype(np.float)
##    nan要素をマスキングで計算に現れずにする
    masked_array = np.ma.masked_array(speed_array,np.isnan(speed_array))
    speed = np.mean(masked_array,axis=0)
    ##        np.nansumでnanを無視した合計を取ってくれる便利なヤツあり
##    maskした後mean取ると--になっているところを無視して平均取ってくれる
    return uid, speed,len(data['climbCategory'])

##keyに基づいてdicからデータを取得
##keyのなかでデータがないものがあれば省く
def make_onedata(dic,keys):
    data = []
    for key in keys:
        d = dic[key]
##        データが存在しない場合無視するためにNoneを代入
        if d == '-1':
            d = None
        data.append(d)
    return data,dic['userid'],dic['climbCategory']

##ユーザデータにラベルを付与
##現在は欠損値があるデータを削除している
def add_labels(tup,uids,labels):
    uid = tup[0]
    data = tup[1]

    if uid not in uids:
        return None
    data = np.array(data['data']).astype(np.float)
##    欠損値を削除
    data = data[~np.isnan(data).any(axis=1)]
    if len(data) == 0:
        print(uid)
        return None
    label = np.array([labels[uids.index(uid)]]*len(data))
    return np.c_[data,label]

##climbCategory毎の回数を記録
##pythonは引数で与えられた変数を直接弄る？
##dictだからメモリが引数として与えられている？
def update_climbCategory(dic,climbCategory):
    if climbCategory in dic.keys():
        dic[climbCategory] += 1
    else:
        dic[climbCategory] = 1
    return dic

##クラスタごとに色分けしてプロット
def plot_data_3D(data,classes):
    fig = plt.figure()
    ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134)
    ax.scatter(data[:,0], data[:,1], data[:,2], c=classes.astype(np.float))
    ax.w_xaxis.set_ticklabels([])
    ax.w_yaxis.set_ticklabels([])
    ax.w_zaxis.set_ticklabels([])
    ax.set_xlabel('Negative Grade')
    ax.set_ylabel('No Grade')
    ax.set_zlabel('Positive Grade')
    plt.show()

##クラスタごとに色分けしてプロット
def plot_data_2D(data,classes,components_label):
    fig = plt.figure()
    data_len = len(components_label)
    for x in range(data_len):
        for y in range(data_len):
            ax = fig.add_subplot(data_len,data_len,1+y+x*data_len)
            if x == y:
                ax.text(0,0,components_label[x])
            else:
                ax.scatter(data[:,x], data[:,y], c=classes.astype(np.float))
    plt.show()

##雑に時間を定義、季節によって変える必要あり
def get_hour(hour):
    key = ""
    if hour >= 6 and hour <= 9:
        key = "morning"
    elif hour >=10 and hour <= 18 :
        key = "noon"
    else:
        key = "night"

    return key

##季節を定義
def get_month(month):
    key = ""
    if month >= 3 and month <= 5:
        key = "spring"
    elif month >=6 and month <= 8 :
        key = "summer"
    elif month >=9 and month <= 11 :
        key = "autumn"
    else:
        key = "winter"

    return key

##セグメントのデータをファイルから作成
def makesegmentdata_from_files(path_labeled):
    keys = ['startDate','climbCategory','DISTANCE']
    ret_list = []
    for label in ['labeled','notlabeled']:
        files = os.listdir(os.path.join(path_labeled,label))
        for file in files:
            data = funcs.read_myjson(os.path.join(path_labeled,label,file),None,None)
            for d in data:
                temp_list = []
                for key in keys:
                    if key == 'DISTANCE':
                        temp_list.append(d[key][0])
                        temp_list.append(d[key][-1] - d[key][0])
                    elif key == 'startDate':
                        tdatetime = datetime.datetime.strptime(d[key], '%Y-%m-%dT%H:%M:%SZ')
                        temp_list.append(get_month(tdatetime.month))
                        temp_list.append(get_hour(tdatetime.hour))
                        temp_list.append(d[key])
                    else:
                        temp_list.append(d[key])
                ret_list.append(temp_list)
    try:
        f = funcs.get_fileobj('middata.csv','w',None)
        writer = csv.writer(f, lineterminator='\n')
        keys = ['month','hour','climbCategory','DISTANCE','consumed_dist']
        writer.writerow(keys)
        writer.writerows(ret_list)
    finally:
        f.close()

##--------------プログラム開始------------------
##マルチスレッド処理の為にこれが必要 http://matsulib.hatenablog.jp/entry/2014/06/19/001050
if __name__ == '__main__':
    funcs.start_program()
    filepath = funcs.get_filepath('middata.csv',None)
    if os.path.exists(filepath) is False:
        print("DATA FROM FILE!!!!!!!!!!!!!!!!!!!!!!!!!!!")
        data = makesegmentdata_from_files('results\\data_for_regression\\20160203_011146')
        print("RESTART")
        exit(-1)
    else:
        df = pd.read_csv(filepath)
        data = np.array(df).astype(np.float)

##    クラスタリング開始
    uids = data[:,0]
    data = data[:,2:5]
    scaled_data = scale(data)
##kmeansでクラスタリング
    kmeans_model = KMeans(n_clusters=9, random_state=1)
    kmeans_model.fit(scale(scaled_data))
    km_labels = np.array(kmeans_model.labels_)
##        plot_data_2D(data,labels,components_label)

##GMMでクラスタリング
##        GMMのチューニング
    gmm = tune_GMM(scaled_data)
    joblib.dump(gmm, funcs.get_filepath('gmmpred.pkl',gmmmodel))
    gm_labels = gmm.predict(scaled_data)
    plot_data_3D(data,gm_labels)

##MeanShiftでクラスタリング
##    ms_model = tune_MS(scaled_data)
    ms_model = MeanShift(bandwidth=0.35299999999999998)
    ms_model.fit(scaled_data)
    ms_labels = ms_model.labels_

    try:
        file = funcs.get_fileobj('labels.csv','w',None)
        writer = csv.writer(file, lineterminator='\n')
        writer.writerow(['uid','9-means','GMM','MeanShift'])
        writer.writerows(np.c_[uids,km_labels,gm_labels,ms_labels])
    finally:
        file.close()
##        kmeansの重心のプロット
##        centers = kmeans_model.cluster_centers_
##        plot_data_2D(centers,np.array(range(cluster_num)),components_label)
    funcs.end_program()


"""
Randomized Search Example.
"""

from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import RandomizedSearchCV

# Loading data
iris = load_iris()
X = iris.data[:,2:]
y = iris.target

# Splitting data in training and test set
X_train, X_test, y_train, y_test = train_test_split(X, y, stratify = y,
                                                    random_state = 7)

# Instantiate nearest neighbors classifier
knn_clf = KNeighborsClassifier()

# Prepare parameter distribution for randomized grid search
param_dist = {'n_neighbors': list(range(3,50,1))}

# Instantiate a randomized search using 10-fold cross validation
rs = RandomizedSearchCV(knn_clf, param_dist, cv=10, n_iter=15)

# Fit the randomized grid search estimator
rs.fit(X_train, y_train)

# Print the results
print("Best parameters:", rs.best_params_)
print("\nResults overview:")
for result in zip(rs.cv_results_['params'], rs.cv_results_['mean_test_score']):
    print(result)


import datetime
from bnw.core.base import get_webui_base

formatters = {
    'comment': None,
    'message': None,
    'recommendation': None,
    'message_with_replies': None,
    'messages': None,
}

def format_message(request, msg):
    result = '+++ [%s] %s:\n' % (
        datetime.datetime.utcfromtimestamp(
            msg['date']).strftime('%d.%m.%Y %H:%M:%S'),
        msg['user'],)
    if msg['tags']:
        result += 'Tags: %s\n' % (', '.join(msg['tags']),)
    if msg['clubs']:
        result += 'Clubs: %s\n' % (', '.join(msg['clubs']),)
    result += '\n%s\n' % (msg['text'],)
    result += '--- %(id)s (%(rc)d) %(base_url)s/p/%(id)s' % {
        'base_url': get_webui_base(request.user),
        'id': msg['id'].upper(),
        'rc': msg['replycount'],
    }
    return result

def format_comment(request, msg, short=False):
    args = {'id': msg['id'].split('/')[1].upper(),
            'author': msg['user'],
            'message': msg['message'].upper(),
            'replyto': None if msg['replyto'] is None else msg['replyto'].upper(),
            'replytotext': msg['replytotext'],
            'text': msg['text'],
            'num': msg.get('num', -1),
            'date': datetime.datetime.utcfromtimestamp(msg['date']).strftime('%d.%m.%Y %H:%M:%S'),
            'web': get_webui_base(request.user),
            }
    formatstring = '+++ [ %(date)s ] %(author)s'
    if msg['replyto']:
        formatstring += ' (in reply to %(replyto)s):\n'
    else:
        formatstring += ' (in reply to %(message)s):\n'
    if not short:
        formatstring += '>%(replytotext)s\n'
    formatstring += '\n%(text)s\n--- %(message)s/%(id)s (%(num)d) %(web)s/p/%(message)s#%(id)s'
    return formatstring % args

def formatter_messages(request, result):
    return 'Search results:\n' + '\n'.join((format_message(request, msg) for msg in result['messages']))

def formatter_message_with_replies(request, result):
    return format_message(request, result['message']) + '\n' + \
        '\n'.join(
        (format_comment(request, c, True) for c in result['replies']))

def formatter_search(request, search_result):
    estimated = search_result['estimated']
    results = search_result['results']
    results.reverse()  # Most relevant items should be at the bottom
    if not results:
        return 'No results found.'
    info = 'Found %d results' % estimated
    if estimated > 20:
        info += ' (displaying first 20)'
    out = [info + ':']
    for res in results:
        out.append('@%s: %s (%s%%)\n%s\n\n#%s %s/p/%s' % (
            res['user'], res['tags_info'], res['percent'], res['text'],
            res['id'], get_webui_base(request.user),
            res['id'].replace('/', '#')))
    return '\n\n'.join(out)

def formatter_subscriptions(request, result):
    return 'Your subscriptions:\n' + '\n'.join(
        (s['type'][4:].ljust(5) + s['target'].rjust(10) + ' ' + s.get('from', '')
         for s in result['subscriptions']))

def formatter_blacklist(request, result):
    return 'Your blacklist:\n' + '\n'.join(
        (s[0].ljust(5) + s[1].rjust(10)
         for s in result['blacklist']))

def formatter_message(request, result):
    return '\n' + format_message(request, result['message'])

def formatter_recommendation(request, result):
    return '\nRecommended by @%s: %s\n' % (result['recommender'], result['recocomment']) + \
        format_message(request, result['message'])

def formatter_comment(request, result):
    return '\n' + format_comment(request, result['comment'])

def formatter_userlist(request, result):
    if not result['users']:
        return 'No more users.'
    return ' ' + ' '.join(
        '@' + u['name'].ljust(10) + ('\n' if i % 5 == 4 else '')
        for i, u in enumerate(result['users'])) + '\nuserlist -p ' + str(result['page'] + 1) + ' -- next page.'

def formatter_settings(request, result):
    return ('Current settings:\n' +
            '\n'.join('%s\t%s' % (k, v) for k, v in result['settings'].iteritems()))

def formatter_clubs(request, result):
    return ' ' + ' '.join(
        '' + u['_id'].ljust(
            15) + ' ' + str(int(u['value'])).ljust(3) + ('\n' if i % 5 == 4 else '')
        for i, u in enumerate(result['clubs']))

def formatter_jids(request, result):
    res = 'JIDs:\n' + ' '.join(u for u in result['jids'])
    if result['pending_jids']:
        res += '\n\nPending JIDs:\n' + ' '.join(u for u in result[
                                                'pending_jids'])
    res += '\n\nActive JID: ' + result['jid']
    return res


from keras.models import load_model
from src.TokenSelector import TokenSelector
from src.Tokenization import Tokenization

import numpy as np
import os
import json

class CheckModel():

    def __init__(self, model_path, tokenization_path, tokenSelector_path):

        raisePathNotExists(model_path)
        raisePathNotExists(tokenization_path)
        raisePathNotExists(tokenSelector_path)

        with open(tokenization_path) as f:
            self.tokenization_params = json.load(f)

        self.model = load_model(model_path)

        with open(tokenSelector_path) as f:
            self.tokenSelector_params = json.load(f)

        self.tokenSelector = TokenSelector(final_char= self.tokenSelector_params["final_char"],
                                           inter_char= self.tokenSelector_params["inter_char"],
                                           signs_to_ignore= self.tokenSelector_params["signs_to_ignore"],
                                           words_to_ignore= self.tokenSelector_params["words_to_ignore"],
                                           map_punctuation= self.tokenSelector_params["map_punctuation"],
                                           letters= "".join(self.tokenSelector_params["letters"]),
                                           sign_not_syllable= self.tokenSelector_params["sign_not_syllable"]
                                           )

        self.tokenSelector.set_params(self.tokenSelector_params)

        self.map_punctuation_inv = dict()
        for key, value in self.tokenSelector_params["map_punctuation"]:
            self.map_punctuation_inv[value] = key

    def predict_word(self, sentence, temperature):

        next_token = ""
        generated = []
        while next_token not in self.tokenSelector_params["index_ends"]:
            x_pred = np.zeros((1, self.tokenization_params["lprime"]))
            for t, token in enumerate(sentence):
                x_pred[0, t] = self.tokenization_params["token_to_index"][token]
            preds = self.model.predict(x_pred, verbose=0)[0]
            next_index = sample(preds, temperature)
            next_token = self.tokenization_params["index_to_token"][str(next_index + 1)]
            generated += [next_token]
            sentence = sentence[1:] + [next_token]

        word = ""
        for token in generated:
            if token in self.map_punctuation_inv:
                word += self.map_punctuation_inv[token]+ " "
            else:
                token = token.replace(self.tokenSelector_params["final_char"], " ")
                token = token.replace(self.tokenSelector_params["inter_char"], "")
                word += token

        return word, generated

    def predict_text(self,
                     seed = "declaro",
                     nwords = 10,
                     temperature = 1.0):

        seed_array = seed.lower().split()

        sentence = []
        text = ""

        for word in seed_array:
            sentence = self.tokenSelector.select(word, sentence)

        for i in range(nwords):
            word, generated = self.predict_word(sentence, temperature)
            text += word
            sentence += generated

        return text

    def get_probability_token(self, token, sentence):
        x_pred = np.zeros((1, self.tokenization_params["lprime"]))
        for t, tk in enumerate(sentence):
            x_pred[0, t] = self.tokenization_params["token_to_index"][tk]
        preds = self.model.predict(x_pred, verbose=0)[0]

        return preds[token]

    def perplexity(self, path_to_test):

        tokenization = Tokenization(path_to_file=None,
                                    final_char=self.tokenSelector_params["final_char"],
                                    final_punc=self.tokenSelector_params["final_punc"],
                                    inter_char=self.tokenSelector_params["inter_char"],
                                    signs_to_ignore=self.tokenSelector_params["signs_to_ignore"],
                                    words_to_ignore=self.tokenSelector_params["words_to_ignore"],
                                    map_punctuation=self.tokenSelector_params["map_punctuation"],
                                    letters=self.tokenSelector_params["letters"],
                                    sign_not_syllable=self.tokenSelector_params["sign_not_syllable"],
                                    )
        tokenization.set_params_experiment(self.tokenization_params)
        max_len = self.tokenization_params["lprime"]
        test_selected = tokenization.select_tokens(path_to_test)

        sentence = []
        token_word = []
        ppl = 0
        qw = 0

        for i,token in enumerate(test_selected):
            token_word += token
            if token[-1] in [self.tokenSelector_params["final_char"], self.tokenSelector_params["final_punc"]]:
                if len(sentence) < 1:
                    sentence += token_word
                    token_word = []
                    continue

                seed = sentence
                probs_word = 1
                ## Calculo la probabilidad de la palabra
                for tok in token_word:
                    probs_word *= self.get_probability_token(tok, seed)
                    seed += [tok]

                ## sumo la probabilidad a la del resto
                ppl += np.log2(probs_word)

                sentence += token_word
                qw += 1
                token_word = []
                if len(sentence) > max_len:
                    sentence = sentence[-max_len:]

        return - ppl / qw

def sample(pred, temperature=1.0):
    pred = np.asarray(pred).astype('float64')
    pred = np.log(pred) / temperature
    exp_pred = np.exp(pred)
    pred = exp_pred / np.sum(exp_pred)
    prob = np.random.multinomial(1, pred, 1)

    return np.argmax(prob)

def raisePathNotExists(path_to_file):
    if not os.path.exists(path= path_to_file):
        raise FileNotFoundError("Path doesn't exists, '{}'".format(path_to_file))


from time import sleep
from os import system
import math
import datetime

def ask_number(txt):
    try:
        global number
        number = input(txt)
        number = int(number)
    except:
        print(f">>>{number}" + " no es un numero\n")
        ask_number(txt)
    return number

def find_age():
    system('cls')
    print("""
************************************************************
PROGRAMA QUE ACEPTA LA FECHA DE NACIMIENTO Y MUESTRE LA EDAD
*************************************************************
\n""")

    date = datetime.datetime.now()
    get_day = date.day
    get_month = date.month
    get_year = date.year

    day = ask_number('Escribe tu dia de naciento: ')
    month = ask_number('Escribe tu mes de naciento: ')
    year = ask_number('Escribe el año de tu naciento: ')

    if year > get_year or day > 31 or month > 12:
        print(f'\nEs imposible que naciera el {day}/{month}/{year}')
        sleep(2)
        return find_age()
    if year == get_year and month > get_month:
        print(f'\nEs imposible que naciera el {day}/{month}/{year}')
        sleep(2)
        return find_age()
    if year == get_year and day > get_day:
        print(f'\nEs imposible que naciera el {day}/{month}/{year}')
        sleep(2)
        return find_age()

    age = get_year - year

    if month == get_month and day <= get_day and age >= 1:
        age = age
        print(f"Tu edad es: {age} años")

    elif month == get_month and day > get_day and age >= 1:
        age = age - 1
        print(f"Tu edad es: {age} años")

    elif month > get_month and age >= 1:
        age = age - 1
        print(f"Tu edad es: {age} años")

    elif month < get_month and age >= 1:
        age = age
        print(f"Tu edad es: {age} años")

    elif month < get_month and age == 0:
        age = get_month - month
        print(f"Tu edad es: {age} Mes/Meses")
    elif month == get_month and day <= get_day and age == 0:
        age = get_day - day
        print(f"Tu edad es: {age} dia/s")

    sleep(1)
    print('\nQUIERES PROBAR DE NUEVOO ? \n 1) SI \n 2) NO')
    option = ask_number('RESPUESTA: ')

    if option == 1:
        find_age()
    else:
        print('\nGRACIAS POR UTILIZARME!')

find_age()


import unidecode
import string
import random
import re

all_characters = string.printable
n_characters = len(all_characters)
limit = 1000000
f = open('speeches.txt', encoding='latin-1').read()
file = unidecode.unidecode(f)
file_len = len(file)
print('file_len =', file_len)

f_s = open('shakespeare_input.txt', encoding='latin-1').read()
file_s = unidecode.unidecode(f_s)
file_len_s = len(file_s)
print('file_len_s =', file_len_s)

# To make inputs out of this big string of data, we will be splitting it into chunks.

chunk_len = 1000

def random_chunk():
    start_index = random.randint(0, file_len - chunk_len)
    end_index = start_index + chunk_len + 1
    return file[start_index:end_index]

def random_chunk_s():
    start_index = random.randint(0, file_len_s - chunk_len)
    end_index = start_index + chunk_len + 1
    return file_s[start_index:end_index]

print(random_chunk())
print(random_chunk_s())

# # Build the Model

import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F

class RNN(nn.Module):
    def __init__(self, input_size=n_characters, hidden_size=128, output_size=n_characters, n_layers=2):
        super(RNN, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.n_layers = n_layers

        self.encoder = nn.Embedding(input_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size, n_layers)
        self.decoder = nn.Linear(hidden_size, output_size)

    def forward(self, input, hidden):
        input = self.encoder(input.view(1, -1))
        output, hidden = self.gru(input.view(1, 1, -1), hidden)
        output = self.decoder(output.view(1, -1))
        #output = F.log_softmax(output, dim=1)
        return output, hidden

    def init_hidden(self):
        return Variable(torch.zeros(self.n_layers, 1, self.hidden_size))

# # Inputs and Targets

# Turn string into list of longs
def char_tensor(string):
    tensor = torch.zeros(len(string)).long()
    for c in range(len(string)):
        tensor[c] = all_characters.index(string[c])
    return Variable(tensor)

print(char_tensor('abcDEF'))

def random_training_set():
    chunk = random_chunk()
    inp = char_tensor(chunk[:-1])
    target = char_tensor(chunk[1:])
    return inp, target

# # Evaluating

def category_from_output(output):
    top_n, top_i = output.data.topk(1) # Tensor out of Variable with .data
    category_i = top_i[0][0]
    return all_characters[category_i], category_i

def evaluate(prime_str='A', predict_len=100, temperature=0.8):
    hidden = decoder.init_hidden()
    prime_input = char_tensor(prime_str)
    predicted = prime_str
    perplexity = 1

    # Use priming string to "build up" hidden state
    for p in range(len(prime_str) - 1):
        _, hidden = decoder(prime_input[p], hidden)
    inp = prime_input[-1]

    for p in range(predict_len):
        output, hidden = decoder(inp, hidden)

        #char_i,top_i = category_from_output(output)
        #print(char_i,top_i)
        #Pick the one with the highest category

        #Sample from the network as a multinomial distribution
        output_dist = output.data.view(-1).div(temperature).exp()
        top_i = torch.multinomial(output_dist, 1)[0]

        # Add predicted character to string and use as next input
        predicted_char = all_characters[top_i]
        predicted += predicted_char
        inp = char_tensor(predicted_char)

    return predicted

def perplexity(input_string):
    hidden = decoder.init_hidden()
    perplexity = 0
    n = len(input_string)

    inp = char_tensor(input_string[0])

    for p in range(n):
        output, hidden = decoder(inp, hidden)

        inp = char_tensor(input_string[p])

        output = F.softmax(output, dim=1)

        output_dist = output.data.view(-1).tolist()

        perplexity += math.log10 ( output_dist [ all_characters.index(input_string[p]) ] )
    perplexity *= -1/n

    return 10**perplexity

# # Training

# A helper to print the amount of time passed:

import time, math

def time_since(since):
    s = time.time() - since
    m = math.floor(s / 60)
    s -= m * 60
    return '%dm %ds' % (m, s)

# The main training function

def train(inp, target):
    hidden = decoder.init_hidden()
    decoder.zero_grad()
    loss = 0

    for c in range(chunk_len):
        output, hidden = decoder(inp[c], hidden)
        loss += criterion(output, target[c].unsqueeze(0))

    loss.backward()
    decoder_optimizer.step()

    return loss.item() / chunk_len

# Then we define the training parameters, instantiate the model, and start training:

n_epochs = 7000
print_every = 100
plot_every = 100
hidden_size = 100
n_layers = 2
lr = 0.005

decoder = RNN(n_characters, hidden_size, n_characters, n_layers)
decoder_optimizer = torch.optim.Adam(decoder.parameters(), lr=lr)
criterion = nn.CrossEntropyLoss()

start = time.time()
all_losses = []
loss_avg = 0

shakespeare_perplexity = []
corpus_preplexity = []

corpus_random_chunk = random_chunk()
shakespeare_random_chunk = random_chunk_s()

#print( ' Trump Speeches Random Text: ',corpus_random_chunk)
#print( ' Shakespeare Random Text: ',shakespeare_random_chunk)
for epoch in range(1, n_epochs + 1):

    loss = train(*random_training_set())
    loss_avg += loss

    if epoch % print_every == 0:
        print('[%s (%d %d%%) %.4f]' % (time_since(start), epoch, epoch / n_epochs * 100, loss))
        print(evaluate('Wh', 100), '\n')
        shakespeare_perplexity.append(perplexity(random_chunk_s()))
        corpus_preplexity.append (perplexity(random_chunk()))

    if epoch % plot_every == 0:
        all_losses.append(loss_avg / plot_every)
        loss_avg = 0

#Save the PyTorch Model

models_generate = decoder
torch.save(models_generate.state_dict(), "/Users/srinivassuri/Documents/cis530/hw6/part3_submission/model_generate")

def evaluate_model(model,prime_str='A', predict_len=100, temperature=0.8):
    hidden = model.init_hidden()
    prime_input = char_tensor(prime_str)
    predicted = prime_str
    perplexity = 1

    # Use priming string to "build up" hidden state
    for p in range(len(prime_str) - 1):
        _, hidden = model(prime_input[p], hidden)
    inp = prime_input[-1]

    for p in range(predict_len):
        output, hidden = model(inp, hidden)

        #Sample from the network as a multinomial distribution
        output_dist = output.data.view(-1).div(temperature).exp()
        top_i = torch.multinomial(output_dist, 1)[0]

        # Add predicted character to string and use as next input
        predicted_char = all_characters[top_i]
        predicted += predicted_char
        inp = char_tensor(predicted_char)

    return predicted

#Load the PyTorch Model back and generate few sentences
model = RNN(n_characters, hidden_size, n_characters, n_layers)
model.load_state_dict(torch.load("/Users/srinivassuri/Documents/cis530/hw6/part3_submission/model_generate"))
model.eval() #To predict

print( evaluate_model(model," ",200,0.5 ) )

#Text generated by the model
print(evaluate('Wh', 100), '\n')
print(evaluate('Wh', 100), '\n')
print(evaluate('Wh', 100), '\n')

#Code from the previous assignment for N-Grams
def start_pad(n):
    ''' Returns a padding string of length n to append to the front of text
        as a pre-processing step to building n-grams '''
    return '~' * n

def ngrams(n, text):
    ''' Returns the ngrams of the text as tuples where the first element is
        the length-n context and the second is the character '''
    m = len(text)

    #print(type(text),len(text),text)

    ngrams_ = []

    if( n == 0 ):
        for i in range(m):
            ngrams_.append( ('',text[i]) )
        return ngrams_

    for i in range(m):
        res = ''
        for j in range(i-n,i,1): #[i-n,i-1]
            if( j < 0 ):
                res += '~'
            else:
                res += text[j]
        ngrams_.append( (res,text[i]) )

    return ngrams_

class NgramModel(object):
    ''' A basic n-gram model using add-k smoothing '''

    def __init__(self, n, k):
        self.vocab = set()
        self.n = n
        self.k = k
        self.counts = {}
        self.contexts ={} #Count of the Contexts
        self.contexts_tokens={} #The actual tokens that occur after the context

    def get_vocab(self):
        ''' Returns the set of characters in the vocab '''
        return self.vocab

    def update(self, text):
        ''' Updates the model n-grams based on text '''
        #print('Updating...',text)
        #Add the Characters to the set if they don't exist
        for ch in text:
            #print('ch is',ch)
            if ch not in self.vocab:
                self.vocab.add(ch)

        #Update the Counts of the ngrams
        ngrams_ = ngrams(self.n, text)

        for (context,char) in ngrams_:
            if (context,char) not in self.counts:
                self.counts[(context,char)] = 1
            else:
                self.counts[(context,char)] += 1

            if context not in self.contexts:
                self.contexts[context] = 1
                self.contexts_tokens[context] = [char]
            else:
                self.contexts[context] += 1
                if char not in self.contexts_tokens[context]:
                    words = self.contexts_tokens[context]
                    words.append(char)
                    self.contexts_tokens[context] = words

    def prob(self, context, char):
        ''' Returns the probability of char appearing after context '''
        # P( char | context ) = P( context, char ) / P( context )
        if context not in self.contexts:
            #print('context not seen',context)
            return 1/(len(self.get_vocab() ))

        if( self.k == 0 ):
            if (context,char) not in self.counts:
                return 0.0

            return self.counts[ (context,char) ] / self.contexts[context]
        else:
            #K Smoothing
            if (context,char) not in self.counts:
                return (self.k)/(self.contexts[context]+(self.k*len(self.get_vocab()) ))

            return (self.counts[ (context,char) ] + self.k) / (self.contexts[context] + (self.k*len(self.get_vocab())) )

    def random_char(self, context):
        ''' Returns a random character based on the given context and the
            n-grams learned by this model '''

        words = sorted(self.get_vocab()) #Sorted Lexicographically

        r = random.random()
        #print(r)
        res = 0

        for word in words:
            #print(word,self.prob( context,word ) )
            if res + self.prob( context,word ) > r:
                return word
            else:
                res += self.prob( context,word )

    def random_text(self, length):
        ''' Returns text of the specified character length based on the
            n-grams learned by this model '''
        #print('Generate text for length',length,self.n,self.k)
        context = start_pad(self.n)

        text = ''

        for i in range(length):
            random_char = self.random_char(context)
            context = context[1:]
            context += random_char
            text += random_char

        return text

    def perplexity(self, text):
        ''' Returns the perplexity of text based on the n-grams learned by
            this model '''
        ngrams_ = ngrams(self.n,text)

        res = 0

        for (context,char) in ngrams_:
            prob = self.prob( context,char )
            if( prob == 0 ):
                return float('inf')
            else:
                res += math.log10( self.prob( context,char ))
        res *= -1/len(text)

        return 10**res

class NgramModelWithInterpolation(NgramModel):
    ''' An n-gram model with interpolation '''

    def __init__(self, n, k):
        self.vocab = set()
        self.n = n
        self.k = k
        self.counts = {}
        self.contexts ={} #Count of the Contexts
        self.contexts_tokens={} #The actual tokens that occur after the context

        self.w = []
        for i in range(self.n + 1): #[n+1] weights
             self.w.append(  1.0/(n+1) )

    def get_vocab(self):
        return self.vocab

    def update(self, text):
        for ch in text:
            if ch not in self.vocab:
                self.vocab.add(ch)

        #Update the Counts of the ngrams
        n = self.n
        while n >=0 :
            ngrams_ = ngrams(n, text)

            for (context,char) in ngrams_:
                if (context,char) not in self.counts:
                    self.counts[(context,char)] = 1
                else:
                    self.counts[(context,char)] += 1

                if context not in self.contexts:
                    self.contexts[context] = 1
                    self.contexts_tokens[context] = [char]
                else:
                    self.contexts[context] += 1
                    if char not in self.contexts_tokens[context]:
                        words = self.contexts_tokens[context]
                        words.append(char)
                        self.contexts_tokens[context] = words
            n = n-1

    def prob(self, context, char):
        res = 0
        for i in range(self.n + 1 ):

            if( i != 0 ):
                context = context[1:]

            if context not in self.contexts:
                res += ( self.w[i]*1/(len(self.get_vocab() )) )
                continue

            if( self.k == 0 ):
                if (context,char) not in self.counts:
                    res += 0.0
                    continue

                res +=  self.w[i] * (self.counts[ (context,char) ] / self.contexts[context] )
            else:
                #K Smoothing
                if (context,char) not in self.counts:
                    res += self.w[i] * (self.k)/(self.contexts[context]+(self.k*len(self.get_vocab()) ))
                    continue

                res += self.w[i] * (self.counts[ (context,char) ] + self.k) / (self.contexts[context] + (self.k*len(self.get_vocab())) )

        return res

def perplexity_model(model,input_string):
    hidden = model.init_hidden()
    perplexity = 0
    n = len(input_string)

    inp = char_tensor(input_string[0])

    for p in range(1,n,1):
        output, hidden = model(inp, hidden)

        inp = char_tensor(input_string[p])

        output = F.softmax(output, dim=1)

        output_dist = output.data.view(-1).tolist()

        perplexity += math.log10 ( output_dist [ all_characters.index(input_string[p]) ] )
    perplexity *= -1/n

    return 10**perplexity

#Create a New N-gramModel
ng = NgramModelWithInterpolation(3,2)

f1_s = open('speeches.txt', encoding='latin-1')
i = 0

for line in f1_s.readlines():
    ng.update(line)
    i +=1

# In[322]:

print('S.no\tngram perplexity\trnn perplexity')
ng_ = []
rnn_ = []

for i in range(20):
    chunk = random_chunk()
    #print(' Chunk is: ', chunk)
    ng_.append(ng.perplexity(chunk))
    rnn_.append(perplexity_model(model,chunk ))
    print(str(i)+'\t'+str(ng_[i])+'\t'+ str(rnn_[i]) )
print('------------------------------------------------------')
print('avg:\t' ,sum(ng_)/20, '\t' , sum(rnn_)/20 )


# This file is part of AltCanvas.
#
# http://code.google.com/p/altcanvas
#
# AltCanvas is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# AltCanvas is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with AltCanvas.  If not, see <http://www.gnu.org/licenses/>.

import gtk
import cairo
from math import pi as PI
import libpub
from libpub.prime.widget import Widget
import libpub.prime.mask as mask
from libpub.prime import utils
from libpub.prime.utils import RGBA

WALLPAPER_PATH = libpub.IMAGE_DIR+'/altpublishr.png'

class Pad(Widget):
    (RECT,ROUNDED_RECT) = range(2)
    (PLAIN,GRAD_EXPLOSION,WALLPAPER) = range(3)

    def __init__(self,w,h,color=RGBA(0,0,0,1),texture=PLAIN,shape=RECT):
        Widget.__init__(self,w,h)
        self.surface = cairo.ImageSurface(cairo.FORMAT_ARGB32,w,h)
        ctx = cairo.Context(self.surface)

        if texture == self.GRAD_EXPLOSION:
            BORDER = int(min(w,h)*0.25)
            border_surface = utils.draw_grad_rect(
                                inner=(BORDER/2,BORDER/2,w-BORDER,h-BORDER),
                                border=BORDER,color=color)

            ctx.set_source_surface(border_surface)
            ctx.paint()

            ctx.set_source_rgba(color.r,color.g,color.b,color.a)
            ctx.rectangle(BORDER/2,BORDER/2,w-BORDER,h-BORDER)
            ctx.fill()

        elif texture == self.PLAIN and shape == self.RECT:
            ctx.set_source_rgba(color.r,color.g,color.b,color.a)
            ctx.rectangle(0,0,w,h)
            ctx.fill()

        elif texture == self.PLAIN and shape == self.ROUNDED_RECT:
            ctx.set_source_rgba(color.r,color.g,color.b,color.a)
            utils.draw_rounded_rect(ctx, 0, 0, w, h)
            '''
            x0 = 0
            y0 = 0
            x1 = x0+w
            y1 = y0+h
            vr = int(min(w,h)/10)

            ctx.set_source_rgba(color.r,color.g,color.b,color.a)

            ctx.move_to(x0+vr,y0)
            ctx.line_to(x1-vr,y0)
            ctx.arc(x1-vr,y0+vr,vr,3*PI/2,0)
            ctx.line_to(x1,y1-vr)
            ctx.arc(x1-vr,y1-vr,vr,0,PI/2)
            ctx.line_to(x0+vr,y1)
            ctx.arc(x0+vr,y1-vr,vr,PI/2,PI)
            ctx.line_to(x0,y0+vr)
            ctx.arc(x0+vr,y0+vr,vr,PI,3*PI/2)

            ctx.fill()
            '''

        elif texture == self.WALLPAPER:
            wpSurface = cairo.ImageSurface.create_from_png(WALLPAPER_PATH)
            ctx.set_source_surface(wpSurface,0,0)
            ctx.paint()