import collections
import random
import re
from math import floor

from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.http import HttpResponse
from django.http.response import JsonResponse
from django.shortcuts import render, get_object_or_404, redirect
from django.urls import reverse

from character.models import Spieler
from .models import *
from .api_views import *

def get_grade_score(correct, max, insg_answered):
    if correct > max or insg_answered == 0 or max == 0:
        return ""

    grade_key = [   # minimal in%, Note
        (95, "A+"),
        (90, "A"),
        (85, "A-"),

        (80, "B+"),
        (75, "B"),
        (70, "B-"),

        (65, "C+"),
        (60, "C"),
        (55, "C-"),

        (50, "D+"),
        (45, "D"),
        (40, "D-"),

        (34, "E+"),
        (28, "E"),
        (20, "E-"),

        (0, "F")
    ]
    percent = round((correct / max) * 100, 2)
    for e in grade_key:
        if percent >= e[0]:
            return e[1]
    return "F-"

def mw_from_grade_list(grade_list):
    note = {"A+": 15, "A": 14, "A-": 13, "B+": 12, "B": 11, "B-": 10,
            "C+": 9, "C": 8, "C-": 7, "D+": 6, "D": 5, "D-": 4, "E+": 3, "E": 2, "E-": 1, "F": 0}
    grades = {}
    for g, n in note.items():
        grades[n] = g

    sum_grades = 0
    num_grades = 0
    for g in grade_list:
        if g in note.keys():
            sum_grades += note[g]
            num_grades += 1

    if num_grades == 0:
        return ""
    return grades[floor(sum_grades / num_grades + 0.5)]

@login_required
def index(request, spieler_id=None):
    # for Phillip's wish to see everyone's timetable
    if spieler_id is not None:
        if not User.objects.filter(username=request.user.username, groups__name='spielleiter').exists():
            return HttpResponse(status=404)
        spielleiter_service = True
        is_spielleiter = False
        spieler = get_object_or_404(Spieler, id=spieler_id)
    # usual case
    else:
        spielleiter_service = False
        is_spielleiter = User.objects.filter(username=request.user.username, groups__name='spielleiter').exists()
        spieler = get_object_or_404(Spieler, name=request.user.username)

    # avoid a thousand queries to the db
    all_published_questions = get_all_published_questions(is_spielleiter)

    if request.method == "GET":
        if not spielleiter_service and QuestionSession.objects.filter(spieler=spieler).exists():
            return redirect("quiz:question")

        subjects = Subject.objects.all().order_by("titel")
        data_dict = collections.OrderedDict()

        for s in subjects:
            data_dict[s.titel] = {"topics": collections.OrderedDict(), "answered_num": 0, "answerable_num": 0, "total_num": 0, "points": 0,
                                  "score": "", "score_tag_class": ""}
            for t in s.topic_set.all().order_by("titel"):
                questions = all_published_questions.filter(topic=t)
                total_num = questions.count()

                # done questions, answered correctly
                answered = []
                # not done questions or answered wrongly with enough time passed
                answerable = []

                # handle questions, where answers have been given correctly, if wrongly only if a certain period of time has passed (sp.answerable_again() )
                spielerQuestion_list = SpielerQuestion.objects.filter(question__in=questions, spieler=spieler)
                for sp in spielerQuestion_list:
                    if sp.answerable_again():
                        answerable.append(sp.question)
                    # if question is finally answered (all levels of intervall done) and will not come back
                    elif sp.num_correctly >= len(intervall.keys()):
                        answered.append(sp.question)

                # add those questions of our topic which haven't been answered jet to answerable
                prev_answered_ids = [sq.question.id for sq in spielerQuestion_list]
                for q in all_published_questions.filter(topic=t).exclude(id__in=prev_answered_ids):
                        answerable.append(q)

                answerable_ids = [q.id for q in answerable]

                # all vals for raw topic set now, grade-specific following
                data_dict[s.titel]["topics"][t.titel] = \
                    {"grades": [], "answered_num": len(set(answered)), "points": 0,
                     "answerable_num": len(set(answerable)), "total_num": total_num, "score": ""}

                # get set (no duplicates) of grades
                grades = sorted(set(all_published_questions.filter(topic=t).values_list('grade', flat=True)))
                answered_ids = [a.id for a in answered]
                for g in grades:

                    query = all_published_questions.filter(topic=t, grade=g)
                    grade_answered_num = 0
                    grade_answerable_num = 0
                    grade_total_num = query.count()
                    grade_points = 0
                    for q in query:
                        if q.id in answered_ids:
                            grade_answered_num += 1
                        elif q.id in answerable_ids:
                            grade_answerable_num += 1

                    # get points to achieve if answered correctly, therefore the offset=+1
                    for sq in SpielerQuestion.objects.filter(spieler=spieler, question__id__in=answerable_ids, question__grade=g):  # is subset of all_published questions.ids, because answerable_ids is already
                        grade_points += sq.get_points(offset=+1)

                        answerable_ids.remove(sq.question.id)

                    first_sq = SpielerQuestion.objects.first()
                    for q in Question.objects.filter(id__in=answerable_ids, grade=g):
                        grade_points += first_sq.get_points(value=1, question=q)

                    # get gradually_answered for note
                    grade_answered_gradually_num = 0
                    grade_answered_num_addon = 0
                    for sq in SpielerQuestion.objects.filter(spieler=spieler, question__topic=t, question__grade=g):
                        if sq.question.is_published():
                            grade_answered_num_addon += 1
                            grade_answered_gradually_num += sq.num_correctly

                    grade_dict = {"grade": g, "answered_num": grade_answered_num, "answerable_num": grade_answerable_num,
                                  "total_num": grade_total_num, "score": get_grade_score(grade_answered_gradually_num, grade_total_num*(len(intervall)-1), grade_answered_num_addon),
                                  "points": grade_points}
                    match = re.match("\w+", grade_dict["score"])
                    grade_dict["score_tag_class"] = ("grade_"+match.group(0)) if match else ""

                    data_dict[s.titel]["topics"][t.titel]["grades"].append(grade_dict)

                # get some fields for topic from its grades
                topic_dict = data_dict[s.titel]["topics"][t.titel]
                topic_dict["score"] = mw_from_grade_list([g["score"] for g in topic_dict["grades"]])

                match = re.match("\w+", topic_dict["score"])
                topic_dict["score_tag_class"] = ("grade_"+match.group(0)) if match else ""

                # get points
                points = 0
                for g in topic_dict["grades"]:
                    points += g["points"]
                topic_dict["points"] += points

            # get some fields for subject from its topics
            for t, e in data_dict[s.titel]["topics"].items():
                data_dict[s.titel]["answered_num"] += e["answered_num"]
                data_dict[s.titel]["answerable_num"] += e["answerable_num"]
                data_dict[s.titel]["total_num"] += e["total_num"]

                # list of topic grades
                list_topic_grades = []

                # get noten of subject from middleval
                for topic, vals in data_dict[s.titel]["topics"].items():
                    list_topic_grades.append(vals["score"])
                data_dict[s.titel]["score"] = mw_from_grade_list(list_topic_grades)
                match = re.match("\w+", data_dict[s.titel]["score"])
                data_dict[s.titel]["score_tag_class"] = ("grade_"+match.group(0)) if match else ""
                data_dict[s.titel]["points"] += e["points"]

        # big list-comprehension (not with topics) for all subjects
        timetable = [[{"entry": s, "answered_num": t["answered_num"], "answerable_num": t["answerable_num"],
                       "total_num": t["total_num"], "score": t["score"], "score_tag_class": t["score_tag_class"],
                       "points": t["points"]}
                     for s, t in data_dict.items()]]

        topics_left = True
        next_index = -1
        while topics_left:
            topics_left = False
            next_index += 1
            next_row = []
            for s_dict in timetable[0]:
                s_titel = s_dict["entry"]
                topics_dict = data_dict[s_titel]["topics"]
                if len(topics_dict.keys()) > next_index:
                    topic_dict = list(topics_dict.values())[next_index]
                    topic_dict["entry"] = list(topics_dict.keys())[next_index]

                    next_row.append(topic_dict)
                    topics_left = True
                else:
                    next_row.append([])
            if topics_left:
                timetable.append(next_row)

        name = ""
        if spielleiter_service:
            name = spieler.get_real_name()

        # gesamtnote concerning points
        ges_points = 0
        ges_questions = 0
        for q in all_published_questions:
            ges_questions += 1

            for i in range(len(intervall)):
                ges_points += first_sq.get_points(value=i, question=q)

        correct_questions = 0
        insg_answered_points = 0
        insg_answered_questions = 0
        gradually_correct_questions = 0
        for sq in SpielerQuestion.objects.filter(spieler=spieler):
            for i in range(len(intervall)):
                insg_answered_points += sq.get_points(value=i, question=sq.question)

            # for question-note
            insg_answered_questions += 1
            gradually_correct_questions += sq.num_correctly
            if sq.num_correctly+1 >= len(intervall):
                correct_questions += 1

        gesamtnote_points = get_grade_score(spieler.quiz_points_achieved, ges_points, insg_answered_points)
        match = re.match("\w+", gesamtnote_points)
        gesamtnote_points_tag_class = ("grade_"+match.group(0)) if match else ""

        note_points = {"correct": spieler.quiz_points_achieved, "gesamt": ges_points, "note": gesamtnote_points,
                       "note_tag_class": gesamtnote_points_tag_class, "erreicht": insg_answered_points}

        # gesamtnote concerning questions

        gesamtnote_questions = get_grade_score(gradually_correct_questions, ges_questions*(len(intervall)-1), insg_answered_questions)
        match = re.match("\w+", gesamtnote_questions)
        gesamtnote_questions_tag_class = ("grade_"+match.group(0)) if match else ""

        # note questions, mw from subjects' noten
        list_subject_grades = []
        for subj in timetable[0]:
            list_subject_grades.append(subj["score"])
        gesamtnote_questions_mw = mw_from_grade_list(list_subject_grades)
        match = re.match("\w+", gesamtnote_questions_mw)
        gesamtnote_questions_mw_tag_class = ("grade_"+match.group(0)) if match else ""

        note_questions = {"correct": correct_questions, "gesamt": ges_questions, "note": gesamtnote_questions,
                          "note_tag_class": gesamtnote_questions_tag_class, "erreicht": insg_answered_questions,
                          "mw_note": gesamtnote_questions_mw, "mw_note_tag_class": gesamtnote_questions_mw_tag_class}

        question_stufe_list = [{"num": 0, "intervall": e["days"], "factor": e["point_factor"]} for e in intervall.values()]   # needed to give an overview/at the end of this view, filled in next-querstion-on..-gathering

        # get info when new questions will be answerable
        sq_s = SpielerQuestion.objects.filter(spieler=spieler).order_by("-date")
        now = timezone.now()
        num_today = 0
        min_timedelta = None
        timedelta_0 = timedelta(days=0)
        for sq in sq_s:
            question_stufe_list[sq.num_correctly]["num"] += 1
            a_again = sq.datetime_when_answerable_again()
            # opened again today, but not jet
            if a_again.date() == now.date() and a_again > now:
                num_today += 1

            # look for the one opened the soonest, a_again should be in the future
            if a_again - now > timedelta_0 and (not min_timedelta or min_timedelta > a_again):
                min_timedelta = a_again

        # format min_timedelta
        min_str = "Alle alten Fragen sind offen."
        # es gibt wieder zu öffnende oder komplett fertige Fragen
        if min_timedelta is not None:
            # nicht alle sind bis zur letzten Stufe bearbeitet, es werden welche offen sein
            if min_timedelta - now < timedelta(days=1000):
                min_str = min_timedelta.strftime("Nächste Frage am %d.%m.%y um %H:%M")
                min_str = "{}, heute kommen noch {} alte Fragen wieder.".format(min_str, num_today)
            # alle sind bis zur letzten Stufe bearbeitet oder bereits offen, es gibt keine, die wieder offen sein werden
            #else: "Alle alten Fragen sind offen."
        #else: "Alle alten Fragen sind offen."

        info = [
            {"link": "", "text": min_str},
            {"link": "", "text": "Neue Fragen gibt's montags und donnerstags."},
            {"link": "", "text": "Falsch beantwortete Fragen sind nach 7 Tagen wieder offen."},
                ]

        context = {"timetable": timetable, "topic": "Auswahl", "akt_punktzahl": spieler.quiz_points,
                   "note_points": note_points, "note_questions": note_questions, "url_score_board": reverse("quiz:scoreBoard"),
                   "infos": info, "question_stufe_list": question_stufe_list,
                   # for Phillip's wish to see everyone's timetable (disable button in template, show Spieler's name, ..)
                   "spielleiter_service": spielleiter_service, "spieler_name": name}
        return render(request, "quiz/index.html", context)

    if request.method == "POST":
        subjects = request.POST.getlist("subject_list[]")
        topics = []
        grades = []

        regex_vals = {"topic_list": ["subject", "topic"], "grade_list": ["subject", "topic", "grade"]}
        regex_template = "{}\[(?P<index>\d+)\]\[{}\]"

        topic_collect_dict = {}
        grade_collect_dict = {}
        for k, v in regex_vals.items():
            for u in v:
                regex = re.compile(regex_template.format(k, u))

                # regex stands until here, go through all keys of request.POST
                for key, val in request.POST.items():
                    match = regex.match(key)
                    if match:
                        index = match.group("index")

                        # store in collect_dicts
                        collect_dict = topic_collect_dict if k == "topic_list" else grade_collect_dict
                        if index not in collect_dict.keys():
                            collect_dict[index] = {}
                        collect_dict[index][u] = val

        topics = list(topic_collect_dict.values())
        grades = list(grade_collect_dict.values())

        # get fitting questions which are answerable

        # get ids of all wrongly answered questions which are not answerable jet and all correctly answered questions
        except_id_list = []
        for sp in SpielerQuestion.objects.filter(spieler=spieler):
            if not sp.answerable_again():
                except_id_list.append(sp.question.id)

        # all chosen questions will be collected in here
        questions = []
        for q in all_published_questions.exclude(id__in=except_id_list):
            # subjects
            if q.topic.subject.titel in subjects:
                questions.append(q)

            # topics
            for e in topics:
                if q.topic.subject.titel == e["subject"] and q.topic.titel == e["topic"]:
                    questions.append(q)

            for e in grades:
                if q.topic.subject.titel == e["subject"] and q.topic.titel == e["topic"] and q.grade == int(e["grade"]):
                    questions.append(q)

        # remove all old sessions
        QuestionSession.objects.filter(spieler=spieler).delete()

        # get a new one with all collected questions, start_num_questions and max_points set
        session = QuestionSession.objects.create(spieler=spieler)
        max_points = 0
        start_num_questions = len(questions)

        # add all chosen questions to session questions_to_answer-pool
        for q in questions:
            DeleteableM2M_Question.objects.get_or_create(session=session, question=q)

        # get start_num_questions and max_points
        for sq in SpielerQuestion.objects.filter(spieler=spieler, question__in=questions):
            max_points += sq.get_points(offset=+1)

            # delete questions which have a sq object from questions to prevent counting them again
            questions.remove(sq.question)

        # all the questions left which haven't got an sq-obj jet and will be answered for the first time
        first_sq = SpielerQuestion.objects.first()
        for q in questions:
            max_points += first_sq.get_points(value=1, question=q)

        session.max_points = max_points
        session.start_num_questions = start_num_questions
        session.save()
        return JsonResponse({"url": reverse("quiz:question")})

@login_required
def question(request):
    spieler = get_object_or_404(Spieler, name=request.user.username)
    session = get_object_or_404(QuestionSession, spieler=spieler)

    if request.method == "GET":
        random.seed()

        if session.actual_question is not None:
            question = session.actual_question
        else:
            if DeleteableM2M_Question.objects.filter(session=session).count() == 0:
                return redirect("quiz:score")

            # get random num for acessing questions_to_answer => random order
            question_pool_size = DeleteableM2M_Question.objects.filter(session=session).count()
            index = random.randrange(question_pool_size)
            # get new actual_question
            question = DeleteableM2M_Question.objects.filter(session=session)[index].question
            get_object_or_404(DeleteableM2M_Question, session=session, question=question).delete()
            # set actual_question, important for POST
            session.actual_question = question
        session.save()
        num_question = session.start_num_questions - session.questions_to_answer.all().count()

        # if answered prev, use sq to determine possible points, else any sq-obj
        actual_sq = SpielerQuestion.objects.filter(spieler=spieler, question=question)
        if actual_sq:
            actual_sq = actual_sq[0]
            points = actual_sq.get_points(offset=+1)
            num_correct = actual_sq.num_correctly
        else:
            points = SpielerQuestion.objects.first().get_points(value=1, question=question)
            num_correct = 0

        answers = [a for a in question.answer_set.all()]
        random.shuffle(answers)

        context = {"topic": "Frage", "question": question, "answers": answers,
                   "start_num_questions": session.start_num_questions, "num_question": num_question,
                   "points": points, "num_correctly": num_correct, "achieved": session.achieved_points, "max_points": session.max_points,
                   # for debugging
                   "spielleiter": request.user.groups.filter(name__iexact="spielleiter").exists()
                   }
        return render(request, "quiz/question.html", context)

    if request.method == "POST":

        answer_ids = request.POST.getlist("answer_ids[]")
        answer_ids = [int(a) for a in answer_ids]
        selected_answers = Answer.objects.filter(id__in=answer_ids)

        # get actual question
        question = session.actual_question

        # case if >1 POST and no GET of "quiz:question" in between:
        if question is None:
            # reload from server in ajax.success
            return JsonResponse({"reload": 1})
        session.actual_question = None
        session.save()

        # check if answers belong to question
        all_answers = question.answer_set.all()

        all_answers_id = [a.id for a in all_answers]
        selected_answers_id = [a.id for a in selected_answers]
        for a in selected_answers:
            if a.id not in all_answers_id:
                return JsonResponse({"message": "Antwort: {} gehört nicht zu der Frage.".format(a.text)}, status=418)

        # collect answerres in lists ordered by given solution (example tt_list: should be true and is answered as true
        tt_list = []
        ff_list = []
        ft_list = []
        for a in all_answers:
            if a.id in selected_answers_id and a.correct:
                tt_list.append([a.id])
            elif a.id not in selected_answers_id and not a.correct:
                ff_list.append([a.id])
            else:
                ft_list.append([a.id, a.message_on_wrong_answered if a.message_on_wrong_answered else ""])

        # make spielerQuestion obj
        sq, _ = SpielerQuestion.objects.get_or_create(spieler=spieler, question=question)

        # add to session.results
        DeleteableM2M_SpielerQuestion.objects.get_or_create(spielerQuestion=sq, session=session)

        # get points, also negative if false answer given
        answered_correctly = len(ft_list) == 0
        sq.answered_correctly(answered_correctly)

        # save in every scenario, so the date is always reset to now, even if not necssary, because answered_correctly() is called
        sq.save()

        points = sq.get_points()
        session.achieved_points += points
        spieler.quiz_points += points
        spieler.save()

        session.save()
        return JsonResponse({"tt_list": tt_list, "ff_list": ff_list, "ft_list": ft_list})

@login_required
def score(request):
    spieler = get_object_or_404(Spieler, name=request.user.username)
    session = get_object_or_404(QuestionSession, spieler=spieler)

    if request.method == "GET":
        if session.start_num_questions == 0:
            session.delete()
            return redirect("quiz:index")

        if session.questions_to_answer.all().count() != 0 or session.actual_question is not None:
            return redirect("quiz:question")

        right_list = []
        wrong_list = []     # necessary?
        for dsq in DeleteableM2M_SpielerQuestion.objects.filter(session=session):
            sq = dsq.spielerQuestion
            if sq.correct and sq.num_correctly != 0:
                right_list.append(sq.question)
            else:
                wrong_list.append(sq.question)

        # list, grades of questions are indices
        score_list = []
        for d in range(14):
            score_list.append({"num": 0, "correct_num": 0, "points": 0})

        for sq in SpielerQuestion.objects.filter(question__in=wrong_list, spieler=spieler):
            score_grade_dict = score_list[sq.question.grade]

            score_grade_dict["num"] += 1
            score_grade_dict["points"] += sq.get_points()

        for sq in SpielerQuestion.objects.filter(question__in=right_list, spieler=spieler):
            score_grade_dict = score_list[sq.question.grade]

            score_grade_dict["num"] += 1
            score_grade_dict["correct_num"] += 1
            score_grade_dict["points"] += sq.get_points()

        percent = round((len(right_list) / session.start_num_questions) * 100, 2)
        score = get_grade_score(len(right_list), session.start_num_questions, session.start_num_questions)
        match = re.match("\w+", score)
        score_tag_class = ("grade_"+match.group(0)) if match else ""

        context = {"score_list": score_list, "topic": "Auswertung", "session": session, "sum_points": spieler.quiz_points_achieved,
                   "percent": percent, "score": score, "score_tag_class": score_tag_class, "right_num": len(right_list),
                   "total_num": session.start_num_questions, "url_mehr_fragen": reverse("quiz:index")}

        return render(request, "quiz/score.html", context)

    if request.method == "POST":
        QuestionSession.objects.filter(spieler=spieler).delete()
        return JsonResponse({})

def score_board(request):
    context = data_score_board(request);
    return render(request, "quiz/score_board.html", context)


from django.shortcuts import render
import json, urllib2
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, HttpResponseRedirect
from django.core.context_processors import csrf
# Create your views here.
from slider.models import *

def home(request):
	c = {}
	return render(request, 'home.html',c)
def slider(request, slider_id):
	c = {}
	slider = Slider.objects.get(id=int(slider_id))
	c['slider'] = json.dumps(slider.slides_as_json())
	return render(request, 'slider.html', c)

@login_required
def sliders(request):
	c = {}
	c['sliders'] = Slider.objects.filter(user=request.user)
	return render(request, 'sliders.html',c)
@login_required
def slider_create(request):
	if request.method == 'POST':
		d = request.POST['data']
		slides = json.loads(d)['slides']
		slider = Slider(user=request.user)
		slider.save()
		i = 0
		for slide in slides:
			s = Slide(image=slide['img'], headeing=slide['heading'], slide_id=i, slider=slider)
			i += 1
			s.save()
		return HttpResponseRedirect('/slider/'+str(slider.id))
	c = {}
	c.update(csrf(request))
	return render(request, 'create_slider.html', c)

def get_rand_image(request):
	d = urllib2.urlopen("http://www.splashbase.co/api/v1/images/random").read()
	d = json.loads(d)
	return HttpResponse(d['url'])

def error404(request):
	return render(request, '404.html', {})


"""
Created by hwyncho on 2017. 08. 24.
Last updated by hwyncho on 2018. 06. 16.
Copyright © 2018 hwyncho. All rights reserved.
==================================================
Convolutional Neural Network.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

import numpy as np
import tensorflow as tf
from PIL import Image

from dataset import Dataset

np.random.seed(777)
tf.set_random_seed(777)

class Cnn:
    def __init__(self):
        self.__load_dataset = False
        self.__load_model = False

        self._dataset = None
        self._model = None

        self._image_width = 48
        self._image_height = 48
        self._image_mode = 'RGB'

        self._dataset_size = 0
        self._image_size = 0
        self._label_size = 0

        self._device = 'cpu'
        self._epoch = 100
        self._batch_size = 100

    def __del__(self):
        del self.__load_dataset
        del self.__load_model

        del self._dataset
        del self._model

        del self._image_width
        del self._image_height
        del self._image_mode

        del self._dataset_size
        del self._image_size
        del self._label_size

        del self._device
        del self._epoch
        del self._batch_size

    def load_dataset(self, path):
        """
        Load the dataset from a specific path.

        :param dataset_path: str
            The path to the dataset to load.
        """
        if not isinstance(path, str):
            raise TypeError("type of 'path' must be str.")

        if not os.path.exists(path):
            raise FileNotFoundError()

        self.__load_dataset = True
        self._dataset = Dataset(path, one_hot=True)
        self._dataset_size = self._dataset.num_examples
        self._image_size = self._dataset.image_size
        self._label_size = self._dataset.label_size

    def load_model(self, path, sess):
        """
        Load the learned model from a specific path.

        :param model_path: str
            The path to the model to load.
        """
        if not isinstance(path, str):
            raise TypeError("type of 'path' must be str.")

        par_dir = os.path.split(path)[0]
        meta_file = '{}.meta'.format(path)
        ckpt_file = '{}/checkpoint'.format(par_dir)

        if not os.path.exists(meta_file):
            raise FileNotFoundError(
                'The meta file is not exist from: {}'.format(par_dir))

        if not os.path.exists(ckpt_file):
            raise FileNotFoundError(
                'The checkpoint file is not exist from: {}'.format(par_dir))

        self.__load_model = True
        self._model = tf.train.import_meta_graph(meta_file, clear_devices=True)
        self._model.restore(sess, tf.train.latest_checkpoint(par_dir))

    def set_device(self, device='cpu'):
        """
        Set up the device with CPU or GPU.

        :param device: str
            'cpu' or 'gpu'
        """
        if device in ['cpu', 'gpu']:
            self._device = device
        else:
            raise ValueError("'device' must be in 'cpu' or 'gpu'.")

    def set_epoch(self, epoch=100):
        """
        Set size of epoch.

        :param epoch: int
            size of epoch
        """
        self._epoch = epoch

    def set_batch_size(self, batch_size=100):
        """
        Set size of batch.

        :param batch_size: int
            size of batch
        """
        self._batch_size = batch_size

    def train(self, dataset_path, model_path='./Models/model'):
        """
        Train Neural-Networks and save model.

        :param dataset_path: str
            the path to load the dataset for train.
        :param model_path: str
            the path to save the learning model.
        """
        if not isinstance(dataset_path, str):
            raise TypeError("type of 'dataset_path' must be str.")

        if not isinstance(model_path, str):
            raise TypeError("type of 'model_path' must be str.")

        self.load_dataset(dataset_path)

        image_width = 48
        image_height = 48
        if self._image_mode == 'RGB':
            image_channel = 3
        elif self._image_mode == 'L':
            image_channel = 1

        learning_rate = 0.001

        with tf.device('/{}:0'.format(self._device)):
            # Input
            with tf.variable_scope('Input_Layer') as scope:
                x = tf.placeholder(tf.float32, shape=[
                                   None, self._image_size], name='images')
                y = tf.placeholder(tf.float32, shape=[
                                   None, self._label_size], name='labels')

            x_2d = tf.reshape(
                x, [-1, image_height, image_width, image_channel], name='images_2d')

            # Hidden Layers
            with tf.variable_scope('Hidden_Layers') as scope:
                conv_0 = tf.layers.conv2d(
                    inputs=x_2d, filters=32, kernel_size=5, padding='same', activation=tf.nn.relu
                )
                pool_0 = tf.layers.max_pooling2d(
                    inputs=conv_0, pool_size=2, strides=2, padding='same'
                )

                conv_1 = tf.layers.conv2d(
                    inputs=pool_0, filters=64, kernel_size=5, padding='same', activation=tf.nn.relu
                )
                pool_1 = tf.layers.max_pooling2d(
                    inputs=conv_1, pool_size=2, strides=2, padding='same'
                )

                conv_2 = tf.layers.conv2d(
                    inputs=pool_1, filters=128, kernel_size=5, padding='same', activation=tf.nn.relu
                )
                pool_2 = tf.layers.max_pooling2d(
                    inputs=conv_2, pool_size=2, strides=2, padding='same'
                )

                conv_3 = tf.layers.conv2d(
                    inputs=pool_2, filters=256, kernel_size=5, padding='same', activation=tf.nn.relu
                )
                pool_3 = tf.layers.max_pooling2d(
                    inputs=conv_3, pool_size=2, strides=2, padding='same'
                )

                flatten = tf.layers.flatten(inputs=pool_3)
                fc = tf.layers.dense(
                    inputs=flatten, units=1024, activation=tf.nn.relu)

            # Output
            with tf.variable_scope('Output_Layer') as scope:
                y_predict = tf.layers.dense(
                    inputs=fc, units=self._label_size, activation=None, name='y_predict')

            cross_entropy = tf.reduce_mean(
                tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_predict))
            train_step = tf.train.AdamOptimizer(
                learning_rate=learning_rate).minimize(cross_entropy)

        with tf.device('/cpu:0'):
            model_saver = tf.train.Saver()

        # set train count
        if self._dataset_size <= self._batch_size:
            train_count = 1
        else:
            if (self._dataset_size % self._batch_size) == 0:
                train_count = int(self._dataset_size / self._batch_size)
            else:
                train_count = int(self._dataset_size / self._batch_size) + 1

        # session configure
        config = tf.ConfigProto(log_device_placement=True)
        if self._device == 'gpu':
            config.gpu_options.allow_growth = True

        with tf.Session(config=config) as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())

            # optimization
            for step in range(self._epoch):
                self._dataset.reset_batch()
                for i in range(train_count):
                    batch_x, batch_y = self._dataset.next_batch(
                        self._batch_size)
                    sess.run(train_step, feed_dict={x: batch_x, y: batch_y})

            # save model
            par_dir = os.path.split(model_path)[0]
            if not os.path.exists(par_dir):
                os.makedirs(par_dir)

            model_saver.save(sess, model_path)
            self.__load_model = True
            print("Model save to '{}'".format(model_path))

            tf.reset_default_graph()

    def eval(self, dataset_path, model_path):
        """
        Evaluate saved model.

        :param dataset_path: str
            the path to load the dataset for evaluate.
        :param model_path: str
            the path to load the learned model.

        :return: dict
        """
        if not isinstance(dataset_path, str):
            raise TypeError("type of 'dataset_path' must be str.")

        if not isinstance(model_path, str):
            raise TypeError("type of 'model_path' must be str.")

        self.load_dataset(dataset_path)

        sess = tf.Session()

        sess.run(tf.global_variables_initializer())
        sess.run(tf.local_variables_initializer())

        self.load_model(model_path, sess)
        graph = tf.get_default_graph()

        with tf.device('/{}:0'.format(self._device)):
            # Input
            x = graph.get_tensor_by_name('Input_Layer/images:0')
            y = graph.get_tensor_by_name('Input_Layer/labels:0')

            y_predict = graph.get_tensor_by_name(
                'Output_Layer/y_predict/BiasAdd:0')

            correct_prediction = tf.equal(
                tf.argmax(y, 1), tf.argmax(y_predict, 1))
            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

        with tf.device('/cpu:0'):
            confusion_matrix = tf.confusion_matrix(
                labels=tf.argmax(y, 1), predictions=tf.argmax(y_predict, 1))

        if self._dataset_size <= self._batch_size:
            test_count = 1
        else:
            if (self._dataset_size % self._batch_size) == 0:
                test_count = int(self._dataset_size / self._batch_size)
            else:
                test_count = int(self._dataset_size / self._batch_size) + 1

        # test
        self._dataset.reset_batch()
        for i in range(test_count):
            batch_x, batch_y = self._dataset.next_batch(self._batch_size)
            if i == 0:
                test_accuracy = sess.run(accuracy, feed_dict={
                                         x: batch_x, y: batch_y})
                test_cm = sess.run(confusion_matrix, feed_dict={
                                   x: batch_x, y: batch_y})
            else:
                test_accuracy += sess.run(accuracy,
                                          feed_dict={x: batch_x, y: batch_y})
                test_cm += sess.run(confusion_matrix,
                                    feed_dict={x: batch_x, y: batch_y})

        test_accuracy /= test_count
        test_cm = test_cm.tolist()

        tf.reset_default_graph()
        sess.close()

        return {
            'accuracy': test_accuracy,
            'confusion_matrix': test_cm
        }

    def query(self, image_path, model_path):
        """
        Input images and predict labels.

        :param images: array
        :param model_path: str

        :return: nothing
        """
        if not isinstance(image_path, str):
            raise TypeError("type of 'image_path' must be str.")

        if not isinstance(model_path, str):
            raise TypeError("type of 'model_path' must be str.")

        image_width = 48
        image_height = 48
        if self._image_mode == 'RGB':
            image_channel = 3
        elif self._image_mode == 'L':
            image_channel = 1

        image = Image.open(image_path).convert(
            mode=self._image_mode).resize((image_width, image_height))
        w, h = image.size

        pixel_list = list()
        for y in range(h):
            for x in range(w):
                pixel = image.getpixel((x, y))
                if isinstance(pixel, tuple):
                    pixel_list += [p for p in pixel]
                else:
                    pixel_list.append(pixel)

        sess = tf.Session()

        sess.run(tf.global_variables_initializer())
        sess.run(tf.local_variables_initializer())

        self.load_model(model_path, sess)
        graph = tf.get_default_graph()

        with tf.device('/{}:0'.format(self._device)):
            # Input
            x = graph.get_tensor_by_name('Input_Layer/images:0')

            y_predict = graph.get_tensor_by_name(
                'Output_Layer/y_predict/BiasAdd:0')

            predict = tf.argmax(y_predict, 1)

            # run session
            result = sess.run(predict, feed_dict={x: [pixel_list]})

        sess.close()

        return result


"""
    Module For Making Fake CCDImage
    Author : Kim Tae Bum
    e-mail : phya.ktaebum@gmail.com
    Lastly Modified : 2017-06-17
"""

import numpy as np
from astropy.modeling.functional_models import Gaussian2D, Sersic2D
from ccdproc import CCDData

class FakeCCD():
    """
        Class of Fake CCD
    """

    def __init__(self, base=100, size=800, readout=10):
        """
        Constructor For Class
        :param base: Base Value of Sky Background
        :param size: Size of Mesh Grid(square)
        :param readout: Standard Deviation of Sky Background
                Recommended: readout ~= base / 10
        Class Members:
            1. x, y = pixel grid with size; default size = 800
            2. star = fake star map
            3. galaxy = fake galaxy map
            4. cr = fake cosmic rays map
            5. sky = fake sky background
            6. std = readout noise of sky background, default = 10
            7. header = header infor for CCDData
        """
        self.size = size
        self.x, self.y = np.mgrid[0:size, 0:size]
        self.star = np.zeros(shape=(size, size), dtype=np.float64)
        self.galaxy = np.zeros(shape=(size, size), dtype=np.float64)
        self.cr = np.zeros(shape=(size, size), dtype=np.float64)
        # Set Sky Background
        self.sky = np.random.normal(base, readout, size=(size, size))
        self.std = readout
        self.header = dict(name='Taebum\'s fake data',
                           size='{:d} by {:d}'.format(size, size),
                           readout=readout,
                           unit='ADU')

    def merge(self):
        """
        :return: Merged Data
        """
        return self.sky + self.galaxy + self.cr + self.star

    def getAll(self):
        """
        :return: Merged Data in CCDData object
        """
        header = self.header.copy()
        header['Data'] = 'sky, cosmic ray, galaxy, star'
        data = self.merge()
        return CCDData(header=header, data=data, unit='adu')

    def getSky(self):
        """
        :return: Just sky background
        """
        header = self.header.copy()
        header['Data'] = 'Sky'
        data = self.merge()
        return CCDData(header=header, data=data, unit='adu')

    def getStar(self):
        """
        :return: Get Only Star Map(include sky background)
        """
        header = self.header.copy()
        header['Data'] = 'sky, star'
        return CCDData(header=header, data=self.sky + self.star, unit='adu')

    def getGalaxy(self):
        """
        :return: Get Only Galaxy Map(include sky background)
        """
        header = self.header.copy()
        header['Data'] = 'sky, galaxy'
        return CCDData(header=header, data=self.sky + self.galaxy, unit='adu')

    def getCR(self):
        """
        :return: Get Only Cosmic Ray Map(in this case, do not include sky background, only pure cr)
        """
        header = self.header.copy()
        header['Data'] = 'Cosmic ray'
        data = self.cr.copy()
        data[data > 0] = 100
        return CCDData(header=header, data=data, unit='adu')

    def set_star(self, num):
        """
        Set num stars in ccd data
        Following Gaussian 2D Distribution
        :param num: number of fake stars
        :return:
        """
        for i in range(num):
            x = np.random.uniform(0, self.size)
            y = np.random.uniform(0, self.size)
            # std = np.random.uniform(3, 5)
            std = 1.5
            amp = np.random.uniform(5 * self.std, 10 * self.std)
            self.star += Gaussian2D.evaluate(self.x, self.y, amplitude=amp,
                                             theta=0, x_mean=x, y_mean=y, x_stddev=std,
                                             y_stddev=std)
        return

    def set_galaxy(self, num):
        """
        Set num galaxies in ccd data
        following Sersic 2D Profile
        :param num: number of fake galaxies
        :return:
        """
        for i in range(num):
            amp = np.random.uniform(0.5 * self.std, 5 * self.std)
            r_eff = np.random.uniform(0, self.size / 100)
            n = np.random.uniform(0.5, 10)
            x_c = np.random.uniform(0, self.size)
            y_c = np.random.uniform(0, self.size)
            eps = np.random.uniform(0, 0.5)
            theta = np.pi * np.random.random() - np.pi - 2
            galaxy = Sersic2D(amp, r_eff, n, x_c, y_c, eps, theta)
            self.galaxy += galaxy(self.x, self.y)
        return

    def set_cr(self, num, prob=0.2):
        """
        Set num Cosmic Rays in CCD Data
        There are two Types of Cosmic Rays
        1. Line Shape
        2. Dot Shape
        :param num: number of cosmic rays
        :param prob: probability of being line cosmic ray, default = 0.2
        :return:
        """
        for i in range(num):
            x = np.random.randint(0, self.size)
            y = np.random.randint(0, self.size)
            theta = np.pi * np.random.random() - np.pi / 2
            length = np.random.randint(int(self.size / 100), int(1.2 * self.size / 100))
            dice = np.random.random()
            if dice < prob:
                self.set_line_cr(x, y, theta, length)
            else:
                self.set_dot_cr(x, y)

        return

    def set_line_cr(self, x_s, y_s, theta, length):
        """
        :param x_s: start x point
        :param y_s: start y point
        :param theta: direction
        :param length: x pixel length
        :return:
        """
        value = np.random.uniform(5 * self.std, 100 * self.std)
        x_prev = x_s
        y_prev = y_s
        for i in range(length):
            x_s = x_prev + 1
            y_s = int(y_prev + np.tan(theta))
            if y_s < 0 or y_s >= self.size or x_s < 0 or x_s >= self.size:
                return
            self.cr[x_s, y_prev:y_s] = value
            x_prev = x_s
            y_prev = y_s

        return

    def set_dot_cr(self, x, y):
        """
        :param x: x center
        :param y: y centerget update
        :return:
        """
        value = np.random.uniform(5 * self.std, 100 * self.std)
        x_g, y_g = np.mgrid[x - 1:x + 2, y - 1:y + 2]
        sli = (x_g >= 0) & (x_g < self.size)
        sli = sli & (y_g >= 0) & (y_g < self.size)
        self.cr[x_g[sli], y_g[sli]] = value

    def analysis(self, masked):
        """
        Determine How LA Cosmic Correctly Masked Fake Cosmic Rays Via Comparison
        :param masked: masked cosmic rays
        :return:
        """
        mark = np.where(masked > 0)
        correct = 0
        as_star = 0
        as_gal = 0
        for i in range(len(mark[0])):
            if self.cr[mark[0][i], mark[1][i]] > 0:
                correct += 1
            else:
                if self.star[mark[0][i], mark[1][i]] >= self.galaxy[mark[0][i], mark[1][i]]:
                    as_star += 1
                else:
                    as_gal += 1
        print('Number of Cosmic Ray Pixels : {:d}'.format(len(np.where(self.cr > 0)[0])))
        print('Correctly Masked Pixels : {:d}'.format(correct))
        print('Incorrectly Masked, as Stars : {:d}'.format(as_star))
        print('Incorrectly Masked, as Galaxies : {:d}'.format(as_gal))


import torch
import itertools as it

def run_epoch(params, phase, num_batches, model, optimizer, scheduler, dataloader, device):

        print(phase.upper())
        if phase == 'train':
            scheduler.step()
            model.train()  # Set model to training mode
        else:
            model.eval()  # Set model to evaluate mode

        running_loss = 0.0

        # Iterate over data once.
        batchidx = 0
        for i, (x, y) in zip(it.count(num_batches),dataloader):
            print(i,end=' ')
            batchidx += 1
            if batchidx > num_batches:
                print(" ")
                break

            y_classmap, y_vectormap = y
            X = torch.Tensor(x.type(torch.float))
            Y_class = torch.Tensor(y_classmap).type(torch.long)
            Y_vector = torch.Tensor(y_vectormap[:, :, :, :])

            """ put input and target to device"""
            Y_class = Y_class.to(device)
            Y_vector = Y_vector.to(device)
            X = X.to(device)

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward
            # track history if only in train
            with torch.set_grad_enabled(phase == 'train'):
                outputs = model(X)

                pred_class_logits = outputs[:, :2, :, :]  # torch.Size([1, 2, 1920, 1184])
                pred_class_vectors = outputs[:, 2:, :, :]  # torch.Size([1, 2, 1920, 1184])

                """ Cross entropy loss """
                cross_entropy_loss_fn = loss = torch.nn.CrossEntropyLoss()
                pred_class_logits = torch.reshape(pred_class_logits,
                                                  (params.batch_size, params.numclasses, params.width * params.height))
                Y_class = torch.reshape(Y_class, (params.batch_size, params.width * params.height))
                cross_entropy_loss = cross_entropy_loss_fn(pred_class_logits, Y_class)

                """ L2 vector loss """
                # pred_class_vectors:  torch.Size([1, 2, 1920, 1184])
                # Y_vector: torch.Size([1, 2, 1920, 1184])
                vector_loss_fn = torch.nn.MSELoss()
                vector_loss = vector_loss_fn(pred_class_vectors, Y_vector)

                total_loss = cross_entropy_loss + vector_loss

                # backward + optimize only if in training phase
                if phase == 'train':
                    total_loss.backward()
                    optimizer.step()

            # statistics
            running_loss += total_loss.item() * X.size(0)

        epoch_loss = running_loss / num_batches

        return epoch_loss


from django import forms
from django.db import models
from .models import Image

class ImageForm(forms.ModelForm):
    class Meta:
        model = Image
        fields = ('title', 'description', 'user', 'image')
        labels = {
            'user':'User'
        }

    def __init__(self, *args, **kwargs):
        super(ImageForm, self).__init__(*args, **kwargs)
        self.fields['user'].empty_label = "-- Keep Unselected for Anonymous --"


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('main', '0003_tamgas_imagebrowse'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='tamgas',
            options={'verbose_name': '\u0442\u0430\u043c\u0433\u0430-'},
        ),
        migrations.RemoveField(
            model_name='tamgas',
            name='imagebrowse',
        ),
    ]


import asyncio
from tcp_tunnel.connection import Connection
from tcp_tunnel import logger

async def tcp_echo_client():
    for i in range(100):
        reader, writer = await asyncio.open_connection('127.0.0.1', 8889)
        gold_reader, gold_writer = await asyncio.open_connection('127.0.0.1', 8000)
        conn = Connection(reader, writer, gold_reader, gold_writer)
        loop.create_task(conn.run_writer())
        loop.create_task(conn.run_reader())
        logger.debug('create conn')

loop = asyncio.get_event_loop()
future = asyncio.ensure_future(tcp_echo_client())
try:
    loop.run_forever()
except KeyboardInterrupt:
    future.cancel()
    loop.run_until_complete(future)
finally:
    # 不管是什么异常，最终都要close掉loop循环
    loop.close()


from keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.preprocessing import sequence

from postag_eng.parser.EnglishNerParser import EnglishNerParser
from postag_eng.vectorizer.Vectorizer import Vectorizer
from keras.utils import np_utils
from postag_eng.recurrentneuralnetwork.RecurrentNeuralNetwork import RecurrentNeuralNetwork

import numpy as np

if __name__ == '__main__':

    print('Reading training data')
    documents = EnglishNerParser().read_file('F:/Utilisateur/Documents/ESGI/Cours/Traitement Automatique du Langage Naturel/eng_train.txt')

    print('Create features')
    vectorizer = Vectorizer(word_embedding_path='F:/Utilisateur/Documents/ESGI/Cours/Traitement Automatique du Langage Naturel/glove.6B.50d.txt')
    word, shape = vectorizer.encode_features(documents)
    labels = vectorizer.encode_annotations(documents)
    print('Loaded {} data samples'.format(len(word)))

    print('Split training/validation')
    max_length = 20
    # --------------- Features_word ----------------
    # 2. Padd sequences
    word = sequence.pad_sequences(word, maxlen=max_length)
    # --------------- Features_shape ----------------
    # 2. Padd sequences
    shape = sequence.pad_sequences(shape, maxlen=max_length)

    # --------------- Labels -------------------
    # 1. Convert to one-hot vectors
    labels = [np_utils.to_categorical(y_group, num_classes=len(vectorizer.labels)) for y_group in labels]
    # 2. (only for sequence tagging) Pad sequences
    labels = sequence.pad_sequences(labels, maxlen=max_length)

    to = int(len(word)*0.8)

    x_train = [np.asarray(word[:to]), np.asarray(shape[:to])]
    y_train = np.array(labels[:to])
    x_validation = [np.asarray(word[to+1:]), np.asarray(shape[to+1:])]
    y_validation = np.array(labels[to+1:])

    print(x_train[1].shape)
    print(x_validation[1].shape)

    print('Building network...')
    model = RecurrentNeuralNetwork.build_sequence(word_embeddings=vectorizer.word_embeddings,
                                          input_shape={'shape': (len(vectorizer.shape2index), 46)},
                                          out_shape=len(vectorizer.labels),
                                          units=100, dropout_rate=0.5)

    print('Train...')
    trained_model_name = 'ner_weights.h5'

    # Callback that stops training based on the loss fuction
    early_stopping = EarlyStopping(monitor='val_loss', patience=10)

    # Callback that saves the best model across epochs
    saveBestModel = ModelCheckpoint(trained_model_name, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')

    model.fit(x_train, y_train,
              validation_data=(x_validation, y_validation),
              batch_size=32,  epochs=50, callbacks=[saveBestModel, early_stopping])

    # # Load the best weights in the model
    # model.load_weights(trained_model_name)
    #
    # # Save the complete model
    # model.save('rnn.h5')


from pyspark.ml.wrapper import JavaEstimator, JavaModel
from pyspark.ml.param.shared import *
from pyspark.mllib.common import inherit_doc
from pyspark.ml.util import keyword_only

@inherit_doc
class KNNRegression(JavaEstimator, HasFeaturesCol, HasLabelCol, HasPredictionCol,
                    HasInputCols, HasThresholds, HasSeed, HasWeightCol):
    @keyword_only
    def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
                 seed=None, topTreeSize=1000, topTreeLeafSize=10, subTreeLeafSize=30, bufferSize=-1.0,
                 bufferSizeSampleSize=list(range(100, 1000 + 1, 100)), balanceThreshold=0.7,
                 k=5, neighborsCol="neighbors", maxNeighbors=float("inf")):
        super(KNNRegression, self).__init__()
        self._java_obj = self._new_java_obj(
            "org.apache.spark.ml.regression.KNNRegression", self.uid)

        self.topTreeSize = Param(self, "topTreeSize", "number of points to sample for top-level tree")
        self.topTreeLeafSize = Param(self, "topTreeLeafSize",
                                     "number of points at which to switch to brute-force for top-level tree")
        self.subTreeLeafSize = Param(self, "subTreeLeafSize",
                                     "number of points at which to switch to brute-force for distributed sub-trees")
        self.bufferSize = Param(self, "bufferSize",
                                "size of buffer used to construct spill trees and top-level tree search")
        self.bufferSizeSampleSize = Param(self, "bufferSizeSampleSize",
                                          "number of sample sizes to take when estimating buffer size")
        self.balanceThreshold = Param(self, "balanceThreshold",
                                      "fraction of total points at which spill tree reverts back to metric tree if "
                                      "either child contains more points")
        self.k = Param(self, "k", "number of neighbors to find")
        self.neighborsCol = Param(self, "neighborsCol", "column names for returned neighbors")
        self.maxNeighbors = Param(self, "maxNeighbors", "maximum distance to find neighbors")

        self._setDefault(topTreeSize=1000, topTreeLeafSize=10, subTreeLeafSize=30, bufferSize=-1.0,
                         bufferSizeSampleSize=list(range(100, 1000 + 1, 100)), balanceThreshold=0.7,
                         k=5, neighborsCol="neighbors", maxNeighbors=float("inf"))

        kwargs = self.__init__._input_kwargs
        self.setParams(**kwargs)

    @keyword_only
    def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
                  seed=None, topTreeSize=1000, topTreeLeafSize=10, subTreeLeafSize=30, bufferSize=-1.0,
                  bufferSizeSampleSize=list(range(100, 1000 + 1, 100)), balanceThreshold=0.7,
                  k=5, neighborsCol="neighbors", maxNeighbors=float("inf")):
        kwargs = self.setParams._input_kwargs
        return self._set(**kwargs)

    def _create_model(self, java_model):
        return KNNRegressionModel(java_model)

class KNNRegressionModel(JavaModel):
    """
    Model fitted by KNNRegression.
    """
    def __init__(self, java_model):
        super(KNNRegressionModel, self).__init__(java_model)

        # note: look at https://issues.apache.org/jira/browse/SPARK-10931 in the future
        self.bufferSize = Param(self, "bufferSize",
                                "size of buffer used to construct spill trees and top-level tree search")
        self.k = Param(self, "k", "number of neighbors to find")
        self.neighborsCol = Param(self, "neighborsCol", "column names for returned neighbors")
        self.maxNeighbors = Param(self, "maxNeighbors", "maximum distance to find neighbors")

        self._transfer_params_from_java()


# -*- coding: utf-8 -*-
"""
Created on Sat Sep 19 20:50:40 2020

@author: luis
"""

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support.ui import Select
import time
from bs4 import BeautifulSoup
import requests
import pandas as pd
from self import self

class Table:
    def __init__(self,driver):
        self.driver=driver

    def personas(self,item):
        try:
            time.sleep(4)
            valor= False
            data = driver.find_elements_by_class_name("wanted_teaser_quick_info")
            boton = data[item]
           # data.send_keys(Keys.ENTER)
            boton.click()
            valor =True

            return valor

        except Exception as e:
              print("Error personas" + str(e))

    def get_data(self,item):
        try:
             time.sleep(5)
             data =[]
             lista =[]
             info=driver.find_element_by_class_name("wanted_top_right")
             divs = info.find_elements_by_tag_name("div")
             nombre = driver.find_element_by_class_name("field-name-title-field")
             for item in divs:
                 lista.append(item.text.replace("\n",""))

             bandera = 0
             bandera2 = 0
             validacion = 0
             Delito =''
             sexo =''
             ojos =''
             nacimiento =''
             origen =''
             nacionalidad =''
             idioma =''
             estado =''
             for elemento in lista:
                 separar = elemento.split(":")

                 if separar[0] in ('DELITO','SEXO ','COLOR DE OJOS','FECHA DE NACIMIENTO','NACIONALIDAD','ORIGEN ÉTNICO','IDIOMAS HABLADOS','ESTADO DEL CASO'):
                      if separar[0] in ('DELITO'):
                          Delito =separar[1]
                          validacion = 1
                      if separar[0] in ('SEXO '):
                           sexo =separar[1]
                           validacion = 1
                      if separar[0] in ('COLOR DE OJOS'):
                          ojos =separar[1]
                          validacion = 1
                      if separar[0] in ('FECHA DE NACIMIENTO'):
                          if bandera  == 0:
                              bandera = 1
                              validacion = 1
                              nacimiento =separar[1]
                      if separar[0] in ('NACIONALIDAD'):
                          nacionalidad =separar[1]
                          validacion = 1
                      if separar[0] in ('ORIGEN ÉTNICO'):
                          origen =separar[1]
                          validacion = 1
                      if separar[0] in ('IDIOMAS HABLADOS'):
                          idioma =separar[1]
                          validacion = 1
                      if separar[0] in ('ESTADO DEL CASO'):
                          if bandera2  == 0:
                              bandera2 = 1
                              validacion = 1
                              estado =separar[1]

             if validacion == 1:
                    data=[nombre.text,Delito,sexo,ojos,nacimiento,nacionalidad,origen,idioma,estado]
             print(data)
             print(len(data))

             return data

        except Exception as e:
              print("An exception occurred get_rows" + str(e))

if "__main__" == __name__:
    PATH = "C:\Pentaho\chromedriver.exe"
    driver = webdriver.Chrome(PATH)
    driver.implicitly_wait(2)
    Url="https://eumostwanted.eu/es"

    driver.get(Url)
    table = Table(driver)
    rows=[]
    time.sleep(2)

    personas = driver.find_elements_by_class_name("wanted_teaser_quick_info")
    print(len(personas))
    total =len(personas)
    column_rows=[]
    for item in range(0,total):
                  valor = table.personas(item)
                  if valor == True:
                      data =table.get_data(item)
                      if data is not None:
                         if len(data) > 0:
                              rows.append(data)

                              driver.forward()
                              driver.back()
    driver.close()
    if len(rows) > 0 :
        try:
            df = pd.DataFrame(rows,columns=['nombre','Delito','sexo','ojos','nacimiento','nacionalidad','origen','idioma','estado'])
            df.to_excel('C:\Pentaho\Europol.xlsx',index=False)

        except:
                print("Error en escribir en el excel")
    else:
        print('ocurrio un error')


import cgi
import os
from google.appengine.api import users
import webapp2
from google.appengine.ext import ndb
import json
from datetime import datetime
import jinja2

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    extensions=['jinja2.ext.autoescape'],
    autoescape=True)

MAIN_PAGE_HTML = """\

    <form action="/gwy/test_submitreading" method="post">

Gateway 1111111111111111111 Serial
<input type="text" name="gateway_serial">

ReadOut
<input type="text" name="readout">

      <input type="submit" value="POST RESULT">

    </form>

"""

class DoUpload(webapp2.RequestHandler):
  def post(self):
      fl = self.request.get('myfile')
      for contr in Contract.all():
          contr.delete()
      for prop in Property.all():
          prop.delete()
      for addr in Address.all():
          addr.delete()
      for pers in Person.all():
          pers.delete()

      self.response.write(fl.decode("latin-1")+"
Hallo Kris
" )
      reader = csv.reader(fl.decode("latin-1").splitlines(), delimiter=";")
      rownum = 0
      for row in reader:
         if (rownum > 0 ):
            self.response.write(row[0] + "
")
            addr = Address(street= row[2], postalCode = row[3], locality = row[4])
            addr.put()
            prop = Property(shortName = row[0], propertyType=int(row[1]), propertyOwner=int(row[5]), address=addr.key() )
            prop.put()
            pers = Person(name=row[6], language=row[8], aanspreekTittel =row[7])
            if row[15] != None and len(row[15])>1:
                addr = Address(street= row[15], postalCode = row[16], locality = row[17])
                addr.put()
                pers.address = addr.key()
            pers.put()
            contr = Contract(tenant = pers.key(), property= prop.key(), startDate = datetime.strptime(row[13],"%d/%m/%Y") , baseIndex=float(row[9]), baseIndexType=int(row[10]), baseRent=float(row[11]), provision=float(row[12]), indexMonth=int(row[14]))
            if row[15] != None and len(row[15])>1:
                contr.useTenantAddress = bool(True)
            else:
                contr.useTenantAddress = bool(False)

            contr.put()
         rownum += 1

class MainPage(webapp2.RequestHandler):
    def get(self):
        self.response.write(MAIN_PAGE_HTML)

app = webapp2.WSGIApplication([
    ('/', MainPage),
    ('/uploadFile.do', DoUpload)
], debug=True)


"""
Класс для работы с файлом конфигурации
Конфигурация должна храниться в формате YAML.
В процессе работы объкет данного класса (с загруженной конфигурацией) передаётся на вход другим этапам пайплайна.
По-умолчанию yaml-файл хранится в директории config.
Config имплементирует класс dictionary питона
"""

import os
import yaml

class Config(dict):

    def __init__(self, path_to_configfile):
        """
        :param path_to_configfile: строка, путь до файла с конфигом
        :return None
        """
        assert os.path.isfile(path_to_configfile), \
            'No configuration file found: {}'.format(path_to_configfile)
        with open(path_to_configfile, "r") as file:
            self.config_ = yaml.safe_load(file)
        # на этом этапе сам объект конфигурации становится словарем
        super(Config, self).__init__(**self.config_)


"""Snips NLU training/recognize methods for Rhasspy."""
import io
import logging
import shutil
import tempfile
import typing
from pathlib import Path

import networkx as nx
import rhasspynlu
from rhasspynlu.intent import Entity, Intent, Recognition
from snips_nlu import SnipsNLUEngine
from snips_nlu.dataset import Dataset
from snips_nlu.default_configs import DEFAULT_CONFIGS

_LOGGER = logging.getLogger("rhasspysnips_nlu")

# -----------------------------------------------------------------------------

def train(
    sentences_dict: typing.Dict[str, str],
    language: str,
    slots_dict: typing.Optional[typing.Dict[str, typing.List[str]]] = None,
    engine_path: typing.Optional[typing.Union[str, Path]] = None,
    dataset_path: typing.Optional[typing.Union[str, Path]] = None,
) -> SnipsNLUEngine:
    """Generate Snips YAML dataset from Rhasspy sentences/slots."""
    slots_dict = slots_dict or {}

    _LOGGER.debug("Creating Snips engine for language %s", language)
    engine = SnipsNLUEngine(config=DEFAULT_CONFIGS[language])

    # Parse JSGF sentences
    _LOGGER.debug("Parsing sentences")
    with io.StringIO() as ini_file:
        # Join as single ini file
        for lines in sentences_dict.values():
            print(lines, file=ini_file)
            print("", file=ini_file)

        intents = rhasspynlu.parse_ini(ini_file.getvalue())

    # Split into sentences and rule/slot replacements
    sentences, replacements = rhasspynlu.ini_jsgf.split_rules(intents)

    for intent_sentences in sentences.values():
        for sentence in intent_sentences:
            rhasspynlu.jsgf.walk_expression(
                sentence, rhasspynlu.number_range_transform, replacements
            )

    # Convert to directed graph *without* expanding slots
    # (e.g., $rhasspy/number)
    _LOGGER.debug("Converting to intent graph")
    intent_graph = rhasspynlu.sentences_to_graph(
        sentences, replacements=replacements, expand_slots=False
    )

    # Get start/end nodes for graph
    start_node, end_node = rhasspynlu.jsgf_graph.get_start_end_nodes(intent_graph)
    assert (start_node is not None) and (
        end_node is not None
    ), "Missing start/end node(s)"

    if dataset_path:
        # Use user file
        dataset_file = open(dataset_path, "w+")
    else:
        # Use temporary file
        dataset_file = typing.cast(
            typing.TextIO, tempfile.NamedTemporaryFile(suffix=".yml", mode="w+")
        )
        dataset_path = dataset_file.name

    with dataset_file:
        _LOGGER.debug("Writing YAML dataset to %s", dataset_path)

        # Walk first layer of edges with intents
        for _, intent_node, edge_data in intent_graph.edges(start_node, data=True):
            intent_name: str = edge_data["olabel"][9:]

            # New intent
            print("---", file=dataset_file)
            print("type: intent", file=dataset_file)
            print("name:", quote(intent_name), file=dataset_file)
            print("utterances:", file=dataset_file)

            # Get all paths through the graph (utterances)
            used_utterances: typing.Set[str] = set()
            paths = nx.all_simple_paths(intent_graph, intent_node, end_node)
            for path in paths:
                utterance = []
                entity_name = None
                slot_name = None
                slot_value = None

                # Walk utterance edges
                for from_node, to_node in rhasspynlu.utils.pairwise(path):
                    edge_data = intent_graph.edges[(from_node, to_node)]
                    ilabel = edge_data.get("ilabel")
                    olabel = edge_data.get("olabel")
                    if olabel:
                        if olabel.startswith("__begin__"):
                            slot_name = olabel[9:]
                            entity_name = None
                            slot_value = ""
                        elif olabel.startswith("__end__"):
                            if entity_name == "rhasspy/number":
                                # Transform to Snips number
                                entity_name = "snips/number"
                            elif not entity_name:
                                # Collect actual value
                                assert (
                                    slot_name and slot_value
                                ), f"No slot name or value (name={slot_name}, value={slot_value})"

                                entity_name = slot_name
                                slot_values = slots_dict.get(slot_name)
                                if not slot_values:
                                    slot_values = []
                                    slots_dict[slot_name] = slot_values

                                slot_values.append(slot_value.strip())

                            # Reference slot/entity (values will be added later)
                            utterance.append(f"[{slot_name}:{entity_name}]")

                            # Reset current slot/entity
                            entity_name = None
                            slot_name = None
                            slot_value = None
                        elif olabel.startswith("__source__"):
                            # Use Rhasspy slot name as entity
                            entity_name = olabel[10:]

                    if ilabel:
                        # Add to current slot/entity value
                        if slot_name and (not entity_name):
                            slot_value += ilabel + " "
                        else:
                            # Add directly to utterance
                            utterance.append(ilabel)
                    elif (
                        olabel
                        and (not olabel.startswith("__"))
                        and slot_name
                        and (not slot_value)
                        and (not entity_name)
                    ):
                        slot_value += olabel + " "

                if utterance:
                    utterance_str = " ".join(utterance)
                    if utterance_str not in used_utterances:
                        # Write utterance
                        print("  -", quote(utterance_str), file=dataset_file)
                        used_utterances.add(utterance_str)

            print("", file=dataset_file)

        # Write entities
        for slot_name, values in slots_dict.items():
            if slot_name.startswith("$"):
                # Remove arguments and $
                slot_name = slot_name.split(",")[0][1:]

            # Skip numbers
            if slot_name in {"rhasspy/number"}:
                # Should have been converted already to snips/number
                continue

            # Keep only unique values
            values_set = set(values)

            print("---", file=dataset_file)
            print("type: entity", file=dataset_file)
            print("name:", quote(slot_name), file=dataset_file)
            print("values:", file=dataset_file)

            slot_graph = rhasspynlu.sentences_to_graph(
                {
                    slot_name: [
                        rhasspynlu.jsgf.Sentence.parse(value) for value in values_set
                    ]
                }
            )

            start_node, end_node = rhasspynlu.jsgf_graph.get_start_end_nodes(slot_graph)
            n_data = slot_graph.nodes(data=True)
            for path in nx.all_simple_paths(slot_graph, start_node, end_node):
                words = []
                for node in path:
                    node_data = n_data[node]
                    word = node_data.get("word")
                    if word:
                        words.append(word)

                if words:
                    print("  -", quote(" ".join(words)), file=dataset_file)

            print("", file=dataset_file)

        # ------------
        # Train engine
        # ------------

        if engine_path:
            # Delete existing engine
            engine_path = Path(engine_path)
            engine_path.parent.mkdir(exist_ok=True)

            if engine_path.is_dir():
                # Snips will fail it the directory exists
                _LOGGER.debug("Removing existing engine at %s", engine_path)
                shutil.rmtree(engine_path)
            elif engine_path.is_file():
                _LOGGER.debug("Removing unexpected file at %s", engine_path)
                engine_path.unlink()

        _LOGGER.debug("Training engine")
        dataset_file.seek(0)
        dataset = Dataset.from_yaml_files(language, [dataset_file])
        engine = engine.fit(dataset)

    if engine_path:
        # Save engine
        engine.persist(engine_path)
        _LOGGER.debug("Engine saved to %s", engine_path)

    return engine

# -----------------------------------------------------------------------------

def recognize(
    text: str,
    engine: SnipsNLUEngine,
    slots_dict: typing.Optional[typing.Dict[str, typing.List[str]]] = None,
    slot_graphs: typing.Optional[typing.Dict[str, nx.DiGraph]] = None,
    **parse_args,
) -> typing.List[Recognition]:
    """Recognize intent using Snips NLU."""
    result = engine.parse(text, **parse_args)
    intent_name = result.get("intent", {}).get("intentName")

    if not intent_name:
        # Recognition failure
        return []

    slots_dict = slots_dict or {}
    slot_graphs = slot_graphs or {}

    recognition = Recognition(
        text=text, raw_text=text, intent=Intent(name=intent_name, confidence=1.0)
    )

    # Replace Snips slot values with Rhasspy slot values (substituted)
    for slot in result.get("slots", []):
        slot_name = slot.get("slotName")
        slot_value_dict = slot.get("value", {})
        slot_value = slot_value_dict.get("value")

        entity = Entity(
            entity=slot_name,
            source=slot.get("entity", ""),
            value=slot_value,
            raw_value=slot.get("rawValue", slot_value),
            start=slot["range"]["start"],
            end=slot["range"]["end"],
        )
        recognition.entities.append(entity)

        if (not slot_name) or (not slot_value):
            continue

        slot_graph = slot_graphs.get(slot_name)
        if not slot_graph and (slot_name in slots_dict):
            # Convert slot values to graph
            slot_graph = rhasspynlu.sentences_to_graph(
                {
                    slot_name: [
                        rhasspynlu.jsgf.Sentence.parse(slot_line)
                        for slot_line in slots_dict[slot_name]
                        if slot_line.strip()
                    ]
                }
            )

            slot_graphs[slot_name] = slot_graph

        entity.tokens = slot_value.split()
        entity.raw_tokens = list(entity.tokens)

        if slot_graph:
            # Pass Snips value through graph
            slot_recognitions = rhasspynlu.recognize(entity.tokens, slot_graph)
            if slot_recognitions:
                # Pull out substituted value and replace in Rhasspy entitiy
                new_slot_value = slot_recognitions[0].text
                entity.value = new_slot_value
                entity.tokens = new_slot_value.split()

    return [recognition]

# -----------------------------------------------------------------------------

def quote(s):
    """Surround with quotes for YAML."""
    return f'"{s}"'


import pickle
from sklearn.manifold import TSNE

data = pickle.load(open('pickles/vectorlists.pickle', 'rb'))
data = data[1:]

for i in range(len(data)):
	data[i] = data[i][1:]

model = TSNE(learning_rate = 100, n_components = 3, n_iter=2000)
transformed = model.fit_transform(data)

#saving in pickle
with open('pickles/TSNE-trained.pickle', 'wb') as f:
	pickle.dump(transformed, f)

print ("Dataset trained for TSNE model")


from apscheduler.schedulers.blocking import BlockingScheduler
import time
import datetime
sched = BlockingScheduler()

@sched.scheduled_job('interval', seconds=10)
def timed_job():
    print('This job is run every three minutes.')
    print(time.ctime())

@sched.scheduled_job('cron', day_of_week='mon-fri', hour='16-19', minute='30-59', second='*/3')
def scheduled_job():
	print('This job is run every 3 mins')

print('before the start funciton')
sched.start()
print("let us figure out the situation")


from flask import Flask, request
import logging
import json
import random

app = Flask(__name__)

logging.basicConfig(level=logging.INFO)

cities = {
    'москва': ['1540737/daa6e420d33102bf6947', '213044/7df73ae4cc715175059e'],
    'нью-йорк': ['1652229/728d5c86707054d4745f', '1030494/aca7ed7acefde2606bdc'],
    'париж': ['1540737/355c802a54c0d3c2ce38','965417/429d880b7c62324751f1']
}

sessionStorage = {}

@app.route('/post', methods=['POST'])
def main():

    logging.info('Request: %r', request.json)

    response = {
        'session': request.json['session'],
        'version': request.json['version'],
        'response': {
            'end_session': False
        }
    }

    handle_dialog(response, request.json)

    logging.info('Request: %r', response)

    return json.dumps(response)

def handle_dialog(res, req):

    user_id = req['session']['user_id']

    if req['session']['new']:

        res['response']['text'] = 'Привет! Назови свое имя!'
        sessionStorage[user_id] = {
            'first_name': None,
            'game_started': False
        }

        return

    first_name = get_first_name(req)

    if sessionStorage[user_id]['first_name'] is None:

        if first_name is None:
            res['response']['text'] = 'Не раслышала имя. Повтори!'
        else:
            sessionStorage[user_id]['first_name'] = first_name
            sessionStorage[user_id]['guessed_cities'] = []
            res['response']['text'] = 'Приятно познакомиться, ' + first_name.title() + '. Я Алиса. ' \
                                                                                'Отгадаешь город по фото?'
            res['response']['buttons'] = [
                {
                    'title': 'Да',
                    'hide': True

                },
                {
                    'title': 'Нет',
                    'hide': True

                },
                {
                    'title': 'Помощь',
                    'hide': True
                }
            ]

    else:

        if not sessionStorage[user_id]['game_started']:

            if 'да' in req['request']['nlu']['tokens']:

                if len(sessionStorage[user_id]['guessed_cities']) == 3:

                    res['response']['text'] = 'Ты отгадал все города!'
                    res['end_session'] = True

                else:

                    sessionStorage[user_id]['game_started'] = True
                    sessionStorage[user_id]['attempt'] = 1
                    play_game(res, req)

            elif 'нет' in req['request']['nlu']['tokens']:
                res['response']['text'] = 'Ну и ладно!'
                res['end_session'] = True
            elif req['request']['original_utterance'].lower() == 'помощь':
                res['response']['text'] = 'Это текст помомщи. Будь смелее и продолжи общение.'
            else:
                res['response']['text'] = 'Не понял ответа! Так да или нет?'
                res['response']['buttons'] = [
                    {
                        'title': 'Да',
                        'hide': True

                    },
                    {
                        'title': 'Нет',
                        'hide': True

                    },
                    {
                        'title': 'Помощь',
                        'hide': True
                    }
                ]
        elif req['request']['original_utterance'].lower() == 'помощь':
                res['response']['text'] = 'Это текст помомщи. Будь смелее и продолжи общение.'
        else:

            play_game(res, req)

def play_game(res, req):

    user_id = req['session']['user_id']
    attempt = sessionStorage[user_id]['attempt']

    if attempt == 1:

        city = list(cities.keys())[random.randint(0, 2)]

        while (city in sessionStorage[user_id]['guessed_cities']):
            city = list(cities.keys())[random.randint(0, 2)]

        sessionStorage[user_id]['city'] = city

        res['response']['card'] = {}
        res['response']['card']['type'] = 'BigImage'
        res['response']['card']['title'] = 'Что это за город?'
        res['response']['card']['image_id'] = cities[city][attempt - 1]
        res['response']['buttons'] = [
            {
                'title': 'Помощь',
                'hide': True
            }
        ]

    else:

        city = sessionStorage[user_id]['city']

        if get_city(req) == city:

            res['response']['text'] = 'Правильно! Сыграем еще?'
            res['response']['buttons'] = [
                {
                    'title': 'Да',
                    'hide': True

                },
                {
                    'title': 'Нет',
                    'hide': True

                },
                {
                    'title': 'Помощь',
                    'hide': True
                }
            ]
            sessionStorage[user_id]['guessed_cities'].append(city)
            sessionStorage[user_id]['game_started'] = False
            return

        else:

            res['response']['text'] = 'Неправильно'
            if attempt == 3:
                res['response']['text'] = 'Вы пытались. Это ' + city.title() + '. Сыграем еще?'
                res['response']['buttons'] = [
                    {
                        'title': 'Да',
                        'hide': True

                    },
                    {
                        'title': 'Нет',
                        'hide': True

                    },
                    {
                        'title': 'Помощь',
                        'hide': True
                    }
                ]
                sessionStorage[user_id]['game_started'] = False
                sessionStorage[user_id]['guessed_cities'].append(city)
                return
            else:
                res['response']['card'] = {}
                res['response']['card']['type'] = 'BigImage'
                res['response']['card']['title'] = 'Неправильно. Вот тебе дополнительное фото'
                res['response']['card']['image_id'] = cities[city][attempt - 1]
                res['response']['buttons'] = [
                    {
                        'title': 'Помощь',
                        'hide': True
                    }
                ]

    sessionStorage[user_id]['attempt'] += 1

def get_city(req):

    for entity in req['request']['nlu']['entities']:

        if entity['type'] == 'YANDEX.GEO':

            if 'city' in entity['value'].keys():
                return entity['value']['city']
            else:
                return None

    return None

def get_first_name(req):

    for entity in req['request']['nlu']['entities']:

        if entity['type'] == 'YANDEX.FIO':

            if 'first_name' in entity['value'].keys():
                return entity['value']['first_name']
            else:
                return None
    return None

if __name__ == '__main__':
    app.run()

Алиса/1 - Помощь с угадыванием.py

1 - Помощь с угадыванием.py


"""vrushabhendra_portfolio URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from vrushabhendra_portfolioapp.views import *

urlpatterns = [
    path('admin/', admin.site.urls),
    path('',homepage,name='homepage'),
    path('projectmms/', project_mms_page, name='project_mms_page'),
    path('projectinstac/', project_instaclone_page, name='project_instaclone_page'),
    path('projectecom/', project_ecom_page, name='project_ecom_page'),
    path('webdevconnect/', mywebconnect_page, name='mywebconnect_page'),
    path('insertcontact/',insert_contactdata, name='Insert_contactdata'),
]


from __future__ import absolute_import, print_function
import os
import sys
import json
import time
import errno
import socket
import signal
import logging
import logging.config
import warnings
import threading
import traceback
import multiprocessing

from kuyruk import importer, signals
from kuyruk.task import get_queue_name
from kuyruk.exceptions import Reject, Discard, ConnectionError

logger = logging.getLogger(__name__)

class Worker(object):
    """Consumes tasks from a queue and runs them.

    :param app: An instance of :class:`~kuyruk.Kuyruk`
    :param args: Command line arguments

    """
    def __init__(self, app, args):
        self.kuyruk = app

        if not args.queues:
            raise ValueError("no queue given")

        self.queues = [get_queue_name(q, local=args.local) for q in args.queues]
        self.shutdown_pending = threading.Event()
        self._pause = False
        self._started = None
        self.consuming = False
        self._current_message = None
        self.current_task = None
        self.current_args = None
        self.current_kwargs = None
        self._heartbeat_exc_info = None
        if self.config.WORKER_MAX_LOAD is None:
            self.config.WORKER_MAX_LOAD = multiprocessing.cpu_count()

        self._pid = os.getpid()
        self._hostname = socket.gethostname()

        signals.worker_init.send(self.kuyruk, worker=self)

    @property
    def queue(self):
        warnings.warn("Worker.queue is deprecated. Use Worker.queues instead.")
        return self.queues[0]

    @property
    def config(self):
        return self.kuyruk.config

    def run(self):
        """Runs the worker and consumes messages from RabbitMQ.
        Returns only after `shutdown()` is called.

        """
        # Lazy import setproctitle.
        # There is bug with the latest version of Python with
        # uWSGI and setproctitle combination.
        # Watch: https://github.com/unbit/uwsgi/issues/1030
        from setproctitle import setproctitle
        setproctitle("kuyruk: worker on %s" % ','.join(self.queues))

        self._setup_logging()

        signal.signal(signal.SIGINT, self._handle_sigint)
        signal.signal(signal.SIGTERM, self._handle_sigterm)
        signal.signal(signal.SIGHUP, self._handle_sighup)
        signal.signal(signal.SIGUSR1, self._handle_sigusr1)
        signal.signal(signal.SIGUSR2, self._handle_sigusr2)

        self._started = os.times()[4]

        for f in (self._watch_load, self._shutdown_timer):
            t = threading.Thread(target=f)
            t.daemon = True
            t.start()

        signals.worker_start.send(self.kuyruk, worker=self)
        self._consume_messages()
        signals.worker_shutdown.send(self.kuyruk, worker=self)

        logger.debug("End run worker")

    def _setup_logging(self):
        if self.config.WORKER_LOGGING_CONFIG:
            logging.config.fileConfig(self.config.WORKER_LOGGING_CONFIG)
        else:
            level = getattr(logging, self.config.WORKER_LOGGING_LEVEL.upper())
            fmt = "%(levelname).1s " \
                  "%(name)s.%(funcName)s:%(lineno)d - %(message)s"
            logging.basicConfig(level=level, format=fmt)

    def _consume_messages(self):
        with self.kuyruk.channel() as ch:
            # Set prefetch count to 1. If we don't set this, RabbitMQ keeps
            # sending messages while we are already working on a message.
            ch.basic_qos(0, 1, True)

            self._declare_queues(ch)
            while not self.shutdown_pending.is_set():
                # Consume or pause
                if self._pause and self.consuming:
                    self._cancel_queues(ch)
                    logger.info('Consumer cancelled')
                    self.consuming = False
                elif not self._pause and not self.consuming:
                    self._consume_queues(ch)
                    logger.info('Consumer started')
                    self.consuming = True

                if self._heartbeat_exc_info:
                    break

                try:
                    ch.connection.send_heartbeat()
                    ch.connection.drain_events(timeout=1)
                except socket.error as e:
                    if isinstance(e, socket.timeout):
                        pass
                    elif e.errno == errno.EINTR:
                        pass  # happens when the process receives a signal
                    else:
                        raise
        logger.debug("End run worker")

    def _consumer_tag(self, queue):
        return "%s:%s@%s" % (queue, self._pid, self._hostname)

    def _declare_queues(self, ch):
        for queue in self.queues:
            logger.debug("queue_declare: %s", queue)
            ch.queue_declare(
                queue=queue, durable=True, auto_delete=False)

    def _consume_queues(self, ch):
        for queue in self.queues:
            logger.debug("basic_consume: %s", queue)
            ch.basic_consume(queue=queue,
                             consumer_tag=self._consumer_tag(queue),
                             callback=self._process_message)

    def _cancel_queues(self, ch):
        for queue in self.queues:
            logger.debug("basic_cancel: %s", queue)
            ch.basic_cancel(self._consumer_tag(queue))

    def _process_message(self, message):
        """Processes the message received from the queue."""
        if self.shutdown_pending.is_set():
            return

        try:
            description = json.loads(message.body)
        except Exception:
            message.channel.basic_reject(message.delivery_tag, requeue=False)
            logger.error("Cannot decode message. Dropping.")
        else:
            logger.info("Processing task: %r", description)
            self._process_description(message, description)

    def _process_description(self, message, description):
        try:
            task = importer.import_object(description['module'],
                                          description['function'])
            args, kwargs = description['args'], description['kwargs']
        except Exception:
            logger.error('Cannot import task')
            exc_info = sys.exc_info()
            signals.worker_failure.send(self.kuyruk, description=description,
                                        exc_info=exc_info, worker=self)
            message.channel.basic_reject(message.delivery_tag, requeue=False)
        else:
            self._process_task(message, description, task, args, kwargs)

    def _process_task(self, message, description, task, args, kwargs):
        try:
            self._current_message = message
            self.current_task = task
            self.current_args = args
            self.current_kwargs = kwargs
            stop_heartbeat = threading.Event()
            heartbeat_thread = threading.Thread(
                target=self._heartbeat_tick,
                args=(message.channel.connection, stop_heartbeat))
            heartbeat_thread.start()
            try:
                self._apply_task(task, args, kwargs)
            finally:
                self._current_message = None
                self.current_task = None
                self.current_args = None
                self.current_kwargs = None
                stop_heartbeat.set()
                heartbeat_thread.join()
        except Reject:
            logger.warning('Task is rejected')
            time.sleep(1)  # Prevent cpu burning
            message.channel.basic_reject(message.delivery_tag, requeue=True)
        except Discard:
            logger.warning('Task is discarded')
            message.channel.basic_reject(message.delivery_tag, requeue=False)
        except ConnectionError:
            pass
        except Exception:
            logger.error('Task raised an exception')
            exc_info = sys.exc_info()
            logger.error(''.join(traceback.format_exception(*exc_info)))
            signals.worker_failure.send(self.kuyruk, description=description,
                                        task=task, args=args, kwargs=kwargs,
                                        exc_info=exc_info, worker=self)
            message.channel.basic_reject(message.delivery_tag, requeue=False)
        else:
            logger.info('Task is successful')
            message.channel.basic_ack(message.delivery_tag)
        finally:
            logger.debug("Task is processed")

    @staticmethod
    def _apply_task(task, args, kwargs):
        """Logs the time spent while running the task."""
        if args is None:
            args = ()
        if kwargs is None:
            kwargs = {}

        start = time.time()
        try:
            task.apply(*args, **kwargs)
        finally:
            end = time.time()
            logger.info("%s finished in %i seconds." % (task.name, end - start))

    def _watch_load(self):
        """Pause consuming messages if lood goes above the allowed limit."""
        while not self.shutdown_pending.is_set():
            load = os.getloadavg()[0]
            if load > self.config.WORKER_MAX_LOAD:
                if self._pause is False:
                    logger.warning(
                        'Load is above the treshold (%.2f/%s), '
                        'pausing consumer', load, self.config.WORKER_MAX_LOAD)
                    self._pause = True
            else:
                if self._pause is True:
                    logger.warning(
                        'Load is below the treshold (%.2f/%s), '
                        'resuming consumer', load, self.config.WORKER_MAX_LOAD)
                    self._pause = False
            time.sleep(1)

    @property
    def uptime(self):
        if self._started is not None:
            return os.times()[4] - self._started

    def _shutdown_timer(self):
        """Counts down from MAX_WORKER_RUN_TIME. When it reaches zero sutdown
        gracefully.

        """
        if not self.config.WORKER_MAX_RUN_TIME:
            return

        while not self.shutdown_pending.is_set():
            remaining = self.config.WORKER_MAX_RUN_TIME - self.uptime
            if remaining > 0:
                time.sleep(remaining)
            else:
                logger.warning('Run time reached zero')
                self.shutdown()
                break

    def shutdown(self):
        """Exits after the current task is finished."""
        logger.warning("Shutdown requested")
        self.shutdown_pending.set()

    def _handle_sigint(self, signum, frame):
        """Shutdown after processing current task."""
        logger.warning("Catched SIGINT")
        self.shutdown()

    def _handle_sigterm(self, signum, frame):
        """Shutdown after processing current task."""
        logger.warning("Catched SIGTERM")
        self.shutdown()

    def _handle_sighup(self, signum, frame):
        """Used internally to fail the task when connection to RabbitMQ is
        lost during the execution of the task.

        """
        logger.warning("Catched SIGHUP")
        raise ConnectionError(self._heartbeat_exc_info)

    @staticmethod
    def _handle_sigusr1(signum, frame):
        """Print stacktrace."""
        print('=' * 70)
        print(''.join(traceback.format_stack()))
        print('-' * 70)

    def _handle_sigusr2(self, signum, frame):
        """Drop current task."""
        logger.warning("Catched SIGUSR2")
        if self._current_message:
            logger.warning("Dropping current task...")
            raise Discard

    def drop_task(self):
        os.kill(os.getpid(), signal.SIGUSR2)

    def _heartbeat_tick(self, connection, stop_event):
        while not stop_event.wait(1):
            try:
                connection.send_heartbeat()
            except socket.timeout:
                pass
            except Exception as e:
                logger.error(e)
                self._heartbeat_exc_info = sys.exc_info()
                os.kill(os.getpid(), signal.SIGHUP)
                break