# -*- coding: utf-8 -*-
import json
import string
from datetime import date, timedelta
from django.contrib.auth.models import User, Group
from django.core.exceptions import ValidationError
from django.db import models, IntegrityError
from django.utils.translation import gettext as _

from stationsplan.utils import random_string
from sp_app import utils

FAR_FUTURE = date(2099, 12, 31)
ONE_DAY = timedelta(days=1)

def date_to_json(date):
    return [date.year, date.month - 1, date.day]

class Company(models.Model):
    name = models.CharField(
        _("Name"),
        max_length=50,
        help_text='Name der Klinik (z.B. "Waldklinik Neustadt")',
    )
    shortname = models.CharField(
        _("Short Name"),
        max_length=10,
        help_text='Abkürzung für die Klinik (z.B. "wkns")',
    )
    region = models.ForeignKey(
        "Region",
        null=True,
        blank=True,
        related_name="companies",
        help_text=_(
            "Region that determines the legal holidays for this company"
        ),
        on_delete=models.PROTECT,
    )

    class Meta:
        verbose_name = _("Company")
        verbose_name_plural = _("Companies")

    def __str__(self):
        return self.name

class UniqueField:
    def unique_error_message(self, model_class, unique_check):
        if model_class == type(self) and len(unique_check) == 2:
            for field, field_name in self.unique_fields:
                if unique_check == (field, "company"):
                    return f'Der {field_name} "{getattr(self, field)}" ist bereits vergeben und darf nicht doppelt vorkommen.'
        return super().unique_error_message(model_class, unique_check)

class Department(models.Model):
    name = models.CharField(_("Name"), max_length=50)
    shortname = models.CharField(
        _("Short Name"), max_length=10, null=True, blank=True
    )
    company = models.ForeignKey(
        Company,
        related_name="departments",
        help_text=_("The top organizational unit"),
        on_delete=models.CASCADE,
    )

    class Meta:
        verbose_name = _("Department")
        verbose_name_plural = _("Departments")

    def __str__(self):
        if self.shortname:
            return f"{self.name} ({self.shortname})"
        return self.name

class Ward(UniqueField, models.Model):
    name = models.CharField(_("Name"), max_length=50)
    shortname = models.CharField(_("Short Name"), max_length=10)
    max = models.IntegerField(help_text=_("maximum staffing"))
    min = models.IntegerField(help_text=_("minimum staffing"))
    everyday = models.BooleanField(
        _("everyday"),
        default=False,
        help_text=_("if True, is to be planned also on free days."),
    )
    freedays = models.BooleanField(
        _("freedays"),
        default=False,
        help_text=_("if True, is to be planned only on free days."),
    )
    weekdays = models.CharField(
        _("weekdays"),
        max_length=7,
        default="",
        blank=True,
        help_text=_(
            "Days of the week when this is to be planned. "
            "(String of digits, 0  for sunday.)"
        ),
    )
    on_different_days = models.BooleanField(
        _("on_different_days"),
        default=False,
        help_text=_("This function can be planned on different days"),
    )
    callshift = models.BooleanField(
        _("callshift"),
        default=False,
        help_text=_("if True, then this function is displayed as call shift"),
    )
    on_leave = models.BooleanField(
        _("on_leave"),
        default=False,
        help_text=_("if True, then persons planned for this are on leave"),
    )
    departments = models.ManyToManyField(
        Department,
        verbose_name=_("Departments"),
        related_name="wards",
        help_text=_("Departments whose schedule contains this ward"),
    )
    company = models.ForeignKey(
        Company, related_name="wards", on_delete=models.CASCADE
    )
    position = models.IntegerField(
        default=1,
        help_text=_(
            "Ordering in the display. Should not be more than two digits."
        ),
    )
    ward_type = models.CharField(
        _("Ward type"),
        max_length=50,
        blank=True,
        default="",
        help_text=_("For sorting the CallTallies"),
    )
    approved = models.DateField(
        default=FAR_FUTURE,
        null=True,
        blank=True,
        help_text=_("The date until which the plan is approved"),
    )
    after_this = models.ManyToManyField(
        "self",
        verbose_name=_("after this"),
        symmetrical=False,
        blank=True,
        help_text=_(
            "if not empty, "
            "only these functions can be planned on the next day"
        ),
        related_name="predecessor",
    )
    not_with_this = models.ManyToManyField(
        "self",
        verbose_name=_("not with this"),
        symmetrical=False,
        blank=True,
        help_text=_("these functions can not be planned on the same day"),
        related_name="shadowing",
    )
    weight = models.IntegerField(
        default=0,
        help_text=_(
            "if this is a call shift, the weight reflects its "
            "burden on the persons doing the shift"
        ),
    )
    active = models.BooleanField(
        _("Active"),
        default=True,
        help_text=_("This function should currently be displayed"),
    )
    in_ical_feed = models.BooleanField(
        _("Part of ical feed"),
        default=False,
        help_text=_("This function should be part of the ical feed"),
    )
    unique_fields = (("name", "Name"), ("shortname", "Kurzname"))

    class Meta:
        verbose_name = _("Task")
        verbose_name_plural = _("Tasks")
        ordering = ["position", "name"]
        unique_together = [["name", "company"], ["shortname", "company"]]

    def __str__(self):
        return self.name

    def toJson(self):
        res = {
            "name": self.name,
            "shortname": self.shortname,
            "id": self.id,
            "min": self.min,
            "max": self.max,
            "everyday": self.everyday,
            "freedays": self.freedays,
            "weekdays": self.weekdays,
            "on_different_days": self.on_different_days,
            "callshift": self.callshift,
            "on_leave": self.on_leave,
            "company_id": self.company_id,
            "position": "%02d" % self.position,
            "after_this": "",
            "not_with_this": "",
            "ward_type": self.ward_type,
            "weight": self.weight,
            "active": self.active,
        }
        if self.pk:
            res["after_this"] = ",".join(
                w.shortname for w in self.after_this.all()
            )
            res["not_with_this"] = ",".join(
                w.shortname for w in self.not_with_this.all()
            )
        if self.approved < FAR_FUTURE:
            res["approved"] = date_to_json(self.approved)
        return res

    def clean(self):
        if self.shortname == "id":
            raise ValidationError(
                {"shortname": _('Wards cannot have the shortname "id".')}
            )
        if "," in self.shortname:
            raise ValidationError(
                {
                    "shortname": _(
                        "Wards cannot have a comma in their shortname."
                    )
                }
            )

class DifferentDay(models.Model):
    """A ward can be planned or not planned out of schedule"""

    day = models.DateField(_("day"), help_text=_("Day that is different"))
    ward = models.ForeignKey(
        Ward, related_name="different_days", on_delete=models.CASCADE
    )
    added = models.BooleanField(
        _("additional"), help_text=_("Planning is additional (not cancelled)")
    )

    class Meta:
        verbose_name = _("Different Day")
        verbose_name_plural = _("Different Days")

class Person(UniqueField, models.Model):
    """A person (worker) who can be planned for work"""

    POSITION_ASSISTENTEN = 1
    POSITION_OBERAERZTE = 2
    POSITION_CHEFAERZTE = 80
    POSITION_ANONYM = 4
    POSITION_EXTERNE = 5
    POSITION_CHOICES = (
        (POSITION_ASSISTENTEN, "Assistenten"),
        (POSITION_OBERAERZTE, "Oberärzte"),
        (POSITION_CHEFAERZTE, "Chefärzte"),
        (POSITION_ANONYM, "Anonym (Abteilung)"),
        (POSITION_EXTERNE, "Externe"),
    )

    name = models.CharField(_("Name"), max_length=50)
    shortname = models.CharField(_("Short Name"), max_length=10)
    start_date = models.DateField(
        _("start date"),
        default=utils.get_first_of_month,
        help_text=_("begin of job"),
    )
    end_date = models.DateField(
        _("end date"), default=FAR_FUTURE, help_text=_("end of job")
    )
    departments = models.ManyToManyField(
        Department, verbose_name=_("Departments"), related_name="persons"
    )
    company = models.ForeignKey(
        Company, related_name="persons", null=True, on_delete=models.CASCADE
    )
    functions = models.ManyToManyField(
        Ward,
        related_name="staff",
        verbose_name=_("Tasks"),
        help_text=_("Functions that he or she can  perform."),
    )
    position = models.IntegerField(
        _("position"),
        default=POSITION_ASSISTENTEN,
        choices=POSITION_CHOICES,
        help_text=_(
            "Ordering in the display. "
            "Should not be more than two digits. "
            "A number greater than 80 means Head of Department"
        ),
    )
    anonymous = models.BooleanField(
        _("Anonymous"),
        default=False,
        help_text=_(
            "if True this person represents multiple other persons, "
            "e.g. a department"
        ),
    )
    email = models.EmailField(_("Email"), null=True, blank=True)
    unique_fields = (("name", "Name"), ("shortname", "Kurzname"))

    class Meta:
        verbose_name = _("Person")
        verbose_name_plural = _("Persons")
        unique_together = [["name", "company"], ["shortname", "company"]]

    def __str__(self):
        return f"{self.name} ({self.shortname})"

    def toJson(self):
        return {
            "name": self.name,
            "shortname": self.shortname,
            "id": self.id,
            "start_date": date_to_json(self.start_date),
            "end_date": date_to_json(self.end_date),
            "functions": [f.shortname for f in self.functions.all()],
            "departments": [d.id for d in self.departments.all()],
            "position": "%02d" % self.position,
            "anonymous": self.anonymous,
            "external": self.position == Person.POSITION_EXTERNE,
        }

    def save(self, *args, **kwargs):
        created = not self.pk
        if created and self.position == Person.POSITION_ANONYM:
            self.anonymous = True
        super(Person, self).save(*args, **kwargs)
        # When a person leaves, their plannings should stop.
        # Plannings with an end (like vacations) are left untouched
        # in case the person changes their mind
        if self.end_date < FAR_FUTURE:
            Planning.objects.filter(
                person=self, start__gt=self.end_date, end=FAR_FUTURE
            ).delete()
            Planning.objects.filter(person=self, end=FAR_FUTURE).update(
                end=self.end_date
            )
        # every person can be on leave
        if created and not self.anonymous:
            self.functions.add(
                *list(
                    Ward.objects.filter(
                        company_id=self.company_id, on_leave=True
                    )
                )
            )

    def current(self):
        return self.end_date >= date.today()

    def inactivate_older_feeds(self, feed_id):
        feedids = list(FeedId.objects.filter(person=self, active=True))
        if len(feedids) < 2:
            return
        current_feed = next(f for f in feedids if f.uid == feed_id)
        older = []
        for f in feedids:
            if f.pk < current_feed.pk:
                f.active = False
                older.append(f)
        if older:
            FeedId.objects.bulk_update(older, ["active"])

class ChangeLogging(models.Model):
    """Logs who has made which changes.
    The change can be for one day (continued==False) or continued
    If continued==True it can have an end date (`until`)
    """

    company = models.ForeignKey(Company, on_delete=models.CASCADE)
    user = models.ForeignKey(User, on_delete=models.CASCADE)
    person = models.ForeignKey(Person, on_delete=models.CASCADE)
    ward = models.ForeignKey(Ward, on_delete=models.CASCADE)
    day = models.DateField()
    added = models.BooleanField()
    continued = models.BooleanField(
        default=True, help_text="If False the change is valid for one day."
    )
    until = models.DateField(
        null=True,
        blank=True,
        help_text="The last day the change is valid. Can be blank.",
    )
    description = models.CharField(max_length=255)
    json = models.CharField(max_length=255)
    change_time = models.DateTimeField(auto_now=True)
    version = models.IntegerField(default=0)
    current_version = 1

    class Meta:
        verbose_name = _("ChangeLogging")
        verbose_name_plural = _("ChangeLoggings")

    def toJson(self):
        data = {
            "person": self.person.shortname,
            "ward": self.ward.shortname,
            "day": self.day.strftime("%Y%m%d"),
            "action": "add" if self.added else "remove",
            "continued": self.continued,
            "pk": self.pk,
        }
        if self.until:
            data["until"] = self.until.strftime("%Y%m%d")
        return data

    def get_json_for_history(self):
        return {
            "user": self.user.get_full_name() or self.user.get_username(),
            "person": self.person.shortname,
            "ward": self.ward.shortname,
            "day": self.day,
            "added": self.added,
            "continued": self.continued,
            "until": self.until,
            "change_time": self.change_time,
        }

    def save(self, *args, **kwargs):
        self.make_description()
        self.version = self.current_version
        super(ChangeLogging, self).save(*args, **kwargs)
        # self.json contains the primary key
        self.json = json.dumps(self.toJson())
        ChangeLogging.objects.filter(pk=self.pk).update(json=self.json)

    def make_description(self):
        day = self.day.strftime("%d.%m.%Y")
        if not self.continued or (self.until and self.until == self.day):
            # am 19.11.2019
            time = f"am {day}"
        elif self.until:
            # von 19.11.2019 bis 20.11.2019
            time = f"von {day} bis {self.until.strftime('%d.%m.%Y')}"
        else:
            # ab 19.11.2019
            time = f"ab {day}"
        self.description = (
            f"{self.user.last_name or self.user.get_username()}: "
            f"{self.person.name} ist {time} "
            f'{"" if self.added else "nicht mehr "}für {self.ward.name}'
            f" eingeteilt"
        )

    def __str__(self):
        return self.description

def process_change(cl):
    """Weave the change into the existing Plannings.

    Return the json dict of the effective change to be returned to the client.
    """
    plannings = Planning.objects.filter(
        person_id=cl.person_id, ward_id=cl.ward_id
    )
    pl_data = dict(person_id=cl.person_id, ward_id=cl.ward_id)
    if cl.added:
        if cl.continued:
            end = cl.until or FAR_FUTURE
            plannings = list(
                plannings.filter(
                    end__gte=cl.day,
                    start__lte=end,
                ).order_by("start")
            )
            if len(plannings):
                # if cl.until is given and is before the end of the last
                #   overlapping planning
                # else it ends with the first planning in this period
                if cl.until:
                    last_planning = plannings[-1]
                    if cl.until < last_planning.end:
                        end = cl.until = last_planning.start - ONE_DAY
                        cl.save()
                        plannings.pop()
                else:
                    end = plannings[0].start - ONE_DAY
                if cl.day > end:  # This can happen if cl.until==None and
                    return {}  # change is in an existing planning
            pl = Planning.objects.create(start=cl.day, end=end, **pl_data)
            if cl.until and len(plannings) > 0:
                Planning.objects.filter(
                    id__in=[_pl.id for _pl in plannings]
                ).update(superseded_by=pl)
        else:  # not continued, one day
            plannings = plannings.filter(start__lte=cl.day, end__gte=cl.day)
            if len(plannings) == 0:
                Planning.objects.create(start=cl.day, end=cl.day, **pl_data)
            else:
                return {}  # change is contained in a Planning

    else:  # removed
        plannings = plannings.filter(
            start__lte=cl.until or cl.day, end__gte=cl.day
        ).order_by("start")
        if len(plannings) == 0:
            return {}
        if cl.continued:
            if cl.until:
                for pl in plannings:
                    if pl.start < cl.day:
                        if pl.end > cl.until:
                            Planning.objects.create(
                                start=cl.until + ONE_DAY, end=pl.end, **pl_data
                            )
                        pl.end = cl.day - ONE_DAY
                        pl.save()
                    elif pl.end > cl.until:
                        pl.start = cl.until + ONE_DAY
                        pl.save()
                    else:
                        pl.delete()  # cave: no undo!!
            else:
                for pl in plannings:  # should be only one planning
                    if pl.start == cl.day:
                        pl.delete()
                    else:
                        pl.end = cl.day - ONE_DAY
                        pl.save()
        else:  # not continued, one day
            for pl in plannings:
                if pl.start == pl.end:  # pl is one day
                    pl.delete()
                elif pl.start == cl.day:  # cut off first day
                    pl.start = cl.day + ONE_DAY
                    pl.save()
                else:
                    if not pl.end == cl.day:
                        Planning.objects.create(
                            start=cl.day + ONE_DAY, end=pl.end, **pl_data
                        )
                    pl.end = cl.day - ONE_DAY
                    pl.save()

    return json.loads(cl.json)

class Planning(models.Model):
    """One time period, where one person is planned for one ward.

    Plannings for the same person and ward should not overlap.
    The time period includes 'start' and 'end',
    i.e. a planning for one day has 'start' and 'end' set to the same date.
    """

    company = models.ForeignKey(Company, on_delete=models.CASCADE)
    person = models.ForeignKey(Person, on_delete=models.CASCADE)
    ward = models.ForeignKey(Ward, on_delete=models.CASCADE)
    start = models.DateField()
    end = models.DateField(default=FAR_FUTURE)
    json = models.CharField(max_length=255)
    version = models.IntegerField(default=0)
    current_version = 1
    superseded_by = models.ForeignKey(
        "self",
        on_delete=models.SET_NULL,
        blank=True,
        null=True,
        related_name="supersedes",
        default=None,
        help_text=_("Later planning that supersedes this one"),
    )
    updateddate = models.DateTimeField(auto_now=True)

    class Meta:
        verbose_name = _("Planning")
        verbose_name_plural = _("Plannings")

    def toJson(self):
        return {
            "person": self.person_id,
            "ward": self.ward_id,
            "start": self.start.strftime("%Y%m%d"),
            "end": self.end.strftime("%Y%m%d"),
        }

    def save(self, *args, **kwargs):
        if not self.pk:
            self.company_id = self.person.company_id
        self.json = json.dumps(self.toJson())
        self.version = self.current_version
        # the planning should not exceed the persons stay
        self.end = min(self.end, self.person.end_date)
        super(Planning, self).save(*args, **kwargs)

    def __str__(self):
        if self.end == FAR_FUTURE:
            return (
                f"{self.person.name} ist ab {self.start:%d.%m.%Y} "
                f"für {self.ward.name} geplant."
            )
        return (
            f"{self.person.name} ist von {self.start:%d.%m.%Y} bis "
            f"{self.end:%d.%m.%Y} für {self.ward.name} geplant."
        )

EMPLOYEE_PERMISSIONS = [
    ("is_company_admin", "is admin of the company"),
    ("is_dep_lead", "is leader of a department"),
    ("is_editor", "is editor for a department"),
]
EMPLOYEE_LEVEL = {
    "is_company_admin": "Admin",
    "is_dep_lead": "Abteilungsleiter/in",
    "is_editor": "Planen",
    "None": "Lesen",
}
EMPLOYEE_GROUP = {
    "is_company_admin": "Company Admin",
    "is_dep_lead": "Department admins",
    "is_editor": "Editors",
}

class Employee(models.Model):
    """Somebody who uses the plan.
    Can be anyone who works there, but also other involved personnel,
    like management etc.
    """

    user = models.OneToOneField(
        User, related_name="employee", on_delete=models.CASCADE
    )
    departments = models.ManyToManyField(
        Department, related_name="employees", verbose_name=_("Department")
    )
    company = models.ForeignKey(
        Company, related_name="employees", on_delete=models.CASCADE
    )
    _level = models.CharField("level", max_length=20, null=True)

    class Meta:
        verbose_name = _("Employee")
        verbose_name_plural = _("Employees")
        permissions = EMPLOYEE_PERMISSIONS

    def __str__(self):
        return "; ".join(
            (
                self.user.get_full_name() or self.user.get_username(),
                ", ".join([d.name for d in self.departments.all()]),
                self.company.name,
            )
        )

    def get_name(self):
        return self.user.get_full_name() or self.user.get_username()

    name = property(fget=get_name)

    def get_level(self):
        """Return level of authorization"""
        return self._level

    def set_level(self, level=None):
        """Set level of authorization

        If level is None or "None", remove all special permissions
        """
        assert level in EMPLOYEE_LEVEL or level in (None, "None")
        groups = dict((g.name, g) for g in Group.objects.all())
        self.user.groups.remove(
            *(groups[name] for name in EMPLOYEE_GROUP.values())
        )
        if level not in (None, "None"):
            # set all permissions up to the given level
            level_found = False
            for permission, docstring in EMPLOYEE_PERMISSIONS:
                if permission == level:
                    level_found = True
                if level_found:
                    self.user.groups.add(groups[EMPLOYEE_GROUP[permission]])
        # account for permission caching
        self.user = User.objects.get(id=self.user.id)
        self._level = None if level == "None" else level
        self.save()

    level = property(fget=get_level, fset=set_level)

    def name_and_level(self):
        # level = self.get_level()
        # if level is None:
        #     return self.name
        return f"{self.name} ({EMPLOYEE_LEVEL[self.level or 'None']})"

class StatusEntry(models.Model):
    """Saves some detail about the current status of the planning
    or the program

    Currently is only for setting the approved date of wards
    """

    name = models.CharField(max_length=30)
    content = models.CharField(max_length=255)
    department = models.ForeignKey(
        Department,
        related_name="status_entries",
        null=True,
        blank=True,
        help_text=_("Can be empty"),
        on_delete=models.SET_NULL,
    )
    company = models.ForeignKey(
        Company,
        related_name="status_entries",
        null=True,
        blank=True,
        help_text=_("Can be empty"),
        on_delete=models.SET_NULL,
    )
    timestamp = models.DateTimeField(null=True, auto_now_add=True)

    def __str__(self):
        return f"{self.name}: {self.content}"

class CalculatedHoliday(models.Model):
    """A Holiday with calculated dates

    If mode is 'abs', the holiday is on that day and month,
    if year is not None, it is on that specific date.

    If mode is 'rel', then the holiday is 'day' days before or after easter.
    """

    name = models.CharField(_("Name"), max_length=50)
    mode = models.CharField(
        _("Mode"),
        max_length=3,
        choices=(("abs", _("Absolute")), ("rel", _("Easter-related"))),
    )
    day = models.IntegerField(
        _("Day"), help_text=_("day of month or distance from Easter in days")
    )
    month = models.IntegerField(_("Month"), null=True, blank=True)
    year = models.IntegerField(_("Year"), null=True, blank=True)

    class Meta:
        verbose_name = _("Calculated Holiday")
        verbose_name_plural = _("Calculated Holidays")

    def __str__(self):
        if self.mode == "abs":
            return f"{self.name}: {self.day}.{self.month}.{self.year or ''}"
        if self.day >= 0:
            return f"{self.name}: {self.day} Tage nach Ostern"
        else:
            return f"{self.name}: {-self.day} Tage vor Ostern"

    def toJson(self):
        return {
            "name": self.name,
            "mode": self.mode,
            "day": self.day,
            "month": self.month or 0,
            "year": self.year or 0,
        }

class Region(models.Model):
    """Aggregates the usual holidays for that region"""

    name = models.CharField(_("Name"), max_length=50)
    shortname = models.CharField(_("Short Name"), max_length=10, unique=True)
    # holidays = models.ManyToManyField(Holiday, verbose_name=_('Holidays'),
    #                                   related_name='regions')
    calc_holidays = models.ManyToManyField(
        CalculatedHoliday, verbose_name=_("Holidays"), related_name="regions"
    )

    class Meta:
        verbose_name = _("Region")
        verbose_name_plural = _("Regions")
        ordering = ["name"]

    def __str__(self):
        return self.name

class FeedId(models.Model):
    """The Id for the URL of an ical feed"""

    uid = models.CharField("ID", max_length=20, unique=True)
    person = models.ForeignKey(
        Person, related_name="feed_ids", on_delete=models.CASCADE
    )
    active = models.BooleanField(default=True)

    class Meta:
        verbose_name = "FeedId"
        verbose_name_plural = "FeedIds"

    def __str__(self):
        pass

    @classmethod
    def new(cls, person):
        """Return a new feed-id for a person.

        The old ones stay active
        """
        while True:
            try:
                uid = random_string(12, string.ascii_letters + string.digits)
                feed = FeedId.objects.create(uid=uid, person=person)
                return feed
            except IntegrityError:
                # uid was already taken
                pass


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jul  5 13:25:27 2019

@author: dcaulley
"""

import pickle
import cv2

import os, sys
import subprocess

import numpy as np
import scipy.spatial.distance as dist

from keras_vggface.vggface import VGGFace

exp_dir            = sys.argv[1]
curr_vid           = sys.argv[2]
template_file      = sys.argv[3]
scoring_type       = sys.argv[4]
threshold          = float(sys.argv[5])

pywork_dir      = os.path.join(exp_dir, 'pywork', curr_vid)
pycrop_dir      = os.path.join(exp_dir, 'pycrop', curr_vid)
pycrop_dir      = '"%s"'%pycrop_dir

faces_file      = os.path.join(pywork_dir,     'faces.pckl')
tracks_file     = os.path.join(pywork_dir,    'tracks.pckl')
alltracks_file  = os.path.join(pywork_dir, 'alltracks.pckl')

offsets_file     = os.path.join(pywork_dir,      'offsets.txt')
activesd_file    = os.path.join(pywork_dir,    'activesd.pckl')
frame_confs_file = os.path.join(pywork_dir, 'frame_confs.pckl')

template_faces = np.load(template_file)

with open(tracks_file, 'rb') as fil:
    track = pickle.load(fil)

with open(faces_file, 'rb') as fil:
    faces = pickle.load(fil)

with open(alltracks_file, 'rb') as fil:
    alltracks = pickle.load(fil)

offsets = np.loadtxt(offsets_file, dtype=str)
confs   = offsets[1:,2].astype(float)
offs    = offsets[1:,1].astype(float)
faces = audio_lip_tracks(tracks, actives_d, frame_confs, offs, confs)

empty_space_list_start = []
empty_space_list_end   = []

resnet50_features = VGGFace(model='resnet50', include_top=False, input_shape=(224, 224, 3),
                            pooling='avg')

def face_verif(features):
    scores = []
    u = features
    for v in template_faces.T:
        if scoring_type == 'cosine':
            scores.append(dist.cosine(u, v))
        else:
            scores.append(dist.euclidean(u, v))
    return np.mean(scores)

def img_reformat(img):
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img = cv2.resize(img, (224,224))
    img = np.expand_dims(img, axis=0)
    return img

def audio_lip_tracks(tracks, dists, frame_confs, offs, confs):
    faces = [ [] for i in range(1000000) ]

    for ii, track in enumerate(tracks):
        if abs(offs[ii]) > 5:
            continue

        for ij, frame in enumerate(track[0][0].tolist()) :
            try:
                if frame_confs[ii][ij] > 1.5:
                    faces[frame].append([ii, frame_confs[ii][ij]])
            except:
                continue
    try:
        faces = faces[:frame]
    except:
        pass
    return faces

#cap = cv2.VideoCapture('/Users/dcaulley/projects/audio_visual_diarization/research_exp/pyavi/out_out/video.avi')

frame_num = 1
all_frames = []
all_rects  = []
for tr in alltracks:
    frames_ = tr[0]
    rects_  = tr[1]

    #pts = np.random.randint(len(frames_),size=10)

    pts = np.arange(len(frames_))
    np.random.shuffle(pts)
    pts = pts[:20]
    pts.sort()

    frames_ = frames_[pts]
    rects_  = rects_[pts]

    all_frames.append(frames_)
    all_rects.append(rects_)
all_frames = np.array(all_frames)

frame_num = 0

cap = cv2.VideoCapture(os.path.join(exp_dir, 'pyavi', curr_vid, 'video.avi'))
fw, fh = cap.get(3), cap.get(4)

face_scores = np.zeros(all_frames.shape, dtype=float)
while True:
    ret, frame = cap.read()
    rows_, cols_ = np.where(all_frames == frame_num)

    for r, c in zip(rows_, cols_):
        y_s, x_s, y_e, x_e = all_rects[r][c,:]
        y_s, x_s, y_e, x_e = int(y_s*fh), int(x_s*fw), int(y_e*fh), int(x_e*fw)
        face = frame[y_s:y_e, x_s:x_e, :]
        face = img_reformat(face)
        features = resnet50_features.predict(face)

        face_scores[r,c] = face_verif(features)

    frame_num +=1

    if not ret:
        break

face_scores = np.mean(face_scores, 1)

new_tracks = []
d = 0
for k, score in enumerate(face_scores):
    if score > threshold:
        k = 2


#순열 조합 , :https://shoark7.github.io/programming/algorithm/Permutations-and-Combinations

import itertools
mylist = [1,2,3]
result = itertools.combinations(mylist, r=2)
result = itertools.combinations_with_replacement(mylist, r=2)

from collections import deque
que = deque(['bb','cc'])
que.append('aaa')
que.popleft()

#회전
def rotate_90(map):
    N = len(map)
    rot = [[0] *N for _ in range(N)]
    for r in range(N):
        for c in range(N):
            rot[c][N-1-r] = map[r][c]

    return rot

def rotate_180(map):
    N = len(map)
    rot = [[0] * N for _ in range(N)]
    for r in range(N):
        for c in range(N):
            rot[N-1-r][N - 1 - c] = map[r][c]

    return rot

def rotate_270(map):
    N = len(map)
    rot = [[0] * N for _ in range(N)]
    for r in range(N):
        for c in range(N):
            rot[N-1-c][r] = map[r][c]

    return rot

# DFS
def dfs(graph, v, visited):
    visited[v] = True
    for i in graph[v]:
        if not visited[i]:
            dfs(graph, i, visited)

# BFS
def bfs(graph, start, visited):
    from collections import deque
    queue = deque([start])
    visited[start] = True
    while queue:
        v = queue.popleft()
        for i in graph[v]:
            if not visited[i]:
                queue.append(i)
                visited[i] = True

#heapq
import heapq

# 정렬 quick sort
def quick_sort(array, start, end):
    if start>= end:
        return
    pivot = start
    left = start +1
    right = end

    while left <= right:
        while left <= end and array[left] <= array[pivot]:
            left +=1

        while right > start and array[right] >= array[pivot]:
            right -=1

        if left >right:
            array[right], array[pivot] = array[pivot], array[right]
        else:
            array[left], array[right] = array[right], array[left]

        quick_sort(array, start, right-1)
        quick_sort(array, right+1, end)

# 최단거리 _ 다익스트라 알고리즘
n =10
distance = [1e9] * n # 10: 노드 개수
graph = [[] for i in range(n+1)]
def dijksta(start):
    import heapq
    q = []
    heapq.heappush(q, (0,start))
    while q:
        now, dist = heapq.heappop(q)
        if distance[now] < dist:
            continue
        for i in graph[now]:
            cost = dist + i[1]
            if cost < distance[i[0]]:
                distance[i[0]] = cost
                heapq.heappush(q, (cost,i[0]))
"""
#최소 신장 트리
"""
# 특정 원소가 속한 집합을 찾기
def find_parent(parent, x):
    # 루트 노드가 아니라면, 루트 노드를 찾을 때까지 재귀적으로 호출
    if parent[x] != x:
        parent[x] = find_parent(parent, parent[x])
    return parent[x]

# 두 원소가 속한 집합을 합치기
def union_parent(parent, a, b):
    a = find_parent(parent, a)
    b = find_parent(parent, b)
    if a < b:
        parent[b] = a
    else:
        parent[a] = b

# 노드의 개수와 간선(Union 연산)의 개수 입력 받기
v, e = map(int, input().split())
parent = [0] * (v + 1) # 부모 테이블 초기화하기

# 모든 간선을 담을 리스트와, 최종 비용을 담을 변수
edges = []
result = 0

# 부모 테이블상에서, 부모를 자기 자신으로 초기화
for i in range(1, v + 1):
    parent[i] = i

# 모든 간선에 대한 정보를 입력 받기
for _ in range(e):
    a, b, cost = map(int, input().split())
    # 비용순으로 정렬하기 위해서 튜플의 첫 번째 원소를 비용으로 설정
    edges.append((cost, a, b))

# 간선을 비용순으로 정렬
edges.sort()

# 간선을 하나씩 확인하며
for edge in edges:
    cost, a, b = edge
    # 사이클이 발생하지 않는 경우에만 집합에 포함
    if find_parent(parent, a) != find_parent(parent, b):
        union_parent(parent, a, b)
        result += cost

"""
# 진수 변환 - 210524_01
"""
#10 진수 -> 2, 8, 16 진수
value = 30
b = format(value, 'b')
o = format(value, 'o')
h = format(value, 'x')

#2,8,16 -> 10 진수
value = 101010
int_b = int(str(value), 2)
int_o = int(str(value), 8)
int_x = int(str(value), 16)


import os
import sys
import pika
import logging
import time
import signal
from enum import IntEnum
from colorformatter import ColorFormatter

class HasToken(IntEnum):
    NONE = 0
    PING = 1
    PONG = 2
    BOTH = 3

class TokenType(IntEnum):
    PING = 1
    PONG = 2

class Token:
    def __init__(self, type_: TokenType, value: int):
        if not isinstance(type_, TokenType):
            raise TypeError("Token type must be an instance of TokenType")
        self.type = type_
        self.value = value

    @classmethod
    def from_bytes(cls, values: bytes):
        type_, value = values.split(b' ')
        return cls(TokenType(int(type_)), int(value))

    def values(self):
        return '{} {}'.format(self.type.value, self.value)

    def __str__(self):
        return "Token ({}, {})".format(self.type.name, self.value)

class Node:
    def __init__(self):
        self.m = 0
        self.hasToken = HasToken.NONE
        self.ping_token = None
        self.pong_token = None
        self.tokens_to_lose = set()
        self.ID = int(os.uname().nodename[4:])
        self.logger = logging.getLogger(__file__)
        self.logger.setLevel(logging.DEBUG)
        handler = logging.StreamHandler()
        handler.setLevel(logging.INFO)
        formatter = ColorFormatter(
            '%(asctime)s:%(msecs)3d - %(levelname)-7s - %(message)s',
            datefmt='%M:%S')
        handler.setFormatter(formatter)
        self.logger.addHandler(handler)

        with open('/etc/hosts', 'r') as f:
            nodes = [l.split(' ')[1] for l in f.readlines()
                     if l.startswith('192.168.10.')]

        neighbor = nodes[self.ID % len(nodes)]
        self.remote_conn = pika.BlockingConnection(
            pika.ConnectionParameters(host=neighbor))
        self.remote_channel = self.remote_conn.channel()

        self.local_conn = pika.BlockingConnection(
            pika.ConnectionParameters(host='localhost'))
        self.local_channel = self.local_conn.channel()
        self.local_channel.queue_declare(queue='token_queue', exclusive=True)

        self.local_channel.basic_consume(self.receive_token, no_ack=True,
                                         queue='token_queue')

    def pass_token(self, token: Token):
        self.m = token.value
        if token.type == TokenType.PONG:
            time.sleep(0.5)
            self.pong_token = None
        else:
            self.ping_token = None
        self.hasToken = HasToken(self.hasToken - token.type)

        if token.type in self.tokens_to_lose:
            self.logger.critical("Lost {} token!".format(token.type.name))
            self.tokens_to_lose.remove(token.type)
        else:
            self.remote_channel.publish(exchange='', routing_key='token_queue',
                                        body=token.values())

    def receive_token(self, ch, method, props, body):
        token = Token.from_bytes(body)
        self.handle_token(token)

    def try_receive_token(self):
        msg = self.local_channel.basic_get(queue='token_queue', no_ack=True)
        if msg[0] is None:
            return False
        self.handle_token(Token.from_bytes(msg[2]))
        return True

    def handle_token(self, token: Token):
        if abs(token.value) < self.m:  # old token
            self.logger.warning(
                "Received old token ({}); deleting".format(token))
            return

        self.logger.info("Received token: {}".format(token))

        try:
            assert self.hasToken not in (HasToken(token.type), HasToken.BOTH)
        except AssertionError:
            err = "AssertionError! {}; HasToken = {}"
            self.logger.critical(err.format(token, self.hasToken.name))
            self.cleanup()
            sys.exit(1)

        self.hasToken = HasToken(self.hasToken + token.type)
        if token.type == TokenType.PING:
            self.ping_token = token
        else:  # PONG
            self.pong_token = token

        if token.value == self.m:
            if token.type == TokenType.PING:  # PONG lost, regenerate it
                self.logger.critical("PONG Token lost, regenerating")
                self.pong_token = self.regenerate(TokenType.PONG, -token.value)
            else:  # PING lost
                self.logger.critical("PING Token lost, regenerating")
                self.ping_token = self.regenerate(TokenType.PING, -token.value)
            self.hasToken = HasToken.BOTH

    def regenerate(self, type_: TokenType, value: int):
        return Token(type_, value)

    def incarnate(self, value: int):
        return Token(TokenType.PING, abs(value)+1),\
            Token(TokenType.PONG, -(abs(value)+1))

    def loop(self):
        if self.ID == 1:
            self.logger.info("First node, initializing tokens")
            self.hasToken = HasToken.BOTH
            self.pass_token(Token(TokenType.PING, 1))
            self.pass_token(Token(TokenType.PONG, -1))

        try:
            while True:
                if self.hasToken == HasToken.NONE:
                    self.local_conn.process_data_events(None)

                if self.hasToken == HasToken.PING:
                    self.logger.warning("Entering critical section")
                    time.sleep(1)
                    self.logger.warning("Leaving critical section")
                    if self.try_receive_token():
                        continue
                    self.pass_token(self.ping_token)
                elif self.hasToken == HasToken.PONG:
                    self.pass_token(self.pong_token)
                elif self.hasToken == HasToken.BOTH:
                    self.logger.warning("Tokens met, incarnating")
                    self.ping_token, self.pong_token = \
                        self.incarnate(self.ping_token.value)
                    self.pass_token(self.ping_token)
                    self.pass_token(self.pong_token)
        except KeyboardInterrupt:
            self.cleanup()

    def cleanup(self):
        print("Cleaning up and exiting")
        self.remote_conn.close()
        self.local_conn.close()

    def lose_token(self, signum, frame):
        self.tokens_to_lose.add(TokenType.PING if signum == signal.SIGUSR1
                                else TokenType.PONG)

if __name__ == '__main__':
    node = Node()
    signal.signal(signal.SIGUSR1, node.lose_token)
    signal.signal(signal.SIGUSR2, node.lose_token)
    time.sleep(1)
    node.loop()


from io import BytesIO
from datetime import datetime
from edfpy.header import Header

def test_read(test_header_bytes):
    """test read header-field bytes from file"""
    expected = {
        'version': '0',
        'patient_id': 'brux2',
        'recording_id': '',
        'startdate': '04.02.02',
        'starttime': '22.07.23',
        'num_header_bytes': 1536,
        'reserved': '',
        'num_records': 100,
        'record_duration': 1.0,
        'num_channels': 5,
    }
    expected_dt = datetime(2002, 2, 4, 22, 7, 23)
    file = BytesIO(test_header_bytes)
    fields = Header.read(file)
    for key in expected:
        assert getattr(fields, key) == expected[key], key

    assert fields.startdatetime == expected_dt

def test_read_edfp(test_edfp_header_bytes):
    file = BytesIO(test_edfp_header_bytes)
    fields = Header.read(file)
    assert fields.filetype == 'EDF+C'

def test_write(test_header_bytes):
    expected = {
        'version': '0',
        'patient_id': 'brux2',
        'recording_id': '',
        'startdate': '04.02.02',
        'starttime': '22.07.23',
        'num_header_bytes': 1536,
        'reserved': '',
        'num_records': 100,
        'record_duration': 1,
        'num_channels': 5,
    }
    fields = Header(**expected)
    file = BytesIO()
    fields.write(file)
    file.seek(0)
    content = file.read(256)
    assert content == test_header_bytes

def test_roundtrip():
    expected = {
        'version': '13',
        'patient_id': 'some-id',
        'recording_id': 'another-id',
        'startdate': '18.02.84',
        'starttime': '11.11.00',
        'num_header_bytes': 1000,
        'reserved': 'abracadabra',
        'num_records': 100,
        'record_duration': 2,
        'num_channels': 3,
    }
    expected_dt = datetime(1984, 2, 18, 11, 11, 0)
    fields = Header(**expected)
    assert expected_dt == fields.startdatetime
    file = BytesIO()
    fields.write(file)
    file.seek(0)
    fields = Header.read(file)
    for key in expected:
        assert getattr(fields, key) == expected[key], key


import os
import re
import sys
import json
import time
import shutil
import subprocess
import configparser
from . import db
from .utils import *
from .models import *
from threading import Thread
from werkzeug.security import generate_password_hash
from flask_login import login_required, current_user
from flask import Blueprint, render_template, request, flash, url_for, \
                                redirect, Response, send_from_directory

# Globals dictionaries to hold output objects.
GAME_SERVERS = {}
INSTALL_SERVERS = {}

# Global last requested output time.
last_request_for_output = int(time.time())

# Initialize view blueprint.
views = Blueprint("views", __name__)

######### Home Page #########

@views.route("/", methods=['GET'])
@views.route("/home", methods=['GET'])
@login_required
def home():
    # Import meta data.
    meta_data = db.session.get(MetaData, 1)
    base_dir = meta_data.app_install_path

    # Import config data.
    config = configparser.ConfigParser()
    config.read(f'{base_dir}/main.conf')

    # Kill any lingering background watch processes
    # Used in case console page is clicked away from.
    kill_watchers(last_request_for_output)

    installed_servers = GameServer.query.all()

    # Fetch dict containing all servers and flag specifying if they're running
    # or not via a util function.
    server_status_dict = get_active_servers(installed_servers)

    return render_template("home.html", user=current_user, \
                        server_status_dict=server_status_dict)

######### Controls Page #########

@views.route("/controls", methods=['GET'])
@login_required
def controls():
    # Import meta data.
    meta_data = db.session.get(MetaData, 1)
    base_dir = meta_data.app_install_path

    # Import config data.
    config = configparser.ConfigParser()
    config.read(f'{base_dir}/main.conf')
    text_color = config['aesthetic']['text_color']
    text_area_height = config['aesthetic']['text_area_height']

    # Pull in commands list from commands.json file.
    commands_list = get_commands()
    if not commands_list:
        flash('Error loading commands.json file!', category='error')
        return redirect(url_for('views.home'))

    # Bootstrap spinner colors.
    bs_colors = ['primary', 'secondary', 'success', 'danger', 'warning', \
                                                        'info', 'light']

    # Collect args from GET request.
    server_name = request.args.get("server")
    script_arg = request.args.get("command")

    # Can't load the controls page without a server specified.
    if server_name == None:
        flash("No server specified!", category="error")
        return redirect(url_for('views.home'))

    # Check that the submitted server exists in db.
    server = GameServer.query.filter_by(install_name=server_name).first()
    # If game server doesn't exist in db, can't load page for it.
    if server == None:
        flash("Invalid game server name!", category="error")
        return redirect(url_for('views.home'))

    # Checks that install dir exists.
    if not os.path.isdir(server.install_path):
        flash("No game server installation directory found!", category="error")
        return redirect(url_for('views.home'))

    cfg_paths = find_cfg_paths(server.install_path)
    if cfg_paths == "failed":
        flash("Error reading accepted_cfgs.json!", category="error")
        cfg_paths = []

    # Object to hold output from any running daemon threads.
    output_obj = OutputContainer([''], False)

    # If this is the first time we're ever seeing the server_name then put it
    # and its associated output_obj in the global GAME_SERVERS dictionary.
    if not server_name in GAME_SERVERS:
        GAME_SERVERS[server_name] = output_obj

    # Set the output object to the one stored in the global dictionary.
    output = GAME_SERVERS[server_name]

    script_path = server.install_path + '/' + server.script_name

    # This code block is only triggered in the event the script_arg param is
    # supplied with the GET request. Aka if a user has clicked one of the
    # control button.
    if script_arg != None:
        # Validate script_arg against contents of commands.json file.
        if is_invalid_command(script_arg):
            flash("Invalid Command!", category="error")
            return redirect(url_for('views.controls', server=server_name))

        # Console option, use tmux capture-pane to get output.
        if script_arg == "c":
            # First check if tmux session is running.
            cmd = ['/usr/bin/tmux', '-L', server.script_name, 'list-session']
            proc = subprocess.run(cmd, capture_output=True, text=True)

            if proc.returncode != 0:
                # Clear any previous output.
                output.output_lines.clear()
                output.output_lines.append(proc.stderr)

                flash("No Console Output!", category='error')
                return redirect(url_for('views.controls', server=server_name))

            # Otherwise, use the `watch` command to keep live console running.
            cmd = ['/usr/bin/watch', '-te', '/usr/bin/tmux', '-L', server.script_name, 'capture-pane', '-pt', server.script_name]
            daemon = Thread(target=shell_exec, args=(server.install_path, cmd, \
                                    output), daemon=True, name='Console')
            daemon.start()
            return redirect(url_for('views.controls', server=server_name))

        # If its not the console command
        else:
            cmd = [f'{script_path}', f'{script_arg}']
            daemon = Thread(target=shell_exec, args=(server.install_path, cmd, \
                                    output), daemon=True, name='Command')
            daemon.start()
            return redirect(url_for('views.controls', server=server_name))

    # Default to no refresh.
    refresh = False

    return render_template("controls.html", user=current_user, \
        server_name=server_name, server_commands=commands_list, \
        text_color=text_color, text_area_height=text_area_height, \
                      bs_colors=bs_colors, cfg_paths=cfg_paths)

######### Install Page #########

@views.route("/install", methods=['GET', 'POST'])
@login_required
def install():
    # Import meta data.
    meta_data = db.session.get(MetaData, 1)
    base_dir = meta_data.app_install_path

    # Import config data.
    config = configparser.ConfigParser()
    config.read(f'{base_dir}/main.conf')
    text_color = config['aesthetic']['text_color']
    text_area_height = config['aesthetic']['text_area_height']

    # Pull in install server list from game_servers.json file.
    install_list = get_servers()
    if not install_list:
        flash('Error loading game_servers.json file!', category='error')
        return redirect(url_for('views.home'))

    # Initialize blank install_name, used for update-text-area.js.
    install_name = ""

    # Kill any lingering background watch processes if page is reloaded.
    kill_watchers(last_request_for_output)

    # Bootstrap spinner colors.
    bs_colors = ['primary', 'secondary', 'success', 'danger', 'warning', \
                                                        'info', 'light']
    aiwrapsh = "auto_install_wrap.sh"

    # Check for / install the main linuxgsm.sh script.
    lgsmsh = "linuxgsm.sh"
    check_and_wget_lgsmsh(f"{base_dir}/{lgsmsh}")

    # Post logic only triggered after install form submission.
    if request.method == 'POST':
        server_script_name = request.form.get("server_name")
        server_full_name = request.form.get("full_name")
        sudo_pass = request.form.get("sudo_pass")

        # Make sure required options are supplied.
        for required_form_item in (server_script_name, server_full_name, sudo_pass):
            if required_form_item == None:
                flash("Missing Required Form Feild!", category="error")
                return redirect(url_for('views.install'))

            # Check input lengths.
            if len(required_form_item) > 150:
                flash("Form field too long!", category='error')
                return redirect(url_for('views.install'))

        # Validate form submission data against install list in json file.
        if install_options_are_invalid(server_script_name, server_full_name):
            flash("Invalid Installation Option(s)!", category="error")
            return redirect(url_for('views.install'))

        # Make server_full_name a unix friendly directory name.
        server_full_name = server_full_name.replace(" ", "_")

        # Used to pass install_name to frontend js.
        install_name = server_full_name

        # Object to hold output from any running daemon threads.
        output_obj = OutputContainer([''], False)

        # If this is the first time we're ever seeing the server_name then put it
        # and its associated output_obj in the global INSTALL_SERVERS dictionary.
        if not server_full_name in INSTALL_SERVERS:
            INSTALL_SERVERS[server_full_name] = output_obj

        # Set the output object to the one stored in the global dictionary.
        output = INSTALL_SERVERS[server_full_name]

        install_exists = GameServer.query.filter_by(install_name=server_full_name).first()

        if install_exists:
            flash('An installation by that name already exits.', category='error')
            return redirect(url_for('views.install'))

        if os.path.exists(server_full_name):
            flash('Install directory already exists.', category='error')
            flash('Did you perhaps have this server installed previously?', \
                                                            category='error')
            return redirect(url_for('views.install'))

        if not get_tty_ticket(sudo_pass):
            flash('Problem with sudo password!', category='error')
            return redirect(url_for('views.install'))

        # Make a new server dir and copy linuxgsm.sh into it then cd into it.
        os.mkdir(server_full_name)
        shutil.copy(lgsmsh, server_full_name)
        shutil.copy(aiwrapsh, server_full_name)

        install_path = base_dir + '/' + server_full_name

        # Add the install to the database.
        new_game_server = GameServer(install_name=server_full_name, \
                install_path=install_path, script_name=server_script_name)
        db.session.add(new_game_server)
        db.session.commit()

        cmd = [f'./{lgsmsh}', f'{server_script_name}']
        proc = subprocess.run(cmd, cwd=install_path, capture_output=True, text=True)

        if proc.returncode != 0:
            output.output_lines.append(proc.stderr)
        output.output_lines.append(proc.stdout)

        cmd = [f'./{aiwrapsh}', f'{server_script_name}']
        install_daemon = Thread(target=shell_exec, args=(install_path, cmd, \
                                output), daemon=True, name='Install')
        install_daemon.start()

    return render_template("install.html", user=current_user, \
            servers=install_list, text_color=text_color, bs_colors=bs_colors, \
            install_name=install_name, text_area_height=text_area_height)

######### Output Page #########

@views.route("/output", methods=['GET'])
@login_required
def no_output():
    # Collect args from GET request.
    server_name = request.args.get("server")

    # Can't load the controls page without a server specified.
    if server_name == None:
        return """{ "error" : "eer can't load page n'@" }"""

    # Can't do anyting if we don't recognize the server_name.
    if server_name not in GAME_SERVERS and server_name not in INSTALL_SERVERS:
        return """{ "error" : "eer never heard of em" }"""

    # Reset the last requested output time.
    last_request_for_output = int(time.time())

    # If its a server in the INSTALL_SERVERS dict set the output object to the
    # one from that dictionary.
    if server_name in INSTALL_SERVERS:
        output = INSTALL_SERVERS[server_name]

    # If its a server in the GAME_SERVERS dict set the output object to the one
    # from that dictionary. Will clober the above after server is installed. Is
    # intentional.
    if server_name in GAME_SERVERS:
        output = GAME_SERVERS[server_name]

    # Returns json for used by ajax code on /controls route.
    return output.toJSON()

######### Settings Page #########

@views.route("/settings", methods=['GET', 'POST'])
@login_required
def settings():
    # Import meta data.
    meta_data = db.session.get(MetaData, 1)
    base_dir = meta_data.app_install_path

    # Kill any lingering background watch processes in case console page is
    # clicked away fromleft.
    kill_watchers(last_request_for_output)

    # Import config data.
    config = configparser.ConfigParser()
    config.read(f'{base_dir}/main.conf')
    text_color = config['aesthetic']['text_color']
    text_area_height = config['aesthetic']['text_area_height']
    remove_files = config['settings'].getboolean('remove_files')

    if request.method == 'POST':
        color_pref = request.form.get("text_color")
        file_pref = request.form.get("delete_files")
        height_pref = request.form.get("text_area_height")

        # Set Remove files setting.
        config['settings']['remove_files'] = 'yes'
        if file_pref == "false":
            config['settings']['remove_files'] = 'no'

        # Set text color setting.
        config['aesthetic']['text_color'] = text_color
        if color_pref != None:
            # Validate color code with regular expression.
            if not re.search('^#(?:[0-9a-fA-F]{1,2}){3}$', color_pref):
                flash('Invalid color!', category='error')
                return redirect(url_for('views.settings'))

            config['aesthetic']['text_color'] = color_pref

        # Set default text area height setting.
        config['aesthetic']['text_area_height'] = text_area_height
        if height_pref != None:
            # Validate textarea height is int.
            try:
                height_pref = int(height_pref)
            except ValueError:
                flash('Invalid Textarea Height!', category='error')
                return redirect(url_for('views.settings'))

            # Check if height pref is in valid range.
            if height_pref > 100 or height_pref < 5:
                flash('Invalid Textarea Height!', category='error')
                return redirect(url_for('views.settings'))

            # Have to cast back to string to save in config.
            config['aesthetic']['text_area_height'] = str(height_pref)

        with open(f'{base_dir}/main.conf', 'w') as configfile:
             config.write(configfile)

        flash("Settings Updated!")
        return redirect(url_for('views.settings'))

    return render_template("settings.html", user=current_user, \
            text_color=text_color, remove_files=remove_files, \
                           text_area_height=text_area_height)

######### About Page #########

@views.route("/about", methods=['GET'])
@login_required
def about():
    # Import meta data.
    meta_data = db.session.get(MetaData, 1)
    base_dir = meta_data.app_install_path

    # Kill any lingering background watch processes.
    # In case console page is clicked away from.
    kill_watchers(last_request_for_output)

    # Import config data.
    config = configparser.ConfigParser()
    config.read(f'{base_dir}/main.conf')
    text_color = config['aesthetic']['text_color']

    return render_template("about.html", user=current_user, \
                                        text_color=text_color)

######### Add Page #########

@views.route("/add", methods=['GET', 'POST'])
@login_required
def add():
    # Import meta data.
    meta_data = db.session.get(MetaData, 1)
    base_dir = meta_data.app_install_path

    # Import config data.
    config = configparser.ConfigParser()
    config.read(f'{base_dir}/main.conf')
    remove_files = config['settings'].getboolean('remove_files')

    # Kill any lingering background watch processes in case console page is
    # clicked away fromleft.
    kill_watchers(last_request_for_output)

    # Set default status_code.
    status_code = 200

    if request.method == 'POST':
        install_name = request.form.get("install_name")
        install_path = request.form.get("install_path")
        script_name = request.form.get("script_name")

        # Check all required args are submitted.
        for required_form_item in (install_name, install_path, script_name):
            if required_form_item == None or required_form_item == '':
                flash("Missing required form field(s)!", category="error")
                return redirect(url_for('views.add'))

            # Check input lengths.
            if len(required_form_item) > 150:
                flash("Form field too long!", category='error')
                return redirect(url_for('views.add'))

        # Make install name unix friendly for dir creation.
        install_name = install_name.replace(" ", "_")

        install_exists = GameServer.query.filter_by(install_name=install_name).first()

        # Try to prevent arbitrary bad input.
        for input_item in (install_name, install_path, script_name):
            if contains_bad_chars(input_item):
                flash("Illegal Character Entered!", category="error")
                flash(r"""Bad Chars: $ ' " \ # = [ ] ! < > | ; { } ( ) * , ? ~ &""", \
                                                            category="error")
                return redirect(url_for('views.add'))

        # Only allow lgsm installs under home dir.
        user_home_dir = os.path.expanduser('~')

        if install_exists:
            flash('An installation by that name already exits.', category='error')
            status_code = 400

        elif not os.path.exists(install_path) or not os.path.isdir(install_path):
            flash('Directory path does not exist.', category='error')
            status_code = 400

        elif os.path.commonprefix((os.path.realpath(install_path),user_home_dir)) != user_home_dir:
            flash(f'Only dirs under {user_home_dir} allowed!', category='error')
            status_code = 400

        elif script_name_is_invalid(script_name):
            flash('Invalid game server script file name!', category='error')
            status_code = 400

        elif not os.path.exists(install_path + '/' + script_name) or \
                not os.path.isfile(install_path + '/' + script_name):
            flash('Script file does not exist.', category='error')
            status_code = 400

        else:
            # Add the install to the database, then redirect home.
            new_game_server = GameServer(install_name=install_name, \
                        install_path=install_path, script_name=script_name)
            db.session.add(new_game_server)
            db.session.commit()

            flash('Game server added!')
            return redirect(url_for('views.home'))

    return render_template("add.html", user=current_user), status_code

######### Delete Route #########

@views.route("/delete", methods=['GET', 'POST'])
@login_required
def delete():
    # Import meta data.
    meta_data = db.session.get(MetaData, 1)
    base_dir = meta_data.app_install_path

    # Import config data.
    config = configparser.ConfigParser()
    config.read(f'{base_dir}/main.conf')
    remove_files = config['settings'].getboolean('remove_files')

    # Delete via POST is for multiple deletions.
    # Post submissions come from delete toggles on home page.
    if request.method == 'POST':
        for server_id, server_name in request.form.items():
            server = GameServer.query.filter_by(install_name=server_name).first()
            if server != None:
                del_server(server, remove_files)
    else:
        server_name = request.args.get("server")
        if server_name == None:
            flash("Missing Required Args!", category="error")
            return redirect(url_for('views.home'))

        server = GameServer.query.filter_by(install_name=server_name).first()
        if server != None:
            del_server(server, remove_files)

    return redirect(url_for('views.home'))

######### Edit Route #########
@views.route("/edit", methods=['GET', 'POST'])
@login_required
def edit():
    # Import meta data.
    meta_data = db.session.get(MetaData, 1)
    base_dir = meta_data.app_install_path

    # The abbreviation cfg will be used to refer to any lgsm game server
    # specific config files. Whereas, the word config will be used to refer to
    # any web-lgsm config info.

    # Import config data.
    config = configparser.ConfigParser()
    config.read(f'{base_dir}/main.conf')
    text_color = config['aesthetic']['text_color']
    text_area_height = config['aesthetic']['text_area_height']

    # Collect args from POST request.
    server_name = request.form.get("server")
    cfg_path = request.form.get("cfg_path")
    new_file_contents = request.form.get("file_contents")
    download = request.form.get("download")

    # Can't load the edit page without a server specified.
    if server_name == None or server_name == "":
        flash("No server specified!", category="error")
        return redirect(url_for('views.home'))

    # Can't load the edit page without a cfg specified.
    if cfg_path == None or cfg_path == "":
        flash("No config file specified!", category="error")
        return redirect(url_for('views.home'))

    # Check that the submitted server exists in db.
    server = GameServer.query.filter_by(install_name=server_name).first()
    # If game server doesn't exist in db, can't load page for it.
    if server == None:
        flash("Invalid game server name!", category="error")
        return redirect(url_for('views.home'))

    # Checks that install dir exists.
    if not os.path.isdir(server.install_path):
        flash("No game server installation directory found!", category="error")
        return redirect(url_for('views.home'))

    # Try to pull script's basename from supplied cfg_path.
    try:
        cfg_file = os.path.basename(cfg_path)
    except:
        flash("Error getting config file basename!", category="error")
        return redirect(url_for('views.home'))

    # Validate cfg_file name is in list of accepted cfgs.
    if is_invalid_cfg_name(cfg_file):
        flash("Invalid config file name!", category="error")
        return redirect(url_for('views.home'))

    # Check that file exists before allowing writes to it. Aka don't allow
    # arbitrary file creation. Even though the above should block creating
    # files with arbitrary names, we still don't want to allow arbitrary file
    # creation anywhere on the file system the app has write perms to.
    if not os.path.isfile(cfg_path):
        flash("No such file!", category="error")
        return redirect(url_for('views.home'))

    # If new_file_contents supplied in post request, write the new file
    # contents to the cfg file.
    if new_file_contents:
        try:
            with open(cfg_path, 'w') as f:
                f.write(new_file_contents.replace('\r', ''))
            flash("Config Updated!", category="success")
        except:
            flash("Error writing to config!", category="error")

    # Read in file contents from cfg file.
    file_contents = ""
    # Try except incase problem with file.
    try:
        with open(cfg_path) as f:
            file_contents = f.read()
    except:
        flash("Error reading config!", category="error")
        return redirect(url_for('views.home'))

    # If is download request.
    if download == "yes":
        basedir, basename = os.path.split(cfg_path)
        return send_from_directory(basedir, basename, as_attachment=True)

    return render_template("edit.html", user=current_user, \
    text_color=text_color, text_area_height=text_area_height, \
                server_name=server_name, cfg_file=cfg_path, \
                file_contents=file_contents, cfg_file_name=cfg_file)


import bs4
import copy
import json
import numpy as np

from dataclasses import dataclass
from typing import Optional, List
from bs4 import BeautifulSoup

@dataclass
class TableCell:
    text: str
    width: Optional[np.int8] = 1
    height: Optional[np.int8] = 1
    linked_top: Optional[bool] = False  # judge if the current cell belongs to the above multi-row cell
    linked_left: Optional[bool] = False  # judge if the current cell belongs to the left multi-column cell

class TableRow:
    def __init__(self, cells: List[TableCell]):
        self._cells = cells
        self._width = self._get_width()
        self._expanded_cells = self._expand_cells()

    def __getitem__(self, i):
        return self.expanded_cells[i]

    def __len__(self):
        return self.width

    def _get_width(self):
        cell_widths = [cell.width for cell in self._cells]
        return np.sum(cell_widths)

    @property
    def width(self):
        return self._width

    @property
    def cells(self):
        return self._cells

    @property
    def expanded_cells(self):
        return self._expanded_cells

    @cells.setter
    def cells(self, cells):
        new_widths = [cell.width for cell in cells]
        assert self._width == new_widths, ValueError('The width must equal to the old one!')
        self._cells = cells
        self._expanded_cells = self._expand_cells()

    @expanded_cells.setter
    def expanded_cells(self, cells):
        raise AttributeError("Assigning values to `expanded_cells` is illegal!")

    def __str__(self):
        cell_line = '\t'.join([cell.text for cell in self._cells]) + '\n'
        return cell_line

    def __repr__(self):
        return self.__str__()

    def text(self):
        return self.__str__()

    def _expand_cells(self):
        expanded_cells = list()
        multicolumn_cache = list()
        cell_idx = 0
        while True:
            if multicolumn_cache:
                cached_cell = copy.deepcopy(multicolumn_cache[0])
                cached_cell.width = 1
                cached_cell.linked_left = True
                expanded_cells.append(cached_cell)
                multicolumn_cache[1] -= 1
                if multicolumn_cache[1] == 0:
                    multicolumn_cache = list()
            else:
                expanded_cells.append(self._cells[cell_idx])
                if self._cells[cell_idx].width > 1:
                    multicolumn_cache = [self._cells[cell_idx], self._cells[cell_idx].width - 1]
                cell_idx += 1
            if cell_idx == len(self._cells) and not multicolumn_cache:
                break
        return expanded_cells

class Table:
    def __init__(self,
                 label: Optional[str] = None,
                 idx: Optional[str] = None,
                 caption: Optional[str] = None,
                 rows: Optional[List[TableRow]] = None,
                 footnotes: Optional[List[str]] = None):

        self._label = label
        self._id = idx
        self._caption = caption
        self._rows = rows
        self._footnotes = footnotes

    @property
    def label(self):
        return self._label if self._label is not None else "<EMPTY>"

    @property
    def id(self):
        return self._id if self._id is not None else "<EMPTY>"

    @property
    def caption(self):
        return self._caption if self._caption is not None else ''

    @property
    def rows(self):
        return self._rows if self._rows is not None else []

    @property
    def footnotes(self):
        return self._footnotes if self._footnotes is not None else []

    @property
    def shape(self):
        return len(self), len(self._rows[0]) if self._rows else 0

    @property
    def width(self):
        return len(self._rows[0]) if self._rows else 0

    @property
    def n_columns(self):
        return self.width

    @property
    def height(self):
        return len(self)

    @property
    def n_rows(self):
        return self.height

    @label.setter
    def label(self, x):
        self._label = x

    @id.setter
    def id(self, x):
        self._id = x

    @caption.setter
    def caption(self, x):
        self._caption = x

    @rows.setter
    def rows(self, x):
        self._rows = x

    @footnotes.setter
    def footnotes(self, x):
        self._footnotes = x

    def __getitem__(self, idx):
        if isinstance(idx, int):
            return self.rows[idx]
        elif isinstance(idx, tuple) and len(idx) == 2:
            r, c = idx
            return self.rows[r].expanded_cells[c]

    def __str__(self):
        lines = self._label if self._label else ''
        lines += f' {self._caption}\n' if self._caption else ''
        rows = ''.join([row.__str__() for row in self._rows])
        lines += rows
        lines += '\n'.join(self._footnotes) if self._footnotes else ''
        return lines

    def __len__(self):
        return len(self._rows)

    def __repr__(self):
        return self.__str__()

    def _repr_html_(self):
        return self.write_html().prettify()

    def text(self):
        return self.__str__()

    def format_rows(self):
        """
        Deal with multi-row cells by appending dummy cells below them in each row
        """
        multirow_cache = dict()
        formatted_rows = list()
        for row in self.rows:
            cells = list()
            cell_idx = 1
            for cell in row.cells:
                while True:
                    if cell_idx in multirow_cache:
                        _, width, text = multirow_cache[cell_idx]
                        multirow_cell = TableCell(text, width, linked_top=True)

                        multirow_cache[cell_idx][0] -= 1
                        assert multirow_cache[cell_idx][0] >= 0
                        if multirow_cache[cell_idx][0] == 0:
                            multirow_cache.pop(cell_idx)

                        # check if we have already appended the multi-row cells before
                        if cell.text == text and cell.linked_top is True:
                            pass
                        else:
                            cell_idx += width
                            cells.append(multirow_cell)
                    else:
                        break

                # cache the properties of multi-row cells
                if cell.height > 1:
                    multirow_cache[cell_idx] = [cell.height - 1, cell.width, cell.text]

                cell_idx += cell.width
                cells.append(cell)
            formatted_rows.append(TableRow(cells))
        self._rows = formatted_rows
        return self

    def body_to_lists(self):
        element_list = list()
        for i in range(self.height):
            row_elements = list()
            for j in range(self.width):
                row_elements.append(self[i, j].text)
            element_list.append(row_elements)
        return element_list

    def write_html(self, root: Optional[bs4.element.Tag] = None):
        soup = BeautifulSoup()

        if root is None:
            root = BeautifulSoup()
        html_table = soup.new_tag('table')
        root.insert(len(root), html_table)

        if self.caption:
            caption = soup.new_tag('caption')
            html_table.insert(len(html_table), caption)
            if self.label and self.label != '<EMPTY>':
                cap_txt = f'{self.label} {self.caption}'
            else:
                cap_txt = self.caption
            caption.insert(0, cap_txt)

        tbody = soup.new_tag('tbody')
        html_table.insert(len(html_table), tbody)
        for row in self.rows:
            table_row = soup.new_tag('tr')
            tbody.insert(len(tbody), table_row)

            for entry in row.cells:
                if entry.linked_top:
                    continue
                table_data = soup.new_tag('td', colspan=str(entry.width), rowspan=str(entry.height))
                table_row.insert(len(table_row), table_data)
                table_data.insert(0, entry.text)

        if self.footnotes:
            tfoot = soup.new_tag('tfoot')
            html_table.insert(len(html_table), tfoot)
            for footnote in self.footnotes:
                table_row = soup.new_tag('tr')
                tfoot.insert(len(tfoot), table_row)
                table_data = soup.new_tag('td', colspan=str(self.rows[0].width) if self.rows else 1)
                table_row.insert(0, table_data)
                table_data.insert(0, footnote)

        return root

    def save_json(self, file_name):
        json_elements = {
            'caption': self.caption,
            'body': self.body_to_lists(),
            'footnotes': self.footnotes
        }
        with open(file_name, 'w', encoding='UTF-8') as f:
            json.dump(json_elements, f, indent=2, ensure_ascii=False)

def set_table_style(root: bs4.element.Tag):
    soup = BeautifulSoup()
    style = soup.new_tag('style')
    root.insert(len(root), style)
    style_string = \
        """
        table {
          font-family: arial, sans-serif;
          border-collapse: collapse;
          width: 90%;
          margin-left: auto;
          margin-right: auto;
          margin-top: 1.5em;
          margin-bottom: 1.5em;
        }

        caption {
          margin-bottom: 0.5em;
        }

        tbody td, th {
          border: 1px solid #dddddd;
          text-align: left;
          padding: 8px;
          font-size: 80%;
        }

        tbody tr:nth-child(even) {
          background-color: #dddddd;
        }

        tfoot td, th{
          border: none;
          text-align: left;
          padding: 8px;
          font-size: 70%;
          line-height: 100%;
        }

        tfoot tr {
          background-color: #ffffff;
        }
        """
    style.insert(0, style_string)
    return root


import numpy as np
import matplotlib.pyplot as plt
import math
from decimal import Decimal, getcontext
from matplotlib.animation import FuncAnimation, PillowWriter

N=91

def RK4(f, y0, t):
    n = len(t)
    # if y0 is a scalar:
    if np.isscalar(y0):
        y = np.zeros(n)
    else:
        y = np.zeros((n, len(y0)))
    y[0] = y0
    for i in range(n-1):
        h = t[i+1] - t[i]
        k1 = h*f(t[i], y[i])
        k2 = h*f(t[i] + h/2, y[i] + k1/2)
        k3 = h*f(t[i] + h/2, y[i] + k2/2)
        k4 = h*f(t[i] + h, y[i] + k3)
        y[i+1] = y[i] + (k1 + 2*k2 + 2*k3 + k4)/6
    return y

f4 = lambda t, y: y**2

C = N/100
h = 1e-6
t = np.linspace(0, 1.5, int(1.5/h))
y = RK4(f4, C, t)
plt.plot(t, y)
plt.xlabel('t')
plt.ylabel('y')
t_last = t[(np.where(np.isfinite(y)))][-1]
plt.title(f'RK4 y(0) = 0.91, end={t_last:.6f}')
plt.show()


from telethon import TelegramClient, events, utils
from telethon.sessions import StringSession
from telethon.tl import types, functions
import asyncio,random,os,time
from PIL import Image
import mimetypes
import io
import json

cliId = 12345
cliHash = "hashhsahhashhashhsahhash"
client = TelegramClient("myuser", cliId, cliHash, receive_updates=False)

index = None
indexMsg = None
me = None

async def newIndex():
    start_index = {
        "supportedTags": [],
        "photo": [],
        "video": []
    }

    json_data = json.dumps(start_index)
    data_bytes = json_data.encode('utf-8')

    msg = await client.send_file("me", file=data_bytes, caption="index2.tg",attributes=[types.DocumentAttributeFilename("index2.tg")])

    pin_msg = await client.pin_message("me", msg)
    return msg

async def uploadIndex():
    global index

    json_str = json.dumps(index)
    json_bytes = json_str.encode('utf-8')

    await client.edit_message("me", indexMsg.id, file = json_bytes)

async def getIndex():
    global index,indexMsg,me
    chat = 'me'
    me = await client.get_me()

    pinnedList = await client.get_messages(chat, None, filter=types.InputMessagesFilterPinned)
    for i in pinnedList:
        if i.media and type(i.media) == types.MessageMediaDocument and i.message == 'index2.tg':
            indexMsg = i
            break

    if not indexMsg:
        ioo = await client.get_me()
        if ioo:
            await newIndex()
            return (await getIndex())
    else:
        json_bytes = await indexMsg.download_media(file=bytes)
        json_str = json_bytes.decode('utf-8')
        index = json.loads(json_str)
        return index

def determine_image_orientation(image_path):
    try:
        with Image.open(image_path) as img:
            exif = img._getexif()

            if exif is not None:
                orientation = exif.get(274)
                if orientation is not None:
                    exif_orientation_codes = {
                        1: 0,
                        3: 0,
                        6: 1,
                        8: 1
                    }

                    if orientation in exif_orientation_codes:
                        return exif_orientation_codes[orientation]

            width, height = img.size
            if width == height:
                return 2
            elif width > height:
                return 1
            else:
                return 0
    except Exception as e:
        print("err:", e)
        return 0

def crop_to_square(image_path):
    img = Image.open(image_path)

    if img.mode != 'RGB':
        img = img.convert('RGB')

    img.thumbnail((320,320))

    img_byte_arr = io.BytesIO()
    img.save(img_byte_arr, format='JPEG')
    img_byte_arr = img_byte_arr.getvalue()
    return img_byte_arr

async def main():
    async with client:
        counter = 1
        await getIndex()
        ent = await client.get_entity("me")

        if ent != None:
            for j in [each for each in os.listdir("./photo") if each.endswith('.jpg')]:
                if counter%10 == 0:
                    await uploadIndex()
                    print("uploaded index")
                counter+=1

                jj = "./photo/"+j
                with open(jj,"rb") as f:

                    thumb = crop_to_square(jj)
                    send_file = await client.send_file(ent, f, thumb=thumb, force_document=True, caption=j, silent=True)
                    item = {
                        "mimeType" : mimetypes.guess_type(j, strict=False)[0],
                        "favorite": False,
                        "trashed": False,
                        "msgId": send_file.id,
                        "date": int(os.path.getctime(jj)),#int(time.time()),
                        "tags": [
                                    #"notags"
                                    eval(input('enter tags like ["tag1","tag2",...]: '))
                            ],
                        "label": j,
                        "orientation": determine_image_orientation(jj),
                        "chatId": me.id,
                        "size": os.path.getsize(jj),
                    }
                    print(item)
                    index["photo"].append(item)

                    print("OK1")
                os.remove(jj) #images are deleted from this folder in order not to upload them again in case of a failure

            print("ok2")
            await uploadIndex()

        print("ok3")

asyncio.run(main())


# -*- coding: utf-8 -*-
"""
Created on Tue Nov  5 14:25:10 2019

@author: user4
"""

# Importing the Libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Importing the Dataset
df = pd.read_csv('Social_Network_Ads.csv')
X = df.iloc[:,2:4].values
y = df.iloc[:,4].values

# Missing Values
df.isna().sum() #we have no issing values, hence no imputation required

#from sklearn.impute import SimpleImputer
#imputer = SimpleImputer(missing_values = np.nan, strategy = "mean")
#X.iloc[:,1:3] = imputer.fit_transform(X.iloc[:,1:3])

# Categorical Data
# We have one categorical data that is Gender. let us label it and one hot encode it
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
label_encoder_x = LabelEncoder()
X[:,0] = label_encoder_x.fit_transform(X[:,0])
# As it has only two labels one hot encoding is not required
#one_hot_encoder = OneHotEncoder(categorical_features = [0])
#X = one_hot_encoder.fit_transform(X).toarray()
#label_encoder_y = LabelEncoder()
#y = label_encoder_y.fit_transform(y)

# Train and Test Split the Data
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X,y, train_size = 0.8, random_state = 1234)

# Feature Scaling them all
from sklearn.preprocessing import StandardScaler
standard_scaler_x = StandardScaler()
X_train = standard_scaler_x.fit_transform(X_train)
X_test = standard_scaler_x.transform(X_test)

# Building the classifier
from sklearn.neighbors import KNeighborsClassifier

#Using euclidean distance in the algorithm
classifier = KNeighborsClassifier(n_neighbors=5, metric='minkowski', p = 2)

classifier.fit(X_train, y_train)

y_pred = classifier.predict(X_test)

# Let's use a confusion matrix to see the accuracy
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)
accuracy = (cm[0,0] + cm[1,1])/sum(sum(cm))
accuracy
# hence we observe around 86.2%

def knn_optimizer(n):
    optimum = np.zeros((n,2))
    for i in np.arange(1,n+1,1):
        classifier = KNeighborsClassifier(n_neighbors=i, metric='minkowski', p = 2)

        classifier.fit(X_train, y_train)

        y_pred = classifier.predict(X_test)

        # Let's use a confusion matrix to see the accuracy
        from sklearn.metrics import confusion_matrix
        cm = confusion_matrix(y_test, y_pred)
        accuracy = (cm[0,0] + cm[1,1])/sum(sum(cm))
        optimum[i-1,0] = i
        optimum[i-1,1] = accuracy
    return optimum

visual = knn_optimizer(15)
plt.plot(visual[:,0],visual[:,1])

#From the plot we can see that we have peaks at K value of 7 and 9

classifier = KNeighborsClassifier(n_neighbors=7, metric='minkowski', p = 2)

classifier.fit(X_train, y_train)

y_pred = classifier.predict(X_test)

#Following is a much better visualization of the prediction
# Visualising the Training set results
from matplotlib.colors import ListedColormap
X_set, y_set = X_train, y_train
X1, X2 = np.meshgrid(np.arange(start = X_set[:, 0].min() - 1, stop = X_set[:, 0].max() + 1, step = 0.01),
                     np.arange(start = X_set[:, 1].min() - 1, stop = X_set[:, 1].max() + 1, step = 0.01))
plt.contourf(X1, X2, classifier.predict(np.array([X1.ravel(), X2.ravel()]).T).reshape(X1.shape),
             alpha = 0.75, cmap = ListedColormap(('red', 'green')))
plt.xlim(X1.min(), X1.max())
plt.ylim(X2.min(), X2.max())
for i, j in enumerate(np.unique(y_set)):
    plt.scatter(X_set[y_set == j, 0], X_set[y_set == j, 1],
                c = ListedColormap(('red', 'green'))(i), label = j)
plt.title('Logistic Regression (Training set)')
plt.xlabel('Age')
plt.ylabel('Estimated Salary')
plt.legend()
plt.show()#we see that the predictions are much better than logistic regression

# Visualising the test set results
from matplotlib.colors import ListedColormap
X_set, y_set = X_test, y_test
X1, X2 = np.meshgrid(np.arange(start = X_set[:, 0].min() - 1, stop = X_set[:, 0].max() + 1, step = 0.01),
                     np.arange(start = X_set[:, 1].min() - 1, stop = X_set[:, 1].max() + 1, step = 0.01))
plt.contourf(X1, X2, classifier.predict(np.array([X1.ravel(), X2.ravel()]).T).reshape(X1.shape),
             alpha = 0.75, cmap = ListedColormap(('red', 'green')))
plt.xlim(X1.min(), X1.max())
plt.ylim(X2.min(), X2.max())
for i, j in enumerate(np.unique(y_set)):
    plt.scatter(X_set[y_set == j, 0], X_set[y_set == j, 1],
                c = ListedColormap(('red', 'green'))(i), label = j)
plt.title('Logistic Regression (Training set)')
plt.xlabel('Age')
plt.ylabel('Estimated Salary')
plt.legend()
plt.show()

Part 3 - Classification/Section 15 - K-Nearest Neighbors (K-NN)/K_Nearest_Neighbors/knn_JMR.py


'''
This holds the file where we're going to dump the dictionary
holding all the delicious inverted index information from a directory
'''

###################################################################
#                           Imports                               #
###################################################################
import json
import pickle
import os
import datetime
import math

###################################################################
#                          Functions                              #
###################################################################

def dumpToPickle(index: dict[str, list], number: int) -> None:
    """Dump dictionary to pickle file."""
    with open(f'dumps/dump{number}.pickle', 'wb') as dump_file:
        pickle.dump(index, dump_file, pickle.HIGHEST_PROTOCOL)

def savePageRanks(index: dict[str, list]) -> None:
    """Dump hash dictionary to json file."""
    with open(f'./pageRanks/pageRankIndex.json', 'w') as json_file:
        print(f"{datetime.datetime.now()} Started Saving Page Ranks")
        json.dump(index, json_file)
        print(f"{datetime.datetime.now()} Finished Saving Page Ranks")

def saveHashes(index: dict[str, list]) -> None:
    """Dump hash dictionary to json file."""
    with open(f'./docHashes/hashIndex.json', 'w') as json_file:
        print(f"{datetime.datetime.now()} Started Saving Hash Index")
        json.dump(index, json_file)
        print(f"{datetime.datetime.now()} Finished Saving Hash Index")

def load_index(index_num: int) -> dict[str, int]:
    """Loads index of character start and end locations for each token in file."""
    with open(f'./indexIndex/index{index_num}.pickle', 'rb') as pickle_data:
        location_index = pickle.load(pickle_data)
    return location_index

def load_page_rank_index() -> dict[int, int]:
    """Loads index of Page Ranks"""
    print("Loading PageRanks Index...")
    with open(f'./pageRanks/pageRankIndex.json', 'r') as json_file:
        page_rank_index = json.load(json_file)
    print("\tFinished")
    return page_rank_index

def load_token_idf_index() -> dict[int, int]:
    """Loads index of token idf index."""
    print("Loading tokenIdfIndex Index...")
    with open(f'./tokenIdfIndex/tokenIdf.pickle', 'rb') as pickle_file:
        token_idf_index = pickle.load(pickle_file)
    print("\tFinished")
    return token_idf_index


import numpy as np
import yfinance as yf
import pandas as pd
from collections import OrderedDict
import pycountry_convert as pc
import concurrent.futures
import matplotlib.pyplot as plt
import csv
from scipy.stats import multivariate_normal, norm

def gather_data(ticker, factors):
    """Reads issuer info"""
    temp_factor = []

    for factor in factors:
        k = yf.Ticker(ticker).info[factor]

        if k == "Consumer Defensive":
            temp_factor.append("Consumer Services")

        if factor == "country":
            country_code = pc.country_name_to_country_alpha2(
                k, cn_name_format="default"
            )
            continent_name = pc.country_alpha2_to_continent_code(country_code)
            temp_factor.append(continent_name)

        else:
            temp_factor.append(k)

    return temp_factor

if __name__ == "__main__":
    issuers = []
    threshhold = []
    model = ["country", "sector"]

    with open('issuer_heavy_correlated.csv') as f:
        read = csv.reader(f, delimiter=',')
        next(read)
        for row in read:
            issuers.append(str(row[0]))
            threshhold.append(float(row[1]))

    all_factors = []

    with concurrent.futures.ThreadPoolExecutor() as executor:
        values = [executor.submit(gather_data, ticker, model) for ticker in issuers]

        concurrent.futures.wait(values, timeout=None, return_when='ALL_COMPLETED')
        for f in values:
            all_factors.append(f.result())

    dict_region = {}
    dict_score = {}
    dict_sect = {}
    for issuer in all_factors:
        if issuer[0] not in dict_region.keys():
            # if issuer[0] == "NA":
            #     issuer[0] = "North America"
            # if issuer[0] == "EU":
            #     issuer[0] = "Europe"
            dict_region[issuer[0]] = 1
        else:
            dict_region[issuer[0]] += 1

        if issuer[1] not in dict_sect.keys():
            dict_sect[issuer[1]] = 1
        else:
            dict_sect[issuer[1]] += 1

    for score in threshhold:
        if score == 0.000141:
            score = "AA"
        if score == 0.000544:
            score = "A"
        if score == 0.001997:
            score = "BBB"
        if score == 0.008538:
            score = "BB"
        if score == 0.2433872:
            score = "CCC"
        if score not in dict_score.keys():
            dict_score[score] = 1
        else:
            dict_score[score] += 1

    label_region = dict_region.keys()
    label_sect = dict_sect.keys()
    label_score = dict_score.keys()
    region_size = []
    sect_size = []
    score_size = []
    for key in label_region:
        region_size.append(dict_region[key])
    for key in label_sect:
        sect_size.append(dict_sect[key])
    for key in label_score:
        score_size.append(dict_score[key])

    fig, (ax1, ax2) = plt.subplots(1, 2)

    ax1.pie(region_size, labels=label_region, autopct='%1.1f%%')
    ax1.set_title("Portfolio region distribution")
    # plt.show()
    ax2.pie(sect_size, labels=label_sect, autopct='%1.1f%%')
    ax2.set_title("Portfolio sector distribution")
    plt.show()
    plt.pie(score_size, labels=label_score, autopct='%1.1f%%')
    plt.title("Portfolio credit score distribution")
    plt.show()


from json import loads
from django.http import HttpResponse
from django.shortcuts import render,redirect
from .models import Swimmer,Award,Fact
from django.core.paginator import Paginator
from django.contrib.auth import login, logout
from .forms import SearchForm, UserRegisterForm, UserLoginForm, UserCreationForm
from django.contrib import messages
from django.db.models import Q

def HomePageRender(request):
    Swimmers = Swimmer.objects.all()
    paginator = Paginator(Swimmers, 12)
    page_number = request.GET.get('page', 1)
    page_obj = paginator.get_page(page_number)
    return render( request, 'main/HomePage.html', {'page_obj': page_obj} )

def Register(request):
    if request.method == 'POST':
        form = UserRegisterForm(request.POST)
        if form.is_valid():
            if form.cleaned_data['password1']==form.cleaned_data['password2']:
                user = form.save()
                login(request, user)
                messages.success(request, 'Вы успешно зарегистрировались')
                return redirect('Login')
            else:
                messages.error(request, 'Ошибка регистрации')
        else:
            messages.error(request, 'Ошибка регистрации')
    else:
        form = UserRegisterForm()
    return render(request, 'main/Register.html', {'form': form})

def Login(request):
    if request.method == 'POST':
        form = UserLoginForm(data=request.POST)
        if form.is_valid():
            user=form.get_user()
            login(request, user)
            messages.success(request, 'Вы успешно вошли')
            return redirect('Home')
        else:
            messages.error(request, 'Ошибка входа')
    else:
        form = UserLoginForm()
    return render(request, 'main/Login.html',  {'form':form})

def Logout(request):
    logout(request)
    return redirect('Login')

def Search(request):
    if request.method == 'POST':
        form = SearchForm(request.POST)

        if form.is_valid():

            qr = form.cleaned_data['search'].split()
            res = list()
            if qr != []:
                for i in qr:

                    a = Swimmer.objects.filter(Q(Name__icontains= i))
                    res.append(*a)
            return render( request, 'main/searchresult.html', {'swimmers':res} )
    else:
        form = SearchForm()
        return render(request, 'main/Search.html', {'form': form})

def SwimmerPage(request, id):
    Swimmers = Swimmer.objects.get(pk=id)
    return render(request, 'main/Swimmer.html', {'swimmer': Swimmers})

def renderadd(request):
    return render(request, 'main/AddSwimmer.html')

def createswimmer(request):
    my_json = request.body.decode('utf8')

    my_json = loads(my_json)
    print(type(my_json))
    sw = Swimmer.objects.create(Name = my_json['Name'], Age = int(my_json['Age']), Weight = int(my_json['Weight']))
    Awards = my_json['Awards']
    Facts = my_json['Facts']

    for award in Awards:
        aw = Award.objects.create(Name = award[0], Date = award[1])
        sw.Awards.add(aw)

    for fact in Facts:
        fc = Fact.objects.create(Name = fact[0], Content = fact[1])
        sw.Facts.add(fc)

    return HttpResponse()