# -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function

from functools import wraps
import logging
from numbers import Number
from time import time
from types import FunctionType

from lab import B
from plum import Dispatcher, Self, Referentiable

__all__ = []

_dispatch = Dispatcher()

@_dispatch(object)
def uprank(x):
    """Ensure that the rank of `x` is 2.

    Args:
        x (tensor): Tensor to uprank.

    Returns:
        tensor: `x` with rank at least 2.
    """
    # Simply return non-numerical inputs.
    if not isinstance(x, B.Numeric):
        return x

    # Now check the rank of `x` and act accordingly.
    rank = B.rank(x)
    if rank > 2:
        raise ValueError('Input must be at most rank 2.')
    elif rank == 2:
        return x
    elif rank == 1:
        return B.expand_dims(x, axis=1)
    else:
        # Rank must be 0.
        return B.expand_dims(B.expand_dims(x, axis=0), axis=1)

@_dispatch(FunctionType)
def uprank(f):
    """A decorator to ensure that the rank of the arguments is two."""

    @wraps(f)
    def wrapped_f(*args):
        return f(*[uprank(x) for x in args])

    return wrapped_f


import Chat
import nltk
from nltk.corpus import wordnet as wn
#nltk.download()
#!/usr/bin/python

def getSynonyms(words):
    synonyms = []
    for word in words:
        for s in wn.synsets(word):
            for l in s.lemmas():
                synonyms.append(l.name())
    return set(synonyms)

# findPersonKeywords is a function that takes a list of words (or user's "chat" in this context)
# returns the list of person names in the given list
def findPersonKeywords(words):
    persons = []

    if 'my' in words:
        myIndex = words.index('my')
        if (words[myIndex+1] !=  "roommate" and words[myIndex+1] != "friend" and words[myIndex+1] != "classmate"):
            persons.append(words[myIndex + 1])

    if 'with' in words:
        withIndex = words.index('with') # returns the index of the word 'with' in the list of words

        if words[withIndex + 1].lower() == 'my' and words[withIndex + 2].lower() != 'him' and words[withIndex + 2].lower() != 'her' and words[withIndex + 2].lower() != 'them':
            if words[withIndex+2] not in persons:
                persons.append(words[withIndex+2])
        if words[withIndex+1] not in persons:
            persons.append(words[withIndex+1])

    '''if 'to' in words:
        toIndex = words.index('to')

        if (words[toIndex - 1].lower() != 'went') and (words[toIndex - 1].lower() != 'how') and (words[toIndex -
        1].lower() != 'go') and (words[toIndex - 1].lower() != 'visit'): if words[toIndex + 1].lower() == 'my' and
        words[toIndex + 2] not in persons: persons.append(words[toIndex + 2]) else: personName = words[toIndex + 1]
        personName = personName.capitalize() if personName not in persons: persons.append(personName) '''

    return persons

# findLocationKeywords is a function that takes a list of words (or user's "chat" in this context)
# returns the list of location names in the given list
def findLocationKeywords(words):
    locations = []
    synonyms = getSynonyms(["travel"])

    if 'to' in words:
        toIndex = words.index('to')
        if (words[toIndex - 1].lower() in synonyms or words[toIndex - 1].lower() == 'went'):
            if (words[toIndex+1] == "the"):
                locationName = words[toIndex + 2].capitalize()
                if locationName not in locations:
                    locations.append(locationName)
            elif (words[toIndex+1]=="to"):
                if (words[toIndex + 1] == "the"):
                    locationName = words[toIndex + 2].capitalize()
                    if locationName not in locations:
                        locations.append(locationName)
            else:
                locationName = words[toIndex + 1].capitalize()
                if locationName not in locations:
                    locations.append(locationName)
    return locations

def determineBranch(words):
    depressedWordsList = ['stress','stressing','depressed','lonely','sad','unhappy', 'not well', 'unwell', 'miserable', 'upset', 'discouraged','broken-hearted','down', 'glum']
    neutralWordsList = ['good','alright','okay', 'fine', 'so-so', 'happy', 'content', 'cheery', 'blessed', 'thrilled']

    depressedWords = getSynonyms(depressedWordsList)
    neutralWords = getSynonyms(neutralWordsList)

    if (any(string in words for string in depressedWords)):
        branch = 'depressed'
        return branch
    if (any(string in words for string in neutralWords)):
        branch = 'neutral'
        return branch
    if 'suicidal' in words:
        branch = 'suicidal'
        return branch
    if 'not' in words:
        notIndex = words.index('not')
        if (words[notIndex+1] == "good") or (words[notIndex+1] == "well") or (words[notIndex+1] == "happy"):
            branch = 'depressed'
            return branch

    branch = Chat.getBranch()
    if branch=="":
        branch = "general"

    return branch

# PersonOrLocation is a function that takes a string (or user's "chat" in this context)
# Returns a dictionary data type with the list of people's names and locations.
def PersonOrLocation(string):
    words = string.split() # removes unnecessary left/right spaces and returns a list of words in the string
    return [findPersonKeywords(words),findLocationKeywords(words)]


# coding:utf-8
"""
@author:hanmy
@file:ParaEsti_N.py
@time:2019/04/24
"""
import numpy as np
import matplotlib.pyplot as plt
from gmm import GMM

# 计算steps次参数估计的权重值的均值和方差
def pi_N(N, pi, mean_1, cov_1, mean_2, cov_2, steps):
    gmm = GMM(N, pi, mean_1, cov_1, mean_2, cov_2)
    pi_steps = np.zeros(shape=steps)
    # 均值
    pi_mu = 0
    # 方差
    pi_sigma = 0

    # 计算steps次估计值和均值
    for i in range(steps):
        D = gmm.dataset()
        pi_learn, _, _, _, _ = gmm.EM(D, N)
        pi_mu += pi_learn
        pi_steps[i] = pi_learn

    pi_mu /= steps

    # 计算steps次方差
    for i in range(steps):
        pi_sigma += (pi_steps[i]-pi_mu)**2

    pi_sigma /= steps

    return pi_mu, pi_sigma

if __name__ == "__main__":
    # 样本点数Ni
    N1 = 100
    N2 = 1000
    N3 = 10000

    pi = 0.8
    mean_1 = np.array([0.0, 0.0])
    cov_1 = np.mat([[1.0, 0.0], [0.0, 1.0]])
    mean_2 = np.array([3.0, 3.0])
    cov_2 = np.mat([[1.0, 0.5], [0.5, 1.0]])

    # 参数估计次数steps
    steps = 10

    Y_mu = []
    Y_sigma = []

    pi_mu_100, pi_sigma_100 = pi_N(N1, pi, mean_1, cov_1, mean_2, cov_2, steps)
    Y_mu.append(pi_mu_100)
    Y_sigma.append(pi_sigma_100)
    print("N=100时学习", steps, "次得到的权重值均值：", pi_mu_100, ",方差：", pi_sigma_100)

    pi_mu_1000, pi_sigma_1000 = pi_N(N2, pi, mean_1, cov_1, mean_2, cov_2, steps)
    Y_mu.append(pi_mu_1000)
    Y_sigma.append(pi_sigma_1000)
    print("N=1000时学习", steps, "次得到的权重值均值：", pi_mu_1000, ",方差：", pi_sigma_1000)

    pi_mu_10000, pi_sigma_10000 = pi_N(N3, pi, mean_1, cov_1, mean_2, cov_2, steps)
    Y_mu.append(pi_mu_10000)
    Y_sigma.append(pi_sigma_10000)
    print("N=10000时学习", steps, "次得到的权重值均值：", pi_mu_10000, ",方差：", pi_sigma_10000)

    Y_mu.append(0.8)
    X = [1, 2, 3, 4]
    plt.bar(X, Y_mu, align='center', tick_label=['N=100', 'N=1000', 'N=10000', 'standard value'], width=0.5)
    plt.ylim((0.7, 0.85))
    plt.xlabel('different N')
    plt.ylabel('mean of pi')
    plt.show()


"""Django Models for tracking the configuration compliance per feature and device."""

import json
import logging

from deepdiff import DeepDiff
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.module_loading import import_string
from hier_config import Host as HierConfigHost
from nautobot.core.models.generics import PrimaryModel
from nautobot.core.models.utils import serialize_object, serialize_object_v2
from nautobot.dcim.models import Device
from nautobot.extras.models import ObjectChange
from nautobot.extras.models.statuses import StatusField
from nautobot.extras.utils import extras_features
from netutils.config.compliance import feature_compliance

from nautobot_golden_config.choices import ComplianceRuleConfigTypeChoice, ConfigPlanTypeChoice, RemediationTypeChoice
from nautobot_golden_config.utilities.constant import ENABLE_SOTAGG, PLUGIN_CFG

LOGGER = logging.getLogger(__name__)
GRAPHQL_STR_START = "query ($device_id: ID!)"

ERROR_MSG = (
    "There was an issue with the data that was returned by your get_custom_compliance function. "
    "This is a local issue that requires the attention of your systems administrator and not something "
    "that can be fixed within the Golden Config plugin. "
)
MISSING_MSG = (
    ERROR_MSG + "Specifically the `{}` key was not found in value the get_custom_compliance function provided."
)
VALIDATION_MSG = (
    ERROR_MSG + "Specifically the key {} was expected to be of type(s) {} and the value of {} was not that type(s)."
)

CUSTOM_FUNCTIONS = {
    "get_custom_compliance": "custom",
    "get_custom_remediation": RemediationTypeChoice.TYPE_CUSTOM,
}

def _is_jsonable(val):
    """Check is value can be converted to json."""
    try:
        json.dumps(val)
        return True
    except (TypeError, OverflowError):
        return False

def _null_to_empty(val):
    """Convert to empty string if the value is currently null."""
    if not val:
        return ""
    return val

def _get_cli_compliance(obj):
    """This function performs the actual compliance for cli configuration."""
    feature = {
        "ordered": obj.rule.config_ordered,
        "name": obj.rule,
    }
    feature.update({"section": obj.rule.match_config.splitlines()})
    value = feature_compliance(
        feature, obj.actual, obj.intended, obj.device.platform.network_driver_mappings.get("netutils_parser")
    )
    compliance = value["compliant"]
    if compliance:
        compliance_int = 1
        ordered = value["ordered_compliant"]
    else:
        compliance_int = 0
        ordered = value["ordered_compliant"]
    missing = _null_to_empty(value["missing"])
    extra = _null_to_empty(value["extra"])
    return {
        "compliance": compliance,
        "compliance_int": compliance_int,
        "ordered": ordered,
        "missing": missing,
        "extra": extra,
    }

def _get_json_compliance(obj):
    """This function performs the actual compliance for json serializable data."""

    def _normalize_diff(diff, path_to_diff):
        """Normalizes the diff to a list of keys and list indexes that have changed."""
        dictionary_items = list(diff.get(f"dictionary_item_{path_to_diff}", []))
        list_items = list(diff.get(f"iterable_item_{path_to_diff}", {}).keys())
        values_changed = list(diff.get("values_changed", {}).keys())
        type_changes = list(diff.get("type_changes", {}).keys())
        return dictionary_items + list_items + values_changed + type_changes

    diff = DeepDiff(obj.actual, obj.intended, ignore_order=obj.ordered, report_repetition=True)
    if not diff:
        compliance_int = 1
        compliance = True
        ordered = True
        missing = ""
        extra = ""
    else:
        compliance_int = 0
        compliance = False
        ordered = False
        missing = _null_to_empty(_normalize_diff(diff, "added"))
        extra = _null_to_empty(_normalize_diff(diff, "removed"))

    return {
        "compliance": compliance,
        "compliance_int": compliance_int,
        "ordered": ordered,
        "missing": missing,
        "extra": extra,
    }

def _verify_get_custom_compliance_data(compliance_details):
    """This function verifies the data is as expected when a custom function is used."""
    for val in ["compliance", "compliance_int", "ordered", "missing", "extra"]:
        try:
            compliance_details[val]
        except KeyError:
            raise ValidationError(MISSING_MSG.format(val)) from KeyError
    for val in ["compliance", "ordered"]:
        if compliance_details[val] not in [True, False]:
            raise ValidationError(VALIDATION_MSG.format(val, "Boolean", compliance_details[val]))
    if compliance_details["compliance_int"] not in [0, 1]:
        raise ValidationError(VALIDATION_MSG.format("compliance_int", "0 or 1", compliance_details["compliance_int"]))
    for val in ["missing", "extra"]:
        if not isinstance(compliance_details[val], str) and not _is_jsonable(compliance_details[val]):
            raise ValidationError(VALIDATION_MSG.format(val, "String or Json", compliance_details[val]))

def _get_hierconfig_remediation(obj):
    """Returns the remediating config."""
    hierconfig_os = obj.device.platform.network_driver_mappings["hier_config"]
    if not hierconfig_os:
        raise ValidationError(f"platform {obj.network_driver} is not supported by hierconfig.")

    try:
        remediation_setting_obj = RemediationSetting.objects.get(platform=obj.rule.platform)
    except Exception as err:  # pylint: disable=broad-except:
        raise ValidationError(f"Platform {obj.network_driver} has no Remediation Settings defined.") from err

    remediation_options = remediation_setting_obj.remediation_options

    try:
        hc_kwargs = {"hostname": obj.device.name, "os": hierconfig_os}
        if remediation_options:
            hc_kwargs.update(hconfig_options=remediation_options)
        host = HierConfigHost(**hc_kwargs)

    except Exception as err:  # pylint: disable=broad-except:
        raise Exception(  # pylint: disable=broad-exception-raised
            f"Cannot instantiate HierConfig on {obj.device.name}, check Device, Platform and Hier Options."
        ) from err

    host.load_generated_config(obj.intended)
    host.load_running_config(obj.actual)
    host.remediation_config()
    remediation_config = host.remediation_config_filtered_text(include_tags={}, exclude_tags={})

    return remediation_config

# The below maps the provided compliance types
FUNC_MAPPER = {
    ComplianceRuleConfigTypeChoice.TYPE_CLI: _get_cli_compliance,
    ComplianceRuleConfigTypeChoice.TYPE_JSON: _get_json_compliance,
    RemediationTypeChoice.TYPE_HIERCONFIG: _get_hierconfig_remediation,
}
# The below conditionally add the custom provided compliance type
for custom_function, custom_type in CUSTOM_FUNCTIONS.items():
    if PLUGIN_CFG.get(custom_function):
        try:
            FUNC_MAPPER[custom_type] = import_string(PLUGIN_CFG[custom_function])
        except Exception as error:  # pylint: disable=broad-except
            msg = (
                "There was an issue attempting to import the custom function of"
                f"{PLUGIN_CFG[custom_function]}, this is expected with a local configuration issue "
                "and not related to the Golden Configuration Plugin, please contact your system admin for further details"
            )
            raise Exception(msg).with_traceback(error.__traceback__)

@extras_features(
    "custom_fields",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class ComplianceFeature(PrimaryModel):  # pylint: disable=too-many-ancestors
    """ComplianceFeature details."""

    name = models.CharField(max_length=100, unique=True)
    slug = models.SlugField(max_length=100, unique=True)
    description = models.CharField(max_length=200, blank=True)

    class Meta:
        """Meta information for ComplianceFeature model."""

        ordering = ("slug",)

    def __str__(self):
        """Return a sane string representation of the instance."""
        return self.slug

@extras_features(
    "custom_fields",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class ComplianceRule(PrimaryModel):  # pylint: disable=too-many-ancestors
    """ComplianceRule details."""

    feature = models.ForeignKey(to="ComplianceFeature", on_delete=models.CASCADE, related_name="feature")

    platform = models.ForeignKey(
        to="dcim.Platform",
        on_delete=models.CASCADE,
        related_name="compliance_rules",
    )
    description = models.CharField(
        max_length=200,
        blank=True,
    )
    config_ordered = models.BooleanField(
        verbose_name="Configured Ordered",
        help_text="Whether or not the configuration order matters, such as in ACLs.",
        default=False,
    )

    config_remediation = models.BooleanField(
        default=False,
        verbose_name="Config Remediation",
        help_text="Whether or not the config remediation is executed for this compliance rule.",
    )

    match_config = models.TextField(
        blank=True,
        verbose_name="Config to Match",
        help_text="The config to match that is matched based on the parent most configuration. E.g.: For CLI `router bgp` or `ntp`. For JSON this is a top level key name.",
    )
    config_type = models.CharField(
        max_length=20,
        default=ComplianceRuleConfigTypeChoice.TYPE_CLI,
        choices=ComplianceRuleConfigTypeChoice,
        help_text="Whether the configuration is in CLI or JSON/structured format.",
    )
    custom_compliance = models.BooleanField(
        default=False, help_text="Whether this Compliance Rule is proceeded as custom."
    )

    @property
    def remediation_setting(self):
        """Returns remediation settings for a particular platform."""
        return RemediationSetting.objects.filter(platform=self.platform).first()

    class Meta:
        """Meta information for ComplianceRule model."""

        ordering = ("platform", "feature__name")
        unique_together = (
            "feature",
            "platform",
        )

    def __str__(self):
        """Return a sane string representation of the instance."""
        return f"{self.platform} - {self.feature.name}"

    def clean(self):
        """Verify that if cli, then match_config is set."""
        if self.config_type == ComplianceRuleConfigTypeChoice.TYPE_CLI and not self.match_config:
            raise ValidationError("CLI configuration set, but no configuration set to match.")

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class ConfigCompliance(PrimaryModel):  # pylint: disable=too-many-ancestors
    """Configuration compliance details."""

    device = models.ForeignKey(to="dcim.Device", on_delete=models.CASCADE, help_text="The device")
    rule = models.ForeignKey(to="ComplianceRule", on_delete=models.CASCADE, related_name="rule")
    compliance = models.BooleanField(blank=True)
    actual = models.JSONField(blank=True, help_text="Actual Configuration for feature")
    intended = models.JSONField(blank=True, help_text="Intended Configuration for feature")
    # these three are config snippets exposed for the ConfigDeployment.
    remediation = models.JSONField(blank=True, help_text="Remediation Configuration for the device")
    missing = models.JSONField(blank=True, help_text="Configuration that should be on the device.")
    extra = models.JSONField(blank=True, help_text="Configuration that should not be on the device.")
    ordered = models.BooleanField(default=False)
    # Used for django-pivot, both compliance and compliance_int should be set.
    compliance_int = models.IntegerField(blank=True)

    def to_objectchange(
        self, action, *, related_object=None, object_data_extra=None, object_data_exclude=None
    ):  # pylint: disable=arguments-differ
        """Remove actual and intended configuration from changelog."""
        if not object_data_exclude:
            object_data_exclude = ["actual", "intended"]
        return ObjectChange(
            changed_object=self,
            object_repr=str(self),
            action=action,
            object_data=serialize_object(self, extra=object_data_extra, exclude=object_data_exclude),
            object_data_v2=serialize_object_v2(self),
            related_object=related_object,
        )

    class Meta:
        """Set unique together fields for model."""

        ordering = ["device", "rule"]
        unique_together = ("device", "rule")

    def __str__(self):
        """String representation of a the compliance."""
        return f"{self.device} -> {self.rule} -> {self.compliance}"

    def compliance_on_save(self):
        """The actual configuration compliance happens here, but the details for actual compliance job would be found in FUNC_MAPPER."""
        if self.rule.custom_compliance:
            if not FUNC_MAPPER.get("custom"):
                raise ValidationError(
                    "Custom type provided, but no `get_custom_compliance` config set, please contact system admin."
                )
            compliance_details = FUNC_MAPPER["custom"](obj=self)
            _verify_get_custom_compliance_data(compliance_details)
        else:
            compliance_details = FUNC_MAPPER[self.rule.config_type](obj=self)

        self.compliance = compliance_details["compliance"]
        self.compliance_int = compliance_details["compliance_int"]
        self.ordered = compliance_details["ordered"]
        self.missing = compliance_details["missing"]
        self.extra = compliance_details["extra"]

    def remediation_on_save(self):
        """The actual remediation happens here, before saving the object."""
        if self.compliance:
            self.remediation = ""
            return

        if not self.rule.config_remediation:
            self.remediation = ""
            return

        if not self.rule.remediation_setting:
            self.remediation = ""
            return

        remediation_config = FUNC_MAPPER[self.rule.remediation_setting.remediation_type](obj=self)
        self.remediation = remediation_config

    def save(self, *args, **kwargs):
        """The actual configuration compliance happens here, but the details for actual compliance job would be found in FUNC_MAPPER."""
        self.compliance_on_save()
        self.remediation_on_save()
        self.full_clean()

        super().save(*args, **kwargs)

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class GoldenConfig(PrimaryModel):  # pylint: disable=too-many-ancestors
    """Configuration Management Model."""

    device = models.OneToOneField(
        to="dcim.Device",
        on_delete=models.CASCADE,
        help_text="device",
        blank=False,
    )
    backup_config = models.TextField(blank=True, help_text="Full backup config for device.")
    backup_last_attempt_date = models.DateTimeField(null=True, blank=True)
    backup_last_success_date = models.DateTimeField(null=True, blank=True)

    intended_config = models.TextField(blank=True, help_text="Intended config for the device.")
    intended_last_attempt_date = models.DateTimeField(null=True, blank=True)
    intended_last_success_date = models.DateTimeField(null=True, blank=True)

    compliance_config = models.TextField(blank=True, help_text="Full config diff for device.")
    compliance_last_attempt_date = models.DateTimeField(null=True, blank=True)
    compliance_last_success_date = models.DateTimeField(null=True, blank=True)

    def to_objectchange(
        self, action, *, related_object=None, object_data_extra=None, object_data_exclude=None
    ):  # pylint: disable=arguments-differ
        """Remove actual and intended configuration from changelog."""
        if not object_data_exclude:
            object_data_exclude = ["backup_config", "intended_config", "compliance_config"]
        return ObjectChange(
            changed_object=self,
            object_repr=str(self),
            action=action,
            object_data=serialize_object(self, extra=object_data_extra, exclude=object_data_exclude),
            object_data_v2=serialize_object_v2(self),
            related_object=related_object,
        )

    @staticmethod
    def get_dynamic_group_device_pks():
        """Get all Device PKs associated with GoldenConfigSetting DynamicGroups."""
        gc_dynamic_group_device_queryset = Device.objects.none()
        for setting in GoldenConfigSetting.objects.all():
            # using "|" should not require calling distinct afterwards
            gc_dynamic_group_device_queryset = gc_dynamic_group_device_queryset | setting.dynamic_group.members

        return set(gc_dynamic_group_device_queryset.values_list("pk", flat=True))

    @classmethod
    def get_golden_config_device_ids(cls):
        """Get all Device PKs associated with GoldenConfig entries."""
        return set(cls.objects.values_list("device__pk", flat=True))

    class Meta:
        """Set unique together fields for model."""

        ordering = ["device"]

    def __str__(self):
        """String representation of a the compliance."""
        return f"{self.device}"

@extras_features(
    "graphql",
)
class GoldenConfigSetting(PrimaryModel):  # pylint: disable=too-many-ancestors
    """GoldenConfigSetting Model definition. This provides global configs instead of via configs.py."""

    name = models.CharField(max_length=100, unique=True)
    slug = models.SlugField(max_length=100, unique=True)
    weight = models.PositiveSmallIntegerField(default=1000)
    description = models.CharField(
        max_length=200,
        blank=True,
    )
    backup_repository = models.ForeignKey(
        to="extras.GitRepository",
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="backup_repository",
        limit_choices_to={"provided_contents__contains": "nautobot_golden_config.backupconfigs"},
    )
    backup_path_template = models.CharField(
        max_length=255,
        blank=True,
        verbose_name="Backup Path in Jinja Template Form",
        help_text="The Jinja path representation of where the backup file will be found. The variable `obj` is available as the device instance object of a given device, as is the case for all Jinja templates. e.g. `{{obj.location.name|slugify}}/{{obj.name}}.cfg`",
    )
    intended_repository = models.ForeignKey(
        to="extras.GitRepository",
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="intended_repository",
        limit_choices_to={"provided_contents__contains": "nautobot_golden_config.intendedconfigs"},
    )
    intended_path_template = models.CharField(
        max_length=255,
        blank=True,
        verbose_name="Intended Path in Jinja Template Form",
        help_text="The Jinja path representation of where the generated file will be places. e.g. `{{obj.location.name|slugify}}/{{obj.name}}.cfg`",
    )
    jinja_repository = models.ForeignKey(
        to="extras.GitRepository",
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="jinja_template",
        limit_choices_to={"provided_contents__contains": "nautobot_golden_config.jinjatemplate"},
    )
    jinja_path_template = models.CharField(
        max_length=255,
        blank=True,
        verbose_name="Template Path in Jinja Template Form",
        help_text="The Jinja path representation of where the Jinja template can be found. e.g. `{{obj.platform.network_driver}}.j2`",
    )
    backup_test_connectivity = models.BooleanField(
        default=True,
        verbose_name="Backup Test",
        help_text="Whether or not to pretest the connectivity of the device by verifying there is a resolvable IP that can connect to port 22.",
    )
    sot_agg_query = models.ForeignKey(
        to="extras.GraphQLQuery",
        on_delete=models.PROTECT,
        null=True,
        blank=True,
        related_name="sot_aggregation",
    )
    dynamic_group = models.OneToOneField(
        to="extras.DynamicGroup",
        on_delete=models.PROTECT,
        related_name="golden_config_setting",
    )

    def __str__(self):
        """Return a simple string if model is called."""
        return f"Golden Config Setting - {self.name}"

    class Meta:
        """Set unique fields for model.

        Provide ordering used in tables and get_device_to_settings_map.
        Sorting on weight is performed from the highest weight value to the lowest weight value.
        This is to ensure only one plugin settings could be applied per single device based on priority and name.
        """

        verbose_name = "Golden Config Setting"
        ordering = ["-weight", "name"]  # Refer to weight comment in class docstring.

    def clean(self):
        """Validate the scope and GraphQL query."""
        super().clean()

        if ENABLE_SOTAGG and not self.sot_agg_query:
            raise ValidationError("A GraphQL query must be defined when `ENABLE_SOTAGG` is True")

        if self.sot_agg_query:
            LOGGER.debug("GraphQL - test  query start with: `%s`", GRAPHQL_STR_START)
            if not str(self.sot_agg_query.query.lstrip()).startswith(GRAPHQL_STR_START):
                raise ValidationError(f"The GraphQL query must start with exactly `{GRAPHQL_STR_START}`")

    def get_queryset(self):
        """Generate a Device QuerySet from the filter."""
        return self.dynamic_group.members

    def device_count(self):
        """Return the number of devices in the group."""
        return self.dynamic_group.count

    def get_url_to_filtered_device_list(self):
        """Get url to all devices that are matching the filter."""
        return self.dynamic_group.get_group_members_url()

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class ConfigRemove(PrimaryModel):  # pylint: disable=too-many-ancestors
    """ConfigRemove for Regex Line Removals from Backup Configuration Model definition."""

    name = models.CharField(max_length=255)
    platform = models.ForeignKey(
        to="dcim.Platform",
        on_delete=models.CASCADE,
        related_name="backup_line_remove",
    )
    description = models.CharField(
        max_length=200,
        blank=True,
    )
    regex = models.CharField(
        max_length=200,
        verbose_name="Regex Pattern",
        help_text="Regex pattern used to remove a line from the backup configuration.",
    )

    clone_fields = ["platform", "description", "regex"]

    class Meta:
        """Meta information for ConfigRemove model."""

        ordering = ("platform", "name")
        unique_together = ("name", "platform")

    def __str__(self):
        """Return a simple string if model is called."""
        return self.name

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class ConfigReplace(PrimaryModel):  # pylint: disable=too-many-ancestors
    """ConfigReplace for Regex Line Replacements from Backup Configuration Model definition."""

    name = models.CharField(max_length=255)
    platform = models.ForeignKey(
        to="dcim.Platform",
        on_delete=models.CASCADE,
        related_name="backup_line_replace",
    )
    description = models.CharField(
        max_length=200,
        blank=True,
    )
    regex = models.CharField(
        max_length=200,
        verbose_name="Regex Pattern to Substitute",
        help_text="Regex pattern that will be found and replaced with 'replaced text'.",
    )
    replace = models.CharField(
        max_length=200,
        verbose_name="Replaced Text",
        help_text="Text that will be inserted in place of Regex pattern match.",
    )

    clone_fields = ["platform", "description", "regex", "replace"]

    class Meta:
        """Meta information for ConfigReplace model."""

        ordering = ("platform", "name")
        unique_together = ("name", "platform")

    def __str__(self):
        """Return a simple string if model is called."""
        return self.name

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
)
class RemediationSetting(PrimaryModel):  # pylint: disable=too-many-ancestors
    """RemediationSetting details."""

    # Remediation points to the platform
    platform = models.OneToOneField(
        to="dcim.Platform",
        on_delete=models.CASCADE,
        related_name="remediation_settings",
    )

    remediation_type = models.CharField(
        max_length=50,
        default=RemediationTypeChoice.TYPE_HIERCONFIG,
        choices=RemediationTypeChoice,
        help_text="Whether the remediation setting is type HierConfig or custom.",
    )

    # takes options.json.
    remediation_options = models.JSONField(
        blank=True,
        default=dict,
        help_text="Remediation Configuration for the device",
    )

    csv_headers = [
        "platform",
        "remediation_type",
    ]

    class Meta:
        """Meta information for RemediationSettings model."""

        ordering = ("platform", "remediation_type")

    def to_csv(self):
        """Indicates model fields to return as csv."""
        return (
            self.platform,
            self.remediation_type,
        )

    def __str__(self):
        """Return a sane string representation of the instance."""
        return str(self.platform)

@extras_features(
    "custom_fields",
    "custom_links",
    "custom_validators",
    "export_templates",
    "graphql",
    "relationships",
    "webhooks",
    "statuses",
)
class ConfigPlan(PrimaryModel):  # pylint: disable=too-many-ancestors
    """ConfigPlan for Golden Configuration Plan Model definition."""

    plan_type = models.CharField(max_length=20, choices=ConfigPlanTypeChoice, verbose_name="Plan Type")
    device = models.ForeignKey(
        to="dcim.Device",
        on_delete=models.CASCADE,
        related_name="config_plan",
    )
    config_set = models.TextField(help_text="Configuration set to be applied to device.")
    feature = models.ManyToManyField(
        to=ComplianceFeature,
        related_name="config_plan",
        blank=True,
    )
    plan_result = models.ForeignKey(
        to="extras.JobResult",
        on_delete=models.CASCADE,
        related_name="config_plan",
        verbose_name="Plan Result",
    )
    deploy_result = models.ForeignKey(
        to="extras.JobResult",
        on_delete=models.PROTECT,
        related_name="config_plan_deploy_result",
        verbose_name="Deploy Result",
        blank=True,
        null=True,
    )
    change_control_id = models.CharField(
        max_length=50,
        blank=True,
        verbose_name="Change Control ID",
        help_text="Change Control ID for this configuration plan.",
    )
    change_control_url = models.URLField(blank=True, verbose_name="Change Control URL")
    status = StatusField(blank=True, null=True, on_delete=models.PROTECT)

    class Meta:
        """Meta information for ConfigPlan model."""

        ordering = ("-created", "device")
        unique_together = (
            "plan_type",
            "device",
            "created",
        )

    def __str__(self):
        """Return a simple string if model is called."""
        return f"{self.device.name}-{self.plan_type}-{self.created}"


import argparse
import logging
import multiprocessing as mp
import os
import pytz
import shutil
from datetime import datetime

from lego_prover.env.chromas import ChromaBridge
from lego_prover.evolver import Evolver
from lego_prover.prover import Prover
import lego_prover.utils as U
from openai_key import *

parser = argparse.ArgumentParser(description='LEGO-Prover')
parser.add_argument('--resume', action='store_true',
                    help='whether to resume from the checkpoint')
parser.add_argument('--data_split', type=str, choices=['valid', 'test'],
                    default='valid', help='data split to use in the miniF2F dataset')
parser.add_argument('--ckpt_dir', type=str, default='checkpoints/lego_prover_valid_2023_10_27',
                    help='path to the checkpoint directory')
parser.add_argument('--isabelle_path', type=str, default='/data2/wanghaiming/Isabelle2022/',
                    help='path to the Isabelle2022 directory')
parser.add_argument('--model_name', type=str, choices=["gpt-3.5-turbo", "gpt-4"],
                    default='gpt-3.5-turbo', help='OpenAI model name')
parser.add_argument('--temperature', type=float, default=0.7,
                    help='temperature for sampling the LLM')
parser.add_argument('--num_prover', type=int, default=3,
                    help='number of prover processes')
parser.add_argument('--num_evolver', type=int, default=8,
                    help='number of evolver processes')
parser.add_argument('--num_attempts', type=int, default=100,
                    help='number of proving attempts for each problem in the dataset')

args = parser.parse_args()

resume = args.resume
data_split = args.data_split
ckpt_dir = args.ckpt_dir
isabelle_path = args.isabelle_path
model_name = args.model_name
temperature = args.temperature
number_of_prover_processes = args.num_prover
number_of_evolver_processes = args.num_evolver
number_of_prover_attempts = args.num_attempts

if os.path.exists(ckpt_dir) and not resume:
    text = input(f"the checkpoint directory {ckpt_dir} is already exist, and" + \
                 f"you are not resuming from it, do you want to delete it? (y/n)")
    if "y" in text.lower():
        shutil.rmtree(ckpt_dir, ignore_errors=True)
        resume = False
    else:
        resume = True

# load miniF2F tasks and resume from the checkpoint
miniF2F_tasks = mp.Queue()
problem_names = []
if resume:
    if os.path.exists(f"{ckpt_dir}/curriculum/completed_tasks.json"):
        completed_tasks = U.load_json(
            f"{ckpt_dir}/curriculum/completed_tasks.json")
    if os.path.exists(f"{ckpt_dir}/curriculum/failed_tasks.json"):
        failed_tasks = U.load_json(f"{ckpt_dir}/curriculum/failed_tasks.json")
    print("Current progress: ", len(completed_tasks) + len(set(failed_tasks)))
else:
    completed_tasks = []
    failed_tasks = []
for name in os.listdir(f"data/full_data/{data_split}"):
    path = os.path.join(f"data/full_data/{data_split}", name)
    context = U.load_json(path)
    problem_names.append((path, len(context["informal_proof"])))
problem_names = sorted(problem_names, key=lambda x: x[1])
problem_names = [pn[0] for pn in problem_names]
problem_names = problem_names * number_of_prover_attempts     # 10 * 20 = 200 sketch
for pn in problem_names:
    if pn in completed_tasks:
        continue
    if pn in failed_tasks:
        failed_tasks.remove(pn)
        continue
    miniF2F_tasks.put(pn)
print(f"Sketch to finish: {miniF2F_tasks.qsize()}")

# setup multiprocessing logger
start_time = datetime.now(pytz.timezone(
    'Asia/Shanghai')).strftime("%Y%m%d_%H%M%S")

os.makedirs(f'logs/prover/{start_time}_logs', exist_ok=True)
for rank in range(number_of_prover_processes):
    logger = logging.getLogger(f'prover-{rank}')
    handler = logging.FileHandler(
        f"logs/prover/{start_time}_logs/rank_{rank}.log")
    formatter = logging.Formatter(
        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
    )
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    logger.setLevel(logging.INFO)

os.makedirs(f'logs/evolver/{start_time}_logs', exist_ok=True)
for evolver_rank in range(number_of_evolver_processes):
    evolver_rank += number_of_prover_processes
    logger = logging.getLogger(f'evolver-{evolver_rank}')
    handler = logging.FileHandler(
        f"logs/evolver/{start_time}_logs/rank_{evolver_rank}.log")
    formatter = logging.Formatter(
        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
    )
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    logger.setLevel(logging.INFO)

# define the function to run the prover and evolver
def run_prover(rank, tasks, skill_manager_lock, curriculum_agent_lock, chroma_bridge):
    server_port = 8051 + rank

    prover = Prover(
        rank=rank,
        isabelle_path=isabelle_path,
        server_port=server_port,
        model_name=model_name,
        skill_manager_lock=skill_manager_lock,
        action_agent_task_max_retries=1,
        curriculum_task_type="queue_curriculum",
        curriculum_agent_lock=curriculum_agent_lock,
        resume=resume,
        temperature=temperature,
        miniF2F_tasks=tasks,
        ckpt_dir=ckpt_dir,
        chroma_bridge=chroma_bridge,
    )
    prover.learn()

def run_evolver(rank, skill_manager_lock, chroma_bridge):
    server_port = 8011 + rank
    evolver = Evolver(
        rank=rank,
        isabelle_path=isabelle_path,
        ckpt_dir=ckpt_dir,
        server_port=server_port,
        data_split=data_split,
        skill_manager_lock=skill_manager_lock,
        model_name=model_name,
        temperature=temperature,
        chroma_bridge=chroma_bridge
    )
    evolver.evolve()

processes = []
skill_manager_lock = mp.Lock()
curriculum_agent_lock = mp.Lock()
chroma_bridge = ChromaBridge(ckpt_path=ckpt_dir, resume=resume)

# creating processes
for rank in range(number_of_prover_processes):
    p = mp.Process(target=run_prover, args=(rank, miniF2F_tasks,
                   skill_manager_lock, curriculum_agent_lock, chroma_bridge))
    processes.append(p)
    p.start()

for rank in range(number_of_evolver_processes):
    rank += number_of_prover_processes
    p = mp.Process(target=run_evolver, args=(
        rank, skill_manager_lock, chroma_bridge))
    processes.append(p)
    p.start()

# completing process
for p in processes:
    p.join()


# Python standard library
import sys
sys.path.append('./../pyqtgraph')

# Scipy
import matplotlib.backends.backend_qt4agg
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from matplotlib.patches import Polygon
from mpl_toolkits.basemap import Basemap
import numpy as np

# PyQt
import PyQt4.QtCore
import PyQt4.QtGui
import pyqtgraph as pg

class GlobePlot(matplotlib.backends.backend_qt4agg.FigureCanvas):
    def __init__(self, parent=None):

        self.fig = plt.figure(figsize=(3,3), dpi = 10)
        self.axes = self.fig.add_subplot(111)

        self.patches = {'west_greenland': None,\
            'bering': None,\
            'severny': None,\
            'hudson': None,\
            'custom': None}

        matplotlib.backends.backend_qt4agg.FigureCanvas.__init__(self, self.fig)
        self.setParent(parent)

        matplotlib.backends.backend_qt4agg.FigureCanvas.setSizePolicy(self,
                                   PyQt4.QtGui.QSizePolicy.Expanding,
                                   PyQt4.QtGui.QSizePolicy.Expanding)
        matplotlib.backends.backend_qt4agg.FigureCanvas.updateGeometry(self)

        self.setup_map()

        self.width, self.height = 565, 565

        self.plot_all_roi_patches()

    def setup_map(self):

        self.map = Basemap(projection='npstere',boundinglat=50,lon_0=270,resolution='l')
        self.toolbar = matplotlib.backends.backend_qt4agg.NavigationToolbar2QT(self, self)
        self.toolbar.hide()
        self.toolbar.pan()

        self.im1 = None

        self.map.drawparallels(np.arange(50.,90.,10.), labels=[0,0,0,0],zorder=50, color='grey', linewidth=5)
        self.map.drawmeridians(np.arange(-180.,181.,20.),latmax=90, labels=[0,0,0,0],zorder=50,color='grey', linewidth=5)
        self.map.drawcoastlines(linewidth=1)
        self.map.fillcontinents(color='chocolate',lake_color='lightblue', alpha=1, zorder=20)
        self.map.drawmapboundary(fill_color='brown', linewidth=1)

    def get_figure_coordinates(self, coords):
        xpt, ypt = self.map(coords[0], coords[1])

        xy_pixels = self.axes.transData.transform([xpt,ypt])
        #xy_pixels = self.fig.transFigure.transform()

        #xy_pixels = self.fig.transFigure.inverted().transform(xy_pixels)

        fig_x = xy_pixels[0]
        fig_y = self.height - xy_pixels[1]

        return fig_x, fig_y

    def set_data(self, lats, longs, data):

        if self.im1:
            self.im1.remove()

        self.im1 = self.map.pcolormesh(longs, lats, data,
                             shading='flat',
                             cmap=cm.seismic,
                             latlon=True,
                            zorder=0)

        self.colorbar = self.map.colorbar(self.im1, "bottom", size="5%", pad="2%")

        self.fig.tight_layout()
        self.draw()

    def plot_roi_patch(self, roi_name, latmin, latmax, lonmin, lonmax):

        x1,y1 = self.map(lonmax, latmax)   # lon_max, lat_max
        x2,y2 = self.map(lonmax, latmin)   # lon_max, lat_min
        xmid1, ymid1 = self.map( (lonmax+lonmin)/2, latmin  )
        x3,y3 = self.map(lonmin, latmin)   # lon_min, lat_min
        x4,y4 = self.map(lonmin, latmax)  # lon_min, lat_max
        xmid2, ymid2 = self.map( (lonmax+lonmin)/2, latmax  )

        boundary=[]
        boundary.append([x1,y1])
        boundary.append([x2,y2])

        for n in range((lonmax-lonmin)*3):
            xx,yy = self.map(lonmax-(n+1)/3., latmin)
            boundary.append([xx,yy])

        boundary.append([x3,y3])
        boundary.append([x4,y4])

        for n in range((lonmax-lonmin)*3):
            xx,yy = self.map(lonmin+(n+1)/3., latmax)
            boundary.append([xx,yy])

        if self.patches[roi_name]:
            self.patches[roi_name].remove()

        self.patches[roi_name] = Polygon(boundary,
            facecolor='yellow', edgecolor='yellow',linewidth = 6, zorder = 1000, alpha = 0.35, ls = '-')
        plt.gca().add_patch(self.patches[roi_name])

    def plot_all_roi_patches(self):
        # West greenland
        latmin, latmax= 66,75
        lonmin, lonmax= 290,310 #220,240
        self.plot_roi_patch('west_greenland', latmin,latmax,lonmin,lonmax)

        # Hudson Bay
        latmin, latmax= 55,65
        lonmin, lonmax= 265,285
        self.plot_roi_patch('hudson', latmin,latmax,lonmin,lonmax)

        # Bering Strait
        latmin, latmax= 63,70
        lonmin, lonmax= 180,199
        self.plot_roi_patch('bering', latmin,latmax,lonmin,lonmax)

        # Severny
        latmin, latmax = 70, 80
        lonmin, lonmax = 50, 75
        self.plot_roi_patch('severny', latmin, latmax, lonmin, lonmax)

class View(PyQt4.QtGui.QMainWindow):
    def __init__(self, parent = None):
        super(View, self).__init__(None)

        self.res = [1280,800]

        self.showFullScreen()
        #self.setGeometry(0,0,800,600)

        self.setup_layout()

        self.setup_time_widgets()
        self.setup_stats_roi_sliders()
        self.setup_globe_plot()
        self.setup_stats_plot()
        self.setup_loc_buttons()
        self.setup_roi_sliders()
        self.setup_conc_labels()

        self.globe_plot.get_figure_coordinates([0,0])

        self.setup_roi_sliders()

    def setup_conc_labels(self):
        self.conc_0_lineedit = PyQt4.QtGui.QLineEdit('0', parent = self)
        self.conc_0_lineedit.setGeometry(50,800,50,55)
        self.conc_0_lineedit.setStyleSheet("""
            .QLineEdit {
                border: 0px solid black;
                border-radius: 10px;
                background-color: rgba(0, 0, 255, 0);
                }
                """)
        self.conc_0_lineedit.show()

        self.conc_50_lineedit = PyQt4.QtGui.QLineEdit('50', parent = self)
        self.conc_50_lineedit.setGeometry(50+565/2,800,50,55)
        self.conc_50_lineedit.setStyleSheet("""
            .QLineEdit {
                border: 0px solid black;
                border-radius: 10px;
                background-color: rgba(255, 255, 255, 0);
                }
                """)
        self.conc_50_lineedit.show()

        self.conc_100_lineedit = PyQt4.QtGui.QLineEdit('100', parent = self)
        self.conc_100_lineedit.setGeometry(50+565,800,50,55)
        self.conc_100_lineedit.setStyleSheet("""
            .QLineEdit {
                border: 0px solid black;
                border-radius: 10px;
                background-color: rgba(255, 0, 0, 0);
                }
                """)
        self.conc_100_lineedit.show()

    def setup_roi_sliders(self):
        self.roi_lat0_slider = PyQt4.QtGui.QSlider(parent = self, orientation = PyQt4.QtCore.Qt.Horizontal)
        self.roi_lat0_slider.setGeometry(715,150,250,50)
        self.roi_lat0_slider.setMinimum(40)
        self.roi_lat0_slider.setMaximum(90)
        self.roi_lat0_slider.show()

        self.roi_long0_slider = PyQt4.QtGui.QSlider(parent = self, orientation = PyQt4.QtCore.Qt.Horizontal)
        self.roi_long0_slider.setGeometry(715,100,250,50)
        self.roi_long0_slider.setMinimum(0)
        self.roi_long0_slider.setMaximum(360) # Total data points
        self.roi_long0_slider.show()

        self.roi_lat1_slider = PyQt4.QtGui.QSlider(parent = self, orientation = PyQt4.QtCore.Qt.Horizontal)
        self.roi_lat1_slider.setGeometry(980,150,250,50)
        self.roi_lat1_slider.setMinimum(40)
        self.roi_lat1_slider.setMaximum(90)
        self.roi_lat1_slider.show()

        self.roi_long1_slider = PyQt4.QtGui.QSlider(parent = self, orientation = PyQt4.QtCore.Qt.Horizontal)
        self.roi_long1_slider.setGeometry(980,100,250,50)
        self.roi_long1_slider.setMinimum(0)
        self.roi_long1_slider.setMaximum(360) # Total data points
        self.roi_long1_slider.show()

        self.roi_lat_lineedit = PyQt4.QtGui.QLineEdit('LAT', parent = self)
        self.roi_lat_lineedit.setGeometry(665,150,50,65)
        self.roi_lat_lineedit.show()

        self.roi_long_lineedit = PyQt4.QtGui.QLineEdit('LONG', parent = self)
        self.roi_long_lineedit.setGeometry(665,100,50,65)
        self.roi_long_lineedit.show()

        self.active_roi_lineedit = PyQt4.QtGui.QLineEdit('', parent = self)
        self.active_roi_lineedit.setGeometry(665,50,200,65)
        self.active_roi_lineedit.show()

        for le in [self.roi_long_lineedit, self.roi_lat_lineedit, self.active_roi_lineedit]:
            le.setStyleSheet("""
            .QLineEdit {
                border: 0px solid black;
                border-radius: 10px;
                background-color: rgba(255, 255, 255, 0);
                }
                """)

    def set_defaults(self):
        # Time

        self.severny_loc_button.clicked.emit(True)

        self.time_month_slider.setSliderPosition(2)
        self.time_month_slider.sliderReleased.emit()

        self.time_year_slider.setSliderPosition(1950)
        self.time_year_slider.sliderReleased.emit()

    def setup_layout(self):

        return

    def setup_time_widgets(self):
        # slider
        self.time_year_slider = PyQt4.QtGui.QSlider(parent = self, orientation = PyQt4.QtCore.Qt.Horizontal)
        self.time_year_slider.setGeometry(50,150,500,50)
        self.time_year_slider.setMinimum(1850)
        self.time_year_slider.setMaximum(2013)
        self.time_year_slider.show()

        self.time_month_slider = PyQt4.QtGui.QSlider(parent = self, orientation = PyQt4.QtCore.Qt.Horizontal)
        self.time_month_slider.setGeometry(50,100,500,50)
        self.time_month_slider.setMinimum(0)
        self.time_month_slider.setMaximum(11) # Total data points
        self.time_month_slider.show()

        # lineedit
        self.time_year_lineedit = PyQt4.QtGui.QLineEdit(parent = self)
        self.time_year_lineedit.setGeometry(550,150,100,65)
        self.time_year_lineedit.setAlignment(PyQt4.QtCore.Qt.AlignHCenter)
        self.time_year_lineedit.setStyleSheet("""
        .QLineEdit {
            border: 0px solid black;
            border-radius: 10px;
            background-color: rgba(255, 255, 255, 0);
            }
        """)
        self.time_year_lineedit.show()

        self.time_month_lineedit = PyQt4.QtGui.QLineEdit(parent = self)
        self.time_month_lineedit.setGeometry(550,100,100,65)
        self.time_month_lineedit.setAlignment(PyQt4.QtCore.Qt.AlignHCenter)
        self.time_month_lineedit.setStyleSheet("""
        .QLineEdit {
            border: 0px solid black;
            border-radius: 10px;
            background-color: rgba(255, 255, 255, 0);
            }
        """)
        self.time_month_lineedit.show()

    def setup_stats_roi_sliders(self):
        pass

    def setup_globe_plot(self):
        self.globe_plot = GlobePlot(self)
        self.globe_plot.setGeometry(50,200,565,565)
        self.globe_plot.show()

    def setup_stats_plot(self):
        self.stats_plot = pg.PlotWidget(parent = self)

        self.stats_plot.setLabel('left', text = 'Mean concentration (units)')
        self.stats_plot.setLabel('bottom', text = 'Time (s)')
        self.stats_plot.showGrid(x = True, y = True, alpha = 0.2)

        self.stats_plot.setGeometry(665,200,565,565)

        pen_0 = PyQt4.QtGui.QPen(PyQt4.QtGui.QColor(238,50,50))
        pen_1 = PyQt4.QtGui.QPen(PyQt4.QtGui.QColor(50,175,238))

        self.stats_plot_item_roi_mar = pg.PlotDataItem(name = 'Mar')
        self.stats_plot_item_roi_mar.setPen(pen_1)

        self.stats_plot.addItem(self.stats_plot_item_roi_mar)

        self.stats_plot_item_roi_sep = pg.PlotDataItem(name = 'Sep')
        self.stats_plot_item_roi_sep.setPen(pen_0)

        self.stats_plot.addItem(self.stats_plot_item_roi_sep)

        self.stats_plot.addLegend()

        #self.stats_plot_item_full = pg.PlotDataItem()
        #self.stats_plot_item_full.setPen(pen_1)

        #self.stats_plot.addItem(self.stats_plot_item_full)

        self.stats_plot.show()

    def setup_loc_buttons(self):
        # West greenland
        self.west_greenland_loc_button = PyQt4.QtGui.QPushButton('X', self)
        self.west_greenland_loc_button.setStyleSheet('''
                            background-color: rgba(255,255,255,0);
                            color: green
                            ''' )
        self.west_greenland_loc_button.setFont(PyQt4.QtGui.QFont("Arial", 20, PyQt4.QtGui.QFont.Bold))
        self.west_greenland_loc_button.setMaximumWidth(30)
        self.west_greenland_loc_button.setMaximumHeight(30)

        self.west_greenland_loc_button.show()

        # Bering
        self.bering_loc_button = PyQt4.QtGui.QPushButton('X', self)
        self.bering_loc_button.setStyleSheet('''
                            background-color: rgba(255,255,255,0);
                            color: green
                            ''' )
        self.bering_loc_button.setFont(PyQt4.QtGui.QFont("Arial", 20, PyQt4.QtGui.QFont.Bold))
        self.bering_loc_button.setMaximumWidth(30)
        self.bering_loc_button.setMaximumHeight(30)

        self.bering_loc_button.show()

        # Hudson
        self.hudson_loc_button = PyQt4.QtGui.QPushButton('X', self)
        self.hudson_loc_button.setStyleSheet('''
                            background-color: rgba(255,255,255,0);
                            color: green
                            ''' )
        self.hudson_loc_button.setFont(PyQt4.QtGui.QFont("Arial", 20, PyQt4.QtGui.QFont.Bold))
        self.hudson_loc_button.setMaximumWidth(30)
        self.hudson_loc_button.setMaximumHeight(30)

        self.hudson_loc_button.show()

        # Severny
        self.severny_loc_button = PyQt4.QtGui.QPushButton('X', self)
        self.severny_loc_button.setStyleSheet('''
                            background-color: rgba(255,255,255,0);
                            color: green
                            ''' )
        self.severny_loc_button.setFont(PyQt4.QtGui.QFont("Arial", 20, PyQt4.QtGui.QFont.Bold))
        self.severny_loc_button.setMaximumWidth(30)
        self.severny_loc_button.setMaximumHeight(30)

        self.severny_loc_button.show()


from django.urls import path
from .views import *

urlpatterns = [
    path('test/', test, name="test"),

# HttpResponse--------------------------------------------------------------------
    path('test-1/', TestViewClass_1.as_view(), name="TestViewClass_1"),

    path('test-2/', TestViewClass_2.as_view(), name="TestViewClass_2"),

    path('test-3/', TestViewClass_3.as_view(subject="Chemistry"), name="TestViewClass_3"),

    path('sub-t-2/', SubClass.as_view(), name="SubClass"),

    path('test-3-1/', TestViewClass_3_1.as_view(), name="TestViewClass_3_1"),

# Class base View --------------------------------------------------------------------
    path('test-4/', TestViewClass_4.as_view(), name="TestViewClass_4"),

    path('test-5/', TestViewClass_5.as_view(), name="TestViewClass_5"),

    path('test-6/', TestViewClass_6.as_view(), name="TestViewClass_6"),

    path('test-7/', TestViewClass_7.as_view(), name="TestViewClass_7"),

    path('test-8/', MyView.as_view(), name="MyView"),

#----------------------------------------------------Template View--------------------------------------------------------

    path('tv-1/', templateview_1.as_view( template_name="viewtest/TemplateView_1.html" ,extra_context ={'course': 'Django'}), name="templateview_1"),
    path('tv-2/', templateview_2.as_view(extra_context ={'course': 'Django', 'fruit': 'Mango'}), name="templateview_2"),
    path('tv-3/', templateview_3.as_view(), name="templateview_3"),

    path('tv-4/<int:id>/', templateview_4.as_view(), name="templateview_4"),

#----------------------------------------------------Redirect View--------------------------------------------------------

    path('rd-1/', RedirectView_1.as_view(url="http://studentportal.diu.edu.bd/#/dashboard1"), name="RedirectView_1"),
    path('rd-2/', RedirectView_2.as_view(), name="RedirectView_2"),
    path('rd-3/', RedirectView_3.as_view(), name="RedirectView_3"),
    path('rd-4/', RedirectView_4.as_view(), name="RedirectView_4"),
    path('rd-5/', RedirectView_5.as_view(pattern_name = 'gen_1'), name="RedirectView_5"),

    path('rd-6/<int:id>/', RedirectView_6.as_view(), name="RedirectView_6"),
    path('rv-result/<int:id>/', R_V.as_view(extra_context ={'address': 'Saver'}), name="R_V"),

    path('rd-7/<slug:product>/', RedirectView_7.as_view(), name="RedirectView_7"),
    path('rv-result-2/<slug:product>/', R_V_2.as_view(extra_context ={'address': 'Saver'}), name="R_V_2"),

    #path('rv-1/<int:id>/', RedirectView_1.as_view(), name="RedirectView_1"),
    #path('rv-5/', RedirectView.as_view(), name="RedirectView"),
    #path('rv-1/', RedirectView_1.as_view(pattern_name ="TestViewClass_1"), name="RedirectView_1"),

    # path('<slug:frute>/', templateview_1.as_view( template_name="test/TemplateView_1.html" ,extra_context ={'course': 'Django'}), name="templateview_rv"),
    path('rv-result/<int:id>/', R_V.as_view(extra_context ={'address': 'Saver'}), name="R_V"),
    #path('rv-result/', R_V.as_view(extra_context ={'address': 'Saver'}), name="R_V"),

]


from __future__ import division, print_function, absolute_import

import pytest
from faker.providers import BaseProvider

from hypothesis import given
from hypothesis.strategytests import strategy_test_suite
from hypothesis.internal.debug import minimal
from hypothesis.extra.fakefactory import fake_factory

class KittenProvider(BaseProvider):

    def kittens(self):
        return u'meow %d' % (self.random_number(digits=10),)

@given(fake_factory(u'kittens', providers=[KittenProvider]))
def test_kittens_meow(kitten):
    assert u'meow' in kitten

@given(fake_factory(u'email'))
def test_email(email):
    assert u'@' in email

@given(fake_factory(u'name', locale=u'en_US'))
def test_english_names_are_ascii(name):
    name.encode(u'ascii')

def test_french_names_may_have_an_accent():
    minimal(
        fake_factory(u'name', locale=u'fr_FR'),
        lambda x: u'é' not in x
    )

def test_fake_factory_errors_with_both_locale_and_locales():
    with pytest.raises(ValueError):
        fake_factory(
            u'name', locale=u'fr_FR', locales=[u'fr_FR', u'en_US']
        )

def test_fake_factory_errors_with_unsupported_locale():
    with pytest.raises(ValueError):
        fake_factory(
            u'name', locale=u'badger_BADGER'
        )

def test_factory_errors_with_source_for_unsupported_locale():
    with pytest.raises(ValueError):
        fake_factory(u'state', locale=u'ja_JP')

def test_fake_factory_errors_if_any_locale_is_unsupported():
    with pytest.raises(ValueError):
        fake_factory(
            u'name', locales=[u'fr_FR', u'en_US', u'mushroom_MUSHROOM']
        )

def test_fake_factory_errors_if_unsupported_method():
    with pytest.raises(ValueError):
        fake_factory(u'spoon')

def test_fake_factory_errors_if_private_ish_method():
    with pytest.raises(ValueError):
        fake_factory(u'_Generator__config')

TestFakeEmail = strategy_test_suite(
    fake_factory(u'email')
)

TestFakeNames = strategy_test_suite(
    fake_factory(u'name')
)

TestFakeEnglishNames = strategy_test_suite(
    fake_factory(u'name', locale=u'en_US')
)

TestStates = strategy_test_suite(
    fake_factory(u'state')
)


#!/usr/bin/env python3

# pyre-strict

import asyncio
import logging
import sys
from typing import Any, Union

import click

__version__ = "0.6.9"
LOG: logging.Logger = logging.getLogger(__name__)

def _handle_debug(
    ctx: Union[click.core.Context, None],
    param: Union[click.core.Option, click.core.Parameter, None],
    debug: Union[bool, int, str],
) -> Union[bool, int, str]:
    """Turn on debugging if asked otherwise INFO default"""
    log_level = logging.DEBUG if debug else logging.INFO
    logging.basicConfig(
        format="[%(asctime)s] %(levelname)s: %(message)s (%(filename)s:%(lineno)d)",
        level=log_level,
    )
    return debug

async def async_main(debug: bool) -> int:
    return 0

# Some typing stub issue I can't work out - https://github.com/pallets/click/issues/2558
# Fix for 8.1.6 dosen't seem to fix me - To look into
@click.command(context_settings={"help_option_names": ["-h", "--help"]})  # type: ignore
@click.option(
    "--debug",
    is_flag=True,
    callback=_handle_debug,
    show_default=True,
    help="Turn on debug logging",
)
@click.pass_context
def main(ctx: click.core.Context, **kwargs: Any) -> None:
    LOG.debug(f"Starting {sys.argv[0]}")
    ctx.exit(asyncio.run(async_main(**kwargs)))

if __name__ == "__main__":  # pragma: no cover
    main()


#!/usr/bin/env python3

import argparse
import dgl
import numpy as np
import pandas as pd
import pickle as pkl
import torch
import torch.nn.functional as F

import ipdb

from gnn.link_prediction import LinkPredictor, compute_lp_loss
from gnn.node_classification import NodeClassifier, NodeClassifierConv

# if torch.cuda.is_available():
#     DEVICE = 'cuda:0'
# else:
#     DEVICE = 'cpu'
DEVICE='cpu'

def preprocess_edges(graph):
    chem = pd.read_csv("./data/chemicals.csv")
    maccs = torch.tensor([[int(x) for x in xx] for xx in chem.maccs]).float().to(DEVICE)
    node_features = {
        'chemical': maccs,
        #'chemical': torch.ones((graph.number_of_nodes(ntype='chemical'))).unsqueeze(1).to(DEVICE),
        'assay': torch.ones((graph.number_of_nodes(ntype='assay'))).unsqueeze(1).to(DEVICE),
        'gene': torch.ones((graph.number_of_nodes(ntype='gene'))).unsqueeze(1).to(DEVICE)
    }
    input_type_map = dict([(x[1], x[0]) for x in graph.canonical_etypes])
    node_sizes = { k: v.shape[1] for k, v in node_features.items() }
    edge_input_sizes = { k: node_features[v].shape[1] for k, v in input_type_map.items() }

    return node_features, node_sizes, edge_input_sizes

def drop_node_return_binary_labels(graph, ntype, node_index, pos_etype, neg_etype):
    """Drop a node from the graph and save its original connectivity as labels
    for supervised learning (especially node classification).

    Parameters
    ----------
    graph : dgl.DGLGraph
        Graph on which to run the operation.
    ntype : str
        Node type of the node to drop from the graph.
    node_index : int
        Integer index of the (typed) node to drop.
    pos_etype : str
        Name of an incoming edge, the source of which will be labeled 1.
    neg_etype : str
        Name of an incoming edge, the source of which will be labeled 0.

    Notes
    -----
    This is not an 'in place' operation. If you want to overwrite the original
    graph, you should reassign to it when calling the function.
    """
    # Get node connectivity and build label indices
    # FIX: Need to get the source node ids only!

    pos_nodes = graph.in_edges(node_index, form='uv', etype=pos_etype)[0]
    neg_nodes = graph.in_edges(node_index, form='uv', etype=neg_etype)[0]

    # Remove node
    new_graph = dgl.remove_nodes(graph, node_index, ntype=ntype)

    # Return both the transformed graph and the labels
    return new_graph, pos_nodes, neg_nodes

def collate_labels(graph, ntype, pos_idxs, neg_idxs, ratio=(0.8, 0.2)):
    """Make training/testing sets along with labels.
    """
    assert sum(ratio) == 1.0

    all_node_idxs = graph.nodes(ntype)

    pos_tensor = torch.cat((pos_idxs.unsqueeze(0), torch.ones_like(pos_idxs).unsqueeze(0)))
    neg_tensor = torch.cat((neg_idxs.unsqueeze(0), torch.zeros_like(neg_idxs).unsqueeze(0)))
    labeled_nodes = torch.cat((pos_tensor, neg_tensor), dim=1)

    idxs = torch.randperm(labeled_nodes.shape[1])
    shuffled = labeled_nodes[:,idxs]

    split_points = (
        round(shuffled.shape[1] * ratio[0]),
    )

    train = shuffled[:, :split_points[0]]
    test = shuffled[:, split_points[0]:]

    return train, test

def construct_negative_graph(graph, k, etype):
    """Construct a negative graph for negative sampling in edge prediction.

    This implementation is designed for heterogeneous graphs - the user specifies
    the edge type on which negative sampling will be performed.

    Parameters
    ----------
    graph : dgl.heterograph.DGLHeterograph
        Graph on which the sampling will be performed.
    k : int
        Number of negative examples to retrieve.
    etype : tuple
        A tuple in the format (subj_node_type, edge_label, obj_node_type) corresponding
        to the edge on which negative sampling will be performed.
    """
    utype, _, vtype = etype
    src, dst = graph.edges(etype=etype)

    neg_src = src.repeat_interleave(k)
    neg_dst = torch.randint(0, graph.num_nodes(vtype), (len(src) * k,))

    return dgl.heterograph(
        {etype: (neg_src, neg_dst)},
        num_nodes_dict= { ntype: graph.num_nodes(ntype) for ntype in graph.ntypes }
    )

def link_prediction(args):
    """Predict edges in a heterogeneous graph given a particular edge type.

    For this implementation, the edge type we are predicting is:
    `('chemical', 'chemicalhasactiveassay', 'assay')`

    There are two approaches for training the network:
    1. Train known edges against a negative sampling of the entire graph, using
       margin loss (or equivalent) to maximize the difference between known edges
       and the background "noise distribution" of randomly sampled edges.
    2. Use a predetermined edge (e.g., `'chemicalhasinactiveassay'`) instead as the
       negative graph. This approach may be more powerful. Cross-entropy loss also
       may be more appropriate than margin loss in this scenario.

    Parameters
    ----------
    args : (namespace output of argparse.parse_args() - see below for details)
    """
    G = dgl.load_graphs(args.graph_file)[0][0]

    k = 5

    node_features, node_sizes, edge_input_sizes = preprocess_edges(G)

    ep_model = EPModel(edge_input_sizes, 20, 5, G.etypes)
    opt = torch.optim.Adam(ep_model.parameters())

    for epoch in range(100):
        neg_G = construct_negative_graph(G, k, ('chemical', 'chemicalhasactiveassay', 'assay'))
        pos_score, neg_score = ep_model(G.to(DEVICE), neg_G.to(DEVICE), node_features, ('chemical', 'chemicalhasactiveassay', 'assay'))

        # margin loss
        loss = compute_ep_loss(pos_score, neg_score)

        opt.zero_grad()
        loss.backward()
        opt.step()

        print("epoch: %3d; margin loss: %.5f" % (epoch, loss.item()))

        # Now, we need to figure out something to do with the trained model!

    ipdb.set_trace()

def node_classification(args, label_assay_idx):
    """Predict node labels in a heterogeneous graph given a particular node type
    and a dataset of known node labels.

    Here, the node type is `'chemical'` and the labels we predict are annotations
    representing edge connectivity to a (deleted) `'assay'` node where 0 is
    `'chemicalhasinactiveassay'` edges and 1 is `'chemicalhasactiveassay'` edges.
    Labels are not provided for nodes with unknown activity annotations to the
    assay of interest.

    Parameters
    ----------
    args : (namespace output of argparse.parse_args() - see below for details)
    label_assay_idx : int
        Index of
    """
    G = dgl.load_graphs(args.graph_file)[0][0]

    # Remove the prediction task node and get labels before doing anything else
    G, pos_nodes, neg_nodes = drop_node_return_binary_labels(G, 'assay', label_assay_idx, 'chemicalhasactiveassay', 'chemicalhasinactiveassay')

    try:
        ratio_pos = len(pos_nodes) / (len(pos_nodes) + len(neg_nodes))
    except ZeroDivisionError:
        ratio_pos = 0

    train, test = collate_labels(G, 'chemical', pos_nodes, neg_nodes)

    train_idx = train[0,:]
    train_labels = train[1,:]
    test_idx = test[0,:]
    test_labels = test[1,:]

    # Remove nodes for ablation analysis
    G = dgl.edge_type_subgraph(G,
      [
        ('chemical', 'chemicalhasinactiveassay', 'assay'),
        ('assay', 'assayinactiveforchemical', 'chemical'),
        ('chemical', 'chemicalhasactiveassay', 'assay'),
        ('assay', 'assayactiveforchemical', 'chemical'),
        ('chemical', 'chemicalbindsgene', 'gene'),
        ('gene', 'genebindedbychemical', 'chemical'),
        ('chemical', 'chemicaldecreasesexpression', 'gene'),
        ('gene', 'expressiondecreasedbychemical', 'chemical'),
        ('chemical', 'chemicalincreasesexpression', 'gene'),
        ('gene', 'expressionincreasedbychemical', 'chemical'),
        ('gene', 'geneinteractswithgene', 'gene'),
        ('gene', 'geneinverseinteractswithgene', 'gene')
      ]
    )

    # Note: We don't do anything with the node features (yet)
    node_features, node_sizes, edge_input_sizes = preprocess_edges(G)

    model = NodeClassifier(G, node_sizes, edge_input_sizes, 2, 2)  # (self, G, node_sizes, edge_input_sizes, hidden_size, out_size)
    # model = NodeClassifierConv(G, node_sizes, edge_input_sizes, 2, 2)  # (self, G, node_sizes, edge_input_sizes, hidden_size, out_size)
    opt = torch.optim.Adam(model.parameters(), lr=0.02, weight_decay=5e-4)

    # compute class weights
    pos_weight = 1 - (len(pos_nodes) / (len(pos_nodes)+len(neg_nodes)))
    neg_weight = len(pos_nodes) / (len(pos_nodes)+len(neg_nodes))
    weights = torch.tensor([neg_weight, pos_weight])

    best_test_acc = 0
    best_f1 = 0

    for epoch in range(1000):
        logits = model(G)
        # logits = model(G, node_features)

        p = F.softmax(logits, dim=1)
        loss = F.cross_entropy(torch.log(p[train_idx]), train_labels)

        pred = logits.argmax(1)

        train_acc = (pred[train_idx] == train_labels).float().mean()
        test_acc = (pred[test_idx] == test_labels).float().mean()

        tp = sum(torch.logical_and(pred[test_idx], test_labels)).item()
        fp = sum(torch.logical_and(pred[test_idx], torch.logical_not(test_labels))).item()
        fn = sum(torch.logical_and(torch.logical_not(pred[test_idx]), test_labels)).item()

        f1 = (tp)/(tp + (0.5*(fp+fn)))

        if best_test_acc < test_acc:
            best_test_acc = test_acc
            # best_f1 = f1
            # best_probs = p
        if best_f1 < f1:
            best_f1 = f1
            best_probs = p

        opt.zero_grad()
        loss.backward()
        opt.step()

        if epoch % 5 == 0:
            try:
                print('Epoch %4d: Loss %.4f, Train Acc %.4f, Test Acc %.4f (Best %.4f), Test F1 %.4f (Best %.4f)' % (
                    epoch,
                    loss.item(),
                    train_acc.item(),
                    test_acc.item(),
                    best_test_acc.item(),
                    f1,
                    best_f1
                ))
            except AttributeError:
                ipdb.set_trace()
                print()

    preds = pd.DataFrame({'proba': p[test_idx][:,1].detach().numpy(), 'true_label': test_labels.detach().numpy()})
    ipdb.set_trace()
    return (best_test_acc.item(), best_f1, preds, ratio_pos)

def main(args):
    if args.task == "lp":
        link_prediction(args)
    elif args.task == "nc":
        assay_index = []
        assay_node_ids = []
        assays = pd.read_csv("./data/assays.csv")
        for ind, a in assays.iterrows():
            assay_index.append(ind)
            assay_node_ids.append(a)

        if args.assay == "all":
            assay_index = []
            assay_node_ids = []
            assays = pd.read_csv("./data/assays.csv")
            for ind, a in assays.iterrows():
                assay_index.append(ind)
                assay_node_ids.append(a)

            test_accs = []
            f1s = []
            ratio_positive_nodes = []

            with open("./gnn_nc_results/nc_results.tsv", 'w') as fp:
                fp.write("assay_index\tassay_node_id\tacc\tf1\tpos_ratio\n")

            for i in assay_index:

                if i == 47:
                    continue  # need to debug this one...

                print("================================")
                print(" ASSAY {0} OF {1}".format(i, len(assay_index)))
                print("================================")
                try:
                    test_acc, f1, preds, rp = node_classification(args, i)
                except:
                    with open("./gnn_nc_results/nc_results_pd.tsv", 'a') as fp:
                        fp.write("ERROR ON ASSAY WITH INDEX: {0}\n".format(i))
                    print("UH OH SOMETHING WENT WRONG")
                    continue
                print()
                test_accs.append(test_acc)
                f1s.append(f1)
                ratio_positive_nodes.append(rp)

                preds.to_csv('gnn_nc_results/{0}.tsv'.format(i), sep='\t', index=False)

                with open("./gnn_nc_results/nc_results.tsv", 'a') as fp:
                    fp.write(f"{i}\t{int(assay_node_ids[i])}\t{test_acc:.4f}\t{f1:.4f}\t{rp:.3f}\n")

            all_assay_results = pd.DataFrame({'assay_index': assay_index, 'assay_node_id': assay_node_ids, 'acc': test_accs, 'f1': f1s})
            all_assay_results.to_csv('gnn_nc_results/nc_results_pd.tsv', sep='\t', index=False)

            ipdb.set_trace()
            print()
        elif args.assay.isdigit():
            assay_idx = int(args.assay)

            test_acc, f1, preds, rp = node_classification(args, assay_idx)

            #preds.to_csv('gnn_nc_results/ablation/{0}.tsv'.format(assay_idx), sep='\t', index=False)

            print(f"{assay_idx}\t{int(assay_node_ids[assay_idx])}\t{test_acc:.4f}\t{f1:.4f}\t{rp:.3f}\n")

            ipdb.set_trace()

        else:
            raise argparse.ArgumentError("Error - `--assay` must be an integer or 'all'.")

if __name__=="__main__":
    tasks = ['nc', 'lp']
    parser = argparse.ArgumentParser(description="Train a heterogeneous RGCN on a prediction task.")
    parser.add_argument("--task", type=str, default="nc",
                        help="Type of prediction task to perform.",
                        choices=tasks)
    parser.add_argument("--graph-file", type=str, default="./data/graph.bin",
                        help="File location where the DGL heterograph is stored.")
    parser.add_argument("--lr", type=float, default=0.01,
                        help="Learning rate for the NN optimizer.")
    parser.add_argument("--assay", )

    parser.set_defaults(validation=True)

    args = parser.parse_args()
    main(args)


from channels.auth import AuthMiddlewareStack
from channels.routing import ProtocolTypeRouter, URLRouter
import landing_page.routing

application = ProtocolTypeRouter({
    # (http->django views is added by default)
    'websocket': AuthMiddlewareStack(
        URLRouter(
            landing_page.routing.websocket_urlpatterns
        )
    ),
})
# Channels
#ASGI_APPLICATION = 'sitotest.routing.application'


#!/usr/bin/env python
# -*- encoding: utf-8 -*-

from pron91pkg import httputil
from bs4 import BeautifulSoup
from pron91pkg import disk
import requests
import shutil
import os
import time
from pron91pkg.FakeHeader import FakeHeader
# 1.get title
#2.get html5 m3u8
#3.download ts files from m3u8
#4.merge m3u8 to one file(ts file)

Sleep_Per_File = 0
Sleep_Per_TioutOut = 10
chunk_size = 512
basePath = "Spider/yezmw/"

def handleVideoContent(url):
    disk.mkdir(basePath)
    content = httputil.fetchContent(url)

    # print(content)

    soup = BeautifulSoup(content , "html.parser")
    results = soup.find_all("span",class_="title")

    global result
    global title
    global hlsViedoUrl
    if len(results)>0:
        targetSoup = results[0]
        title = str(targetSoup.text)
        title = httputil.__escape_file_name_str(title)
    results = soup.find_all("div",class_="dz")
    if len(results) >0:
        targetSoup = results[0]
        childSoup = targetSoup.find("p")
        hlsViedoUrl = str(childSoup.text)

    result = {
        "title":title,
        "hlsViedoUrl":hlsViedoUrl
    }
    return result

# url = "http://yezmw.com/video/show/id/4239"
#
# handleVideoContent(url)

def decodeM3u8File(title,hlsVideoUrl):

    folderPath = basePath + title +"/"
    disk.mkdir(folderPath)
    #create floder
    index = hlsVideoUrl.rfind("/")
    targetPath = folderPath+hlsVideoUrl[index+1:]
    baseURL = hlsVideoUrl[:index+1]

    #remove m3u8 file
    try:
        file = open(targetPath, 'r')
        file.close()
        os.remove(targetPath)
    except FileNotFoundError:
        pass

    #remove convert.m3u8 file
    try:
        file = open(folderPath +"convert.m3u8", 'r')
        file.close()
        os.remove(folderPath +"convert.m3u8")
    except FileNotFoundError:
        pass

    #download m3u8 file
    response = requests.get(hlsVideoUrl, stream=True)
    with open(targetPath, 'wb') as out_file:
        shutil.copyfileobj(response.raw, out_file)
        del response
    #decode m3u8 file
    hlsFile = open(targetPath,"r+")
    outFile = open(folderPath +"convert.m3u8","w+")
    # print(hlsFile.name)

    line = hlsFile.readline()
    lineCount = 0
    while(line!= ''):
        if "#" in line:
            pass
        else:
            line = baseURL + line
            outFile.write(line)
            lineCount = lineCount + 1
        line = hlsFile.readline()

    print("decodeM3u8File end")
    hlsFile.close()
    outFile.close()
    return lineCount

def startdownloadVideo(referUrl,name,linecount):

    fakeHeader = FakeHeader()
    name = name.replace("\n","")

    folderPath = basePath + name +"/"
    downloadPath = folderPath + "parts/"
    disk.mkdir(downloadPath)
    downloadFile = open(folderPath+"convert.m3u8","r+")
    line = downloadFile.readline()
    line = line.replace("\n","")

    index = line.rfind(".")
    type = line[index:]
    type = type.replace("\n","")
    # xxx.ts
    downloadPath = downloadPath + name + type
    # remove xxx.ts file
    try:
        file = open(downloadPath, 'r')
        file.close()
        os.remove(downloadPath)
    except FileNotFoundError:
        pass

    outFile = open(downloadPath,"wb+")
    recordNum = 0;
    i = 0
    while(line!= ''):
        try:
            line = line.replace("\n","")
            partUrl = line

            recordNum =  recordNum + 1

            print(partUrl)
            print("正在下载片段 " + str(recordNum) + " "+str(format(recordNum/linecount*100,".2f")) + "%")
            request_headers = fakeHeader.buildFakeHeader(referer=referUrl)
            response = requests.get(partUrl, stream=True,timeout=5,headers = request_headers)

            # print(request_headers)
            # print(response.status_code)

            for chunk in response.iter_content(chunk_size):
                outFile.write(chunk)
            del response

        except requests.exceptions.ReadTimeout:
            time.sleep(Sleep_Per_TioutOut)
            continue
        except requests.exceptions.ConnectionError:
            time.sleep(Sleep_Per_TioutOut)
            continue
        time.sleep(Sleep_Per_File)

        line = downloadFile.readline()
    downloadFile.close()
    outFile.close()
    pass

# hlsVideoUrl ="http://video1.feimanzb.com:8091/20171215/RKI-413-C/550kb/hls/index.m3u8"
# index = hlsVideoUrl.rfind("/")
# targetPath=hlsVideoUrl[index+1:]
# baseURL = hlsVideoUrl[:index+1]
#
# print(targetPath)
# print(baseURL)
# # response = requests.get(hlsVideoUrl, stream=True)
# # with open(targetPath, 'wb') as out_file:
# #     shutil.copyfileobj(response.raw, out_file)
# #     del response

# hlsFile = open(targetPath,"r+")
# outFile = open("convert.m3u8","w+")
# print(hlsFile.name)
#
# line = hlsFile.readline()
# while(line!= ''):
#     if "#" in line:
#         pass
#     else:
#         line = baseURL + line
#         outFile.write(line)
#     line = hlsFile.readline()
#
#
#
# print("end")
# hlsFile.close()
# outFile.close()

# url = "http://video1.feimanzb.com:8091/20171215/RKI-413-C/550kb/hls/gUYZa0Kw2426000.ts"
# index = url.rfind(".")
# print(url[index:])
url="https://www.avbyebye.com"
url="http://www.allitebooks.com/powershell-for-sql-server-essentials/"
content = httputil.fetchContent(url)
print(content)


# Задание №7

import json

with open("lesson_5_hw_77.json", "w") as j_file:
    with open("lesson_5_hw_7.txt", "r") as f_o:
        subjects = {}
        middle = {}
        k, o = 0, 0
        line = f_o.read().split("\n")
        for i in line:
            i = i.split()
            profit = int(i[2]) - int(i[3])
            subjects[i[0]] = profit
            if profit > 0:
                k += profit
                o += 1
            middle["avegare"] = k / o
        all_list = [subjects, middle]
    json.dump(all_list, j_file)


#!/usr/bin/python
# -*- coding: utf-8 -*-
""" @brief  Precision and Recall
    @ref    https://en.wikipedia.org/wiki/Precision_and_recall
            Modified from https://github.com/lyst/lightfm and https://github.com/jfkirk/tensorrec
    @author <ariel kalingking> akalingking@gmail.com """
import numpy as np
import pandas as pd
from scipy import sparse

__all__ = ["precision_at_k", "recall_at_k", "rank_matrix"]

def rank_matrix(mat):
    assert isinstance(mat, (np.ndarray,))
    mat_ = pd.DataFrame(data=mat)
    mat_ = mat_.rank(axis=1, ascending=False)
    return mat_.values

def precision_at_k(predicted_ranks, interactions, k=10, preserve_rows=False):
    assert isinstance(interactions, (sparse.coo_matrix, sparse.csr_matrix))
    assert isinstance(predicted_ranks, (np.ndarray,))

    relevant = interactions > 0
    ranks = sparse.csr_matrix(predicted_ranks * relevant.A)
    ranks.data = np.less(ranks.data, (k + 1), ranks.data)

    precision = np.squeeze(np.array(ranks.sum(axis=1))).astype(float) / k

    if not preserve_rows:
        precision = precision[relevant.getnnz(axis=1) > 0]

    return precision.mean()

def recall_at_k(predicted_ranks, interactions, k=10, preserve_rows=False):
    assert isinstance(interactions, (sparse.coo_matrix, sparse.csr_matrix))
    assert isinstance(predicted_ranks, (np.ndarray,))

    relevant = interactions > 0
    ranks = sparse.csr_matrix(predicted_ranks * relevant.A)
    ranks.data = np.less(ranks.data, (k + 1), ranks.data)
    retrieved = np.squeeze(relevant.getnnz(axis=1))
    hit = np.squeeze(np.array(ranks.sum(axis=1)))

    if not preserve_rows:
        hit = hit[relevant.getnnz(axis=1) > 0]
        retrieved = retrieved[relevant.getnnz(axis=1) > 0]

    return (hit.astype(float) / retrieved.astype(float)).mean()