from dataclasses import dataclass
from typing import Any, List, Tuple

import hypothesis.strategies as st
import pytest
from hypothesis import given
from omegaconf import DictConfig, ListConfig
from omegaconf.errors import ValidationError

from hydra_zen import (
    DefaultBuilds,
    builds,
    hydrated_dataclass,
    instantiate,
    make_config,
    mutable_value,
)
from hydra_zen._compatibility import HYDRA_SUPPORTS_OBJECT_CONVERT

get_obj_path = DefaultBuilds._get_obj_path

def f_three_vars(x, y, z):
    return x, y, z

def wrapper(obj):
    return obj

@given(
    convert=st.sampled_from(["none", "all", "partial"]),
    recursive=st.booleans(),
    sig=st.booleans(),
    partial=st.none() | st.booleans(),
    meta=st.none() | st.just(dict(meta=1)),
    wrappers=st.none() | st.just(wrapper) | st.lists(st.just(wrapper)),
    kwargs=st.dictionaries(keys=st.sampled_from(["x", "y", "z"]), values=st.floats()),
    name=st.none() | st.sampled_from(["NameA", "NameB"]),
)
def test_builds_sets_hydra_params(
    convert, recursive, sig, partial, name, meta, wrappers, kwargs
):
    out = builds(
        f_three_vars,
        hydra_convert=convert,
        hydra_recursive=recursive,
        populate_full_signature=sig,
        zen_partial=partial,
        zen_meta=meta,
        zen_wrappers=wrappers,
        zen_dataclass={"cls_name": name},
        **kwargs,
    )

    assert out._convert_ == convert  # type: ignore
    assert out._recursive_ == recursive  # type: ignore
    if name is not None:
        assert out.__name__ == name
    else:
        assert "Builds_f_three_vars" in out.__name__

def f_for_convert(**kwargs):
    return kwargs

class NotSet:
    pass

@dataclass
class A:
    x: Any = (1, 2)

@pytest.mark.parametrize("via_hydrated_dataclass", [False, True])
@pytest.mark.parametrize(
    "convert, expected_types",
    [
        # "Passed objects are DictConfig and ListConfig"
        ("none", (DictConfig, ListConfig, DictConfig, int)),
        # default should be same as 'none'
        (NotSet, (DictConfig, ListConfig, DictConfig, int)),
        # "Passed objects are converted to dict and list, with
        #  the exception of Structured Configs (and their fields)."
        ("partial", (dict, list, DictConfig, int)),
        # "Passed objects are dicts, lists and primitives without
        # a trace of OmegaConf containers"
        ("all", (dict, list, dict, int)),
    ],
)
def test_hydra_convert(
    convert: str, expected_types: List[type], via_hydrated_dataclass: bool
):
    """Tests that the `hydra_convert` parameter produces the expected/documented
    behavior in hydra."""

    kwargs = dict(hydra_convert=convert) if convert is not NotSet else {}

    if not via_hydrated_dataclass:
        out = instantiate(
            builds(
                f_for_convert,
                a_dict=dict(x=1),
                a_list=[1, 2],
                a_struct_config=A,
                an_int=1,
                zen_convert={"dataclass": False},
                **kwargs,
            )
        )
    else:

        @hydrated_dataclass(f_for_convert, **kwargs)
        class MyConf:
            a_dict: Any = mutable_value(dict(x=1))
            a_list: Any = mutable_value([1, 2])
            a_struct_config: Any = A
            an_int: int = 1

        out = instantiate(MyConf)

    actual_types = tuple(
        type(out[i]) for i in ("a_dict", "a_list", "a_struct_config", "an_int")
    )
    assert actual_types == expected_types

    if convert in ("none", "partial", NotSet):
        expected_field_type = ListConfig
    else:
        expected_field_type = list

    assert type(out["a_struct_config"]["x"]) is expected_field_type

@pytest.mark.xfail(
    not HYDRA_SUPPORTS_OBJECT_CONVERT,
    raises=ValueError,
    reason="Hydra 1.3.0 required for _convert_ = 'object'",
)
def test_object_convert():
    Conf = make_config(
        a=builds(dict, b=1), hydra_convert="object"
    )  # should raise if local Hydra version is < 1.3.0
    assert isinstance(instantiate(Conf), Conf)

@pytest.mark.xfail(
    not HYDRA_SUPPORTS_OBJECT_CONVERT,
    raises=ValueError,
    reason="Hydra 1.3.0 required for _convert_ = 'object'",
)
def test_object_convert_hydrated():
    @hydrated_dataclass(dict, hydra_convert="object")
    class A:
        ...

    assert A._convert_ == "object"  # type: ignore

def f_for_recursive(**kwargs):
    return kwargs

@pytest.mark.parametrize("via_hydrated_dataclass", [False, True])
@pytest.mark.parametrize("recursive", [False, True, NotSet])
def test_recursive(recursive: bool, via_hydrated_dataclass: bool):
    target_path = get_obj_path(f_for_recursive)

    kwargs = dict(hydra_recursive=recursive) if recursive is not NotSet else {}

    if not via_hydrated_dataclass:
        out = instantiate(
            builds(
                f_for_recursive,
                x=builds(f_for_recursive, y=1),
                **kwargs,
            )
        )

    else:

        @hydrated_dataclass(f_for_recursive)
        class B:
            y: int = 1

        @hydrated_dataclass(f_for_recursive, **kwargs)
        class A:
            x: Any = B

        out = instantiate(A)

    if recursive is NotSet or recursive:
        assert out == {"x": {"y": 1}}
    else:
        assert out == {"x": {"_target_": target_path, "y": 1}}

def f(x: Tuple[int]):
    return x

class C:
    def __init__(self, x: int):
        self.x = x

def g(x: C):
    return x

def g2(x: List[C]):
    return x

def test_type_checking():
    conf = builds(f, populate_full_signature=True)(x=("hi",))  # type: ignore
    with pytest.raises(ValidationError):
        instantiate(conf)

    # should be ok
    instantiate(builds(f, populate_full_signature=True)(x=(1,)))

    # nested configs should get validated too
    with pytest.raises(ValidationError):
        instantiate(builds(g, x=builds(C, x="hi")))  # Invalid: `C.x` must be int

    # should be ok
    instantiate(builds(g, x=builds(C, x=1)))

    conf_C = builds(C, x=1)

    # should be ok
    instantiate(builds(g2, x=[conf_C, conf_C]))


from django.db.models import Q
from django.http import HttpResponseRedirect
from django.shortcuts import render
from schedule_advisor.forms import ClassSearchForm, AdviseeChoiceForm
from schedule_advisor.models import Schedule, Class, User
from django.contrib import messages

def search_view(request):
    results = None
    if request.method == "POST":
        form = ClassSearchForm(request.POST)
        if form.is_valid():
            # results = ClassSearchForm.get_results(form.cleaned_data)
            results = ClassSearchForm.get_results_from_database(form.cleaned_data)
    else:
        form = ClassSearchForm()
    return render(
        request, "schedule_advisor/index.html", {"form": form, "results": results}
    )
def home_view(request):
    return render(
        request, "schedule_advisor/home.html"
    )

def schedule_view(request):
    days_of_week = [
        "Sunday",
        "Monday",
        "Tuesday",
        "Wednesday",
        "Thursday",
        "Friday",
        "Saturday",
    ]
    days_from_abbrev = {day[:2]: day for day in days_of_week}
    if not request.user.is_authenticated:
        messages.add_message(request, messages.ERROR, "You are not logged in!")
        return HttpResponseRedirect("/")
    u = User.objects.get_or_create(username=request.user.username)[0]
    if u.is_advisor:
        messages.add_message(request, messages.ERROR, "You are not a student!")
        return HttpResponseRedirect("/")
    schedule = Schedule.objects.get_or_create(connected_user_id=u.id)[0]
    schedule_by_day = {day: list() for day in days_of_week}
    for cl in schedule.classes.all():
        for day in cl.meeting_days():
            if day not in days_from_abbrev:
                schedule_by_day[day] = [cl]
            else:
                schedule_by_day[days_from_abbrev[day]].append(cl)
    for day, classes in schedule_by_day.items():
        if day not in days_of_week:
            classes.sort(key=lambda c: c.name)
            continue
        abbrev = day[:2]
        classes.sort(
            key=lambda c: tuple(
                int(i) for i in c.meeting_days()[abbrev][0].split(".")[:3]
            )
        )
    return render(
        request,
        "schedule_advisor/schedule.html",
        {
            "schedule_by_day": schedule_by_day,
            "days_of_week": days_of_week,
            "schedule": schedule,
        },
    )

def class_schedule_change_view(request):
    u = User.objects.get_or_create(username=request.user.username)[0]
    if u.is_advisor:
        messages.add_message(request, messages.ERROR, "You are not a student!")
        return HttpResponseRedirect("/")
    if request.method == "POST":
        action = request.POST["action"]
        try:
            if action == "Add to Schedule":
                schedule = Schedule.objects.get_or_create(
                    connected_user_id=request.user.id
                )[0]
                new_class = Class.objects.get_or_create(
                    semester=request.POST["semester"],
                    class_number=int(request.POST["class_number"]),
                )[0]
                schedule.add_class(new_class)
                schedule.reset_advisor()
                messages.add_message(
                    request, messages.SUCCESS, "Course successfully added to schedule!"
                )
            elif action == "Remove from Schedule":
                schedule = Schedule.objects.get_or_create(
                    connected_user_id=request.user.id
                )[0]
                old_class = Class.objects.get_or_create(
                    semester=request.POST["semester"],
                    class_number=int(request.POST["class_number"]),
                )[0]
                schedule.remove_class(old_class)
                schedule.reset_advisor()
                messages.add_message(
                    request,
                    messages.SUCCESS,
                    "Course successfully removed from schedule!",
                )
            else:
                messages.add_message(
                    request, messages.ERROR, "Unable to update schedule!"
                )
        except:
            messages.add_message(request, messages.ERROR, "Unable to update schedule!")
        # https://stackoverflow.com/questions/35796195/how-to-redirect-to-previous-page-in-django-after-post-request
        return HttpResponseRedirect(request.headers.get("referer", "/"))

def advisor_view(request):
    u = User.objects.get_or_create(username=request.user.username)[0]
    if not u.is_advisor:
        messages.add_message(request, messages.ERROR, "You are not an advisor!")
        return HttpResponseRedirect("/")
    else:
        schedules = None
        days_of_week = None
        schedule_by_day = None
        query = None
        advisees = User.objects.filter(advisor=request.user)
        if request.method == "GET":
            query = request.GET.get("q", None)
            if query:
                query = User.objects.filter(
                    Q(is_advisor=False)
                    & (
                        Q(first_name__icontains=query)
                        | Q(last_name__icontains=query)
                        | Q(username__icontains=query)
                    )
                )
        elif request.method == "POST":
            advisee_id = request.POST.get("advisee_id")
            schedules = Schedule.objects.filter(connected_user_id=advisee_id)
            days_of_week = [
                "Sunday",
                "Monday",
                "Tuesday",
                "Wednesday",
                "Thursday",
                "Friday",
                "Saturday",
            ]
            days_from_abbrev = {day[:2]: day for day in days_of_week}
            u = User.objects.get_or_create(id=advisee_id)[0]
            schedule = Schedule.objects.get_or_create(connected_user_id=u.id)[0]
            schedule_by_day = {day: list() for day in days_of_week}
            for cl in schedule.classes.all():
                for day in cl.meeting_days():
                    if day not in days_from_abbrev:
                        schedule_by_day[day] = [cl]
                    else:
                        schedule_by_day[days_from_abbrev[day]].append(cl)
            for day, classes in schedule_by_day.items():
                if day not in days_of_week:
                    classes.sort(key=lambda c: c.name)
                    continue
                abbrev = day[:2]
                classes.sort(
                    key=lambda c: tuple(
                        int(i) for i in c.meeting_days()[abbrev][0].split(".")[:3]
                    )
                )
        return render(
            request,
            "schedule_advisor/advisor_schedules.html",
            {
                "advisees": advisees,
                "schedules": schedules,
                "days_of_week": days_of_week,
                "schedule_by_day": schedule_by_day,
                "query": query,
            },
        )

def advisee_change_view(request):
    u = User.objects.get_or_create(username=request.user.username)[0]
    if not u.is_advisor:
        messages.add_message(request, messages.ERROR, "You are not an advisor!")
        return HttpResponseRedirect("/")
    else:
        action = request.POST["action"]
        try:
            if action == "Add":
                advisee = User.objects.get_or_create(username=request.POST["advisee"])[
                    0
                ]
                advisor = User.objects.get_or_create(username=request.POST["advisor"])[
                    0
                ]
                advisee.advisor = advisor
                advisee.save()
                messages.add_message(
                    request,
                    messages.SUCCESS,
                    f"You are now the advisor for {advisee.get_full_name()}!",
                )
            elif action == "Remove":
                advisee = User.objects.get_or_create(username=request.POST["advisee"])[
                    0
                ]
                advisee.advisor = None
                advisee.save()
                messages.add_message(
                    request,
                    messages.SUCCESS,
                    f"You are no longer the advisor for {advisee.get_full_name()}!",
                )
            else:
                messages.add_message(
                    request, messages.ERROR, "Unable to update schedule!"
                )
        except:
            messages.add_message(
                request,
                messages.ERROR,
                "Unable to edit advisor information for student!",
            )
        # https://stackoverflow.com/questions/35796195/how-to-redirect-to-previous-page-in-django-after-post-request
        return HttpResponseRedirect(request.headers.get("referer", "/"))

def class_schedule_visible_view(request):
    if request.method == "POST":
        try:
            u = User.objects.get_or_create(username=request.user.username)[0]
            if u.is_advisor:
                messages.add_message(request, messages.ERROR, "You are not a student!")
                return HttpResponseRedirect("/")
            if not u.advisor:
                messages.add_message(request, messages.ERROR, "You do not have an advisor!")
                return HttpResponseRedirect("/")
            s = Schedule.objects.get_or_create(connected_user=u)[0]
            visibility = request.POST.get("visible", False)
            s.visible = visibility == "on"
            s.reset_advisor()
            messages.add_message(
                request,
                messages.SUCCESS,
                f"Your schedule is{'' if visibility else ' no longer'} visible to {u.advisor.get_full_name()}!",
            )
        except:
            messages.add_message(
                request,
                messages.ERROR,
                "Unable to edit schedule visibility!",
            )
        return HttpResponseRedirect(request.headers.get("referer", "/"))

def schedule_approval_view(request):
    u = User.objects.get_or_create(username=request.user.username)[0]
    if not u.is_advisor:
        messages.add_message(request, messages.ERROR, "You are not an advisor!")
        return HttpResponseRedirect("/")
    if request.method == "POST":
        try:
            schedule = Schedule.objects.get_or_create(id=int(request.POST["schedule"]))[0]
            schedule_user: User = schedule.connected_user
            if schedule_user.advisor != u:
                messages.add_message(request, messages.ERROR, "You are not this student's advisor!")
                return HttpResponseRedirect("/schedules")
            decision = request.POST["decision"]
            if decision == "true":
                decision = True
                messages.add_message(
                    request, messages.SUCCESS, "You have approved this schedule!"
                )
            elif decision == "false":
                decision = False
                messages.add_message(
                    request, messages.SUCCESS, "You have rejected this schedule!"
                )
            else:
                decision = None
            schedule.approved = decision
            schedule.save()
        except:
            messages.add_message(
                request,
                messages.ERROR,
                "Unable to update schedule approval status!",
            )
    return HttpResponseRedirect(request.headers.get("referer", "/"))


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Use iterative parsing to process the map file and
find out not only what tags are there, but also how many, to get the
feeling on how much of which data is expected within the map.

The output should be a dictionary with the tag name as the key
and number of times this tag can be encountered in the map as value.
"""
import xml.etree.ElementTree as ET
import pprint

#OSMFILE = "../example.osm"
OSMFILE = "../example_sf.osm"
#OSMFILE = "../san-francisco.osm"

def count_tags(filename):
    """Returns dictionary of different tags and corresponding count of how
    many times that tag was seen"""
    xml_dict = {}
    context = ET.iterparse( filename )
    for event, elem in context:
        if elem.tag in xml_dict:
            xml_dict[elem.tag] = xml_dict[elem.tag] + 1
        else:
            xml_dict.update({elem.tag : 1})
    return xml_dict

def test():

    tags = count_tags(OSMFILE)
    pprint.pprint(tags)

if __name__ == "__main__":
    test()


"""
1027. Two City Scheduling
A company is planning to interview 2n people. Given the array costs where costs[i] = [aCost_i, bCost_i], the cost of flying the i-th person to city a is aCost_i, and the cost of flying the i-th person to city b is bCost_i.

Return the minimum cost to fly every person to a city such that exactly n people arrive in each city.

Example1:
Input: costs = [[10,20],[30,200],[400,50],[30,20]]
Output: 110
Explanation:
The first person goes to city A for a cost of 10.
The second person goes to city A for a cost of 30.
The third person goes to city B for a cost of 50.
The fourth person goes to city B for a cost of 20.

The total minimum cost is 10 + 30 + 50 + 20 = 110 to have half the people interviewing in each city.

Example2:
Input: costs = [[259,770],[448,54],[926,667],[184,139],[840,118],[577,469]]
Output: 1859

Example3:
Input: costs = [[515,563],[451,713],[537,709],[343,819],[855,779],[457,60],[650,359],[631,42]]
Output: 3086

Constraints:
2 * n == costs.length
2 <= costs.length <= 100
costs.length is even.
1 <= aCost_i, bCost_i <= 1000
"""

"""
Note:
1. Greedy: O(nlogn) time | O(n) space - where n is the length of array costs
"""

import unittest
from typing import List
class Solution:
    def twoCitySchedCost(self, costs: List[List[int]]) -> int:
        costs.sort(key = lambda x: x[1] - x[0])
        n = len(costs) // 2
        total = 0
        for i, (aCost, bCost) in enumerate(costs):
            if i <= n-1:
                total += bCost
            else:
                total += aCost
        return total

# Unit Tests
import unittest
funcs = [Solution().twoCitySchedCost]
class TestTwoCitySchedCost(unittest.TestCase):
    def testTwoCitySchedCost1(self):
        for func in funcs:
            costs = [[10,20],[30,200],[400,50],[30,20]]
            self.assertEqual(func(costs=costs), 110)

    def testTwoCitySchedCost2(self):
        for func in funcs:
            costs = [[259,770],[448,54],[926,667],[184,139],[840,118],[577,469]]
            self.assertEqual(func(costs=costs), 1859)

    def testTwoCitySchedCost3(self):
        for func in funcs:
            costs = [[515,563],[451,713],[537,709],[343,819],[855,779],[457,60],[650,359],[631,42]]
            self.assertEqual(func(costs=costs), 3086)

if __name__ == "__main__":
    unittest.main()


import os
import sys
import requests
import random
import string
from glob import glob
from time import perf_counter
from . import process_question_with_teprolin, \
    remove_diacritics, frequent_verbs

_bing_search_url = 'https://api.bing.microsoft.com/v7.0/search?q=#QUERY#&cc=RO'
_bing_api_key = os.getenv('BING_SEARCH_V7_KEY')
_debug = True

# 1. Check for valid Bing API key
if not _bing_api_key:
    raise RuntimeError(
        "No Bing API key was found in environment variable BING_SEARCH_V7_KEY!")
# end if

# 2. Create multiprocess Bing query caches
_cache_unique_key = ''

while len(_cache_unique_key) < 8:
    _cache_unique_key += random.choice(string.ascii_lowercase + string.digits)
# end while

_bing_cache_file = os.path.join('cache', f'bing-{_cache_unique_key}.txt')
_bing_master_cache_file = os.path.join('cache', f'bing-master.txt')
_bing_cache = {}
_cache_additions = 0

def _add_cache_file(cache_file: str):
    """Adds a Bing cache file to the current cache."""

    with open(cache_file, mode='r', encoding='utf-8') as f:
        line = f.readline().strip()

        while True:
            query = line
            query_hits = []
            line = f.readline().strip()

            while line and line != '#EOR#':
                hit_dict = {}
                hit_dict['name'] = line
                hit_dict['url'] = f.readline().strip()
                hit_dict['snippet'] = f.readline().strip()
                hit_dict['date'] = f.readline().strip()
                hit_dict['rank'] = len(query_hits)
                query_hits.append(hit_dict)
                line = f.readline().strip()
            # end while

            if not line:
                break
            # end if

            _bing_cache[query] = query_hits
            line = f.readline().strip()
        # end for
    # end with
# end def

# 2.1 Load the cache for Bing queries
for f in glob(os.path.join('cache', 'bing-*.txt')):
    _add_cache_file(f)

    if not f.endswith(_bing_master_cache_file):
        print(f'Deleting previous cache file [{f}]', file=sys.stderr, flush=True)
        os.remove(f)
    # end if
# end for

def _save_cache_file(cache_file: str):
    """Saves the `_bing_cache` dict to the `cache_file` file."""

    with open(cache_file, mode='w', encoding='utf-8') as f:
        for query in sorted(list(_bing_cache.keys())):
            print(query, file=f)

            for hit_dict in _bing_cache[query]:
                print(hit_dict['name'], file=f)
                print(hit_dict['url'], file=f)
                print(hit_dict['snippet'], file=f)
                print(hit_dict['date'], file=f)
            # end for

            print('#EOR#', file=f, flush=True)
        # end for
    # end with
# end def

# 2.2 Write master file back
_save_cache_file(_bing_master_cache_file)

def _generate_query_1(tokens: list) -> str:
    """Takes a processed question and generates a web search query from it.
    For now, just choose the nouns in the question."""

    query = []

    for tid, wf, lem, ctag, head, deprel in tokens:
        if ctag == 'NOUN' or ctag == 'PROPN' or ctag == 'ADJ' or \
                ctag == 'NUM' or ctag == 'ADV' or \
                (ctag == 'VERB' and lem not in frequent_verbs):
            query.append(wf)
        # end if
    # end for

    return ' '.join(query)

def bing_search_v7(question: str, given_query: str) -> list:
    """Performs a Bing V7 search for the given question.
    If question is in cache, no search request is sent.
    If `given_query` is not empty, use it instead of generating the query.
    This is useful when testing the QA system with human generated queries."""

    if given_query:
        return _do_the_web_search(given_query)
    # end if

    # 1. Process question
    q_tokens = process_question_with_teprolin(question)

    # 2. Generate query
    q_query = _generate_query_1(q_tokens)
    ndq_query = remove_diacritics(q_query)

    if q_query == ndq_query:
        if q_query in _bing_cache:
            # Spare Bing API querying...
            return _bing_cache[q_query]
        else:
            return _do_the_web_search(q_query)
        # end if
    else:
        results = []

        if q_query in _bing_cache:
            # Spare Bing API querying...
            results.extend(_bing_cache[q_query])
        else:
            results.extend(_do_the_web_search(q_query))
        # end if

        if ndq_query in _bing_cache:
            # Spare Bing API querying...
            results.extend(_bing_cache[ndq_query])
        else:
            results.extend(_do_the_web_search(ndq_query))
        # end if

        return _remove_duplicate_hits(results)
    # end if

def _do_the_web_search(query: str) -> list:
    """Does the Bing search and returns the list of results."""
    global _cache_additions

    # 1. Populate query link
    bing_query_url = _bing_search_url.replace('#QUERY#', query)

    if _debug:
        # 2. Do the query
        print(f'Querying Bing with query [{query}]', file=sys.stderr, flush=True)
        start = perf_counter()
    # end if
    response = requests.get(bing_query_url, headers={
                            "Ocp-Apim-Subscription-Key": _bing_api_key})

    if _debug:
        elapsed = 1000 * (perf_counter() - start)
        print(f' -> took {elapsed:.3f}ms', file=sys.stderr, flush=True)
    # end if

    if response.status_code == 200:
        bing_json = response.json()

        if 'webPages' in bing_json and \
                'value' in bing_json['webPages']:
            results = []
            rank = 0

            for hit in bing_json['webPages']['value']:
                hit_dict = {'rank': rank}
                hit_dict['name'] = hit['name']
                hit_dict['url'] = hit['url']
                hit_dict['snippet'] = hit['snippet']
                hit_dict['date'] = hit['dateLastCrawled']
                results.append(hit_dict)
                rank += 1
            # end for

            _bing_cache[query] = results
            _cache_additions += 1

            if _cache_additions % 10 == 0:
                _save_cache_file(_bing_cache_file)
            # end if

            return results
    else:
        print(
            f'Error querying Bing with [{query}; error code {response.status_code}]',
            file=sys.stderr, flush=True)
        return []

def _remove_duplicate_hits(search_results: list) -> list:
    results = []
    url_set = set()

    for hit in search_results:
        if hit['url'] not in url_set:
            results.append(hit)
            url_set.add(hit['url'])
        # end if
    # end for

    return results


from django.conf import settings
import django

from dota2Private.settings import DATABASES, INSTALLED_APPS
settings.configure(DATABASES=DATABASES, INSTALLED_APPS=INSTALLED_APPS)
django.setup()

import time
from schedule import Scheduler
import threading
from privateApp.database_fetcher import getGamesDota2Api, getmatchesDetailsStratz
from privateApp.dabase_writer import add_to_database
from nordvpn_switcher import rotate_VPN
from django.db import connection

SECRET_KEY = 'django-insecure-uf0*ru4@iah!v($!i-(fm$4vqjd$x=!p^qeliqm$v5%n6=j4*u'

from privateApp import fetch_and_write
from privateApp.models import *
run = 0

def fetch_and_write():

    global run

    try:
        Match.objects.all().first()
    except Exception:
        connected = False
        print("database connection failed")
        connection.close()
        cursor = connection.cursor()
    else:
        connected = True
    if not run ==429 and connected == True:
        with open(r'privateApp/lastSequence.txt') as f:
            maxid = f.readline
            maxid =maxid()
        allIds=[]
        last_seq = 0
        for i in range(0,9):
            resd2, matchIds, temp_last_seq = getGamesDota2Api(maxid)
            if resd2 ==200:
                allIds+=matchIds
                last_seq = temp_last_seq
                maxid=temp_last_seq
            elif i > 2:
                break
            else:
                time.sleep(2)

        print("Found "+str(len(allIds))+" matches")
        if not len(allIds) == 0:
            print(str(allIds[len(allIds)-1]))
            chunks = [allIds[x:x+100] for x in range(0, len(allIds), 100)]
            success=True
            for chunk in chunks:
                resstratz, data = getmatchesDetailsStratz(chunk,maxid)
                run = resstratz
                if resstratz==200:
                    add_to_database(data)

                else:
                    success=False
                    break

            if success and last_seq!=0:
                try:
                    with open('privateApp/lastSequence.txt', "w") as myfile:
                        myfile.write(last_seq)
                except Exception as e:
                    print('File error main loop '+ str(e))
                    try:
                        with open('privateApp/lastSequence.txt', "w") as myfile:
                            myfile.write(last_seq)
                    except Exception as e:
                        print('File error main loop '+ str(e))
        elif resd2 != 200:
            time.sleep(5)

    else:
        if(run==429):
            rotate_VPN()
        time.sleep(30)
        run = 200

while True:
    fetch_and_write()
    time.sleep(1)


from django.conf.urls import url, include
from django.contrib import admin

from .views import Home, Base, Team, Landingpage
from videos.views import VideoListView
from videos.views import (
    VideoDetailView ,
    VideoCreateView ,
    VideoUpdateView ,
    VideoDeleteView ,
)

from .views import UsercreateView , UserCreateDoneView

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^accounts/', include('django.contrib.auth.urls')),
    url(r'^accounts/register/$', UsercreateView.as_view(), name='register'),
    url(r'^accounts/register/done', UserCreateDoneView.as_view(), name='register_done' ),
    url(r'^$', Home.as_view()),
    url(r'^base/$', Base.as_view()),
    url(r'^team/$', Team.as_view()),
    url(r'^videos/$', VideoListView.as_view(), name='videos'),
    url(r'^videos/(?P<pk>\d+)/$', VideoDetailView.as_view(), name='video-detail'),
    url(r'^videos/create/$', VideoCreateView.as_view(), name='video-create'),
    url(r'^videos/(?P<pk>\d+)/update/$', VideoUpdateView.as_view(), name='video-update'),
    url(r'^videos/(?P<pk>\d+)/delete/$', VideoDeleteView.as_view(), name='video-delete'),
    url(r'^index/$', Landingpage.as_view(), name='Landingpage'),
    url(r'^photo/', include('photo.urls', namespace='photo')),
]


from pyird.image.pattern_model import median_XY_profile
import numpy as np
import pkg_resources
from pyird.utils import irdstream
import pathlib
import matplotlib.pyplot as plt
from pyird.image.channel import image_to_channel_cube, channel_cube_to_image, eopixel_split, eopixel_combine
from pyird.image.bias import bias_subtract_image
from pyird.image.hotpix import identify_hotpix
import astropy.io.fits as pyf

# hotpixel mask
datadir = pathlib.Path('/home/kawahara/pyird/data/dark/')
anadir = pathlib.Path('/home/kawahara/pyird/data/dark/')
dark = irdstream.Stream2D('targets', datadir, anadir)
dark.fitsid = [41018]
for data in dark.rawpath:
    im = pyf.open(str(data))[0].data
im_subbias = bias_subtract_image(im)
hotpix_mask, obj = identify_hotpix(im_subbias)

# Load data
mode = 'faint'
if mode == 'dark':
    datadir = pathlib.Path('/home/kawahara/pyird/data/dark/')
    anadir = pathlib.Path('/home/kawahara/pyird/data/dark/')
    target = irdstream.Stream2D('targets', datadir, anadir)
    target.fitsid = [41018]
elif mode == 'faint':
    datadir = pathlib.Path('/home/kawahara/pyird/data/samples/REACH/')
    anadir = pathlib.Path('/home/kawahara/pyird/data/samples/REACH/')
    target = irdstream.Stream2D('targets', datadir, anadir)
    # target.fitsid=[47103]
    target.fitsid = [47077]
else:
    print('no mode')
    import sys
    sys.exit()

# Load an image
for datapath in target.rawpath:
    im = pyf.open(str(datapath))[0].data

# image for calibration
calim = np.copy(im)

# aperture
if mode == 'faint':
    from pyird.io.iraf_trace import read_trace_file
    pathC = (pkg_resources.resource_filename('pyird', 'data/samples/aprefC'))
    path_c = (pkg_resources.resource_filename('pyird', 'data/samples/apref_c'))
    y0, interp_function, xmin, xmax, coeff = read_trace_file([pathC, path_c])

    from pyird.image.mask import trace
    from pyird.image.trace_function import trace_legendre
    mask = trace(im, trace_legendre, y0, xmin, xmax, coeff)
    calim[mask] = np.nan
    calim[hotpix_mask] = np.nan

#############################################################
# REMOVAL CODE

model_im = median_XY_profile(calim)
corrected_im = im-model_im
print(np.sum(corrected_im))

###########################################
fig = plt.figure(figsize=(8, 4))
ax1 = fig.add_subplot(131)
if mode == 'dark':
    vmax = -8.0
    vmin = -15.0
elif mode == 'faint':
    vmax = 10.0
    vmin = -15.0

cc = ax1.imshow(im, vmin=vmin, vmax=vmax)
plt.colorbar(cc, shrink=0.55)
ax1.set_title('Raw image')

ax2 = fig.add_subplot(132)
cc = ax2.imshow(model_im, vmin=vmin, vmax=vmax)
plt.colorbar(cc, shrink=0.55)
ax2.set_title('Model image')

ax3 = fig.add_subplot(133)
cc = ax3.imshow(corrected_im, vmin=-3.0, vmax=4.0)
plt.colorbar(cc, shrink=0.55)
ax3.set_title('pattern corrected')
plt.savefig('pattern_correction.png')
plt.show()


# leetcode-字谜分组.py
# 给定一个字符串数组，将字母异位词组合在一起。字母异位词指字母相同，但排列不同的字符串。

# 示例:

# 输入: ["eat", "tea", "tan", "ate", "nat", "bat"],
# 输出:
# [
#   ["ate","eat","tea"],
#   ["nat","tan"],
#   ["bat"]
# ]
# 说明：

# 所有输入均为小写字母。
# 不考虑答案输出的顺序。

"""
思路:
长度,~集合~,,count

en ..
想到了排序整个strs
但没有想到排序单个str呢...

看清关键再哪喽..

解答即参考...

"""
from collections import defaultdict
class Solution(object):
    def groupAnagrams(self, strs):
        """
        :type strs: List[str]
        :rtype: List[List[str]]
        """

        # 参考
        sorted_words = defaultdict(list)

        for word in strs:
            sorted_list = [c for c in word]
            sorted_list.sort()
            sorted_word = "".join(sorted_list)

            sorted_words[sorted_word].append(word)

        return list(sorted_words.values())

        # i = 0
        # ans = []

        # while i<len(strs):
        #     tmps = strs[i]
        #     ans.append([strs[i]])
        #     strs.pop(i)

        #     j = i
        #     while j<len(strs):
        #         if len(tmps) == len(strs[j]):
        #             if not any([tmps.count(c) != strs[j].count(c) for c in set(tmps)]):
        #                 ans[-1].append(strs[j])
        #                 strs.pop(j)
        #                 j-=1

        #         j += 1

        # return ans


"""
Spaces defined for the algorithms to user
It might be a good idea to experiment with some of the distributions here
"""

from math import log
from hyperopt import hp
from hyperopt.pyll.base import scope

LR_LOG_LOWER = log(0.0001)
LR_LOG_UPPER = log(0.1)

# NOTE: Do not use keyword arguments for hyperopt distributions. It causes a crash when using ATPE

SVD_DEFAULT_SPACE = {
    "n_factors": scope.int(hp.quniform("n_factors", 1, 200, 1)),
    "n_epochs": scope.int(hp.quniform("n_epochs", 1, 200, 1)),
    "lr_bu": hp.loguniform("lr_bu", LR_LOG_LOWER, LR_LOG_UPPER),
    "lr_bi": hp.loguniform("lr_bi", LR_LOG_LOWER, LR_LOG_UPPER),
    "lr_pu": hp.loguniform("lr_pu", LR_LOG_LOWER, LR_LOG_UPPER),
    "lr_qi": hp.loguniform("lr_qi", LR_LOG_LOWER, LR_LOG_UPPER),
    "reg_bu": hp.loguniform("reg_bu", LR_LOG_LOWER, LR_LOG_UPPER),
    "reg_bi": hp.loguniform("reg_bi", LR_LOG_LOWER, LR_LOG_UPPER),
    "reg_pu": hp.loguniform("reg_pu", LR_LOG_LOWER, LR_LOG_UPPER),
    "reg_qi": hp.loguniform("reg_qi", LR_LOG_LOWER, LR_LOG_UPPER),
}

SVDPP_SPACE = {
    **SVD_DEFAULT_SPACE,
    "lr_yj": hp.loguniform("lr_yj", LR_LOG_LOWER, LR_LOG_UPPER),
    "reg_yj": hp.loguniform("reg_yj", LR_LOG_LOWER, LR_LOG_UPPER),
}

# Conditioally define similarity options space since `shrinkage` only applies
# for `pearson_baseline` similarity.
@scope.define
def define_similarity_options_space(name, user_based, min_support, shrinkage):
    space = {"name": name, "user_based": user_based, "min_support": min_support}
    if name == "pearson_baseline":
        space.update({"shrinkage": shrinkage})

    return space

SIMILARITY_OPTIONS_SPACE = scope.define_similarity_options_space(
    hp.choice("name", ["cosine", "msd", "pearson", "pearson_baseline"]),
    hp.choice("user_based", [False, True]),
    scope.int(hp.quniform("min_support", 1, 100, 1)),
    scope.int(hp.quniform("shrinkage", 1, 300, 1)),
)

BSL_OPTIONS_SPACE = hp.choice(
    "bsl_options",
    [
        {
            "method": "als",
            "reg_i": hp.uniform("reg_i", 1, 100),
            "reg_u": hp.uniform("reg_u", 1, 100),
            "n_epochs": scope.int(hp.quniform("n_epochs", 5, 200, 1)),
        },
        {
            "method": "sgd",
            "reg": hp.loguniform("reg", LR_LOG_LOWER, LR_LOG_UPPER),
            "learning_rate": hp.loguniform("learning_rate", LR_LOG_LOWER, LR_LOG_UPPER),
        },
    ],
)

BASELINE_ONLY_SPACE = {"bsl_options": BSL_OPTIONS_SPACE}

KNN_DEFAULT_SPACE = {
    "k": scope.int(hp.quniform("k", 1, 500, 1)),
    "min_k": scope.int(hp.quniform("min_k", 1, 50, 1)),
    "sim_options": SIMILARITY_OPTIONS_SPACE,
}

KNN_BASELINE_SPACE = {**KNN_DEFAULT_SPACE, "bsl_options": BSL_OPTIONS_SPACE}

NMF_DEFAULT_SPACE = {
    "n_factors": scope.int(hp.quniform("n_factors", 1, 500, 1)),
    "n_epochs": scope.int(hp.quniform("n_epochs", 5, 200, 1)),
    "lr_bu": hp.loguniform("lr_bu", LR_LOG_LOWER, LR_LOG_UPPER),
    "lr_bi": hp.loguniform("lr_bi", LR_LOG_LOWER, LR_LOG_UPPER),
    "reg_bu": hp.loguniform("reg_bu", LR_LOG_LOWER, LR_LOG_UPPER),
    "reg_bi": hp.loguniform("reg_bi", LR_LOG_LOWER, LR_LOG_UPPER),
    "reg_pu": hp.loguniform("reg_pu", LR_LOG_LOWER, LR_LOG_UPPER),
    "reg_qi": hp.loguniform("reg_qi", LR_LOG_LOWER, LR_LOG_UPPER),
    "biased": hp.choice("biased", [False, True]),
}

CO_CLUSTERING_DEFAULT_SPACE = {
    "n_cltr_u": scope.int(hp.quniform("n_cltr_u", 1, 1000, 1)),
    "n_cltr_i": scope.int(hp.quniform("n_cltr_i", 1, 100, 1)),
    "n_epochs": scope.int(hp.quniform("n_epochs", 5, 200, 1)),
}

HPO_SPACE_MAP = {
    "svd": SVD_DEFAULT_SPACE,
    "svdpp": SVDPP_SPACE,
    "nmf": NMF_DEFAULT_SPACE,
    "knn_baseline": KNN_BASELINE_SPACE,
    "knn_basic": KNN_DEFAULT_SPACE,
    "knn_with_means": KNN_DEFAULT_SPACE,
    "knn_with_z_score": KNN_DEFAULT_SPACE,
    "co_clustering": CO_CLUSTERING_DEFAULT_SPACE,
    "slope_one": None,
    "baseline_only": BASELINE_ONLY_SPACE,
    "normal_predictor": None,
}


import numpy as np
import matplotlib.pyplot as plt

def MetNaiva(f, a, b, n, x):
    X = np.linspace(a, b, n+1)
    Y = f(X)
    A = np.vander(X)
    C = np.linalg.solve(A, Y)
    C = np.flip(C)
    y = 0
    for i in range(n+1):
        y += C[i] * x ** i
    return y

def f(x):
    return np.exp(2*x)
#
a = - 1
b = 1

def PlotInterpolare(f, a, b, n, metoda):
    plt.title('n = {val}'.format(val=n))
    X = np.linspace(a, b, n+1)
    x_grafic = np.linspace(a, b, 50*n+1)
    Pn = []
    for x in x_grafic:
        Pn.append(metoda(f, a, b, n, x))
    plt.plot(x_grafic, f(x_grafic), c='g', linewidth=1)
    plt.plot(x_grafic, Pn, c='r', linestyle='dotted', linewidth=1)
    for el in X:
        plt.scatter(el, f(el), c='b')
    plt.legend(['Functia f', 'Polinomul de interpolare Lagrange'])
    plt.show()

for i in range(1, 5):
    PlotInterpolare(f, a, b, i, MetNaiva)

def PlotInterpolareEroare(f, a, b, n, metoda):
    plt.title('Eroare absoluta pt n = {val}'.format(val=n))
    X = np.linspace(a, b, n+1)
    x_grafic = np.linspace(a, b, 50*n+1)
    Pn = []
    for x in x_grafic:
        Pn.append(metoda(f, a, b, n, x))
    plt.plot(x_grafic, abs(f(x_grafic) - Pn), c='r')
    plt.show()

for i in range(1, 5):
    PlotInterpolareEroare(f, a, b, i, MetNaiva)

#Rezolvare Ex2

def MetLagrange(f, a, b, n, x):
    X = np.linspace(a, b, n + 1)
    Y = f(X)
    y = 0
    for k in range(n+1):
        L = 1
        for i in range(n + 1):
            if i != k:
                L = L * (x-X[i])/(X[k] - X[i])
        y = y + L * Y[k]
    return y

for i in range(1, 5):
    PlotInterpolare(f, a, b, i, MetLagrange)

for i in range(1, 5):
    PlotInterpolareEroare(f, a, b, i, MetLagrange)


# 실버 1

import sys
from collections import deque

input = sys.stdin.readline

m, n = map(int, input().split())
dx, dy = [-1, 1, 0, 0, -1, -1, 1, 1], [0, 0, -1, 1, -1, 1, -1, 1]
board = [input().split() for _ in range(m)]
result = 0

for i in range(m):
    for j in range(n):
        if board[i][j] == "1":
            board[i][j] = "0"
            queue = deque([(i, j)])

            while queue:
                x, y = queue.popleft()
                for k in range(8):
                    nx, ny = x + dx[k], y + dy[k]
                    if 0 <= nx < m and 0 <= ny < n and board[nx][ny] == "1":
                        board[nx][ny] = "0"
                        queue.append((nx, ny))

            result += 1

print(result)


import os
import subprocess
import collections
import re

# private
def d2nt(dictionary):
    return collections.namedtuple('metadata', dictionary.keys())(**dictionary)

def export_env(key, value):
    with open(os.getenv("GITHUB_ENV"), "a") as f:
        f.write("%s=%s\n" % (key, value))

class Repo(object):
    def __init__(self, name, url, branch_rx=None, extraction_rx=None):
        print("[Repo Object] Initializing repo %s with URL %s…" % (name, url))
        self.name = name
        self.url = url
        self.commit = None
        self.branch_rx = branch_rx
        self.extraction_rx = extraction_rx

        self._latest_commit = None
        self._branches = None
        self._tags = None

    @property
    def latest_commit(self):
        if self._latest_commit is None:
            print("[Repo Object] Fetching latest commit for %s…" % self.name)
            p = subprocess.check_output([
                "git",
                "ls-remote",
                self.url
            ]).decode("utf8")
            for line in p.split("\n"):
                if "HEAD" in line:
                    self._latest_commit = line[:40]
        return self._latest_commit

    @property
    def branches(self):
        if self._branches is None:
            print("[Repo Object] Fetching branches for %s…" % self.name)
            p = subprocess.check_output([
                "git", "ls-remote", "--heads", self.url
            ]).decode("utf8")
            branches = []
            for line in p.split("\n"):
                if line.strip() == "":
                    continue

                match = line.split()

                hash = match[0]
                name = match[1]

                branches.append((hash, name))
            self._branches = branches
        return self._branches

    @property
    def tags(self):
        if self._tags is None:
            print("[Repo Object] Fetching tags for %s…" % self.name)
            p = subprocess.check_output([
                "git", "ls-remote", "--tags", "--sort=v:refname",
                self.url
            ]).decode("utf8")

            tags = []
            for line in p.split("\n"):
                if line.strip() == "":
                    continue

                match = line.split()

                hash = match[0]
                name = match[1].split("/")[2]

                tags.append((hash, name))
            self._tags = tags
        return self._tags

    # Helper Functions
    def match_branch(self, line):
        match = re.match(self.branch_rx, line)
        if match is not None:
            return match[1]

    def match_line(self, line):
        match = re.match(self.extraction_rx, line)
        if match is not None:
            return match[1]

    def out_of_date(self):
        return self.commit != self.latest_commit

origin = os.getenv("REPO_URL")
repo = Repo("Openlane", origin)

# public
gh = d2nt({
    "origin": origin,
    "branch": os.getenv("BRANCH_NAME"),
    "image": os.getenv("IMAGE_NAME"),
    "root": os.getenv("GITHUB_WORKSPACE"),
    "pdk": os.getenv("PDK_ROOT"),
    "tool": os.getenv("TOOL"),
    "event": d2nt({
        "name": os.getenv("GITHUB_EVENT_NAME")
    }),
    "export_env": export_env,
    "Repo": Repo,
    "openlane": repo
})


from scipy.misc import factorial as fac
from sys import argv
import time
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
import matplotlib.pyplot as plt
from pyrap.tables import table
import astropy.io.fits as fits
from astropy import wcs
from pyuvdata import UVBeam
import os
import sys
import glob
import argparse
import shutil
import copy
import healpy
import scipy.stats as stats
import pytz
import datetime
import ephem
from scipy import interpolate
from astropy.time import Time
import pylab as plt
from astropy.table import Table
from scipy import interpolate
from scipy.interpolate import interp1d

def beam_interp_func(theta, phi, data):
            # convert to radians
            theta = copy.copy(theta) * np.pi / 180.0
            phi = copy.copy(phi) * np.pi / 180.0
            shape = theta.shape
            beam_interp = [healpy.get_interp_val(m, theta.ravel(), phi.ravel(), lonlat=False).reshape(shape) for m in data]
            return np.array(beam_interp)

def lookupNearest(theta_s,phi_s,theta,phi):

  index = np.where((np.abs(theta-theta_s)==np.min(np.abs(theta-theta_s))) &
                   (np.abs(phi-phi_s)==np.min(np.abs(phi-phi_s))))

  return index

# generate random sky models
RA_inner=[]
DE_inner=[]
while len(RA_inner)<100:
   RA_cen=np.random.randint(-55,55,1)
   DE_cen=np.random.randint(-35,35,1)
   r=np.sqrt(RA_cen**2+DE_cen**2)
   if r<30:
        RA_inner.append(RA_cen)
        DE_inner.append(DE_cen)

RA_outer=[]
DE_outer=[]
while len(RA_outer)<100:
   RA_cen=np.random.randint(-55,55,1)
   DE_cen=np.random.randint(-35,35,1)
   r=np.sqrt(RA_cen**2+DE_cen**2)
   if r>40:
        RA_outer.append(RA_cen)
        DE_outer.append(DE_cen)

plt.plot(RA_inner,DE_inner,'ob')
plt.plot(RA_outer,DE_outer,'or')
RA_inner=np.array(RA_inner)[:,0]
DE_inner=np.array(DE_inner)[:,0]

RA_outer=np.array(RA_outer)[:,0]
DE_outer=np.array(DE_outer)[:,0]

path_to_rand='/vault-ike/ntsikelelo/simulated_ms_files/Artificial_sky_models/20_random_distributed_sources/'
np.save(path_to_rand+'RA_cen.npy',RA_inner)
np.save(path_to_rand+'DE_cen.npy',DE_inner)
np.save(path_to_rand+'RA_cen_outer.npy',RA_outer)
np.save(path_to_rand+'DE_cen_outer.npy',DE_outer)

# pointing direction
ms = table('/home/ntsikelelo/zen.2458115.31193.xx.HH.uvR.ms.split')
fld = table('/home/ntsikelelo/zen.2458115.31193.xx.HH.uvR.ms.split'+'::FIELD')
radec0 = fld.getcol('PHASE_DIR').squeeze().reshape(1,2)
radec0 = np.tile(radec0, (60,1))
fld.close()
point_ra=np.rad2deg(radec0[0,0])
point_dec=np.rad2deg(radec0[0,1])
path_to_rand='/vault-ike/ntsikelelo/simulated_ms_files/Artificial_sky_models/20_random_distributed_sources/'

RA_cen=np.zeros(shape=(200))
DE_cen=np.zeros(RA_cen.shape)
RA_cen[0:100]=np.load(path_to_rand+'RA_cen.npy')
DE_cen[0:100]=np.load(path_to_rand+'DE_cen.npy')
RA_cen[100:200]=np.load(path_to_rand+'RA_cen_outer.npy')
DE_cen[100:200]=np.load(path_to_rand+'DE_cen_outer.npy')
DE=DE_cen+point_dec
RA=RA_cen+point_ra
flux=np.ones(DE.shape)
flux[0:100]=flux[100:200]*1e-2
flux[100:200]=flux[100:200]*1e2
spectral_index=np.ones(DE.shape)*(-0.7)

for o in range (flux.shape[0]):
    sky_model= open(path_to_rand+"model_sky_gains_30_rand_original_full.txt","w+")
    sky_model.write("#format:name ra_d dec_d i q u v emaj_d emin_d pa_d spi freq0"+"\n")
    sources=[]
    for k in range (flux.shape[0]):
        sources.append("SRC"+str(k)+" "+str(RA[k])+" "+str(DE[k])+" "+str(flux[k])+" "+"0"+" "+"0"+" "+"0"+" "+"0"+" "+"0"+" "+"0"+" "+str(-0.7)+" "+"150e6")
    for i in range(len(sources)):
        sky_model.write(sources[i]+"\n")
    sky_model.close()

#  #HERA beam
# print("Now simulating with HERA_beam")
# uvb = UVBeam()
# uvb.read_beamfits('/home/ntsikelelo/NF_HERA_power_beam_healpix128.fits')
# pol_ind = np.where(uvb.polarization_array ==-6)[0][0]
# beam_maps = np.abs(uvb.data_array[0, 0, pol_ind, :, :])
# beam_freqs = uvb.freq_array.squeeze() / 1e6
# Nbeam_freqs = len(beam_freqs)
# beam_nside = healpy.npix2nside(beam_maps.shape[1])

# theta = np.sqrt( (RA - point_ra)**2 + (DE - point_dec)**2 ) # center origin at the array dec and RA
# phi = np.arctan2((DE-point_dec), (RA-point_ra)) + np.pi

# pb = beam_interp_func(theta, phi,beam_maps)

# data_freqs=np.linspace(100,200,1024)
# Ndata_freqs = len(data_freqs)

# beam_values=np.zeros(shape=(1024, DE_cen.shape[0]))
# for source in range (pb.shape[1]):
#     pb_interp = interpolate.interp1d(beam_freqs,pb[:,source], kind='cubic')
#     beam_values[:,source]=pb_interp(data_freqs)

# gains=np.zeros((1,8,1024,DE_cen.shape[0],1), dtype=complex)
# gains[:,:,:,:,0]=np.sqrt(beam_values)
# np.save(path_to_rand+'Gains_HERA_Fagnoni_beam_full.npy', gains)

n_chan=1024
#         mutual coupling beams
beam_name=np.array(['06','22','26'])

for k in range (len(beam_name)):
    print("Now simulating beam "+beam_name[k])
    uvb = UVBeam()
    uvb.read_beamfits('/home/ntsikelelo/HERA-Beams/NF_CrossCouplingBeams/NF_HERA-Dipole_ccbeam_port'+beam_name[k]+'_E-field_phasecorrected.fits')
    uvb.peak_normalize()
    print("beam normalised")

    beam_maps = uvb.data_array[0, 0, 0, :, :, :]
    beam_freqs = uvb.freq_array.squeeze() / 1e6
    Nbeam_freqs = len(beam_freqs)
    phi_grid,theta_grid=np.meshgrid(uvb.axis1_array,uvb.axis2_array) ## phi,theta

    theta = np.sqrt( (RA_cen)**2 + (DE_cen)**2 ) # source origin at the array dec and RA
    phi = np.arctan2((DE_cen), (RA_cen)) + np.pi

    pb=np.zeros((beam_freqs.shape[0],DE_cen.shape[0]),dtype=complex)
    for source in range (len(phi)):
        index=lookupNearest(np.deg2rad(theta[source]),np.deg2rad(phi[source]),theta_grid,phi_grid)
        index=np.array(index)[:,0]
        pb[:,source]=beam_maps[:,index[0],index[1]]

        ##phase at zenith
    theta_0 = np.sqrt( (0)**2 + (0)**2 ) # center origin at the array dec and RA
    phi_0 = np.arctan2((0), (0)) + np.pi
    index_0=lookupNearest(np.deg2rad(theta_0),np.deg2rad(phi_0),theta_grid,phi_grid)
    index_0=np.array(index_0)[:,0]
    pb_0=beam_maps[:,index_0[0],index_0[1]]

    for source in range(DE_cen.shape[0]):
        pb[:,source]=pb[:,source]*(np.conjugate(pb_0)/np.abs(pb_0))

    data_freqs=np.linspace(100,200,n_chan)
    Ndata_freqs = len(data_freqs)
    beam_values=np.zeros(shape=(n_chan, DE_cen.shape[0]),dtype=complex)

    print("interpolating over freq")
    for source in range (pb.shape[1]):
        pb_interp_abs = interpolate.interp1d(beam_freqs,np.abs(pb[:,source]), kind='cubic')
        pb_interp_phase = interpolate.interp1d(beam_freqs,pb[:,source], kind='cubic')
        phase=np.angle(pb_interp_phase(data_freqs))
        amp=pb_interp_abs(data_freqs)
        beam_values[:,source]=amp*np.exp(1j*phase)

    gains=np.zeros((1,1,n_chan,DE_cen.shape[0],1), dtype=complex)
    gains[:,:,:,:,0]=beam_values
    np.save(path_to_rand+'sky_model_mutual_'+beam_name[k]+'_full.npy', gains)

HERA_beam_mutual_06=np.load(path_to_rand+'sky_model_mutual_06_full.npy')
HERA_beam_mutual_22=np.load(path_to_rand+'sky_model_mutual_22_full.npy')
HERA_beam_mutual_26=np.load(path_to_rand+'sky_model_mutual_26_full.npy')

## mutual_coupling_gain
mutual_gains=np.zeros(shape=(1,8,1024,HERA_beam_mutual_06.shape[3],1),dtype=complex)
mutual_gains[:,0,:,:,:]=HERA_beam_mutual_06
mutual_gains[:,1,:,:,:]=HERA_beam_mutual_26
mutual_gains[:,2,:,:,:]=HERA_beam_mutual_26
mutual_gains[:,3,:,:,:]=HERA_beam_mutual_22
mutual_gains[:,4,:,:,:]=HERA_beam_mutual_22
mutual_gains[:,5,:,:,:]=HERA_beam_mutual_22
mutual_gains[:,6,:,:,:]=HERA_beam_mutual_22
mutual_gains[:,7,:,:,:]=HERA_beam_mutual_22

np.save(path_to_rand+'Gains_mutual_coupling_EEC_ECC_CCC_full.npy',mutual_gains)


"""studentlife URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""

from django.urls import path
from career import views

urlpatterns = [
    path('index/', views.index,name="index"),
    path('login/', views.login, name="login"),
    path('contract/', views.contract, name="contract"),
    path('education/', views.education, name="education"),
    path('registation/', views.registation, name="registation"),
    path('allstu/', views.allstu, name="allstu"),
    path('about/', views.about, name="about"),
    path('bank/', views.bank, name="bank"),
    path('books/', views.books, name="books"),

]


#-------------------------------------------------------------------------------
# Name:        module1
# Purpose:
#
# Author:      alex
#
# Created:     29/08/2019
# Copyright:   (c) alex 2019
# Licence:     <your licence>
#-------------------------------------------------------------------------------

import csv
import json
#import unidecode
import unicodedata

with open('dictionary.csv') as csv_file:
    csv_reader = csv.reader(csv_file, delimiter=';')
    header = csv_reader.next()
    dic = {}
    for name in header:
        uname = unicode(name, "utf-8")
        uname=unicodedata.normalize('NFKD', uname).encode('ascii','ignore')
        dic[uname.upper()] = {}
    for row in csv_reader:
        for idx,name in enumerate(header):
            uname = unicode(name, "utf-8")
            uname=unicodedata.normalize('NFKD', uname).encode('ascii','ignore')
            value = row[idx].strip()
            if value != '':
                uvalue = unicode(value, "utf-8")
                nuvalue=unicodedata.normalize('NFKD', uvalue).encode('ascii','ignore')
                letter = nuvalue[0].upper()# unidecode.unidecode(value)[0]

                uvalues = uvalue.split("/")
                if len(uvalues) == 0:
                    uvalues = uvalue.split("\\")
                if len(uvalues) == 0:
                    uvalues = uvalue.split(",")
                if len(uvalues) == 0:
                    uvalues = [uvalue]

                for item in uvalues:
                    try:
                        dic[uname.upper()][letter].append(item.strip())
                    except:
                        dic[uname.upper()][letter] = [item.strip()]

with open(r'../plugin/list_of_words.js', 'w') as f:
    f.write("var stopCheats = ")
    json.dump (dic, f, indent=4, sort_keys=True)# , encoding='Win-1252')
    f.write(";\n\n")

    #var stopCheats = [


import os
import sys
import tempfile
import logging
import fcntl
import errno
import signal
from signal import SIGABRT,  SIGILL, SIGINT, SIGSEGV, SIGTERM, signal, SIGQUIT
import atexit

def signal_handler(signal, frame):
    signal.signal(signal.SIGINT, signal_handler)
    signal.signal(signal.SIGQUIT, signal_handler)
    signal.signal(signal.SIGTSTP, signal_handler)

def single_process():
    filename = os.path.abspath(sys.argv[0])
    lock_file_name = filename.lstrip('/').replace('/', '_') + '.lock'

    lock_file_path = os.path.join(
        tempfile.gettempdir(), lock_file_name)
    fd = os.open(lock_file_path, os.O_CREAT | os.O_RDWR, 0o660)
    try:
        fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except IOError as e:
        if e.errno in (errno.EACCES, errno.EAGAIN):
            logging.error(
                '%s RUNNING . QUIT' % filename
            )
            sys.exit(0)
        else:
            raise

    def exit(signal, frame):
        os.close(fd)
        os.remove(lock_file_path)
        sys.exit(signal)

    for sig in (SIGABRT,  SIGILL, SIGINT, SIGSEGV, SIGTERM, SIGQUIT):
        signal(sig, exit)
    atexit.register(lambda: exit(0, 0))

if __name__ == "__main__":

    single_process()

    def test():
        from time import sleep
        from datetime import datetime
        for i in range(3):
            print(datetime.now())
            sleep(1)
    test()


import DosePrediction.Train.config as config
from typing import Optional

from monai.inferers import sliding_window_inference
from monai.data import DataLoader, list_data_collate

import pytorch_lightning as pl
from pytorch_lightning.callbacks.model_checkpoint import ModelCheckpoint
from pytorch_lightning.callbacks import TQDMProgressBar
from pytorch_lightning.loggers import MLFlowLogger

import bitsandbytes as bnb

from DosePrediction.Models.Networks.models_experiments import *
from DosePrediction.DataLoader.dataloader_OpenKBP_monai import get_dataset
from DosePrediction.Evaluate.evaluate_openKBP import *
from DosePrediction.Train.loss import GenLoss

os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
torch.backends.cudnn.benchmark = True

class LitProgressBar(TQDMProgressBar):
    def init_validation_tqdm(self):
        bar = super().init_validation_tqdm()
        bar.set_description("running validation...")
        return bar

class OpenKBPDataModule(pl.LightningDataModule):
    def __init__(self, crop_flag, image_size):
        super().__init__()
        self.val_data = None
        self.train_data = None
        self.crop_flag = crop_flag
        self.image_size = image_size

    def setup(self, stage: Optional[str] = None):
        # Assign train/val datasets for use in dataloaders
        self.train_data = get_dataset(path=config.MAIN_PATH + config.TRAIN_DIR, state='train',
                                      size=100, cache=True, crop_flag=self.crop_flag, image_size=self.image_size)

        self.val_data = get_dataset(path=config.MAIN_PATH + config.VAL_DIR, state='val',
                                    size=20, cache=True)

    def train_dataloader(self):
        return DataLoader(self.train_data, batch_size=config.BATCH_SIZE, shuffle=True,
                          num_workers=config.NUM_WORKERS, collate_fn=list_data_collate, pin_memory=True)

    def val_dataloader(self):
        return DataLoader(self.val_data, batch_size=1, shuffle=False,
                          num_workers=config.NUM_WORKERS, pin_memory=True)

class TestOpenKBPDataModule(pl.LightningDataModule):
    def __init__(self):
        super().__init__()

    def setup(self, stage: Optional[str] = None):
        # Assign val datasets for use in dataloaders
        self.test_data = get_dataset(path=config.MAIN_PATH + config.VAL_DIR, state='test',
                                     size=20, cache=True)

    def test_dataloader(self):
        return DataLoader(self.test_data, batch_size=1, shuffle=False,
                          num_workers=config.NUM_WORKERS, pin_memory=True)

class TestModel(pl.LightningModule):
    def __init__(
            self,
            config_param,
            image_size=128,
            sw_batch_size=1,
            huber=False,
    ):
        super().__init__()
        self.config_param = config_param
        self.save_hyperparameters()

        # OAR + PTV + CT => dose

        # self.model_ = SharedUNetRModelA(
        #     in_channels_a=9,
        #     in_channels_b=9,
        #     out_channels=1,
        #     img_size=(config.IMAGE_SIZE, config.IMAGE_SIZE, config.IMAGE_SIZE),
        #     feature_size=16,
        #     hidden_size=768,
        #     mlp_dim=3072,
        #     num_heads=12,
        #     pos_embed="perceptron",
        #     norm_name="instance",
        #     res_block=True,
        #     conv_block=True,
        #     dropout_rate=0.0)
        #
        # self.model_ = SharedUNetRModel(
        #     in_channels_a=9,
        #     in_channels_b=25,
        #     out_channels=1,
        #     img_size=(config.IMAGE_SIZE, config.IMAGE_SIZE, config.IMAGE_SIZE),
        #     feature_size=16,
        #     hidden_size=768,
        #     mlp_dim=3072,
        #     num_heads=12,
        #     pos_embed="perceptron",
        #     norm_name="instance",
        #     res_block=True,
        #     conv_block=True,
        #     dropout_rate=0.0)
        #
        # self.model_ = SharedUNetModel(in_ch=9, out_ch=1,
        #                               list_ch=[-1, 16, 32, 64, 128, 256, 512],
        #                               mode_decoder=2,
        #                               mode_encoder=2)

        self.model_ = VitGenerator(
            in_ch=9,
            out_ch=1,
            mode_decoder=1,
            mode_encoder=1,
            feature_size=16,
            img_size=(image_size, image_size, image_size),
            num_layers=8,  # 4, 8, 12
            num_heads=6,  # 3, 6, 12,
            mode_multi_dec=True,
            act=config_param["act"],
            multiS_conv=config_param["multiS_conv"], )

        # self.model_ = SharedEncoderModel(in_ch=9,
        #                                  out_ch=1,
        #                                  mode_decoder=1,
        #                                  mode_encoder=1,
        #                                  img_size=(config.IMAGE_SIZE, config.IMAGE_SIZE, config.IMAGE_SIZE),
        #                                  num_layers=config_param["num_layers"],  # 4, 8, 12
        #                                  num_heads=config_param["num_heads"]  # 3, 6, 12
        #                                  )
        #
        # self.model_ = Model(in_ch=9, out_ch=1,
        #                     list_ch_A=[-1, 16, 32, 64, 128, 256],
        #                     list_ch_B=[-1, 32, 64, 128, 256, 512],
        #                     mode_decoder_A=4,
        #                     mode_decoder_B=4,
        #                     mode_encoder_A=2,
        #                     mode_encoder_B=1)
        #
        # self.model_ = ModelMonai(in_ch=9, out_ch=1,
        #                          list_ch_A=(16, 32, 64, 128, 256),
        #                          list_ch_B=(32, 64, 128, 256, 512))

        self.lr = config_param["lr"]
        self.weight_decay = config_param["weight_decay"]

        self.loss_function = GenLoss(im_size=image_size)
        self.huber = huber
        # Moving average loss, loss is the smaller, the better
        self.eps_train_loss = 0.01

        self.best_average_val_index = -99999999.
        self.average_val_index = None
        self.metric_values = []

        self.best_average_train_loss = 99999999.
        self.moving_train_loss = None
        self.train_epoch_loss = []

        self.max_epochs = 1300
        self.check_val = 5
        self.warmup_epochs = 1

        self.image_size = image_size

        self.sw_batch_size = sw_batch_size

        self.list_DVH_dif = []
        self.list_dose_metric = []
        self.dict_DVH_dif = {}
        self.ivs_values = []

    def forward(self, x):
        return self.model_(x)

    def training_step(self, batch, batch_idx):
        torch.cuda.empty_cache()
        input_ = batch['Input'].float()
        target = batch['GT']

        # train
        output = self(input_)
        torch.cuda.empty_cache()

        loss = self.loss_function(output, target, casecade=False, huber=self.huber)

        if self.moving_train_loss is None:
            self.moving_train_loss = loss.item()
        else:
            self.moving_train_loss = \
                (1 - self.eps_train_loss) * self.moving_train_loss \
                + self.eps_train_loss * loss.item()

        tensorboard_logs = {"train_loss": loss.item()}

        return {"loss": loss, "log": tensorboard_logs}

    def training_epoch_end(self, outputs):
        torch.cuda.empty_cache()
        train_mean_loss = torch.stack([x["loss"] for x in outputs]).mean()
        if train_mean_loss < self.best_average_train_loss:
            self.best_average_train_loss = train_mean_loss
        self.train_epoch_loss.append(train_mean_loss.detach().cpu().numpy())
        self.logger.log_metrics({"train_mean_loss": train_mean_loss}, self.current_epoch + 1)
        torch.cuda.empty_cache()

    def validation_step(self, batch_data, batch_idx):
        torch.cuda.empty_cache()
        input_ = batch_data["Input"].float()
        target = batch_data["GT"]
        gt_dose = np.array(target[:, :1, :, :, :].cpu())
        possible_dose_mask = np.array(target[:, 1:, :, :, :].cpu())

        roi_size = (self.image_size, self.image_size, self.image_size)

        prediction = sliding_window_inference(
            input_, roi_size, self.sw_batch_size, lambda x: self.forward(x)[0]
        )
        torch.cuda.empty_cache()
        loss = self.loss_function(prediction, target, mode='val', huber=self.huber)

        # for vit generator
        prediction_b = np.array(prediction.cpu())

        # Post-processing and evaluation
        mask = np.logical_or(possible_dose_mask < 1, prediction_b < 0)
        prediction_b[mask] = 0
        dose_score = 80. * get_3D_Dose_dif(prediction_b.squeeze(0), gt_dose.squeeze(0),
                                           possible_dose_mask.squeeze(0))

        return {"val_loss": loss, "val_metric": dose_score}

    def validation_epoch_end(self, outputs):
        torch.cuda.empty_cache()
        avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean()
        mean_dose_score = - np.stack([x["val_metric"] for x in outputs]).mean()
        if mean_dose_score > self.best_average_val_index:
            self.best_average_val_index = mean_dose_score
        self.metric_values.append(mean_dose_score)

        self.log("mean_dose_score", mean_dose_score, logger=False)
        self.log("val_loss", avg_loss, logger=False)
        self.logger.log_metrics({"mean_dose_score": mean_dose_score}, self.current_epoch + 1)
        self.logger.log_metrics({"val_loss": avg_loss}, self.current_epoch + 1)

        tensorboard_logs = {"val_metric": mean_dose_score}
        torch.cuda.empty_cache()

        return {"log": tensorboard_logs}

    def configure_optimizers(self):
        optimizer = bnb.optim.Adam8bit(self.model_.parameters(), lr=self.lr,
                                       weight_decay=self.weight_decay)
        return optimizer

    def test_step(self, batch_data, batch_idx):
        torch.cuda.empty_cache()
        input_ = batch_data["Input"].float()
        target = batch_data["GT"]
        gt_dose = target[:, :1, :, :, :].cpu()
        possible_dose_mask = target[:, 1:, :, :, :].cpu()

        roi_size = (self.image_size, self.image_size, self.image_size)

        prediction = sliding_window_inference(
            input_, roi_size, self.sw_batch_size, lambda x: self.forward(x)[0]
        ).cpu()
        torch.cuda.empty_cache()

        # Post-processing and evaluation
        prediction[np.logical_or(possible_dose_mask < 1, prediction < 0)] = 0
        prediction = 80. * prediction

        dose_dif, dvh_dif, self.dict_DVH_dif, ivs_values = get_Dose_score_and_DVH_score_batch(
            prediction, batch_data, list_DVH_dif=self.list_DVH_dif, dict_DVH_dif=self.dict_DVH_dif, ivs_values=None)
        self.list_DVH_dif.append(dvh_dif)

        torch.cuda.empty_cache()
        save_results = True
        if save_results:
            ckp_re_dir = os.path.join('YourSampleImages/DosePrediction' + 'ablation')
            if batch_idx < 12:
                plot_DVH(prediction, batch_data, path=os.path.join(ckp_re_dir, 'dvh_{}.png'.format(batch_idx)))

                # Post-processing and evaluation
                gt_dose[possible_dose_mask < 1] = 0

                predicted_img = torch.permute(prediction[0].cpu(), (1, 0, 2, 3))
                gt_img = torch.permute(gt_dose[0], (1, 0, 2, 3))
                name_p = batch_data['file_path'][0].split("/")[-2]

                for i in range(len(predicted_img)):
                    predicted_i = predicted_img[i][0].numpy()
                    gt_i = 80. * gt_img[i][0].numpy()
                    error = np.abs(gt_i - predicted_i)

                    # Create a figure and axis object using Matplotlib
                    fig, axs = plt.subplots(3, 1, figsize=(4, 10))
                    plt.subplots_adjust(wspace=0, hspace=0)

                    # Display the ground truth array
                    axs[0].imshow(gt_i, cmap='jet')
                    # axs[0].set_title('Ground Truth')
                    axs[0].axis('off')

                    # Display the prediction array
                    axs[1].imshow(predicted_i, cmap='jet')
                    # axs[1].set_title('Prediction')
                    axs[1].axis('off')

                    # Display the error map using a heatmap
                    axs[2].imshow(error, cmap='jet')
                    # axs[2].set_title('Error Map')
                    axs[2].axis('off')

                    save_dir = os.path.join(ckp_re_dir, '{}_{}'.format(name_p, batch_idx))
                    if not os.path.isdir(save_dir):
                        os.mkdir(save_dir)

                    fig.savefig(os.path.join(save_dir, '{}.jpg'.format(i)), bbox_inches="tight")

        self.list_dose_metric.append(dose_dif)
        return {"dose_dif": dose_dif}

    def test_epoch_end(self, outputs):

        mean_dose_metric = np.stack([x["dose_dif"] for x in outputs]).mean()
        std_dose_metric = np.stack([x["dose_dif"] for x in outputs]).std()
        mean_dvh_metric = np.mean(self.list_DVH_dif)
        # for key in self.dict_DVH_dif.keys():
        #     for metric in self.dict_DVH_dif[key]:
        #         self.dict_DVH_dif[key][metric] = np.mean(self.dict_DVH_dif[key][metric])

        print(mean_dose_metric, mean_dvh_metric)
        print('----------------------Difference DVH for each structures---------------------')
        print(self.dict_DVH_dif)

        self.ivs_values = np.array(self.ivs_values)

        self.log("mean_dose_metric", mean_dose_metric)
        self.log("std_dose_metric", std_dose_metric)
        return self.dict_DVH_dif

def main(act, crop_flag, sw_batch_size, image_size, huber, resume, databricks, ckp):
    # Initialise the LightningModule
    openkbp_ds = OpenKBPDataModule(crop_flag=crop_flag, image_size=image_size)
    config_param = {
        "num_layers": 8,
        "num_heads": 6,
        "act": act,
        "multiS_conv": True,
        "lr": 3e-4,
        "weight_decay": 1e-4,
    }
    net = TestModel(
        config_param,
        image_size=image_size,
        huber=huber,
        sw_batch_size=sw_batch_size,
    )

    # Set up checkpoints
    checkpoint_callback = ModelCheckpoint(
        dirpath=ckp,
        save_last=True, monitor="mean_dose_score", mode="max",
        every_n_epochs=net.check_val,
        auto_insert_metric_name=True,
    )

    # Set up logger
    if not resume:
        mlflow_logger = MLFlowLogger(
            experiment_name='EXPERIMENT_NAME',
            tracking_uri="databricks",
            run_name=databricks
        )
    else:
        mlflow_logger = MLFlowLogger(
            experiment_name='EXPERIMENT_NAME',
            tracking_uri="databricks",
            run_id=databricks
        )

    # Initialise Lightning's trainer.
    trainer = pl.Trainer(
        devices=[0],
        accelerator="gpu",
        max_epochs=net.max_epochs,
        check_val_every_n_epoch=net.check_val,
        callbacks=[checkpoint_callback],
        logger=mlflow_logger,
        default_root_dir=ckp,
        # enable_progress_bar=True,
        # log_every_n_steps=net.check_val,
    )

    # Train
    if not resume:
        trainer.fit(net, datamodule=openkbp_ds)
    else:
        trainer.fit(net,
                    datamodule=openkbp_ds,
                    ckpt_path=os.path.join(ckp, 'last.ckpt'))

    return net

if __name__ == '__main__':
    net_ = main()


import scrapy

### Year, rating, next pages to be done

class rogerSpider(scrapy.Spider):
    name = 'roger'
    start_urls = ['https://www.rogerebert.com/movies/A']
    def parse(self, response):
        SET_SELECTOR = '.details'

        for object in response.css(SET_SELECTOR):
                NAME_SELECTOR = 'a ::text'
                yield{
                    'name': object.css(NAME_SELECTOR).extract_first(),
                    }


from cryptography.fernet import Fernet

def encrypt_message(message, key):
    fernet = Fernet(key)
    encrypted_message = fernet.encrypt(message.encode())
    return encrypted_message

def write_to_encrypted_file(file_path, content, key):
    encrypted_content = encrypt_message(content, key)
    try:
        with open(file_path, 'a') as file:
            file.write(str(encrypted_content) + '\n')
        print("Successfully wrote encrypted data to the file:", file_path)
    except IOError as e:
        print("An error occurred while writing to the file:", str(e))

def decrypt_message(encrypted_message, key):
    fernet = Fernet(key)
    decrypted_message = fernet.decrypt(encrypted_message).decode()
    return decrypted_message

def read_from_encrypted_file(file_path, key):
    try:
        with open(file_path, 'rb') as file:
            encrypted_content = file.read()
            decrypted_content = decrypt_message(encrypted_content, key)
            return decrypted_content
    except IOError as e:
        print("An error occurred while reading from the file:", str(e))
        return None


import manim

from typing import List

class GeneralPositionScene(manim.ThreeDScene):
    def construct(self):
        plane = manim.NumberPlane()
        axis_config = {
            "x_range": (-10, 10, 1),
            "y_range": (-10, 10, 1),
            "z_range": (-10, 10, 1),
        }

        axes = manim.ThreeDAxes(**axis_config)
        self.set_camera_orientation(
            phi=-130*manim.DEGREES,
            theta=10*manim.DEGREES,
            distance=10,
        )
        # circle = manim.Circle()
        # square = manim.Square()
        # square.flip(manim.RIGHT)
        # square.rotate(-3 * manim.TAU / 8)
        # circle.set_fill(manim.PINK, opacity=0.5)

        vec_coordinates: List[List[int]] = [
            [2, 2], [0, 5], [1, 3],
        ]
        vec_colors = [
            manim.ORANGE,
            manim.GREEN_C,
            manim.YELLOW_C,
        ]
        self.add(plane)
        self.add(axes)

        for vec_coords, vec_color in zip(vec_coordinates, vec_colors):
            v = manim.Vector(vec_coords)
            v.set_color(vec_color)
            # v.coordinate_label(color=manim.ORANGE)
            self.add(v)

        self.begin_ambient_camera_rotation(rate=0.1)
        self.wait(12)
        # self.play(manim.Create(square))
        # self.play(manim.Transform(square, circle))
        # self.play(manim.FadeOut(square))

class NotGeneralPositionScene(manim.ThreeDScene):
    def construct(self):
        pass


from pydantic import validator, AnyHttpUrl

from tracardi.service.plugin.domain.register import Plugin, Spec, MetaData, Documentation, PortDoc, Form, FormGroup, \
    FormField, FormComponent
from tracardi.service.plugin.runner import ActionRunner
from tracardi.service.plugin.domain.result import Result
from tracardi.service.notation.dot_template import DotTemplate
from tracardi.service.plugin.domain.config import PluginConfig

class Configuration(PluginConfig):
    type: str = "success"
    message: str
    hide_after: str
    position_x: str
    position_y: str
    uix_mf_source: AnyHttpUrl = "http://localhost:8686"

    @validator("message")
    def should_no_be_empty(cls, value):
        if len(value) == 0:
            raise ValueError("Message should not be empty")
        return value

    @validator("hide_after")
    def hide_after_should_be_numeric(cls, value: str):
        if not value.isnumeric():
            raise ValueError("This value should be numeric")
        return value

def validate(config: dict):
    return Configuration(**config)

class SnackBarUx(ActionRunner):

    config: Configuration

    async def set_up(self, init):
        self.config = validate(init)

    async def run(self, payload: dict, in_edge=None) -> Result:
        dot = self._get_dot_accessor(payload)
        template = DotTemplate()

        message = template.render(self.config.message, dot)

        self.ux.append({
            "tag": "div",
            "props": {
                "class": "tracardi-uix-snackbar",
                "data-type": self.config.type,
                "data-message": message,
                "data-vertical": self.config.position_y,
                "data-horizontal": self.config.position_x,
                "data-auto-hide": self.config.hide_after
            }
        })
        self.ux.append({"tag": "script", "props": {"src": f"{self.config.uix_mf_source}/uix/snackbar/index.js"}})

        return Result(port="payload", value=payload)

def register() -> Plugin:
    return Plugin(
        start=False,
        spec=Spec(
            module=__name__,
            className='SnackBarUx',
            inputs=["payload"],
            outputs=["payload"],
            init={
                "type": "success",
                "message": "",
                "hide_after": 6000,
                "position_x": "center",
                "position_y": "bottom",
                "uix_mf_source": "http://localhost:8686"
            },
            version='0.6.1',
            license="MIT",
            author="Risto Kowaczewski",
            form=Form(groups=[
                FormGroup(
                    name="Widget Message Configuration",
                    fields=[
                        FormField(
                            id="message",
                            name="Pop-up message",
                            description="Provide message that will be shown on the web page.",
                            component=FormComponent(type="textarea", props={"label": "message"})
                        ),
                        FormField(
                            id="type",
                            name="Alert type",
                            description="Select alert type.",
                            component=FormComponent(type="select", props={"label": "Alert type", "items": {
                                "error": "Error",
                                "warning": "Warning",
                                "success": "Success",
                                "info": "Info"
                            }})
                        ),
                        FormField(
                            id="hide_after",
                            name="Hide message after",
                            description="Type number of milliseconds the message must be visible. Default: 6000. 6sec.",
                            component=FormComponent(type="text", props={"label": "hide after"})
                        ),
                        FormField(
                            id="position_y",
                            name="Vertical position",
                            description="Select where would you like to place the message.",
                            component=FormComponent(type="select", props={"label": "Vertical position", "items": {
                                "bottom": "Bottom",
                                "top": "Top"
                            }})
                        ),
                        FormField(
                            id="position_x",
                            name="Horizontal position",
                            description="Select where would you like to place the message.",
                            component=FormComponent(type="select", props={"label": "Horizontal position", "items": {
                                "left": "Left",
                                "center": "Center",
                                "right": "Right"
                            }})
                        ),
                    ]),
                FormGroup(
                    name="Widget Source Location",
                    fields=[
                        FormField(
                            id="uix_mf_source",
                            name="Micro frontend source location",
                            description="Provide URL where the micro frontend source is located. Usually it is the "
                                        "location of Tracardi API. Type different location if you use CDN.",
                            component=FormComponent(type="text", props={"label": "URL"})
                        ),
                    ],
                ),
            ]),

        ),
        metadata=MetaData(
            name='Show snack bar',
            desc='Shows snack bar pop-up on the front end.',
            icon='react',
            group=["UIX Widgets"],
            documentation=Documentation(
                inputs={
                    "payload": PortDoc(desc="This port takes payload object.")
                },
                outputs={"payload": PortDoc(desc="This port returns input payload object.")}
            ),
            frontend=True
        )
    )


#!/usr/bin/python3.4
# -*- coding: utf-8 -*-
__author__ = 'Prostakov Alexey'

import os, glob
from pymongo import MongoClient
import sys
import configparser
import smev
import logging
import protokol
import datetime

"""
Описание
**********************

Входные данные
**********************

Выходные данные
**********************

"""
logging.basicConfig(filename='tester.log', filemode='a', level=logging.DEBUG,
    format='%(asctime)s %(levelname)s: %(message)s')

def printTest(post):
    print("%s выполнено за %s сек. ошибок %s."
                % (post['name'], post['data']['Итого'], post['errors']))
    if post['errors']:
        logging.error("%s Выполнено за %s ошибок %s."
            % (post['name'], post['data']['Итого'], post['errors']))
    else:
        logging.info("%s Выполнено за %s без ошибок."
            % (post['name'], post['data']['Итого']))

def readConfig(file="config.ini"):
    '''
    :param file: имя файла конфигурации
    :return: список ИС для тестирования, словарь настроек к БД, кол-во ошибок
    '''
    err = 0
    ISList = list()
    BD = dict()
    if os.access(file, os.F_OK):
        # выполняется если найден конфигурационный файл
        config_str = open(file, encoding='utf-8', mode='r').read()
        # удалить признак кодировки
        config_str = config_str.replace(u'\ufeff', '')
        # чтение конфигурационного файла
        Config = configparser.ConfigParser()
        Config.read_string(config_str)
        sections = Config.sections()
        # читаем все секции
        for section in sections:
            i = Config[section]
            # это секция про ИС, их может быть несколько
            if section.count('IS'):
                IS = dict()
                IS['snd_name'] = i.get('name', fallback='СОЦИНФОРМТЕХ')
                IS['snd_code'] = i.get('mnemonic', fallback='SOCP01711')
                IS['oktmo'] = i.get('OKTMO', fallback='70000000')
                IS['url'] = i.get('URL', fallback='/socportal/')
                IS['adr'] = i.get('address', fallback='')
                IS['port'] = i.get('port', fallback='80')
                IS['servicecode'] = i.get('SERVICE_CODE', fallback='123456789')
                IS['SpravID'] = i.get('SpravID', fallback='1')
                IS['comment'] = i.get('comment', fallback='')
                IS['protocol'] = i.get('protocol', fallback='no')
                ISList.append(IS)
            # это про подсоединение к БД
            elif section == 'BD':
                BD['adr'] = i.get('address', fallback='')
                BD['port'] = i.get('port', fallback='27017')
                BD['BD'] = i.get('dataBase', fallback='')
                BD['protocol'] = i.get('protocol', fallback='')
        # проверим заполнение секции о БД
        if len(BD.keys()) == 0:
            print('В конфигурационном файле отсутствует обязательная секция о БД.')
            err += 1
        else:
            for key in BD.keys():
                if BD[key] == '':
                    print("Параметр %s не должен быть пустой, заполните его в конфигурационном файле %s" % (key, file))
                    err += 1
        # проверим заполнение сведений об ИС
        if len(ISList) == 0:
            print('В конфигурационном файле отсутствует обязательная секция об ИС.')
            err += 1
        else:
            for IS in ISList:
                for key in IS.keys():
                    if IS[key] == '':
                        print("В секции сведение об ИС параметр %s не должен быть пустой, заполните его в конфигурационном файле %s" % (key, file))
                        err += 1
    # нет конфигурационного файла
    else:
        print("Ошибка! Не найден конфигурационный файл")
        err = 1
    return ISList, BD, err

if __name__ == '__main__':
    logging.info("Программа запущена")
    # чтение конфигурационного файла
    ISList, BD, err = readConfig('config.ini')
    if err > 0 or ISList is None or BD is None:
        logging.critical("При чтении конфигурационного файла произошли ошибки. Программа остановлена")
        exit(1)
    else:
        logging.info("Конфигурационный файл прочитан успешно.")

    # соединяемся с БД для записи протокола
    try:
        client = MongoClient(BD['adr'], int(BD['port']))
        db = client[BD['BD']]
        collProt = db[BD['protocol']]
    except:
        logging.critical("Ошибка при соединении с БД. Программа остановлена")
        Type, Value, Trace = sys.exc_info()
        logging.critical("""Тип ошибки: %s Текст: %s""" % (Type, Value))
        exit(2)
    logging.info('Успешное соединения с БД')

    # перебираем все ИС из конф. файла
    for IS in ISList:
        err = 0
        logging.info('Обрабатываем ИС: %s' % IS['snd_name'])
        # получаем версии ПО
        IS['version'], errMsg = smev.getVersion(IS)

        if errMsg :
            logging.error(errMsg)
            continue
        if IS['protocol'] == 'yes':
            post, err1 = (protokol.getData(IS))
            err += err1
            print(post)
            if err1:
                logging.error('Получение протокола для %s не выполнено, возникли ошибки' % post['comment'])
            else:
                logging.info('Получение протокола для %s выполнено успешно' % post['comment'])
                logging.debug('Для записи подготовлено: %s' % post)
                #collProt.insert_one(post)
                logging.debug('Было записано c ID=%s' % post['_id'])
    # закрывает соединения с БД
    client.close()
    exit(0)


from setuptools import setup
import setuptools
import os
from gnutools.utils import parent, listfiles
import numpy as np
from setuptools import Extension
def get_packages(root):
    """
    Get the list of packages availables in a root

    :param root: package root
    :return list:
    """
    root = os.path.realpath(root)
    proot = parent(root) + "/"
    py_files = [file.rsplit(proot)[1] for file in listfiles(root)]
    packages = list(np.unique([parent(file).replace("/", ".") for file in py_files]))
    # return list(np.unique([parent(file).replace("/", ".").split(".{name_root}.".format(name_root=name(root)))[1]
    #                        for file in py_files]))
    return packages

print(get_packages('/mnt/IYOProjects/IYO/kenlm-publish/kenlm'))

setup(
    name="kenlm",
    version="1.0",
    long_description="",
    include_package_data=True,
    url='https://9dw-lab.jp',
    license='9DWLab',
    author='jcadic',
    # python_requires='>=3.6' and '<3.7',
    packages=["kenlm", "kenlm.bin"],
    # setup_requires=setup_requires,
    #install_requires=install_requires,
    # extra_requires=setup_requires,
    # ext_modules=[Extension('kenlm.bin', ['kenlm/bin/lmplz'])],
    author_email='j.cadic@9dw-lab.jp',
    description='KenLM',
    # platforms="linux_debian_10_x86_64",
    classifiers=[
        "Programming Language :: Python :: 3.6",
        "License :: OSI Approved :: MIT License",
    ]
)


import pixiv_auth
import json
from pixivpy3 import *

api = AppPixivAPI()

def tag_fixer(tag):
    tags = []
    translated_tags = []
    for i in tag:
        translated_tags.append(i.get("translated_name"))
        tags.append(i.get("name"))

    return tags, translated_tags

def pixiv_url_fixer(img):
    pic_url = img.image_urls.medium
    pic_url = pic_url.replace("pximg.net","pixiv.cat")
    return pic_url

def access_token_renewer(refresh_token):
    access = pixiv_auth.refresh(refresh_token)
    return access

def check_access_token_valid(refresh, access):
    api.set_auth(refresh_token=refresh, access_token=access)
    user = api.user_detail("57233775")
    try:
        if user["user_id"] == "Karlhsiao":
            return False
        else:
            return True
    except:
        return True

def info_printer(title, pic_url, bookmarks, original_tags, id):
    print("[%s]" % (title))
    print("original link: https://pixiv.net/artworks/" + str(id))
    print("bookmarks: %s" % (str(bookmarks)))
    print("tags:", original_tags)
    print("link: %s" % (pic_url))
    print("")

def turn_page_bookmarks(json_result):
    next_qs = api.parse_qs(json_result.next_url)
    if next_qs == None:
        return None
    json_result = api.user_bookmarks_illust(**next_qs)
    return json_result

def is_number(x):
    try:
        x = int(x)
        return True
    except:
        return False

def find_user_settings(user):
    with open("user_settings.json", "r") as fhandle:
        user_settings = json.load(fhandle)

        for user_id in user_settings:
            try:

                if user_id["user"] == user:
                    personal_detail = user_id["personal_detail"]
                    r18allow = user_id["r18allow"]
                    pic_amount = user_id["picture_amount"]
                    p_details = user_id["print_details"]
                    favorite = user_id["favorite"]
                    like_limit = user_id["like_limit"]

                    return personal_detail, r18allow, pic_amount, p_details, favorite, like_limit

            except:
                return False

        return False

def user_registered(user_settings, user):
    for users in user_settings:
        if user == users["user"]:
            return True

    return False


from .utils import create_transformer_masks
from .encoder import TransformerEncoder
from .decoder import TransformerDecoder

import torch
import torch.nn as nn

class Transformer(nn.Module):
    def __init__(self,
                 num_layers,
                 d_model, num_head,
                 intermediate_dim,
                 input_vocab_size,
                 target_vocab_size,
                 src_max_len,
                 tgt_max_len,
                 padding_idx,
                 shared_emb_layer=None, # Whether use embeeding layer from encoder
                 rate=0.1):
        super(Transformer, self).__init__()

        self.pad_idx = padding_idx

        # (vocab_size, emb_dim)
        self.embedding_layer = nn.Embedding(input_vocab_size, d_model)

        self.encoder = TransformerEncoder(num_layers, d_model, num_head,
                                          intermediate_dim,
                                          input_vocab_size,
                                          src_max_len,
                                          rate)

        if shared_emb_layer is True:
            self.shared_emb_layer = self.embedding_layer
        else:
            self.shared_emb_layer = shared_emb_layer
        print(self.shared_emb_layer)
        self.decoder = TransformerDecoder(num_layers, d_model, num_head,
                                         intermediate_dim,
                                         target_vocab_size,
                                         tgt_max_len,
                                         self.shared_emb_layer,  # share embedding
                                         rate)
        self.final_layer = nn.Linear(d_model, target_vocab_size)

    def forward(self, src, tgt, training, enc_padding_mask,
                look_ahead_mask, dec_padding_mask, cuda):
        """Forward propagate for transformer.

        Args:
          src: (batch_size, src_max_len)

        """
        # Mapping
        src = self.embedding_layer(src)

        # (batch_size, inp_seq_len, d_model)
        enc_out = self.encoder(src, training, enc_padding_mask, gpu=cuda) #.cuda()

        # if cuda:
        #     enc_out = enc_out.cuda()
        # print("type of decoder input", type(tgt))
        # print("decoder input", tgt)
        # (batch_size, tgt_seq_len, d_model)
        dec_output, dec_attn = self.decoder(x=tgt,
                                            enc_output=enc_out,
                                            training=training,
                                            look_ahead_mask=look_ahead_mask,
                                            padding_mask=dec_padding_mask,
                                            gpu=cuda)

        # (batch_size, tgt_seq_len, target_vcoab_size)
        final_output = self.final_layer(dec_output)

        return final_output, dec_attn

    def sample(self, inp, max_len, temperature, training, sos_idx, eos_idx, cuda):
        """Forward propagate for transformer.

        Args:
          inp
          max_len
          temperature
          sos_idx
          eos_idx

        Returns:
          out: (batch_size, max_len)
        """
        if torch.is_tensor(inp):
            pass
        else:
            inp = torch.tensor(inp)

        #if cuda:
        #    inp = inp.cuda()

        # Gumbel-Softmax tricks
        batch_size = inp.shape[0]
        #sampled_ids = torch.zeros(batch_size, max_len).type(torch.LongTensor)

        # (batch_size, 1)
        # Create a tensor on CPU by default
        output = torch.tensor([sos_idx]*batch_size).unsqueeze(1)
        if cuda:
            output=output.cuda()
        assert output.shape[-1] == 1

        for i in range(max_len-1):
            # enc_pad_mask, combined_mask, dec_pad_mask
            enc_padding_mask, combined_mask, dec_padding_mask = create_transformer_masks(inp, output, self.pad_idx,gpu=cuda)
            # predictions.shape == (batch_size, seq_len, vocab_size)
            predictions, _ = self.forward(inp,    # (bathc_size, 1)
                                          output,  # (batch_size, 1-TO-MAXLEN)
                                          training,
                                          enc_padding_mask,
                                          combined_mask,
                                          dec_padding_mask,
                                          cuda=cuda)

            # Select the last word from the seq_len dimension
            # (batch_size, 1, vocab_size) to (batch_size, voacb_size)
            predictions = predictions[: ,-1:, :].squeeze()
            # print("preds", predictions.shape)

            # (batch_size, 1)
            #assert inp.shape[-1] = 1
            gumbel_distribution = gumbel_softmax_sample(predictions, temperature,gpu=cuda)
            # (batch_size, vocab_size)
            # print("gumbel", gumbel_distribution.shape)

            # (batch_sizes) to (bathc_size, 1)
            predicted_idx = torch.argmax(gumbel_distribution, dim=-1).unsqueeze(1)

            # print("pred idx", predicted_idx.shape)
            output = torch.cat((output, predicted_idx), 1)

            # Update along with col
            #sampled_ids[:,i] = predicted_idx.squeeze()
        #print(sampled_ids==output[:,1:])
        return output

def sample_gumbel(shape, eps=1e-20, device=None):
    """Sample from Gumbel(0, 1)"""
    if device:
        device="cuda"
    # The drawn nosie is created by default in CPU
    noise = torch.rand(shape, device=device)
    return -torch.log(-torch.log(noise+eps)+eps)

def gumbel_softmax_sample(logits, temperature, gpu):
    """Sample from Gumbel softmax distribution.
    Reference:
        1. Gumbel distribution: https://en.wikipedia.org/wiki/Gumbel_distribution
        2. Inverse Tranform Sampling: https://en.wikipedia.org/wiki/Inverse_transform_sampling
    """
    y = logits + sample_gumbel(shape=logits.shape, device=gpu)
    return nn.functional.softmax(y/temperature, dim=-1)

if __name__ == "__main__":
    transf = Transformer(2,512,8,2048,8500, 8000,10000,6000, -100, None)

    i = torch.randint(0, 200, (64,38))
    tgt_i = torch.randint(0,200, (64, 36))

    output, attn = transf(i,
                          tgt_i,
                          training=False,
                          enc_padding_mask=None,
                          look_ahead_mask=None,
                          dec_padding_mask=None)

    print(output.shape)