from copy import deepcopy

ACTIVE = '#'
INACTIVE = '.'
ROUNDS = 6

def next_state(cubes, x, y, z):
    count = 0
    for z_off in range(-1, 2):
        for x_off in range(-1, 2):
            for y_off in range(-1, 2):
                if cubes[z + z_off][x + x_off][y + y_off] == ACTIVE:
                    count += 1

    if cubes[z][x][y] == ACTIVE:
        count -= 1

    if cubes[z][x][y] == ACTIVE:
        if count in [2, 3]:
            return ACTIVE
    else:
        if count == 3:
            return ACTIVE

    return INACTIVE

    # Read input
with open('file.in', 'rt') as fin:
    lines = list(map(lambda line: line.strip(), fin.readlines()))

x_size = len(lines) + 2 * ROUNDS + 2
y_size = len(lines[0]) + 2 * ROUNDS + 2
z_size = 1 + 2 * ROUNDS + 2

# Init cubes [z][x][y]
cubes = [[[INACTIVE for _ in range(y_size)]
          for _ in range(x_size)] for _ in range(z_size)]

z = int(z_size / 2)
for x in range(len(lines)):
    for y in range(len(lines[0])):
        cubes[z][x + ROUNDS + 1][y + ROUNDS + 1] = lines[x][y]

# Compute rounds
for _ in range(ROUNDS):
    new_cubes = deepcopy(cubes)

    for z in range(1, z_size - 1):
        for x in range(1, x_size - 1):
            for y in range(1, y_size - 1):
                new_cubes[z][x][y] = next_state(cubes, x, y, z)

    cubes = new_cubes

# Compute res
res = 0
for z in range(1, z_size - 1):
    for x in range(1, x_size - 1):
        for y in range(1, y_size - 1):
            if cubes[z][x][y] == '#':
                res += 1

print(res)


import logging
import re
from datetime import datetime
from pprint import pprint
from typing import List, Optional, Union

from dictalchemy.utils import fromdict
# from opennem.core.loader import load_data
from pydantic import BaseModel
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import MultipleResultsFound

from opennem.core.loader import load_data
from opennem.core.stats.store import init_stats
from opennem.db import SessionLocal
from opennem.db.load_fixtures import load_fixtures
from opennem.db.models.opennem import Facility, Location, Revision, Station
from opennem.importer.aemo_gi import gi_import
from opennem.importer.aemo_rel import rel_import
from opennem.importer.emissions import import_emissions_csv
from opennem.importer.fueltechs import init_fueltechs
from opennem.importer.interconnectors import import_nem_interconnects
from opennem.importer.mms import mms_import
from opennem.importer.osm import init_osm
from opennem.importer.photos import import_photos_from_fixtures
from opennem.importer.registry import registry_import
from opennem.importer.rooftop import rooftop_facilities
from opennem.importer.wikidata import wikidata_join_mapping, wikidata_photos
from opennem.schema.stations import StationSet
from opennem.workers.facility_data_ranges import update_facility_seen_range

from .comparator import compare_record_differs

logger = logging.getLogger(__name__)
s = SessionLocal()

def load_revisions(stations):
    pass

def create_revision(model):
    # s = session()

    # make_transient(model)

    clone = model.asdict(exclude_pk=True)
    # clone.pop("id")
    # model.fromdict(clone)

    # pylint: disable=no-member
    clone_model = Station().fromdict(clone)

    return clone_model

def get_schema_name(record: object) -> str:
    class_name = str(record.__class__.__name__)

    class_name = re.sub("Schema", "", class_name)

    class_name = class_name.lower()

    return class_name

def revision_factory_join_field(record_instance: BaseModel) -> str:
    class_name = str(record_instance.__class__.__name__)

    FIELD_MAP = {
        "StationSchema": "station_id",
        "FacilitySchema": "facility_id",
        "LocationSchema": "location_id",
    }

    if class_name in FIELD_MAP.keys():
        return FIELD_MAP[class_name]

    raise Exception("Could not map record %s", record_instance)

def revision_factory(
    record: BaseModel,
    field_names: Union[str, List[str]],
    created_by: str,
) -> Optional[Revision]:

    if isinstance(field_names, str):
        field_names = [field_names]

    revision_data = {}

    for field_name in field_names:
        # field_type = get_schema_name(record)
        field_value = getattr(record, field_name)

        if isinstance(field_value, BaseModel):
            _value_dict = field_value.dict()

            # we only store the primary keys in associations rather
            # than the full record.
            if "id" in _value_dict:
                field_value = _value_dict["id"]

            elif "code" in _value_dict:
                field_value = _value_dict["code"]

            else:
                logger.error("Could not serialize data value %s", field_value)
                field_value = False

        if field_value is not False:
            revision_data[field_name] = field_value

    if not record.id:
        logger.error("Require a code to create a revision: %s", record)
        return None

    if len(revision_data.keys()) < 1:
        logger.error("Require data to lookup for code %s", record.code)
        return None

    column_name = revision_factory_join_field(record)

    __query = (
        s.query(Revision).filter(getattr(Revision, column_name) == record.id)
        # .filter(Revision.code == record.code)
    )

    for data_name, data_value in revision_data.items():
        if isinstance(data_value, str):
            __query = __query.filter(Revision.changes[data_name].as_string() == data_value)
        if isinstance(data_value, bool):
            __query = __query.filter(Revision.changes[data_name].as_boolean() == data_value)
        if isinstance(data_value, int):
            __query = __query.filter(Revision.changes[data_name].as_integer() == data_value)
        if isinstance(data_value, float):
            __query = __query.filter(Revision.changes[data_name].as_float() == data_value)

    revision_lookup = None

    try:
        revision_lookup = __query.one_or_none()
    except MultipleResultsFound:
        logger.info(
            "Revision exists: %s %s %s",
            record.code,
            field_name,
            field_value,
        )
        return None

    if revision_lookup:
        return None

    revision = Revision(
        created_by=created_by,
        changes=revision_data,
    )

    # s.add(revision)
    # s.commit()

    return revision

def load_revision(records, created_by):
    logger.info("Running db test")

    for station_record in records:
        station_model = s.query(Station).filter(Station.code == station_record.code).one_or_none()

        if not station_model:
            logger.info(f"New station {station_record.name} {station_record.code}")

            station_dict = station_record.dict(
                include={"code", "network_name", "name", "location"}
            )
            # pprint(station_dict)

            # pylint:disable no-member
            station_model = fromdict(Station(), station_dict)
            station_model.approved = False
            station_model.created_by = created_by

            # location
            if "location" in station_dict and station_dict["location"]:
                station_model.location = fromdict(
                    Location(), station_dict["location"], exclude=["id"]
                )

                if station_record.location.lat and station_record.location.lng:
                    station_model.location.geom = station_record.location.geom

            s.add(station_model)
            s.commit()

            station_record.id = station_model.id

            # revision = revision_factory(
            #     station_record, ["code", "name", "network_name"], created_by,
            # )

            # if revision:
            #     station_model.revisions.append(revision)

            # s.add(station_model)
            # s.commit()

        else:
            for field in ["name"]:
                if getattr(station_model, field) != getattr(station_record, field):
                    revision_factory(station_record, field, created_by)

        for facility in station_record.facilities:
            facility_model = s.query(Facility).filter(Facility.code == facility.code).first()

            if not facility_model:
                logger.info("New facility %s => %s", station_record.name, facility.code)

                facility_dict = facility.dict(
                    include={
                        "code",
                        "network",
                        # "network_id",
                        "dispatch_type",
                        "station",
                        # "station_id",
                        # "status",
                        "network_code",
                        "network_region",
                        "network_name",
                    }
                )
                # pprint(station_dict)

                # pylint:disable no-member
                facility_model = fromdict(Facility(), facility_dict)
                facility_model.network_id = facility.network.code
                facility_model.approved = False
                facility_model.created_by = created_by

                s.add(facility_model)
                s.commit()

                facility.id = facility_model.id

                # @NOTE don't create revisions for new facilities
                # revision = revision_factory(
                #     facility, ["code", "dispatch_type"], created_by
                # )

                # if revision:
                #     facility_model.revisions.append(revision)

                # s.add(facility_model)
                # s.commit()

            else:
                facility.id = facility_model.id

            for field in [
                "fueltech",
                "status",
                "capacity_registered",
            ]:
                revision = None

                if compare_record_differs(facility_model, facility, field):
                    # logger.info(
                    #     "%s and %s differ",
                    #     getattr(facility, field),
                    #     getattr(facility_model, field),
                    # )

                    revision = revision_factory(facility, field, created_by)

                if revision:
                    facility_model.revisions.append(revision)

            s.add(facility_model)

            station_model.facilities.append(facility_model)
            s.add(station_model)
            s.commit()

def db_test():
    s.query(Revision).delete()

    mms = mms_import()
    rel = rel_import()
    gi = gi_import()

    load_revision(mms, "aemo.mms.202006")
    load_revision(rel, "aemo.rel.2020006")
    load_revision(gi, "aemo.gi.202006")

def registry_init() -> None:
    registry = registry_import()
    session = SessionLocal()

    for station in registry:
        station_dict = station.dict(exclude={"id"})

        station_model = session.query(Station).filter_by(code=station.code).one_or_none()

        if not station_model:
            # pylint:disable no-member
            station_model = fromdict(Station(), station_dict)
            station_model.approved = True
            station_model.approved_at = datetime.now()
            station_model.approved_by = "opennem.registry"
            station_model.created_by = "opennem.registry"

        # location
        station_model.location = fromdict(
            Location(),
            station_dict["location"],
            exclude=["id"],
        )

        if station.location.lat and station.location.lng:
            station_model.location.geom = "SRID=4326;POINT({} {})".format(
                station.location.lng, station.location.lat
            )

        session.add(station_model)
        session.commit()

        for fac in station.facilities:
            f = (
                session.query(Facility)
                .filter_by(code=fac.code)
                .filter_by(network_id=fac.network.code)
                .one_or_none()
            )

            if not f:
                logger.info("Added facility {} {}".format(fac.code, fac.network.code))
                f = Facility(
                    **fac.dict(
                        exclude={
                            "id",
                            "fueltech",
                            "status",
                            "network",
                            "revision_ids",
                            "scada_power",
                        }
                    )
                )
                f.approved_by = "opennem.registry"
                f.created_by = "opennem.registry"
                f.approved_at = datetime.now()
                f.created_at = datetime.now()

            f.network_id = fac.network.code

            if fac.fueltech:
                f.fueltech_id = fac.fueltech.code

            if fac.network.code:
                f.network_code = fac.network.code

            f.status_id = fac.status.code

            f.approved = True

            if station_model.id:
                f.station_id = station_model.id
            else:
                station_model.facilities.append(f)
                session.add(station_model)

            session.add(f)

        try:
            session.commit()
        except IntegrityError as e:
            logger.error(e)

def opennem_init() -> None:
    session = SessionLocal()
    station_data = load_data("opennem_stations.json", from_project=True)
    stations = StationSet()

    for s in station_data:
        stations.add_dict(s)

    for station in stations:
        add_or_update: str = "Updating"
        station_model = session.query(Station).filter_by(code=station.code).one_or_none()

        if not station_model:
            add_or_update = "Adding"
            station_model = Station(code=station.code)
            station_model.approved = True
            station_model.approved_at = datetime.now()
            station_model.approved_by = "opennem.init"
            station_model.created_by = "opennem.init"

        logger.debug("{} station: {}".format(add_or_update, station.code))

        station_model.description = station.description
        station_model.name = station.name
        station_model.network_name = station.network_name

        if not station_model.location:
            station_model.location = Location()

        if station.location:
            station_model.location.locality = station.location.locality
            station_model.location.state = station.location.state
            station_model.location.postcode = station.location.postcode
            station_model.location.country = station.location.country

        if station.location.lat and station.location.lng:
            station_model.location.geom = "SRID=4326;POINT({} {})".format(
                station.location.lng, station.location.lat
            )

        for fac in station.facilities:
            facility_model = (
                session.query(Facility)
                .filter_by(code=fac.code)
                .filter_by(network_id=fac.network.code)
                .one_or_none()
            )

            if not facility_model:
                facility_model = Facility(code=fac.code, network_id=fac.network.code)

            facility_model.network_region = fac.network_region
            facility_model.fueltech_id = fac.fueltech.code
            facility_model.status_id = fac.status.code
            facility_model.dispatch_type = fac.dispatch_type
            facility_model.capacity_registered = fac.capacity_registered
            facility_model.registered = fac.registered
            facility_model.unit_id = fac.unit_id
            facility_model.unit_number = fac.unit_number
            facility_model.unit_alias = fac.unit_alias
            facility_model.unit_capacity = fac.unit_capacity
            facility_model.emissions_factor_co2 = fac.emissions_factor_co2
            facility_model.approved = True
            facility_model.approved_by = "opennem.importer"

            facility_model.created_by = "opennem.init"
            facility_model.approved_by = "opennem.init"

            session.add(facility_model)
            station_model.facilities.append(facility_model)
            logger.debug(" => Added facility {}".format(fac.code))

        session.add(station_model)
        session.commit()

def test_revisions() -> None:
    registry = registry_import()
    # mms = mms_import()

    k = registry.get_code("KWINANA")

    station = Station(
        code=k.code,
        name=k.name,
        network_name=k.network_name,
    )
    s.add(station)
    s.commit()

    # pylint: disable=no-member
    station_clone = station.asdict()
    station_clone.pop("id")

    pprint(station_clone)

    station = create_revision(station)

    station.location = Location(
        address1=k.location.address1,
        address2=k.location.address2,
        locality=k.location.locality,
        state=k.location.state,
        postcode=k.location.postcode,
    )

    s.add(station)
    s.commit()

    station = create_revision(station)

    station.name = "Kwinana Edited"

    s.add(station)
    s.commit()

def import_facilities() -> None:
    opennem_init()
    logger.info("Opennem stations imported")

def init() -> None:
    """
    These are all the init steps required after a db has been initialized
    """
    load_fixtures()
    logger.info("Fixtures loaded")

    registry_init()
    logger.info("Registry initialized")

    opennem_init()
    logger.info("Opennem facilities initialized")

    rooftop_facilities()
    logger.info("Rooftop stations initialized")

    import_emissions_csv()
    logger.info("Emission data initialized")

    init_fueltechs()
    logger.info("Initialized fueltechs")

    import_nem_interconnects()
    logger.info("Initialized interconnectors")

    wikidata_join_mapping()
    logger.info("Imported wikidata for stations")

    wikidata_photos()
    logger.info("Imported wikidata photos")

    import_photos_from_fixtures()
    logger.info("Imported photos from wikidata")

    init_osm()
    logger.info("Initialized osm")

    init_stats()
    logger.info("Stats data initialized")

    update_facility_seen_range()
    logger.info("Ran seen range")


import spidev, time

spi = spidev.SpiDev()
spi.open(0,0)
def analog_read(channel):
 r = spi.xfer2([1, (8 + channel) << 4, 0])
 adc_out = ((r[1]&3) << 8) + r[2]
 return adc_out
while True:
 reading = analog_read(0)
 print(reading)
 time.sleep(0.2)
 if reading >470 :
         print('alcohol detected...')
         time.sleep(0.2)
 else :
         print('not...')
         time.sleep(0.2)
 '''
 time.sleep(1)
 voltage = reading * 3.3/1024
 print(voltage)

 temp_c = voltage * 100
 temp_f = temp_c * 9.0 / 5.0 + 32
 print("Temp C=%f\t\tTemp f=%f" % (temp_c, temp_f))
 time.sleep(1)
'''


# -*- coding: utf-8 -*-
# Hello! Good luck!

##############
#   TASK 1   #
##############
import json
import math
from datetime import *
class Ticket(object):
    def __init__(
        self,
        ticket_id,    # the unique identifier of the ticket;
        event_date,   # the date of the event;
        event_time,   # the time when event occurs;
        event_name,   # the event name -> obvious one :);
        client,       # the INSTANCE of the client;
        room_number,  # the room number in which event happens;
    ):
        self.ticket_id = ticket_id
        self.event_date = event_date
        self.event_time = event_time
        self.event_name = event_name
        self.client = client
        self.room_number = room_number
    def get_ticket_data(self):
        j = self.__dict__
        j['client'] = self.client.__dict__
        return json.dumps(j)
# SUBTASK 1
# Define the Client class with attributes:
# first_name, last_name, birth_date, sex

class Client(object):
    def __init__(self, first_name, last_name, birth_date, sex):
        self.first_name = first_name
        self.last_name = last_name
        self.birth_date = birth_date
        self.sex = sex
    def can_watch_pegi(self, movie):
        movie_age = movie.pegi
        client_age = date.today()
        birth_date = self.birth_date.split('-')
        birth_date = date(int(birth_date[0]), int(birth_date[1]), int(birth_date[2]))
        client_age = math.floor((client_age - birth_date).days / 365)
        if client_age >= movie_age: return True
        return False
    def get_ticket_ids(self):
        return SomeInterface.get_tickets_ids(self)
# SUBTASK 2
# define a method on Ticket: get_ticket_data, which returns dumped JSON object
# with all data on ticket, and all data on client;
# EXAMPLE:
#     {
#         "event_time": "20:00",
#         "event_name": "The Killer",
#         "room_number": "Sala 7",
#         "client":
#             {
#                 "birth_date": "1985-01-23",
#                 "first_name": "Romuald",
#                 "last_name": "Lechon",
#                 "sex": "MALE"
#             },
#         "ticket_id": "12345678",
#         "event_date": "2015-09-10"
#     }
c = Client('Romuald', 'Lechon', '1985-01-23', 'MALE')
t = Ticket("12345678", "2015-09-10", "20:00", "The Killer", c, "Sala 7")
print(t.get_ticket_data())
# SUBTASK 3
# Write a method (name: can_watch_pegi ;)) on client which check if he is able
# to watch the movie - according to PEGI
# (https://pl.wikipedia.org/wiki/Pan_European_Game_Information)
# (yep, I know - PEGI is for games, but who cares) - assume following age
# marks: 3, 7, 12, 16, 18;
# Method should return True or False -> True means that client is older than
# restriction and can watch the movie;
class Movie(object):
    def __init__(self, title, pegi):
        self.title = title
        self.pegi = pegi
m = Movie('Big Short', 16)
print(c.can_watch_pegi(m))
# SUBTASK 4
# Propose a structure where client instance will have some interface to get the
# all ticket ids - whenever added, which returns all ticket ids;
# EXAMPLE (it's just an example, remember - "some interface" - be creative):
# c = Client(...)
# t = Ticket('12345678', client=c, ...)
# t = Ticket('8765431', client=c, ...)
# t = Ticket('12343141', client=c, ...)
# c.get_ticket_ids()
# OUTPUT: ['12345678', '8765431', '12343141']

class SomeInterface(object):
    tickets_list = []
    @staticmethod
    def get_tickets_ids(client):
        temp = []
        for element in SomeInterface.tickets_list:
            if element.client is client:
                temp.append(element.ticket_id)
        return temp
t = Ticket("12345678", "2015-09-10", "20:00", "The Killer", c, "Sala 7")
SomeInterface.tickets_list.append(t)
t = Ticket("8765431", "2015-09-10", "20:00", "The Killer", c, "Sala 7")
SomeInterface.tickets_list.append(t)
t = Ticket("12343141", "2015-09-10", "20:00", "The Killer", c, "Sala 7")
SomeInterface.tickets_list.append(t)
print(c.get_ticket_ids())

##############
#   TASK 2   #
##############

# SUBTASK 1
# do it in pythonic way;
# it just adds the index to the list element on this index
# and return a new list;
# TIP: shorter is better;

def iterate_over_list(some_list):
    i = 0
    new_list = []
    for element in some_list:
        n_element = int(element) + i
        new_list.append(n_element)
        i = i + 1

    return new_list

print(iterate_over_list([1, 2, 3]))
def iterate_over_list(some_list):
    new_list = [ int(some_list[x]) + x for x in range(0, len(some_list))]
    return new_list

print(iterate_over_list([1, 2, 3]))

# SUBTASK 2
# and how handle this?
# in such case - method should repeat string index times;
# input -> ['a', 'b', 'c']; output: ['', 'b', 'cc'];
def iterate_over_list(some_list):
    new_list = [ some_list[x] * x for x in range(0, len(some_list))]
    return new_list
try:
    iterate_over_list(['a', 'b', 'c'])
except ValueError:
    print('Ups. We have an error here.')
print(iterate_over_list(['a', 'b', 'c']))
##############
#   TASK 3   #
##############

# Imagine that you have a database table, defined as follows:
# TABLE poi
#   COLUMN id SERIAL (primary_key)
#   COLUMN name TEXT
#   COLUMN location_city TEXT
#   COLUMN location_country TEXT
#   COLUMN latitude NUMERIC
#   COLUMN longitude NUMERIC

# Write an SQL query which returns the aggregated poi by location_city column;
# We want to know how many poi (Pint of interest is in London)
# So we want to get, something like this:
# city|pois_count
# London|123
# Warsaw|541
# Moscow|32
"""
select location_city, count(*) from poi
group by location_city

"""

# SUBTASK 1
# Assume that above query will be done quite frequently, additionally
# there will be also a query which will be filtering (WHERE) on location_city.
# What can we do to improve database performance?
"""
create or replace view widok as select location_city, count(*) from poi
group by location_city
"""


# -*- coding: utf-8 -*-
import mimetypes
from typing import Optional, Any

from django.core.files.storage import DefaultStorage
from django.http import Http404, HttpResponse
from rest_framework import status
from rest_framework.exceptions import ValidationError
from rest_framework.parsers import (
    FileUploadParser, FormParser,
    MultiPartParser
)
from rest_framework.permissions import AllowAny
from rest_framework.response import Response
from rest_framework.viewsets import ViewSet

class StorageWrapper:
    def __init__(self):
        self.storage = DefaultStorage()

    def save(self, filename: str, file_like):
        return self.storage.save(filename, file_like)

    def list(self, ext: str = None):
        _, files = self.storage.listdir("/")
        if ext:
            files = [f for f in files if f.endswith(ext)]
        return files

    def exists(self, filename: str):
        return self.storage.exists(filename)

    def find(self, filename: str) -> Optional[str]:
        for file_ in self.list():
            if file_ == filename:
                return file_
        return None

    def write_to(self, filename: str, callback: Any):
        f = self.storage.open(filename)
        callback(f.read())
        f.close()

    def remove(self, filename: str):
        self.storage.delete(filename)

class FilesViewSet(ViewSet):
    permission_classes = (AllowAny,)
    parser_classes = (MultiPartParser, FormParser, FileUploadParser)
    lookup_field = 'filename'
    lookup_value_regex = r'[\w\s.\-_]+'

    def __init__(self, *args, **kwargs):
        self.client = StorageWrapper()
        super().__init__(*args, **kwargs)

    def finalize_response(self, request, response, *args, **kwargs):
        return super().finalize_response(request, response, *args, **kwargs)

    def list(self, _):
        ext = self.request.query_params.get('ext', None)
        files = [{'name': filename} for filename in self.client.list(ext=ext)]
        return Response({'results': files})

    def create(self, request):
        files = request.FILES
        common_files = set(files.keys()) & set(self.client.list())
        if common_files:
            raise ValidationError(
                'File with name(s) `{}` already exists.'.format(
                    ','.join(common_files)
                ), code='unique'
            )
        created_files = []
        for filename, in_mem_file in request.FILES.items():
            self.client.save(filename, in_mem_file)
            created_files.append({'name': filename})
        return Response(created_files, status=status.HTTP_201_CREATED)

    def retrieve(self, request, filename=None):
        file_ = self.client.find(filename)
        if not file_:
            raise Http404
        content_type, encoding = mimetypes.guess_type(filename)
        if content_type is None:
            content_type = 'application/octet-stream'
        response = HttpResponse(content_type=content_type)
        response['Content-Disposition'] = (
            'attachment; filename="{}"'.format(filename)
        )
        self.client.write_to(filename, response.write)
        return response

    def update(self, request, filename=None):
        if not self.client.exists(filename):
            raise Http404

        self.client.remove(filename)
        return Response({'status': 'ok'})

    def destroy(self, request, filename=None):
        if not self.client.exists(filename):
            raise Http404
        self.client.remove(filename)
        return Response({'status': 'ok'}, status=status.HTTP_204_NO_CONTENT)


## This file will scrape goodreads popular book lists to fill out a list of ~5,000 books to scrape
## This will be recorded in a .txt file named "book_list.txt"
## The structure of the file will be:

##    BOOK ID  |  BOOK TITLE  |  BOOK AUTHOR  |   BOOK HREF LINK |  BOOK LIST NAME |  LIST HREF LINK | COLLECTED | DATE/TIME COLLECTED

#-----------------------------------------------------------------------------#
#                                  IMPORTS                                    #
#-----------------------------------------------------------------------------#

import csv
import time
from bs4 import BeautifulSoup
from bs4 import SoupStrainer
from selenium import webdriver
from collections import Counter
import datetime

#-----------------------------------------------------------------------------#
#                               PRE-DEFINES                                   #
#-----------------------------------------------------------------------------#

data_fields = ['Book ID', 'Book Title', 'Book Author', 'Book Link',
               'Book List Name', 'Book List Link', 'Collected', 'Date/Time Collected']

# Let's create a booklist from the most popular books:
year_list = [2010+i for i in range(1,11)]
genre_list = ['fiction-books',
              'mystery-thriller-books',
              'historical-fiction-books',
              'fantasy-books',
              'romance-books',
              'science-fiction-books',
              'horror-books',
              'humor-books',
              'nonfiction-books',
              'memoir-autobiography-books',
              'history-biography-books',
              'science-technology-books',
              # 'food-cookbooks', # Who tries to find more cookbooks...
              'graphic-novels-comics',
              'poetry-books',
              'debut-novel',
              'young-adult-fiction-books',
              'young-adult-fantasy-books',
              # all of the genres below are only for a few years...
              'debut-goodreads-author',
              'of-the-best']
choiceawards_list = []

for curr_year in reversed(year_list):
    for curr_genre in genre_list:
        choiceawards_list.append('https://www.goodreads.com/choiceawards/best-' + curr_genre + '-' + str(curr_year))

master_ID_check = []

#-----------------------------------------------------------------------------#
#                            MODULE FUNCTIONS                                 #
#-----------------------------------------------------------------------------#

# From thread_spinner.py
def get_page(fnc_url_link):
    ##Below is the old code that WORKED
    driver.get(fnc_url_link)

    soup_toreturn = BeautifulSoup(driver.page_source, 'lxml')

    time.sleep(1)

    return soup_toreturn

def extract_book_ids(in_soup):
    awarded_books = in_soup.select('.js-tooltipTrigger.tooltipTrigger')
    out_ids = [i.attrs['data-resource-id'] for i in awarded_books]

    out_hrefs = []
    out_title = []
    out_authors = []
    for cur_content in awarded_books:
        temp_content = cur_content.contents[1]

        out_hrefs.append( 'https://www.goodreads.com' + temp_content.attrs['href'].split('?')[0] )
        shared_str = temp_content.contents[0].attrs['alt'].split(' by ')
        out_title.append( shared_str[0] )
        out_authors.append(shared_str[1])
    out_list = []

    for i in range(len(out_ids)):
        # Check that the ID has not been encountered
        if out_ids[i] not in master_ID_check:
            master_ID_check.append(out_ids[i])
            out_list.append((out_ids[i], out_title[i], out_authors[i], out_hrefs[i]))
        else:
            print('Skipping ID {0}, its already in the list!'.format(out_ids[i]))

    return out_list

def create_log_file(fileName):
    # writing to csv file
    with open(fileName, 'w') as csvfile:
        # creating a csv writer object
        csvwriter = csv.writer(csvfile)

        # writing the fields
        csvwriter.writerow(data_fields)

def add_list_to_txt(list_link, tuple_list):
    list_name = list_link.split('/')[-1]

    with open('book_list.csv', 'a') as csvfile: #open in append mode
        csvwriter = csv.writer(csvfile)
        for curr_tuple in tuple_list:
            row_to_write = [curr_tuple[0], curr_tuple[1], curr_tuple[2], curr_tuple[3],
                            list_name, list_link, '0', 'N/A']
            csvwriter.writerow( row_to_write )

def exctract_csv_to_list():
    with open('book_list.csv', 'r') as csvfile: #open in read mode
        csvreader = csv.reader(csvfile)
        out_list = []

        for row in csvreader:
            out_list.append(row)

    return out_list

def get_latest_id():
    with open('book_list.csv', 'r') as csvfile: #open in read mode
        csvreader = csv.reader(csvfile)

        # Assign vars to ERRORCODES so we know when the list is finished
        next_indx_toCollect = 'ERRORCODE'
        next_ID_toCollect = 'ERRORCODE'

        for i, row in enumerate(csvreader):
            if row[6] == '0':
                next_indx_toCollect = i
                next_ID_toCollect = row[0]
                break

    return next_indx_toCollect, next_ID_toCollect

def write_back_list( list_to_write ):
    # Overwrite the csv file
    with open('book_list.csv', 'w') as csvfile:
        # creating a csv writer object
        csvwriter = csv.writer(csvfile)

        # writing the fields
        for curr_row in list_to_write:
            csvwriter.writerow( curr_row )

def mark_line_as_finished(index_toMark):
    with open("book_list.csv", "r+") as f:
        csvreader = csv.reader(f)

        contents = []
        # gives you a list of the lines
        for row in csvreader:
            contents.append(row)

        # delete the old line and insert the new one
        temp_list = contents.pop(index_toMark)

        # Now modify the list to change the 'collected' and 'dates' columns
        temp_list[6] = '1'
        temp_list[7] = str( datetime.datetime.now() )

        contents.insert(index_toMark, temp_list)
        # # join all lines and write it back
        # contents = "".join(contents)

        write_back_list( contents )
        pass

# -----------------------------------------------------------------------------#
#                                   MAIN                                       #
# -----------------------------------------------------------------------------#

# Create proper main structure
if __name__ == "__main__":

    chrome_options = webdriver.chrome.options.Options()
    driver = webdriver.Chrome(options=chrome_options)

    # if True:
    create_log_file('book_list.csv')
    print(choiceawards_list)

    for curr_link in choiceawards_list:

        # currSoup = get_page('https://www.goodreads.com/choiceawards/best-young-adult-fantasy-books-2020')
        currSoup = get_page(curr_link)
        curr_tuple_list = extract_book_ids(currSoup)

        # Add to the text file the collected book info with the respective list
        add_list_to_txt(curr_link, curr_tuple_list)

    master_list = exctract_csv_to_list()

    pass


"""
Given an array of integers nums and an integer target, return indices of the two numbers
such that they add up to target.

You may assume that each input would have exactly one solution, and you may not use the same element twice.

You can return the answer in any order.

Example 1:

Input: nums = [2,7,11,15], target = 9
Output: [0,1]
Output: Because nums[0] + nums[1] == 9, we return [0, 1].
"""

def twoSum(nums, target):
    i = 0
    j = 1

    while j < len(nums):
        if (nums[i] + nums[j] == target):
            print(range(nums[i]) + range(nums[j]))

print(twoSum([2, 7, 11, 15], 9))


# -*- coding:UTF-8 -*-
class Solution(object):
    def numEnclaves(self, A):
        m = len(A)
        n = len(A[0])
        if not A:
            return 0
        dirs = [[1, 0], [-1, 0], [0, 1], [0, -1]]

        def dfs(A, i, j):
            A[i][j] = 0
            for dir in dirs:
                x, y = i + dir[0], j + dir[1]
                if (0 <= x < m and 0 <= y < n and A[x][y] == 1):
                    dfs(A, x, y)

        for i in range(m):
            if A[i][0] == 1:
                dfs(A, i, 0)
            if A[i][n - 1] == 1:
                dfs(A, i, n - 1)
        # for i in A:
        #     print(i)
        for j in range(n):
            if A[0][j] == 1:
                dfs(A, 0, j)
            if A[m - 1][j] == 1:
                dfs(A, m - 1, j)
        # for i in A:
        #     print(i)
        count = 0
        for i in range(m):
            for j in range(n):
                if A[i][j] == 1:
                    count += 1
        return count

if __name__ == '__main__':
    # 返回网格中无法在任意次数的移动中离开网格边界的陆地单元格的数量
    A = [[0, 0, 0, 0], [1, 0, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]]
    print(Solution().numEnclaves(A))


import shutil
import xml.etree.ElementTree as ET
import pandas as pd
import glob
import numpy as np
import os
from PIL import Image

# class picture():
#     # location = >array yolo format
#     def __init__(self,name,x,y,width,height):
#         self.name = name
#         self.x = x
#         self.y = y
#         self.width = width
#         self.height = height

#  전체 파일 통합
def merge(dir_path, move_path):
    path, dirs, files = next(os.walk(dir_path))
    file_count = 0
    for i in dirs:
        directory = os.path.join(dir_path, i)
        path, dirs, files = next(os.walk(directory))
        file_count += len(files)
    print(file_count)
    fname = ['mushroom{}.jpg'.format(i) for i in range(file_count)]
    f = 0
    for files_path in os.listdir(dir_path):
        files_path = os.path.join(dir_path, files_path)
        files_path = files_path + '/*.jpg'
        for file_path in glob.glob(files_path):
            dst = os.path.join(move_path, fname[f])
            f = f + 1
            print(dst)
            shutil.copyfile(file_path, dst)

# txt -> yolo csv
def get_point(dir_path):

    picture_name = []
    picture_width = []
    picture_height = []
    mushroom_xmins = []
    mushroom_xmaxs = []
    mushroom_ymins = []
    mushroom_ymaxs = []

    width = 0
    height =0
    dirtxt_path = dir_path + '/*.txt'

    for filename in glob.glob(dirtxt_path):
        fn = open(filename)
        image_name = filename.split('.')[0]
        image_name = image_name.split('\\')[-1]
        image_name = image_name+'.jpg'
        image = Image.open(os.path.join(dir_path,image_name))
        width, height = image.size
        lines = fn.readlines()
        for line in lines:
            line = line.replace('\n', '')
            location = line.split(' ')
            picture_name.append(image_name)
            picture_width.append(width)
            picture_height.append(height)

            centerx= int(float(location[1])*width)
            centery= int(float(location[2])*height)
            mushroom_width = int(float(location[3])*width/2)
            mushroom_height = int(float(location[4])*height/2)

            mushroom_xmin = centerx - mushroom_width
            mushroom_xmax = centerx + mushroom_width
            mushroom_ymin = centery - mushroom_height
            mushroom_ymax = centery + mushroom_height

            mushroom_xmins.append(mushroom_xmin)
            mushroom_xmaxs.append(mushroom_xmax)
            mushroom_ymins.append(mushroom_ymin)
            mushroom_ymaxs.append(mushroom_ymax)
    result = pd.DataFrame(
        {'filename': picture_name, 'width': picture_width, 'height': picture_height,'class':'mushroom',
         'xmin': mushroom_xmins, 'ymin': mushroom_ymins,'xmax':mushroom_xmaxs,'ymax':mushroom_ymaxs})
    result.to_csv('./third_csv_1.csv')

# xml -> coco csv
def get_xml(dir_path, dir_num):
    num = [0, 353, 750, 1073]
    num_sum = np.sum(num[0:dir_num], dtype=np.int32)
    mushroom_name = []
    mushroom_x = []
    mushroom_y = []
    mushroom_width = []
    mushroom_height = []

    dir_path = dir_path + '/*.xml'

    for filename in glob.glob(dir_path):

        tree = ET.parse(filename)
        tree = tree.getroot()
        size = tree.find('size')
        whole_width = int(size.find('width').text)
        whole_height = int(size.find('height').text)
        filename = filename.split('.', 1)
        filename = filename[0].split('\\')[1]
        filename = int(filename.split('_')[0]) + num_sum
        filename = 'mushroom' + str(filename)
        for line in tree.findall('object'):
            line = line.find('bndbox')
            min_width = int(line.find('xmin').text)
            min_height = int(line.find('ymin').text)
            max_width = int(line.find('xmax').text)
            max_height = int(line.find('ymax').text)

            #
            mushroom_name.append(filename)
            mushroom_x.append(min_width/whole_width)
            mushroom_y.append(min_height/whole_height)
            mushroom_width.append(max_width/whole_width)
            mushroom_height.append(max_height/whole_height)
    # mushrooms = np.array(mushroom_name,mushroom_x,mushroom_y,mushroom_width,mushroom_height)
    return pd.DataFrame({'name': mushroom_name, 'min_x': mushroom_x, 'min_y': mushroom_y, 'max_x': mushroom_width,
                         'max_y': mushroom_height})

def get_pictures(names, file_path):
    for value in names:
        print(value)

# yolo csv -> coco csv
def convert_coco(file_path, picture_path):
    data = pd.read_csv(file_path)
    names = []
    minx = []
    miny = []
    maxx = []
    maxy = []
    for index, row in data.iterrows():
        name = row['name'] + '.jpg'
        path = os.path.join(picture_path, name)
        image = Image.open(path)
        width, height = image.size
        # check int values
        min_x =  (row['x'] - row['width'] / 2)
        max_x =  (row['x'] + row['width'] / 2)
        min_y =  (row['y'] - row['height'] / 2)
        max_y =  (row['y'] + row['height'] / 2)
        minx.append(min_x)
        miny.append(min_y)
        maxx.append(max_x)
        maxy.append(max_y)
        names.append(row['name'])
    return pd.DataFrame({'file': names, 'min_x': minx, 'min_y': miny, 'max_x': maxx, 'max_y': maxy})

# txt file to coco csv file
def merge_sample(dir_path,picture_path):

    mushroom_name = []
    mushroom_minwidth = []
    mushroom_maxwidth = []
    mushroom_minheight = []
    mushroom_maxheight = []
    dir_path = dir_path + '/*.txt'
    sequence = sorted(glob.glob(dir_path))
    file_count = len(sequence)
    fname = ['mushroom{}.jpg'.format(i) for i in range(file_count)]
    for file,picture_name in zip(sequence,fname):
        fn = open(file)

        filename = file.split('.', 1)
        filename = filename[0].split('\\')[1]
        filename = os.path.join(picture_path,filename+'.jpg')
        dst = os.path.join('D:/merge1',picture_name)
        shutil.copy(filename, dst)
        lines = fn.readlines()
        for line in lines:
            line = line.replace('\n', '')
            location = line.split(' ')
            mushroom_name.append(picture_name)
            mushroom_minwidth.append(float(location[1])-float(location[3])/2)
            mushroom_maxwidth.append(float(location[1])+float(location[3])/2)
            mushroom_minheight.append(float(location[2])-float(location[4])/2)
            mushroom_maxheight.append(float(location[2])+float(location[4])/2)
    record = pd.DataFrame({'name': mushroom_name,'min_x':mushroom_minwidth,'max_x':mushroom_maxwidth,'min_y':mushroom_minheight,'max_y':mushroom_maxheight})
    record.to_csv('D:/example.csv')

def xml_to_csv(path):
    """Iterates through all .xml files (generated by labelImg) in a given directory and combines
    them in a single Pandas dataframe.

    Parameters:
    ----------
    path : str
        The path containing the .xml files
    Returns
    -------
    Pandas DataFrame
        The produced dataframe
    """

    xml_list = []
    for xml_file in glob.glob(path + '/*.xml'):
        tree = ET.parse(xml_file)
        root = tree.getroot()
        for member in root.findall('object'):
            value = (root.find('filename').text,
                     int(root.find('size')[0].text),
                     int(root.find('size')[1].text),
                     member[0].text,
                     int(member[4][0].text),
                     int(member[4][1].text),
                     int(member[4][2].text),
                     int(member[4][3].text)
                     )
            xml_list.append(value)
    column_name = ['filename', 'width', 'height',
                   'class', 'xmin', 'ymin', 'xmax', 'ymax']
    xml_df = pd.DataFrame(xml_list, columns=column_name)
    xml_df.to_csv('./third_csv_2.csv')

# get_point('D:third')
xml_to_csv('D:third')

# merge_sample('D:/third','D:/Mushrooms/Boletus')
# name_array = get_point('D:/third', 3)
# name_array.to_csv('D:/third.csv')
# get_pictures(name_array)
# merge('D:\Mushrooms','D:\merge')
# name = get_xml('D:/third',3)
# name.to_csv('D:/third_xml1.csv')
# coco = convert_coco('D:/third.csv', 'D:/merge')
# coco.to_csv('D:/third_xml.csv')


import os
import sys
import click
import logging
from .config import Config

log = logging.getLogger("a2ml")

CONTEXT_SETTINGS = dict(auto_envvar_prefix='A2ML')
PROVIDERS = ['auger', 'google', 'azure']
PROVIDERS_META = '|'.join(PROVIDERS)

class Context(object):

    def __init__(self, name='config', path=None, debug=False):
        super(Context, self).__init__()

        self.config = Config(name=name, path=path)
        self.name = self.config.name

        if len(self.name) > 0:
            self.name = "{:<9}".format('[%s]' % self.name)
        self.debug = self.config.get('debug', debug)
        self.runs_on_server = False

    def get_providers(self, provider = None):
        if provider:
            providers = provider
        else:
            providers = self.config.get('providers', [])

        if isinstance(providers, (str,)):
            providers = [p.strip() for p in providers.split(',')]

        if isinstance(providers, (list,)):
            for p in providers:
                if p not in PROVIDERS:
                    raise Exception('Provider %s is not supported.' % p)
            return providers

        raise Exception('Expecting list of providers in config.yaml\providers')

    def copy(self, name):
        new = Context(name, self.config.path, self.debug)
        new.runs_on_server = self.runs_on_server
        #new.config = Config(name)
        return new

    def log(self, msg, *args, **kwargs):
        log.info('%s%s' %(self.name, msg), *args, **kwargs)

    def debug(self, msg, *args, **kwargs):
        log.debug('%s%s' %(self.name, msg), *args, **kwargs)

    def error(self, msg, *args, **kwargs):
        log.error('%s%s' %(self.name, msg), *args, **kwargs)

    @staticmethod
    def setup_logger(format='%(asctime)s %(name)s | %(message)s'):
        logging.basicConfig(
            stream=sys.stdout,
            datefmt='%H:%M:%S',
            format=format,
            level=logging.INFO)

pass_context = click.make_pass_decorator(Context, ensure=True)


def maxarea(height):
    ma = 0

    for i in range(len(height)):
        for j in range(1 + i, len(height)):
            area = (j - i) * min(height[i], height[j])
            # print(area)

            ma = max(ma, area)
    return ma

# call function
height = [1, 8, 6, 2, 5, 4, 8, 3, 7]
y = maxarea(height)
print(y)


#!/usr/bin/env python3
import argparse

class Day7:
    """ Day 7: Handy Haversacks """
    _BAG = "shiny gold"

    def __init__(self, input_file):
        self._bag_rules = dict()
        self.process(input_file)

    def find_bag(self, color: str) -> int:
        for bag in self._bag_rules[color].keys():
            if bag == self._BAG:
                return 1
            elif bag == "none":
                pass
            else:
                if self.find_bag(bag):
                    return 1
        return 0

    def count_bag(self, color: str) -> int:
        count = 0
        for _, bag in enumerate(self._bag_rules[color].items()):
            bag_c, bag_n = bag
            if bag_c == "none":
                return 0
            else:
                count += self.count_bag(bag_c) * bag_n + bag_n
        return count

    def process(self, input_file):
        with open(input_file, "r") as input:
            for line in input:
                line = line.rstrip()
                rule = line.split(" bags contain ")
                main_bag = rule[0]
                contain = rule[1]\
                    .replace("bags", "bag") \
                    .replace(", ", "") \
                    .replace(".", "") \
                    .split(" bag")
                self._bag_rules[main_bag] = dict()
                for cont in contain:
                    if len(cont) == 0:
                        break
                    elif cont == "no other":
                        self._bag_rules[main_bag]["none"] = 0
                    else:
                        nb, bag = cont.split(" ", 1)
                        self._bag_rules[main_bag][bag] = int(nb)
        bag_nb = 0
        for bag in self._bag_rules.keys():
            if bag == self._BAG:
                continue
            bag_nb += self.find_bag(bag)
        print(f"Part 1: {bag_nb}")
        print(f"Part 2: {self.count_bag(self._BAG)}")

def parse_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('input', help="File containing day input data")
    return parser.parse_args()

if __name__ == "__main__":
    args = parse_arguments()
    Day7(args.input)


from parabase import Parameter, ConfigBase, ParameterDisplay, ParameterInput, Sep
import sys  # sys._getframe().f_code.co_name = {File}_{Class}_{digit}_{function}_{digit}
import itertools

sup_sub_template = "{word}{sup}{sub}"

s_n_crb_ssb = sup_sub_template.format(word="N", sub="CRB", sup="SSB")
s_freq_ssb_gscn = sup_sub_template.format(word="FREQ", sub="SSB", sup="GSCN")
s_scs_crb_ssb = sup_sub_template.format(word="SCS", sub="CRB", sup="SSB")

_scs_dict = {15: "15kHz", 30: "30kHz", 60: "60kHz", 120: "120kHz", 240: "240kHz"}

#############################################################################
# start add parameter
#############################################################################
t = Sep()

t = ParameterDisplay('scs_ssb')
t.default = "120kHz"
t.dict = _scs_dict
t.spec  = ''

t = ParameterDisplay('subCarrierSpacingCommon')
t.default = "120kHz"
t.dict = _scs_dict
t.spec  = 'MIB -> subCarrierSpacingCommon'

t = Sep()

t = Parameter('nrbs')
t.range = [24, 48]
t.spec = "number of RBs"

t = Parameter('offset')
t.range = [-21, -20, 24, 48]
t.spec = "Offset (RBs)"

t = Sep()

t = Parameter('kssb')
t.desc = "kSSB"
t.range = list(range(12))
t.spec = 'MIB -> ssb-SubcarrierOffset'

t = ParameterDisplay('scs_kssb')
t.default = "120kHz"
t.dict = _scs_dict
t.spec  = ''

#############################################################################
# end add parameter
#############################################################################

class Config(ConfigBase):
    #############################################################################
    # start add consistency check
    #############################################################################
    def nrbs_change(self, event=None):
        m = self.mvalue
        if m == 24:
            self.set_range('offset', [-21, -20, 24])
        if m == 48:
            self.set_range('offset', [-21, -20, 48])
        self.offset_change()

    def offset_change(self, event=None):
        m = self.mvalue
        if m == -20:
            self.set_range('kssb', [0])
        elif m == -21:
            self.set_range('kssb', range(1, 12))
        else:
            self.set_range('kssb', range(12))


"""
Реализуйте рекурсивную функцию нарезания прямоугольника с заданными пользователем сторонами a и b
на квадраты с наибольшей возможной на каждом этапе стороной.
Выведите длины ребер получаемых квадратов и кол-во полученных квадратов.
"""

a = int(input())
b = int(input())
i = 0
t = 2
# Упорядочиваем к виду а всегда меньше b

if a<=b:
    a1=a
    b1=b
else:
    a1=b
    b1=a

while a1>=1:
    i+=1
    t=a1
    print('Сторона квадрата:' + str(a1))
    a1=b1-a1
    b1=t

print('Количество квадратов:' + str(i))


# -*- coding: utf-8 -*-

import logging
from time import perf_counter

from dataiku.customrecipe import get_recipe_resource
from spacy_tokenizer import MultilingualTokenizer
from wordcloud_visualizer import WordcloudVisualizer
from plugin_config_loading import load_plugin_config_wordcloud

# Load config
params = load_plugin_config_wordcloud()
resource_path = get_recipe_resource()
output_folder = params["output_folder"]
df = params["df"]

# Load tokenizer
tokenizer = MultilingualTokenizer()

# Load wordcloud visualizer
worcloud_visualizer = WordcloudVisualizer(
    tokenizer=tokenizer,
    text_column=params["text_column"],
    font_path=resource_path,
    language=params["language"],
    language_column=params["language_column"],
    subchart_column=params["subchart_column"],
)

# Prepare data and count tokens for each subchart
frequencies = worcloud_visualizer.tokenize_and_count(df)

# Save wordclouds to folder
start = perf_counter()
logging.info("Generating wordclouds...")

for temp, output_file_name in worcloud_visualizer.generate_wordclouds(frequencies):
    output_folder.upload_data(output_file_name, temp.getvalue())

end = perf_counter()
logging.info(f"Generating wordclouds: Done in {end - start:.2f} seconds.")


#!/usr/bin/env python3

import torch
from torch.distributions import Bernoulli
from ..distributions import MultivariateNormal
from ..utils.quadrature import GaussHermiteQuadrature1D
from ..functions import log_normal_cdf, normal_cdf
from .likelihood import Likelihood

class BernoulliLikelihood(Likelihood):
    r"""
    Implements the Bernoulli likelihood used for GP classification, using
    Probit regression (i.e., the latent function is warped to be in [0,1]
    using the standard Normal CDF \Phi(x)). Given the identity \Phi(-x) =
    1-\Phi(x), we can write the likelihood compactly as:

    .. math::
        \begin{equation*}
            p(Y=y|f)=\Phi(yf)
        \end{equation*}
    """
    def __init__(self):
        super().__init__()
        self.quadrature = GaussHermiteQuadrature1D()

    def forward(self, input):
        if not isinstance(input, MultivariateNormal):
            raise RuntimeError(
                "BernoulliLikelihood expects a multi-variate normally distributed latent function to make predictions"
            )

        mean = input.mean
        var = input.variance
        link = mean.div(torch.sqrt(1 + var))
        output_probs = normal_cdf(link)
        return Bernoulli(probs=output_probs)

    def variational_log_probability(self, latent_func, target):
        likelihood_func = lambda locs: log_normal_cdf(locs.mul(target.unsqueeze(-1)))
        res = self.quadrature(likelihood_func, latent_func)
        return res.sum()

    def pyro_sample_y(self, variational_dist_f, y_obs, sample_shape, name_prefix=""):
        import pyro

        f_samples = variational_dist_f(sample_shape)
        y_prob_samples = torch.distributions.Normal(0, 1).cdf(f_samples)
        y_dist = pyro.distributions.Bernoulli(y_prob_samples)
        pyro.sample(name_prefix + "._training_labels", y_dist.independent(1), obs=y_obs)


import pandas as pd
import numpy as np
import copy

class TriangleGroupBy:
    def __init__(self, old_obj, by):
        obj = copy.deepcopy(old_obj)
        if by != -1:
            indices = obj.index.groupby(by).indices
            new_index = obj.index.groupby(by).count().index
        else:
            indices = {'All': np.arange(len(obj.index))}
            new_index = pd.Index(['All'], name='All')
        groups = [indices[item] for item in sorted(list(indices.keys()))]
        old_k_by_new_k = np.zeros(
            (len(obj.index.index), len(groups)), dtype='bool')
        for num, item in enumerate(groups):
            old_k_by_new_k[:, num][item] = True
        old_k_by_new_k = np.swapaxes(old_k_by_new_k, 0, 1)
        for i in range(3):
            old_k_by_new_k = old_k_by_new_k[..., np.newaxis]
        self.old_k_by_new_k = old_k_by_new_k
        obj.kdims = np.array(list(new_index))
        obj.key_labels = list(new_index.names)
        self.obj = obj

    def quantile(self, q, axis=1, *args, **kwargs):
        """ Return values at the given quantile over requested axis.  If
            Triangle is convertible to DataFrame then pandas quantile
            functionality is used instead.

        Parameters
        ----------
        q: float or array-like, default 0.5 (50% quantile)
            Value between 0 <= q <= 1, the quantile(s) to compute.
        axis:  {0, 1, ‘index’, ‘columns’} (default 1)
            Equals 0 or ‘index’ for row-wise, 1 or ‘columns’ for column-wise.

        Returns
        -------
            Triangle

        """
        x = self.obj.values*self.old_k_by_new_k
        ignore_vector = np.sum(np.isnan(x), axis=1, keepdims=True) == \
            x.shape[1]
        x = np.where(ignore_vector, 0, x)
        self.obj.values = \
            getattr(np, 'nanpercentile')(x, q*100, axis=1, *args, **kwargs)
        self.obj.values[self.obj.values == 0] = np.nan
        return self.obj

class TrianglePandas:
    def to_frame(self, *args, **kwargs):
        """ Converts a triangle to a pandas.DataFrame.  Requires an individual
        index and column selection to appropriately grab the 2D DataFrame.

        Returns
        -------
            pandas.DataFrame representation of the Triangle.
        """
        axes = [num for num, item in enumerate(self.shape) if item > 1]
        if self.shape[:2] == (1, 1):
            return self._repr_format()
        elif len(axes) == 2 or len(axes) == 1:
            tri = np.squeeze(self.values)
            axes_lookup = {0: self.kdims, 1: self.vdims,
                           2: self.origin, 3: self.ddims}
            if len(axes) == 2:
                return pd.DataFrame(tri, index=axes_lookup[axes[0]],
                                    columns=axes_lookup[axes[1]]).fillna(0)
            if len(axes) == 1:
                return pd.Series(tri, index=axes_lookup[axes[0]]).fillna(0)
        else:
            raise ValueError('len(index) and len(columns) must be 1.')

    def plot(self, *args, **kwargs):
        """ Passthrough of pandas functionality """
        return self.to_frame().plot(*args, **kwargs)

    @property
    def T(self):
        return self.to_frame().T

    def quantile(self, q, *args, **kwargs):
        if self.shape[:2] == (1, 1):
            return self.to_frame().quantile(q, *args, **kwargs)
        return TriangleGroupBy(self, by=-1).quantile(q, axis=1)

    def groupby(self, by, *args, **kwargs):
        """ Group Triangle by index values.  If the triangle is convertable to a
        DataFrame, then it defaults to pandas groupby functionality.

        Parameters
        ----------
        by: str or list
            The index to group by

        Returns
        -------
            GroupBy object (pandas or Triangle)
        """
        if self.shape[:2] == (1, 1):
            return self.to_frame().groupby(*args, **kwargs)
        return TriangleGroupBy(self, by)

    def append(self, other, index):
        """ Append rows of other to the end of caller, returning a new object.

        Parameters
        ----------
        other : Triangle
            The data to append.
        index:
            The index label(s) to assign the appended data.

        Returns
        -------
            New Triangle with appended data.
        """
        return_obj = copy.deepcopy(self)
        x = pd.DataFrame(list(return_obj.kdims), columns=return_obj.key_labels)
        new_idx = pd.DataFrame([index], columns=return_obj.key_labels)
        x = x.append(new_idx, sort=True)
        x.set_index(return_obj.key_labels, inplace=True)
        return_obj.values = np.append(return_obj.values, other.values, axis=0)
        return_obj.kdims = np.array(x.index.unique())
        return return_obj

    def rename(self, axis, value):
        """ Alter axes labels.

        Parameters
        ----------
            axis: str or int
                A value of 0 <= axis <= 4 corresponding to axes 'index',
                'columns', 'origin', 'development' respectively.  Both the
                int and str representation can be used.
            value: list or str
                List of new labels to be assigned to the axis. List must be of
                same length of the specified axis.

        Returns
        -------
            Triangle with relabeled axis.
        """
        value = [value] if type(value) is str else value
        if axis == 'index' or axis == 0:
            self.index = value
        if axis == 'columns' or axis == 1:
            self.columns = value
        if axis == 'origin' or axis == 2:
            self.origin = value
        if axis == 'development' or axis == 3:
            self.development = value
        return self

    def astype(self, dtype, inplace=True):
        '''
        Copy of the array, cast to a specified type.

        Parameters
        ----------
            dtype : str or dtype
                Typecode or data-type to which the array is cast.
            copy : bool, optional
                By default, astype always returns a newly allocated array.
        Returns
        -------
            Triangle as new datatype.
        '''
        obj = copy.deepcopy(self) if inplace is True else self
        obj.values = obj.values.astype(dtype)
        return obj

def add_triangle_agg_func(cls, k, v):
    ''' Aggregate Overrides in Triangle '''
    def agg_func(self, *args, **kwargs):
        if self.shape[:2] == (1, 1):
            return getattr(pd.DataFrame, k)(self.to_frame(), *args, **kwargs)
        else:
            return getattr(TriangleGroupBy(self, by=-1), k)(axis=1)
    set_method(cls, agg_func, k)

def add_groupby_agg_func(cls, k, v):
    ''' Aggregate Overrides in GroupBy '''
    def agg_func(self, axis=1, *args, **kwargs):
        obj = copy.deepcopy(self.obj)
        x = np.broadcast_to(
            self.obj.values,
            (self.old_k_by_new_k.shape[0], *self.obj.values.shape)) * \
            self.old_k_by_new_k
        ignore_vector = np.sum(np.isnan(x), axis=1, keepdims=True) == \
            x.shape[1]
        x = np.where(ignore_vector, 0, x)
        obj.values = \
            getattr(np, v)(x, axis=1, *args, **kwargs)
        obj.values[obj.values == 0] = np.nan
        return obj
    set_method(cls, agg_func, k)

def add_df_passthru(cls, k):
    '''Pass Through of pandas functionality '''
    def df_passthru(self, *args, **kwargs):
        return getattr(pd.DataFrame, k)(self.to_frame(), *args, **kwargs)
    set_method(cls, df_passthru, k)

def set_method(cls, func, k):
    ''' Assigns methods to a class '''
    func.__doc__ = 'Refer to pandas for ``{}`` functionality.'.format(k)
    func.__name__ = k
    setattr(cls, func.__name__, func)

df_passthru = ['to_clipboard', 'to_csv', 'to_excel', 'to_json',
               'to_html', 'to_dict', 'unstack', 'pivot', 'drop_duplicates',
               'describe', 'melt', 'pct_chg']
agg_funcs = ['sum', 'mean', 'median', 'max', 'min', 'prod', 'var', 'std']
agg_funcs = {item: 'nan'+item for item in agg_funcs}

for item in df_passthru:
    add_df_passthru(TrianglePandas, item)

for k, v in agg_funcs.items():
    add_triangle_agg_func(TrianglePandas, k, v)
    add_groupby_agg_func(TriangleGroupBy, k, v)


__author__ = 'ma0722'

class Solution:
    # @param {integer} num
    # @return {string}
    def intToRoman(self, num):
        roman = {1000: 'M', 900: 'CM', 500: 'D', 400: 'CD', 100: 'C', 90: 'XC', 50: 'L', 40: 'XL', 10: 'X', 9: 'IX', 5: 'V', 4: 'IV', 1: 'I'}
        s = ""
        for i in sorted(roman.keys(), reverse=True):
            while num >= i:
                s += roman[i]
                num -= i
        return s


import os
import glob
import json
import urllib
import urllib2
from datetime import datetime

from os.path import isfile, join

dir_path = os.path.dirname(os.path.realpath(__file__))
DSP_HOST = os.getenv('DSP_HOST', 'http://127.0.0.1:8090')
YUSUF_HOST = os.getenv('YUSUF_HOST', 'http://127.0.0.1:8010')
token = os.getenv('DSP_TOKEN', '811')

org_data_dir = "{d}/data/orgs/".format(d=dir_path)
ser_data_dir = "{d}/data/sers/".format(d=dir_path)
rewrite_dir = "{d}/data/rewrites/".format(d=dir_path)
logo_dir = "{d}/data/logos/".format(d=dir_path)
service_glossary_dir = "{d}/data/service_glossary/".format(d=dir_path)
all_dir = "{d}/data/all_locations/".format(d=dir_path)

print(all_dir)

for directory in [org_data_dir, ser_data_dir, rewrite_dir, logo_dir, service_glossary_dir, all_dir]:

    if not os.path.exists(directory):
        os.makedirs(directory)

    for f in glob.glob(directory + '*.*'):
        os.remove(f)

# loop through all orgs
# if file is modified in last 20 mins
# download the logo with a new name (<id> _org_log)

# for root, dirs, files in os.walk(data_dir):
#     for filename in files:
#         if not filename.endswith(".json"):
#             continue

#         full_path = '{d}{f}'.format(d=data_dir, f=filename)
#         rewrite = False
#         with open(full_path) as f:
#             # time = os.path.getmtime(full_path)
#             # if datetime.fromtimestamp(time).date() < datetime.today().date():
#             #     removable_files.append(full_path)

#             data = json.load(f)
#             is_success = True if ('success' in data) and (data['success'] == True) else False
#             if not is_success:
#                 removable_files.append(full_path)
#                 continue

#             details = data['details']
#             org_id = details.get('id')

#             has_services = True if len(details.get('services', [])) > 0 else False
#             has_cd = True if len(details.get('contact_details', [])) > 0 else False
#             org_logo = details.get('logo_url')

#             if org_logo:
#                 file_name, file_ext = os.path.splitext(org_logo)
#                 org_logo_filename = u"{l}{id}_org_logo{e}".format(l=logo_dir, id=org_id, e=file_ext)
#                 if not os.path.exists(org_logo_filename):
#                     with open(org_logo_filename, 'w+'):
#                         pass
#                 urllib.urlretrieve(org_logo, org_logo_filename)

#             for service in details.get('services', []):
#                 services_in_detail = service.get('services', [])
#                 for service_detail in services_in_detail:
#                     ser_id = service_detail.get('id')
#                     service_logo = service_detail.get('logo_url')
#                     service_detail['service_flow_steps'] = []
#                     rewrite = True

#                     if not service_detail.get('delivery_enabled', False):
#                         service_detail.update({
#                             'delivery_enabled': False
#                         })
#                         rewrite = True

#                     if not service_detail.get('service_steps'):
#                         service_detail.update({
#                             'service_steps': []
#                         })
#                         rewrite = True
#                     # if service_detail.get('service_flow_steps'):
#                     #     service_detail.update({
#                     #         'service_flow_steps': []
#                     #     })
#                     #     rewrite = True

#                     if not service_logo:
#                         continue

#                     # file_name, file_ext = os.path.splitext(service_logo)
#                     # ser_log_filename = u"{l}{id}_ser_logo{e}".format(
#                     #     l=logo_dir, id=ser_id, e=file_ext)

#                     # if not os.path.exists(ser_log_filename):
#                     #     with open(ser_log_filename, 'w+'):
#                     #         pass
#                     # urllib.urlretrieve(service_logo, ser_log_filename)

#             if has_services and not has_cd:
#                 no_cd_files.append(full_path)

#             if not all([has_services, has_cd]):
#                 removable_files.append(full_path)

#         print rewrite
#         if rewrite:
#             with open(full_path, 'w') as the_file:
#                 the_file.write(json.dumps(data))

# fncd = open("no_cd.txt", "w")
# fncd.write(str(no_cd_files))
# fnser = open("no_cd_ser.txt", "w")
# fnser.write(str(removable_files))

# for rem in removable_files:
#     try:
#         os.remove(rem)
#     except:
#         pass

def get_orgs():
    print(DSP_HOST, token)
    res = urllib2.urlopen(
        '{h}/arthur/org_pages?token={t}'.format(h=DSP_HOST, t=token))
    response = json.load(res)
    org_rewrites_file = '{d}{o}.txt'.format(d=rewrite_dir, o='org_rewrites')

    if not os.path.exists(org_data_dir):
        os.makedirs(org_data_dir)

    rewrites_string = ''

    if response and 'success' in response:
        orgs = response.get('orgs')
        if not orgs:
            return

        for org in orgs:
            filename = org.get('id')
            full_path = '{d}{f}.json'.format(d=org_data_dir, f=filename)
            org_slug = org.get('url_slug')

            if filename and org_slug:
                new_string = "\norganization/{f}/     /organization/{s}/   301!".format(
                    f=filename, s=org_slug)
                rewrites_string = rewrites_string + new_string

            with open(full_path, 'w+') as f:
                json.dump(org, f)

            org_logo = org.get('logo_url')
            if not org_logo:
                continue

            file_name, file_ext = os.path.splitext(org_logo)
            org_logo_filename = u"{l}{id}_org_logo{e}".format(
                l=logo_dir, id=filename, e=file_ext)

            if not os.path.exists(org_logo_filename):
                with open(org_logo_filename, 'w+'):
                    pass

            try:
                urllib.urlretrieve(org_logo, org_logo_filename)
            except Exception as e:
                print("failed getting logo for {s}, fix the logo url {o}".format(
                    s=org_slug, o=org_logo_filename))

def get_services():
    res = urllib2.urlopen(
        '{h}/arthur/ser_pages?token={t}'.format(h=DSP_HOST, t=token))
    response = json.load(res)

    ser_rewrites_file = '{d}{o}.txt'.format(d=rewrite_dir, o='ser_rewrites')
    if not os.path.exists(ser_data_dir):
        os.makedirs(ser_data_dir)

    rewrites_string = ''
    if response and 'success' in response:
        sers = response.get('services')

        for ser in sers:
            filename = ser.get('service').get('id')
            if not filename:
                continue

            ser_slug = ser.get('service').get('url_slug')

            if filename and ser_slug:
                new_string = "\nservice/{f}/     /service/{s}/   301!".format(
                    f=filename, s=ser_slug)
                rewrites_string = rewrites_string + new_string

            full_path = '{d}{f}.json'.format(d=ser_data_dir, f=filename)

            with open(full_path, 'w+') as f:
                json.dump(ser, f)

            service_logo = ser.get('service').get('logo_url')
            # if not service_logo:
            continue

            file_name, file_ext = os.path.splitext(service_logo)
            ser_logo_filename = u"{l}{id}_ser_logo{e}".format(
                l=logo_dir, id=filename, e=file_ext)

            if not os.path.exists(ser_logo_filename):
                with open(ser_logo_filename, 'w+'):
                    pass
            urllib.urlretrieve(service_logo, ser_logo_filename)

def get_service_glossary_items():
    res = urllib2.urlopen(
        '{h}/arthur/service_glossary_overview?token={t}'.format(h=DSP_HOST, t=token))
    sg_items = json.load(res)

    if sg_items and 'success' in sg_items:
        items = sg_items.get('details')
        for item in items:
            serice_glossary_item_slug = item.get('service_name_slug')
            ser_glossary_item_file_name = u"{d}{n}.json".format(
                d=service_glossary_dir, n=serice_glossary_item_slug)
            with open(ser_glossary_item_file_name, 'w+') as file_to_write:
                json.dump(item, file_to_write)

def get_all_locations():
    res = urllib2.urlopen(
        '{h}/arthur/all_locations?token={t}'.format(h=DSP_HOST, t=token))
    orgs = json.load(res)
    all_orgs_file = u"{d}{n}.json".format(d=all_dir, n="orgs")
    items = orgs.get('details')

    if orgs and 'success' in orgs:
        with open(all_orgs_file, 'w+') as file_to_write:
            json.dump(orgs, file_to_write)

def write_standard_redirects():
    print(DSP_HOST, YUSUF_HOST)
    redir_string = """/org_images/* http://storage.googleapis.com/evergov-prod-dsp-uploads/organization_logo_uploads/:splat 200!

/api/yusuf/* {y}/:splat 200!

/quick_pay/* https://pay.papergov.com/:splat 200!

/pay/* https://pay.papergov.com/:splat 200!

/knowledge-base/*  https://learn.papergov.com/knowledge-base/:splat 200!

/blog/* https://learn.papergov.com/blog/:splat 200!

/help/* https://learn.papergov.com/help-center/:splat 200!

/community/* https://disqus.com/home/forums/papergov/:splat 200!

/api/dsp/* {d}/:splat 200!

https://papergov.netlify.com/* https://papergov.com/:splat 301!

http://papergov.netlify.com/* https://papergov.com/:splat 301!

/app/*  /app/auth/callback/  200
/app/*  /app/  200
/app/*  /app/profile/  200
/deep_link/*  /deep_link/  200
/search/*  /search/  200

""".format(y=YUSUF_HOST, d=DSP_HOST)

    # res = urllib2.urlopen('{h}/arthur/redirects?token={t}'.format(h=DSP_HOST, t=token))
    # result = json.load(res)

    # if not result.get('success'):
    #     print("failed getting redirects")
    #     return

    # orgs = result.get('orgs')
    # services = result.get('sers')

    # for org_id in orgs:
    #     org_revs = orgs[org_id]
    #     if not org_revs:
    #         continue

    #     latest_rev = org_revs[-1]
    #     other_revs = org_revs[:-1]

    #     redir_string = redir_string + "\norganization/{f}/     /{s}/   301!".format(
    #         f=org_id, s=latest_rev)

    #     for rev in other_revs:
    #         redir_string = redir_string + "\n/{f}/     /{s}/   301!".format(
    #             f=rev, s=latest_rev)

    # for ser_id in services:
    #     ser_revs = services[ser_id]
    #     if not ser_revs:
    #         continue

    #     latest_rev = ser_revs[-1]
    #     other_revs = ser_revs[:-1]

    #     redir_string = redir_string + "\nservice/{f}/     /{s}/   301!".format(
    #         f=ser_id, s=latest_rev)

    #     for rev in other_revs:
    #         redir_string = redir_string + "\n/{f}/     /{s}/   301!".format(
    #             f=rev, s=latest_rev)

    with open('_redirects', 'w+') as f:
        f.write(redir_string)

write_standard_redirects()
print("redirects done")
get_orgs()
print("orgs done")
get_services()
print("services done")
get_service_glossary_items()
print("service glossary pages done")
get_all_locations()
print("all locations done")


"""
Author: Tran Dinh Hoang
Date: 05/07/2021
Program: Project_04_page33.py
Problem:
    4.  Open an IDLE window, and enter the program from Figure 1-7 that computes
        the area of a rectangle. Load the program into the shell by pressing the F5 key,
        and correct any errors that occur. Test the program with different inputs by
        running it at least three times.

* * * * * ============================================================================================= * * * * *

Solution:
    --> Display: "  Enter with width: 10
                    Enter with height: 5
                    The area is 15 square units. "
"""

width = int(input("Enter with width: "))
height = int(input("Enter with height: "))
area = width + height
print("The area is", area, "square units.")


from django.shortcuts import render, get_object_or_404, redirect
from django.utils import timezone

from .models import Post
from .forms import PostForm

def post_list(request):
    posts = Post.objects.filter(published_date__lte=timezone.now()).order_by('published_date')
    return render(request, 'blog/post_list.html', {'posts': posts})

def post_detail(request, pk):
    post = get_object_or_404(Post, pk=pk)
    return render(request, 'blog/post_detail.html', {'post': post})

def post_edit_or_new(request, pk=None):
    if pk:
        post = get_object_or_404(Post, pk=pk)
    else:
        post = None

    if request.method == "POST":
        form = PostForm(request.POST, instance=post)
        if form.is_valid():
            post = form.save(user=request.user)
            post.save()
            return redirect('post_detail', pk=post.pk)
    else:
        form = PostForm(instance=post)
    return render(request, 'blog/post_edit.html', {'form': form, 'post': post})


import math
import torch
from torch import nn
import torch.nn.functional as F

from collections import OrderedDict

class NbsCls(nn.Module):
    def __init__(self, in_feat, num_classes):
        super().__init__()
        self.in_feat = in_feat
        self.fc_out = nn.Linear(in_feat, num_classes)
        self.num_classes = num_classes

    def forward(self, x, alpha):
        out1 = x
        if isinstance(alpha, int):
            res_ = torch.zeros([alpha, out1.size(0), self.num_classes]).cuda()
            for i in range(alpha):
                w = torch.rand_like(out1).cuda()
                res = self.fc_out(out1 * w)
                res_[i] += res
            return res_
        else:
            out2 = torch.exp(-F.interpolate(alpha[:, None], self.in_feat))[:, 0]
            return self.fc_out(out1 * out2)

class NbsConvNet(nn.Module):
    def __init__(self, backbone, classifier):
        super().__init__()
        self.backbone = backbone
        self.classifer = classifier

    def forward(self, x, w=None):
        out = self.backbone(x)
        if out.size(-1) != 1:
            out = F.relu(out, inplace=True).mean([2, 3])
        else:
            out = out.squeeze()
        return self.classifer(out, w)

class GeneralConvNet(nn.Module):
    def __init__(self, backbone, classifier, last_drop=.0):
        super().__init__()
        self.backbone = backbone
        self.fc = classifier
        self.dropout = nn.Dropout(p=last_drop)

    def forward(self, x):
        out = self.backbone(x)
        if out.size(-1) != 1:
            out = F.relu(out, inplace=True).mean([2, 3])
        else:
            out = out.squeeze()
        out = self.dropout(out)
        return self.fc(out)

class BackboneGetter(nn.Sequential):
    def __init__(self, model, return_layer):
        if not set([return_layer]).issubset([name for name, _ in
                                             model.named_children()]):
            raise ValueError("return_layer is not present in model")

        layers = OrderedDict()
        for name, module in model.named_children():
            layers[name] = module
            if name == return_layer:
                break

        super().__init__(layers)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.bias.data.zero_()

def get_conv(backbone, return_layer, classifier, model_type, drop_rate=0.0):
    if model_type == 'mcd':
        backbone = BackboneGetter(backbone(drop_rate), return_layer)
    else:
        backbone = BackboneGetter(backbone(.0), return_layer)
    if model_type == 'nbs':
        model = NbsConvNet(backbone, classifier)
    else:
        model = GeneralConvNet(backbone, classifier, drop_rate)
    return model


import sys
import string

def check_avoids(names, avoids, avoid_type, ignore="") :

    html_text = "
\n"
    hits = 0
    for name in names :
        for anywhere in avoids["anywhere"]:
            if anywhere in ignore :
                pass
            elif (anywhere.lower() in name[1:-1].lower()):
                line = ""
                pos = (name.lower()).index(anywhere.lower())
                hits += 1
                for x in range(len(name)) :
                    if x == pos :
                        line += "" + name[x]
                        #html_text += "" + name[x]
                        if len(anywhere) == 1 :
                            #html_text += ""
                            line += ""
                    elif x == pos + len(anywhere)-1 :
                        line += name[x] + ""
                        #html_text += name[x] + ""
                    else :
                        line += name[x]
                        #html_text += name[x]
                if line not in html_text:
                    html_text += line +"\t%s\n" % ('"'+anywhere+'"')

        for prefix in avoids["prefix"] :
            if name[:len(prefix)].lower() == prefix.lower() :
                html_text += ""
                pos = 0
                hits += 1
                for x in range(len(name)) :
                    if x == pos :
                        html_text += "" + name[x]
                        if len(prefix) == 1 :
                            html_text += ""
                    elif x == pos + len(prefix)-1 :
                        html_text += name[x] + ""
                    else :
                        html_text += name[x]

                html_text += "\t%s\n" % (prefix+"-")

        for infix in avoids["infix"] :
            if infix in ignore :
                pass
            elif (infix.lower() in name[1:-1].lower()):# or (len(infix) == 1 and infix.lower() in name.lower()) or avoid_type == "project" and infix.lower() in name.lower():
                line = ""
                pos = (name.lower()).index(infix.lower())
                hits += 1
                for x in range(len(name)) :
                    if x == pos :
                        line += "" + name[x]
                        #html_text += "" + name[x]
                        if len(infix) == 1 :
                            #html_text += ""
                            line += ""
                    elif x == pos + len(infix)-1 :
                        line += name[x] + ""
                        #html_text += name[x] + ""
                    else :
                        line += name[x]
                        #html_text += name[x]
                if line not in html_text:
                    html_text += line +"\t%s\n" % ("-"+infix+"-")

        for suffix in avoids["suffix"]:
            if suffix in ignore :
                pass
            elif name[-len(suffix):].lower() == suffix.lower():
                html_text += ""
                pos = (name.lower()).index(suffix)
                hits += 1
                for x in range(len(name)):
                    if x == pos :
                        html_text += "" + name[x]
                        if len(suffix) == 1 :
                            html_text += ""
                    elif x == pos + len(suffix)-1:
                        html_text += name[x] + ""
                    else:
                        html_text += name[x]

                html_text += "\t%s\n" % ("-"+suffix)

    if hits == 0 :
        html_text += "None"
    return html_text + "
"

def check_internal_names(names, internal_names):
    html_text = "
"
    hits = 0
    for n in names:
        for i in internal_names :
            if "(" in i:
                i = i.split("(")[0].strip(string.whitespace)

            if n.lower() in i.lower() or i.lower() in n.lower():
                hits += 1
                line = n + " : "  + i + "\n"
                html_text += line

    if hits == 0 :
        html_text += "None"
    return html_text + "
"

def check_competitor_names(names, competitor_names):
    html_text = "
"
    hits = 0

    conflict_hits = []
    for name in names :
        html_text += ""
        for c in competitor_names :
            c = c.strip("\n")
            if "(" in c:
                c = c.split("(")[0].strip(string.whitespace)

            # Check for similar prefixes
            if c[0:3].lower() == name[0:3].lower() :
                prefix_hit = c[0:3]
                pos = 0
                hits += 1
                for x in range(len(name)) :
                    if x == pos :
                        html_text += "" + name[x]
                        if len(prefix_hit) == 1 :
                            html_text += ""
                    elif x == pos + len(prefix_hit)-1 :
                        html_text += name[x] + ""
                    else :
                        html_text += name[x]

                html_text += " : " + c + "\n"

            # Check for similar 4-letter word parts
            infix_conflict = False
            for l in range (0, len(c)-3) :
                if c[l:l+4].lower() in name.lower() :
                    infix_conflict = c[l:l+4]
                    pos = l
            if infix_conflict :
                html_text += ""
                pos = (name.lower()).index(infix_conflict.lower())
                hits += 1
                for x in range(len(name)) :
                    if x == pos :
                        html_text += "" + name[x]
                        if len(infix_conflict) == 1 :
                            html_text += ""
                    elif x == pos + len(infix_conflict)-1 :
                        html_text += name[x] + ""
                    else :
                        html_text += name[x]

                html_text +=  " : " + c + "\n"

            # Checks for similar combo hits (first and last 3 letters)
            if (c[0] + c[-3:]).lower() == (name[0] + name[-3:]).lower() :
                conflict_part1 = c[0]
                conflict_part2 = c[-3:]
                pos = 0
                pos2 = name.lower().index(conflict_part2.lower())
                hits += 1
                for x in range(len(name)) :
                    if x == pos :
                        html_text += "" + name[x] + ""
                    elif x == pos2 :
                        html_text += "" + name[x]
                    elif x == pos2 + len(conflict_part2)-1 :
                        html_text += name[x] + ""
                    else :
                        html_text += name[x]

                html_text += " : " + c + "\n"
    if hits == 0 :
        html_text += "None"
    return html_text + "
"


# -*- coding: utf-8 -*-
#:
#: Author: redkern
#: Date: 26/12/2016
#: Version: 0.1
#: License: MIT
#:

"""This module contains all the common functions used in the rsa module."""

import os
import random
import gmpy2
import functools

## Prime number generation
def get_prime(nbit):
    """Get a random prime number coded on size bits."""
    rng = random.SystemRandom()
    n = rng.getrandbits(nbit - 2) # n is nbit-2 random bits
    # this computation is made to that when you make the product of multiple
    # primes, you `nbit * number of primes` coded number
    n = 2 ** (nbit - 1) + n*2 + 1
    return gmpy2.next_prime(n)

def bit_size(n):
    """Returns the number of bits necessary to store the integer n."""
    if n == 0:
        return 1
    s = 0
    while n:
        s += 1
        n >>= 1 # shift MSB to left so that when it's 0, it returns s
    return s

def bytes_size(n):
    """Returns the number of bytes used to store the integer n."""
    q, m = divmod(bit_size(n), 8)
    if n == 0 or m == 0:
        q += 1
    return q

## Arithmetic
def extended_gcd(a, b):
    """Extended Euclidean algortihm

    Args:
        a: greatest number
        b: lowest number

    Returns:
        gcd, x, y such as gcd(a,b) = x*a + y*b
    """
    lastremainder, remainder = abs(a), abs(b)
    x, lastx, y, lasty = 0, 1, 1, 0
    while remainder:
        lastremainder, (quotient, remainder) = remainder, divmod(lastremainder, remainder)
        x, lastx = lastx - quotient*x, x
        y, lasty = lasty - quotient*y, y
    return lastremainder, lastx * (-1 if a < 0 else 1), lasty * (-1 if b < 0 else 1)

def mod_inverse(a, m):
    """Computes the inverse of a mod m."""
    g, x, y = extended_gcd(a, m)
    if g != 1:
        raise ValueError("a is not inversible")
    return x % m

def totient(factors):
    """Euler's phi function calculation.

    Args:
        factors: list of prime factorization of an integer

    Returns:
        phi(n)
    """
    phi = 1
    for factor in factors:
        phi = phi * (factor - 1)

    return phi

def exp_by_squaring(base, exponent, modulus):
    """Fast exponentiation."""
    # Converting the exponent to its binary form
    bin_exp = [int(x) for x in bin(exponent)[2:]]

    # Application of the square and multiply algorithm
    result = 1
    for i in bin_exp:
        result = (result * result) % modulus
        if i == 1:
            result = (result * base) % modulus

    return result

def crt(n, a):
    """Chinese remainer theorem

    Args:
        n: list of modulus
        a: list of coefficients

    Returns:
        the unique solution
    """
    s = 0
    prod = functools.reduce(lambda a, b: a * b, n)

    for n_i, a_i in zip(n, a):
        p = prod / n_i
        s = s + a_i * mod_inverse(p, n_i) * p

    return int(s % prod)

def is_probable_prime(n):
    """Miller-Rabin primality test.

    A return value of False means n is certainly not prime. A return value of
    True means n is very likely a prime.
    """
    _mrpt_num_trials = 5 # number of bases to test

    assert n >= 2
    # special case 2
    if n == 2:
        return True
    # ensure n is odd
    if n % 2 == 0:
        return False
    # write n-1 as 2**s * d
    # repeatedly try to divide n-1 by 2
    s = 0
    d = n-1
    while True:
        quotient, remainder = divmod(d, 2)
        if remainder == 1:
            break
        s += 1
        d = quotient
    assert(2**s * d == n-1)

    # test the base a to see whether it is a witness for the compositeness of n
    def try_composite(a):
        if pow(a, d, n) == 1:
            return False
        for i in range(s):
            if pow(a, 2**i * d, n) == n-1:
                return False
        return True # n is definitely composite

    for i in range(_mrpt_num_trials):
        a = random.randrange(2, n)
        if try_composite(a):
            return False

    return True # no base tested showed n as composite

if __name__ == "__main__":
    pass


# coding: utf-8

# In[1]:

import string
import numpy as np
from random import shuffle
import math
import codecs

# ## Спецсимволы

# In[11]:

# symbol we use to replace unknown symbols
UNKNSYM = u'*'
# symbol we use for padding
NOSYM = u'_'

# ## Кодирование слов в бинарные матрицы

# In[12]:

def encode_data(x, maxlen, vocab, vocab_size, check):
    """
    Iterate over the loaded data and create a matrix of size maxlen x vocabsize

    This is then placed in a 3D matrix of size data_samples x maxlen x vocab_size.
    Each character is encoded into a one-hot array. Chars not in the vocab
    are encoded into an all zero vector.
    """

    input_data = np.zeros((len(x), maxlen, vocab_size))

    for dix, sent in enumerate(x):

        counter = 0
        sent_array = np.zeros((maxlen, vocab_size))

        try:
            chars = list(sent.lower())  # .replace(' ', ''))
        except:
            print("ERROR " + str(dix) + " " + str(sent))
            continue

        counter = 0
        for c in chars:
            if counter >= maxlen:
                break
            else:
                char_array = np.zeros(vocab_size, dtype=np.int)
                if c in check:
                    ix = vocab[c]
                else:
                    ix = vocab[UNKNSYM]
                char_array[ix] = 1

                sent_array[counter, :] = char_array
                counter += 1

        char_array = np.zeros(vocab_size, dtype=np.int)
        char_array[vocab[NOSYM]] = 1
        while counter < maxlen:
            sent_array[counter, :] = char_array
            counter += 1

        input_data[dix, :, :] = sent_array

    return input_data

# ## Декодирование слов в бинарные матрицы

# In[13]:

def decode_data(matrix, reverse_vocab):
    """
        data_samples x maxlen x vocab_size
    """
    try:
        return "".join(
            [reverse_vocab[np.argmax(row)] for encoded_matrix in matrix for row in encoded_matrix])
    except:
        return "ERROR"

# ##  Словарь номеров символов и т. п.

# In[9]:

def create_vocab_set():
    alphabet = (list(NOSYM + u"йцукенгшщзхъёфывапролджэячсмитьбю-" + UNKNSYM))
    vocab_size = len(alphabet)
    check = set(alphabet)

    vocab = {}
    reverse_vocab = {}

    for ix, t in enumerate(alphabet):
        vocab[t] = ix
        reverse_vocab[ix] = t

    return vocab, reverse_vocab, vocab_size, check

# ## Генератор батчей из файла

# In[3]:

def simple_batch_generator(filename, vocab, vocab_size, vocab_check, maxlen, batch_size):
    with codecs.open(filename, "rt", encoding="utf-8") as data:
        run = True
        while run:
            x_sample = np.zeros((batch_size, 3), dtype=np.object)
            y_sample = np.zeros((batch_size,),   dtype=np.object)
            for i in range(batch_size):
                line = data.readline()
                if line:
                    words = line.strip("\n").split(",")
                    x_sample[i] = words[:3]
                    y_sample[i] = words[3]
                else:
                    run = False
                    break
            if not run:
                break

            input_data0 = encode_data(x_sample[:, 0], maxlen, vocab, vocab_size, vocab_check)
            input_data1 = encode_data(x_sample[:, 1], maxlen, vocab, vocab_size, vocab_check)
            input_data2 = encode_data(x_sample[:, 2], maxlen, vocab, vocab_size, vocab_check)

            input_data = np.concatenate([input_data0, input_data1, input_data2], axis=2)
            y_for_fitting = encode_data(y_sample, maxlen, vocab, vocab_size, vocab_check)

            yield (input_data, y_for_fitting, x_sample, y_sample)

# In[ ]:

def complex_batch_generator(filename, vocab, vocab_size, vocab_check, maxlen, batch_size):
    with codecs.open(filename, "rt", encoding="utf-8") as data:
        run = True
        while run:
            x_sample = np.zeros((batch_size, 3), dtype=np.object)
            y_sample = np.zeros((batch_size,),   dtype=np.object)
            for i in range(batch_size):
                line = data.readline()
                if line:
                    words = line.strip("\n").split(",")
                    x_sample[i] = words[:3]
                    y_sample[i] = words[3]
                else:
                    run = False
                    break
            if not run:
                break

            input_data0 = encode_data(x_sample[:, 0], maxlen, vocab, vocab_size, vocab_check)
            input_data1 = encode_data(x_sample[:, 1], maxlen, vocab, vocab_size, vocab_check)
            input_data2 = encode_data(x_sample[:, 2], maxlen, vocab, vocab_size, vocab_check)

            input_data = [input_data0, input_data1, input_data2]
            y_for_fitting = encode_data(y_sample, maxlen, vocab, vocab_size, vocab_check)

            yield (input_data, y_for_fitting, x_sample, y_sample)

def generatorlike_batch_generator(filename, vocab, vocab_size, vocab_check, maxlen,
                                  batch_size, sub_length):
    with codecs.open(filename, "rt", encoding="utf-8") as data:
        run = True
        while run:
            x_sample = np.zeros((batch_size, 3), dtype=np.object)
            y_sample = np.zeros((batch_size,),   dtype=np.object)
            for i in range(batch_size):
                line = data.readline()
                if line:
                    words = line.strip("\n").split(",")
                    x_sample[i] = words[:3]
                    y_sample[i] = words[3]
                else:
                    run = False
                    break
            if not run:
                break

            input_data1 = encode_data(x_sample[:, 0], maxlen, vocab, vocab_size, vocab_check)
            input_data2 = encode_data(x_sample[:, 1], maxlen, vocab, vocab_size, vocab_check)
            input_data3 = encode_data(x_sample[:, 2], maxlen, vocab, vocab_size, vocab_check)

            y_for_fitting = encode_data(y_sample, maxlen, vocab, vocab_size, vocab_check)
            for shift in xrange(1, maxlen - sub_length):
                input_data4 = np.zeros((batch_size, sub_length, vocab_size))
                y           = np.zeros((batch_size, sub_length, vocab_size))
                for i in xrange(batch_size):
                    input_data4[i] = y_for_fitting[i][shift: shift + sub_length]
                    y[i]           = y_for_fitting[i][shift: shift + sub_length]

                input_data = [input_data1, input_data2, input_data3, input_data4]
                yield (input_data, y, x_sample, y_sample)

# In[11]:

# def load_relations(train_file, test_file):
#     train = pd.read_csv(train_file, header=None, encoding="utf-8")
#     train = train.dropna()
#     x_train = np.array(train.ix[:, 0:2])
#     y_train = np.array(train.ix[:, 3])
#
#     test = pd.read_csv(test_file, header=None, encoding="utf-8")
#     test = test.dropna()
#     x_test = np.array(test.ix[:, 0:2])
#     y_test = np.array(test.ix[:, 3])
#
#     return (x_train, y_train), (x_test, y_test)

# In[14]:

# def shuffle_matrix(x, y):
#     stacked = np.hstack((np.matrix(x).T, np.matrix(y).T))
#     np.random.shuffle(stacked)
#     xi = np.array(stacked[:, 0]).flatten()
#     yi = np.array(stacked[:, 1]).flatten()
#     return xi, yi

# In[ ]:

# if __name__ == '__main__':
#     prepare_relations("data/relations.pairs.test.tsv")