import json
import socket
import time
import sys
import random
import numpy
import threading
from Priority_Queue import PriorityQueue
import logging
import datetime

logging.basicConfig(filename='YACS_logs.log', filemode='w',
                    format='%(message)s', level=logging.INFO)

# Priority Queue that contains all the jobs
jobs_pq = PriorityQueue()
pq_lock = threading.Lock()
workers_lock = {}
sockets = {}
lock = threading.Lock()

# Function to send tasks to the worker
def send_tasks(data, port):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect(('localhost', port))
    s.send(data.encode())
    s.close()

# Function that runs on a separate thread and listens to requests
def listen_to_requests():
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('localhost', 5000))
    s.listen(1)

    while(1):
        try:
            host, _ = s.accept()
            with host:
                data = host.recv(1024)
                # Structure the jobs to this format:
                # [
                #  {'0': [[('0_M0', 2)], [('0_R0', 4), ('0_R1', 1)]]},
                #  {'1': [[('1_M0', 1)], [('1_R0', 2), ('1_R1', 4)]]}
                # ]
                x = json.loads(data)
                d = dict()
                p = []
                l = []
                for i in x['map_tasks']:
                    l.append((i['task_id'], i['duration']))
                    lock.acquire()
                    date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                    logging.info(date + " " + "task arrival" + " " +
                                 x["job_id"] + " " + i["task_id"])
                    lock.release()
                p.append(l)
                r = []
                for i in x['reduce_tasks']:
                    r.append((i['task_id'], i['duration']))
                    lock.acquire()
                    date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                    logging.info(date + " " + "task arrival" +
                                 " " + x["job_id"] + " " + i["task_id"])
                    lock.release()

                p.append(r)
                d[x['job_id']] = p

                pq_lock.acquire()
                jobs_pq.insert(d)
                pq_lock.release()
            print("Received Request")
        except KeyboardInterrupt:
            break

# Function to schedule tasks using Round-Robin Algorithm
def handle_roundrobin(workers):
    # Initialise prev_worker_id to -1
    prev_worker_id = -1

    while(1):
        time.sleep(0.05)
        pq_lock.acquire()
        task = jobs_pq.getTask()
        pq_lock.release()
        # If the queue is empty or
        # if reduce tasks cannot be run because map tasks of a job are still running
        if(task == None):
            continue

        found = False
        while(not found):
            worker_id = (prev_worker_id + 1) % len(worker_ids)
            count = 0
            # Iterate from prev_worker_id+1 through pre_worker_id
            while(count < len(worker_ids)):
                time.sleep(0.1)
                count += 1
                workers_lock[worker_id+1].acquire()
                freeSlotisAvailable = workers[worker_id+1]["free_slots"]
                workers_lock[worker_id+1].release()
                # If a free slot is available, assign the task to the worker
                if(freeSlotisAvailable):
                    prev_worker_id = worker_id
                    workers_lock[worker_id+1].acquire()
                    workers[worker_id+1]["free_slots"] -= 1
                    workers_lock[worker_id+1].release()

                    print("Task %s assigned to worker %d" %
                            (str(task), worker_id+1))
                    #####################
                    # Send task to worker
                    send_tasks(str(task[0])+","+str(task[1]),
                                workers[worker_id+1]["port"])
                    #####################
                    found = True
                    break
                worker_id = (worker_id + 1) % len(worker_ids)

# Function to schedule tasks using Random Algorithm
def handle_random(workers, worker_ids):

    while(1):
        time.sleep(0.05)
        pq_lock.acquire()
        task = jobs_pq.getTask()
        pq_lock.release()
        # If the queue is empty or
        # if reduce tasks cannot be run because map tasks of a job are still running
        if(task == None):
            continue
        worker_found = False
        # Make a random choices until a worker with free slots is found
        while(not worker_found):
            worker_id = random.choice(worker_ids)
            workers_lock[worker_id].acquire()
            freeSlotisAvailable = workers[worker_id]["free_slots"]
            workers_lock[worker_id].release()
            # If a free slot is available, assign the task to the worker
            if(freeSlotisAvailable):
                worker_found = True
                workers_lock[worker_id].acquire()
                workers[worker_id]["free_slots"] -= 1
                workers_lock[worker_id].release()

                print("Task %s assigned to worker %d" %
                        (str(task), worker_id))
                #####################
                # Send task to worker
                send_tasks(str(task[0])+","+str(task[1]),
                            workers[worker_id]["port"])
                #####################

    print("Exitting random")

# Function to schedule tasks using Least-Loaded Algorithm
def handle_LL(workers):
    while(1):
        time.sleep(0.05)
        pq_lock.acquire()
        task = jobs_pq.getTask()
        pq_lock.release()
        # If the queue is empty or
        # if reduce tasks cannot be run because map tasks of a job are still running
        if(task == None):
            continue
        worker_found = False
        max_slots = 0
        max_id = 0
        while(not worker_found):
            # Iterate through all workers select a worker with the maximum number of free slots
            for worker_id in workers:
                workers_lock[worker_id].acquire()
                if(workers[worker_id]["free_slots"] > max_slots):
                    max_slots = workers[worker_id]["free_slots"]
                    max_id = worker_id
                workers_lock[worker_id].release()
            # If the maximum number of free slots is greater than 0,
            # assign the task to the worker
            if(max_slots):
                worker_found = True
                workers_lock[max_id].acquire()
                workers[max_id]["free_slots"] -= 1
                workers_lock[max_id].release()
                print("Task %s assigned to worker %d" %
                        (str(task), max_id))
                #####################
                # Send task to worker
                send_tasks(str(task[0])+","+str(task[1]),
                            workers[max_id]["port"])
                #####################

# Function that runs on a separate thread and listens for updates from the worker
# Worker sends an update upon completion of execution of the task
def listen_to_workers():
    print("Listening to workers")
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('localhost', 5001))
    s.listen(1)

    while(1):
        try:
            host, _ = s.accept()
            with host:
                msg = host.recv(1024)
                msg = msg.decode()
                # Message received from the worker contains:
                # start time of the task, end time of the task,
                # taskId and worker_id that executed the task
                start_time, end_time, task_id, worker_id = msg.split(',')
                worker_id = int(worker_id)
                print("Message %s received from worker %d " %
                      (task_id, worker_id))
                lock.acquire()
                job_id = task_id.split("_")[0]
                # Log the starting and ending time of the task
                logging.info(start_time + " " + "starting task" +
                             " " + job_id + " " + task_id + " " + str(worker_id))
                logging.info(end_time + " " + "ending task" + " " +
                             job_id + " " + task_id + " " + str(worker_id))
                lock.release()

                # Increment the number of free slots in the worker
                workers_lock[worker_id].acquire()
                workers[worker_id]["free_slots"] += 1
                workers_lock[worker_id].release()

                # Remove the task from the job
                pq_lock.acquire()
                jobs_pq.popTask(task_id)
                pq_lock.release()
        except KeyboardInterrupt:
            break

    print("NOT LISTENING TO UPDATES FROM WORKERS ANYMORE")
    s.close()

if __name__ == '__main__':
    if(len(sys.argv) != 3):
        print("Usage: python master.py <path to config file> <name of algorithm: RR,LL,RANDOM>")
        exit()
    args = sys.argv
    path = args[1]
    algo = args[2]
    logging.info(args[2])

    if (algo != "RR") and (algo != "RANDOM") and (algo != "LL"):
        print("INVALID ALGORITHM! ENTER RR,LL or RANDOM")
        exit()

    f = open(path, 'r')
    data = json.load(f)

    print("Connect with Workers..............")
    workers = {}
    worker_ids = []
    for worker in data["workers"]:
        print(worker)
        # Dictionary indexed by the worker_id which contains:
        # Total number of slots available in a worker, port to which it is connnected and
        # the number of free slots available
        workers[worker["worker_id"]] = {
            "slots": worker["slots"],
            "port": worker["port"],
            "free_slots": worker["slots"]
        }
        worker_ids.append(worker["worker_id"])
        # Dictionary indexed by worker_id which contains the lock for each worker
        workers_lock[worker["worker_id"]] = threading.Lock()

    print("Connection with Workers successful!!\n")

    # Create and start a thread to listen to requests and listen to workers for updates
    requests_listener = threading.Thread(target=listen_to_requests)
    worker_listener = threading.Thread(target=listen_to_workers)
    requests_listener.start()
    worker_listener.start()

    print("Continue processing on the master thread to assign jobs")
    time.sleep(5)
    if algo == "RR":
        handle_roundrobin(workers)
    elif algo == "RANDOM":
        handle_random(workers, worker_ids)
    elif algo == "LL":
        handle_LL(workers)

    worker_listener.join()
    requests_listener.join()


from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import SuspiciousOperation
from django.db import IntegrityError
from django.db import transaction
from django.db.models import Prefetch
from django.db.models import Q
from django.shortcuts import get_object_or_404
from django.utils.translation import ugettext_lazy as _

from accession import localsettings
from accession.namebuilder import NameBuilderManager
from descriptor.comment import CommentController
from descriptor.describable import DescriptorsBuilder
from descriptor.models import Layout, Descriptor
from igdectk.rest.handler import *
from igdectk.rest.response import HttpResponseRest
from main.models import Language, EntitySynonymType
from permission.utils import get_permissions_for
from classification.models import ClassificationEntry

from .models import Accession, AccessionSynonym, AccessionClassificationEntry, AccessionPanel
from .base import RestAccession

class RestAccessionAccession(RestAccession):
    regex = r'^accession/$'
    name = 'accession'

class RestAccessionSearch(RestAccessionAccession):
    regex = r'^search/$'
    suffix = 'search'

class RestAccessionAccessionCount(RestAccessionAccession):
    regex = r'^count/$'
    name = 'count'

class RestAccessionId(RestAccessionAccession):
    regex = r'^(?P<acc_id>[0-9]+)/$'
    suffix = 'id'

class RestAccessionIdComment(RestAccessionId):
    regex = r'^comment/$'
    suffix = 'comment'

class RestAccessionIdCommentId(RestAccessionIdComment):
    regex = r'^(?P<com_id>[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12})/$'
    suffix = 'id'

@RestAccessionAccession.def_auth_request(Method.POST, Format.JSON, content={
    "type": "object",
    "properties": {
        "name": AccessionSynonym.NAME_VALIDATOR,
        "naming_options": {"type": "array", 'minItems': 0, 'maxItems': 10, 'additionalItems': {'type': 'any'}, 'items': []},
        # "code": AccessionSynonym.CODE_VALIDATOR,
        "layout": {"type": "number"},
        "primary_classification_entry": {"type": "number"},
        "descriptors": {"type": "object"},
        "language": AccessionSynonym.LANGUAGE_VALIDATOR
    },
}, perms={
    'accession.add_accession': _("You are not allowed to create an accession")
})
def create_accession(request):
    name = request.data['name']
    naming_options = request.data['naming_options']
    # code = request.data['code']
    layout_id = int_arg(request.data['layout'])
    primary_classification_entry_id = int_arg(request.data['primary_classification_entry'])
    descriptors = request.data['descriptors']
    language = request.data['language']

    if not Language.objects.filter(code=language).exists():
        raise SuspiciousOperation(_("The language is not supported"))

    naming_variables = {}
    code = NameBuilderManager.get(NameBuilderManager.GLOBAL_ACCESSION).pick(naming_variables, naming_options)

    # check uniqueness of the code for any type of synonym
    if AccessionSynonym.objects.filter(name=code).exists():
        raise SuspiciousOperation(_("The code of the accession is already used as a synonym name"))

    # check uniqueness of the code
    if Accession.objects.filter(code=code).exists():
        raise SuspiciousOperation(_("The code of the accession is already used"))

    if name == code:
        raise SuspiciousOperation(_("The code and the name of the accession must be different"))

    content_type = get_object_or_404(ContentType, app_label="accession", model="accession")
    layout = get_object_or_404(Layout, id=layout_id, target=content_type)

    try:
        with transaction.atomic():
            # common properties
            accession = Accession()
            accession.name = name
            accession.code = code
            accession.layout = layout

            # primary classification entry
            primary_classification_entry = get_object_or_404(ClassificationEntry, id=primary_classification_entry_id)
            accession.primary_classification_entry = primary_classification_entry

            # descriptors
            descriptors_builder = DescriptorsBuilder(accession)

            descriptors_builder.check_and_update(layout, descriptors)
            accession.descriptors = descriptors_builder.descriptors

            accession.save()

            # update owner on external descriptors
            descriptors_builder.update_associations()

            # primary classifications in M2M
            AccessionClassificationEntry.objects.create(
                accession=accession, classification_entry=primary_classification_entry, primary=True)

            # initial synonym GRC code
            grc_code = AccessionSynonym(
                entity=accession,
                name=code,
                synonym_type_id=localsettings.synonym_type_accession_code,
                language='en')
            grc_code.save()

            # primary synonym if defined
            primary_name = AccessionSynonym(
                entity=accession,
                name=name,
                synonym_type_id=localsettings.synonym_type_accession_name,
                language=language)
            primary_name.save()

            accession.synonyms.add(grc_code)
            accession.synonyms.add(primary_name)

            # add related classification entries
            classification_entry_bulk = []
            for classification_entry in primary_classification_entry.related.all():
                ace = AccessionClassificationEntry(
                    primary=False,
                    accession=accession,
                    classification_entry=classification_entry
                )

                classification_entry_bulk.append(ace)

            AccessionClassificationEntry.objects.bulk_create(classification_entry_bulk)

    except IntegrityError as e:
        Descriptor.integrity_except(Accession, e)

    response = {
        'id': accession.pk,
        'name': accession.name,
        'code': accession.code,
        'layout': layout.id,
        'primary_classification_entry': primary_classification_entry.id,
        'descriptors': accession.descriptors,
        'synonyms': [
            {
                'id': grc_code.id,
                'name': grc_code.name,
                'synonym_type': grc_code.synonym_type_id,
                'language': grc_code.language
            },
            {
                'id': primary_name.id,
                'name': primary_name.name,
                'synonym_type': primary_name.synonym_type_id,
                'language': primary_name.language
            }
        ]
    }

    return HttpResponseRest(request, response)

@RestAccessionAccessionCount.def_auth_request(Method.GET, Format.JSON, perms={
    'accession.list_accession': _("You are not allowed to list the accessions")
})
def get_accession_list_count(request):
    from main.cursor import CursorQuery
    cq = CursorQuery(Accession)

    if request.GET.get('search'):
        search = json.loads(request.GET['search'])
        cq.filter(search)

    if request.GET.get('filters'):
        filters = json.loads(request.GET['filters'])
        cq.filter(filters)

    cq.m2m_to_array_field(
        relationship=AccessionPanel.accessions,
        selected_field='accessionpanel_id',
        from_related_field='id',
        to_related_field='accession_id',
        alias='panels'
    )

    cq.m2m_to_array_field(
        relationship=Accession.classifications_entries,
        selected_field='classification_entry_id',
        from_related_field='id',
        to_related_field='accession_id',
        alias='classifications'
    )

    cq.set_synonym_model(AccessionSynonym)

    count = cq.count()

    results = {
        'count': count
    }

    return HttpResponseRest(request, results)

@RestAccessionAccession.def_auth_request(Method.GET, Format.JSON, perms={
    'accession.list_accession': _("You are not allowed to list the accessions")
})
def get_accession_list(request):
    results_per_page = int_arg(request.GET.get('more', 30))
    cursor = json.loads(request.GET.get('cursor', 'null'))
    limit = results_per_page
    sort_by = json.loads(request.GET.get('sort_by', '[]'))

    if not len(sort_by) or sort_by[-1] not in ('id', '+id', '-id'):
        order_by = sort_by + ['id']
    else:
        order_by = sort_by

    from main.cursor import CursorQuery
    cq = CursorQuery(Accession)

    if request.GET.get('search'):
        search = json.loads(request.GET['search'])
        cq.filter(search)

    if request.GET.get('filters'):
        filters = json.loads(request.GET['filters'])
        cq.filter(filters)

    cq.m2m_to_array_field(
        relationship=AccessionPanel.accessions,
        selected_field='accessionpanel_id',
        from_related_field='id',
        to_related_field='accession_id',
        alias='panels'
    )

    cq.m2m_to_array_field(
        relationship=Accession.classifications_entries,
        selected_field='classification_entry_id',
        from_related_field='id',
        to_related_field='accession_id',
        alias='classifications'
    )

    cq.set_synonym_model(AccessionSynonym)

    cq.prefetch_related(Prefetch(
        "synonyms",
        queryset=AccessionSynonym.objects.all().order_by('synonym_type', 'language')
    ))

    cq.select_related('primary_classification_entry->name', 'primary_classification_entry->rank')

    cq.cursor(cursor, order_by)
    cq.order_by(order_by).limit(limit)

    accession_items = []

    synonym_types = dict(EntitySynonymType.objects.filter(target_model=ContentType.objects.get_for_model(Accession)).values_list('id', 'name'))

    for accession in cq:
        a = {
            'id': accession.pk,
            'name': accession.name,
            'code': accession.code,
            'primary_classification_entry': accession.primary_classification_entry_id,
            'layout': accession.layout_id,
            'descriptors': accession.descriptors,
            'synonyms': {},
            'primary_classification_entry_details': {
                'id': accession.primary_classification_entry.id,
                'name': accession.primary_classification_entry.name,
                'rank': accession.primary_classification_entry.rank_id,
            }
        }

        for synonym in accession.synonyms.all():
            synonym_type_name = synonym_types.get(synonym.synonym_type_id)
            a['synonyms'][synonym_type_name] = {
                'id': synonym.id,
                'name': synonym.name,
                'synonym_type': synonym.synonym_type_id,
                'language': synonym.language
            }

        accession_items.append(a)

    results = {
        'perms': [],
        'items': accession_items,
        'prev': cq.prev_cursor,
        'cursor': cursor,
        'next': cq.next_cursor,
    }

    return HttpResponseRest(request, results)

@RestAccessionId.def_auth_request(Method.GET, Format.JSON, perms={
    'accession.get_accession': _("You are not allowed to get an accession")
})
def get_accession_details_json(request, acc_id):
    """
    Get the details of an accession.
    """

    accession = Accession.objects.get(id=int(acc_id))

    # check permission on this object
    perms = get_permissions_for(request.user, accession.content_type.app_label, accession.content_type.model,
                                accession.pk)
    if 'accession.get_accession' not in perms:
        raise PermissionDenied(_('Invalid permission to access to this accession'))

    result = {
        'id': accession.id,
        'name': accession.name,
        'code': accession.code,
        'primary_classification_entry': accession.primary_classification_entry_id,
        'synonyms': [],
        'layout': accession.layout_id,
        'descriptors': accession.descriptors,
        'panels': []
    }

    for panel in accession.panels.all():
        result['panels'].append({
            'id': panel.id,
            'name': panel.name,
            'layout': panel.layout.pk if panel.layout else None,
            'descriptors': panel.descriptors,
            'accessions_amount': panel.accessions.count()
        })

    for s in AccessionSynonym.objects.filter(entity=accession.id).order_by('synonym_type', 'language'):
        result['synonyms'].append({
            'id': s.id,
            'name': s.name,
            'synonym_type': s.synonym_type_id,
            'language': s.language,
        })

    return HttpResponseRest(request, result)

@RestAccessionSearch.def_auth_request(Method.GET, Format.JSON, ('filters',), perms={
    'accession.search_accession': _("You are not allowed to search accessions")
})
def search_accession(request):
    """
    Quick search for an accession with a exact or partial name and layout of descriptor.
    It is possible to have multiple results for a same accession because of the multiples synonyms.

    The filters can be :
        - name: value to look for the name field.
        - method: for the name 'ieq' or 'icontains' for insensitive case equality or %like% respectively.
        - layout: id of the descriptor layout to look for.
        - fields: list of fields to look for.
    """
    filters = json.loads(request.GET['filters'])

    results_per_page = int_arg(request.GET.get('more', 30))
    cursor = request.GET.get('cursor')
    limit = results_per_page

    if cursor:
        cursor = json.loads(cursor)
        cursor_name, cursor_id = cursor
        qs = Accession.objects.filter(Q(name__gt=cursor_name))
    else:
        qs = Accession.objects.all()

    name_method = filters.get('method', 'ieq')
    if 'layout' in filters['fields']:
        layout = int_arg(filters['layout'])

        if name_method == 'ieq':
            qs = qs.filter(Q(synonyms__name__iexact=filters['name']))
        elif name_method == 'icontains':
            qs = qs.filter(Q(synonyms__name__icontains=filters['name']))

        qs = qs.filter(Q(layout_id=layout))
    elif 'name' in filters['fields']:
        if name_method == 'ieq':
            qs = qs.filter(synonyms__name__iexact=filters['name'])
        elif name_method == 'icontains':
            qs = qs.filter(synonyms__name__icontains=filters['name'])
    elif 'code' in filters['fields']:
        if name_method == 'ieq':
            qs = qs.filter(code__iexact=filters['code'])
        elif name_method == 'icontains':
            qs = qs.filter(code__icontains=filters['code'])

    qs = qs.prefetch_related(
        Prefetch(
            "synonyms",
            queryset=AccessionSynonym.objects.exclude(
                synonym_type=localsettings.synonym_type_accession_code).order_by('synonym_type', 'language'))
    )

    qs = qs.order_by('name').distinct()[:limit]

    items_list = []

    for accession in qs:
        label = accession.name

        for synonym in accession.synonyms.all():
            label += ', ' + synonym.name

        a = {
            'id': accession.id,
            'label': label,
            'value': accession.code
        }

        items_list.append(a)

    if len(items_list) > 0:
        # prev cursor (asc order)
        obj = items_list[0]
        prev_cursor = (obj['value'], obj['id'])

        # next cursor (asc order)
        obj = items_list[-1]
        next_cursor = (obj['value'], obj['id'])
    else:
        prev_cursor = None
        next_cursor = None

    results = {
        'perms': [],
        'items': items_list,
        'prev': prev_cursor,
        'cursor': cursor,
        'next': next_cursor,
    }

    return HttpResponseRest(request, results)

@RestAccessionId.def_auth_request(Method.PATCH, Format.JSON, content={
        "type": "object",
        "properties": {
            "primary_classification_entry": {"type": "integer", "required": False},
            "entity_status": Accession.ENTITY_STATUS_VALIDATOR_OPTIONAL,
            "descriptors": {"type": "object", "required": False}
        },
    },
    perms={
      'accession.change_accession': _("You are not allowed to modify an accession"),
    })
def patch_accession(request, acc_id):
    accession = get_object_or_404(Accession, id=int(acc_id))

    entity_status = request.data.get("entity_status")
    descriptors = request.data.get("descriptors")

    result = {
        'id': accession.id
    }

    try:
        with transaction.atomic():
            if 'primary_classification_entry' in request.data:
                primary_classification_entry_id = int(request.data['primary_classification_entry'])
                primary_classification_entry = get_object_or_404(ClassificationEntry, id=primary_classification_entry_id)

                # update FK
                accession.primary_classification_entry = primary_classification_entry
                result['primary_classification_entry'] = primary_classification_entry.id

                # and replace from classification entry M2M previous primary
                accession_classification_entry = get_object_or_404(
                    AccessionClassificationEntry, accession=accession, primary=True)

                accession_classification_entry.classification_entry = primary_classification_entry
                accession_classification_entry.save()

                accession.update_field('primary_classification_entry')

            if entity_status is not None and accession.entity_status != entity_status:
                accession.set_status(entity_status)
                result['entity_status'] = entity_status
                accession.update_field('entity_status')

            if descriptors is not None:
                # update descriptors
                descriptors_builder = DescriptorsBuilder(accession)

                descriptors_builder.check_and_update(accession.layout, descriptors)

                accession.descriptors = descriptors_builder.descriptors
                result['descriptors'] = accession.descriptors

                descriptors_builder.update_associations()

                accession.update_descriptors(descriptors_builder.changed_descriptors())
                accession.update_field('descriptors')

            accession.save()
    except IntegrityError as e:
        Descriptor.integrity_except(Accession, e)

    return HttpResponseRest(request, result)

@RestAccessionId.def_auth_request(Method.DELETE, Format.JSON, perms={
    'accession.delete_accession': _("You are not allowed to delete an accession"),
})
def delete_accession(request, acc_id):
    accession = get_object_or_404(Accession, id=int(acc_id))

    accession.synonyms.all().delete()
    accession.delete()

    return HttpResponseRest(request, {})

@RestAccessionIdComment.def_auth_request(Method.GET, Format.JSON, perms={
      'accession.get_accession': _("You are not allowed to get an accession"),
    })
def get_accession_comment_list(request, acc_id):
    sort_by = json.loads(request.GET.get('sort_by', '[]'))

    if not len(sort_by) or sort_by[-1] not in ('id', '+id', '-id'):
        order_by = sort_by + ['id']
    else:
        order_by = sort_by

    accession = get_object_or_404(Accession, id=int(acc_id))

    comment_controller = CommentController(accession)
    results = comment_controller.list_comments(order_by)

    return HttpResponseRest(request, results)

@RestAccessionIdCommentId.def_auth_request(Method.DELETE, Format.JSON, perms={
    'accession.change_accession': _("You are not allowed to modify an accession"),
})
def remove_accession_comment(request, acc_id, com_id):
    accession = get_object_or_404(Accession, id=int(acc_id))

    comment_controller = CommentController(accession)
    comment_controller.remove_comment(com_id)

    return HttpResponseRest(request, {})

@RestAccessionIdComment.def_auth_request(Method.POST, Format.JSON, content=Accession.COMMENT_VALIDATOR, perms={
  'accession.change_accession': _("You are not allowed to modify an accession"),
})
def add_accession_comment(request, acc_id):
    accession = get_object_or_404(Accession, id=int(acc_id))

    comment_controller = CommentController(accession)
    result = comment_controller.add_comment(request.data['label'], request.data['value'])

    return HttpResponseRest(request, result)

@RestAccessionIdCommentId.def_auth_request(Method.PATCH, Format.JSON, content=Accession.COMMENT_VALIDATOR, perms={
    'accession.change_accession': _("You are not allowed to modify an accession"),
})
def patch_accession_comment(request, acc_id, com_id):
    accession = get_object_or_404(Accession, id=int(acc_id))

    comment_controller = CommentController(accession)
    result = comment_controller.update_comment(com_id, request.data['label'], request.data['value'])

    return HttpResponseRest(request, result)


from telethon.sync import TelegramClient
from telethon.tl.functions.channels import CreateChannelRequest, CheckUsernameRequest, UpdateUsernameRequest
from telethon.tl.types import InputPeerChannel
from telethon.tl.types import DocumentAttributeVideo
from telethon import functions

class TelethonHelpers(TelegramClient):
    def __init__(self, api_id, api_hash, client_name, restrict_users=False) :
        self.client = self.get_client(api_id, api_hash, client_name, init=True)
        self.restrict_users = restrict_users

    def get_client(self, api_id, api_hash, client_name, init=False):
        if init == False and self.client:
            return self.client
        else:
            client = TelegramClient.__init__(client_name, api_id, api_hash)
            client.start()
            return client

    def get_chat_id(self, entity_name=None, entity_id=None):
        dialogs = self.client.get_dialogs()
        for dialog in dialogs:
            if (entity_name and dialog.name == entity_name) \
                or (entity_id and dialog.message.peer_id == entity_id):
                return dialog

        return None

    def create_group_if_not_exist(self, entity_name, group_about, messages, avatar=None):
        chat_id = self.get_chat_id(entity_name=entity_name)

        if not chat_id:
            chat_id = self.create_group(entity_name, group_about)
            if avatar:
                upload_file_result = self.client.upload_file(file=avatar)
                try:
                    result = self.client(functions.channels.EditPhotoRequest(
                        channel=chat_id,
                        photo=upload_file_result
                    ))
                except Exception as e:
                    print(f'Error setting group avatar.\n{e}')

            for message in messages:
                if message.get('type') == 'message':
                    self.send_message(entity_name, message.get('message'))
                else:
                    self.send_file(entity_name, message.get('path'))

    def send_message(self, entity_name, message):
        try:
            dialog = self.get_chat_id(entity_name=entity_name)

            if dialog:
                response = self.client.send_message(entity=dialog, message=message)
            else:
                print(f'No chat found - {entity_name}')
        except Exception as e:
            print(f'Error while sending telegram message\n{e}')

    def send_file(self, entity_name, file, text=None):
        try:
            dialog = self.get_chat_id(entity_name=entity_name)
            if dialog:
                attributes = ()
                if '.mp4' in file:
                    attributes=(DocumentAttributeVideo(0, 0, 0),)
                response = self.client.send_file(entity=dialog, file=file, attributes=attributes, caption=text)
            else:
                print(f'No chat found - {entity_name}')
        except Exception as e:
            print(f'Error while sending telegram message\n{e}')

    def create_group(self, group_name, group_about=None):
        createdPrivateChannel = self.client(CreateChannelRequest(group_name,group_about,megagroup=True))

        newChannelID = createdPrivateChannel.__dict__["chats"][0].__dict__["id"]
        newChannelAccessHash = createdPrivateChannel.__dict__["chats"][0].__dict__["access_hash"]
        desiredPublicUsername = "myUsernameForPublicChannel"
        checkUsernameResult = self.client(CheckUsernameRequest(InputPeerChannel(channel_id=newChannelID, access_hash=newChannelAccessHash), desiredPublicUsername))
        if(checkUsernameResult==True):
            publicChannel = self.client(UpdateUsernameRequest(InputPeerChannel(channel_id=newChannelID, access_hash=newChannelAccessHash), desiredPublicUsername))
        if self.restrict_users:
            try:
                self.client.edit_permissions(entity=group_name, view_messages=True, send_games=False, send_gifs=False, send_inline=False, send_media=False, send_messages=False, send_polls=False, send_stickers=False, change_info=False)
            except Exception as e:
                print(f'Error while trying to edit group permissions.\nError: {e}')
        return newChannelID


"""
Contains the Plotsettings-view classes for the GUI without buttons etc.
Not created using a .ui file due to the fact that we want to display the widgets in a tree-view.
Might be better to create a custom model/view with widgets for each type.
"""
import logging

from PySide6 import QtCore, QtGui, QtWidgets

from mvts_analyzer.graphing.plotter.plot_wrapper import QPlotter
from mvts_analyzer.widgets import (combobox_list, datetime_range,
                                   range_slider_with_box,
                                   range_sliders_with_boxes, variable_range)
from mvts_analyzer.widgets.collapsible_groupbox import \
    CollapsibleGroupBoxLayout
from mvts_analyzer.windows.labeler_window import LabelerWindowView

#pylint: disable=invalid-name #Allow camelcase for Qt objects

log = logging.getLogger(__name__)

class PlotSettingsInnerView(QtWidgets.QTreeWidget):
	"""
	Only the settings without buttons etc.
	Not created using a .ui file due to the fact that we want to display the widgets in a tree-view.
	Might be better to create a custom model/view with widgets for each type.
	"""

	fftToggleBtnChanged = QtCore.Signal(bool)
	fftBrightnessChanged = QtCore.Signal(float)
	fftQualityChanged = QtCore.Signal(float)
	fftColumnSelectorChanged = QtCore.Signal(str)
	fftLineRangeChanged = QtCore.Signal(object) #[newleft, newright]
	fftTransparencyChanged = QtCore.Signal(float)
	fftColorMapChanged = QtCore.Signal(str)

	labelerToggleChanged = QtCore.Signal(bool)
	normalizationToggleChanged = QtCore.Signal(bool)

	plotDomainChanged = QtCore.Signal(object) #Emits [min, max] of sliders
	plotListChanged = QtCore.Signal(object)
	plottedLabelsChanged = QtCore.Signal(object)

	fontSizeChanged = QtCore.Signal(object)
	viewDomainSliderChanged = QtCore.Signal(object)
	viewDomainBoxesChanged = QtCore.Signal(object) #[str, str]

	xAxisChanged = QtCore.Signal(str)
	plotTypeChanged = QtCore.Signal(str)
	selectionGapFillMsChanged = QtCore.Signal(int)

	def __init__(self, *args, **kwargs):
		super(PlotSettingsInnerView, self).__init__(*args, **kwargs)
		self.setWindowTitle('Treeview')
		self.resize(700, 1200)
		self.setColumnWidth(0, 150)
		self.setHeaderLabels(["Name", "Value"])

		#=============== FFT ======================
		self.fft_header = QtWidgets.QTreeWidgetItem(self.invisibleRootItem())
		self.fft_header.setText(0, "Fourier")
		self.fft_header.setExpanded(False)

		self.fft_toggle = QtWidgets.QTreeWidgetItem(self.fft_header)
		self.fft_toggle.setText(0, "On/Off")
		self.fft_toggle_btn = QtWidgets.QCheckBox()
		self.fft_toggle_btn.stateChanged.connect(lambda: self.fftToggleBtnChanged.emit(self.fft_toggle_btn.isChecked()))
		self.fft_toggle.treeWidget().setItemWidget(self.fft_toggle, 1, self.fft_toggle_btn)

		self.fft_column_header = QtWidgets.QTreeWidgetItem(self.fft_header)
		self.fft_column_header.setText(0, "FFT Column")
		self.fft_column_combobox = QtWidgets.QComboBox()
		self.fft_column_combobox.currentTextChanged.connect(self.fftColumnSelectorChanged)
		self.fft_column_header.treeWidget().setItemWidget(self.fft_column_header, 1, self.fft_column_combobox)

		self.fft_lines_range_tree_item = QtWidgets.QTreeWidgetItem(self.fft_header)
		self.fft_lines_range_tree_item.setText(0, "Lines Range")
		self.fft_lines_range_slider = range_sliders_with_boxes.RangeSlidersWithBoxes()
		self.fft_lines_range_slider.rangeEdited.connect(self.fftLineRangeChanged)
		self.fft_lines_range_tree_item.treeWidget().setItemWidget(
			self.fft_lines_range_tree_item, 1, self.fft_lines_range_slider)

		self.fft_brightness = QtWidgets.QTreeWidgetItem(self.fft_header)
		self.fft_brightness_slider = range_slider_with_box.RangeSliderWithBox(text_parser=float)
		self.fft_brightness.treeWidget().setItemWidget(self.fft_brightness, 1, self.fft_brightness_slider)
		self.fft_brightness_slider.valueEdited.connect(lambda x: self.fftBrightnessChanged.emit(float(x)))
		self.fft_brightness.setText(0, "Brightness")

		self.fft_quality = QtWidgets.QTreeWidgetItem(self.fft_header)
		self.fft_quality_slider = range_slider_with_box.RangeSliderWithBox(
			text_parser=float, text_converter=lambda x: str(round(x, 2)))
		self.fft_quality.treeWidget().setItemWidget(self.fft_quality, 1, self.fft_quality_slider)
		self.fft_quality_slider.valueEdited.connect(lambda x: self.fftQualityChanged.emit(float(x)))
		self.fft_quality.setText(0, "Quality")

		self.fft_color_map = QtWidgets.QTreeWidgetItem(self.fft_header)
		self.fft_color_map_dropdown = QtWidgets.QComboBox()
		self.fft_color_map.treeWidget().setItemWidget(self.fft_color_map, 1, self.fft_color_map_dropdown)
		self.fft_color_map_dropdown.currentTextChanged.connect(self.fftColorMapChanged) #TODO: check
		self.fft_color_map.setText(0, "Color Map")

		#============== Plot domain ==============
		self.plot_domain = QtWidgets.QTreeWidgetItem(self.invisibleRootItem())
		self._plot_domain_dt_widget = datetime_range.DateTimeRange(enforce_limits=False)
		self._plot_domain_num_widget = range_sliders_with_boxes.RangeSlidersWithBoxes()

		self.plot_domain_widget = variable_range.VariableRange(self._plot_domain_num_widget, self._plot_domain_dt_widget)

		self.plot_domain.treeWidget().setItemWidget(self.plot_domain, 1, self.plot_domain_widget)
		self.plot_domain_widget.rangeEdited.connect(self.plotDomainChanged)

		self.plot_domain.setText(0, "Plot Domain")

		#============== Plotlist ============
		self.plots_header = QtWidgets.QTreeWidgetItem(self.invisibleRootItem())
		self.plots_header.setText(0, "Plot Columns")
		self.plots_header.setExpanded(True)

		self.plot_footer = QtWidgets.QTreeWidgetItem(self.plots_header)
		self.plot_selector_list = combobox_list.ComboboxList()
		self.plot_selector_list.set_widgetcount(4)
		self.plot_selector_list.selectionsChanged.connect(self.plotListChanged)
		self.plot_footer.treeWidget().setItemWidget(self.plot_footer, 1, self.plot_selector_list)

		#TODO: If we don't fold/unfold, size doesn't propagate correctly - do this in a better way
		self.plot_selector_list.widgetCountChanged.connect(lambda: self.plots_header.setExpanded(False))
		self.plot_selector_list.widgetCountChanged.connect(
			lambda: QtCore.QTimer.singleShot(1, lambda: self.plots_header.setExpanded(True)))

		#============= Plotted Labels List ===============
		self.plotted_labels_header = QtWidgets.QTreeWidgetItem(self.invisibleRootItem())
		self.plotted_labels_header.setText(0, "Label Columns")
		self.plotted_labels_header.setExpanded(True)

		self.plotted_labels_footer = QtWidgets.QTreeWidgetItem(self.plotted_labels_header)
		self.plotted_labels_selection_list = combobox_list.ComboboxList()
		self.plotted_labels_selection_list.set_widgetcount(2)
		self.plotted_labels_selection_list.selectionsChanged.connect(self.plottedLabelsChanged)
		self.plotted_labels_footer.treeWidget().setItemWidget(
			self.plotted_labels_footer, 1, self.plotted_labels_selection_list)

		#TODO: If we don't fold/unfold, size doesn't propagate correctly - do this in a better way
		self.plotted_labels_selection_list.widgetCountChanged.connect(lambda: self.plotted_labels_header.setExpanded(False))
		self.plotted_labels_selection_list.widgetCountChanged.connect(
			lambda: QtCore.QTimer.singleShot(1, lambda: self.plotted_labels_header.setExpanded(True)))

		#============= Font size ============
		self.font_size_tree_item = QtWidgets.QTreeWidgetItem(self.invisibleRootItem())
		self.font_size_selector = range_slider_with_box.RangeSliderWithBox(text_parser=int, text_converter=str)
		self.font_size_selector.valueEdited.connect(self.fontSizeChanged)
		self.font_size_tree_item.treeWidget().setItemWidget(self.font_size_tree_item, 1, self.font_size_selector)
		self.font_size_tree_item.setText(0, "Font Size")

		#============== X-axis =============
		self.x_axis = QtWidgets.QTreeWidgetItem(self.invisibleRootItem())
		self.x_axis_combobox = QtWidgets.QComboBox()
		self.x_axis_combobox.currentTextChanged.connect(self.xAxisChanged)
		self.x_axis.treeWidget().setItemWidget(self.x_axis, 1, self.x_axis_combobox)
		self.x_axis.setText(0, "X-Axis")

		#============== Plot Type (Scatter/Line) ===========
		self.plot_type_tree_item = QtWidgets.QTreeWidgetItem(self.invisibleRootItem())
		# plot_type_combobox = QtWidgets.QTreeWidgetItem(plot_type)
		self.plot_type_tree_item.setText(0, "Plot Type")
		self.plot_type_combobox = QtWidgets.QComboBox()
		self.plot_type_combobox.addItems(["Line", "Scatter"])
		self.plot_type_combobox.currentTextChanged.connect(self.plotTypeChanged)
		self.plot_type_tree_item.treeWidget().setItemWidget(self.plot_type_tree_item, 1, self.plot_type_combobox)

		#============== Coloring Type =======================
		self.plot_colors_tree_item = QtWidgets.QTreeWidgetItem(self.invisibleRootItem())
		self.plot_colors_tree_item.setText(0, "Plot Colors")

		self.plot_colors_VLayout = QtWidgets.QHBoxLayout()

		self.plot_colors_method_ComboBox = QtWidgets.QComboBox()
		self.plot_colors_method_ComboBox.addItems(["Based On Column", "Based On Labels"])
		self.plot_colors_VLayout.addWidget(self.plot_colors_method_ComboBox)

		self.plot_colors_StackedWidget = QtWidgets.QStackedWidget()
		self.plot_colors_VLayout.addWidget(self.plot_colors_StackedWidget)

		self.plot_colors_column_emptylabel =  QtWidgets.QLabel()
		self.plot_colors_StackedWidget.addWidget(self.plot_colors_column_emptylabel)

		self.plot_colors_column_ComboBox =  QtWidgets.QComboBox()
		self.plot_colors_column_ComboBox.addItems(["None"])
		self.plot_colors_StackedWidget.addWidget(self.plot_colors_column_ComboBox)

		self.plot_colors_widget = QtWidgets.QWidget()
		self.plot_colors_widget.setLayout(self.plot_colors_VLayout)

		self.plot_colors_tree_item.treeWidget().setItemWidget(self.plot_colors_tree_item, 1, self.plot_colors_widget)

		#=========== Selection ====================
		self.plot_select_gapfill_ms_tree_item = QtWidgets.QTreeWidgetItem(self.invisibleRootItem())
		self.plot_select_gapfill_ms_tree_item.setText(0, "Selection Threshold (ms)")
		self.plot_select_gapfill_slider_with_box = range_slider_with_box.RangeSliderWithBox(
			text_parser=int, text_converter=str)
		self.plot_select_gapfill_ms_tree_item.treeWidget().setItemWidget(
			self.plot_select_gapfill_ms_tree_item, 1, self.plot_select_gapfill_slider_with_box)

		#=========== Shortcuts / actions ==============

class PlotSettings(QtWidgets.QWidget):
	"""
	The outer plotsettings - includes the scrollable inside as well as the update/save buttons on the top/bottom
	"""
	fftZoomViewSignal = QtCore.Signal()
	replotViewSignal = QtCore.Signal()
	replotViewSignal = QtCore.Signal()
	updateSignal = QtCore.Signal()
	saveDfSignal = QtCore.Signal()
	toggleLabelerSignal = QtCore.Signal()

	def __init__(self, *args, **kwargs):
		super(PlotSettings, self).__init__(*args, **kwargs)
		self.settings_layout = QtWidgets.QVBoxLayout(self)

		#===============Config-tree-gui====================
		# self.config_gui = config_gui.ConfigTree(self.plot_data.config)
		# print(f"Child found: {self.config_gui.findChild(conf_base.ConfBase, 'view/xlim')}")
		self.inner_settings = PlotSettingsInnerView()

		#=============== Labeler ============
		# self.labeler_groupbox = QtWidgets.QGroupBox("Labeler")
		self.labeler_groupbox = CollapsibleGroupBoxLayout("Labeler")
		# self.labeler_groupbox.setCheckable(False)
		self.labeler_groupbox.setSizePolicy(QtWidgets.QSizePolicy.Policy.Minimum, QtWidgets.QSizePolicy.Policy.Maximum)
		self.labeler_groupbox.setContentsMargins(0, 0, 0, 0)
		self.labeler_groupbox.setStyleSheet('QGroupBox  {border: 0.5px solid #8b909a;}')

		# self.labeler_layout = QtWidgets.QHBoxLayout()

		self.labeler_window_view = LabelerWindowView()
		self.labeler_groupbox.addWidget(self.labeler_window_view)
		# self.labeler_groupbox.setLayout(self.labeler_layout)
		# self.labeler_window_view.setSizePolicy(QtWidgets.QSizePolicy.Policy.Minimum, QtWidgets.QSizePolicy.Policy.Maximum)

		#=============Update-bar===================
		self.update_button = QtWidgets.QPushButton(text="Update")
		# self.update_button.clicked.connect(self.on_update_click)
		self.update_button.clicked.connect(self.updateSignal)

		#, QtCore.QSize(), QtGui.QIcon.Mode.Normal, QtGui.QIcon.State.On)
		self.update_button.setIcon(QtGui.QIcon(":/Icons/icons/Tango Icons/actions/view-refresh.svg"))
		self.replot_view_button = QtWidgets.QPushButton(text="Replot View (X)")
		self.replot_view_button.clicked.connect(self.replotViewSignal)
		self.replot_view_button.setIcon(QtGui.QIcon(":/Icons/icons/Custom Icons/toview.svg"))

		self.fft_zoom_button = QtWidgets.QPushButton(text="Replot View (Vertical range FFT)")
		self.fft_zoom_button.clicked.connect(self.fftZoomViewSignal)
		self.fft_zoom_button.setIcon(QtGui.QIcon(":/Icons/icons/Custom Icons/fftview.svg"))

		self.sub_btn_layout = QtWidgets.QHBoxLayout()
		self.btn_layout = QtWidgets.QVBoxLayout()

		self.sub_btn_layout.addWidget(self.replot_view_button)
		self.sub_btn_layout.addWidget(self.fft_zoom_button)

		self.btn_layout.addLayout(self.sub_btn_layout)
		self.btn_layout.addWidget(self.update_button)

		#===========Adding everything to layout===========
		self.settings_layout.addWidget(self.inner_settings)
		self.settings_layout.addWidget(self.labeler_groupbox)
		self.settings_layout.addLayout(self.btn_layout)

		self.setLayout(self.settings_layout)

class GraphSettingsView(QtWidgets.QWidget):
	"""
	The graph-settings view - most outer widget that contains the plotter and the settings and divides the available
	space using a splitter
	"""
	xlimChangedSignal = QtCore.Signal()
	# def __init__(self, plot_data, matplot_plotter, *args, **kwargs):
	def __init__(self, plotter : QPlotter, *args, **kwargs):
		super(GraphSettingsView, self).__init__(*args, **kwargs)
		log.debug("initializing GraphSettings view")

		#################GUI##################
		self.main_layout = QtWidgets.QVBoxLayout(self)

		#=================Layout Splitter (plot left + settings right)=================
		self.plot_splitter = QtWidgets.QSplitter(self)
		self.plot_splitter.setOrientation(QtCore.Qt.Orientation.Horizontal)
		self.main_layout.addWidget(self.plot_splitter)

		# #================= Plot adder =============
		self.plot_adder_layout = QtWidgets.QHBoxLayout()
		self.main_layout.addLayout(self.plot_adder_layout)

		################################## Matplotlib plot layout (left) #########################################
		self.plotter = plotter
		self.plot_splitter.addWidget(self.plotter)

		#################################### Plot Settings layout  ###############################################
		self.plot_settings = PlotSettings()
		self.plot_splitter.addWidget(self.plot_settings)

		self.setLayout(self.main_layout)

		#==========Some shortcuts ===========
		self.actionCopy_Figure_To_Clipboard = QtGui.QAction(plotter)
		self.actionCopy_Figure_To_Clipboard.setObjectName("actionCopy_Figure_To_Clipboard")
		self.actionCopy_Figure_To_Clipboard.setShortcut("Ctrl+C")
		self.addAction(self.actionCopy_Figure_To_Clipboard)

		self.actionCopy_Figure_To_Clipboard.triggered.connect(lambda x : "Trying to copy figure in graphsettingsview")


# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request, FormRequest
from bs4 import BeautifulSoup
import requests
from selenium import webdriver
import pandas as pd
import json
import re
from UCrawler.items import UcrawlerItem

class NtpuSpider(scrapy.Spider):
    name = 'NTPU'
    allowed_domains = ['sea.cc.ntpu.edu.tw']
    start_urls = ['https://sea.cc.ntpu.edu.tw/pls/dev_stud/course_query_all.CHI_query_common']
    post_urls = ['https://sea.cc.ntpu.edu.tw/pls/dev_stud/course_query_all.queryByAllConditions']

    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36','referer':'http://www.ntpu.edu.tw/chinese/'}

    def start_requests(self):
        res = requests.get(self.start_urls[0], headers = self.headers)
        soup = BeautifulSoup(res.text, 'lxml')
        dept_table = soup.select('select')[1].select('option')[1:]
        latest_semester = soup.find('select', {'name':"qYear"}).select('option')[1]['value']

        for option in dept_table:
            FormData = {
            'qdept': option['value'],
            'qYear':latest_semester,
            'qTerm':'1',
            'seq1':'A',
            'seq2':'M'
            }
            yield scrapy.FormRequest(self.post_urls[0], formdata=FormData, encoding='big5')

    def parse(self, response):
        soup = BeautifulSoup(response.body, 'lxml')
        table = soup.find('table')

        if table != None:
            df_course = pd.read_html(str(table))[0]
            TableExit = True
            # duplicate rows having more than one for_depts and obligatory_tf
            del_row_idxs = []
            add_rows = []
            def duplicateDepts(row):
                if not '通' in row['必選修別']:
                    forDepts = row['應修系級'].split()
                    obligatory_tfs = row['必選修別'].split()

                    if len(forDepts) > 1:
                        del_row_idxs.append(row.name)

                        for i in range(0, len(forDepts)):
                            row['應修系級'] = forDepts[i]
                            row['必選修別'] = obligatory_tfs[i]
                            add_rows.append(dict(row))

            df_course.apply(duplicateDepts,axis=1)

            # print('preprocess len = ', len(df_course))
            add_df = pd.DataFrame(add_rows)
            df_course = pd.concat([df_course, add_df])
            df_course.drop(del_row_idxs, inplace=True)
            # print('processed len = ', len(df_course))

            # match columns
            for row in df_course.iterrows():
                # print(type(row))
                # 1.replace pd.null by None 2. transfer to str type
                def preprocess(item):
                    if pd.isnull(item):
                        return None
                    else:
                        return str(item)
                row = pd.Series(row[1]).apply(preprocess)

                courseItem = UcrawlerItem()
                ## department, obligatory_tf
                courseItem['department'] = row['開課系所'] if row['開課系所'] != None else None
                courseItem['obligatory_tf'] = True if row['必選修別'] == '必' else False

                ## for_dept, grade
                if not '通' in row['必選修別']:
                    for_dept = re.sub("\d", "",  row['應修系級']) if row['應修系級'] != None else None
                    grade = row['應修系級'].replace(for_dept, "") if row['應修系級'] != None else None
                else:
                    for_dept = ", ".join(row['應修系級'].split()) if row['應修系級'] != None else None
                    grade = None

                courseItem['for_dept'] = for_dept
                courseItem['grade'] = grade

                ## title, note
                if '備註' in row['科目名稱']:
                    title = row['科目名稱'].split("備註")[0]
                    note = row['科目名稱'].split("備註")[1].replace("：","")
                    if note == "":
                        note = None
                else:  ##basically, this will not happen
                    title = row['科目名稱']
                    note = None

                courseItem['title'] = title
                courseItem['note'] = note

                ## professor
                if row['授課教師'] != None:
                    row['授課教師'] = row['授課教師'].replace("張�睇�","張恒豪").replace("林�琝�","林恒志")
                    profs = list(row['授課教師'].split())
                else:
                    profs = []

                ##deal with error code
                #if '�'.encode() in str(row['授課教師']).encode():
                #    print(row[['科目名稱','授課教師']])
                courseItem['professor'] = profs

                ## time, location
                if row['上課時間、教室'] != None:
                    Ctime, location = self.parse_timeAndLocation(row['上課時間、教室'])
                else:
                    Ctime = None
                    location = None
                courseItem['time'] = Ctime
                courseItem['location'] = location

                ## credits
                courseItem['credits'] = float(row['學分']) if row['學分'] != None else None

                ## code, campus
                courseItem['code'] = row['課程流水號'] if row['課程流水號'] != None else None
                courseItem['campus'] = 'NTPU'
                courseItem['category'] = self.parse_catgory(courseItem['for_dept'], courseItem['obligatory_tf'])

                yield courseItem

    @staticmethod
    def parse_timeAndLocation(TimeAndLocation):
        if len(TimeAndLocation.replace('\t', ' ').split()) >= 1:
            timeAndClassroom = TimeAndLocation.replace('\t', ' ')
            items = timeAndClassroom.split()

            timeDist = []
            locationLi = []
            for item in items:
                if '週一' in item or '週二' in item or '週三' in item or '週四' in item or '週五' in item or '週六' in item or '週日' in item:
                    timeObj = {}
                    if '週一' in item:
                        timeObj['day'] = 1
                    elif '週二' in item:
                        timeObj['day'] = 2
                    elif '週三' in item:
                        timeObj['day'] = 3
                    elif '週四' in item:
                        timeObj['day'] = 4
                    elif '週五' in item:
                        timeObj['day'] = 5
                    elif '週六' in item:
                        timeObj['day'] = 6
                    elif '週日' in item:
                        timeObj['day'] = 7
    #                 re_words = re.compile(u"[\u4e00-\u9fa5]+")
                    re_words = re.compile(u"[\d]+")
                    res = list(pd.Series(re.findall(re_words, item)).apply(lambda x : int(x)))
                    timeObj['time'] = [i for i in range(res[0],res[1]+1)]
                    timeDist.append(timeObj)
                else:
                    if not '維護' in item:
                        locationLi.append(item)
            Ctime = timeDist if timeDist != [] else None
            location = locationLi if locationLi != [] else None
        else:
            Ctime = None
            location = None

        return Ctime, location

    @staticmethod
    def parse_catgory(for_dept, obligatory_tf):  ##應修系級
        if "體育" in for_dept:
            return "體育類"
        elif "通識" in for_dept:
            return "通識類"
        elif "共同" in for_dept or "軍訓" in for_dept:
            return "其他類"
        else:
            return "必修類" if obligatory_tf else "選修類"


from tkinter import *
from PIL import Image, ImageTk
from random import randint
window = Tk()
window.title("Rock paper Scissors")
window.configure(background="black")

rock_l = ImageTk.PhotoImage(Image.open("rock left.png"))
paper_l = ImageTk.PhotoImage(Image.open("paper left.png"))
scissors_l = ImageTk.PhotoImage(Image.open("scissors left.png"))

rock_r = ImageTk.PhotoImage(Image.open("rock right.png"))
paper_r = ImageTk.PhotoImage(Image.open("paper right.png"))
scissors_r = ImageTk.PhotoImage(Image.open("scissors right.png"))

label_player = Label(window, image=scissors_r)
label_computer = Label(window, image=scissors_l)
label_computer.grid(row=1, column=0)
label_player.grid(row=1, column=4)

compuer_score = Label(window, text=" 0 ", font=(
    'arial', 60, "bold"),  fg="blue")
player_score = Label(window, text=" 0 ", font=('arial', 60, "bold"), fg="blue")

compuer_score.grid(row=1, column=1)
player_score.grid(row=1, column=3)

player_indicator = Label(window, font=(
    'arial', 40, "bold"), text="PLAYER", bg="cyan", fg="blue")
computer_indicator = Label(window, font=(
    'arial', 40, "bold"), text="COMPUTER", bg="cyan", fg="blue")

computer_indicator.grid(row=0, column=1)
player_indicator.grid(row=0, column=3)

final_message = Label(window, font=('arial', 40, "bold"),
                      text="Start", bg="red", fg="white")
final_message.grid(row=3, column=2)

def msg_update(a):
    final_message['text'] = a

def Comp_update():
    final = int(compuer_score['text'])
    final += 1
    compuer_score['text'] = str(final)

def Player_update():
    final = int(player_score['text'])
    final += 1
    player_score['text'] = str(final)

def winner(p, c):
    if (p == c):
        msg_update("Its a tie")
    elif p == "rock":
        if c == "paper":
            msg_update("Computer Wins !!!")
            Comp_update()
        else:
            msg_update("Player Wins !!!")
            Player_update()
    elif p == "paper":
        if (c == "scissors"):
            msg_update("Computer Wins !!!")
            Comp_update()
        else:
            msg_update("Player Wins !!!")
            Player_update()
    elif p == "scissors":
        if c == "rock":
            msg_update("Computer Wins !!!")
            Comp_update()
        else:
            msg_update("Player Wins !!!")
            Player_update()
    else:
        pass

to_select = ["rock", "paper", "scissors"]

def choice_update(a):
    choice_comp = to_select[randint(0, 2)]
    if (choice_comp == "rock"):
        label_computer.configure(image=rock_l)
    elif (choice_comp == "paper"):
        label_computer.configure(image=paper_l)
    else:
        label_computer.configure(image=scissors_l)

    if (a == "rock"):
        label_player.configure(image=rock_r)
    elif (a == "paper"):
        label_player.configure(image=paper_r)
    else:
        label_player.configure(image=scissors_r)

    winner(a, choice_comp)

button_rock = Button(window, text="Rock", width=20, height=4, font=(
    "arial", 20, "bold"), bg="magenta", fg="black", command=lambda: choice_update("rock")).grid(row=2, column=1)
button_paper = Button(window, text="Paper", width=20, height=4, font=(
    "arial", 20, "bold"), bg="magenta", fg="black", command=lambda: choice_update("paper")).grid(row=2, column=2)
button_sci = Button(window, text="Scissors", width=20, height=4, font=(
    "arial", 20, "bold"), bg="magenta", fg="black", command=lambda: choice_update("scissors")).grid(row=2, column=3)
window.mainloop()


#!/usr/bin/env python

from __future__ import absolute_import
import visa
import logging
import serial
import useserial
from sys import platform

class GPIBDetector ( object ) :
	def __init__ ( self ) :
		logger = logging.getLogger ( u'myLogger' )
		resources = [""] * 1
		try :
			self._rm1 = visa.ResourceManager ( )
			resources = self._rm1.list_resources ( )
		except :
			logger.debug ( u"  Failed to open ni-visa" )
		try :
			if useserial.haveserial :
			    self._rm2 = visa.ResourceManager ( u'@py' )
			    resources += self._rm2.list_resources ( )
		except :
			logger.debug ( u"  Failed to open py-visa" )

		self.identifiers = {}
		logger.debug ( u"  Probing devices..." )
		for res in resources:
			logger.debug ( u"   Found %s", res )
			if not ( res.startswith ( u"ASRL" ) or res.startswith ( u"GPIB" ) ) :
				continue

			dev = None
			if res.startswith ( u"ASRL" ) and useserial.haveserial :
				logger.debug ( u"   Opening serial connection to %s", res )
				try :
					# 5000 msecs needed to catch slow devices...
					dev = self._rm2.open_resource ( res, baud_rate = 19200, data_bits = 8, timeout = 5000 )
				except :
					logger.debug ( u"   Could not open serial connection to %s", res )
			if res.startswith ( u"GPIB" ) :
				logger.debug ( u"   Opening GPIB connection to %s", res )
				try :
					dev = self._rm1.open_resource ( res )
				except :
					logger.debug ( u"   Could not open GPIB connection to %s", res )
			if not ( dev == None ) :
				idn = dev.query ( u"*IDN?" )
				logger.debug ( u"   Got device identification: %s", idn )
				self.identifiers[res] = idn
				dev.close ( )

	def get_resname_for ( self, search ) :
		for key, value in self.identifiers.items ( ) :
			if search in value :
				return key
		return None

if __name__ == u"__main__" :
	import pprint

	detector = GPIBDetector ( )

	pprint.pprint ( detector.identifiers )


import os

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.nn.functional import interpolate
from loguru import logger
from tqdm import tqdm

import wandb

from draw_concat import draw_concat
from generate_noise import generate_spatial_noise
from mario.level_utils import group_to_token, one_hot_to_ascii_level, token_to_group
from mario.tokens import TOKEN_GROUPS as MARIO_TOKEN_GROUPS
from mariokart.tokens import TOKEN_GROUPS as MARIOKART_TOKEN_GROUPS
from models import calc_gradient_penalty, save_networks

def update_noise_amplitude(z_prev, real, opt):
    """ Update the amplitude of the noise for the current scale according to the previous noise map. """
    RMSE = torch.sqrt(F.mse_loss(real, z_prev))
    return opt.noise_update * RMSE

def train_single_scale(D, G, reals, generators, noise_maps, input_from_prev_scale, noise_amplitudes, opt):
    """ Train one scale. D and G are the current discriminator and generator, reals are the scaled versions of the
    original level, generators and noise_maps contain information from previous scales and will receive information in
    this scale, input_from_previous_scale holds the noise map and images from the previous scale, noise_amplitudes hold
    the amplitudes for the noise in all the scales. opt is a namespace that holds all necessary parameters. """
    current_scale = len(generators)
    real = reals[current_scale]

    if opt.game == 'mario':
        token_group = MARIO_TOKEN_GROUPS
    else:  # if opt.game == 'mariokart':
        token_group = MARIOKART_TOKEN_GROUPS

    nzx = real.shape[2]  # Noise size x
    nzy = real.shape[3]  # Noise size y

    padsize = int(1 * opt.num_layer)  # As kernel size is always 3 currently, padsize goes up by one per layer

    if not opt.pad_with_noise:
        pad_noise = nn.ZeroPad2d(padsize)
        pad_image = nn.ZeroPad2d(padsize)
    else:
        pad_noise = nn.ReflectionPad2d(padsize)
        pad_image = nn.ReflectionPad2d(padsize)

    # setup optimizer
    optimizerD = optim.Adam(D.parameters(), lr=opt.lr_d, betas=(opt.beta1, 0.999))
    optimizerG = optim.Adam(G.parameters(), lr=opt.lr_g, betas=(opt.beta1, 0.999))
    schedulerD = torch.optim.lr_scheduler.MultiStepLR(optimizer=optimizerD, milestones=[1600, 2500], gamma=opt.gamma)
    schedulerG = torch.optim.lr_scheduler.MultiStepLR(optimizer=optimizerG, milestones=[1600, 2500], gamma=opt.gamma)

    if current_scale == 0:  # Generate new noise
        z_opt = generate_spatial_noise([1, opt.nc_current, nzx, nzy], device=opt.device)
        z_opt = pad_noise(z_opt)
    else:  # Add noise to previous output
        z_opt = torch.zeros([1, opt.nc_current, nzx, nzy]).to(opt.device)
        z_opt = pad_noise(z_opt)

    logger.info("Training at scale {}", current_scale)
    for epoch in tqdm(range(opt.niter)):
        step = current_scale * opt.niter + epoch
        noise_ = generate_spatial_noise([1, opt.nc_current, nzx, nzy], device=opt.device)
        noise_ = pad_noise(noise_)

        ############################
        # (1) Update D network: maximize D(x) + D(G(z))
        ###########################
        for j in range(opt.Dsteps):
            # train with real
            D.zero_grad()

            output = D(real).to(opt.device)

            errD_real = -output.mean()
            errD_real.backward(retain_graph=True)

            # train with fake
            if (j == 0) & (epoch == 0):
                if current_scale == 0:  # If we are in the lowest scale, noise is generated from scratch
                    prev = torch.zeros(1, opt.nc_current, nzx, nzy).to(opt.device)
                    input_from_prev_scale = prev
                    prev = pad_image(prev)
                    z_prev = torch.zeros(1, opt.nc_current, nzx, nzy).to(opt.device)
                    z_prev = pad_noise(z_prev)
                    opt.noise_amp = 1
                else:  # First step in NOT the lowest scale
                    # We need to adapt our inputs from the previous scale and add noise to it
                    prev = draw_concat(generators, noise_maps, reals, noise_amplitudes, input_from_prev_scale,
                                       "rand", pad_noise, pad_image, opt)

                    # For the seeding experiment, we need to transform from token_groups to the actual token
                    if current_scale == (opt.token_insert + 1):
                        prev = group_to_token(prev, opt.token_list, token_group)

                    prev = interpolate(prev, real.shape[-2:], mode="bilinear", align_corners=False)
                    prev = pad_image(prev)
                    z_prev = draw_concat(generators, noise_maps, reals, noise_amplitudes, input_from_prev_scale,
                                         "rec", pad_noise, pad_image, opt)

                    # For the seeding experiment, we need to transform from token_groups to the actual token
                    if current_scale == (opt.token_insert + 1):
                        z_prev = group_to_token(z_prev, opt.token_list, token_group)

                    z_prev = interpolate(z_prev, real.shape[-2:], mode="bilinear", align_corners=False)
                    opt.noise_amp = update_noise_amplitude(z_prev, real, opt)
                    z_prev = pad_image(z_prev)
            else:  # Any other step
                prev = draw_concat(generators, noise_maps, reals, noise_amplitudes, input_from_prev_scale,
                                   "rand", pad_noise, pad_image, opt)

                # For the seeding experiment, we need to transform from token_groups to the actual token
                if current_scale == (opt.token_insert + 1):
                    prev = group_to_token(prev, opt.token_list, token_group)

                prev = interpolate(prev, real.shape[-2:], mode="bilinear", align_corners=False)
                prev = pad_image(prev)

            # After creating our correct noise input, we feed it to the generator:
            noise = opt.noise_amp * noise_ + prev
            fake = G(noise.detach(), prev, temperature=1 if current_scale != opt.token_insert else 1)

            # Then run the result through the discriminator
            output = D(fake.detach())
            errD_fake = output.mean()

            # Backpropagation
            errD_fake.backward(retain_graph=False)

            # Gradient Penalty
            gradient_penalty = calc_gradient_penalty(D, real, fake, opt.lambda_grad, opt.device)
            gradient_penalty.backward(retain_graph=False)

            # Logging:
            if step % 10 == 0:
                wandb.log({f"D(G(z))@{current_scale}": errD_fake.item(),
                           f"D(x)@{current_scale}": -errD_real.item(),
                           f"gradient_penalty@{current_scale}": gradient_penalty.item()
                           },
                          step=step, sync=False)
            optimizerD.step()

        ############################
        # (2) Update G network: maximize D(G(z))
        ###########################

        for j in range(opt.Gsteps):
            G.zero_grad()
            fake = G(noise.detach(), prev.detach(), temperature=1 if current_scale != opt.token_insert else 1)
            output = D(fake)

            errG = -output.mean()
            errG.backward(retain_graph=False)
            if opt.alpha != 0:  # i. e. we are trying to find an exact recreation of our input in the lat space
                Z_opt = opt.noise_amp * z_opt + z_prev
                G_rec = G(Z_opt.detach(), z_prev, temperature=1 if current_scale != opt.token_insert else 1)
                rec_loss = opt.alpha * F.mse_loss(G_rec, real)
                rec_loss.backward(retain_graph=False)  # TODO: Check for unexpected argument retain_graph=True
                rec_loss = rec_loss.detach()
            else:  # We are not trying to find an exact recreation
                rec_loss = torch.zeros([])
                Z_opt = z_opt

            optimizerG.step()

        # More Logging:
        if step % 10 == 0:
            wandb.log({f"noise_amplitude@{current_scale}": opt.noise_amp,
                       f"rec_loss@{current_scale}": rec_loss.item()},
                      step=step, sync=False, commit=True)

        # Rendering and logging images of levels
        if epoch % 500 == 0 or epoch == (opt.niter - 1):
            if opt.token_insert >= 0 and opt.nc_current == len(token_group):
                token_list = [list(group.keys())[0] for group in token_group]
            else:
                token_list = opt.token_list

            img = opt.ImgGen.render(one_hot_to_ascii_level(fake.detach(), token_list))
            img2 = opt.ImgGen.render(one_hot_to_ascii_level(
                G(Z_opt.detach(), z_prev, temperature=1 if current_scale != opt.token_insert else 1).detach(),
                token_list))
            real_scaled = one_hot_to_ascii_level(real.detach(), token_list)
            img3 = opt.ImgGen.render(real_scaled)
            wandb.log({f"G(z)@{current_scale}": wandb.Image(img),
                       f"G(z_opt)@{current_scale}": wandb.Image(img2),
                       f"real@{current_scale}": wandb.Image(img3)},
                      sync=False, commit=False)

            real_scaled_path = os.path.join(wandb.run.dir, f"real@{current_scale}.txt")
            with open(real_scaled_path, "w") as f:
                f.writelines(real_scaled)
            wandb.save(real_scaled_path)

        # Learning Rate scheduler step
        schedulerD.step()
        schedulerG.step()

    # Save networks
    torch.save(z_opt, "%s/z_opt.pth" % opt.outf)
    save_networks(G, D, z_opt, opt)
    wandb.save(opt.outf)
    return z_opt, input_from_prev_scale, G


# %%
import pandas as pd

# https://eresearch.fidelity.com/eresearch/markets_sectors/sectors/sectors_in_market.jhtml
market_cap_by_sector = pd.Series(
    {
        "Communication Services": 5.37,
        "Consumer Discretionary": 7.5,
        "Consumer Staples": 4.12,
        "Energy": 1.76,
        "Financials": 5.95,
        "Health Care": 6.89,
        "Industrials": 4.46,
        "IT": 12.31,
        "Materials": 2.1,
        "Real Estate": 1.35,
        "Utilities": 1.49,
    }
)

num_stock_by_sector = (30 * (market_cap_by_sector / market_cap_by_sector.sum())).round()
print(num_stock_by_sector)
# %%

import yfinance as yf

stock_list = [
    "T",
    "EA",
    "VZ",
    "BBY",
    "KMX",
    "CMG",
    "F",
    "KO",
    "WMT",
    "CVX",
    "JPM",
    "MS",
    "C",
    "JNJ",
    "MRK",
    "PFE",
    "CVS",
    "AAL",
    "FDX",
    "UPS",
    "AAPL",
    "AMD",
    "MSFT",
    "NVDA",
    "ORCL",
    "QCOM",
    "INTC",
    "EMN",
    "AVB",
    "SO",
]

stock_close = yf.download(stock_list, start="2010-01-01", end="2020-01-01")["Close"]
# print(stock_data.index[stock_data["Close"].isna().any(axis=1)]) # market closed on 2012-10-29
stock_close = stock_close.dropna(axis=0, how="all")

stock_ret = (stock_close - stock_close.shift()) / stock_close.shift()
stock_ret = stock_ret.dropna()
stock_cov = stock_ret.cov()
stock_corr = stock_ret.corr()
# %%
import os

newpath = os.getcwd() + "\\output"
if not os.path.exists(newpath):
    os.makedirs(newpath)

filename = newpath + "\\data.xlsx"
with pd.ExcelWriter(filename) as writer:
    stock_close.to_excel(writer, sheet_name="stock_close")
    stock_ret.to_excel(writer, sheet_name="stock_ret")
    stock_cov.to_excel(writer, sheet_name="stock_cov")
    stock_corr.to_excel(writer, sheet_name="stock_corr")


from flask import Flask
from flask import request
from faker import Faker
import csv

app = Flask(__name__)

@app.route('/')
def hello_world():
    return '
Hello, World
'

@app.route('/requirements/')
def get_requirements():
    with open('requirements.txt') as file:
        text = ''
        line = file.readline()

        while line != '':
            text += (line.strip()) + ' \n'
            line = file.readline()
        return text

@app.route('/generate-users/')
def generate_users():
    dict_name_email = {}
    fake = Faker()
    amount = request.args.get('amount', 100, type=int)
    for i in range(amount):
        dict_name_email[fake.name()] = fake.email()
    return dict_name_email

@app.route('/mean/')
def get_avg_value():
    list_index, list_height, list_weight = [], [], []
    with open('hw.csv') as file:
        reader = csv.reader(file)
        for row in reader:
            if row:
                list_index.append(row[0])
                list_height.append(row[1])
                list_weight.append(row[2])
        index = max(list_index[1:], key=int)
        avg_hight = (sum([float(i) for i in list_height[1:]]) * 2.54) / int(index)
        avg_weight = (sum([float(i) for i in list_weight[1:]]) * 0.454) / int(index)
    return f'Average height is {avg_hight} cm, average weight is {avg_weight} kg'

if __name__ == '__main__':
    app.run(debug=True)


import requests

bus_addr = '<bus_shelter_address>'

host = '<om2m_server_address>'
port = '8080'

ae_name = 'bus_shelter'
headers = {
    'X-M2M-Origin': '<om2m-id>:<om2m-passwd>',
    'Content-Type': '',
    'Cache-Control': 'no-cache',
}
containers = ['led1', 'led2', 'awning', 'inverter']

def init_script():
    # create ae
    headers['Content-Type'] = 'application/json;ty=2'
    data = {
        'm2m:ae': {
            'api': 'bus-shelter',
            'rn': ae_name,
            'lbl' : [],
            'rr': True
        }
    }
    res = requests.post(f'http://{host}:{port}/~/in-cse', json=data, headers=headers)
    print('[CREATE AE]', res.status_code)

    if res.status_code != 201:
        return 0
    ae_addr = res.json()['m2m:ae']['ri']

    for container in containers:
        # create container
        headers['Content-Type'] = 'application/json;ty=3'
        data = {
            'm2m:cnt': {
                'rn': container
            }
        }
        res = requests.post(f'http://{host}:{port}/~{ae_addr}', json=data, headers=headers)
        print(f'[CREATE {container} CONTAINER]', res.status_code)

        if res.status_code != 201:
            return ae_addr
        cnt_addr = res.json()['m2m:cnt']['ri']

        # create content instance
        headers['Content-Type'] = 'application/json;ty=4'
        data = {
            'm2m:cin': {
                'cnf': f'{container}-status',
                'con': 0
            }
        }
        res = requests.post(f'http://{host}:{port}/~{cnt_addr}', json=data, headers=headers)
        print(f'[CREATE {container} CONTENT INSTANCE]', res.status_code)
        if res.status_code != 201:
            return ae_addr

        # create subscription
        headers['Content-Type'] = 'application/json;ty=23'
        data = {
            'm2m:sub': {
                'rn': f'{container}-sub',
                'nu': [f'http://{bus_addr}:3002/{container}'],
                'nct': 2
            }
        }
        res = requests.post(f'http://{host}:{port}/~{cnt_addr}', json=data, headers=headers)
        print(f'[CREATE {container} SUBSCRIPTION]', res.status_code)
        if res.status_code != 201:
            return

    print('Successfully initialize OM2M server')

if __name__ == "__main__":
    ret = init_script()

    if ret is not 0:
        requests.delete(f'http://{host}:{port}/~{ret}', headers={'X-M2M-Origin': 'admin:admin', 'Cache-Control': 'no-cache'})


from typing import Union
from enum import Enum

from fastapi import FastAPI, Request

app = FastAPI()

# https://fastapi.tiangolo.com/es/advanced/custom-response/#redirectresponse
from fastapi.responses import RedirectResponse
@app.get("/")
def redirect_to_home(request: Request):
    #return {"raw_url": request.url._url}
    return RedirectResponse(request.url._url+"home")

@app.get("/hello")
def read_root():
    return {"Hello": "World"}

@app.get("/integer/{int_id}")
def read_item(int_id: int, q: Union[str, None] = None):
    return {"int_id": int_id, "q": q}

# https://fastapi.tiangolo.com/tutorial/path-params/#path-convertor
@app.get("/file/path/{file_path:path}")
async def show_file_path(file_path: str):
    return {"file_path": file_path}

# https://fastapi.tiangolo.com/advanced/custom-response/#fileresponse -> https://stackoverflow.com/a/62939520
from fastapi.responses import FileResponse
@app.get("/file/{file_path:path}")
async def respond_file_content(file_path: str):
    return FileResponse("app/"+file_path)

# https://fastapi.tiangolo.com/tutorial/path-params/#return-enumeration-members
class ModelName(str, Enum):
    bonus = "bonus"
    user = "user"
    group = "group"

@app.get("/models/{model_name}")
async def get_model(model_name: ModelName):
    if model_name is ModelName.bonus:
        return {"model_name": model_name, "message": "Seems like you got an BONus!"}

    if model_name.value == "user":
        return {"model_name": model_name, "message": "You want's to get the User?"}

    return {"model_name": model_name, "message": "That is the model you are looking for?"}

# https://fastapi.tiangolo.com/advanced/custom-response/#html-response
from fastapi.responses import HTMLResponse
@app.get("/home/", response_class=HTMLResponse)
async def read_items():
    return """

            <title>Some HTML in here</title>

Look, some HTML Markup!

    """


import regex

from typing import Dict, List, TypedDict
from aocd import get_data

NUMBERS_RE = regex.compile(r" (\d+)")

class RulesWithEntries(TypedDict):
    rules: Dict[str, str]
    entries: List[str]

def parse(raw: str) -> RulesWithEntries:
    """
    Precompute regex, `47: 7 10 | 5 11 \\n aaaaaaaa`
    -> `{'rules': {'47': '(?:(?:7 10 )|(?: 5 11))'} , 'entries': 'aaaaaaaa'}`.
    """
    raw_rules, raw_entries = raw.strip().split("\n\n")
    rules = {}
    for rule in raw_rules.split("\n"):
        key, under_rules = rule.split(":", 1)
        if under_rules[2] in "ab":
            rules[key] = under_rules[2]
        elif rule.find("|") != -1:
            rules[key] = f"(?:(?:{under_rules.replace(' |', ')|(?:')}))"
        else:
            rules[key] = f"(?:{under_rules})"
    return {"rules": rules, "entries": raw_entries.split("\n")}

def run(rules: Dict[str, str], entries: List[str]):
    """Compute the regex and count how many entry match it."""
    rule_0 = rules["0"]
    match = NUMBERS_RE.search(rule_0)
    while match:
        rule_0 = NUMBERS_RE.sub(rules[match[1]], rule_0, count=1)
        match = NUMBERS_RE.search(rule_0)
    rule_0_re = regex.compile(rule_0)
    return sum(1 for line in entries if rule_0_re.fullmatch(line))

def part_one(rules_entries: RulesWithEntries):
    """Valid word number with regex."""
    return run(rules_entries["rules"], rules_entries["entries"])

def part_two(rules_entries: RulesWithEntries):
    """Valid word number with a recursive patched regex."""
    patched_rules = dict(rules_entries["rules"])
    patched_rules.update({"8": " 42+?",
                          "11": "(?P<rec>(?: 42 31)|(?: 42(?&rec) 31))"})
    return run(patched_rules, rules_entries["entries"])

if __name__ == "__main__":
    print("--- Day 19: Monster Messages ---")
    puzzle = parse(get_data(day=19, year=2020))
    print("Part One:", part_one(puzzle))
    print("Part Two:", part_two(puzzle))


import os
import pathlib
import tempfile
import subprocess

import pytest

from o2g.osm.exporter import TransitDataExporter
from o2g.gtfs.gtfs_writer import GTFSWriter
from o2g.gtfs import gtfs_dummy

@pytest.fixture
def transit_data(request):
    root_dir = pathlib.Path(__file__).parents[2]
    filepath = root_dir.joinpath('resources', 'osm', 'freiburg.osm.bz2')
    tde = TransitDataExporter(str(filepath))
    tde.process()
    return tde

@pytest.fixture
def dummy_transit_data(transit_data):
    return \
        gtfs_dummy.create_dummy_data(transit_data.routes,
                                     transit_data.stops)

@pytest.fixture
def gtfs_writer(transit_data):
    w = GTFSWriter()

    w.add_agencies(transit_data.agencies)
    w.add_stops(transit_data.stops)
    w.add_routes(transit_data.routes)

    return w

@pytest.fixture
def dummy_gtfs_writer(transit_data, dummy_transit_data):
    w = GTFSWriter()

    # Patch agencies to pass transitfeed check
    patched_agencies = gtfs_dummy.patch_agencies(transit_data.agencies)

    w.add_agencies(patched_agencies)
    w.add_stops(transit_data.stops)
    w.add_routes(transit_data.routes)
    w.add_shapes(transit_data.shapes)

    w.add_trips(dummy_transit_data.trips)
    w.add_stop_times(dummy_transit_data.stop_times)
    w.add_calendar(dummy_transit_data.calendar)

    return w

@pytest.fixture
def dummy_zipfeed(dummy_gtfs_writer):
    filename = '{}.zip'.format(tempfile.mktemp())
    print('Writing GTFS feed to %s' % filename)
    dummy_gtfs_writer.write_zipped(filename)

    return filename

def test_write_zipped(gtfs_writer):
    filename = tempfile.mktemp()
    print('Writing GTFS feed to %s' % filename)
    gtfs_writer.write_zipped(filename)

    assert os.path.exists(filename)

def test_duplicate_trips(dummy_transit_data):
    dups = []
    for trip in dummy_transit_data.trips:
        assert trip['trip_id'] not in dups
        dups.append(trip['trip_id'])

def test_validation(dummy_zipfeed):
    """Run transitfeed over the generated feed."""
    # transitfeed is a python2 application. So we need to run it outside
    # python3 process. Moreover, it is not available in pip repository.
    # Therefore we have to clone it from git and eventually run it in a
    # process of its own. Finally we parse the standard output and look
    # for errors. We ignore the warnings for now.
    if not os.path.exists('transitfeed'):
        subprocess.check_call(
            ['git', 'clone', '-b', '1.2.16', '--single-branch',
             'https://github.com/google/transitfeed'])

    assert os.path.exists('transitfeed/feedvalidator.py')

    p = subprocess.Popen(['python2.7', 'transitfeed/feedvalidator.py', '-n',
                         dummy_zipfeed],
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    out, err = p.communicate()

    print("Google Transitfeed's output:\n{}".format(out.decode('utf8')))

    assert 'error' not in out.decode('utf8')
    assert 'errors' not in out.decode('utf8')

def test_shape_id_in_trips(transit_data, dummy_transit_data):
    shape_ids = [shape.shape_id for shape in transit_data.shapes]
    for trip in dummy_transit_data.trips:
        assert trip['shape_id'] in shape_ids


from collections import abc
import functools
import inspect
import sys

from keystoneclient import exceptions as keystone_exceptions
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import excutils
from oslo_utils import uuidutils
import pecan
import wsme

from solum.i18n import _

LOG = logging.getLogger(__name__)

exc_log_opts = [
    cfg.BoolOpt('fatal_exception_format_errors',
                default=False,
                help='make exception message format errors fatal')
]

def list_opts():
    yield None, exc_log_opts

CONF = cfg.CONF
CONF.register_opts(exc_log_opts)

def wrap_exception(notifier=None, publisher_id=None, event_type=None,
                   level=None):
    """This decorator wraps a method to catch any exceptions.

    It logs the exception as well as optionally sending
    it to the notification system.
    """
    def inner(f):
        def wrapped(self, context, *args, **kw):
            # Don't store self or context in the payload, it now seems to
            # contain confidential information.
            try:
                return f(self, context, *args, **kw)
            except Exception as e:
                with excutils.save_and_reraise_exception():
                    if notifier:
                        call_dict = inspect.getcallargs(f, *args, **kw)
                        payload = dict(exception=e,
                                       private=dict(args=call_dict)
                                       )

                        # Use a temp vars so we don't shadow
                        # our outer definitions.
                        temp_level = level
                        if not temp_level:
                            temp_level = notifier.ERROR

                        temp_type = event_type
                        if not temp_type:
                            # If f has multiple decorators, they must use
                            # functools.wraps to ensure the name is
                            # propagated.
                            temp_type = f.__name__

                        notifier.notify(context, publisher_id, temp_type,
                                        temp_level, payload)

        return functools.wraps(f)(wrapped)
    return inner

OBFUSCATED_MSG = _('Your request could not be handled '
                   'because of a problem in the server. '
                   'Error Correlation id is: %s')

def wrap_controller_exception(func, func_server_error, func_client_error):
    """This decorator wraps controllers methods to handle exceptions:

    - if an unhandled Exception or a SolumException with an error code >=500
    is catched, raise a http 5xx ClientSideError and correlates it with a log
    message

    - if a SolumException is catched and its error code is <500, raise a http
    4xx and logs the exception in debug mode

    """
    @functools.wraps(func)
    def wrapped(*args, **kw):
        try:
            return func(*args, **kw)
        except Exception as excp:
            LOG.error(excp)
            http_error_code = 500
            if hasattr(excp, 'code'):
                http_error_code = excp.code

            if http_error_code >= 500:
                # log the error message with its associated
                # correlation id
                log_correlation_id = uuidutils.generate_uuid()
                LOG.error("%s:%s", log_correlation_id, str(excp))
                # raise a client error with an obfuscated message
                func_server_error(log_correlation_id, http_error_code)
            else:
                # raise a client error the original message
                func_client_error(excp, http_error_code)
    return wrapped

def wrap_wsme_controller_exception(func):
    """This decorator wraps wsme controllers to handle exceptions."""
    def _func_server_error(log_correlation_id, status_code):
        raise wsme.exc.ClientSideError(
            str(OBFUSCATED_MSG % log_correlation_id), status_code)

    def _func_client_error(excp, status_code):
        raise wsme.exc.ClientSideError(str(excp), status_code)

    return wrap_controller_exception(func,
                                     _func_server_error,
                                     _func_client_error)

def wrap_pecan_controller_exception(func):
    """This decorator wraps pecan controllers to handle exceptions."""
    def _func_server_error(log_correlation_id, status_code):
        pecan.response.status = status_code
        pecan.response.text = str(OBFUSCATED_MSG % log_correlation_id)
        # message body for errors is just a plain text message
        # The following code is functionally equivalent to calling:
        #
        #     pecan.override_template(None, "text/plain")
        #
        # We do it this way to work around a bug in our unit-test framework
        # in which the mocked request object isn't properly mocked in the pecan
        # core module (gilbert.plz@oracle.com)
        pecan.request.pecan['override_template'] = None
        pecan.request.pecan['override_content_type'] = 'text/plain'

    def _func_client_error(excp, status_code):
        pecan.response.status = status_code
        pecan.response.text = str(excp)

        # The following code is functionally equivalent to calling:
        #
        #     pecan.override_template(None, "text/plain")
        #
        # We do it this way to work around a bug in our unit-test framework
        # in which the mocked request object isn't properly mocked in the pecan
        # core module (gilbert.plz@oracle.com)
        pecan.request.pecan['override_template'] = None
        pecan.request.pecan['override_content_type'] = 'text/plain'

    return wrap_controller_exception(func,
                                     _func_server_error,
                                     _func_client_error)

def wrap_wsme_pecan_controller_exception(func):
    """Error handling for controllers decorated with wsmeext.pecan.wsexpose:

    Controllers wrapped with wsme_pecan.wsexpose don't throw
    exceptions but handle them internally. We need to intercept
    the response and mask potentially sensitive information.
    """

    @functools.wraps(func)
    def wrapped(*args, **kw):
        ret = func(*args, **kw)
        ismapping = isinstance(ret, abc.Mapping)
        if (pecan.response.status_code >= 500 and ismapping):

            log_correlation_id = uuidutils.generate_uuid()
            LOG.error("%s:%s", log_correlation_id, ret.get("faultstring",
                                                           "Unknown Error"))
            ret['faultstring'] = str(OBFUSCATED_MSG % log_correlation_id)
        return ret

    return wrapped

def wrap_keystone_exception(func):
    """Wrap keystone exceptions and throw Solum specific exceptions."""
    @functools.wraps(func)
    def wrapped(*args, **kw):
        try:
            return func(*args, **kw)
        except keystone_exceptions.AuthorizationFailure:
            raise AuthorizationFailure(
                client=func.__name__, message="reason: %s" % sys.exc_info()[1])
        except keystone_exceptions.ClientException:
            raise AuthorizationFailure(
                client=func.__name__,
                message="unexpected keystone client error occurred: %s"
                        % sys.exc_info()[1])
    return wrapped

class SolumException(Exception):
    """Base Solum Exception

    To correctly use this class, inherit from it and define
    a 'msg_fmt' property. That msg_fmt will get printf'd
    with the keyword arguments provided to the constructor.

    """
    message = _("An unknown exception occurred.")
    code = 500

    def __init__(self, **kwargs):
        self.kwargs = kwargs

        if CONF.fatal_exception_format_errors:
            assert isinstance(self.msg_fmt, str)

        try:
            self.message = self.msg_fmt % kwargs
        except KeyError:
            # kwargs doesn't match a variable in the message
            # log the issue and the kwargs
            LOG.exception(_('Exception in string format operation'),
                          extra=dict(
                              private=dict(
                                  msg=self.msg_fmt,
                                  args=kwargs
                                  )
                              )
                          )

            if CONF.fatal_exception_format_errors:
                raise

    def __str__(self):
        return self.message

class ResourceLimitExceeded(SolumException):
    msg_fmt = _("Resource limit exceeded. Reason: %(reason)s")

class BadRequest(SolumException):
    msg_fmt = _("The request is malformed. Reason: %(reason)s")
    code = 400

class ObjectNotFound(SolumException):
    msg_fmt = _("The %(name)s %(id)s could not be found.")

class ObjectNotUnique(SolumException):
    msg_fmt = _("The %(name)s already exists.")

class RequestForbidden(SolumException):
    msg_fmt = _("The request is forbidden. Reason: %(reason)s")
    code = 403

class ResourceNotFound(ObjectNotFound):
    msg_fmt = _("The %(name)s resource %(id)s could not be found.")
    code = 404

class ResourceExists(ObjectNotUnique):
    msg_fmt = _("The %(name)s resource already exists.")
    code = 409

class ResourceStillReferenced(SolumException):
    msg_fmt = _("The %(name)s resource cannot be deleted because one or more"
                " resources reference it.")
    code = 409

class UnsupportedMediaType(SolumException):
    msg_fmt = _("\'%(name)s\' is not a supported media type for the %(method)s"
                " method of this resource")
    code = 415

class Unprocessable(SolumException):
    msg_fmt = _("Server is incapable of processing the specified request.")
    code = 422

class PlanStillReferenced(ResourceStillReferenced):
    msg_fmt = _("Plan %(name)s cannot be deleted because one or more"
                " Assemblies reference it.")

class LPStillReferenced(ResourceStillReferenced):
    msg_fmt = _("Languagepack %(name)s cannot be deleted because one or more"
                " applications reference it.")

class NotImplemented(SolumException):
    msg_fmt = _("The requested operation is not implemented.")
    code = 501

class AuthorizationFailure(SolumException):
    msg_fmt = _("%(client)s connection failed. %(message)s")

class InvalidObjectSizeError(Exception):
    msg_fmt = _("Invalid object size.")

class MaxRetryReached(Exception):
    msg_fmt = _("Maximum retries has been reached.")


import shutil

import pydicom
import os
import numpy as np
import json

def apply_boolean_mask(array, array_criterion, criterion, value):
    array.copy()
    if array.shape != array_criterion.shape:
        print("Размеры не совпадают")
        return
    if len(array.shape) == 2:
        for i in range(len(array)):
            for j in range(len(array[i])):
                if criterion(array_criterion[i][j]):
                    array[i][j] = value
    elif len(array.shape) == 3:
        for i in range(len(array)):
            for j in range(len(array[i])):
                for k in range(len(array[i][j])):
                    if criterion(array_criterion[i][j][k]):
                        array[i][j][k] = value
    return array

def tag2int(x):
    # get x[0] as in int is x is a 'pydicom.multival.MultiValue', otherwise get int(x)
    if isinstance(x, pydicom.multival.MultiValue):
        x = x[0]
    x = str(x)
    x = x.split(',')[0]
    return int(float(x))

def data2tags(data):
    windowing = {}
    try:
        windowing['window_center'] = tag2int(data[('0028', '1050')].value)  # window center
        windowing['window_width'] = tag2int(data[('0028', '1051')].value)  # window width
    except KeyError:
        windowing['window_center'] = 2048
        windowing['window_width'] = 4096
    try:
        windowing['intercept'] = tag2int(data[('0028', '1052')].value)  # window center
        windowing['slope'] = tag2int(data[('0028', '1053')].value)  # window width
    except KeyError:
        windowing['intercept'] = 0
        windowing['slope'] = 1
    try:
        windowing['image_mode'] = str(data[('0028', '0004')].value)  # monochrome1 or not
    except KeyError:
        pass
    return windowing

def reformat_survey(src_path):
    data, scan = read_survey(src_path)
    tags = data2tags(data)
    scan = np.array(scan, dtype=np.uint16).tobytes()
    return scan, tags

def read_survey(path):
    survey = pydicom.dcmread(path)
    scan = survey.pixel_array
    return survey, scan

def create_empty_json_files(dicom_paths):
    for dicom_path in dicom_paths:
        json_path = dicom_path.replace('.dcm', '.json')
        with open(json_path, 'w') as f:
            json.dump({"contours": [],
                       "points": [],
                       "rulers": [],
                       "polygons": [[]]}, f)

def get_paths(dst_folder, web_path):
    dicom_file_names = os.listdir(dst_folder)
    dicom_file_names = list(filter(lambda x: x.split('.')[-1] == 'dcm', dicom_file_names))
    dicom_file_paths = ["/".join([web_path, dicom_file_name]) for dicom_file_name in dicom_file_names]
    dicom_file_paths.sort()
    json_file_paths = [x.replace(".dcm", ".json") for x in dicom_file_paths]
    return dicom_file_paths, json_file_paths

def rename_dicoms(paths_to_dicoms):
    new_paths = []
    for path in paths_to_dicoms:
        single_slice = pydicom.dcmread(path)
        number = single_slice.InstanceNumber - 1
        new_path = path.split('/')[:-1]
        new_path.append(str(number).rjust(4, '0') + '.dcm')
        new_path = '/'.join(new_path)
        os.rename(path, new_path)
        new_paths.append(new_path)
    return new_paths