"""
Data Transform Plugin
"""

import _pickle as cPickle
from datetime import datetime, timedelta
from difflib import SequenceMatcher
import logging
import os
import re

from bson.objectid import ObjectId
import psycopg2
from pymongo import MongoClient

from airflow.models import BaseOperator
from airflow.plugins_manager import AirflowPlugin

class NRSRTransformOperator(BaseOperator):
    """
    Data Transform
    """

    def __init__(self, data_type, period, daily,
                 postgres_url, mongo_settings, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.data_type = data_type
        self.period = period
        self.daily = daily

        mongo_client = MongoClient(mongo_settings['uri'])
        mongo_db = mongo_client[mongo_settings['db']]
        self.mongo_col = mongo_db[mongo_settings['col']]
        self.mongo_outcol = mongo_db[mongo_settings['outcol']]
        self.postgres_url = postgres_url

    def _fields_to_dict(self, fields):
        """
        Convert list of fields to mongodb field dict
        """
        return {x: 1 for x in fields}

    def _insert_documents(self, documents, remove=[]):
        """
        Insert into MongoDB
        """

        if remove:
            self.mongo_outcol.remove({'type': {'$in': remove}})

        self.mongo_outcol.insert_many(documents)

    def _get_documents(self, fields_dict, *aggregation, unwind=None, projection=None):
        """
        Get MongoDB Cursor
        """

        filter_dict = {'type': self.data_type}
        if self.period:
            filter_dict['period_num'] = self.period
        if self.daily and self.data_type not in ['daily_club']:
            now = datetime.utcnow() - timedelta(days=30)
            filter_dict['_id'] = {'$gte': ObjectId.from_datetime(now)}

        if unwind and projection:
            docs = self.mongo_col.aggregate([
                {'$match': filter_dict},
                unwind,
                projection
            ])
        elif aggregation:
            aggregation = [{'$match': filter_dict}] + list(aggregation)
            docs = self.mongo_col.aggregate(aggregation)
        else:
            docs = self.mongo_col.find(filter_dict, fields_dict)
        return docs

    def _copy_doc(self, document):
        return cPickle.loads(cPickle.dumps(document, -1))

    def _get_wanted_keys(self, document, keys):
        """Reduce dictionary to wanted keys only"""
        return {k: v for k, v in document.items() if k in keys}

    def transform_members(self):
        """
        Transform MPs
        """

        fields_list = [
            'external_id',
            'forename',
            'surname',
            # 'title',
            'stood_for_party',
            # 'born',
            # 'nationality',
            # 'residence',
            # 'county',
            # 'email',
            'image_urls',
            'period_num',
            'url',
            'memberships',
            'type'
        ]
        fields_dict = self._fields_to_dict(fields_list)

        new_docs = []

        residence_replacements = {
            'Tvrdošín-Medvedzie': 'Medvedzie',
            'Kostolná-Záriečie': 'Kostolná - Záriečie',
            'Ivánka pri Dunaji': 'Ivanka pri Dunaji',
            'Nový Život - Eliášovce': 'Eliášovce',
            'Michalovce - Čečehov': 'Čečehov',
            'Hnúšťa - Likier': 'Likier'
        }
        county_replacements = {
            'Trenčín': 'Trenčiansky',
            'Nitra': 'Nitriansky',
        }
        nationality_replacements = {
            'slovenská': 'slovak',
            'maďarská': 'hungarian',
            'rómska': 'romani',
            'rusínska': 'rusyn',
            'ruská': 'russian',
            'česká': 'czech',
            'ukrajinská': 'ukrainian',
            '': 'unknown'
        }

        # pg_conn = psycopg2.connect(self.postgres_url)
        # pg_cursor = pg_conn.cursor()

        # wanted_keys = [
        #     'external_id', 'period_num', 'forename', 'surname', 'title',
        #     'email', 'born', 'nationality',
        #     'external_photo_url', 'stood_for_party', 'url', 'type'
        # ]
        wanted_keys = [
            'external_id', 'external_photo_url', 'period_num',
            'forename', 'surname', 'stood_for_party', 'url', 'type'
        ]

        for doc in self._get_documents(fields_dict):
            new_doc = self._copy_doc(doc)
            for key, val in new_doc.items():
                if isinstance(val, str):
                    new_doc[key] = val.strip()
            new_doc['external_photo_url'] = new_doc['image_urls'][0]
            # GDPR
            # if new_doc['residence'] in residence_replacements:
            #     new_doc['residence'] = residence_replacements[new_doc['residence']]

            # if new_doc['external_id'] == 184:
            #     new_doc['residence'] = 'Bratislava'

            # if new_doc['residence'] == '':
            #     new_doc['residence'] = 'Neuvedené'

            # if new_doc['county'] in county_replacements:
            #     new_doc['county'] = county_replacements[new_doc['county']]

            # if new_doc['nationality'] in nationality_replacements:
            #     new_doc['nationality'] = nationality_replacements[new_doc['nationality']]

            # region = new_doc['county'] if 'kraj' in new_doc['county'] else '{} kraj'.format(new_doc['county'])
            # residence_query = """
            #     SELECT V.id, V.full_name, D.name, R.name
            #     FROM geo_village V
            #     INNER JOIN geo_district D ON V.district_id = D.id
            #     INNER JOIN geo_region R ON (D.region_id = R.id)
            #     WHERE V.full_name = '{village}' AND R.name = '{region}'
            #     """.format(village=new_doc['residence'], region=region)
            # pg_cursor.execute(residence_query)
            # villages = pg_cursor.fetchall()
            # residence_id = None
            # if not villages:
            #     raise Exception("Residence can't be paired for {}".format(new_doc))

            # elif len(villages) > 1 and new_doc['external_id'] == 708:
            #     for village in villages:
            #         if village[2] == 'Žarnovica':
            #             residence_id = village[0]
            #     if not residence_id:
            #         raise Exception(
            #             "Residence doesn't match on {} for {}".format(new_doc, villages))
            # else:
            #     residence_id = villages[0][0]

            # if not residence_id:
            #     raise Exception("residence_id for {} is None".format(new_doc))

            new_doc = self._get_wanted_keys(new_doc, wanted_keys)
            # TODO: broken nrsr data workaround
            if new_doc['external_id'] == 1080:
                new_doc['stood_for_party'] = 'OBYČAJNÍ ĽUDIA a nezávislé osobnosti (OĽANO), NOVA, Kresťanská únia (KÚ), ZMENA ZDOLA'
            if new_doc['external_id'] == 874:
                new_doc['stood_for_party'] = 'SMER – sociálna demokracia'
            if new_doc['external_id'] == 966:
                new_doc['stood_for_party'] = 'Sloboda a Solidarita'
            # new_doc['residence_id'] = residence_id
            new_docs.append(new_doc)
        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_member_changes(self):
        """
        Transform MP changes
        """
        fields_list = [
            'external_id',
            'date',
            'period_num',
            'change_type',
            'change_reason',
            'type'
        ]
        fields_dict = self._fields_to_dict(fields_list)

        new_docs = []
        change_type = {
            "Mandát sa neuplatňuje": 0,
            "Mandát vykonávaný (aktívny poslanec)": 1,
            "Mandát náhradníka zaniknutý": 2,
            "Mandát náhradníka vykonávaný": 3,
            "Mandát náhradníka získaný": 4,
            "Mandát zaniknutý": 5,
            "Mandát nadobudnutý vo voľbách": 6
        }

        for doc in self._get_documents(fields_dict):
            new_doc = self._copy_doc(doc)
            new_doc['change_type'] = change_type[doc['change_type']]
            del new_doc['_id']
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_presses(self):
        """
        Transform Press data
        """
        fields_list = [
            'title',
            'press_num',
            'group_num',
            'period_num',
            'press_type',
            'date',
            'attachments_urls',
            'attachments_names',
            'url',
            'type'
        ]

        fields_dict = self._fields_to_dict(fields_list)

        press_type_replacements = {
            'Návrh zákona': 1,
            'Iný typ': 2,
            'Informácia': 7,
            'Správa': 5,
            'Petícia': 3,
            'Medzinárodná zmluva': 4,

        }

        wanted_keys = ['press_type', 'title', 'press_num', 'date', 'period_num', 'url', 'type']
        new_docs = []
        for doc in self._get_documents(fields_dict):
            new_doc = self._copy_doc(doc)
            new_doc['press_type'] = press_type_replacements[new_doc['press_type']]
            new_doc = self._get_wanted_keys(new_doc, wanted_keys)
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_sessions(self):
        """
        Transform Session data
        """

        fields_list = ['url', 'period_num', 'external_id', 'name', 'program_points', 'type']
        fields_dict = self._fields_to_dict(fields_list)

        state_replacements = {
            'Prerokovaný bod programu': 0,
            'Neprerokovaný bod programu': 1,
            'Presunutý bod programu': 2,
            'Stiahnutý bod programu': 3,
            'Prerušené rokovanie o bode programu': 4,
        }

        wanted_keys = ['external_id', 'period_num', 'name', 'type', 'url', 'program_points', 'session_num']
        new_docs = []
        for doc in self._get_documents(fields_dict):
            new_doc = self._copy_doc(doc)
            try:
                new_doc['session_num'] = int(''.join(re.findall(r'\d+', new_doc['name'][:15])))
            except ValueError:
                new_doc['session_num'] = None

            program_points = []
            for point in new_doc['program_points']:
                program_points.append({
                    'state': state_replacements[point['state']],
                    'point_num': point['progpoint'],
                    'press_num': point['parlpress'],
                    'text1': point['text'][0],
                    'text2': point['text'][1],
                    'text3': point['text'][2],
                })
            new_doc['program_points'] = program_points
            new_doc = self._get_wanted_keys(new_doc, wanted_keys)
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_club_members(self):
        """
        Transform club members
        """

        max_voting = list(
            self.mongo_col.find({
                'type': 'voting'
            }, {
                'datetime': 1
            }).sort([('datetime', -1)]).limit(1))[0]['datetime'].replace(
                hour=12, minute=0, second=0, microsecond=0)

        # TODO(Jozef): the if/else condition won't work on last day of period
        aggregation = [
            {
                '$unwind': '$clubs'
            },
            {
                '$project': {
                    'period_num': 1,
                    'date': 1,
                    'club_name': {
                        '$arrayElemAt': ['$clubs', 0]
                    },
                    'club_values': {
                        '$arrayElemAt': ['$clubs', 1]
                    }
                }
            }, {
                '$unwind': '$club_values'
            }, {
                '$sort': {
                    'club_values': 1,
                    'date': 1
                }
            },
            {
                '$group': {
                    '_id': {
                        'period_num': '$period_num',
                        'member_external_id': '$club_values',
                        'club': '$club_name',
                        'date': '$date',
                    },
                }
            },
            {
                '$project': {
                    '_id': '$_id',
                    'period_num': '$_id.period_num',
                    'member_external_id': '$_id.member_external_id',
                    'date': '$_id.date',
                    'club': '$_id.club',
                }
            },
            {'$sort': {'_id.period_num': 1, '_id.member_external_id': 1, '_id.date': 1}}
        ]
        member_sequence = {}
        docs = list(self._get_documents({}, *aggregation))
        docs_len = len(docs)

        pg_conn = psycopg2.connect(self.postgres_url)
        pg_cursor = pg_conn.cursor()

        active_query = """
        SELECT 1 FROM person_person PP
            INNER JOIN parliament_member PM ON PM.person_id = PP.id
            INNER JOIN parliament_memberactive PMA ON PMA.member_id = PM.id
        WHERE PMA.start <= '{date}' AND (PMA.end > '{date}' OR PMA.end IS NULL)
        AND PP.external_id = {external_id}
        LIMIT 1
        """

        for index, doc in enumerate(docs):
            _id = doc['_id']
            period_num = _id['period_num']
            club_key = _id['club']
            member_key = _id['member_external_id']

            if period_num not in member_sequence:
                member_sequence[period_num] = {}

            if club_key not in member_sequence[period_num]:
                member_sequence[period_num][club_key] = {}

            if member_key not in member_sequence[period_num][club_key]:
                member_sequence[period_num][club_key][member_key] = []

            member_link = member_sequence[period_num][club_key][member_key]

            if index < docs_len - 1:
                next_item = docs[index + 1]['_id']
            else:
                next_item = None

            if member_link:
                # last was ended, create new
                if member_link[-1][2]:
                    member_link.append([_id['date'], None, False])
            else:
                member_link.append([_id['date'], None, False])

            current = member_link[-1]

            if next_item:
                if next_item['member_external_id'] == member_key and next_item['club'] != club_key:
                    if _id['date'] == max_voting:
                        current[1] = None
                    else:
                        current[1] = _id['date']
                    current[2] = True
                elif next_item['member_external_id'] != member_key:
                    pg_cursor.execute(active_query.format(date=max_voting, external_id=member_key))
                    db_result = pg_cursor.fetchall()
                    if not db_result:
                        current[1] = _id['date']
                        current[2] = True

        if pg_conn is not None:
            pg_conn.close()

        new_docs = []
        for period_num, clubs in member_sequence.items():
            for club, members in clubs.items():
                for member, changes in members.items():
                    for change in changes:
                        new_docs.append({
                            'type': self.data_type,
                            'period_num': period_num,
                            'member_external_id': member,
                            'club': club,
                            'start': change[0],
                            'end': change[1],
                        })

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_votings(self):
        """
        Transform votings and votes
        """

        fields_list = [
            'external_id', 'topic', 'datetime', 'session_num', 'voting_num',
            'result', 'period_num', 'press_num', 'url', 'votes', 'type'
        ]
        fields_dict = self._fields_to_dict(fields_list)

        result_replacements = {
            "Návrh prešiel": 0,
            "Návrh neprešiel": 1,
            "Parlament nebol uznášaniaschopný": 2
        }

        vote_replacements = {
            "-": -1,
            "Z": 0,
            "P": 1,
            "?": 2,
            "N": 3,
            "0": 4
        }

        new_docs = []
        for doc in self._get_documents(fields_dict):
            new_doc = self._copy_doc(doc)
            new_doc['result'] = result_replacements[new_doc['result']]
            for vote in new_doc['votes']:
                vote['vote'] = vote_replacements[vote['vote']]
            del new_doc['_id']
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_bills(self):
        """
        Transform bills
        """
        fields_list = [
            'type',
            'external_id',
            'period_num',
            'press_num',
            'proposer',
            'delivered',
            'current_state',
            'current_result',
            'category_name',
            'url',
            'type',
        ]

        fields_dict = self._fields_to_dict(fields_list)

        current_state_replacements = {
            "Evidencia": 0,
            "Uzavretá úloha": 1,
            "Rokovanie NR SR": 8,
            "Rokovanie gestorského výboru": 9,
            "I. čítanie": 2,
            "II. čítanie": 3,
            "III. čítanie": 4,
            "Redakcia": 5,
            "Výber poradcov k NZ": 10,
            "Stanovisko k NZ": 7
        }

        current_result_replacements = {
            "(NR SR nebude pokračovať v rokovaní o návrhu zákona)": 0,
            "(NZ vzal navrhovateľ späť)": 1,
            "(Zákon vyšiel v Zbierke zákonov)": 7,
            "(NZ nebol schválený)": 5,
            "(Zákon bol vrátený prezidentom)": 8,
            "(Zápis spoločnej správy výborov)": 4,
            "(Zapísané uznesenie výboru)": 9,
            "(NZ postúpil do II. čítania)": 11,
            "(NZ postúpil do redakcie)": 3,
            "(Výber právneho poradcu)": 10,
            "(Pripravená informácia k NZ)": 6
        }

        category_name_replacements = {
            "Novela zákona": 0,
            "Návrh nového zákona": 1,
            "Iný typ": 2,
            "Petícia": 3,
            "Medzinárodná zmluva": 4,
            "Správa": 5,
            "Ústavný zákon": 6,
            "Informácia": 7,
            "Návrh zákona o štátnom rozpočte": 8,
            "Zákon vrátený prezidentom": 9,

        }

        new_docs = []
        for doc in self._get_documents(fields_dict):
            if 'category_name' not in doc:
                # empty bill with no details
                continue
            press_title = ''
            press_num = ''
            if 'press_num' in doc:
                presses = list(self.mongo_col.find({
                    'type': 'press', 'press_num': doc['press_num'], 'period_num': doc['period_num']}))
                if not presses:
                    raise Exception("Missing press for {}".format(doc))
                if len(presses) > 1:
                    raise Exception("Multiple presses for {}".format(doc))
                else:
                    press_title = presses[0]['title']
                    press_num = presses[0]['press_num']

            doc['press_num'] = press_num
            names = []
            proposers = []
            doc['proposer_nonmember'] = ''
            proposer_type = 'NULL'
            if 'proposer' in doc:
                if doc['proposer'].startswith('poslanci NR SR'):
                    doc['proposer_nonmember'] = 'poslanci NR SR'
                    proposer_type = 0
                    try:
                        names = re.match(r'^.*\(([^)]+)\)$', doc['proposer']).groups()[0].split(', ')
                    except AttributeError:
                        names = []
                elif doc['proposer'].startswith('vláda'):
                    proposer_type = 1
                    propre = re.match(r'vláda \((.*)\)', doc['proposer'])
                    if propre and len(propre.groups()) == 1:
                        doc['proposer_nonmember'] = propre.groups()[0]
                    else:
                        doc['proposer_nonmember'] = doc['proposer']
                elif doc['proposer'].startswith('výbor'):
                    proposer_type = 2
                    doc['proposer_nonmember'] = doc['proposer']
                query = """
                    SELECT M.id, S.forename, S.surname FROM parliament_member M
                    INNER JOIN parliament_period P ON M.period_id = P.id
                    INNER JOIN person_person S ON M.person_id = S.id
                    WHERE P.period_num = {period_num}
                    AND S.surname = '{surname}' AND S.forename LIKE '{forename}%'
                    """
                pg_conn = psycopg2.connect(self.postgres_url)
                pg_cursor = pg_conn.cursor()
                for name in names:
                    name_list = name.split('.\xa0')
                    pg_cursor.execute(
                        query.format(
                            period_num=doc['period_num'],
                            forename=name_list[0],
                            surname=name_list[1]
                        )
                    )
                    rows = pg_cursor.fetchall()
                    if len(rows) == 1:
                        proposers.append(rows[0][0])
                    elif len(rows) == 0:
                        raise Exception("Missing member in {}".format(query))
                    else:
                        try:
                            similarities = []
                            for row in rows:
                                similarities.append(
                                    SequenceMatcher(
                                        None, '{} {}'.format(row[1], row[2]),
                                        press_title).ratio())
                            max_ratio = max(similarities)
                            proposers.append(rows[similarities.index(max_ratio)][0])
                        except Exception as exc:
                            raise Exception(
                                "Similarities search failed on {} with {}".format(doc, exc))
                pg_conn.close()
                doc['proposers'] = proposers

            new_doc = self._copy_doc(doc)
            new_doc['proposer_type'] = proposer_type
            try:
                new_doc['current_state'] = current_state_replacements[new_doc['current_state']]
            except KeyError:
                new_doc['current_state'] = None
            try:
                new_doc['current_result'] = current_result_replacements[new_doc['current_result']]
            except KeyError:
                new_doc['current_result'] = None
            new_doc['category_name'] = category_name_replacements[new_doc['category_name']]

            new_doc = self._get_wanted_keys(new_doc, [
                'type', 'period_num', 'press_num', 'external_id', 'delivered',
                'current_state', 'current_result', 'proposers',
                'proposer_nonmember', 'proposer_type', 'category_name', 'url'
            ])
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    # def transform_bill_process_steps(self):
    #     """
    #     Transform bill process steps
    #     """
    #     fields_list = [
    #         'type',

    #         'period_num',

    #         'external_id',
    #         'bill_id',
    #         'body_label',
    #         'meeting_session_num',
    #         'meeting_resolution',
    #         'step_result',

    #         'committees_label',
    #         'slk_label',
    #         'coordinator_label',
    #         'coordinator_meeting_date',??
    #         'coordinator_name',??
    #         'discussed_label',
    #         'sent_standpoint',
    #         'sent_label',
    #         'act_num_label'

    #         'url'
    #     ]

    def transform_debate_appearances(self):
        """
        Debate Appearances
        """
        fields_list = [
            'type',
            'external_id',
            'period_num',
            'debater_name',
            'debater_party',
            'debater_role',
            'start',
            'end',
            'session_num',
            'press_num',
            'appearance_type',
            'appearance_type_addition',
            'video_short_url',
            'text',
        ]

        fields_dict = self._fields_to_dict(fields_list)

        appearance_type_replacements = {
            "-": 0,
            "Doplňujúca otázka / reakcia zadávajúceho": 1,
            "Prednesenie interpelácie": 2,
            "Prednesenie otázky": 3,
            "Uvádzajúci uvádza bod": 4,
            "Vstup predsedajúceho": 5,
            "Vystúpenie": 6,
            "Vystúpenie s faktickou poznámkou": 7,
            "Vystúpenie s procedurálnym návrhom": 8,
            "Vystúpenie spoločného spravodajcu": 9,
            "Vystúpenie v rozprave": 10,
            "Zodpovedanie otázky": 11,
        }

        new_docs = []
        for doc in self._get_documents(fields_dict):
            doc['appearance_type'] = appearance_type_replacements[doc['appearance_type']]
            name_list = doc['debater_name'].split(', ')
            doc['debater_forename'] = name_list[1]
            doc['debater_surname'] = name_list[0]
            if not 'text' in doc:
                doc['text'] = ''
            else:
                doc['text'] = ''.join(doc['text'])
            if 'debater_party' not in doc:
                doc['debater_party'] = None
            try:
                party = re.match(r'^\(+(.+)+\)', doc['debater_party'])
                if party:
                    doc['debater_party'] = party.groups()[0]
            except TypeError:
                pass

            if 'debater_role' not in doc:
                doc['debater_role'] = ''
            doc['debater_role'] = doc['debater_role'][2:]

            if 'NRSR' in doc['debater_role'] or 'NR SR' in doc['debater_role']:
                doc['parliament_member'] = True
            else:
                doc['parliament_member'] = False

            new_docs.append(
                self._get_wanted_keys(doc, [
                    'type', 'period_num', 'session_num', 'press_num', 'external_id',
                    'start', 'end', 'debater_forename', 'debater_surname', 'debater_role',
                    'parliament_member', 'debater_party', 'appearance_type',
                    'appearance_type_addition', 'video_short_url', 'text'
                ])
            )

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_interpellations(self):
        """
        Transform Interpellations
        """
        fields_list = [
            'type',
            'external_id',
            'period_num',
            'status',
            'asked_by',
            'description',
            'recipients',
            'date',
            'responded_by',
            'interpellation_session_num',
            'response_session_num',
            'responded_by',
            'press_num',
            'url'
        ]
        fields_dict = self._fields_to_dict(fields_list)

        status_replacements = {
            "Príjem odpovede na interpeláciu": 0,
            "Rokovanie o interpelácii": 1,
            "Uzavretá odpoveď na interpeláciu": 2,
            "Interpelácia na expedíciu": 3,
        }
        new_docs = []
        for doc in self._get_documents(fields_dict):
            doc['status'] = status_replacements[doc['status']]
            asked_list = doc['asked_by'].split(', ')
            doc['asked_by_forename'] = asked_list[1]
            doc['asked_by_surname'] = asked_list[0]

            new_docs.append(
                self._get_wanted_keys(doc, [
                    'type', 'external_id', 'period_num', 'status',
                    'asked_by_forename', 'asked_by_surname', 'description',
                    'recipients', 'date', 'responded_by', 'interpellation_session_num',
                    'response_session_num', 'responded_by', 'press_num', 'url'
                ])
            )

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_amendments(self):
        """Transform amendments"""
        fields_list = [
            'type',
            'external_id',
            'period_num',
            'press_num',
            'session_num',
            'submitter',
            'other_submitters',
            'date',
            'signed_members',
            'voting_external_id',
            'url'
        ]

        fields_dict = self._fields_to_dict(fields_list)
        new_docs = []
        for doc in self._get_documents(fields_dict):
            doc['submitter'] = doc['submitter'].split(', ')[::-1]

            if 'other_submitters' in doc:
                doc['other_submitters'] = [x.split(', ')[::-1] for x in doc['other_submitters']]

            if 'signed_members' in doc:
                doc['signed_members'] = [x.split(', ')[::-1] for x in doc['signed_members']]
            new_docs.append(self._get_wanted_keys(doc, fields_list))

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_committees(self):
        """Committees and memberships"""
        fields_list = [
            '_id',
            'type',
            'period_num',
            'external_id',
            'name',
            'description',
            'url',
            'members'
        ]

        roles = {
            'predsedníčka': 0,
            'predseda': 1,
            'podpredsedníčka': 2,
            'podpredsedkyňa': 2, # incorrect phrase promoted to nrsr.sk
            'podpredseda': 3,
            'členka': 4,
            'člen': 5,
            'náhradná členka': 6,
            'náhradný člen': 7,
            'overovateľka': 8,
            'overovateľ': 9
        }

        fields_dict = self._fields_to_dict(fields_list)
        new_docs = []
        for doc in self._get_documents(fields_dict):
            doc['description'] = ''.join([x.strip() for x in doc['description']])
            for member in doc['members']:
                member['role'] = roles[member['role']]
                member['start'] = doc['_id'].generation_time
            new_docs.append(self._get_wanted_keys(doc, fields_list))
        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

    def transform_committee_schedules(self):
        """Transform committee sessions and session points"""
        fields_list = [
            'type',
            'committee_name',
            'period_num',
            'date',
            'time',
            'place',
            'points'
        ]
        fields_dict = self._fields_to_dict(fields_list)
        new_docs = []
        pg_conn = psycopg2.connect(self.postgres_url)
        pg_cursor = pg_conn.cursor()
        for doc in self._get_documents(fields_dict):
            times = doc['time'].split(' - ')
            if len(times) == 1:
                time_start = times[0]
                time_end = None
            else:
                time_start, time_end = times

            pg_cursor.execute(
                """
                SELECT PC.id FROM parliament_committee PC
                LEFT JOIN parliament_period PP ON PP.id = PC.period_id
                WHERE PC.name = '{committee_name}' AND PP.period_num = {period_num}
                """.format(
                    committee_name=doc['committee_name'],
                    period_num=doc['period_num']
                )
            )
            row = pg_cursor.fetchone()

            try:
                dt_start = datetime.strptime('{} {}'.format(doc['date'], time_start),
                                  '%d. %m. %Y %H:%M')
            except ValueError:
                dt_start = datetime.strptime('{} {}'.format(doc['date'], time_start),
                                  '%d.%m.%Y %H:%M')

            try:
                dt_end = datetime.strptime('{} {}'.format(doc['date'], time_end),
                                  '%d. %m. %Y %H:%M') if time_end else None
            except ValueError:
                dt_end = datetime.strptime('{} {}'.format(doc['date'], time_end),
                                  '%d.%m.%Y %H:%M') if time_end else None

            new_doc = {
                'type': doc['type'],
                'period_num': doc['period_num'],
                'place':
                ''.join([x.strip() for x in doc['place']]),
                'start': dt_start,
                'end': dt_end,
                'committee_id':
                row[0]
            }
            points = []
            pg_cursor.execute(
                """
                SELECT PRESS.id, PRESS.press_num FROM parliament_press PRESS
                LEFT JOIN parliament_period PER ON PER.id = PRESS.period_id
                WHERE PER.period_num = {period_num}
                """.format(period_num=doc['period_num'])
            )
            presses = {x[1]: x[0] for x in pg_cursor.fetchall()}
            for point in doc['points']:
                index = None
                text = ''.join([x.strip() for x in point['text']])
                if not text:
                    continue
                is_index = re.match(r'^([0-9]\. )([\s\S]*)$', text)
                if is_index:
                    index = is_index.groups()[0].strip().replace('.', '')
                    text = ''.join(is_index.groups()[1:])

                press_id = None
                if point['press_num']:
                    press_id = presses[str(point['press_num'])]
                points.append({
                    'press_id': press_id,
                    'index': index,
                    'topic': text
                })
            new_doc['points'] = points
            new_docs.append(new_doc)

        if new_docs:
            self._insert_documents(new_docs, remove=[self.data_type])

        if pg_conn is not None:
            pg_conn.close()

    def execute(self, context):
        """Operator Executor"""
        if self.data_type == 'member':
            self.transform_members()
        elif self.data_type == 'member_change':
            self.transform_member_changes()
        elif self.data_type == 'press':
            self.transform_presses()
        elif self.data_type == 'session':
            self.transform_sessions()
        # elif self.data_type == 'club':
        #     data_frame = self.transform_clubs()
        elif self.data_type == 'daily_club':
            self.transform_club_members()
        elif self.data_type == 'voting':
            self.transform_votings()
        elif self.data_type == 'bill':
            self.transform_bills()
        # elif self.data_type == 'bill_step':
        #     self.transform_bill_process_steps()
        elif self.data_type == 'debate_appearance':
            self.transform_debate_appearances()
        elif self.data_type == 'interpellation':
            self.transform_interpellations()
        elif self.data_type == 'amendment':
            self.transform_amendments()
        elif self.data_type == 'committee':
            self.transform_committees()
        elif self.data_type == 'committeeschedule':
            self.transform_committee_schedules()

class NRSRTransformPlugin(AirflowPlugin):

    name = 'nrsr_transform_plugin'
    operators = [NRSRTransformOperator]
    hooks = []
    executors = []
    macros = []
    admin_views = []
    flask_blueprints = []
    menu_links = []


#ASPiRE UoWM 2019
#CanSat in Greece 2019

import os
import string
import time
import datetime
import serial
import board
import busio
import smbus
import adafruit_bmp280
from picamera import PiCamera, Color

cameracapture = True

#Telemetry serial port
Telemetry_SerialConnection = serial.Serial(port = '/dev/ttyUSB0', baudrate = 9600, timeout = 1)

# Create library object using Bus I2C port
i2c = busio.I2C(board.SCL, board.SDA)

bus = smbus.SMBus(1) # or bus = smbus.SMBus(0) for older version boards
#Device_Address = 0x68 # MPU6050 device address

if cameracapture == True:
	camera = PiCamera()
	camera.resolution = (3264, 2448)

bmp280 = adafruit_bmp280.Adafruit_BMP280_I2C(i2c)

#GPS Serial port
GPS_port = "/dev/ttyAMA0"
GPS_SerialConnection = serial.Serial(GPS_port, baudrate = 9600, timeout = 1)

def timestamp():
	return time.time()

def read_GPS():
	GPS_SerialConnection.flushInput()
	GPS_Data = GPS_SerialConnection.readline() #Read raw data from GPS chipset (NEO-6M)
	GPS_DataString = str(GPS_Data) #Raw data is binary. We need string to process.

	if 'GGA,' in GPS_DataString:
		GPGGAStringDecoded = GPS_DataString[18:30] + "(Latitude)|" + GPS_DataString[32:44] + "(Longitude)"
		f = open("nmea_strings.txt", "a")
		f.write(GPS_DataString + "\n")
		f.close()
		return "|" + GPGGAStringDecoded

def read_GPS_Latitude():
	GPS_SerialConnection.flushInput()
	GPS_Data = GPS_SerialConnection.readline() #Read raw data from GPS chipset (NEO-6M)
	GPS_DataString = str(GPS_Data) #Raw data is binary. We need string to process.

	if 'GGA,' in GPS_DataString:
		Latitude = "Lat" + GPS_DataString[18:30] + "\n"
		return Latitude

def read_GPS_Longitude():
	GPS_SerialConnection.flushInput()
	GPS_Data = GPS_SerialConnection.readline() #Read raw data from GPS chipset (NEO-6M)
	GPS_DataString = str(GPS_Data) #Raw data is binary. We need string to process.

	if 'GGA,' in GPS_DataString:
		Longitude = "Lon" + GPS_DataString[32:44] + "\n"
		return Longitude

def read_BMP280():
	bmp280.sea_level_pressure = 1013.25
	sensordata = "|Temperature=%0.1f(C)" %bmp280.temperature + "|Pressure=%0.1f(hPa)" %bmp280.pressure + "|Altitude=%0.2f(m)" %bmp280.altitude
	return sensordata

def read_BMP280_Temperature():
	bmp280.sea_level_pressure = 1013.25
	sensordata = "Tmp%0.1f(C)\n" %bmp280.temperature
	return sensordata

def read_BMP280_Pressure():
	bmp280.sea_level_pressure = 1013.25
	sensordata = "Prs%0.1f(hPa)\n" %bmp280.pressure
	return sensordata

def read_BMP280_Altiture():
	bmp280.sea_level_pressure = 1013.25
	sensordata = "Alt%0.2f(m)\n" %bmp280.altitude
	return sensordata

def main():
	try:
		i = 0 #Counter for data lines

		while True:
			#Main Loop Start
			while True:
				gpsDataForFileTransmission = read_GPS()
				if type(gpsDataForFileTransmission) is str:
					break

			#mpu6050DataForFileTransmission = read_MPU6050()
			bmp280DataForFileTransmission = read_BMP280()
			finalData = "|" + str(i) + "|" + str(timestamp()) + gpsDataForFileTransmission + bmp280DataForFileTransmission
			print(finalData)
			i = i + 1
			f = open("measurements.txt", "a")
			f.write(finalData + "\n")
			f.close()

			TEMP = read_BMP280_Temperature()
			Telemetry_SerialConnection.write(TEMP.encode())
			time.sleep(1)
			PRESS = read_BMP280_Pressure()
			Telemetry_SerialConnection.write(PRESS.encode())
			time.sleep(1)
			ALT = read_BMP280_Altiture()
			Telemetry_SerialConnection.write(ALT.encode())
			time.sleep(1)
			while True:
				LAT = read_GPS_Latitude()
				if type(LAT) is str:
					break
			Telemetry_SerialConnection.write(LAT.encode())
			time.sleep(1)
			while True:
				LON = read_GPS_Longitude()
				if type(LON) is str:
					break
			Telemetry_SerialConnection.write(LON.encode())
			time.sleep(1)
			if cameracapture == True:
				camera.start_preview()
				time.sleep(1)
				print("Capturing format: jpeg | picture\n")
				camera.capture('images/image%s.jpg' %str(timestamp()), 'jpeg')
				camera.stop_preview()

			time.sleep(1)
			#Main Loop End
	except KeyboardInterrupt:
		GPS_SerialConnection.close()
		Telemetry_SerialConnection.close()

if __name__ == "__main__":
	main()


from django.urls import path

from . import views

urlpatterns = [
  path('', views.home, name='home'),
  path('admission/', views.admission, name='admission'),
  path('academics/cs_bsc/', views.csBSc, name='cs-bsc'),
  path('academics/se_bsc/', views.seBSc, name='se-bsc'),
  path('academics/cs_msc/', views.csMSc, name='cs-msc'),
  path('academics/se_msc/', views.seMSc, name='se-msc'),
  path('academics/courses/', views.courses, name='courses-list'),
  path('about/', views.about, name='about'),
  path('contact/', views.contact, name='contact-us'),
]


"""This file contains modules to draw pairs of similar instances."""

from ampligraph.discovery import find_nearest_neighbours
import pandas as pd
import numpy as np
import copy
from scipy.spatial import distance

def get_matrix_similarity_pairs(model,instances_tc,mode='mixed',instances_t0='instances_t0',instances_t1='instances_t1'):
    """
    Getting the euclidean distance between pairs of studied instances of a target class.
    The instances of a pair can differ on the treatment (mode=treatment_sort), or created independently of their treatment values. (mode=mixed)

    Parameters :
    model : ampligraph EmbeddingModel
    all_people : list of instances studied
    mode : str
    instances_t0 : list of instances
    instances_t1 : list of instances

    Returns :
    df : pandas dataframe
        With euclidean distance between all pairs of instances
    df_to_numpy : numpy array
        array version of the df
    """
    if mode == 'mixed':
        df = pd.DataFrame(columns=instances_tc,index=instances_tc)
        for t_ in instances_tc:
            neighbors, dist = find_nearest_neighbours(model,
                                                entities=[t_],
                                                n_neighbors=len(instances_tc)-1,
                                                entities_subset=instances_tc)
            for i in range(len(neighbors[0])):
                df.loc[neighbors[0][i],t_] = round(dist[0][i],3)

        df = df.fillna(100)
        df_to_numpy = df.to_numpy()
        np.fill_diagonal(df_to_numpy, 100)
        return df,df_to_numpy

    elif mode == 'treatment_sort':
        df_ = pd.DataFrame(columns=instances_t0,index=instances_t1)
        for t_ in instances_t0:
            neighbors, dist = find_nearest_neighbours(model,
                                                entities=[t_],
                                                n_neighbors=len(instances_t1)-1,
                                                entities_subset=instances_t1)
            for i in range(len(neighbors[0])):
                df_.loc[neighbors[0][i],t_] = round(dist[0][i],3)

        df_ = df_.fillna(200)
        df_to_numpy = df_.to_numpy()
        return df_,df_to_numpy

def get_pairs_from_matrix_and_threshold(df,distance_threshold,strategy='greedy',mode='mixed'):
    """
    Returns the similar pairs of a target class given a threshold.

    Parameters :
    df : pandas dataframe
        With euclidean distance between all pairs of instances
    distance_threshold : float
        Threshold on the distance for selecting similar instances
    strategy : string (by default = 'greedy')
        Strategy to obtain the pairs. Can be greedy or optimal

    Returns :
    pairs_similar_instances : list
        All pairs of similar instances
    """
    df__ = copy.deepcopy(df)
    df_to_numpy = df__.to_numpy()
    min_value = df_to_numpy.min()
    pairs_similar_instances = []

    if strategy == 'greedy':
        if min_value >= distance_threshold:
            return pairs_similar_instances
        else:
            while min_value < distance_threshold:
                # get position
                position = np.where(df_to_numpy==min_value)

                # add to set of pairs
                first_instance = list(df.columns)[position[1][0]]
                second_instance = list(df.index)[position[0][0]]
                pairs_similar_instances.append([first_instance,second_instance])

                # replace value
                if mode=='mixed': # squared matrix - we update 2 values
                    df_to_numpy[position[0][0]] = [100]
                    df_to_numpy[:,position[1][0]] = [100]
                elif mode == 'treatment_sort': # only one value to update
                    df_to_numpy[position[0][0]] = [100]

                #update min
                min_value = df_to_numpy.min()

            return pairs_similar_instances

    elif strategy == 'optimal':
        return pairs_similar_instances

    else:
        print("This stratedy does not exist, please switch to greedy or optimal.")
        return None

def get_pairs_from_matrix_and_proportion(df,number_total_instances,proportion=0.05,mode='mixed'):
    """
    Returns the closer pairs of a target class given a proportion of pairs to create.

    Parameters :
    df : pandas dataframe
        With euclidean distance between all pairs of instances
    proportion : float
        Proportion of pairs to create

    Returns :
    pairs_closer_instances : list
        The closer pairs of instances given the proportion
    """
    df__ = copy.deepcopy(df)
    df_to_numpy = df__.to_numpy()
    pairs_closer_instances = []

    number_possible_pairs = min(df.shape[0],df.shape[1])
    number_pairs_total = number_total_instances*(number_total_instances-1)/2
    number_to_reach = number_pairs_total*proportion

    if number_possible_pairs < number_to_reach:
        number_to_reach = number_possible_pairs

    print('Number of pairs to build : ',number_to_reach)

    while len(pairs_closer_instances) < number_to_reach:
        # get position
        min_value = df_to_numpy.min()
        max_value = df_to_numpy.max()
        position = np.where(df_to_numpy==min_value)

        # add to set of pairs
        first_instance = list(df.columns)[position[1][0]]
        second_instance = list(df.index)[position[0][0]]
        pairs_closer_instances.append([first_instance,second_instance])

        # replace value
        if mode=='mixed': # squared matrix - we update 2 values
            df_to_numpy[position[0][0]] = [max_value]
            df_to_numpy[:,position[1][0]] = [max_value]
        elif mode == 'treatment_sort': # only one value to update
            df_to_numpy[position[0][0]] = [max_value]

    return pairs_closer_instances

def get_distance_for_degree(dic_distance_per_d,degree):
    if degree in list(dic_distance_per_d.keys()):
        all_values_degree = dic_distance_per_d[degree]
        return np.mean(all_values_degree)
    else:
        print('Degree can not be found')
        return None

def get_distance_for_pair(pair_,model,type_distance='euclidian'):
    """
    Returns the distance between the embedding vectors of a pair.

    The type of the distance can be specified in : euclidian, TO_COMPLETE
    """
    array_pair = np.array(pair_)
    embedding_pair = model.get_embeddings(entities=np.array(array_pair))
    if type_distance=='euclidian':
        return distance.euclidean(embedding_pair[0],embedding_pair[1])
    else:
        return None


import pandas as pd
import numpy as np
from validate_utf8 import find_utf8_errors
import os
import glob
from collections import namedtuple
import json
import jsonschema
import rich.traceback
rich.traceback.install()

from rich.console import Console
console = Console(record=True, force_terminal=True)
console.print("Testing ðŸ’µ[yellow]rich[/] console")
console.log([1, "str"], 'https://github.com/willmcgugan/rich/blob/9cba2027f4/tests/test_color_triplet.py', {"k": 'v', "date": '00:46:54'}, None, '2009-11-27T00:00:00.000100-06:39')

COVID_19_REPO_FOLDER = 'COVID-19'
if not os.path.exists(COVID_19_REPO_FOLDER):
    COVID_19_REPO_FOLDER = '../' + COVID_19_REPO_FOLDER + '.git'
    if not os.path.exists(COVID_19_REPO_FOLDER):
        raise Exception('Could not find "{0}" folder'.format(COVID_19_REPO_FOLDER))

SCHEMAS_FOLDER = 'schemas'

ErrorEntry = namedtuple("ErrorEntry", ("filename", "errors"))

schemas = {}
def getJsonSchemaFor(filename):
    basename = os.path.basename(filename)
    schemaName, ext = os.path.splitext(basename)
    suffix = '-latest'
    if schemaName.endswith(suffix): schemaName = schemaName[:-len(suffix)]
    schemaName = os.path.join(SCHEMAS_FOLDER, schemaName + '.schema')
    #print(schemaName)
    if schemaName in schemas:
        schema = schemas[schemaName]
    else:
        if not os.path.exists(schemaName):
            raise Exception('No schema file named "{0}".'.format(schemaName))
        with open(schemaName, 'r') as jsonFile:
            schema = json.load(jsonFile)
            schemas[schemaName] = schema
            print('[LOAD SCHEMA] {0}'.format(os.path.basename(schemaName)))

    return schema

if __name__ == '__main__':
    print('Hello Github Actions Python')

    print('[COVID FOLDER] {0}'.format(COVID_19_REPO_FOLDER))

    print()

    csvFiles = glob.glob(COVID_19_REPO_FOLDER + '/*/*.csv')
    jsonFiles = glob.glob(COVID_19_REPO_FOLDER + '/*/*.json')

    errorsList = []

    filesToValidate = csvFiles + jsonFiles
    print('[UTF8] Validating utf8 in {0} files...'.format(len(filesToValidate)))
    for file in filesToValidate:
        with open(file, 'rb') as f:
            fileContent = f.read()
            errors = find_utf8_errors(fileContent)
            if errors:
                errorsList.append(ErrorEntry(file, errors))

    print('- Errors found: {0}'.format(len(errorsList)))
    for i, entry in enumerate(errorsList):
        print('  [{0:02}] {1}'.format(i, entry.filename))
        for error in entry.errors:
            print('     ' + error.extract)
        print()
    if not errorsList:
        print()

    verboseJsonErrors = True
    errorsList = []

    print('[JSON] Validating json schema in {0} files...'.format(len(jsonFiles)))
    for file in jsonFiles:
        schema = getJsonSchemaFor(file)
        #print(file)
        #print(file, schema)
        with open(file, 'r') as f:
            instance = json.load(f)
            try:
                jsonschema.validate(instance=instance, schema=schema, format_checker=jsonschema.FormatChecker())
            except jsonschema.exceptions.ValidationError as e:
                errorsList.append(ErrorEntry(file, e))

    print('- Errors found: {0}'.format(len(errorsList)))
    for i, entry in enumerate(errorsList):
        print('  [{0:02}] {1}'.format(i, entry.filename))
        if not verboseJsonErrors:
            print('       ' + entry.errors.message)
        else:
            error = str(entry.errors)
            error = error.replace('\n', '\n       ')
            print('       ' + error)
        print()


import multiprocessing
from config import server_conf

bind = f'{server_conf.HOST}:{str(server_conf.WSGI_PORT)}'

timeout = 30

worker_class = 'gevent'

workers = multiprocessing.cpu_count() * 2 + 1

threads = 3

worker_connections = 1000


#!/usr/bin/env python3

import csv

import bs4
import dateutil.parser
import requests

from collections import namedtuple
from datetime import timedelta
from itertools import chain

days = [
'http://www.tomorrowland.com/en/friday-22-july',
'http://www.tomorrowland.com/en/saturday-23-july',
'http://www.tomorrowland.com/en/sunday-24-july',
]

TimeSlot = namedtuple('TimeSlot', ['stage', 'artist', 'time_start', 'time_end', 'duration'])

def fetch_day(url, parser="lxml"):
    day_html = requests.get(url).text
    soup = bs4.BeautifulSoup(day_html, parser)
    return soup

def parse_day(url):
    soup = fetch_day(url)
    day_str = soup.find('h1', class_='page-title').text
    day = dateutil.parser.parse(day_str)

    stage_titles = {}
    for title in soup.find_all('div', class_='stage-logo'):
        stage_titles[title['data-id']] = title.find('img', class_='stage-icon')['title']

    stages = soup.find_all('div', class_='stage')
    return (parse_stage(stage, stage_titles, day) for stage in stages)

def parse_stage(stage, titles, day):
    timeslots = stage.find('div', class_='artists').find_all('div', class_="time-slot performance")
    stage_id = stage['data-id']
    stage_title = titles[stage_id]

    return (parse_timeslot(timeslot, day, stage_title) for timeslot in timeslots)

def parse_timeslot(timeslot, day, stage):
    artist = timeslot.find('a').text
    time_start, time_end = timeslot.find('span', class_='time').text.split('-')
    set_start = parse_time(day, time_start)
    set_end = parse_time(day, time_end)

    return TimeSlot(stage, artist, set_start, set_end, set_end - set_start)

def parse_time(day, time):
    # push times before 6am to the following day
    if int(time.partition(':')[0]) < 6:
        day = day + timedelta(days=1)

    return dateutil.parser.parse(time, default=day)

def main():
    sets = (parse_day(day) for day in days)

    # completely flatten the day[stage[set]] nested list structure
    flat_sets = chain.from_iterable((chain.from_iterable(sets)))

    # write out file
    with open("tomorrowland2016.csv", 'w') as outfile:
        csvfile = csv.writer(outfile)
        csvfile.writerow(TimeSlot._fields)
        csvfile.writerows(flat_sets)

if __name__ == '__main__':
    main()


import asyncio
import os
import youtube_dl

from datetime import datetime
from pyrogram import Client, filters
from pytgcalls import GroupCall
from Python_ARQ import ARQ

from .functions import (
    transcode,
    download_and_transcode_song,
    convert_seconds,
    time_to_seconds,
    generate_cover,
    generate_cover_square
)

from .configs import (
    API_ID as api_id,
    API_HASH as api_hash,
    SUDO_CHAT_ID as sudo_chat_id,
    ARQ_API,
    HEROKU
)

from .misc import (
    HELP_TEXT,
    START_TEXT,
    REPO_TEXT,
    DONATION_TEXT
)

if HEROKU:
    from .configs import SESSION_STRING

if not HEROKU:
    app = Client('ktgvc', api_id=api_id, api_hash=api_hash)
else:
    app = Client(SESSION_STRING, api_id=api_id, api_hash=api_hash)

group_calls = GroupCall(None, path_to_log_file='')
cmd_filter = lambda cmd: filters.command(cmd, prefixes='/')

# Arq Client
arq = ARQ(ARQ_API)

# File raw music
raw_filename = 'input.raw'

queue = []  # This is where the whole song queue is stored
playing = False  # Tells if something is playing or not

@app.on_message(filters.text & cmd_filter('start'))
async def start(_, message):
    await message.reply_text(START_TEXT)

@app.on_message(filters.text & cmd_filter('help'))
async def helps(_, message):
    await message.reply_text(HELP_TEXT)

@app.on_message(filters.text & cmd_filter('repo'))
async def repo(_, message):
    await message.reply_text(REPO_TEXT)

@app.on_message(filters.text & cmd_filter('ping'))
async def ping(_, message):
    start = datetime.now()
    msg = await send('`âš¡`')
    end = datetime.now()
    latency = (end - start).microseconds / 1000
    await msg.edit(f"**Jinda hu sir itni jaldi nhi marunga**\n`{latency} ms me aapke bato ka reply deta hu dekh rhe ho`")

@app.on_message(filters.text & cmd_filter('donation'))
async def donation(_, message):
    await message.reply_text(DONATION_TEXT)

@app.on_message(filters.text & cmd_filter('join'))
async def join(_, message):
    if group_calls.is_connected:
        await message.reply_text('Bot already join hai ab kya tere ghar me aajaye?')
        return
    group_calls.client = app
    await group_calls.start(message.chat.id)
    await message.reply_text('Khusi manao jaoin ho gya XD')

@app.on_message(filters.text & cmd_filter('mute'))
async def mute(_, message):
    group_calls.set_is_mute(is_muted=True)
    await message.reply_text('Succsessfully muted \nab nhi bol payega ye XD')

@app.on_message(filters.text & cmd_filter('unmute'))
async def unmute(_, message):
    group_calls.set_is_mute(is_muted=False)
    await message.reply_text('Succsessfully unmuted \nja bol le wo bhi kya yaad rakhega')

@app.on_message(filters.text & cmd_filter('volume'))
async def volume(_, message):
    if len(message.command) < 2:
        await message.reply_text('You forgot to pass volume (1-200)')

    await group_calls.set_my_volume(volume=int(message.command[1]))
    await message.reply_text(f'Volume changed to {message.command[1]}')

@app.on_message(filters.text & cmd_filter('stop'))
async def stop(_, message):
    global playing
    group_calls.stop_playout()
    queue.clear()
    playing = False
    await message.reply_text('Chal band kar diya gana XD')

@app.on_message(filters.text & cmd_filter('leave'))
async def leave(_, message):
    global playing
    if not group_calls.is_connected:
        await message.reply_text('Abe mai hu kha jo leave karu! Ajib aadmi ho tum bhi')
        return
    await group_calls.stop()
    queue.clear()
    playing = False
    group_calls.input_filename = ''
    await message.reply_text('Bahut gande ho tum mujhe wha se leave karwa diya :~(')

@app.on_message(filters.text & cmd_filter('kill'))
async def killbot(_, message):
    await send("__**Mar diya ðŸ˜ˆ!__**")
    quit()

@app.on_message(filters.text & cmd_filter('play'))
async def queues(_, message):
    if not group_calls.is_connected:
        await message.reply_text('Abe pahle bot ko VC to join karwa!')
        return
    usage = "**Usage:**\n__**/play youtube Song_Name**__"
    if len(message.command) < 3:
        await message.reply_text(usage)
        return
    text = message.text.split(None, 2)[1:]
    service = text[0]
    song_name = text[1]
    requested_by = message.from_user.first_name
    services = ["youtube", "deezer", "saavn"]
    if service not in services:
        await message.reply_text(usage)
        return
    if len(queue) > 0:
        await message.reply_text("__**Added To Queue.__**")
        queue.append({"service": service, "song": song_name,
                      "requested_by": requested_by})
        await play()
        return
    queue.append({"service": service, "song": song_name,
                  "requested_by": requested_by})
    await play()

@app.on_message(filters.text & cmd_filter('skip'))
async def skip(_, message):
    global playing
    if len(queue) == 0:
        await message.reply_text("__**Queue Is Empty. #sedlyf Shi bta teri girlfriend/boyfriend nhi na hai?**__")
        return
    playing = False
    await message.reply_text("__**Hue hue Hta diya wo gana!**__")
    await play()

@app.on_message(filters.text & cmd_filter('queue'))
async def queue_list(_, message):
    if len(queue) != 0:
        i = 1
        text = ""
        for song in queue:
            text += f"**{i}. Platform:** __**{song['service']}**__ | **Song:** __**{song['song']}**__\n"
            i += 1
        await message.reply_text(text)
    else:
        await message.reply_text("__**Queue Is Empty. #sedlyf Shi bta teri girlfriend/boyfriend nhi na hai?**__")

# Queue handler

async def play():
    global queue, playing
    while not playing:
        await asyncio.sleep(2)
        if len(queue) != 0:
            service = queue[0]["service"]
            song = queue[0]["song"]
            requested_by = queue[0]["requested_by"]
            if service == "youtube":
                playing = True
                del queue[0]
                try:
                    await ytplay(requested_by, song)
                except Exception as e:
                    print(str(e))
                    await send(str(e))
                    playing = False
            elif service == "saavn":
                playing = True
                del queue[0]
                try:
                    await jiosaavn(requested_by, song)
                except Exception as e:
                    print(str(e))
                    await send(str(e))
                    playing = False
            elif service == "deezer":
                playing = True
                del queue[0]
                try:
                    await deezer(requested_by, song)
                except Exception as e:
                    print(str(e))
                    await send(str(e))
                    playing = False

# Deezer----------------------------------------------------------------------------------------

async def deezer(requested_by, query):
    global playing
    m = await send(f"__**Searching for {query} on Deezer.**__")
    try:
        songs = await arq.deezer(query, 1)
        title = songs[0].title
        duration = convert_seconds(int(songs[0].duration))
        thumbnail = songs[0].thumbnail
        artist = songs[0].artist
        url = songs[0].url
    except Exception as e:
        await m.edit("__**Found No Song Matching Your Query.**__")
        playing = False
        print(str(e))
        return
    await m.edit("__**Generating Thumbnail.**__")
    await generate_cover_square(requested_by, title, artist, duration, thumbnail)
    await m.edit("__**Downloading And Transcoding.**__")
    await download_and_transcode_song(url)
    await m.delete()
    m = await app.send_photo(
        chat_id=sudo_chat_id,
        photo="final.png",
        caption=f"**Playing** __**[{title}]({url})**__ **Via Deezer.**",
    )
    os.remove("final.png")
    group_calls.input_filename = raw_filename
    await asyncio.sleep(int(songs[0]["duration"]))
    await m.delete()
    playing = False

# Jiosaavn--------------------------------------------------------------------------------------

async def jiosaavn(requested_by, query):
    global playing
    m = await send(f"__**Searching for {query} on JioSaavn.**__")
    try:
        songs = await arq.saavn(query)
        sname = songs[0].song
        slink = songs[0].media_url
        ssingers = songs[0].singers
        sthumb = songs[0].image
        sduration = songs[0].duration
        sduration_converted = convert_seconds(int(sduration))
    except Exception as e:
        await m.edit("__**Found No Song Matching Your Query.**__")
        print(str(e))
        playing = False
        return
    await m.edit("__**Processing Thumbnail.**__")
    await generate_cover_square(
        requested_by, sname, ssingers, sduration_converted, sthumb
    )
    await m.edit("__**Downloading And Transcoding.**__")
    await download_and_transcode_song(slink)
    await m.delete()
    m = await app.send_photo(
        chat_id=sudo_chat_id,
        caption=f"**Playing** __**{sname}**__ **Via Jiosaavn.**",
        photo="final.png",
    )
    os.remove("final.png")
    group_calls.input_filename = raw_filename
    await asyncio.sleep(int(sduration))
    await m.delete()
    playing = False

# Youtube Play-----------------------------------------------------------------------------------

async def ytplay(requested_by, query):
    global playing
    ydl_opts = {"format": "bestaudio"}
    m = await send(f"__**{query} Ye jo tera gana hai usko mai youtube pe dhundh rha hu wait kar**__")
    try:
        results = await arq.youtube(query, 1)
        link = f"https://youtube.com{results[0].url_suffix}"
        title = results[0].title
        thumbnail = results[0].thumbnails[0]
        duration = results[0].duration
        views = results[0].views
        if time_to_seconds(duration) >= 1800:
            await m.edit("__**Abe yaar bas 30min ke andar ke hi gane bajwa mere se Din bhar majduri karwata hai saram nhi aati?**__")
            playing = False
            return
    except Exception as e:
        await m.edit("__**Abe kaun se duniya ka gana diye ho be mila hi nhi #sedlyf mujhe pakka pta hai teri koi girlfriend nhi hogi hai na?**__")
        playing = False
        print(str(e))
        return
    await m.edit("__**Thumbnail bna rha hu wait kar**__")
    await generate_cover(requested_by, title, views, duration, thumbnail)
    await m.edit("__**Youtube devta se gane la rha hu XD**__")
    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
        info_dict = ydl.extract_info(link, download=False)
        audio_file = ydl.prepare_filename(info_dict)
        ydl.process_info(info_dict)
    await m.edit("__**Chhu mantar ye jadu chu hai gana bajadu mai**__")
    os.rename(audio_file, "audio.webm")
    transcode("audio.webm")
    await m.delete()
    m = await app.send_photo(
        chat_id=sudo_chat_id,
        caption=f"**Abhi ye baj rha hai** __**[{title}]({link})**__ **Youtube, SATYA aur Shubham devta ki kripa se**",
        photo="final.png",
    )
    os.remove("final.png")
    group_calls.input_filename = raw_filename
    await asyncio.sleep(int(time_to_seconds(duration)))
    playing = False
    await m.delete()

async def send(text):
    m = await app.send_message(sudo_chat_id, text=text, disable_web_page_preview=True)
    return m

print('[INFO] Bot is running...\n')
app.run()


from api.models import Incident, AffectedStation, Notification, Station
from django.utils import timezone

def create_incident(affected_stations, tweet_id):
    incidents = Incident.objects.filter(status=1)
    if incidents:
        # If there is active incidents
        # Check which stations are related to active incidents
        # Update incidents
        inc_updated = []
        for station in affected_stations:
            for incident in incidents:
                if AffectedStation.objects.filter(incident=incident, affected_station=station):
                    if incident.id not in inc_updated:

                        obj = Notification(tweet_id=tweet_id,
                                           incident=incident, date=timezone.now())
                        obj.save()
                        inc_updated.append(incident.id)
                    affected_stations = affected_stations.exclude(pk=station.pk)
        # If there are stations not related to active incidents
        # Create new incident
        if len(affected_stations) > 0:
            incident = Incident(status=1)
            incident.save()
            for station in affected_stations:
                obj = AffectedStation(
                    incident=incidents[0], affected_station=station)
                obj.save()

            noti = Notification(tweet_id=tweet_id,
                                incident=incident, date=timezone.now())
            noti.save()
    else:
        # If there is no active incidents
        # Create incident
        incident = Incident(status=1)
        incident.save()

        # Create notification

        noti = Notification(tweet_id=tweet_id,
                            incident=incident, date=timezone.now())
        noti.save()

        # Add affected stations to incident
        for station in affected_stations:
            af_st = AffectedStation(
                incident=incident, affected_station=station)
            af_st.save()

def full_incident(line, tweet, type="O"):
    """
    Change the status of all stations of a line to the given type.
    """
    stations_of_line = Station.objects.filter(line=line)
    if type == "O":
        incidents = Incident.objects.filter(status=1)
        if incidents:
            for incident in incidents:
                affected_stations = AffectedStation.objects.filter(
                    incident=incident)
                affected_stations_len = len(affected_stations)
                station_count = 0
                for station in affected_stations:
                    if station in stations_of_line:
                        station_count += 1
                        station.status = "O"
                        station.save()
                if station_count == affected_stations_len:
                    # If the incident stations are all related to the line
                    incident.status = 0
                if station_count > 0:

                    notif = Notification(tweet_id=tweet,
                                         incident=incident, date=timezone.now())
                    notif.save()
                incident.updated_at = timezone.now()
                incident.save()
    else:
        for station in stations_of_line:
            station.status = "M"
            station.save()
        incident = Incident(status=1)
        incident.save()
        for station in stations_of_line:
            affected_station = AffectedStation(
                incident=incident, affected_station=station)
            affected_station.save()

        notif = Notification(tweet_id=tweet,
                             incident=incident, date=timezone.now())
        notif.save()


# -*- coding: utf-8 -*-
"""
Created on Wed Jun  3 12:11:37 2020

@author: Mack Pro
"""

import math
import random

import matplotlib.pyplot as plt
import numpy as npy

import volmdlr as volmdlr
import volmdlr.primitives2D as primitives2D
import volmdlr.primitives3D as primitives3D

rmin, rmax = 10, 100
posmin, posmax = -100, 100
x1, y1, z1 = random.randrange(posmin, posmax, 1)/100, random.randrange(posmin, posmax, 1)/100, random.randrange(posmin, posmax, 1)/100

r1 = random.randrange(rmin, rmax, 1)/1000
c1 = volmdlr.Point3D([x1,y1,z1])

x3, y3, z3 = random.randrange(posmin, posmax, 1)/100, random.randrange(posmin, posmax, 1)/100, random.randrange(posmin, posmax, 1)/100

n1 = volmdlr.Vector3D([x3,y3,z3])
n1 = n1.unit_vector() #Normalize the normal if it is not the case
plane1 = volmdlr.Plane3D.from_normal(c1, n1)

frame1 = volmdlr.Frame3D(c1, plane1.vectors[0], plane1.vectors[1], n1) #Frame in the center of the cylinder
cylsurface1 = volmdlr.CylindricalSurface3D(frame1, r1)

hmin, hmax = -50, 50

h1 = random.randrange(hmin, hmax, 5)/100

center2d = c1.To2D(c1, plane1.vectors[0], plane1.vectors[1])
#Classic Contour
segbh = volmdlr.LineSegment2D(center2d, center2d + volmdlr.Point2D((0,h1))) #### Minus Pt2D because of Step adaptation
circlestart = volmdlr.LineSegment2D(segbh.points[1], segbh.points[1]+volmdlr.Point2D((2*math.pi*3/4,0))) #You can change 2*pi by an other angle
seghb = volmdlr.LineSegment2D(circlestart.points[1],circlestart.points[1]-segbh.points[1])
circlend = volmdlr.LineSegment2D(seghb.points[1],segbh.points[0])
edges = [segbh, circlestart, seghb, circlend]
points = edges[0].points
contours =  [volmdlr.Contour2D(edges)]

cyl1 = volmdlr.CylindricalFace3D(contours, cylsurface1, points)

pts1, tangle1 = cyl1.triangulation(resolution=12)

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
[pt.MPLPlot(ax=ax) for pt in pts1]

# number_holes = 5

# outer_circle = volmdlr.Circle2D(volmdlr.O2D, 0.06)

# circles = []
# delta_angle = 2*math.pi/number_holes
# inner_circle = volmdlr.Circle2D(volmdlr.O2D, 0.04)
# first_circle = volmdlr.Circle2D(volmdlr.Point2D((0, 0.05)), 0.005)
# circles = [inner_circle, first_circle]
# for i in range(1, number_holes):
#     circles.append(first_circle.Rotation(volmdlr.O2D, i*delta_angle))

# xn, yn, zn = random.randrange(posmin, posmax, 1)/100, random.randrange(posmin, posmax, 1)/100, random.randrange(posmin, posmax, 1)/100
# xc, yc, zc = random.randrange(posmin, posmax, 1)/100, random.randrange(posmin, posmax, 1)/100, random.randrange(posmin, posmax, 1)/100

# n = volmdlr.Vector3D([xn,yn,zn])
# n.Normalize()
# c = volmdlr.Point3D((xc,yc,zc))
# plane = volmdlr.Plane3D.from_normal(c, n)
# contours = [volmdlr.Contour2D([outer_circle])]#, inner_circle])]#, volmdlr.Contour2D(circles)]
# planeface = volmdlr.PlaneFace3D(contours,  plane)

# p1, p2 = cyl1.minimum_distance_points_plane(planeface)
# # print(p1.point_distance(p2))
# # print(p1)
# # print(p2)
# pts2, t2 = planeface.triangulation()
# [pt.MPLPlot(ax=ax) for pt in pts2]
# p1.MPLPlot(ax=ax, color='r')
# p2.MPLPlot(ax=ax, color='b')

##### extrusion 1

p1=volmdlr.Point2D((0, 0))
p2=volmdlr.Point2D((0.1, 0.))
p3=volmdlr.Point2D((0.1, 0.2))
p4=volmdlr.Point2D((0.05, 0.1))
p5=volmdlr.Point2D((0.,0.21))
p6=volmdlr.Point2D((0.05, 0.05))

p7 = volmdlr.Point2D((0.06, 0.05))
p8 = volmdlr.Point2D((0.04, 0.07))

radius = {0: 0.01, 2: 0.01, 3: 0.015}

outer_profile = primitives2D.ClosedRoundedLineSegments2D([p1, p2, p3, p4, p5], radius)
#hole = volmdlr.Circle2D(p6, 0.01)
#inner_profile = primitives2D.RoundedLineSegments2D([p6, p7, p8], {0: 0.5}, closed = True)
l1 = volmdlr.LineSegment2D(p6, p7)
l2 = volmdlr.LineSegment2D(p7, p8)
l3 = volmdlr.LineSegment2D(p8, p6)
c2 = volmdlr.Contour2D([l1,l2,l3])

#c1 = volmdlr.Contour2D([outer_profile])
#c2 = volmdlr.Contour2D([inner_profile])

# f, a = outer_profile.MPLPlot()
# c2.MPLPlot(a)

profile=primitives3D.ExtrudedProfile(volmdlr.O3D, volmdlr.Y3D, volmdlr.Z3D, outer_profile, [c2], volmdlr.X3D*0.1, name = 'extrusion')
dmin, p1 ,p2 = cyl1.minimum_distance(profile.faces[0], return_points=True)
face_min = profile.faces[0]
# print('dmin', dmin)
for face in profile.faces[1:] :
    dtest, ptest1, ptest2 = cyl1.minimum_distance(face, return_points=True)
    # print('dtest', dtest)
    if dtest < dmin :
        p1, p2 = ptest1, ptest2
        dmin = dtest
        face_min = face

# print('>>>>>>>>> distance minimale', dmin)

shell = volmdlr.Shell3D([cyl1])
vol = volmdlr.VolumeModel([shell, profile, p1, p2])
vol.babylonjs_from_script()
# m = volmdlr.VolumeModel([shell, profile])
# m.babylonjs()


# Author: Quentin Bertrand <quentin.bertrand@inria.fr>
#         Mathurin Massias <mathurin.massias@gmail.com>
#         Salim Benchelabi
# License: BSD 3 clause

from abc import abstractmethod

import numpy as np
from numba import float64
from numba.experimental import jitclass
from numba.types import bool_

from andersoncd.utils import ST

class BasePenalty():
    @abstractmethod
    def value(self, w):
        """Value of penalty at vector w."""

    @abstractmethod
    def prox_1d(self, value, stepsize, j):
        """Proximal operator of penalty for feature j."""

    @abstractmethod
    def subdiff_distance(self, w, grad, ws):
        """Distance of gradient to subdifferential of penalty for features
        in the working set ws.

        Parameters
        ----------
        w: array, shape (n_features,)
            Coefficient vector.
        grad: array, shape (ws.shape[0],)
            Gradient of the datafit at w.
        ws: array
            Indices of features in the working set.

        Returns
        -------
        distances: array, shape (ws.shape[0],).

        """

    @abstractmethod
    def is_penalized(self, n_features):
        """Mask corresponding to penalized features."""

spec_L1 = [
    ('alpha', float64),
]

@jitclass(spec_L1)
class L1(BasePenalty):
    def __init__(self, alpha):
        self.alpha = alpha

    def value(self, w):
        """ alpha * ||w||_1
        """
        return self.alpha * np.sum(np.abs(w))

    def prox_1d(self, value, stepsize, j):
        return ST(value, self.alpha * stepsize)

    def subdiff_distance(self, w, grad, ws):
        subdiff_dist = np.zeros_like(grad)
        for idx, j in enumerate(ws):
            if w[j] == 0:
                # distance of - grad_j to  [-alpha, alpha]
                subdiff_dist[idx] = max(0, np.abs(grad[idx]) - self.alpha)
            else:
                # distance of - grad_j to alpha * sign(w[j])
                subdiff_dist[idx] = np.abs(
                    - grad[idx] - np.sign(w[j]) * self.alpha)
        return subdiff_dist

    def is_penalized(self, n_features):
        return np.ones(n_features, bool_)

spec_L1_plus_L2 = [
    ('alpha', float64),
    ('l1_ratio', float64),
]

@jitclass(spec_L1_plus_L2)
class L1_plus_L2(BasePenalty):
    def __init__(self, alpha, l1_ratio):
        self.alpha = alpha
        self.l1_ratio = l1_ratio

    def value(self, w):
        """ alpha * (l1_ratio * ||w||_1 + 0.5 * (1 - l1_ratio) * ||w||_2^2)
        """
        value = self.l1_ratio * self.alpha * np.sum(np.abs(w))
        value += (1 - self.l1_ratio) * self.alpha / 2 * np.sum(w ** 2)
        return value

    def prox_1d(self, value, stepsize, j):
        prox = ST(value, self.l1_ratio * self.alpha * stepsize)
        prox /= (1 + stepsize * (1 - self.l1_ratio) * self.alpha)
        return prox

    def subdiff_distance(self, w, grad, ws):
        subdiff_dist = np.zeros_like(grad)
        for idx, j in enumerate(ws):
            if w[j] == 0:
                # distance of - grad_j to alpha * l1_ratio * [-1, 1]
                subdiff_dist[idx] = max(
                    0, np.abs(grad[idx]) - self.alpha * self.l1_ratio)
            else:
                # distance of - grad_j to alpha * l_1 ratio * sign(w[j]) +
                # alpha * (1 - l1_ratio) * w[j]
                subdiff_dist[idx] = np.abs(
                    - grad[idx] -
                    self.alpha * (self.l1_ratio *
                                  np.sign(w[j]) + (1 - self.l1_ratio) * w[j]))
        return subdiff_dist

    def is_penalized(self, n_features):
        return np.ones(n_features).astype(bool_)

spec_WeightedL1 = [
    ('alpha', float64),
    ('weights', float64[:]),
]

@jitclass(spec_WeightedL1)
class WeightedL1(BasePenalty):
    def __init__(self, alpha, weights):
        self.alpha = alpha
        self.weights = weights.astype(np.float64)

    def value(self, w):
        """ alpha * ||weights * w||_1
        """
        return self.alpha * np.sum(np.abs(w) * self.weights)

    def prox_1d(self, value, stepsize, j):
        return ST(value, self.alpha * stepsize * self.weights[j])

    def subdiff_distance(self, w, grad, ws):
        subdiff_dist = np.zeros_like(grad)
        for idx, j in enumerate(ws):
            if w[j] == 0:
                # distance of - grad_j to alpha * weights[j] * [-1, 1]
                subdiff_dist[idx] = max(
                    0, np.abs(grad[idx]) - self.alpha * self.weights[j])
            else:
                # distance of - grad_j to alpha * weights[j] * sign(w[j])
                subdiff_dist[idx] = np.abs(
                    - grad[idx] - self.alpha * self.weights[j] * np.sign(w[j]))
        return subdiff_dist

    def is_penalized(self, n_features):
        return self.weights != 0

spec_MCP = [
    ('alpha', float64),
    ('gamma', float64),
]

@jitclass(spec_MCP)
class MCP_pen(BasePenalty):
    def __init__(self, alpha, gamma):
        self.alpha = alpha
        self.gamma = gamma

    def value(self, w):
        """
        With x >= 0
        pen(x) = alpha * x - x^2 / (2 * gamma) if x =< gamma * alpha
                 gamma * alpha 2 / 2           if x > gamma * alpha
        value = sum_{j=1}^{n_features} pen(|w_j|)

        For more details see
        Coordinate descent algorithms for nonconvex penalized regression,
        with applications to biological feature selection, Breheny and Huang.
        """
        s0 = np.abs(w) < self.gamma * self.alpha
        value = np.full_like(w, self.gamma * self.alpha ** 2 / 2.)
        value[s0] = self.alpha * np.abs(w[s0]) - w[s0]**2 / (2 * self.gamma)
        return np.sum(value)

    def prox_1d(self, value, stepsize, j):
        tau = self.alpha * stepsize
        g = self.gamma / stepsize  # what does g stand for ?
        if np.abs(value) <= tau:
            return 0.
        if np.abs(value) > g * tau:
            return value
        return np.sign(value) * (np.abs(value) - tau) / (1. - 1./g)

    def subdiff_distance(self, w, grad, ws):
        subdiff_dist = np.zeros_like(grad)
        for idx, j in enumerate(ws):
            if w[j] == 0:
                # distance of grad to alpha * [-1, 1]
                subdiff_dist[idx] = max(0, np.abs(grad[idx]) - self.alpha)
            elif np.abs(w[j]) < self.alpha * self.gamma:
                # distance of grad_j to (alpha - abs(w[j])/gamma) * sign(w[j])
                subdiff_dist[idx] = np.abs(np.abs(grad[idx]) - (
                    self.alpha - np.abs(w[j])/self.gamma))
            else:
                # distance of grad to 0
                subdiff_dist[idx] = np.abs(grad[idx])
        return subdiff_dist

    def is_penalized(self, n_features):
        return np.ones(n_features, bool_)


from django.shortcuts import render, redirect
from django.views import View
from django.contrib import messages
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.http import HttpResponseRedirect
from django.utils.html import strip_tags
from django.utils.decorators import method_decorator

from backend.forms import Seat
from backend.models import Agency, Vehicle
from backend.forms import SeatForm
from core.utils.decorators import MustLogin

class SeatListView(PermissionRequiredMixin, View):
    permission_required = [
        "backend.view_seat",
    ]
    template = "backend/lists/seats.html"

    @method_decorator(MustLogin)
    def get(self, request, *args, **kwargs):
        if request.user.is_staff or request.user.is_superuser:
            seats = Seat.objects.all().order_by('-id')
        elif request.user.is_agency_admin:
            seats = Seat.objects.filter(vehicle__agency=request.user.agency)
        context = {'seats': seats}
        return render(request, self.template, context)

class CreateUpdateSeat(PermissionRequiredMixin, View):
    permission_required = [
        "backend.add_seat",
        "backend.change_seat",
    ]

    template = "backend/create_update_seat.html"

    @method_decorator(MustLogin)
    def get(self, request, *args, **kwargs):
        seat_id = request.GET.get('seat_id')
        vehicle_id = request.GET.get('vehicle_id')
        if request.user.is_staff or request.user.is_superuser:
            seat = Seat.objects.filter(id=seat_id).first()
            vehicle = Vehicle.objects.filter(id=vehicle_id).first()  # noqa
            vehicles = Vehicle.objects.all()
        elif request.user.is_agency_admin:
            seat = Seat.objects.filter(id=seat_id, vehicle__agency=request.user.agency).first()  # noqa
            vehicle = Vehicle.objects.filter(id=vehicle_id, agency=request.user.agency).first()  # noqa
            vehicles = Vehicle.objects.filter(agency=request.user.agency)
        context = {
            "seat": seat,
            "vehicle": vehicle,
            "vehicles": vehicles,
        }  # noqa
        return render(request, self.template, context)

    @method_decorator(MustLogin)
    def post(self, request, *args, **kwargs):
        seat_id = request.POST.get('seat_id')
        vehicle_id = request.POST.get('vehicle_id')
        if request.user.is_staff or request.user.is_superuser:
            vehicle = Vehicle.objects.filter(id=vehicle_id).first()
        elif request.user.is_agency_admin:
            vehicle = Vehicle.objects.filter(id=vehicle_id, agency=request.user.agency).first()  # noqa

        if seat_id:
            # seat exists - update seat
            if request.user.is_staff or request.user.is_superuser:
                seat = Seat.objects.filter(id=seat_id).first()
            elif request.user.is_agency_admin:
                seat = Seat.objects.filter(id=seat_id, vehicle__agency=request.user.agency).first()  # noqa
            form = SeatForm(request.POST, request.FILES, instance=seat)  # noqa
            if form.is_valid():
                new_seat = form.save(commit=False)
                new_seat.vehicle = vehicle
                new_seat.save()
                messages.success(request, 'Seat Detail Updated Successfully.')  # noqa
                return redirect('backend:seats')
            else:
                for field, error in form.errors.items():
                    message = f"{field.title()}: {strip_tags(error)}"
                    break
                messages.warning(request, message)
                return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
        else:
            # it's a new seat - create seat
            form = SeatForm(request.POST, request.FILES)
            if form.is_valid():
                seat = form.save(commit=False)
                seat.vehicle = vehicle
                seat.save()
                messages.success(request, 'New Seat Created Successfully.')  # noqa
                return redirect('backend:seats')
            else:
                for field, error in form.errors.items():
                    message = f"{field.title()}: {strip_tags(error)}"
                    break
                messages.warning(request, message)
                return HttpResponseRedirect(request.META.get("HTTP_REFERER"))

class DeleteSeat(PermissionRequiredMixin, View):
    permission_required = [
        "backend.delete_seat",
    ]

    @method_decorator(MustLogin)
    def get(self, request, *args, **kwargs):
        return redirect('backend:seats')

    @method_decorator(MustLogin)
    def post(self, request, *args, **kwargs):
        seat_id = request.POST.get('seat_id')
        if request.user.is_staff or request.user.is_superuser:
            seat = Seat.objects.filter(id=seat_id).first()
        elif request.user.is_agency_admin:
            seat = Seat.objects.filter(
                id=seat_id, vehicle__agency=request.user.agency).first()
        seat.delete()
        messages.success(request, 'Vehicle Seat Deleted Successfully.')
        return HttpResponseRedirect(request.META.get("HTTP_REFERER"))


import ee
from functools import partial

from hydrafloods import filtering

def fwdet(water_img, dem, band=None, outlier_test=False, force_projection=False):
    """Implementation of the Flood Water Depth Estimation Tool
    Used to calculate water depth given a water map and elevation data
    Original paper: https://doi.org/10.5194/nhess-19-2053-2019
    Earth Engine paper: https://doi.org/10.1109/LGRS.2020.3031190

    args:
        water_img (ee.Image): earth engine image object representing the water extent. Assumes values of 1 equal water.
        dem (ee.Image): earth engine image object representing elevation
        band (str | None): band name to use for algorithm, if set to `None` will use first band in image. default = None
        outlier_test (bool): flag used to find and filter outlier dem values around water boundaries. default = False
        force_projection (bool): flag used to force the output image projection to that of the input dem. default = False

    returns:
        ee.Image: image of water depth in meters
    """

    proj = dem.projection()
    res = proj.nominalScale().multiply(1.5)

    # select band to use for algorithm
    if band is None:
        water_img = water_img.select([0]).selfMask()

    else:
        water_img = water_img.select(ee.String(band)).selfMask()

    if outlier_test:
        filled = filtering.modified_median_zscore(dem)

        expand = water_img.focal_max(
            kernel=ee.Kernel.square(radius=res, units="meters")
        )

        dem_mask = dem.updateMask(water_img.gt(0))

        boundary = dem_mask.add(expand)

        filled = filtering.modified_median_zscore(boundary, filled)

    else:
        filled = dem

    # cumulative cost model

    mod = filled.updateMask(water_img.mask().eq(0))
    source = mod.mask()

    val = 10000
    push = 5000

    cost0 = ee.Image(val).where(source, 0).cumulativeCost(source, push)
    cost1 = ee.Image(val).where(source, 1).cumulativeCost(source, push)
    cost2 = mod.unmask(val).cumulativeCost(source, push)

    costFill = cost2.subtract(cost0).divide(cost1.subtract(cost0))

    costSurface = mod.unmask(0).add(costFill)

    # Kernel for low-pass filter
    boxcar = ee.Kernel.square(radius=3, units="pixels", normalize=True)

    # Floodwater depth calculation and smoothing using a low-pass filter
    depths = (
        costSurface.subtract(filled).convolve(boxcar).updateMask(water_img.mask())
    ).rename("depth")

    # force min values to be 0
    depths = depths.max(ee.Image.constant(0))

    # force the output project to be that of the input dem if force_projection is true
    if force_projection:
        depths = depths.reproject(proj)

    return depths

def fwdet_experimental(
    water_img,
    dem,
    iter=10,
    band=None,
    pixel_edge=0,
    boundary_definition="mean",
    smooth_depths=True,
    force_projection=False,
):
    """Experimantal changes to the implementation of the Flood Water Depth Estimation Tool
    Used to calculate water depth given a water map and elevation data

    Difference from original FwDET implementation include ...

    args:
        water_img (ee.Image): earth engine image object representing the water extent.
        dem (ee.Image): earth engine image object representing elevation
        iter (int|ee.Number, optional): keyword to set number of iterations to fill up every pixel.
            The algorithm will calculate minimum iterations needed and use the max between the two.
            default = 10
        band (str | None,optional): band name to use for algorithm, if set to `None` will use first band in image. default = None
        pixel_edge (int): value that represents the boundary/edge of water. Note: must be client-side int object. default = 0
        boundary_definition (str): method for defining the elevation at water edges. Options are 'mean' or 'nearest'
            Note: 'nearest' is the FwDET original method, 'mean' is an updated method. default=mean
        smooth_depths (bool): flag to control if the resulting water depths should be smoothed or not, uses a 3x3 pixel mean. default=True
        force_projection (bool): flag used to force the output image projection to that of the input dem. default = False

    returns:
        ee.Image: image of water depth in meters
    """
    proj = dem.projection()

    res = water_img.projection().nominalScale().multiply(1.5)

    # select band to use for algorithm
    if band is None:
        water_img = water_img.select([0])

    else:
        water_img = water_img.select(ee.String(band))

    # detect water edge
    watermap_edge = (
        water_img.selfMask().unmask(-999).focal_min(res, "square", "meters").eq(-999)
    )
    watermap_edge = watermap_edge.updateMask(water_img.unmask(0))
    watermap_edge = watermap_edge.selfMask()
    # watermap_edge = watermap_edge.reproject(water_img.projection());

    # detect watermap extent outer edge (not to be used in algorithm)
    if pixel_edge > 0:
        outer_edge = (
            water_img.mask()
            .unmask(0)
            .gte(0)
            .unmask(0, False)
            .gt(0)
            .focal_min(res.multiply(ee.Number(pixel_edge).add(1)), "square", "meters")
            .eq(0)
        )
        outer_edge = outer_edge.updateMask(outer_edge)
        # outer_edge = outer_edge.reproject(water_img.projection())
        # update watermap edge
        watermap_edge = watermap_edge.updateMask(outer_edge.unmask(0).eq(0))
        # watermap_edge = watermap_edge.reproject(water_img.projection());
    else:
        outer_edge = water_img.mask().unmask(0).lt(0)

    DEM = dem  # being super lazy here by setting a variable vs renaming things...

    # get elevation at edge
    # DEM_watered_edge = watermap_edge.multiply(DEM); # simply using boundary cell elevations
    DEM_masked = DEM.updateMask(
        water_img.unmask(0).eq(0).add(watermap_edge.unmask(0)).eq(1)
    )
    DEM_watered_edge = watermap_edge.multiply(
        DEM_masked.focal_mean(res, "square", "meters")
    )
    DEM_watered_edge = DEM_watered_edge  # .reproject(water_img.projection())
    # calculate (approximation of) minimum needed iterations to fill up every pixel
    req_iter = (
        water_img.eq(0)
        .add(watermap_edge)
        .fastDistanceTransform(50, "pixels", "squared_euclidean")
        .sqrt()
        .round()
        .updateMask(water_img)
        .updateMask(outer_edge.unmask(0).eq(0))
        .reproject(water_img.projection())
    )
    req_iter = req_iter.reduceRegion(
        reducer=ee.Reducer.max(),
        geometry=water_img.geometry().bounds(1e3),
        scale=res,
        maxPixels=1e12,
    ).get("distance")
    # set iterations to be used
    iter = ee.Number(req_iter).max(iter)
    # construct list to iterate over (list contents unused, just for iterations)
    iter_list = ee.List.sequence(0, iter)

    # use look up which algorithm to use for defining the boundary elevation values
    boundary_algos = {"mean": _mbce, "nearest": _nbce}
    f_boundary = partial(
        boundary_algos[boundary_definition], mask=water_img, resolution=res
    )

    # apply boundary condition algo, starting with edge values and filling it up further with each iteration
    DEM_watered_fill = ee.Image(iter_list.iterate(f_boundary, DEM_watered_edge))
    DEM_watered = DEM_watered_fill.reproject(water_img.projection())
    # derive water depths
    depths = DEM_watered.subtract(DEM).max(0)

    # smooth water depths
    # Cohen et al. (2018) use a 3x3 pixel mean (but FwDET-GEE seems to use a boxcar kernel?)
    if smooth_depths:
        depths = depths.reduceNeighborhood(
            reducer=ee.Reducer.mean(),
            kernel=ee.Kernel.square(res, units="meters"),
            optimization="boxcar",  # optimization for mean.
        )

    # force the output project to be that of the input dem if force_projection is true
    if force_projection:
        depths = depths.reproject(proj)

    return depths.rename("depth").set("depth_iter", iter)

# Mean Boundary Cell Elevation (MBCE), adapted from NBCE above
# MBCE function
def _mbce(i, img, mask=None, resolution=30):
    """Mean Boundary Cell Elevation (MBCE), unpublished A. Haag (2021)"""
    # obtain mean boundary cell elevation
    new_img = ee.Image(img).focal_mean(resolution, "square", "meters")
    # make sure the original / previous step values are kept intact (comment out for more smoothing)
    # new_img = new_img.where(ee.Image(img).gt(-999), ee.Image(img));
    # make sure it does not expand to other side of floodmap (where there is no water)
    return new_img.updateMask(mask.unmask(0))  # .reproject(floodmap.projection())

# Nearest Boundary Cell Elevation (NBCE) algorithm
def _nbce(i, img, mask=None, resolution=30):
    """Nearest Boundary Cell Elevation (NBCE), Cohen et al. (2018)"""
    # obtain nearest boundary cell elevations using different kernels
    new_img_0 = ee.Image(img).focal_min(resolution, "square", "meters")
    new_img_1 = ee.Image(img).focal_min(resolution, "plus", "meters")
    new_img_2 = ee.Image(img).focal_min(resolution, "cross", "meters")
    # make sure the original / previous step values are kept intact
    new_img_0 = new_img_0.where(ee.Image(img).gt(-999), ee.Image(img))
    new_img_1 = new_img_1.where(ee.Image(img).gt(-999), ee.Image(img))
    new_img_2 = new_img_2.where(ee.Image(img).gt(-999), ee.Image(img))
    # add the NBCE in right order (minimum distance at the end, overwrites diagonal)
    new_img = new_img_0.where(new_img_2.gt(-999), new_img_2)
    new_img = new_img.where(new_img_1.gt(-999), new_img_1)
    # make sure it does not expand to other side of floodmap (where there is no water)
    return new_img.updateMask(mask.unmask(0))

def downscale_wfraction(fraction, dem):
    """Algorithm to downscale water fraction maps using DEM data
    Converts fractional water data to higher resolution binary water data
    Paper: https://doi.org/10.1016/j.rse.2013.03.015

    args:
        fraction (ee.Image): earth engine image object representing the fractional water extent within each pixel.
        dem (ee.Image): earth engine image object representing elevation

    returns:
        ee.Image: downscaled binary image of water
    """

    resolution = fraction.projection().nominalScale()

    water_present = fraction.gt(0.0)

    h_min = dem.updateMask(water_present).focal_min(resolution, "square", "meters")
    h_max = dem.updateMask(water_present).focal_max(resolution, "square", "meters")

    water_high = h_min.add(h_max.subtract(h_min).multiply(fraction))

    return dem.gte(h_min).And(dem.lt(water_high))


import pandas as pd
from datetime import datetime

def date_to_orbit(date):
    #establish path to orbit reference file
    filename = "./VEX_data_files/orbit_ref.tab"
    #open orbit reference file into pandas dataframe
    with open(filename) as f:
        orbit_ref = pd.read_csv(f,sep='\t',header=None,names = ['orbit','start','end','BS_in','BS_out'])
    #find orbit start and end times from whole file
    orbit_ref['start'] = pd.to_datetime(orbit_ref['start'])
    orbit_ref['end'] = pd.to_datetime(orbit_ref['end'])
    #make desired date a datetime instead of a string
    date = datetime(int(date[0:4]),int(date[5:7]),int(date[8:10]))

    #for each orbit in orbit reference file
    for i in orbit_ref.index:
        #if desired date is within two orbits, break
        if (date >= orbit_ref['start'][i]) and (date <= orbit_ref['end'][i]):
            orbit_num = orbit_ref['orbit'][i]
            break
    f.close()
    #return orbit number of date
    return orbit_num


from .views import *
from django.urls import path

urlpatterns=[
    path('search/',ArticleList.as_view(),name='search'),
    path('type_ahead_wikis/',WikiSearchAPI.as_view(),name='type_ahead_wikis'),
    path('article/',FetchArticle.as_view(), name='article')
]


import time
from paddle import Pad
from turtle import Turtle,Screen
from ball import Ball

screen=Screen()
screen.setup(width=800,height=600)
screen.bgcolor("black")

ball=Ball()

rpad=Pad(380,0)
lpad=Pad(-380,0)

screen.listen()
screen.onkey(rpad.goup,"Up")
screen.onkey(rpad.godown,"Down")

screen.onkey(lpad.goup,"w")
screen.onkey(lpad.godown,"s")

gameison=True
while gameison:
    time.sleep(0.05)
    screen.update()
    ball.move()
    if ball.ycor()>=290 or ball.ycor()< -290:
        ball.bounce()

    if ball.distance(rpad)<50 and ball.xcor()>350 or ball.distance(rpad)<50 or ball.xcor() < -350 :
        print("made contact")
        ball.bounce_back()

screen.exitonclick()


import pygame
from pygame.locals import*
import sys

from scripts.config import _
from scripts.menu import *

def AireCraftTutorial(screen):

    running = True
    scroll_y = 0
    scroll_speed = 15

    while running:
        screen.fill((0,0,0))

        draw_text(screen, 'move mouse and click on left buttom for simple fire & right buttom for extra fire.', _.font, (255, 255, 255), 0, 20+scroll_y)
        drawImage(screen, "resources/images/airecraft/ac_2.png", 0, 80+scroll_y)
        drawImage(screen, "resources/images/mira.gif", 0, 250+scroll_y)
        draw_text(screen, 'press [Echap] to return', _.font, (255, 255, 255), 0, 420+scroll_y)

        for event in pygame.event.get():
            if event.type == QUIT:
                pygame.quit()
                sys.exit()
            if event.type == KEYDOWN:
                if event.key == K_ESCAPE:
                    running = False
                elif event.key == K_UP: # up
                    scroll_y += scroll_speed
                elif event.key == K_DOWN: # down
                    scroll_y -= scroll_speed
            if event.type == pygame.MOUSEBUTTONDOWN:
                if event.button == 4: # up
                    scroll_y += scroll_speed
                if event.button == 5: # down
                    scroll_y -= scroll_speed

            if scroll_y > 0:
                scroll_y = 0

        pygame.display.update()
        _.mainClock.tick(_.FPS)