"""This module defines the Essence Narayan16 API"""

from pathlib import Path
from typing import List

import numpy as np
from astropy.table import Table

from ..base_classes import DefaultParser, PhotometricRelease
from ..exceptions import InvalidObjId
from ..utils import downloads, unit_conversion

def _format_table_to_sncosmo(data_table: Table) -> Table:
    """Format a data table for use with SNCosmo

    Args:
        data_table: A data table returned by ``get_data_for_id``

    Returns:
        The same data in a new table following the SNCosmo data model
    """

    out_table = Table()
    out_table.meta = data_table.meta

    out_table['time'] = unit_conversion.convert_to_jd(data_table['MJD'], format='MJD')
    out_table['band'] = ['csp_dr3_' + band for band in data_table['Passband']]
    out_table['zp'] = np.full(len(data_table), 25)
    out_table['zpsys'] = np.full(len(data_table), 'ab')
    out_table['flux'] = data_table['Flux']
    out_table['fluxerr'] = np.max(
        [data_table['Fluxerr_hi'], data_table['Fluxerr_lo']], axis=0)

    return out_table

class Narayan16(DefaultParser, PhotometricRelease):
    """The ``Narayan16`` class provides access to photometric data for 213
    Type Ia supernovae discovered by the ESSENCE survey at redshifts
    0.1 <= z <= 0.81 between 2002 and 2008. It includes R and I band photometry
    measured from images obtained using the MOSAIC II camera at the CTIO
    Blanco telescope. (Source: Narayan et al. 2016)

    Deviations from the standard UI:
        - None

    Cuts on returned data:
        - None
    """

    # General metadata
    survey_name = 'Equation of State: Supernovae trace Cosmic Expansion'
    survey_abbrev = 'ESSENCE'
    release = 'Narayan16'
    survey_url = 'http://www.ctio.noao.edu/essence/'
    publications = ('Narayan et al. 2016',)
    ads_url = 'https://ui.adsabs.harvard.edu/abs/2016ApJS..224....3N/abstract'

    # Photometric metadata (Required for photometric data, otherwise delete)
    band_names = ('essence_narayan16_R', 'essence_narayan16_I')
    zero_point = (27.5, 27.5)

    def __init__(self):
        """Define local and remote paths of data"""

        super().__init__()

        # Local paths
        self._table_dir = self._data_dir / 'vizier'
        self._photometry_dir = self._table_dir / 'lcs'
        self._filter_dir = self._data_dir / 'filters'

        # Define urls for remote data
        self._table_url = 'http://cdsarc.u-strasbg.fr/viz-bin/nph-Cat/tar.gz?J/ApJS/224/3'
        self._filter_urls = (
            'https://www.noao.edu/kpno/mosaic/filters/asc6004.f287.r04.txt',
            'https://www.noao.edu/kpno/mosaic/filters/asc6028.f287.r04.txt'
        )

        self._filter_file_names = ('R_band.dat', 'I_band.dat')

    def _get_available_ids(self) -> List[str]:
        """Return a list of target object IDs for the current survey"""

        files = self._photometry_dir.glob('*.dat')
        return sorted(Path(f).name.split('.')[0] for f in files)

    # noinspection PyUnusedLocal
    def _get_data_for_id(self, obj_id: str, format_table: bool = True) -> Table:
        """Returns data for a given object ID

        Args:
            obj_id: The ID of the desired object
            format_table: Format for use with ``sncosmo`` (Default: True)

        Returns:
            An astropy table of data for the given ID
        """

        if obj_id not in self.get_available_ids():
            raise InvalidObjId()

        # Get photometric data
        path = self._photometry_dir / f'{obj_id}.W6yr.clean.nn2.Wstd.dat'
        data_table = Table.read(
            path, format='ascii',
            names=['Observation', 'MJD', 'Passband', 'Flux', 'Fluxerr_lo', 'Fluxerr_hi']
        )

        # Get meta data
        table_6 = self.load_table(6)
        object_metadata = table_6[table_6['ESSENCE'] == obj_id][0]
        ra, dec = unit_conversion.hourangle_to_degrees(
            rah=object_metadata['RAh'],
            ram=object_metadata['RAm'],
            ras=object_metadata['RAs'],
            dec_sign=object_metadata['DE-'],
            decd=object_metadata['DEd'],
            decm=object_metadata['DEm'],
            decs=object_metadata['DEs']
        )

        data_table.meta['obj_id'] = obj_id
        data_table.meta['ra'] = ra
        data_table.meta['dec'] = dec
        data_table.meta['z'] = object_metadata['zSNID']
        data_table.meta['z_err'] = object_metadata['e_zSNID']
        del data_table.meta['comments']

        if format_table:
            data_table = _format_table_to_sncosmo(data_table)

        return data_table

    def _download_module_data(self, force: bool = False, timeout: float = 15):
        """Download data for the current survey / data release

        Args:
            force: Re-Download locally available data
            timeout: Seconds before timeout for individual files/archives
        """

        downloads.download_tar(
            url=self._table_url,
            out_dir=self._table_dir,
            skip_exists=self._table_dir,
            mode='r:gz',
            force=force,
            timeout=timeout
        )

        for filter_file, filter_url in zip(self._filter_file_names, self._filter_urls):
            downloads.download_file(
                url=filter_url,
                destination=self._filter_dir / filter_file,
                force=force,
                timeout=timeout
            )


import requests
import json
import datetime

class NikeDataCollecter():
    """Nikeのログデータ収集用クラス"""
    __init_act_url = "https://api.nike.com/sport/v3/me/activities/after_time/0"
    __next_act_url = "https://api.nike.com/sport/v3/me/activities/after_id/"
    __uniq_act_url = "https://api.nike.com/sport/v3/me/activity/"
    __id_cache_path = "work/act_ids"
    __save_dir = "log"
    __token_path = "work/token"

    def __fetch_nike_api(self, url, params=None):
        """ nike.comからfetchする共通api """
        with open(self.__token_path) as f:
            bearer_token = f.read().strip()

        headers = {'Authorization': 'Bearer {}'.format(bearer_token)}
        r_get = requests.get(url=url, headers=headers, params=params)
        r_get = r_get.json()
        if(r_get.get("error_id") is None):
            return r_get
        else:
            return False

    def __fetch_save(self, id):
        """fetchのメイン関数

        Notes:
        --------
        走行データをfetchして__save_dirに.jsonとして保存する
        保存条件:heart_rateが含まれること

        Returns:
        --------
        bool :
            True:Saveした
            False:Saveしなかった
        """
        params = {'metrics':'all'}
        url= self.__uniq_act_url + id
        data = self.__fetch_nike_api(url, params=params)
        if data is not False:
            # 保存する条件を決める
            need_save = "heart_rate" in data.get("metric_types")
            if(need_save):
                epoch = data.get("start_epoch_ms")
                epoch_utc = datetime.datetime.fromtimestamp(epoch/1000) # utc時刻へ変換
                with open('{}/act_{}.json'.format(self.__save_dir, epoch_utc), 'w') as f:
                    json.dump(data, f, indent=4)
                return True
        return False

    def __extract_all_actid(self):
        """Nike Apiに問い合わせして全てのActivity idリストを取得する"""
        url = self.__init_act_url
        data = self.__fetch_nike_api(url)
        if data is False:
            return None
        act_ids = [d.get("id") for d in data["activities"]]

        # idリストが1ページに収まらない場合は次ページが存在
        # after_idを使って次ページからもidを取得
        after_id = data["paging"].get("after_id")
        while(after_id is not None):
            url = self.__next_act_url+ after_id
            data = self.__fetch_nike_api(url)
            ids = [d.get("id") for d in data["activities"]]
            act_ids.extend(ids)
            after_id = data["paging"].get("after_id")
        return act_ids

    def __fetch_main(self):
        #キャッシュ済みデータの読み出し
        with open(self.__id_cache_path, 'r') as f:
            cached_ids = [d.replace('\n', '') for d in f.readlines()]    #改行コードを変換
            if cached_ids is None:
                cached_ids = []

        #全てのactivity_idを問い合わせて取得
        all_ids = self.__extract_all_actid()
        if all_ids is None:
            print("all_ids fetch fail")
            return False

        #未キャッシュ≒fetchが必要
        non_cache_ids = list(set(all_ids) - set(cached_ids))
        if non_cache_ids == []:
            print("No need fetch")
            return False

        #読み出し＆キャッシュ開始
        for id in non_cache_ids:
            ret = self.__fetch_save(id)
            print("{}:{}".format("saved" if ret else "unsaved", id))

        #処理したidリストを保存
        f = open(self.__id_cache_path, 'w')
        f.write('\n'.join(all_ids))
        f.close()
        return True

    def get_save_dir(self):
        """保存したディレクトリパスのgetter"""
        return self.__save_dir

    def fetch(self):
        """fetchの公開呼び出し関数

        Note:
        -------
        Nike.comからデータ取得とキャッシングを実行.fetch失敗時もそのまま抜ける.

        fetchが失敗する例:
        -------
            ・BearerTokenの有効期限切れ
            ・Nike.comにアクセスできない
        """
        if self.__fetch_main():
            print("fetch done.")
        else:
            print("fetch process abort")


#!/usr/bin/python3

import cgi, cgitb
import _mysql
import sys
import helperHTML
import helperSession
import database.db_config as db_config
import mlog
from helperRegEx import Validator

TAG = "DO_ADD_PAYEE"
mlog.debug(TAG, "Trying to add payee..")
cgitb.enable()
print(helperHTML.get_html_init())
print(helperHTML.get_html_start_preset())

#Validate session..
if(helperSession.any_session_active() == False):
    #Show session inactive..
    mlog.debug(TAG,"In do_add_payee with no active session. So prompting to sign in again..")
    print(helperHTML.get_html_invalid_session_preset())
    print(helperHTML.get_html_end_preset())
    sys.exit()

account_number = helperSession.get_session_accout_no()
payee_name = ""
payee_account = 0
payee_bank = ""
payee_branch = ""
payee_ifsc_code = ""
mlog.debug(TAG,"Fetching form values..")
try:
    form_entries = cgi.FieldStorage()
    payee_name = form_entries.getvalue("payee_name")
    payee_account = form_entries.getvalue("payee_account")
    payee_bank = form_entries.getvalue("payee_bank")
    payee_branch = form_entries.getvalue("payee_branch")
    payee_ifsc_code = form_entries.getvalue("payee_ifsc_code")
except Exception as e:
    mlog.error(TAG,"Error: " + str(e))

#--------------------------------------------MENU DETAILS----------------------------------------------------
#Show MENU..
print('''

            DASHBOARD
            Add Payee
            Remove Payee
            Deposit Money
            Money Transfer
            Logout

    ''')
print("
")
print("
 Add Payee status:

")
print("
")
#------------------------------------------------------------------------------------------------------------

error_message = None
if(payee_name == None or payee_account == None or payee_bank == None or payee_branch == None) or payee_ifsc_code == None:
    error_message = "

Failed to add payee! Invalid attributes provided. Add payee again with valid attributes
"
else:
    payee_validator = Validator()
    if payee_validator.validate_generic_name(payee_name,6,15) == False:
        error_message = payee_validator.message
    elif int(payee_account)<10000 or int(payee_account) > 999999:
        error_message = "
Account number must be greater than 10000 and less than 999999!
"
    elif payee_validator.validate_bank_name(payee_bank,3,10) == False:
        error_message = payee_validator.message
    elif payee_validator.validate_branch_name(payee_branch,5,10) == False:
        error_message = payee_validator.message
    elif payee_validator.validate_ifsc_code(payee_ifsc_code) == False:
        error_message = payee_validator.message

mlog.error(TAG,"Add payee validator error message: " + str(error_message))
if error_message != None:
    print('''

Failed to add payee:

    ''')
    print(error_message)
    print('''

''')
    print(helperHTML.get_html_end_preset())
    sys.exit()

try:
    mlog.debug(TAG, "Establishing database connection..")
    conn = _mysql.connect(db_config.db_host,db_config.db_user,db_config.db_password,db_config.db_name)
except Exception as e:
    mlog.error(TAG,"Unable to conenct to MyBanking Database.")
    print('''

Unable to conenct to MyBanking Database.

                BACK

        ''')
    print(helperHTML.get_html_end_preset())
#    print(e)
    sys.exit()

all_recs = None
add_payee_failed = False
try:
    sql_insert = "INSERT INTO payee_list(payee_name,owner_account,payee_account,payee_bank,payee_branch,payee_ifsc_code)"
    sql_insert += " VALUES('"+ payee_name +"', "+ str(account_number) +", "+ payee_account +", '"+ payee_bank +"', '"+ payee_branch +"', '"+ payee_ifsc_code +"')"
    conn.query(sql_insert)
    all_recs = conn.store_result()
except Exception as e:
    add_payee_failed = True
    mlog.error(TAG,"Unable to add payee to database:"+str(e))

print('''
''')
print('''
''')# to give a margin.
if(add_payee_failed == True) :
    print("

Unable to add payee to MyBanking Database.!
")
else:
    print("

Successfully added payee!
")
print("
")
print("
")

conn.close()
print(helperHTML.get_html_end_preset())


import argparse
import boto3
import datetime
import pytz

default_objs = set(["rubrik_cluster_lock.txt", "rubrik_encryption_key_check.txt"])

class S3Client(object):
    def __init__(self, profile):
        self._session = boto3.Session(profile_name=profile)
        self.s3 = self._session.resource("s3")

    def list_buckets(self, visitor):
        for b in self.s3.buckets.all():
            #print("processing bucket '{}'".format(b.name))

            # try:
            #     num = sum(1 for _ in b.objects.limit(3))
            #     if num == 0:
            #         print("failed to load objects for {}".format(b.name))
            # except Exception as ex:
            #     print("failed to load objects for {}: {}".format(b.name, ex))
            #     continue

            visitor(b)

    @staticmethod
    def delete_bucket(bucket):
        print("deleting {}".format(bucket.name))
        try:
            bucket.objects.all().delete()
        except Exception as ex:
            print("failed to delete objects {}".format(ex))
            return

        try:
            bucket.delete()
        except Exception as ex:
            print("failed to delete bucket {}".format(ex))
            return

    @staticmethod
    def is_bucket_too_old(bucket, days):
        num = sum(1 for _ in bucket.objects.limit(3))
        if num > 2:
            # has objects other than the default objects
            return False

        days_ago = pytz.UTC.localize(datetime.datetime.now()) - datetime.timedelta(days=days)
        for o in bucket.objects.all():
            if o.key not in default_objs:
                # has objects other than the default objects
                return False

            if o.last_modified > days_ago:
                # object has been modified recently
                return False

        if bucket.creation_date > days_ago:
            # buckets has been created recently
            return False

        return True

    @staticmethod
    def is_bucket_deletable(bucket, prefix, days):
        if prefix != "" and bucket.name.startswith(prefix):
            return S3Client.is_bucket_too_old(bucket, 5)

        return S3Client.is_bucket_too_old(bucket, days)

    @staticmethod
    def delete_bucket_if_necessary(bucket):
        if not S3Client.is_bucket_too_old(bucket, 21):
            return

        print("bucket {} will be deleted".format(bucket.name))
        S3Client.delete_bucket(bucket)

def parse_args():
    parser = argparse.ArgumentParser(description='cleanup the s3 buckets for AppFlows')
    parser.add_argument(
        '--profile', '-p',
        dest='profile',
        type=str,
        help='the name of the AWS profile')
    parser.add_argument(
        '--name', '-n',
        dest='prefix',
        type=str,
        default="",
        help='the prefix of the AWS s3 buckets to be check and removed')
    parser.add_argument(
        '--days', '-d',
        dest='days',
        type=str,
        help='the age of the AWS s3 buckets to be removed (days)')
    parser.add_argument(
        '--not-dry-run',
        dest='not_dry_run',
        action='store_true',
        help='delete the found buckets')
    return parser.parse_args()

def main():
    args = parse_args()
    names_to_be_deleted = []

    buckets_to_be_deleted = []

    def handle(bucket):
        print("\"{}\",".format(bucket.name))
        if "appflows-ui-" in bucket.name:
            buckets_to_be_deleted.append(bucket)
        if not S3Client.is_bucket_deletable(bucket, args.prefix, 5):
            return

        #print("found bucket {} to be deleted".format(bucket.name))
        #buckets_to_be_deleted.append(bucket)

    s3 = S3Client(args.profile)
    s3.list_buckets(handle)

    print("buckets to be deleted:")
    for b in buckets_to_be_deleted:
        print("\t{}".format(b.name))

    if args.not_dry_run:
        print("deleting buckets:")
        for b in buckets_to_be_deleted:
            S3Client.delete_bucket(b)

if __name__ == "__main__":
    main()


import argparse

def add_frequencies(dict_count, in_path, out_path, word="w1"):
    print("gets frequencies for results file")
    assert word == "w1" or word == "w2", "invalid word "
    with open(out_path, 'w') as wf:
        wf.write("{}\t{}\t{}\t{}\t{}\t{}\n".format("relation", word, "rank", "counts", "pos", "morph"))
        with open(in_path, 'r') as rf:
            next(rf)
            for l in rf:
                l = l.strip()
                line = l.split("\t")
                if word == 'w1':
                    tw = line[1]
                else:
                    tw = line[2]
                try:
                    counts = dict_count[tw]
                    for c in counts:
                        wf.write("{}\t{}\t{}\t{}\t{}\t{}\n".format(line[0], tw, line[2], c[3], c[1], c[2]))
                except:
                    print("{} not in frequency dict".format(tw))
                    wf.write("{}\t{}\t{}\t{}\t{}\t{}\n".format(line[0],tw, line[2], "NA", "NA", "NA"))

def add_frequencies_to_words(dict_count, in_path, out_path, word):
    print("gets frequencies for word file")
    print("word type", word)
    print("in path", in_path)
    assert word == "w1" or word == "w2", "invalid word "
    i = 0
    with open(out_path, 'w') as wf:
        wf.write("{}\t{}\t{}\t{}\t{}\n".format("relation", word, "counts", "pos", "morph"))
        with open(in_path, 'r', encoding='utf8') as rf:
            next(rf)
            try:
                for l in rf:
                    i+=1
                    print("line",i, l)
                    l = l.strip()
                    line = l.split("\t")
                    if word == "w1":
                        tw = line[1].lower()
                    elif word == "w2":
                        tw = line[2].lower()
                    else:
                        print("invalid word")
                    if tw in dict_count:
                        counts = dict_count[tw]
                        for c in counts:
                            if len(c) > 2:
                                wf.write("{}\t{}\t{}\t{}\t{}\n".format(line[0], tw, c[-1], c[1], c[2]))
                    else:
                        print("{} not in frequency dict".format(tw))
                        wf.write("{}\t{}\t{}\t{}\t{}\n".format(line[0], tw, "NA", "NA", "NA"))
            except:
                print("problem reading file")
def read_count_dict(in_path):
    #("word form", "pos", "morph", "count"
    dict_count = dict()
    with open(in_path, 'r') as rf:
        next(rf)
        for l in rf:
            l = l.strip()
            line = l.split("\t")
            if line[0].lower() not in dict_count:
                dict_count[line[0].lower()] = []
            dict_count[line[0].lower()].append((line))
    return dict_count
def main(args):
    count_dict = read_count_dict(args.frequencies)
    print("finished reading count dict")
    if args.type_file =="results":
        add_frequencies(count_dict, args.file_path, args.out_path, word =args.word_type)
    else:
        add_frequencies_to_words(count_dict, args.file_path, args.out_path, word=args.word_type)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("frequencies")
    parser.add_argument("file_path")
    parser.add_argument("out_path")
    parser.add_argument("word_type")
    parser.add_argument("type_file", help="results or words")
    args = parser.parse_args()
    main(args)


from typing import Tuple, List

def load_data() -> Tuple[int, List[int], int]:
    file = open('./input.txt', 'rt')
    period = int(file.readline())
    data = [int(x) for x in file.readline().split()]
    price = int(file.readline())
    return period, data, price

def search_shopping_day(left: int, right: int, data: List[int], price: int):
    if price > data[right - 1]:
        return -1
    mid = (left + right) // 2

    if left == right:
        return mid // 2

    if mid == 0:
        return mid + 1

    if data[mid] >= price > data[mid - 1]:
        return mid + 1
    elif price < data[mid]:
        return search_shopping_day(left, mid, data, price)
    else:
        return search_shopping_day(mid+1, right, data, price)

if __name__ == '__main__':
    days, savings, cost = load_data()
    f = search_shopping_day(0, days, savings, cost)
    d = search_shopping_day(f, days, savings, cost*2)
    print(f, d)


from typing import List

class Solution:
    def hIndex(self, citations: List[int]) -> int:
        citations = citations[::-1]
        start, end = 0, len(citations) - 1
        while start <= end:
            mid = (start + end) // 2
            h = mid + 1
            if h == citations[mid]:
                return h
            elif h > citations[mid]:
                end = mid - 1
            else:
                start = mid + 1

        return start


import os
import json
import torch
from transformers import AutoConfig, AutoTokenizer, AutoModel
from sentence_transformers import SentenceTransformer, util

def main():

    model_path = "/home/zb/ChatGLM-6B-main/chatglm-6b-int4"
    a_model_path = "/home/zb/ChatGLM-6B-main/ptuning/output/task2-chatglm-6b-pt-128-2e-2/checkpoint-3000"

    tokenizer = AutoTokenizer.from_pretrained(a_model_path, trust_remote_code=True)

    # Fine-tuning 后的表现测试
    config = AutoConfig.from_pretrained(model_path, trust_remote_code=True, pre_seq_len=128)
    model = AutoModel.from_pretrained(model_path, config=config, trust_remote_code=True)
    # 此处使用你的 ptuning 工作目录
    prefix_state_dict = torch.load(os.path.join(a_model_path, "pytorch_model.bin"))
    new_prefix_state_dict = {}
    for k, v in prefix_state_dict.items():
        new_prefix_state_dict[k[len("transformer.prefix_encoder."):]] = v
    model.transformer.prefix_encoder.load_state_dict(new_prefix_state_dict)

    #V100 机型上可以不进行量化
    print(f"Quantized to 4 bit")
    model = model.quantize(4)
    model = model.half().cuda()
    model.transformer.prefix_encoder.float()
    model = model.eval()

    bert_model_path = "task1_sbert/output/bert-2023-05-31_02-45-44"
    bert_model = SentenceTransformer(model_name_or_path=bert_model_path)

    with open('datasets/processed/qa_test2_task2.json', 'r', encoding='utf-8') as fr, open('datasets/res/task2.txt', 'w', encoding='utf-8') as fw:
        for q in fr:
            query = json.loads(q)['query']
            best_response = None
            best_score = 0
            for _ in range(3):
                response, _ = model.chat(tokenizer, repr(query), history=[], top_p=1.5)
                # Compute embedding for both query and reply
                query_embedding = bert_model.encode(query, convert_to_tensor=True)
                reply_embedding = bert_model.encode(response, convert_to_tensor=True)
                # Compute cosine-similarities
                cosine_scores = util.cos_sim(query_embedding, reply_embedding)
                if cosine_scores>best_score:
                    best_score = cosine_scores
                    best_response = response
                # print(response, cosine_scores, best_score)

            fw.write(best_response)
            fw.write('\n')

if __name__=='__main__':
    main()


from datetime import timedelta, date as dt
from dateutil import rrule, relativedelta
from dal import autocomplete

from django import forms
from django.forms import ModelForm
from dal import autocomplete

from openrepairplatform.event.models import Event, Activity, Condition
from openrepairplatform.location.models import Place
from openrepairplatform.user.models import CustomUser
from openrepairplatform.user.models import Organization

class EventForm(ModelForm):
    date = forms.DateField(
        initial=dt.today(),
        widget=forms.DateInput(attrs={"type": "date"}, format="%Y-%m-%d"),
    )
    starts_at = forms.TimeField(widget=forms.TimeInput(attrs={"type": "time"}))
    ends_at = forms.TimeField(widget=forms.TimeInput(attrs={"type": "time"}))
    publish_at = forms.DateTimeField(
        initial=dt.today(),
        widget=forms.DateTimeInput(format="%Y-%m-%d %H:%M:%S"),
    )

    def __init__(self, *args, **kwargs):
        self.orga = kwargs.pop("orga")
        super().__init__(*args, **kwargs)
        self.fields["organizers"] = forms.ModelMultipleChoiceField(
            queryset=(
                self.orga.actives.all() | self.orga.admins.all() | self.orga.volunteers.all()
            ),
            widget=autocomplete.ModelSelect2Multiple(url='/' + self.orga.slug + '/user_orga_autocomplete/'),
            required=False,
        )
        self.fields["conditions"] = forms.ModelMultipleChoiceField(
            queryset=self.orga.conditions,
            widget=autocomplete.ModelSelect2Multiple(url='/event/' + self.orga.slug + '/condition_orga_autocomplete/'),
            required=False,
        )
        self.fields["activity"] = forms.ModelChoiceField(
            widget=autocomplete.ModelSelect2(url='activity_autocomplete'),
            queryset=Activity.objects.all()
        )

    class Meta:
        model = Event
        fields = [
            "activity",
            "is_free",
            "available_seats",
            "booking",
            "allow_stuffs",
            "collaborator",
            "external",
            "external_url",
            "description",
            "location",
            "date",
            "starts_at",
            "ends_at",
            "publish_at",
            "needed_organizers",
            "organizers",
            "conditions",
        ]
        widgets = {
            'location': autocomplete.ModelSelect2(url='place_autocomplete'),
        }

class RecurrentEventForm(forms.ModelForm):
    recurrent_type = forms.ChoiceField(
        choices=[("WEEKLY", "semaine"), ("MONTHLY", "mois")], label="Par"
    )
    days = forms.MultipleChoiceField(
        choices=Event.DAYS,
        widget=forms.CheckboxSelectMultiple(),
        label="Jour(s)",
    )
    weeks = forms.MultipleChoiceField(
        choices=Event.WEEKS,
        widget=forms.CheckboxSelectMultiple(attrs={"class": "for-month"}),
        label="La ou les semaines",
        required=False,
    )
    starts_at = forms.TimeField(
        label="De", widget=forms.TimeInput(attrs={"type": "time"})
    )
    ends_at = forms.TimeField(
        label="À", widget=forms.TimeInput(attrs={"type": "time"})
    )
    date = forms.DateField(
        initial=dt.today(),
        widget=forms.DateInput(attrs={"type": "date"}, format="%Y-%m-%d"),
        label="Date de début",
    )
    end_date = forms.DateField(
        widget=forms.DateInput(attrs={"type": "date"}, format="%Y-%m-%d"),
        label="Date de fin",
    )
    period_before_publish = forms.ChoiceField(
        choices=[
            (1, "1 jour avant"),
            (2, "2 jours avant"),
            (7, "Une semaine avant"),
            (14, "Deux semaines avant"),
            (21, "Trois semaines avant"),
            (28, "Quatre semaines avant"),
            (35, "Cinq semaines avant"),
            (42, "Six semaines avant"),
        ],
        label="Publication",
    )

    def __init__(self, *args, **kwargs):
        self.orga = kwargs.pop("orga")
        super().__init__(*args, **kwargs)
        self.fields["organizers"] = forms.ModelMultipleChoiceField(
            queryset=(
                self.orga.actives.all() | self.orga.admins.all() | self.orga.volunteers.all()
            ).distinct(),
            widget=autocomplete.ModelSelect2Multiple(url='/' + self.orga.slug + '/user_orga_autocomplete/'),
            required=False,
        )
        self.fields["conditions"] = forms.ModelMultipleChoiceField(
            queryset=self.orga.conditions,
            widget=autocomplete.ModelSelect2Multiple(url='/event/' + self.orga.slug + '/condition_orga_autocomplete/'),
            required=False,
        )
        self.fields["activity"] = forms.ModelChoiceField(
            widget=autocomplete.ModelSelect2(url='activity_autocomplete'),
            queryset=Activity.objects.all()
        )

    def clean_weeks(self):
        recurrent_type = self.cleaned_data["recurrent_type"]
        error_message = (
            "Vous devez renseigner au moins une semaine de récurrence."
        )
        if recurrent_type == "MONTHLY":
            if not self.cleaned_data["weeks"]:
                self.add_error("weeks", error_message)
                raise forms.ValidationError(error_message)
        return self.cleaned_data["weeks"]

    def manage_recurrence(self, dates):
        objects = []
        for date in dates:
            params = {
                "date": date.date(),
                "starts_at": self.cleaned_data["starts_at"],
                "ends_at": self.cleaned_data["ends_at"],
                "organization": self.orga,
                "location": self.cleaned_data["location"],
                "publish_at": (
                    date
                    - timedelta(
                        days=int(self.cleaned_data["period_before_publish"])
                    )
                ),
                "activity": self.cleaned_data["activity"],
                "available_seats": self.cleaned_data["available_seats"],
            }
            event = Event.objects.create(**params)
            event.organizers.add(*list(self.cleaned_data["organizers"]))
            event.conditions.add(*list(self.cleaned_data["conditions"]))
            objects.append(event)
        return objects

    def get_rule_list(self):
        if self.cleaned_data["weeks"]:
            weekdays = [
                getattr(relativedelta, day)(int(week))
                for week in self.cleaned_data["weeks"]
                for day in self.cleaned_data["days"]
            ]
        else:
            weekdays = [
                getattr(rrule, day) for day in self.cleaned_data["days"]
            ]
        rule = list(
            rrule.rrule(
                getattr(rrule, self.cleaned_data["recurrent_type"]),
                byweekday=weekdays,
                dtstart=max(self.cleaned_data["date"], dt.today()),
                until=self.cleaned_data["end_date"],
            )
        )
        return rule

    def save(self, commit=False):
        dates = self.get_rule_list()
        objects = self.manage_recurrence(dates)
        return len(objects)

    class Meta:
        model = Event
        fields = [
            "activity",
            "is_free",
            "available_seats",
            "booking",
            "allow_stuffs",
            "collaborator",
            "external",
            "external_url",
            "description",
            "organizers",
            "needed_organizers",
            "conditions",
            "location",
            "recurrent_type",
            "date",
            "days",
            "weeks",
            "starts_at",
            "ends_at",
            "end_date",
            "period_before_publish",
        ]
        widgets = {
            'location': autocomplete.ModelSelect2(url='place_autocomplete'),
        }

class ActivityForm(ModelForm):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fields["picture"].widget.attrs.update({"class": "form-control"})

    class Meta:
        model = Activity
        exclude = ["slug", "organization"]

class ConditionForm(ModelForm):
    class Meta:
        model = Condition
        exclude = ["slug", "organization"]

class EventSearchForm(forms.Form):
    starts_after = forms.DateField(
        widget=forms.DateInput(attrs={"type": "date"}, format="%Y-%m-%d"),
        required=False,
        label="Commence après le"
    )
    starts_before = forms.DateField(
        widget=forms.DateInput(attrs={"type": "date"}, format="%Y-%m-%d"),
        required=False,
        label="Commence avant le"
    )

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        future_events = Event.future_published_events()
        self.fields["activity"] = forms.ModelChoiceField(
            required=False,
            queryset=Activity.objects.filter(
                events__in=future_events
            ).distinct(),
            widget=autocomplete.ModelSelect2(url='event:future_event_activity_autocomplete'),
            label="Activité"
        )
        self.fields["place"] = forms.ModelChoiceField(
            required=False,
            queryset=Place.objects.filter(
                events__in=future_events
            ).distinct(),
            widget=autocomplete.ModelSelect2(url='event:future_event_place_autocomplete',
            forward=['activity']),
            label="Lieu",
        )
        self.fields["organization"] = forms.ModelChoiceField(
            required=False,
            queryset=Organization.objects.filter(
                events__in=future_events
            ).distinct(),
            label="Organisateur",
        )


import string
import redis
import random

r = redis.StrictRedis()

def main():
    for x in range(5000):
        create_list_of_words()

def create_list_of_words():
    key = random_string(random.randint(10,25))
    num_items = random.randint(5, 1000)
    pipe = r.pipeline()
    for x in range(num_items):
        value = random_string(random.randint(5, 100))
        pipe.rpush(key, value)
    pipe.execute()

def random_string(length) :
    return ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(length))

if __name__ == '__main__':
    main()


import json
import re
import csv
# import gensim

QUESTION_TYPES = ['Pure Questioning', 'Rhetorical Questioning', 'Assertive Questioning']

discourse_markers = [
    'about',
    'accordingly',
    'additionaly',
    'after',
    'afterward',
    'afterwards',
    'albeit',
    'also',
    'alternatively',
    'although',
    'and',
    'as',
    'because',
    'before',
    'besides',
    'beyond',
    'both',
    'but',
    'by',
    'consequently',
    'conversely',
    'despite',
    'earlier',
    'else',
    'except',
    'finally',
    'for',
    'from',
    'further',
    'furthermore',
    'given',
    'hence',
    'however',
    'if',
    'in',  # I am not sure if that is considered a discourse marker. Double-check
    'indeed',
    'instead',
    'later',
    'lest',
    'like',
    'likewise',
    'meantime',
    'meanwhile',
    'moreover',
    'nevertheless',
    'next',
    'nonetheless',
    'nor',
    'on',  # not a discourse marker in phrasal verbs such as "get on"
    'once',
    'only',
    'or',
    'plus',
    'previously',
    'rather',
    'regardless',
    'separately',
    'similarly',
    'simultaneously',
    'since',
    'so',
    'specifically',
    'still',
    'subsequently',
    'then',
    'thereafter',
    'thereby',
    'though',
    'thus',
    'till',
    'ultimately',
    'unless',
    'until',
    'upon',
    'whatever',
    'when',
    'whenever',
    'where',
    'whereas',
    'whether',
    'while',
    'with',
    'without',
    'yet'
]

HEADER_ROW = [
    "Testrun",
    "question_number",
    "question_json_ID",
    "question_part",
    "response_number",
    "response_json_ID",
    "response_part",
    "response_text",
    "response_locutions",
    "response_length",
    "response_loc_count",  # number of locutions in the response
    "response_dm_count",  # number of discourse markers
    "response_em_count",  # number of epistemic markers
    "response_assertion_count",  # number of assertions
    "response_question_count",  # number of questions
    "response_MA_count",
    "response_RA_count",
    "response_CA_count",
    "response_unconnected_count",  # unconnected propositions in the answer
    "response_DefaultIlloc_count",
    "response_rs_count",  # reported speech count
    "Resp_ranking"
]

# Read CSV file
with open("FriPa_new_dub.csv") as fp:
    reader = csv.reader(fp, delimiter=",", quotechar='"')
    csv_FriPa = [row for row in reader]

with open("qt30.csv") as fp:
    reader = csv.reader(fp, delimiter=",", quotechar='"')
    csv_qt30 = [row for row in reader]

def find_indexes(lst, target, current_indexes=[]):
    indexes = []
    for index, item in enumerate(lst):
        if isinstance(item, list):
            # If the item is a sublist, recursively call the function
            sub_indexes = find_indexes(item, target, current_indexes + [index])
            indexes.extend(sub_indexes)
        elif item.lower() == target:
            # If the item is the target string, add the current index
            indexes.append(current_indexes[-1])
    return list(set(indexes))

def extract_node_ids(s):
    pattern = r'(\d+_\d+)'
    matches = re.findall(pattern, s)
    return matches

def get_statistics(text):
    # words = gensim.utils.simple_preprocess(str(text), deacc=True, min_len=1)
    # above is legacy code. Use when the text contains spans
    words = text.split()
    # Number of discourse indicators
    discourse_markers_count = 0
    epistemic_markers_count = []
    for word in words:
        if word in discourse_markers:
            discourse_markers_count += 1
    # Count epistemic expressions
    epistemic_markers_count.extend(re.findall(
        r'(?:I|We|we|One|one|They|they)(?:\s\w+)?(?:\s\w+)?\s(?:believes?|think|thinks|means?|worry|worries|knows?|guesse?s?|assumes?|wonders?|feels?)\b(?:that)?',
        text))
    epistemic_markers_count.extend(re.findall(
        r'(?:I|We|we|One|one)\s(?:don\'t|\sdoesn\'t\sdo\snot|\sdoes\snot)\s(?:believe|think|mean|worry|know|guess|assume|wonder|feel)\b(?:that)?',
        text))
    epistemic_markers_count.extend(
        re.findall(r'(?:It|it)\sis\s(?:believed|known|assumed|thought)\b(?:that)?', text))
    epistemic_markers_count.extend(
        re.findall(r'(?:I|We|we)\s(?:am|are|was|were)(?:\sjust)?\s(?:thinking|guessing|wondering)\b(?:that)?',
                   text))
    epistemic_markers_count.extend(
        re.findall(r'(?:I\'m|[Ww]e\'re)(?:\sjust)?\s(?:thinking|guessing|wondering)\b(?:that)?', text))
    epistemic_markers_count.extend(
        re.findall(r'(?:I|We|we|One|one)(?:\s\w+)?\s(?:do|does)\snot\s(?:believe?|think|know)\b(?:that)?',
                   text))
    epistemic_markers_count.extend(
        re.findall(r'(?:I|We|we|One|one)\swould(?:\s\w+)?(?:\snot)?\ssay\b(?:that)?', text))
    epistemic_markers_count.extend(
        re.findall(r'(?:I\sam|I\'m|We\sare|We\'re)(?:\s\w+)?\s(?:afraid|sure|confident)\b(?:that)?', text))
    epistemic_markers_count.extend(re.findall(
        r'(?:My|my|Our|our)\s(?:personal\s)?(?:experience|opinion|belief|view|knowledge|worry|worries|concerns?|guesse?s?|position|perception)(?:\son\s\w+)?\s(?:is|are)\b(?:that)?',
        text))
    epistemic_markers_count.extend(re.findall(r'[Ii]n\s(?:my|our)(?:\s\w+)?\s(?:view|opinion)\b', text))
    epistemic_markers_count.extend(re.findall(r'[Fr]rom\s(?:my|our)\s(?:point\sof\sview|perspective)\b', text))
    epistemic_markers_count.extend(re.findall(r'As\sfar\sas\s(?:I|We|we)\s(?:am|are)\sconcerned', text))
    epistemic_markers_count.extend(
        re.findall(r'(?:I|We|we|One|one)\s(?:can|could|may|might)(?:\s\w+)?\sconclude\b(?:that)?', text))
    epistemic_markers_count.extend(re.findall(r'I\s(?:am\swilling\sto|must)\ssay\b(?:that)?', text))
    epistemic_markers_count.extend(re.findall(r'"One\s(?:can|could|may|might)\ssay\b(?:that)?', text))
    epistemic_markers_count.extend(re.findall(r'[Oo]ne\s(?:can|could|may|might)\ssay\b(?:that)?', text))
    epistemic_markers_count.extend(re.findall(r'[Ii]t\sis\s(?:obvious|(?:un)?clear)\b', text))
    epistemic_markers_count.extend(re.findall(r'[Ii]t(?:\sjust)?\s(?:seems|feels|looks)', text))
    epistemic_markers_count.extend(re.findall(r'[Pp]ersonally\s(?:for\sme|speaking)', text))
    epistemic_markers_count.extend(re.findall(r'[Ff]rankly|[Hh]onestly|[Cc]learly', text))
    return discourse_markers_count, len(epistemic_markers_count)

with open('output.csv', 'a', encoding='utf-8') as f:
    csv_writer = csv.writer(f)
    csv_writer.writerow(HEADER_ROW)
    # used to write the header once and only once

for i in csv_FriPa[1:]:  # first two rows are examples (15.08.2023)
    resp_json = i[11]
    if resp_json == "":  # if no Resp_json exist, we skip it - the stance is different
        if len(i[4].split()) != 1:
            with open('output.csv', 'a', encoding='utf-8') as f:
                csv_writer = csv.writer(f)
                csv_writer.writerow([
                    i[0],
                    i[1],
                    '',
                    i[2],
                    i[3],
                    '',
                    '',
                    i[9],
                    '',
                    len(i[9].split()),
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    '',
                    i[13]
                ])
        continue
    date_fripa = "cutietestrun" + i[0].lower()
    date_fripa_s = "cutiestestrun" + i[0].lower()
    question_number = i[1]
    answer_number = i[3]
    answer_speaker = i[8]
    response_json_ID = ""
    question_json_ID = ""
    response_text = i[9]
    statistics = get_statistics(response_text)
    response_dm_count = statistics[0]
    response_em_count = statistics[1]
    response_length = len(response_text.split())  # count the ammount of words
    answer_part = i[4]
    question_part = i[2]
    node_list = []
    response_loc_count = 0
    response_assertion_count = 0
    response_question_count = 0
    response_MA_count = 0
    response_RA_count = 0
    response_CA_count = 0
    response_unconnected_count = 0
    response_DefaultIlloc_count = 0
    response_rs_count = 0
    Resp_ranking = i[13]
    # looking for corpus in qt30 map
    date_indexes_qt30 = find_indexes(csv_qt30, date_fripa)
    # some of the rows have cutieStestrun in them. Next line calls a function again in that case
    if not date_indexes_qt30:
        date_indexes_qt30 = find_indexes(csv_qt30, date_fripa_s)
    for d in date_indexes_qt30:
        # search for json_ids using parts for answers
        if csv_qt30[d][9] == answer_part or csv_qt30[d][9] == "part " + answer_part:
            if len(csv_qt30[d][11]) <= 6:  # there are texts instead of corpus numbers sometimes in the column
                json_corpus = csv_qt30[d][11]
                response_json_ID = json_corpus  # writing to outer scope
                try:
                    with open("jsons/" + json_corpus + ".json", encoding='utf-8-sig') as f:
                        json_data = json.load(f)
                except FileNotFoundError:
                    print("File " + json_corpus + " does not exist.")
                node_list = extract_node_ids(resp_json)  # extracted all nodes referring to text
                response_loc_count = len(node_list)
                nodes_YA_from_locutions = []
                #  First, we collect all the edges that contain our locution nodes in their fromID
                for edge in json_data['AIF']['edges']:
                    fromID = edge["fromID"]
                    toID = edge["toID"]
                    if fromID in node_list:
                        nodes_YA_from_locutions.append(toID)
                #  Now, we find all nodes where our edges are going to, to check if it is Assertion
                for node in json_data["AIF"]["nodes"]:
                    if node["nodeID"] in nodes_YA_from_locutions and node["type"] != "YA":
                        nodes_YA_from_locutions.remove(node["nodeID"])
                    if node["text"] == "Asserting" and node["nodeID"] in nodes_YA_from_locutions:
                        response_assertion_count += 1
                    if node["text"] in QUESTION_TYPES and node["nodeID"] in nodes_YA_from_locutions:
                        response_question_count += 1
                #  creating a search for I-nodes to later find connected CA, MA and RA nodes
                nodes_I_from_YA = []
                for edge in json_data['AIF']['edges']:
                    fromID = edge["fromID"]
                    toID = edge["toID"]
                    if fromID in nodes_YA_from_locutions:
                        nodes_I_from_YA.append(toID)
                for node in json_data["AIF"]["nodes"]:
                    if node["nodeID"] in nodes_I_from_YA and node["type"] != "I":
                        nodes_I_from_YA.remove(node["nodeID"])
                        #  searching for reported speech assuming that L to L situations are unique
                        if node["type"] == "L":
                            response_rs_count += 1
                nodes_MA_from_I = []
                nodes_RA_from_I = []
                nodes_CA_from_I = []
                end_nodes = []
                unconnected_nodes = nodes_I_from_YA.copy()
                #  searching nodes and those with types RA, MA, CA are appended to an array
                for edge in json_data['AIF']['edges']:
                    fromID = edge["fromID"]
                    toID = edge["toID"]
                    if fromID in nodes_I_from_YA:
                        end_nodes.append(toID)
                        if fromID in unconnected_nodes:
                            unconnected_nodes.remove(fromID)
                unconnected_nodes_total = unconnected_nodes.copy()
                for node in unconnected_nodes_total:
                    counter_to = 0
                    for edge in json_data['AIF']['edges']:
                        if node == edge["toID"]:
                            counter_to += 1
                    if counter_to != 1:
                        unconnected_nodes.remove(node)
                response_unconnected_count = len(unconnected_nodes)
                for node in json_data["AIF"]["nodes"]:
                    if node["nodeID"] in end_nodes:
                        if node["type"] == "MA":
                            nodes_MA_from_I.append(node["nodeID"])
                        elif node["type"] == "RA":
                            nodes_RA_from_I.append(node["nodeID"])
                        elif node["type"] == "CA":
                            nodes_CA_from_I.append(node["nodeID"])
                response_MA_count = len(nodes_MA_from_I)
                response_RA_count = len(nodes_RA_from_I)
                response_CA_count = len(nodes_CA_from_I)
                #  search for all incoming edges in I-nodes to filter for Default Illocutings
                nodes_to_MA = []
                default_illocutings_nodes = []
                for edge in json_data['AIF']['edges']:
                    fromID = edge["fromID"]
                    toID = edge["toID"]
                    if toID in nodes_MA_from_I:
                        nodes_to_MA.append(fromID)
                for node in json_data["AIF"]["nodes"]:
                    if node["nodeID"] in nodes_to_MA and node["text"] == "Default Illocuting":
                        default_illocutings_nodes.append(node)
                response_DefaultIlloc_count = len(default_illocutings_nodes)
        # same search for json ID in qt30 file, but now for the questions
        if csv_qt30[d][9] == question_part or csv_qt30[d][9] == "part " + question_part:
            if len(csv_qt30[d][11]) <= 6:  # there are texts instead of corpus numbers sometimes in the column
                json_corpus = csv_qt30[d][11]
                question_json_ID = json_corpus  # writing to outer scope
    with open('output.csv', 'a', encoding='utf-8') as f:
        csv_writer = csv.writer(f)
        csv_writer.writerow([
            date_fripa,
            question_number,
            question_json_ID,
            question_part,
            answer_number,
            response_json_ID,
            answer_part,
            response_text,
            node_list,
            response_length,
            response_loc_count,
            response_dm_count,
            response_em_count,
            response_assertion_count,
            response_question_count,
            response_MA_count,
            response_RA_count,
            response_CA_count,
            response_unconnected_count,
            response_DefaultIlloc_count,
            response_rs_count,
            Resp_ranking
        ])


import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import UserAgent as ua
import time
import firebase_admin
from firebase_admin import credentials, firestore
import re

if __name__ == "__main__":

    cities = ['台中','基隆','彰化','屏東','新竹市','苗栗','南投','雲林','嘉義縣','嘉義市','屏東','宜蘭','花蓮','台東','台北','新北','台南','高雄','新竹縣']

    start_time = time.time()

    for city in cities:
        #url = 'https://www.booking.com/searchresults.zh-tw.html?ss=' + city + '&nflt=review_score%3D90%3Breview_score%3D80&order=bayesian_review_score'
        url = 'https://www.booking.com/searchresults.zh-tw.html?ss=' + city + '&nflt=review_score%3D80%3Breview_score%3D90%3Breview_score%3D70&order=bayesian_review_score'
        print(f'\nurl: {url}')

        user_agent = ua.UserAgent().get_latest('Chrome') # modified by Peter
        print(f"user_agent: {user_agent}")
        options = Options()
        options.add_argument("--headless")  # Remove this if you want to see the browser (Headless makes the chromedriver not have a GUI)
        options.add_argument("--window-size=1920,1080")
        options.add_argument("user-agent=%s"%user_agent)
        options.add_argument("--no-sandbox")
        options.add_argument("--disable-extensions")
        driver = webdriver.Chrome(options = options)
        driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", { # 清空 window.navigator
            "source": """
                Object.defineProperty(navigator, 'webdriver', {
                get: () => undefined
                })
            """
        }) # added by Peter
        driver.implicitly_wait(20)   # added by Peter
        driver.get(url)
        #print(driver.page_source)
        page_content = driver.page_source #取得網頁內的內容
        soup = BeautifulSoup(page_content,'html.parser')

        hotel_info_list = []

        hotels = soup.find_all('div',class_='f6431b446c a15b38c233')
        web_pages = soup.find_all('div',class_='e952b01718')
        scores = soup.find_all('div',class_='a3b8729ab1 d86cee9b25')

        driver.close() #關閉網頁
        driver.quit()

        for i in range(len(hotels)):
            hotel_info_dict = {}
            hotel_name = hotels[i].text
            hotel_info_dict['飯店名稱'] = hotel_name
            #print(hotel_name)

            hotel_score = scores[i].text
            hotel_info_dict['評分'] = hotel_score
            #print(hotel_score)

            original_web = web_pages[i].find('a').get('href')
            hotel_web = original_web.split('?')[0]
            hotel_info_dict['訂房網站'] = hotel_web
            print(hotel_web)
            hotel_info_list.append(hotel_info_dict)

            url = hotel_web
            #print(f'\nurl: {url}')

            #user_agent = ua.UserAgent().get() # marked by Peter
            #print(f"user_agent: {user_agent}")
            options = Options()
            options.add_argument("--headless")  # Remove this if you want to see the browser (Headless makes the chromedriver not have a GUI)
            options.add_argument("--window-size=1920,1080")
            options.add_argument("user-agent=%s"%user_agent)
            options.add_argument("--no-sandbox")
            options.add_argument("--disable-extensions")
            driver = webdriver.Chrome(options = options)
            driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", { # 清空 window.navigator
            "source": """
                Object.defineProperty(navigator, 'webdriver', {
                get: () => undefined
                })
            """
            }) # added by Peter
            driver.implicitly_wait(20)  # added by Peter
            driver.get(url)
            # print(driver.page_source)
            WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR,"span.a5a5a75131"))) # added by Peter
            page_content = driver.page_source #取得網頁內的內容
            soup = BeautifulSoup(page_content,'html.parser')

            services = soup.find('div',class_='hp--popular_facilities js-k2-hp--block').find_all('span',class_='a5a5a75131')
            service_list = []
            for service in services:
                hotel_service = service.text
                service_list.append(hotel_service)
            hotel_info_dict['熱門設施'] = service_list

            #print(f"{hotel_info_dict['飯店名稱']} {service_list}")
            #print(services)

            rooms = soup.find_all('div',class_='ed14448b9f b817090550 e7f103ee9e')
            room_list = []
            for room in rooms:
                hotel_room = room.text.strip()
                hotel_room = hotel_room.replace(' ','')
                #找到第一個出現1 2的位置
                first_one_index = hotel_room.find('1')
                first_two_index = hotel_room.find('2')

                #如果有找到1 2，確定添加(的位置
                if first_one_index != -1 or first_two_index != -1:
                    if first_one_index != -1 and (first_two_index == -1 or first_one_index < first_two_index):
                        insert_index = first_one_index
                    else:
                        insert_index = first_two_index

                    hotel_room = hotel_room[:insert_index] + '(' + hotel_room[insert_index:]
                hotel_room += ')'
                #print(hotel_room)
                room_list.append(hotel_room)

            hotel_info_dict['房型'] = room_list
            #print(room_list)

            checkin = soup.find('div', {'id': 'checkin_policy'})
            checkin_time = checkin.find_all('p')[1].text.strip().replace(' ','')
            hotel_info_dict['入住時間'] = checkin_time
            #print(checkin_time)

            checkout = soup.find('div', {'id': 'checkout_policy'})
            checkout_time = checkout.find_all('p')[1].text.strip().replace(' ','')
            hotel_info_dict['退房時間'] = checkout_time
            #print(checkout_time)

            driver.close() #關閉網頁
        driver.quit() #關閉瀏覽器
    #print(hotel_info_list)

        print('將旅宿資料寫入 Firestore !')
        print('資料寫入中...')
        if not firebase_admin._apps:
            cred = credentials.Certificate("hotel-1a77a-firebase-adminsdk-bnri1-fd5cb200db.json")
            firebase_admin.initialize_app(cred)
        db = firestore.client()

        for i in range(len(hotel_info_list)):
            # document 由 firestore hash 產生
            db.collection(city + '推薦飯店').add(hotel_info_list[i])
        db.close()
        print('旅宿資料寫入完畢 !')

        hotel_info_list = []

end_time = time.time()
elapsed_time = end_time - start_time
#print(elapsed_time)
#print(len(hotel_info_list))


from django.http import JsonResponse
from lxml import html
import requests
from .models import News

# Create your views here.

def test(request):
    """
    Fetches and returns the news from front page in json format
    """
    response = requests.get('https://www.indiatoday.in/technology/news')

    data = {'news': []}
    #   Dictionary so that to save the input in json format.

    if response:

        tree = html.fromstring(response.content)

        img_src = tree.xpath('/html/body/div[1]/main/div[1]/section/div[3]/div[1]/div/div/img/@src')
        headings = tree.xpath('/html/body/div[1]/main/div[1]/section/div[3]/div[1]/div/div[2]/h3/@title')
        content = tree.xpath('/html/body/div[1]/main/div/section/div[3]/div[1]/div/div/p/text()')

        for i in range(len(img_src)):

            data_exist = News.objects.filter(heading=headings[i])

            if not data_exist:
                data_save_obj = News.objects.create(heading=headings[i], content=content[i], img_src=img_src[i])
                data_save_obj.save()

            data['news'].append({
                'heading': headings[i],
                'content': content[i],
                'img_src': img_src[i],
            })

        return JsonResponse(data)

    else:

        data['news'].append({
            'error': 'There is no response from the request.get() method'
        })

        return JsonResponse(data)


"""Module that uses machine learning techniques to categorise transactions."""
from nltk.stem.snowball import SnowballStemmer
from .database import db, Transaction, Category
from sklearn import model_selection
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import SelectPercentile, f_classif
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import accuracy_score
from sklearn import preprocessing
from sklearn import svm
from flask_login import current_user
from flask import session
from dateutil.parser import parse
import string

def classification_score(group_id):
    """Calculate transaction classification score for group.

    This is used for checking algorithm from command line.
    """
    feature_data, label_data = collect_data_for_group(group_id)
    data_size = len(feature_data)
    features_train, features_test, labels_train, labels_test = split_data(
        feature_data, label_data
    )
    features_train, features_test, num_features = vectorize_data(
        features_train, features_test
    )
    features_train, features_test = feature_selection(
        features_train, features_test, labels_train
    )
    predict = svm_predict(features_train, labels_train, features_test)
    score = accuracy_score(labels_test, predict)
    return score, data_size, num_features

def predict_categories():
    """Predict transaction categories."""
    features_train, labels_train = collect_data()
    features_test = get_test_features()
    if len(features_train) == 0:
        predict = ["Unspecified Expense" for _ in features_test]
        return predict
    features_train, features_test, num_features = vectorize_data(
        features_train, features_test
    )
    features_train, features_test = feature_selection(
        features_train, features_test, labels_train
    )
    predict = svm_predict(features_train, labels_train, features_test)
    return predict

def predict_columns():
    """Predict transaction column labels."""
    transactions = session["uploaded_transactions"]
    result = ["ignore" for _ in range(len(transactions[0]))]
    header_row = False
    for i, value in enumerate(transactions[0]):
        if "date" in value.lower():
            result[i] = "date"
            header_row = True
        elif "description" in value.lower() or "narration" in value.lower():
            result[i] = "description"
            header_row = True
        elif (
            "dr" in value.lower() and "cr" in value.lower() and len(value) < 6
        ):
            result[i] = "drcr"
            header_row = True
        elif "debit" in value.lower():
            result[i] = "dr"
            header_row = True
        elif "dr" in value.lower() and len(value) < 5:
            result[i] = "dr"
            header_row = True
        elif "credit" in value.lower():
            result[i] = "cr"
            header_row = True
        elif "cr" in value.lower() and len(value) < 5:
            result[i] = "cr"
            header_row = True
    if not header_row:
        for i, value in enumerate(transactions[1]):
            if is_number(value):
                result[i] = "drcr"
            elif is_date(value):
                result[i] = "date"
            elif len(value) > 10:
                result[i] = "description"
    return result, header_row

def is_date(string):
    """Is string a date."""
    try:
        parse(string)
        return True
    except ValueError:
        return False

def is_number(s):
    """Is string a number."""
    try:
        float(s)
        return True
    except ValueError:
        return False

def collect_data():
    """Get existing transaction descriptions and categories."""
    feature_data = []
    label_data = []
    transactions = current_user.group().transactions
    for transaction in transactions:
        description = stem_description(transaction.description)
        feature_data.append(description)
        label_data.append(transaction.category.catname)
    # print('*******************************************************')
    # for num, feature in enumerate(feature_data):
    #     if 'myer' in feature:
    #         print(feature, label_data[num])
    # print('*******************************************************')
    return feature_data, label_data

def get_test_features():
    """Get test features from uploaded transactions."""
    features_test = []
    transactions = session["uploaded_transactions"]
    for transaction in transactions:
        description = " ".join(transaction)
        description = stem_description(description)
        features_test.append(description)
    # print('*******************************************************')
    # for num, feature in enumerate(features_test):
    #     if 'myer' in feature:
    #         print(num, ':', feature)
    # print('*******************************************************')
    return features_test

def collect_data_for_group(group_id):
    """Read transactions (descriptions and categories) from database.

    Add string of space separated stemmed words to feature_data list
    Add category to label_data list
    """
    feature_data = []
    label_data = []
    transactions = (
        db.session.query(Transaction, Category)
        .filter(Transaction.group_id == group_id)
        .filter(Transaction.catno == Category.catno)
        .all()
    )
    for transaction, category in transactions:
        description = stem_description(transaction.description)
        feature_data.append(description)
        label_data.append(category.catname)
    return feature_data, label_data

def stem_description(description):
    """Stem the transaction description."""
    translator = str.maketrans("", "", string.punctuation)
    description = description.translate(translator)  # Remove punctuation
    stemmer = SnowballStemmer("english")
    description_list = description.split()
    stemmed_list = [stemmer.stem(word) for word in description_list]
    return " ".join(stemmed_list)

def split_data(feature_data, label_data):
    """Split data into train and test."""
    (
        features_train,
        features_test,
        labels_train,
        labels_test,
    ) = model_selection.train_test_split(
        feature_data, label_data, test_size=0.1, random_state=42
    )
    return features_train, features_test, labels_train, labels_test

def vectorize_data(features_train, features_test):
    """Vectorize the feature data ie. create bag of words.

    Go from strings to lists of numbers
    (trans_no, word_no) freq_of_word_no
     vectorizer.get_feature_names()[word_no] is feature name

    """
    vectorizer = TfidfVectorizer(
        sublinear_tf=True, max_df=0.5, stop_words="english"
    )
    features_train_transformed = vectorizer.fit_transform(features_train)
    features_test_transformed = vectorizer.transform(features_test)
    num_features = len(vectorizer.get_feature_names())
    return features_train_transformed, features_test_transformed, num_features

def feature_selection(features_train, features_test, labels_train):
    """Feature selection."""
    selector = SelectPercentile(f_classif, percentile=100)
    selector.fit(features_train, labels_train)
    features_train_transformed = selector.transform(features_train).toarray()
    features_test_transformed = selector.transform(features_test).toarray()
    return features_train_transformed, features_test_transformed

def naive_bayes_predict(features_train, labels_train, features_test):
    """Naive Bayes."""
    clf = GaussianNB()
    # t0 = time()
    clf.fit(features_train, labels_train)
    # print("training time:", round(time() - t0, 3), "s")
    # t0 = time()
    predict = clf.predict(features_test)
    # print("predict time:", round(time() - t0, 3), "s")
    return predict

def svm_predict(features_train, labels_train, features_test):
    """SVM algorithm."""
    scaler = preprocessing.StandardScaler().fit(features_train)
    features_train = scaler.transform(features_train)
    features_test = scaler.transform(features_test)
    clf = svm.SVC()
    # t0 = time()
    clf.fit(features_train, labels_train)
    # print("training time:", round(time() - t0, 3), "s")
    # t0 = time()
    predict = clf.predict(features_test)
    # print("predict time:", round(time() - t0, 3), "s")
    return predict


import logging
from package.module_1 import some_function

entry_logger = logging.getLogger("entry logger")
entry_logger.addHandler(logging.StreamHandler())
entry_logger.info("this message won't be printed")

module_logger = logging.getLogger("package.module_1")
print(module_logger.getEffectiveLevel())
print(entry_logger.getEffectiveLevel())

entry_logger.info("this message will not be printed")
some_function()


"""
Methods in this file are used to open URLs in a web browser to assist with data inputation.
"""
from typing import List
from urllib.request import urlopen
from urllib.error import HTTPError
import webbrowser as web
import validators

def open_url(urls: List[str]) -> None:
    """
    Open URL in web browser. If first URL doesn't work, try second, so on and so forth.
    """
    errors = []
    for url in urls:
        try:
            if validators.url(url):  # type: ignore
                html = urlopen(url)
                web.open(url)
                html.close()
                return
            else:
                errors.append("Validation Error")
        except HTTPError as e:
            errors.append(e)
    print(f"No valid URLs: {errors}")


import cv2
import PIL.Image as PImage
import matplotlib.pyplot as plt
from matplotlib import transforms
import numpy as np
import math
import os
import httpx
import keras_ocr

from image_utils import load_images

# def paragraph_extract_preprocess(image : np.ndarray, min_size=(1080, 1080)):
#     """
#     """
#     image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
#     kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
#     image = cv2.GaussianBlur(image, kernel.shape, 0)
#     image = cv2.medianBlur(image, 5)
#     thres, image = cv2.threshold(image, 0, 204, cv2.THRESH_BINARY + cv2.THRESH_OTSU)

#     plt.imshow(image, cmap='gray')
#     plt.show()
#     return image

# def extract_paragraphs(image : np.ndarray):
#     # Erode để phân tách dòng
#     kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (11,1))
#     image = cv2.erode(image, kernel, iterations=4)
#     contours, hierarchy = cv2.findContours(image, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
#     plt.imshow(image, cmap='gray')
#     plt.show()

    # avg_height = 0
    # for contour in contours:
    #     x, y, width, height = cv2.boundingRect(contour)
    #     cv2.rectangle(image, (x, y, width, height), color=(0, 0, 255), thickness=3)
    #     avg_height += height
    # avg_height = int(avg_height /(12 * len(contours))) + 1
    # print(avg_height)

    # kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1,avg_height))
    # image = cv2.erode(image, kernel, iterations=4)
    # plt.imshow(image, cmap='gray')
    # plt.show()

    # contours, hierarchy = cv2.findContours(image, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    # return contours

def segment_image_by_lines(image : np.ndarray, line_boxes) -> list:
    """tách ảnh theo dòng, trả về ảnh crop theo dòng

    image: ảnh gốc
    line_boxes: danh sách vị trí các dòng, trả về ở extract_text_lines()
    """
    # plt.imshow(img, cmap='gray')
    # plt.show()
    # preprocessed_img = paragraph_extract_preprocess(img)
    # # print(pytesseract.image_to_osd(preprocessed_img))
    # # offset = detect_orientation(preprocessed_img)
    # # reoriented_img = orient_image(preprocessed_img, offset)
    # # reoriented_original_img = orient_image(img, offset)

    # line_crops = []
    # y_list = []
    # contours = extract_paragraphs(preprocessed_img)
    # for contour in contours:
    #     x, y, width, height = cv2.boundingRect(contour)
    #     if height < img.shape[1]/100:
    #         continue

    #     # cv2.rectangle(img, (x, y, width, height), color=(0, 0, 255), thickness=3)
    #     line_crops.append(img[y : y+height, x : x+width])
    #     y_list.append(y)

    # line_crops.pop(0) # Contour đầu tiên là toàn bộ ảnh, bỏ ra vì ko cần
    # y_list.pop(0)
    # # Sắp xếp từ trên xuống dưới
    # args_sorted = np.argsort(y_list)
    # line_crops = [line_crops[i] for i in args_sorted]
    # return line_crops

    image = cv2.cvtColor(image, code=cv2.COLOR_RGB2GRAY)
    line_crops = []
    for line_box in line_boxes:
        rect = cv2.boundingRect(line_box)
        x, y, w, h = rect
        crop = image[y : y+h, x : x+w].copy()

        line_box = line_box - line_box.min(axis=0)
        mask = np.zeros((h, w), dtype=np.uint8)
        cv2.drawContours(mask, [line_box], -1, (255, 255, 255), -1, cv2.LINE_AA)
        # print(crop)
        line_image = cv2.bitwise_and(crop, crop, mask=mask)

        background = np.ones_like(crop, dtype=np.uint8)*255
        cv2.bitwise_not(background, background, mask=mask)
        line_image = line_image + background
        line_crops.append(line_image)

    # image = cv2.cvtColor(image, code=cv2.COLOR_RGB2GRAY)
    # line_crops = []
    # for boxes in line_boxes:
    #     points = []
    #     for point in boxes:
    #         points.append(point)

    #     points = np.asarray(points, dtype=np.uint8)
    #     print(points)

    #     rect = cv2.boundingRect(points)
    #     x, y, w, h = rect
    #     crop = image[y : y+h, x : x+w]

    #     mask = np.zeros((h,w), dtype=np.uint8)
    #     for box in line_boxes:
    #         cv2.drawContours(mask, [box], -1, (255,0,0), -1, cv2.LINE_AA)

    #     line_image = cv2.bitwise_and(crop, crop, mask=mask)

    #     background = np.ones_like(crop, dtype=np.uint8)*255
    #     cv2.bitwise_not(background, background, mask=mask)
    #     line_image = line_image + background
    #     line_crops.append(line_image)

    return line_crops

def separate_lines(image : np.ndarray, threshold = 200):
    '''tìm ranh giới các dòng, sắp xếp từ trên xuống dưới. Trả về list các (y1, y2), y1 là cận trên dòng, y2 là cận dưới dòng

    image: ảnh rgb
    '''
    image = cv2.cvtColor(image, code=cv2.COLOR_RGB2GRAY)
    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
    image = cv2.GaussianBlur(image, kernel.shape, 0)
    image = cv2.medianBlur(image, 5)
    thres, image = cv2.threshold(image, 0, 204, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)

    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (11,1))
    image = cv2.dilate(image, kernel, iterations=4)
    histogram = cv2.reduce(image, dim=1, rtype=cv2.REDUCE_AVG)
    histogram = [math.pow(x[0], 2) for x in histogram]
    # plt.imshow(image,cmap='gray')
    plt.plot(histogram)
    plt.savefig("debug_images/histogram.png")

    H,W = image.shape[:2]
    uppers = [y for y in range(H-1) if histogram[y]<=threshold and histogram[y+1]>threshold]
    lowers = [y for y in range(H-1) if histogram[y]>threshold and histogram[y+1]<=threshold]

    debug_image = cv2.cvtColor(image.copy(), cv2.COLOR_GRAY2BGR)
    for y in uppers:
        cv2.line(debug_image, (0,y), (W, y), (255,0,0), 1)

    for y in lowers:
        cv2.line(debug_image, (0,y), (W, y), (0,255,0), 1)

    cv2.imwrite("debug_images/line_sep_res.png", debug_image)

    # plt.imshow(image)
    # plt.show()

    lines = [(y1, y2) for y1, y2 in zip(uppers, lowers)]
    return lines

def keras_recognize(image : np.ndarray):
    """nhận diện vị trí chữ bằng Keras_ocr

    image: ảnh, dạng numpy array
    """
    debug_image = image.copy()
    image, scale_factor = preprocess_osd(image)
    image = cv2.cvtColor(image, code=cv2.COLOR_GRAY2RGB)
    recognizer = keras_ocr.detection.Detector()
    prediction_groups = recognizer.detect([image])

    groups = []
    for prediction in prediction_groups:
        boxes = []
        for box in prediction:
            unscaled_box = [(x / scale_factor, y / scale_factor) for x, y in box]
            unscaled_box = np.array(unscaled_box, dtype=np.int32)
            boxes.append(unscaled_box)
        groups.append(boxes)

    debug_image = keras_ocr.tools.drawBoxes(debug_image, boxes, color=(255,0,0), thickness=1, boxes_format='boxes')
    cv2.imwrite("debug_images/osd_result.png", debug_image)

    return groups

def categorize_by_lines(lines : list, prediction_groups):
    """Phân nhóm các hộp chứa chữ theo dòng

    lines: danh sách dòng, trả về từ separate_lines()
    prediction_groups: danh sách nơi có chữ, trả về từ keras_recognize()
    """
    box_by_lines = [[] for line in lines]
    for prediction in prediction_groups:
        for box in prediction:
            v1, v2, v3, v4 = box
            centroid = (v1[1] + v2[1] + v3[1] + v4[1]) / 4
            line_idx = 0
            for line_y1, line_y2 in lines:
                if line_y1 <= centroid <= line_y2:
                    # print("box: ", box, "IS in line", line_idx, "because", line_y1, line_y2)
                    box_by_lines[line_idx].append(box)
                    break
                else:
                    # print("box: ", box, "NOT in line", line_idx, "because", line_y1, line_y2)
                    line_idx += 1
    return box_by_lines

def preprocess_osd(image : np.ndarray, min_size=(2160,2160)):
    if len(image.shape) == 3:
        image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    total_scale_factor = 1

    min_width, min_height = min_size
    height, width = image.shape[:2]

    if width < min_width:
        scale_factor = min_width / width
        height = int(height * scale_factor)
        width = min_width
        total_scale_factor *= scale_factor
    if height < min_height:
        scale_factor = min_height / height
        width = int(width * scale_factor)
        height = min_height
        total_scale_factor *= scale_factor

    # while width < min_width or height < min_height:
    #     width *= 2
    #     height *= 2
    #     total_scale_factor *= 2

    image = cv2.resize(src=image, dsize=(width, height), interpolation=cv2.INTER_CUBIC)
    return image, total_scale_factor

def preprocess_ocr(image : np.ndarray, min_size=(320,320)):
    """Tiền xử lý ảnh trước khi OCR, trả về ảnh đã xử lý

    image: ảnh gốc, dạng numpy array
    min_size: kích cỡ tối thiểu (chiều cao, chiều dài). Đối với OCR nên để ở khoảng
    """
    if len(image.shape) == 3:
        image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)

    image, scale_factor = preprocess_osd(image, min_size=min_size)
    thres, image = cv2.threshold(image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
    # kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
    # image = cv2.GaussianBlur(image, kernel.shape, 0)
    # thres, image = cv2.threshold(image, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
    # image = cv2.morphologyEx(image, cv2.MORPH_CLOSE, kernel)
    return image

def send_to_tika(image : np.ndarray):
    is_success, image_buffer = cv2.imencode(".jpeg", image)
    image_bytes = image_buffer.tobytes()
    with httpx.Client() as client:
        headers = {
            "Accept": "text/plain",
            "X-Tika-OCRPageSegMode": "7", #single text line
            "X-Tika-OCRLanguage": "vie"
        }
        return client.put(url="http://10.15.10.123:9998/tika", data=image_bytes, headers=headers, timeout=None).text

def extract_text_lines(image : np.ndarray):
    """tìm dòng, trả về polygon bao quanh mỗi chữ dưới dạng list

    image: ảnh dạng numpy array
    """
    lines = separate_lines(image)
    prediction_groups = keras_recognize(image)
    box_by_lines = categorize_by_lines(lines, prediction_groups)
    line_boxes = []
    for line in box_by_lines:
        line_box = []
        for box in line:
            for point in box:
                line_box.append(point)
        if len(line_box) == 0:
            continue
        box = np.asarray(line_box, dtype=np.int32)
        rectangle = cv2.minAreaRect(box.copy())
        box = cv2.boxPoints(rectangle)
        box = np.array(box, dtype=np.int32)
        line_boxes.append(box)

    debug_image = image.copy()
    debug_image = cv2.polylines(debug_image, line_boxes, isClosed=True, color=(0, 0, 255), thickness=2)
    cv2.imwrite("debug_images/polygon.png", debug_image)

    return line_boxes

def read_image_by_lines(image : np.ndarray):
    """OCR ảnh, sắp xếp theo từng dòng

    image: ảnh, dạng numpy array
    """
    plt.imshow(image)
    plt.savefig("debug_images/source.png")
    results = []
    line_boxes = extract_text_lines(image)
    line_crops = segment_image_by_lines(image, line_boxes)
    for line_crop in line_crops:
        line_crop = preprocess_ocr(line_crop)
        result = send_to_tika(line_crop)
        results.append(result)

    print(results)
    return results

def main():
    img = load_images()
    img = img[3]
    results = []
    line_boxes = read_image_by_lines(img)

    # box = [[910,1969],
    #         [934,1946],
    #         [1000,2016],
    #         [976,2039],
    #         [1086,1958],
    #         [1124,1958],
    #         [1124,1987],
    #         [1086,1987],
    #         [1067,2028],
    #         [1153,1952],
    #         [1177,1978],
    #         [1092,2055],
    #         [1004,2018],
    #         [1080,2018],
    #         [1080,2047],
    #         [1004,2047],
    #         [1149,2038],
    #         [1211,2038],
    #         [1211,2066],
    #         [1149,2066],
    #         [1092,2043],
    #         [1137,2043],
    #         [1137,2066],
    #         [1092,2066]]

    # box = np.asarray(box, dtype=np.int32)
    # rectangle = cv2.minAreaRect(box.copy())
    # print(rectangle)
    # read_image_by_lines(img)

if __name__ == '__main__':
    main()


from .analysis import polyfit
import matplotlib
import matplotlib.pyplot as plt

"""This module contains a collection of functions related to
plotting water level data.

"""

def plot_water_levels(station, dates, levels):
    """Function to plot station water level data against time, with plot lines for low and high water level marks
    Axis are labelled, title is the station name

    Params:
    - station: MonitoringStation object
    - dates: list of datetime objects
    - levels: list of floats
    """

    # Plot level data
    plt.plot(dates, levels)

    # Add low/high water level marks
    lows = [station.typical_range[0]]*len(dates)
    highs = [station.typical_range[1]]*len(dates)

    plt.plot(dates, lows)
    plt.plot(dates, highs)

    # Add axis labels, rotate date labels and add plot title
    plt.xlabel('date')
    plt.ylabel('water level (m)')
    plt.xticks(rotation=45);
    plt.title(station.name)

    # Display plot
    plt.tight_layout()  # This makes sure plot does not cut off date labels
    plt.show()

def plot_water_level_with_fit(station, dates, levels, p):
    """function that plots the water level data and the best-fit polynomial"""

    # Plot water level data
    plt.plot(dates, levels)

    # Get best-fit polynomial
    poly, d0 = polyfit(dates, levels, p)

    # Plot polynomial on same graph
    dates_float = matplotlib.dates.date2num(dates)
    d0_float = matplotlib.dates.date2num([d0])[0]
    plt.plot(dates, poly(dates_float-d0_float))

    # Add low/high water level marks
    lows = [station.typical_range[0]]*len(dates)
    highs = [station.typical_range[1]]*len(dates)

    plt.plot(dates, lows)
    plt.plot(dates, highs)

    # Add axis labels, rotate date labels and add plot title
    plt.xlabel('date')
    plt.ylabel('water level (m)')
    plt.xticks(rotation=45);
    plt.title(station.name)

    # Display plot
    plt.show()

def polynomial_coeff(station, dates, levels, p):
    """function that returns the best-fit polynomial equation"""

    # Get best-fit polynomial
    poly, d0 = polyfit(dates, levels, p)

    return poly


import pytest

from llama_hub.docstring_walker.base import DocstringWalker
from llama_index import Document

SOME_CLASS_WITH_DOCSTRING = """
'''Basic module with a class definition'''

class Person:

    '''Basic class to represent a person
    '''

    def __init__(self, name: str, surname: str, age: int):
        self.name = name
        self.surname = surname
        self.age = age

    @property
    def full_name(self) -> str:
        '''A property for getting person fullname

        Returns
        -------
        str
            Full name: concatencation of name and surname
        '''
        return self.name + ' ' + self.surname

    def greet(self, other: str) -> str:
        '''Greeting function for a person

        Parameters
        ----------
        other : str
            Other person name.

        Returns
        -------
        str
            Greeting string.
        '''
        return 'Hello ' + other + ' , my name is ' + self.full_name
"""

MALFORMED_FILE = """
def addtwo(a, b):
    return a + b kasdlkjas
"""

SOME_FUNCTION = """
def add_numbers(a, b):
    return a+b
"""

def test_reading_module_with_class(mocker):
    # Given
    mocker.patch("os.path.exists", return_value=True)
    walker = DocstringWalker()
    mocker.patch(
        "os.walk",
        return_value=[
            ("somepath", "", ["somefile1.py"]),
        ],
    )
    mocker.patch.object(
        walker, "read_module_text", return_value=SOME_CLASS_WITH_DOCSTRING
    )

    # When
    docs = walker.load_data("somepath")

    # Then
    assert len(docs) == 1
    assert isinstance(docs[0], Document)
    assert docs[0].text.startswith("Module name: somefile")
    assert docs[0].text.endswith("Greeting string.\n")

def test_dont_fail_on_malformed_file(mocker):
    # Given
    mocker.patch("os.path.exists", return_value=True)
    walker = DocstringWalker()

    mocker.patch(
        "os.walk",
        return_value=[
            ("somepath", "", ["somefile.py"]),
        ],
    )
    mocker.patch.object(walker, "read_module_text", return_value=MALFORMED_FILE)

    # When
    docs = walker.load_data("somepath", fail_on_malformed_files=False)

    # Then
    assert len(docs) == 0

def test_fail_on_malformed_file(mocker):
    # Given
    mocker.patch("os.path.exists", return_value=True)
    walker = DocstringWalker()

    mocker.patch(
        "os.walk",
        return_value=[
            ("somepath", "", ["somefile.py"]),
        ],
    )
    mocker.patch.object(walker, "read_module_text", return_value=MALFORMED_FILE)

    # Then
    with pytest.raises(SyntaxError):
        walker.load_data("somepath", fail_on_malformed_files=True)

def test_reading_multiple_modules(mocker):
    # Given
    mocker.patch("os.path.exists", return_value=True)
    walker = DocstringWalker()
    mocker.patch(
        "os.walk",
        return_value=[
            ("somepath", "", ["somefile1.py", "somefile2.py"]),
        ],
    )
    mocker.patch.object(
        walker, "read_module_text", return_value=SOME_CLASS_WITH_DOCSTRING
    )

    # When
    docs = walker.load_data("somepath")

    # Then
    assert len(docs) == 2
    assert isinstance(docs[0], Document)
    assert all([doc.text.startswith("Module name: somefile") for doc in docs])
    assert all([doc.text.endswith("Greeting string.\n") for doc in docs])


import json
import smtplib
from email.mime.text import MIMEText
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart

class StatusMail(object):

    def __init__(self, email_config_path):
        with open(email_config_path) as config:
            self.email_config = json.load(config)

        self.smtp_host = self.email_config["smtp_details"]["host"]
        self.mail_from = "manish.6.singh.ext@nokia.com"
        self.mail_to = "manish.6.singh.ext@nokia.com"
        self.mail_cc = []
        self.mail_initial = self.email_config["email_details"]["mail_initial"]
        self.mail_greeting = self.email_config["email_details"]["mail_greeting"]
        self.mail_signature_header = self.email_config["email_details"]["mail_signature_header"]
        self.mail_signature = self.email_config["email_details"]["mail_signature"]
        self.mail_footer = self.email_config["email_details"]["mail_footer"]

    def shoot_mail(self, image_path, status):
        file_path = image_path
        file_status = status

        mail_subject = self.email_config["email_details"]["mail_subject"].replace("task_status", file_status)
        mail_body = self.email_config["email_details"]["main_body"].replace("task_status", file_status)

        msg_root = MIMEMultipart('related')
        msg_root['Subject'] = mail_subject
        msg_root['From'] = self.mail_from
        msg_root['To'] = self.mail_to

        msg_root.preamble = 'This is a multi-part message in MIME format.'
        msg_alternative = MIMEMultipart('alternative')
        msg_root.attach(msg_alternative)

        html_body = self.mail_initial + "
" + "
" + self.mail_greeting + "
" + "
" + mail_body + "
" + "
" + "
"
        html_footer = "
" + "
" + self.mail_signature_header + "
" + self.mail_signature + "
" + "
" + self.mail_footer
        msg_text = MIMEText(html_body + '
' + html_footer, 'html')
        msg_alternative.attach(msg_text)

        fp = open(file_path, 'rb')
        msg_image = MIMEImage(fp.read())
        fp.close()

        msg_image.add_header('Content-ID', '<image1>')
        msg_root.attach(msg_image)

        smtp = smtplib.SMTP(self.smtp_host)
        mail_receipt = [self.mail_to] + self.mail_cc
        smtp.sendmail(self.mail_from, mail_receipt, msg_root.as_string())
        smtp.quit()

# if __name__ == "__main__":
#
#     images_path_ = r"C:\Users\m4singh\PycharmProjects\GraphAutomationMRTG\ErrorDetection&Mail\InputImages"
#     shoot_mail(images_path_)


######Introduction to classes########################
class example:
  def __init__(self, x, y):
    self.num1 = x
    self.num2 = y

  def sum(self):
     s = self.num1 + self.num2
     return(s)##or print

  def diff(self):
     d = self.num1-self.num2
     return(d)

  def sum_diff(self):
      k = self.diff()+self.sum()
      return(k)

p1 = example(2, 36)

p1.sum()
p1.diff()
p1.sum_diff()

######Create a class to estimate a regression model#############
import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
x , y , coef  = datasets.make_regression(n_samples=100,#number of samples
                                      n_features=1,#number of features
                                      n_informative=1,#number of useful features
                                      noise=10,#bias and standard deviation of the guassian noise
                                      coef=True,#true coefficient used to generated the data
                                      random_state=0) #set for same data points for each run
coef
data_x = x.flatten()
data_y = y.flatten()
plt.scatter(data_x,data_y,c ="blue")

class regression:
    def __init__(self,response,covariate):
        self.x = covariate
        self.y = response
    ######Estimate the parameters######### y = a + bx
    def coefficients(self):
        b = np.sum((self.x-np.mean(self.x))*(self.y-np.mean(self.y)))/np.sum(np.power(self.x-np.mean(self.x),2))
        a = np.mean(self.y) - b*np.mean(self.x)
        return([a,b])
    #####Estimate the fitted values########## y_hat = a_hat + b_hat*x
    def fitted(self):
        y_hat = self.coefficients()[0] + self.coefficients()[1]*self.x
        return(y_hat)
    ####Estimate the residuels############# e = y_hat-y
    def residuals(self):
        res = self.fitted() - self.y
        return(res)
    ####Plot standardized residuals vs fitted values#######
    def diagnostic_plot(self):
        std_res = self.residuals()/np.std(self.residuals())
        print(plt.scatter(r.fitted(),std_res , c="blue"))
    ####Predict function###################################
    def predict(self,new_x):
        y_pred = self.coefficients()[0] + self.coefficients()[1]*new_x
        return(y_pred)

r  = regression(data_y,data_x)
r.coefficients()
r.fitted()
r.residuals()
r.diagnostic_plot()
r.predict(np.array([5,4,-1]))


# dummy module for use with my logger examples
import logging
from logging import config
from src.conf.logging_dictionary import dict_log_config

logging.config.dictConfig(dict_log_config)
logger = logging.getLogger("utility_logger")

def add(x, y):
    return x + y

class Adder(object):
    """
    Object that does the same as the function above, just for shits and grins
    """
    def add_numbers(self, a, b):
        logger.info('adding numbers in my method')
        return a + b


'''
Created on 04.03.2010

@author: Alex
'''

import os.path
import sys
import tempfile
import shutil
import stat

from CDUtils import UserException

# --------------------------------------------

def OsExec(cmd, printCmd = False) -> int:
    if printCmd: print(cmd)
    res = os.system('"' + cmd + '"')
    if printCmd: print('')
    return res

# --------------------------------------------

def Main(args : [str]):

    psplit = os.path.split
    pjoin  = os.path.join

    CConfigDir = 'conf'

    if len(args) != 1:
        print('Usage: script <repository path>')
        exit()

    def Exec(cmd, stageName):
        res = OsExec(cmd, True)
        if res != 0: raise UserException("Stage '{0}' error!".format(stageName))

    reposPath = args[0]
    reposShortName = psplit(reposPath)[1]
    reposUrl = 'svn://127.0.0.1/' + reposShortName
    tmpDir = tempfile.mkdtemp()

    # 1. copy config to temp
    tmpConfig = pjoin( tmpDir, CConfigDir )
    reposConfig = pjoin(reposPath, CConfigDir)
    shutil.copytree(reposConfig, tmpConfig)

    # 1a. remove read-only
    def RemoveReadOnly(path):
        fullPath = pjoin(reposPath, path)
        os.chmod(fullPath, stat.S_IREAD | stat.S_IWRITE)

    RemoveReadOnly('format')
    RemoveReadOnly('db/format')

    # 2. svn export
    tmpExportPath = pjoin(tmpDir, reposShortName)
    cmd = 'svn export {url} "{path}"'.format(url = reposUrl, path = tmpExportPath)
    Exec(cmd, 'export')

    #3. rm repository
    shutil.rmtree(reposPath)

    #4. create repository
    Exec( 'svnadmin create "{0}"'.format(reposPath), 'create' )

    #5. restore config
    shutil.rmtree(reposConfig)
    shutil.copytree(tmpConfig, reposConfig)

    #6. import
    cmd = 'svn import "{path}" {url} -m "{msg}"'.format(path = tmpExportPath,
                                                        url = reposUrl,
                                                        msg = 'clean up')
    Exec(cmd, 'import')

    #7. clean temp
    shutil.rmtree(tmpDir)

    print('Done!')

# --------------------------------------------

if __name__ == '__main__':
    Main(sys.argv[1:])


from django.shortcuts import render
from rest_framework import viewsets
from .models import Movie, Genre
from .serializers import MovieSerializer, GenreSerializer
from rest_framework.pagination import LimitOffsetPagination

# Create your views here.

class GenreViewSet(viewsets.ModelViewSet):
    queryset = Genre.objects.all().order_by('name')
    serializer_class = GenreSerializer

class MovieViewSet(viewsets.ModelViewSet):
    queryset = Movie.objects.all().order_by('name')
    serializer_class = MovieSerializer
    pagination_class = LimitOffsetPagination

    def get_queryset(self):
        qs = super().get_queryset()
        name = self.request.query_params.get('name', None)
        if name is not None:
            qs = qs.filter(name__icontains=name)

        genre = self.request.query_params.get('genre', None)
        if genre is not None:
            qs = qs.filter(genre__name__icontains=genre)

        director = self.request.query_params.get('director', None)
        if director is not None:
            qs = qs.filter(director__icontains=director)

        return qs


from PIL import Image
import json

calibrationNumber = 50
titleHeight = 65  # npr Zadaci visestrukog izbora
colors = {
    'Brojevi i algebra': (229, 177, 32)
    # ostale boje
}

def isColorSame(rgbColor1, rgbColor2):
    isSame = rgbColor1[0] > rgbColor2[0]-calibrationNumber and rgbColor1[0] < rgbColor2[0]+calibrationNumber and rgbColor1[1] > rgbColor2[1] - \
        calibrationNumber and rgbColor1[1] < rgbColor2[1]+calibrationNumber and rgbColor1[2] > rgbColor2[2] - \
        calibrationNumber and rgbColor1[2] < rgbColor2[2]+calibrationNumber
    return isSame

def isTitle(yCoordinate, borderColor):
    pixelColor = image.getpixel((width*0.33, yCoordinate+5))

    # if title (Zadatci visestrukog izbora)
    if isColorSame(borderColor, pixelColor):
        return True
    else:
        return False

def findYBorders(image, borderColor, height, width):
    borders = []
    y = 1
    returnedBorders = []
    while y != height-1:
        pixelColor = image.getpixel((width*0.33, y))

        if isColorSame(borderColor, pixelColor):

            # if title (Zadatci visestrukog izbora)
            if isTitle(y, borderColor):

                # check if title is the first border
                if len(borders) != 0:
                    borders.append(y)

                borders.append(y + titleHeight)
                y += titleHeight
            else:
                borders.append(y+3)
                y += 3
        y += 1

    if len(borders) % 2 != 0:
        print('ERROR - neparni broj bordera')
        return

    for i in range(0, len(borders), 2):
        # if gap greater than 100px (if not title)
        if borders[i+1] - borders[i] > 100:
            returnedBorders.append((borders[i], borders[i+1]))

    return returnedBorders

def cutPage(image, frame):
    left, right = frame

    for i in range(len(borders)):
        top, bottom = borders[i]
        cutPage = image.crop((left, top, right, bottom-6))
    return cutPage

def isTextOnBorder(image, y, width):
    for x in range(width-1):
        pixelColor = image.getpixel((x, y))

        if isColorSame((255, 255, 255), pixelColor) == False:
            return True
    return False

def cutZadatci(cutPage, height, width, borderColor):
    zadatciY = []
    returiningZadatciY = []
    zadatci = []
    y = 0

    # get Y of zadatci
    while y < height:
        x = 0
        while x < int(width/15):
            pixelColor = cutPage.getpixel((x, y))
            if isColorSame(borderColor, pixelColor):
                zadatciY.append(y - 20)
                y += 20
                x = 0
            else:
                x += 1
        y += 1

    # structure Y of zadatci
    for i in range(len(zadatciY)):
        zadatakY = zadatciY[i]
        if i == len(zadatciY) - 1:
            nextzadatakY = height-1
        else:
            nextzadatakY = zadatciY[i+1]

        returiningZadatciY.append((zadatakY, nextzadatakY))

    # cut zadatci
    for i in range(len(returiningZadatciY)):
        top, bottom = returiningZadatciY[i]

        # if zadatak is past border
        while True:
            if isTextOnBorder(cutPage, top, width):
                top -= 10
            elif isTextOnBorder(cutPage, bottom, width):
                bottom -= 10
            else:
                break

        cutZadatak = cutPage.crop((0, top, width, bottom))
        zadatci.append(cutZadatak)
    return zadatci

with open('book.json') as f:
    book = json.load(f)

for i in range(1):  # len(book)
    obj = book[i]
    print(obj['zadatci'])

    for page in range(obj['zadatci'][0], obj['zadatci'][1]):

        image = Image.open('knjiga/{}.jpg'.format(page))
        width, height = image.size
        borderColor = colors[obj['cjelina']]

        if (page % 2 == 0):
            frame = [208, 1068]
        else:
            frame = [134, 993]

        borders = findYBorders(image, borderColor, height, width)

        framedPage = cutPage(image, frame)

        zadatci = cutZadatci(
            framedPage, framedPage.size[1], framedPage.size[0], borderColor)
        print(zadatci)


import socket
import json
import struct
import time
from datetime import datetime
import sys	#for exit

def readblob(the_socket,timeout=2):
    #make socket non blocking
    the_socket.setblocking(0)

    #total data partwise in an array
    total_data=[]
    data=''

    #beginning time
    begin=time.time()
    while 1:
        #if you got some data, then break after timeout
        if total_data and time.time()-begin > timeout:
            break

        #if you got no data at all, wait a little longer, twice the timeout
        elif time.time()-begin > timeout*2:
            break

        #recv something
        try:
            data = the_socket.recv(8192)
            if data:
                total_data.append(data)
                #change the beginning time for measurement
                begin=time.time()
            else:
                #sleep for sometime to indicate a gap
                time.sleep(0.1)
        except:
            pass

    #join all parts to make final string
    return total_data

HOST = '127.0.0.1'  # Standard loopback interface address (localhost)
PORT = 8009        # Port to listen on (non-privileged ports are > 1023)
BUFFER_SIZE = 4096

with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
    s.bind((HOST, PORT)) #Bind system socket
    s.listen(3) #Listen for up to 3 connections
    print("Listening on %s:%s..." % (HOST, str(PORT)))

    while True:

        conn, addr = s.accept()
        with conn:
            print('Connected by', addr)
            data = readblob(conn)
            jdata = ''
            for i in data:
                jdata += i.decode('utf-8')

            # datetime object containing current date and time
            now = datetime.now()
            # dd-mm-YY_at_H-M-S
            dt_string = now.strftime("%d-%m-%Y_at_%H-%M-%S")
            jFile = open(dt_string+".json","w")
            jFile.write(jdata)
            jFile.close()

            print("Connection received from %s..." % str(addr))
            conn.sendall('OK'.encode("ascii"))
            conn.close()