"""Create dataset to be used for ASR pipelines for BizSpeech Dataset.

Author:
Anand C U
"""

import json
import logging
import pathlib
import shutil
import sys
import multiprocessing
#from multiprocessing import Process

import numpy as np
import speechbrain as sb
import torchaudio
import webdataset as wds
from hyperpyyaml import load_hyperpyyaml
from tqdm import tqdm

logger = logging.getLogger(__name__)
the_queue = multiprocessing.Queue()

def time_str_to_seconds(time_str):
    """
    Return time format in string to secods for easy airthmetic operations.

    Args : time_str : str with format hh:mm:ss
    """
    time_str = str(time_str)
    millisecs = 0
    if "." in time_str:
        time_str, millisecs = time_str.split(".")
    return int(time_str.split(":")[
        0]) * 3600 + int(time_str.split(":")[1]) * 60 + int(time_str.split(":")[2]) + float(millisecs) / 1000

class create_DataSet:
    """Class to create susets of Bizspeech dataset according to required native/nonnative speaker ratio, qna/presentation ratio."""

    def __init__(self, hparams: dict, included_events: list = [], excluded_events: list = []):
        """Init function.

        Args:
            hparams (dict): Hyperparameters dictionary, usually contains contents from 'dataset.yaml' loaded to a dict.
            included_events (list, optional): Specify the eventIDs to choose from first before moving on to other data. Defaults to [].
            excluded_events (list, optional): Specify the eventIDs to exlude when creating subset. Might include invalid data, etc.. Defaults to [].
        """

        if hparams["non_CEO_utt"]:
            logger.info(
                "Native/ Non Native Criteria is not well defined for Non CEOs. The criteria ratios may not be accurate with this enabled")
        self.dataset_dict = {}
        self.split_dicts = {"train": {}, "val": {}, "test": {}}
        self.dataPath = hparams["data_folder"]
        self.non_CEO_utt = hparams["non_CEO_utt"]
        self.bizspeech_metadata = self.load_Bizspeech_metadata()
        self.utterance_duration_limit = hparams["utterance_duration_limit"]
        self.audio_filetype = hparams["audio_filetype"]
        trainValTest = hparams["trainValTest"]
        if not included_events:
            included_events = list(self.bizspeech_metadata.keys())
        event_list = list(set(included_events) - set(excluded_events))
        event_list = np.sort(event_list)
        np.random.seed(hparams["dataset_seed"])
        np.random.shuffle(event_list)
        logger.info(f"Picking data from {len(event_list)} events.")
        totalDuration = hparams["hours_reqd"] * 60 * 60 * 1000
        nonnative = hparams["nonnative"]
        qna = hparams["qna"]
        '''
        self.duration_dict_limit = {
            "nativepresentation": totalDuration * (1 - nonnative) * (1 - qna),
            "nonnativepresentation": totalDuration * nonnative * (1 - qna),
            "nativeq-and-a": totalDuration * (1 - nonnative) * qna,
            "nonnativeq-and-a": totalDuration * nonnative * qna
        }'''
        self.duration_dict_limit = {
            "nativepresentation": totalDuration,
            "nonnativepresentation": totalDuration,
            "nativeq-and-a": totalDuration,
            "nonnativeq-and-a": totalDuration
        }
        self.duration_dict_progress = {"nativepresentation": 0,
                                       "nonnativepresentation": 0,
                                       "nativeq-and-a": 0,
                                       "nonnativeq-and-a": 0
                                       }
        progress_complete = self.dataset_compile_from_event_list(event_list)
        if not progress_complete:
            if included_events:
                if hparams["strict_included"]:
                    logger.info(
                        "Not all the categories were completed with the given criteria. Try freeing up the parameters and split ratios or reduce the number of hours of data.")
                else:
                    logger.info(
                        "Using events from superset apart from included events to complete the required dataset.")
                    included_events_whole = list(
                        self.bizspeech_metadata.keys())
                    event_list = list(
                        (set(included_events_whole) - set(excluded_events)) - set(included_events))
                    event_list = np.sort(event_list)
                    np.random.seed(hparams["dataset_seed"])
                    np.random.shuffle(event_list)
                    progress_complete = self.dataset_compile_from_event_list(
                        event_list)
                    if not progress_complete:
                        logger.info(
                            "Superset was insufficient for the given data split ratios and number of hours. Try reducing them to get the required data split.")
            else:
                logger.info(
                    "Not all the categories were completed with the given criteria. Try freeing up the parameters and split ratios or reduce the number of hours of data.")

        utterance_list = list(self.dataset_dict.keys())
        idxes_list = np.split(utterance_list, [int(trainValTest[0] * len(utterance_list)), int(
            (trainValTest[0] + trainValTest[1]) * len(utterance_list)), len(utterance_list)])
        for event in idxes_list[0]:
            self.split_dicts["train"][event] = self.dataset_dict[event]
        for event in idxes_list[1]:
            self.split_dicts["val"][event] = self.dataset_dict[event]
        for event in idxes_list[2]:
            self.split_dicts["test"][event] = self.dataset_dict[event]

        if hparams["sorting"] == "ascending":
            self.split_dicts["val"] = {k: self.split_dicts["val"][k] for k in sorted(
                self.split_dicts["val"].keys(), key=lambda x: self.split_dicts["val"][x]["duration"])}
            self.split_dicts["train"] = {k: self.split_dicts["train"][k] for k in sorted(
                self.split_dicts["train"].keys(), key=lambda x: self.split_dicts["train"][x]["duration"])}
            self.split_dicts["test"] = {k: self.split_dicts["test"][k] for k in sorted(
                self.split_dicts["test"].keys(), key=lambda x: self.split_dicts["test"][x]["duration"])}
        elif hparams["sorting"] == "descending":
            reverse = True
            self.split_dicts["val"] = {k: self.split_dicts["val"][k] for k in sorted(
                self.split_dicts["val"].keys(), key=lambda x: self.split_dicts["val"][x]["duration"], reverse=reverse)}
            self.split_dicts["train"] = {k: self.split_dicts["train"][k] for k in sorted(
                self.split_dicts["train"].keys(), key=lambda x: self.split_dicts["train"][x]["duration"], reverse=reverse)}
            self.split_dicts["test"] = {k: self.split_dicts["test"][k] for k in sorted(
                self.split_dicts["test"].keys(), key=lambda x: self.split_dicts["test"][x]["duration"], reverse=reverse)}
        else:
            # No sorting required
            pass

    def load_Bizspeech_metadata(self):
        """Load Metadata from json file. Returns dictionary with eventID as key."""
        with open('metadata/BizSpeech_MetaData.json', 'r') as fh:
            result_dict = json.load(fh)
        return result_dict

    def parse_to_json(self, filepath):
        """Write the subset Dataset to a json file.

        'eventID-sentenceID' as key and start_time, end_time, text, speakerID, category as values.

        Args:
        ----
            - filepath : Location for the output JSONs.

        """
        with open(filepath + "/train.json", "w") as fh:
            json.dump(self.split_dicts["train"], fh)
        with open(filepath + "/val.json", "w") as fh:
            json.dump(self.split_dicts["val"], fh)
        with open(filepath + "/test.json", "w") as fh:
            json.dump(self.split_dicts["test"], fh)

    def load_audio(self, audio: str, start: str, end: str, preprocess_audio):
        """Load the audio signal.

        Args:
            audio (str): Filepath of the audio file
            start (str): Start timestamp of the audio utterance
            end (str): End timestamp of the audio utterance
            preprocess_audio ([type]): [description]

        Returns:
            sig: Audio signal cropped and preprocessed.
        """

        '''
        if copy_to_local:
            event_id = str(pathlib.Path(audio).resolve().parent).split("/")[-1]
            dest_dir = pathlib.Path(
                local_dest_dir).resolve().joinpath(event_id)
            dest_dir.mkdir(parents=True, exist_ok=True)
            dest_dir = dest_dir.joinpath("recording." + audio_filetype)
            if not dest_dir.is_file():
                audio = shutil.copy(audio, dest_dir)
        '''
        metadata = torchaudio.info(audio)
        start = int(time_str_to_seconds(start) * metadata.sample_rate)
        end = int(time_str_to_seconds(end) * metadata.sample_rate)
        num_frames = end - start
        audio, fs = torchaudio.load(
            audio, num_frames=num_frames, frame_offset=start)
        audio = audio.transpose(0, 1)
        sig = preprocess_audio(audio, metadata.sample_rate)
        return sig

    def json_to_webdataset_tar(self, dataset: str, dataset_dir: str, number_of_shards: int, use_compression:bool, preprocess_audio):
        """Convert dict input to webdataset tar format.

        Args:
            dataset_dict (dict): Dictionary with the dataset parameters
            hparams (dict): Hyperparameters dictionary loaded from YAML
        """
        def writeTar(queue):
            while True:
                fname, dataset_dict = queue.get(True)
                if fname is None:
                    break
                logger.info("Writing " + str(fname) + ".")
                with wds.TarWriter(fname, compress=use_compression) as sink:
                    for audio_utt in dataset_dict:
                        sig = self.load_audio(audio=dataset_dict[audio_utt]["audio"], start=dataset_dict[audio_utt]
                                                ["start"], end=dataset_dict[audio_utt]["end"], preprocess_audio=preprocess_audio)
                        sample = {
                            "__key__": audio_utt,
                            "wav.pyd": sig,
                            "meta.json": {
                                "speaker_id": dataset_dict[audio_utt]["spkID"],
                                "category": dataset_dict[audio_utt]["category"],
                                "utterance_id": audio_utt,
                                "txt": dataset_dict[audio_utt]["txt"],
                            },
                        }
                        sink.write(sample)

        dataset_dict = self.split_dicts[dataset]
        if use_compression:
            pattern = str(dataset_dir + "/" + "bizspeech_shard" + "-%06d.tar.gz")
        else:
            pattern = str(dataset_dir + "/" + dataset + "_shard" + "-%06d.tar")

        if dataset == "train":
            shard_name_list = [pattern % i for i in range(number_of_shards)]
            per_shard = int(len(dataset_dict)/number_of_shards)
            split_indices = [
                (i+1)*per_shard for i in range(number_of_shards-1)] + [len(dataset_dict)]
            dataset_keys_splits = [list(a) for a in np.split(
                np.array(list(dataset_dict.keys())), split_indices)]
            dataset_values_splits = [[dataset_dict[key] for key in dataset_keys_split]
                                     for dataset_keys_split in dataset_keys_splits]
            dataset_dict_splits = [{dataset_keys_splits[i][j]: dataset_values_splits[i][j] for j in range(
                len(dataset_keys_splits[i]))} for i in range(len(dataset_keys_splits))]
            #p = Pool(cpu_count() - 1)
            #print(tuple(zip(shard_name_list, dataset_dict_splits)))
            #r = list(p.imap(writeTar, tuple(zip(shard_name_list, dataset_dict_splits))))
            # r.join()
            logger.info("The node has " + str(multiprocessing.cpu_count()) + "cores.")
            the_pool = multiprocessing.Pool(multiprocessing.cpu_count()-1, writeTar,(the_queue,))
            #jobs = []
            for i in range(number_of_shards):
                the_queue.put((shard_name_list[i], dataset_dict_splits[i]))
            for i in range(multiprocessing.cpu_count()-1):
                the_queue.put((None, None))
                #jobs.append(Process(target=writeTar, args=(
                #    shard_name_list[i], dataset_dict_splits[i])))
            #for j in jobs:
            #    j.start()
            #for j in jobs:
            #    j.join()
            # prevent adding anything more to the queue and wait for queue to empty
            the_queue.close()
            the_queue.join_thread()

            # prevent adding anything more to the process pool and wait for all processes to finish
            the_pool.close()
            the_pool.join()
            print("Completed tar generation")
            #Parallel(n_jobs=number_of_shards)(delayed(writeTar)(shard_name_list[i], dataset_dict_splits[i]) for i in range(number_of_shards))
        else:
            writeTar(pattern % 0, dataset_dict)

    def dataset_compile_from_event_list(self, event_list):
        """Iterate over given event list and create the dataset dictionary based on data from TimeAligned Transcript.

        Args:
        ----
            - event_list: List of eventIDs that need to e processed.

        """
        for i, event in enumerate(event_list):
            event_metadata = self.bizspeech_metadata[event]
            if event_metadata["partition"] == 5:
                category = "native"
            else:
                category = "nonnative"
            ceo = event_metadata["ceoID"]
            transcript = np.genfromtxt(
                self.dataPath + "/" + event + "/transcript_timealigned.txt", names=True, dtype=None, delimiter="\t", encoding='utf-8')
            for i, row in enumerate(transcript[:-1]):
                if row["SpeakerID"] == ceo or self.non_CEO_utt:
                    category_with_session = category + row["Session"]
                    if self.duration_dict_progress[category_with_session] < self.duration_dict_limit[category_with_session]:
                        if row["Sentence"] != "":
                            try:
                                sentence_ID = row["SentenceID"]
                                utterance_ID = event + "-" + sentence_ID
                                utterance_end_time = transcript[i +
                                                                1]["SentenceTimeGen"]
                                end_time_in_secs = time_str_to_seconds(
                                    utterance_end_time)
                                start_time_in_secs = time_str_to_seconds(
                                    row["SentenceTimeGen"])
                                duration = int(
                                    (end_time_in_secs - start_time_in_secs) * 1000)
                                if int(duration / 1000) < self.utterance_duration_limit:
                                    self.dataset_dict[utterance_ID] = {
                                        "audio": self.dataPath + "/" + event + "/recording." + self.audio_filetype,
                                        "start": row["SentenceTimeGen"],
                                        "end": utterance_end_time,
                                        "duration": duration,
                                        "spkID": row["SpeakerID"],
                                        "txt": row["Sentence"],
                                        "category": category_with_session
                                    }
                                    self.duration_dict_progress[category_with_session] += duration
                            except:
                                continue
            if i % 100 == 0:
                print("Dataset Creation Progress")
                progress_complete = True
                total_duration = 0
                for k in self.duration_dict_progress:
                    progress = self.duration_dict_progress[k] / \
                        self.duration_dict_limit[k] * 100
                    total_duration += self.duration_dict_progress[k]
                    print(k, progress)
                    progress_complete *= progress > 100
                if progress_complete:
                    logger.info(f"Finished compiling dataset from {i} events.")
                    logger.info(f"Total duration in the dataset is {total_duration/3600000} hours.")
                    break

        return progress_complete

def prepare_bizspeech_speechbrain(hparams: dict):
    """Entrypoint for speechbrain function that uses a method directly as a parameter.

    Args:
        hparams ([type]): [description]
    """
    data_folder = hparams["local_dataset_folder"]
    training_file = pathlib.Path(hparams["train_annotation"])
    valid_file = pathlib.Path(hparams["valid_annotation"])
    test_file = pathlib.Path(hparams["test_annotation"])
    if training_file.is_file() and valid_file.is_file() and test_file.is_file():
        logger.info(f"Training JSON/CSV files exists.")
        if hparams["use_wds"]:
            train_shards = [str(f) for f in sorted(
                pathlib.Path(data_folder).glob("train_shard-*.tar*"))]
            val_shards = [str(f) for f in sorted(
                pathlib.Path(data_folder).glob("val_shard-*.tar*"))]
            test_shards = [str(f) for f in sorted(
                pathlib.Path(data_folder).glob("test_shard-*.tar*"))]
            if train_shards and val_shards and test_shards:
                logger.info(
                    f"Using webdataset and tar shards are already available.")
    else:
        dest_dir = pathlib.Path(
            hparams["local_dataset_folder"]).resolve().parent
        dest_dir.mkdir(parents=True, exist_ok=True)

        if hparams["exclude_event_json"]:
            sb.utils.data_utils.download_file(
                hparams["exclude_event_json"], hparams["local_dataset_folder"] + "/exclude_list.json")
            with open(hparams["local_dataset_folder"] + "/exclude_list.json") as fh:
                exclude_list = [item for sublist in list(
                    json.load(fh).values()) for item in sublist]
        if hparams["include_event_json"]:
            sb.utils.data_utils.download_file(
                hparams["include_event_json"], hparams["local_dataset_folder"] + "/include_list.json")
            with open(hparams["local_dataset_folder"] + "/include_list.json") as fh:
                include_list = [item for sublist in list(
                    json.load(fh).values()) for item in sublist]
        datasetObj = create_DataSet(
            hparams, included_events=include_list, excluded_events=exclude_list)
        datasetObj.parse_to_json(hparams["local_dataset_folder"])
        logger.info(
            f'Created JSON dataset files under {hparams["local_dataset_folder"]} directory.')
        if hparams["use_wds"]:
            logger.info("Starting to create tars compatible for webdataset")
            datasetObj.json_to_webdataset_tar(
                "train", hparams["local_dataset_folder"], hparams["number_of_shards"], hparams["use_compression"], hparams["preprocess_audio"])
            logger.info("Train tar shards written out!")
            datasetObj.json_to_webdataset_tar(
                "val", hparams["local_dataset_folder"], hparams["number_of_shards"], hparams["use_compression"], hparams["preprocess_audio"])
            logger.info("Validation tar shards written out!")
            datasetObj.json_to_webdataset_tar(
                "test", hparams["local_dataset_folder"], hparams["number_of_shards"], hparams["use_compression"], hparams["preprocess_audio"])
            logger.info("Test tar shards written out!")

    if hparams["copy_to_local"]:
        local_path = hparams["local_dest_dir"]
        local_train_shards = [str(f) for f in sorted(
                pathlib.Path(local_path).glob("train_shard-*.tar*"))]
        local_val_shards = [str(f) for f in sorted(
                pathlib.Path(local_path).glob("val_shard-*.tar*"))]
        local_test_shards = [str(f) for f in sorted(
                pathlib.Path(local_path).glob("test_shard-*.tar*"))]
        if local_train_shards and local_val_shards and local_test_shards:
            logger.info(f"Local drive already has the required data.")
        else:
            logger.info(f"Copying to local drive.")
            train_shards = [str(f) for f in sorted(
                    pathlib.Path(data_folder).glob("train_shard-*.tar*"))]
            val_shards = [str(f) for f in sorted(
                    pathlib.Path(data_folder).glob("val_shard-*.tar*"))]
            test_shards = [str(f) for f in sorted(
                    pathlib.Path(data_folder).glob("test_shard-*.tar*"))]
            dest_dir = pathlib.Path(local_path).resolve()
            dest_dir.mkdir(parents=True, exist_ok=True)
            for shard_file in tqdm(train_shards + val_shards + test_shards):
                shutil.copy(shard_file, dest_dir)
    logger.info(f"Dataset is ready to use.")

if __name__ == '__main__':
    # Load hyper parameters file with command-line overrides
    # Reading command line arguments
    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])

    with open(hparams_file) as fin:
        hparams = load_hyperpyyaml(fin, overrides)

    # Create experiment directory
    sb.create_experiment_directory(
        experiment_directory=hparams["output_folder"],
        hyperparams_to_save=hparams_file,
        overrides=overrides,
    )

    # Data preparation, to be run on only one process.
    sb.utils.distributed.run_on_main(
        prepare_bizspeech_speechbrain,
        kwargs={
            "hparams": hparams,
        },
    )


import pytest
from django.db import transaction

from common.models.transactions import Transaction
from common.tests import factories
from common.tests.util import serialize_xml
from common.xml.util import remove_transactions
from common.xml.util import renumber_records
from common.xml.util import renumber_transactions

pytestmark = pytest.mark.django_db

def test_record_renumbering_produces_valid_import_data(import_xml, export_xml):
    """Tests that when the renumbering function is used, the XML file it
    produces is valid and when imported produces records in the database that
    are correctly renumbered."""
    code_type = factories.AdditionalCodeTypeFactory.create()
    workbasket = factories.QueuedWorkBasketFactory.create()
    first = factories.AdditionalCodeFactory.create(
        type=code_type,
        transaction__workbasket=workbasket,
    )
    second = factories.AdditionalCodeFactory.create(
        type=code_type,
        transaction__workbasket=workbasket,
    )

    for instance in (first, second):
        assert not type(instance).objects.filter(sid=instance.sid + 2).exists()

    xml = export_xml(workbasket=workbasket)
    renumber_records(
        xml,
        second.sid + 1,
        "oub:additional.code",
        "oub:additional.code.sid",
    )
    import_xml(serialize_xml(xml))

    for instance in (first, second):
        new_instance = type(instance).objects.get(sid=instance.sid + 2)
        assert new_instance.type == instance.type
        assert new_instance.code == instance.code
        assert new_instance.version_group != instance.version_group

def test_record_renumbering_only_renumbers_correct_records(import_xml, export_xml):
    """
    Tests that when the renumbering function is used, only records that are
    identified *by* the ID number is renumbered, and adding attributes that.

    *reference* the record do not trigger renumbering.
    """
    measure = factories.MeasureFactory.create()
    with transaction.atomic():
        model = factories.FootnoteAssociationMeasureFactory.create(
            footnoted_measure=measure,
        )
        xml = export_xml(workbasket=model.transaction.workbasket)
        transaction.set_rollback(True)

    assert not measure.footnotes.exists()

    renumber_records(xml, measure.sid + 1, "oub:measure", "oub:measure.sid")
    import_xml(serialize_xml(xml))
    assert measure.footnotes.exists()

def test_transaction_renumbering_produces_valid_import_data(import_xml, export_xml):
    """Tests that when the transaction renumbering function is used, the XML
    file it produces is valid and when imported produces transactions in the
    database that are correctly renumbered."""
    with transaction.atomic():
        model = factories.AdditionalCodeTypeFactory.create()
        xml = export_xml(workbasket=model.transaction.workbasket)
        original_order = model.transaction.order
        transaction.set_rollback(True)

    assert not Transaction.objects.filter(order=original_order).exists()
    assert not Transaction.objects.filter(order=original_order + 1).exists()
    renumber_transactions(xml, original_order + 1)
    import_xml(serialize_xml(xml))

    txn = Transaction.objects.get(order=original_order + 1)
    assert txn.tracked_models.get().sid == model.sid

def test_transaction_removing_produces_valid_import_data(import_xml, export_xml):
    """Tests that when the transaction filtering function is used, the XML file
    it produces is valid and when imported does not contain the removed
    transactions."""
    code_type = factories.AdditionalCodeTypeFactory.create()
    with transaction.atomic():
        workbasket = factories.QueuedWorkBasketFactory.create()
        first = factories.AdditionalCodeFactory.create(
            type=code_type,
            transaction__workbasket=workbasket,
        )
        second = factories.AdditionalCodeFactory.create(
            type=code_type,
            transaction__workbasket=workbasket,
        )
        xml = export_xml(workbasket=workbasket)
        transaction.set_rollback(True)

    assert not Transaction.objects.filter(order=first.transaction.order).exists()
    assert not Transaction.objects.filter(order=second.transaction.order).exists()

    remove_transactions(xml, "oub:additional.code.sid", [str(first.sid)])
    import_xml(serialize_xml(xml))

    assert not Transaction.objects.filter(order=first.transaction.order).exists()
    assert Transaction.objects.filter(order=second.transaction.order).exists()
    assert not type(first).objects.filter(sid=first.sid).exists()
    assert type(second).objects.filter(sid=second.sid).exists()


API_KEY = "5694590247:AAEDwug01HoBKJOrkFrXfi1_Klo7DAXeNPs"

#import libraries
from skimage.io import imread
import telebot
from telebot import types
import pickle

import string
import numpy as np
import sqlite3
import urllib

from util import JSONParser
from util import process_image
from util import feet_to_shoe

#global variable
user_data = {}

# database connect
conn = sqlite3.connect('./data/data_sepatu.db', check_same_thread=False)
conn.row_factory = lambda cursor, row: row[0]
cur = conn.cursor()

#load model
with open('model/model_chatbot.pkl', 'rb') as handle:
    model = pickle.load(handle)

#API Telegram Bot
bot = telebot.TeleBot(API_KEY)

# load data
path = "data/intents.json"
jp = JSONParser()
jp.parse(path)
df = jp.get_dataframe()

def preprocess(chat):
    # lowercase transform
    chat = chat.lower()
    tandabaca = tuple(string.punctuation)
    chat = "".join(ch for ch in chat if ch not in tandabaca)
    return chat

"""
generate button with items from the database
"""

def makeButtons(user):

    markup = types.InlineKeyboardMarkup()
    if (user.get('buy_state') == 'gender'):
        selected = cur.execute("SELECT DISTINCT gender FROM shoes").fetchall()
        for button in selected:
            btt = types.InlineKeyboardButton(text=button, callback_data=button)
            markup.row(btt)

    if (user.get('buy_state') == 'brand'):
        selected = cur.execute(
            f"SELECT DISTINCT brand FROM shoes WHERE gender ='{user.get('gender')}'"
        ).fetchall()
        for button in selected:
            btt = types.InlineKeyboardButton(text=button, callback_data=button)
            markup.row(btt)

    if (user.get('buy_state') == 'shoe_name'):
        selected = cur.execute(
            f"SELECT name FROM shoes WHERE gender = '{user.get('gender')}' and brand = '{user.get('brand')}'"
        ).fetchall()

        for button in selected:
            btt = types.InlineKeyboardButton(text=button, callback_data=button)
            markup.row(btt)

    if (user.get('buy_state') == 'color'):
        selected = cur.execute(
            f"SELECT DISTINCT color FROM shoes WHERE gender = '{user.get('gender')}' and brand = '{user.get('brand')}'and name = '{user.get('name')}'"
        ).fetchall()

        for button in selected:
            btt = types.InlineKeyboardButton(text=button, callback_data=button)
            markup.row(btt)

    if (user.get('buy_state') == 'tersedia'):
        selected = ['Yes', 'Cancel', 'ambil ulang']
        for button in selected:
            btt = types.InlineKeyboardButton(text=button, callback_data=button)
            markup.row(btt)
    if (user.get('buy_state') == 'tidak_tersedia'):
        selected = ['Cancel Pembelian', 'Foto Ulang']
        for button in selected:
            btt = types.InlineKeyboardButton(text=button, callback_data=button)
            markup.row(btt)

    return markup

# function for updating
def update_user(user, key, value):
    user[key] = value

def resetUser(user_id):
    global user_data
    user_data[str(user_id)] = {}

# function to update bot state
def update_state(chat, pipe, user):
    global bot_state
    chat = preprocess(chat)
    res = pipe.predict_proba([chat])
    max_prob = max(res[0])
    if max_prob < .2:
        user['bot_state'] = 'error'
        bot_state = 'error'
    else:
        max_id = np.argmax(res[0])
        pred_tag = pipe.classes_[max_id]
        user['bot_state'] = pred_tag
        bot_state = pred_tag

def bot_response(jp):
    global bot_state
    if bot_state == "error":
        return "Maaf kak bisa Diperjelas lagi ngga kak masalahnya ? \n\
        saya kurang paham :(", None
    else:
        return jp.get_response(bot_state), bot_state

"""
Handling all message that send to machine
and handle the input process and become the response of the bot
"""

@bot.message_handler(regexp='[a-z]+')
def greet(message):
    global user_data

    #set state user
    user_data[str(message.from_user.id)] = user_data.get(
        str(message.from_user.id), {})
    user = user_data[str(
        message.from_user.id)]  #get unique user id and its data

    if (user.get('bot_state') is None and message.text == '/start'):
        update_user(user, 'bot_state', 'start')
        bot.reply_to(
            message,
            "selamat datang di pelayanan pembelian Get Shoe, ada yang bisa dibantu ?"
        )
    elif (message.text == '/reset'):
        resetUser(message.from_user.id)
        bot.reply_to(
            message,
            "bot reset"
        )

    else:
        update_state(message.text, model, user)
        bot.reply_to(message, bot_response(jp))
        if (user.get("bot_state") == 'cancel'):
            resetUser(message.from_user.id)
        if (user.get("bot_state") == 'ambil'
                and user.get("buy_state") == 'tersedia'):
            resetUser(message.from_user.id)
        if (user.get('bot_state') == 'beli' and user.get('buy_state') is None):
            update_user(user, 'buy_state', 'gender')
            bot.send_message(message.chat.id,
                             "Pilih Gender",
                             reply_markup=makeButtons(user))

# proses beli
"""
Handling shoping session
1. choose gender
2. Choose brand
3. choose shoe name
4. choose color
5. size
"""

@bot.callback_query_handler(func=lambda call: True)
def callback_brand(call):
    global user_data
    user = user_data[str(call.from_user.id)]  #get unique user id and its data

    if (user.get('bot_state') == 'beli'):
        if (user.get('buy_state') == "gender"):
            update_user(user, 'gender', call.data)
            update_user(user, 'buy_state', 'brand')
            bot.send_message(call.message.chat.id,
                             f"Sepatu {user.get('gender')},Pilih Merek Sepatu",
                             reply_markup=makeButtons(user))

        if (user.get('buy_state') == "brand"
                and user.get('gender') is not call.data):
            update_user(user, 'brand', call.data)
            update_user(user, 'buy_state', 'shoe_name')
            bot.send_message(
                call.message.chat.id,
                f"Merek Sepatu {user.get('brand')} ,Pilih Nama Sepatu",
                reply_markup=makeButtons(user))

        if (user.get('buy_state') == "shoe_name"
                and user.get('brand') is not call.data):
            update_user(user, 'name', call.data)
            update_user(user, 'buy_state', 'color')
            bot.send_message(
                call.message.chat.id,
                f"Nama Sepatu {user.get('name')}, Pilih Warna Sepatu",
                reply_markup=makeButtons(user))

        if (user.get('buy_state') == "color"
                and user.get('name') is not call.data):
            update_user(user, 'color', call.data)
            update_user(user, 'buy_state', 'size')
            bot.send_message(
                call.message.chat.id,
                "Kita Bantu Pilih ukuran Sepatunya ya kak ?\n coba kakak kirimin foto kaki kakak diatas kertas HVS A4, seperti contoh berikut"
            )
            bot.send_photo(call.message.chat.id,
                           photo=open('./example.jpg', 'rb'))

        if (user.get('buy_state') == "color"
                and user.get('name') is not call.data):
            update_user(user, 'color', call.data)
            update_user(user, 'buy_state', 'size')
            print(str(user))
            bot.send_message(
                call.message.chat.id,
                "Kita Bantu Pilih ukuran Sepatunya ya kak ?\n coba kakak kirimin foto kaki kakak diatas kertas HVS A4, seperti contoh berikut"
            )
            bot.send_photo(call.message.chat.id,
                           photo=open('./example.jpg', 'rb'))

@bot.message_handler(content_types=['photo'])
def photo(message):
    global user_data
    # get file path
    user = user_data[str(
        message.from_user.id)]  #get unique user id and its data
    shoe_id = cur.execute(
        f"SELECT product_id FROM shoes WHERE gender = '{user.get('gender')}' and brand = '{user.get('brand')}' and color = '{user.get('color')}'"
    ).fetchone()

    if (user['bot_state'] == "beli" and (user.get('buy_state') == 'size' or user.get('buy_state') == 'tidak_tersedia' or user.get('buy_state') == 'tersedia')):
        bot.reply_to(message, "Mohon Tunggu Sebentar ya kak...")
        # open Image
        fileID = message.photo[-1].file_id
        file_info = bot.get_file(fileID)
        path_file = 'https://api.telegram.org/file/bot{0}/{1}'.format(
            API_KEY, file_info.file_path)

        # add image
        oimg = imread(path_file)

        # Preprocessing
        result = process_image(oimg)
        result = "%.2f" % result

        # output
        shoe_size = feet_to_shoe(float(result))
        update_user(user, 'size', shoe_size)
        user['size'] = shoe_size
        print("feet size (cm): ", result)

        # get shoe availabality
        get_shoe = cur.execute(
            f"SELECT size FROM product_stock WHERE product_id = {shoe_id}"
        ).fetchone()

        # Get shoe image
        shoe_image = cur.execute(
            f"SELECT image FROM shoes WHERE product_id = {shoe_id}").fetchone(
            )
        shoe_price = cur.execute(
            f"SELECT price FROM shoes WHERE product_id = {shoe_id}").fetchone(
            )
        f = open('product.jpg', 'wb')
        f.write(urllib.request.urlopen(shoe_image).read())
        f.close()
        # if shoe not found
        if (get_shoe is None or shoe_size is None):
            update_user(user, 'buy_state', 'tidak_tersedia')
            bot.reply_to(
                message, "ukuran kaki kamu " + str(result) +
                f" cm\n Maaf kak, sepatu dengan ukuran kaki kakak tidak ditemukan"
            )
        #if shoe found
        else:
            update_user(user, 'buy_state', 'tersedia')

            bot.reply_to(message, "ukuran kaki kamu " + str(result) + " cm")

            bot.send_photo(
                message.chat.id,
                photo=open('./product.jpg', 'rb'),
                caption=
                f"kakak memesan sepatu untuk {user['gender']} merek {user['brand']}  nama sepatu {user['name']} warna {user['color']} ukuran sepatu {shoe_size} tersedia dengan harga : {shoe_price}"
            )

        bot.send_message(
            message.chat.id,
            "Jika Menurut kakak Ukuran Kaki tidak cocok, bisa langsung kirim lagi foto dengan cara yang sama, atau mau ambil yang ini? atau kakak bisa cancel pemesanan."
        )

    else:
        bot.reply_to(
            message,
            "Maaf kak, kakak bisa pesen dulu sepatunya, coba kirim chat 'Mau Beli Sepatu !'"
        )

bot.polling()


from core.common.names import *
import core.common.resources as cr
import core.common.utils as utils
from core.event_holder import EventHolder
from core.rope import *
import webbrowser
import asyncio
#import numpy
global square_wave
square_wave = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 6, 8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 8, 6, 4, 2, 0]
global yoster_font
cr.screen = pg.display.set_mode([1280, 720])
pg.display.set_caption("Tanoko's fishing")
icon =  pygame.image.load("icon.png")
icon.set_colorkey([0, 0, 0])
pg.display.set_icon(icon)
"""
event holder takes care of getting the events and
the game's fps
"""
async def main():
    cr.event_holder = EventHolder()
    cr.event_holder.determined_fps = 60
    global mouse_collider
    mouse_collider = Collider(0, 61.3)
    mouse_collider.size = 50
    cover = utils.scale_image(pygame.image.load("cover.png"))
    fish_sprites = [[pygame.image.load("assets/Spritesheets/fish1/fish-000"+str(i)+".png").convert() for i in range(1, 4)], [pygame.image.load("assets/Spritesheets/fish3/fish-000"+str(i)+".png").convert() for i in range(1, 4)], [pygame.image.load("assets/Spritesheets/fish4/fish-000"+str(i)+".png").convert() for i in range(1, 4)], [pygame.image.load("assets/Spritesheets/fish2/fish2-00"+str(i)+".png").convert() for i in range(3)]]
    fish_sprites[0] = [utils.scale_image(f, 3) for f in fish_sprites[0]]
    fish_sprites[1] = [utils.scale_image(f, 3) for f in fish_sprites[1]]
    fish_sprites[2] = [utils.scale_image(f, 3) for f in fish_sprites[2]]
    fish_sprites[3] = [utils.scale_image(f, 3) for f in fish_sprites[3]]
    [utils.swap_color(sprite, [102, 57, 49], [0, 0, 0]) for sprite in fish_sprites[0]]
    [sprite.set_colorkey([0, 0, 0]) for sprite in fish_sprites[0]]
    [sprite.set_colorkey([0, 0, 0]) for sprite in fish_sprites[1]]
    [sprite.set_colorkey([0, 0, 0]) for sprite in fish_sprites[2]]
    [sprite.set_colorkey([0, 0, 0]) for sprite in fish_sprites[3]]
    pygame.init()
    yoster_font = pygame.font.Font('assets/Fonts/yoster.ttf', 64)
    button_font = pygame.font.Font('assets/Fonts/yoster.ttf', 35)
    time_font = pygame.font.Font('assets/Fonts/yoster.ttf', 16)
    time_text = time_font.render("Time", False, [1, 1 ,1], [0, 0, 0])
    time_text.set_colorkey([0, 0, 0])
    win_text = yoster_font.render("You Won!", True, [255, 255 ,255], [0, 0, 0])
    win_text.set_colorkey([0, 0, 0])
    win_game_text = yoster_font.render("You Beat the Game!", True, [255, 255 ,255], [0, 0, 0])
    win_game_text.set_colorkey([0, 0, 0])
    lose_text = yoster_font.render("You Lost!", True, [255, 255 ,255], [0, 0, 0])
    lose_text.set_colorkey([0, 0, 0])
    lose_text2 = button_font.render("Press Backspace to return to Menu", True, [255, 255 ,255], [0, 0, 0])
    lose_text2.set_colorkey([0, 0, 0])
    lose_text3 = button_font.render("and click Play to start next level", True, [255, 255 ,255], [0, 0, 0])
    lose_text3.set_colorkey([0, 0, 0])
    length_text = pygame.font.Font('assets/Fonts/yoster.ttf', 24)
    length_text = time_font.render("Length", False, [1, 1 ,1], [0, 0, 0])
    length_text.set_colorkey([0, 0, 0])
    title_text = yoster_font.render("Little Fish", True, [255, 255 ,255], [0, 0, 0])
    title_text.set_colorkey([0, 0, 0])
    start_text = button_font.render("Play", True, [255, 255 ,255], [0, 0, 0])
    start_text.set_colorkey([0, 0, 0])
    cred_text = button_font.render("Credits", True, [255, 255 ,255], [0, 0, 0])
    cred_text.set_colorkey([0, 0, 0])
    quit_text = button_font.render("Quit", True, [255, 255 ,255], [0, 0, 0])
    quit_text.set_colorkey([0, 0, 0])
    back_text = button_font.render("Back", True, [255, 255 ,255], [0, 0, 0])
    back_text.set_colorkey([0, 0, 0])

    button_font.align = pygame.FONT_CENTER
    code_credit = button_font.render("Code and Game Design by Two119", True, [255, 255 ,255], [0, 0, 0])
    art_credit = button_font.render("Art by MasterGemu", True, [255, 255 ,255], [0, 0, 0])
    music_credit = button_font.render("Music and SFX by travsul", True, [255, 255 ,255], [0, 0, 0])
    button_font.align = pygame.FONT_LEFT
    credit_screen = cover.copy()
    code_credit.set_colorkey([0, 0, 0])
    art_credit.set_colorkey([0, 0, 0])
    music_credit.set_colorkey([0, 0, 0])
    credit_screen.blit(art_credit, [utils.center_pos(art_credit)[0], utils.center_pos(art_credit)[1]+32])
    credit_screen.blit(code_credit, [utils.center_pos(code_credit)[0], utils.center_pos(code_credit)[1]-32])
    credit_screen.blit(music_credit, [utils.center_pos(music_credit)[0], utils.center_pos(music_credit)[1]+96])
    def sawtooth_sample(amplitude, freq, samplerate, i):
        value = atan(tan(2.0 * pi * float(freq) * (float(i) / float(samplerate))))
        return amplitude * value

    def plot_sawtooth(num_samples, frequency):
        # Generate the samples
        samples = []
        for i in range(num_samples):
            samples.append(sawtooth_sample(1.0, frequency, 44100, i))
        return samples
    def triangle(length, amplitude):
        section = length // 4
        for direction in (1, -1):
            for i in range(section):
                yield i * (amplitude / section) * direction
            for i in range(section):
                yield (amplitude - (i * (amplitude / section))) * direction
    sawtooth = plot_sawtooth(800, 300)
    triangle_wave = list(triangle(100, 50))
    for i in range(len(triangle_wave)):
        triangle_wave.append(triangle_wave[i])
    for i in range(len(triangle_wave)):
        triangle_wave.append(triangle_wave[i])
    class Fish:
        def __init__(self, x, elevation, type_):
            self.pos = [x, elevation]
            self.orig_pos = [x, elevation]
            self.speed = 4
            self.crossed = False
            self.pattern = random.randint(0, 2)
            self.type = random.randint(0, len(fish_sprites)-2)
            if random.randint(0, 12) == 12:
                self.type= len(fish_sprites)-1
                self.speed = 8
            self.stage = -1
            self.adders = [4, 1, 0]
            self.adding = False
            self.delay = 0
            self.sprite = [s.copy() for s in fish_sprites[self.type]]
            self.frame = 0
            self.masks = [pygame.mask.from_surface(surf) for surf in self.sprite]
            self.caught = False
            self.food_req = random.randint(8, 10)
            self.alpha_surf = pygame.Surface([self.sprite[0].get_width(), self.sprite[0].get_height()])
            self.alpha_surf.fill([0, 0, 0])
            self.alpha = 0
            self.score = 1
            if self.food_req == 8:
                self.alpha = 0
                self.score = 1
            if self.food_req == 9:
                self.alpha = 60
                self.score = 0.6
            if self.food_req == 10:
                self.alpha = 120
                self.score = 0.3
            if self.type == len(fish_sprites)-1:
                self.score+=2
            self.alpha_surf.set_alpha(self.alpha)
            #[self.sprite[n].blit(self.alpha_surf, [0, 0]) for n in range(len(self.sprite))]
        def update(self):
            global square_wave
            self.stage+=self.adders[self.pattern]
            self.delay+=1
            if self.delay%6==0:
                self.frame+=1
                if (self.frame>len(self.sprite)-1):
                    self.frame = 0
            if (self.pattern == 1):
                if self.stage > len(triangle_wave)-1:
                    self.stage = 0
            if not self.caught:
                self.pos[0]-=self.speed
                if (self.pattern==0):
                    self.pos[1]=self.orig_pos[1]+(sin(radians(self.stage))*60)
                    #self.sprite = pygame.transform.rotate(fish_sprites[self.type], degrees(asin(sin(radians(self.stage)))))
                if (self.pattern==1):
                    self.pos[1]=self.orig_pos[1]+(triangle_wave[self.stage])
                if (self.pattern==2):
                    if (self.delay%3==0):
                        self.stage+=1
                        if (self.stage>len(square_wave)-1):
                            self.stage=0
                    self.pos[1]=self.orig_pos[1]+(square_wave[self.stage])*10

            if player.fishing and player.recreated:

                if player.rope is not None:
                    if player.rope.lowest_point is not None:
                        bait_pos = [player.rope.lowest_point[0]-player.bait_sprite.get_width()/2, player.rope.lowest_point[1]]
                        if (self.masks[self.frame].overlap(player.bait_mask, [bait_pos[0]-self.pos[0], bait_pos[1]-self.pos[1]])!=None and ((not player.fished))) or (self.caught):
                            self.pos = bait_pos
                            if not self.caught:
                                if self.type!=len(fish_sprites)-1:
                                    player.fish+=1
                                else:
                                    player.fish+=2
                                player.fished = True
                            self.caught = True
            if self.masks[self.frame].overlap(player.mask, [player.player_pos[0]-self.pos[0], player.player_pos[1]-self.pos[1]])!=None:
                self.crossed = True
            cr.screen.blit(self.sprite[self.frame], self.pos)
            if (self.pos[0]<0):
                self.crossed = True
    class FishManager:
        def __init__(self):
            self.fishes = [Fish(random.randint(1280, 1280*2), random.randint(475, 575), random.randint(0, 1)) for i in range(8)]
        def update(self):
            num_fish = -1
            for fish in self.fishes:
                num_fish+=1
                if not fish.crossed:
                    fish.update()
                else:
                    self.fishes[num_fish].__init__(1380, random.randint(475, 575), random.randint(0, 1))

    fish_manager = FishManager()
    def outline_mask(img, loc, color=[255,255,255]):
        mask = pygame.mask.from_surface(img)
        mask_outline = mask.outline()
        n = 0
        for point in mask_outline:
            mask_outline[n] = (point[0] + loc[0], point[1] + loc[1])
            n += 1
        pygame.draw.polygon(cr.screen,color,mask_outline,3)
    def get_outline_mask(img, loc):
        mask = pygame.mask.from_surface(img)
        mask_outline = mask.outline()
        n = 0
        for point in mask_outline:
            mask_outline[n] = (point[0] + loc[0], point[1] + loc[1])
            n += 1
        return mask_outline
    class Slider:
        def __init__(self, pos, font, div):
            self.pos = pos
            self.font = pygame.font.SysFont(font, 18)
            self.div = div
            self.rect_surf = pygame.Surface((32, 32))
            self.mask = pygame.mask.from_surface(self.rect_surf)
            self.outlines = self.mask.outline()
            self.bg_surf = pygame.Surface((32, 32))
            self.bg_surf.set_colorkey((0, 0, 0))
            self.x_offset = 100
            self.bar_surf = pygame.Surface((120, 20))
            self.bar_surf.fill([99, 155, 255])
        def update(self):
            self.bg_surf.fill((0, 0, 0))
            self.overall_rect = pygame.Rect(self.pos[0], self.pos[1]-10, 120, 40)
            self.bar_rect = pygame.Rect(self.pos[0], self.pos[1], 120, 20)
            self.slider_rect = pygame.Rect(self.pos[0]+self.x_offset, self.pos[1]-10, 20, 40)
            if self.overall_rect.collidepoint(pygame.mouse.get_pos()) and pygame.mouse.get_pressed()[0]:
                self.x_offset = pygame.mouse.get_pos()[0]-self.pos[0]
            if ((pg.K_LSHIFT in pressed_keys) or (pg.K_RSHIFT in pressed_keys)):
                self.x_offset-=10
            if ((pg.K_LCTRL in pressed_keys) or (pg.K_RCTRL in pressed_keys)):
                self.x_offset+=10
            if self.x_offset > 100:
                self.x_offset = 100
            if self.x_offset < 80:
                self.x_offset = 80
            #pygame.draw.rect(cr.screen, (0, 0, 255), self.overall_rect)
            pygame.draw.rect(cr.screen, [99, 155, 255], self.bar_rect)
            outline_mask(self.bar_surf, [self.bar_rect.x, self.bar_rect.y])
            pygame.draw.rect(cr.screen, (255, 255, 255), self.slider_rect)
            self.value = round(self.x_offset/100, 1)
            if not self.div:
                self.value *= 10
            value = self.font.render(str(self.value), False, (1, 1, 1), (0, 0, 0))
            value.set_colorkey((0, 0, 0))
            pygame.draw.lines(self.bg_surf, (1, 1, 1), True, self.outlines, 3)
            self.bg_surf.blit(value, [self.bg_surf.get_width()/2-value.get_width()/2, self.bg_surf.get_height()/2-value.get_height()/2])
            cr.screen.blit(self.bg_surf, [self.pos[0]+self.overall_rect.w+25, self.pos[1]-(self.bar_rect.h/4)])
    class Button:
        def __init__(self, position, textures, function):
            self.textures = textures
            self.onlick = function[0]
            self.args = function[1]
            self.pos = position
            self.screen = cr.screen
            self.current = 0
            self.rect = self.textures[self.current].get_rect(topleft=self.pos)
            self.click_delay = 0
            self.max_delay = 500
            self.delaying = False
            self.clicksound = pygame.mixer.Sound("assets/Audio/click.ogg")
        def update(self):
            self.current = 0
            if self.delaying:
                self.click_delay += 1
            if self.click_delay >= self.max_delay:
                self.delaying = False
            if self.rect.collidepoint(pygame.mouse.get_pos()):
                if pygame.mouse.get_pressed()[0]:
                    if not self.delaying:
                        self.onlick(self.args)
                        self.clicksound.play()
                self.current = 1
            self.screen.blit(self.textures[self.current], self.pos)
            self.rect = self.textures[self.current].get_rect(topleft=self.pos)
    def get_pixel(surf : pygame.Surface, color):
        for j in range(surf.get_height()):
            for i in range(surf.get_width()):
                if surf.get_at([i, j]).r == color[0] and surf.get_at([i, j]).g == color[1] and surf.get_at([i, j]).b == color[2]:
                    return [i, j]
    global game_state
    game_state = 0
    class Player:
        def __init__(self) -> None:
            self.throw_anim = []
            self.idle_anim = []
            self.pull_anim = []
            self.length_bar = pygame.Surface([225*2.2, 30])
            self.length_bar.set_colorkey([0, 0, 0])
            self.length_bar.fill([255, 255, 255])
            self.fished = False
            self.bait_slider = Slider([(1280-180)/2, 70], "Arial", True)
            self.restart = False
            self.bait_sprite = pygame.image.load("assets/Spritesheets/bait.png").convert()
            self.bait_sprite.set_colorkey([255, 255, 255])
            self.orig_bait_sprite = pygame.image.load("assets/Spritesheets/bait.png").convert()
            self.orig_bait_sprite.set_colorkey([255, 255, 255])
            for i in range(6):
                self.throw_anim.append(pygame.transform.flip(utils.scale_image(pg.image.load("assets/Spritesheets//throw/throw"+str(i)+".png").convert()), False, False))
                self.throw_anim[i].set_colorkey([255, 255, 255])
            for i in range(5):
                self.idle_anim.append(pygame.transform.flip(utils.scale_image(pg.image.load("assets/Spritesheets//idle/idle00"+str(i)+".png").convert()), False, False))
                self.idle_anim[i].set_colorkey([0, 0, 0])
            self.idle_anim[0].set_colorkey([255, 255, 255])
            for i in range(8):
                self.pull_anim.append(pygame.transform.flip(utils.scale_image(pg.image.load("assets/Spritesheets//pull/pull00"+str(i)+".png").convert()), False, False))
                self.pull_anim[i].set_colorkey([255, 255, 255])
            self.boat_sprite = pygame.transform.flip(utils.scale_image(pg.image.load("assets/Spritesheets//boat.png").convert()), False, False)
            self.boat_sprite.set_colorkey([255, 255, 255])
            self.pos = [0, 360 - self.boat_sprite.get_height()+8]
            self.line_length = 70
            self.bait_size = 1
            self.scores = []
            self.fish = 0
            self.rope = None
            self.fishing_line_angle = 90
            self.fishing = False
            self.rope_screen = pygame.Surface([1280, 720])
            self.rope_screen.set_colorkey([0, 0, 0])
            self.rope_mask = None
            self.recreated = False
            self.recall_fishing_line = False
            self.cover_rect = pygame.Rect(0, 720, 1280, 720)
            self.set_angle = False
            self.down_rect = pygame.Rect(0, 360, 720, 360)
            self.y_points = []
            self.old_len = 0
            self.moving = True
            self.frame = -1
            self.delay = 0
            self.state = 0
            self.length_bar_pos = [(1280-495)/2, 50]
            self.length_bar_outline = get_outline_mask(self.length_bar, self.length_bar_pos)
            self.length_bar.fill([0, 0, 0])
            self.casting_sound = pygame.mixer.Sound('assets/Audio/casting.ogg')
            self.winding_sound = pygame.mixer.Sound('assets/Audio/winding_rod.ogg')
            self.player_pos = [self.pos[0]+self.boat_sprite.get_width()-self.idle_anim[0].get_width()+20, self.pos[1]-10]
            self.rope_throw_positions = [[get_pixel(sprite, [103, 57, 49])[0]+self.player_pos[0], get_pixel(sprite, [103, 57, 49])[1]+self.player_pos[1]] for sprite in self.throw_anim]
            self.rope_pull_positions = [[get_pixel(sprite, [103, 57, 49])[0]+self.player_pos[0], get_pixel(sprite, [103, 57, 49])[1]+self.player_pos[1]] for sprite in self.pull_anim]
            self.idle_pos = [get_pixel(self.idle_anim[0], [103, 57, 49])[0]+self.player_pos[0], get_pixel(self.idle_anim[0], [103, 57, 49])[1]+self.player_pos[1]]
            self.delays = [12, 8, 8]
            self.bait_mask = pygame.mask.from_surface(self.bait_sprite)
            self.angle_pointer = utils.scale_image(pygame.transform.flip(pygame.image.load("assets/Spritesheets/arrow.png").convert(), True, False))
            self.angle_pointer.set_colorkey([255, 255, 255])
            self.time_lims = [15, 20, 25, 30]
            self.num_fish = [3, 4, 5, 7]
            self.level = 0
            self.time_bar = pygame.Surface([1240, 20])
            self.time_bar_pos = [20, 10]
            self.time_bar_outline = get_outline_mask(self.time_bar, self.time_bar_pos)
            self.time_bar.set_colorkey([0, 0, 0])
            #self.time_bar.blit(time_text, [(1240-time_text.get_width())/2, (20-time_text.get_height())/2])
            self.length_bar.blit(length_text, [(self.length_bar.get_width()-length_text.get_width())/2, (self.length_bar.get_height()-length_text.get_height())/2])
            self.time = time.time()
            self.orig_time = time.time()
        def create_fishing_line(self):
            self.rope = Rope(self.rope_throw_positions[0][0], self.rope_throw_positions[0][0], self.line_length, self.line_length//11)
        def prepare_fishing_line(self):
            self.rope.gravity.x = resultant_gravitational_force*cos(radians(self.fishing_line_angle))
            self.rope.gravity.y = resultant_gravitational_force*sin(radians(self.fishing_line_angle))
        def draw_boat(self):
            cr.screen.blit(self.boat_sprite, self.pos)
        def update(self):
            self.delay +=1
            self.length_bar.fill([0, 0, 0])
            if mouse_pressed_keys[0] and self.rope == None and not self.fishing:
                self.create_fishing_line()
                self.fishing=True
                self.prepare_fishing_line()
            if self.delay%self.delays[self.state]==0:
                self.frame+=1
            if self.state == 0:
                if self.frame>(len(self.idle_anim)-1):
                    self.frame = 0
                else:
                    if self.rope!=None:
                        self.rope.points[0].pos.x = self.idle_pos[0]
                        self.rope.points[0].pos.y = self.idle_pos[1]
                if self.rope != None:
                    if self.delay%12==0:
                        self.frame= 0
                        #self.rope.points[0].pos.x = self.rope_throw_positions[self.frame][0]
                        #self.rope.points[0].pos.y = self.rope_throw_positions[self.frame][1]
            elif self.state == 1:
                if self.frame>(len(self.throw_anim)-1):
                    self.frame = 0
                    self.state = 0
                else:
                    if self.rope!=None:
                        self.rope.points[0].pos.x = self.rope_throw_positions[self.frame][0]
                        self.rope.points[0].pos.y = self.rope_throw_positions[self.frame][1]
            elif self.state == 2:
                if self.frame>(len(self.pull_anim)-1):
                    self.frame = 0
                    self.state = 0
                else:
                    if self.rope!=None:
                        self.rope.points[0].pos.x = self.rope_pull_positions[self.frame][0]
                        self.rope.points[0].pos.y = self.rope_pull_positions[self.frame][1]

            if self.state==0:
                cr.screen.blit(self.idle_anim[self.frame], self.player_pos)
                self.mask = pygame.mask.from_surface(self.idle_anim[self.frame])
            elif self.state==1:
                cr.screen.blit(self.throw_anim[self.frame], self.player_pos)
                self.mask = pygame.mask.from_surface(self.throw_anim[self.frame])
            elif self.state==2:
                cr.screen.blit(self.pull_anim[self.frame], self.player_pos)
                self.mask = pygame.mask.from_surface(self.pull_anim[self.frame])
            self.draw_boat()
            pygame.draw.rect(self.length_bar, [255, 255, 255], pygame.Rect(0, 0, self.line_length*2.2, 30))
            self.length_bar.blit(length_text, [(self.length_bar.get_width()-length_text.get_width())/2, (self.length_bar.get_height()-length_text.get_height())/2])
            cr.screen.blit(self.length_bar, self.length_bar_pos)
            pygame.draw.polygon(cr.screen,[255, 255, 255],self.length_bar_outline,3)
            self.time = time.time()-self.orig_time
            pygame.draw.rect(self.time_bar, [255, 255, 255], pygame.FRect(0, 0, self.time*(1240/self.time_lims[self.level]), 30))
            time_text = time_font.render("Time : "+str(round(self.time))+" out of " +str(self.time_lims[self.level])+ " s", False, [1, 1 ,1], [0, 0, 0])
            time_text.set_colorkey([0, 0, 0])
            fish_text = time_font.render("Fish : "+str(self.fish)+" out of " +str(self.num_fish[self.level]), False, [1, 1 ,1], [0, 0, 0])
            fish_text.set_colorkey([0, 0, 0])
            self.time_bar.blit(time_text, [(1240-time_text.get_width())/2, (20-time_text.get_height())/2])
            cr.screen.blit(self.time_bar, self.time_bar_pos)
            self.time_bar.fill([0, 0, 0])
            pygame.draw.polygon(cr.screen,[255, 255, 255],self.time_bar_outline,3)
            cr.screen.blit(fish_text, [(1280-fish_text.get_width())/2, self.length_bar_pos[1]+50])
            if self.fishing:
                self.rope_screen.fill([0, 0, 0])
                self.prepare_fishing_line()
                self.down_rect.y=360+self.line_length
                self.old_len = len(self.y_points)
                self.rope.update([mouse_collider], delta_time=cr.event_holder.dt/10)
                if not (self.rope.lowest_point[0] in self.y_points):
                    self.y_points.append(self.rope.lowest_point[0])
                #self.moving = True
                if cr.event_holder.mouse_held_keys[0] and (pg.K_UP in held_keys) and not self.recreated:
                    if (self.line_length<225):
                        self.line_length+=3
                if cr.event_holder.mouse_held_keys[0] and (pg.K_DOWN in held_keys) and not self.recreated:
                    if (self.line_length>70):
                        self.line_length-=3

                if not mouse_pressed_keys[0] and not cr.event_holder.mouse_held_keys[0] and not self.recreated:
                    self.create_fishing_line()
                    self.recreated = True
                    self.y_points = []
                    self.state = 1
                    self.casting_sound.play()
                if not self.recreated:
                    if (pg.K_RIGHT in pressed_keys) or (pg.K_RIGHT in held_keys):
                        if self.fishing_line_angle > 32:
                            self.fishing_line_angle -= 2

                        #print(self.fishing_line_angle)
                    if (pg.K_LEFT in pressed_keys) or (pg.K_LEFT in held_keys):
                        if self.fishing_line_angle < 143:
                            self.fishing_line_angle += 2
                        #print(self.fishing_line_angle)
                    arrow_surf = pygame.transform.rotate(self.angle_pointer, 0-self.fishing_line_angle)
                    cr.screen.blit(arrow_surf, [self.idle_pos[0]-(arrow_surf.get_width()/2), self.idle_pos[1]-(arrow_surf.get_height()/2)])
                if self.recreated:
                    #print(self.rope.moving)
                    self.rope.draw(self.rope_screen)

                    if (self.recall_fishing_line):
                        if (self.old_len-len(self.y_points) == 0):
                            self.moving = False
                        if not self.set_angle:
                            self.fishing_line_angle = 90
                            self.set_angle = True
                        if not self.moving and self.delay%3==0:
                            if len(self.rope.points) > 1:
                                self.rope.points.pop(len(self.rope.points)-1)
                                self.rope.points[len(self.rope.points)-1].child = None
                                #self.rope.lowest_point = [self.rope.points[len(self.rope.points)-1].pos.x, self.rope.points[len(self.rope.points)-1].pos.y]
                            else:
                                self.rope.points = []
                        if (self.rope.points == []):
                            self.cover_rect.y = 720
                            self.rope = None
                            self.rope_screen.fill([0, 0, 0])
                            self.recreated = False
                            self.recreated = False
                            self.recall_fishing_line = False
                            self.set_angle = False
                            self.fishing = False
                            self.y_points = []
                            self.moving = True
                            self.line_length = 70
                            self.fished = False
                        pygame.draw.rect(self.rope_screen, [0, 0, 0], self.cover_rect)

                    if (pg.K_SPACE in pressed_keys) or (pg.K_SPACE in held_keys) and not self.recall_fishing_line:
                        self.recall_fishing_line = True
                        self.state = 2
                        self.winding_sound.play()
                        #self.fishing = False
            #self.bait_slider.update()

    #103
    cloud_sprites = [utils.scale_image(pygame.image.load("assets/Spritesheets//bg/cloud1.png").convert(), 2), utils.scale_image(pygame.image.load("assets/Spritesheets//bg/cloud2.png").convert(), 1)]
    for cloud_sprite in cloud_sprites:
        cloud_sprite.set_colorkey([106, 190, 48])

    class Cloud:
        def __init__(self, x, elevation, type_):
            self.pos = [x, elevation]
            self.type = type_
            self.speed = 1.5
            self.crossed = False
        def update(self):
            self.pos[0]+=self.speed
            cr.screen.blit(cloud_sprites[self.type], self.pos)
            if (self.pos[0]>1180+cloud_sprites[self.type].get_width()/2):
                self.crossed = True
        def regen(self):
            self.pos = [random.randint(-(1280+cloud_sprites[self.type].get_width()), 0-cloud_sprites[self.type].get_width()), random.randint(0, 150)]
            self.type = random.randint(0, 1)
            self.crossed = False
    class CloudManager:
        def __init__(self):
            self.clouds = [Cloud(random.randint(-1280, 0), random.randint(0, 150), random.randint(0, 1)) for i in range(20)]
        def update(self):
            num_cloud = -1
            for cloud in self.clouds:
                num_cloud+=1
                if not cloud.crossed:
                    cloud.update()
                else:
                    self.clouds[num_cloud].regen()
    class WaveManager:
        def __init__(self):
            self.wave_sprites = []
            self.wave_sprite_scale = 4
            for i in range(10):
                self.wave_sprites.append(utils.scale_image(pygame.image.load("assets/Spritesheets//wave/wave00"+str(i)+".png").convert(), self.wave_sprite_scale))
                self.wave_sprites[i].set_colorkey([0, 0, 0])
            for i in range(6):
                self.wave_sprites.append(utils.scale_image(pygame.image.load("assets/Spritesheets//wave/wave01"+str(i)+".png").convert(), self.wave_sprite_scale))
                self.wave_sprites[i+10].set_colorkey([0, 0, 0])
            self.waves = []
            for i in range(6):
                self.waves.append([[i*228, 360-self.wave_sprites[0].get_height()+50], 0])
            self.delay = 0
        def update(self):
            self.delay+=1
            for wave in self.waves:
                cr.screen.blit(self.wave_sprites[wave[1]], wave[0])
                if self.delay%8==0:
                    wave[1]+=1
                    if (wave[1]>(len(self.wave_sprites)-1)):
                        wave[1] = 0
    cloud_manager = CloudManager()
    wave_manager = WaveManager()
    player = Player()
    ocean_rect = pygame.Rect(0, 360, 1280, 360)
    ocean_color = [99, 155, 255]
    sea_bed = utils.scale_image(pygame.image.load("assets/Spritesheets//bg/seabed.png").convert())
    utils.swap_color(sea_bed, [69, 40, 60], [0, 0, 0])
    sea_bed.set_colorkey([0, 0, 0])
    global level
    level = 0
    def start(any):
        global game_state
        game_state = 1
        player.__init__()
        global level
        player.level = level
        fish_manager.__init__()
    def credit(any):
        global game_state
        game_state = 2
    def end(any):
        if not __import__("sys").platform == "emscripten":
            raise SystemExit
    def back(any):
        global game_state
        game_state = 0
    global win_state
    win_state = None
    button_sprites = utils.SpriteSheet(utils.scale_image(pygame.image.load("assets/Spritesheets/buttons.png").convert(), 6), [2, 1])
    start_button = Button([(button_sprites.sheet[0][0].get_width())/2, (500-title_text.get_height())/2+25], [button_sprites.sheet[0][i].copy() for i in range(len(button_sprites.sheet[0]))], [start, 1])
    credits_button = Button([(button_sprites.sheet[0][0].get_width())/2, (500-title_text.get_height())/2+125], [button_sprites.sheet[0][i].copy() for i in range(len(button_sprites.sheet[0]))], [credit, 1])
    end_button = Button([(button_sprites.sheet[0][0].get_width())/2, (500-title_text.get_height())/2+225], [button_sprites.sheet[0][i].copy() for i in range(len(button_sprites.sheet[0]))], [end, 1])
    back_button = Button([utils.center_pos(button_sprites.sheet[0][0])[0],utils.center_pos(button_sprites.sheet[0][0])[1]+200], [button_sprites.sheet[0][i].copy() for i in range(len(button_sprites.sheet[0]))], [back, 1])
    start_button.textures[0].blit(start_text, [(start_button.textures[0].get_width()-start_text.get_width())/2, (start_button.textures[0].get_height()-start_text.get_height())/2])
    start_button.textures[1].blit(start_text, [(start_button.textures[0].get_width()-start_text.get_width())/2, (start_button.textures[0].get_height()-start_text.get_height())/2+4])
    credits_button.textures[0].blit(cred_text, [(credits_button.textures[0].get_width()-cred_text.get_width())/2, (credits_button.textures[0].get_height()-cred_text.get_height())/2])
    credits_button.textures[1].blit(cred_text, [(credits_button.textures[0].get_width()-cred_text.get_width())/2, (credits_button.textures[0].get_height()-cred_text.get_height())/2+4])
    end_button.textures[0].blit(quit_text, [(end_button.textures[0].get_width()-quit_text.get_width())/2, (end_button.textures[0].get_height()-quit_text.get_height())/2])
    end_button.textures[1].blit(quit_text, [(end_button.textures[0].get_width()-quit_text.get_width())/2, (end_button.textures[0].get_height()-quit_text.get_height())/2+4])
    back_button.textures[0].blit(back_text, [(back_button.textures[0].get_width()-back_text.get_width())/2, (back_button.textures[0].get_height()-back_text.get_height())/2])
    back_button.textures[1].blit(back_text, [(back_button.textures[0].get_width()-back_text.get_width())/2, (back_button.textures[0].get_height()-back_text.get_height())/2+4])
    screenshot = None
    bg_channel = pygame.mixer.Channel(0)
    bg_channel.play(pygame.mixer.Sound('assets/Audio/background_track.ogg'), -1)
    bg_channel.set_volume(0.7)
    wave_channel = pygame.mixer.Channel(1)
    wave_channel.play(pygame.mixer.Sound('assets/Audio/waves.ogg'), -1)
    while not cr.event_holder.should_quit:
        cr.screen.fill((120, 171, 200))
        cr.event_holder.get_events()
        pressed_keys = cr.event_holder.pressed_keys
        mouse_pressed_keys = cr.event_holder.mouse_pressed_keys
        held_keys = cr.event_holder.held_keys
        released_keys = cr.event_holder.released_keys
        pygame.draw.rect(cr.screen, ocean_color, ocean_rect)
        cr.screen.blit(sea_bed, [0, 720-sea_bed.get_height()])
        cloud_manager.update()
        #print(cr.event_holder.final_fps)
        if game_state == 1:
            if win_state == None:
                player.update()
        wave_manager.update()
        if game_state == 1:

            if win_state == None:
                if player.fishing and player.recreated:
                    cr.screen.blit(player.rope_screen, (0, 0))
                    if player.rope is not None:
                        if player.rope.lowest_point is not None:
                            cr.screen.blit(player.bait_sprite, [player.rope.lowest_point[0]-player.bait_sprite.get_width()/2, player.rope.lowest_point[1]])
                            pygame.draw.circle(cr.screen, [155, 173, 183], player.rope.lowest_point, 5)
                            #print(player.fishing_line_angle, (utils.angle_between([player.idle_pos, player.rope.lowest_point])//1))
                            #print(player.rope.lowest_point[1]-player.rope.orig_pos[1])
                fish_manager.update()
                if (player.time > player.time_lims[level]) and win_state == None:
                    win_state = 0
                    if screenshot == None:
                        screenshot = cr.screen.copy()
                        alph_surf = pygame.Surface([1280, 720])
                        alph_surf.set_alpha(128)
                        screenshot.blit(alph_surf, [0, 0])
                if (player.time <= player.time_lims[level]) and player.fish >= player.num_fish[player.level]:
                    win_state = 1
                    if screenshot == None:
                        screenshot = cr.screen.copy()
                        alph_surf = pygame.Surface([1280, 720])
                        alph_surf.set_alpha(128)
                        screenshot.blit(alph_surf, [0, 0])
            if screenshot != None:
                cr.screen.blit(screenshot, [0, 0])
                if win_state == 1:
                    if player.level < len(player.num_fish)-1:
                        cr.screen.blit(win_text, [(1280-win_text.get_width())/2, (720-win_text.get_height())/2])
                        cr.screen.blit(lose_text3, [(1280-lose_text3.get_width())/2, (720-lose_text2.get_height())/2+150])
                    else:
                        cr.screen.blit(win_game_text, [(1280-win_game_text.get_width())/2, (720-win_game_text.get_height())/2])
                if win_state == 0:
                    cr.screen.blit(lose_text, [(1280-lose_text.get_width())/2, (720-lose_text.get_height())/2])
                cr.screen.blit(lose_text2, [(1280-lose_text2.get_width())/2, (720-lose_text2.get_height())/2+100])
                if pg.K_BACKSPACE in held_keys:
                    game_state = 0
                    player.__init__()
                    fish_manager.__init__()
                    screenshot = None
                    if win_state == 1:
                        level +=1
                        if level > len(player.num_fish)-1:
                            level = 0
                    win_state = None
        elif game_state == 0:
            cr.screen.blit(cover, [0 ,0])
            #cr.screen.blit(title_text, [(1280-title_text.get_width())/2, (720-title_text.get_height())/2-75])
            start_button.update()
            credits_button.update()
            end_button.update()
        elif game_state == 2:
            cr.screen.blit(credit_screen, [0 ,0])
            back_button.update()
        #cr.screen.blit(pygame.transform.flip(cr.screen, True, False), (0, 0))
        pg.display.update()
        await asyncio.sleep(0)
asyncio.run( main() )


import re
from operator import itemgetter

import pdfplumber
from django.core.management import BaseCommand
from pdfplumber.utils import cluster_objects

from office.utils import title_case
from psalter.models import PsalmVerse, Psalm
from website import settings

def collate_line(line_chars, tolerance=3):
    coll = []
    last_x1 = None
    for char in sorted(line_chars, key=itemgetter("x0")):
        if (last_x1 is not None) and (char["x0"] > (last_x1 + tolerance)):
            coll += " "
        last_x1 = char["x1"]
        coll.append(char)
    return coll

def remove_smart_quotes(text):
    return text.replace("\u2018", "'").replace("\u2019", "'").replace("\u201c", '"').replace("\u201d", '"')

# covert to title case
def clean(input):
    input = title_case(input)
    input = remove_smart_quotes(input)
    input = input.replace("\n", " ").replace("\r", "")
    input = re.sub(" +", " ", input)
    input = input.strip()
    return input

def clean_collect(input):
    input = remove_smart_quotes(input)
    input = input.replace("Amen.", "").replace(" Amen.", "")
    input = input.replace("Amen.", "").replace("Amen", "")
    input = re.sub(" +", " ", input)
    input = input.strip()
    input = f"
{input} Amen.
"

    return input

def number(input):
    input = re.sub("[^0-9]", "", input)
    if input == "2323":
        input = "23"
    input = int(input)
    return input

def remove_number(input):
    input = re.sub("^[0-9]+\.", "", input)
    input = input.replace("Benedict of Nursia", "")
    input = input.strip()
    input = input.split("  ")
    input = input[0]

    input = clean(input)
    return input

def line_to_text(line):
    text = [obj["text"] for obj in line if type(obj) != str and type(obj) != list and obj["object_type"] == "char"]
    text = "".join(text)
    return text.strip()

def line_should_be_included(line):
    # if line[0]["fontname"] == "LPJIDQ+ACaslonPro-Italic-SC700":
    #     return False
    text = line_to_text(line).lower()
    skip_list = [
        "morning prayer",
        "evening prayer",
        "the psalter",
    ]
    for skip in skip_list:
        if skip in text:
            return False
    return True

def import_tle_psalter():
    files = ["tle328psalms1-50.pdf", "tle328psalms51-100.pdf", "tle328psalms101-150.pdf"]
    return import_psalter(files, trad=True)

def import_cont_psalter():
    files = ["BCP2019.pdf"]
    return import_psalter(files, trad=False)

def import_psalter(files, trad=False):
    lines = []
    for file in files:
        with pdfplumber.open(f"{settings.BASE_DIR}/psalter/management/commands/{file}") as pdf:
            for i, pdf_page in enumerate(pdf.pages):
                if not trad and (i < 277 or i > 475):
                    continue
                doctop_clusters = cluster_objects(pdf_page.chars, itemgetter("doctop"), 3)
                temp_lines = [collate_line(line_chars, 3) for line_chars in doctop_clusters]
                lines = lines + temp_lines
    lines = [line for line in lines if line_should_be_included(line)]

    psalm_number = 0
    latin = ""
    first_half = ""
    second_half = ""
    verse_number = 0
    first_half_complete = False
    had_asterisk = False
    fonts = []
    for i, line in enumerate(lines):
        for obj in line:
            try:
                fonts.append(obj["fontname"])
            except TypeError:
                pass

        if "Italic" in line[0]["fontname"]:
            continue
        temp_line = line_to_text(line)
        replace_lord = False
        if "Lord" in temp_line:
            for i, letter in enumerate(line):
                try:
                    o_in_lord = "NO"
                    try:
                        if letter["text"] == "o":
                            if (
                                line[i + 1]["text"] == "r"
                                and line[i + 2]["text"] == "d"
                                and line[i - 1]["text"] == "L"
                            ):
                                o_in_lord = "YES"
                    except IndexError:
                        pass
                    if letter["text"] == "o" and o_in_lord == "YES" and letter["width"] > 6:
                        replace_lord = True
                    # if letter['text'] in ["L", "o", "r", "d"]:
                    #     print(letter['text'], psalm_number, verse_number, letter['width'], o_in_lord)
                except TypeError:
                    pass
        line = line_to_text(line)
        if replace_lord:
            line = line.replace("Lord", "Lᴏʀᴅ")
        if re.match("[0-9]+\s", line) or line.isnumeric() or i >= len(lines) - 2:
            if verse_number != 0 and psalm_number != 0 and first_half:
                # print(psalm_number, verse_number, first_half, i, len(lines))
                psalm = Psalm.objects.get_or_create(number=psalm_number)[0]
                verse = PsalmVerse.objects.get_or_create(psalm=psalm, number=verse_number)
                verse = verse[0]
                if trad:
                    verse.first_half_tle = first_half
                    verse.second_half_tle = second_half
                else:
                    verse.first_half = first_half
                    verse.second_half = second_half
                verse.save()
                first_half = ""
                second_half = ""
                first_half_complete = False
                had_asterisk = False
        if re.match("[0-9]+\s", line):
            verse_number = re.search("[0-9]+\s", line).group()
            line = line.replace(verse_number, "").strip()
            verse_number = number(verse_number)
        if line.isnumeric():
            psalm_number = number(line)
        if "*" in line:
            line = line.replace("*", "").strip()
            had_asterisk = True
        if not first_half_complete and not line.isnumeric():
            first_half = first_half + " " + line
        elif not line.isnumeric():
            second_half = second_half + " " + line
        if had_asterisk:
            first_half_complete = True
            had_asterisk = False
    fonts = set(fonts)

class Command(BaseCommand):
    help = "Import all the collects"

    def handle(self, *args, **options):
        import_tle_psalter()
        import_cont_psalter()


import json

from django.conf import settings
from django.contrib.gis.geos.factory import fromstr
from django.core.urlresolvers import reverse
from django.db import connection
from django.db.models import Count
from django.db.models.query_utils import Q
from django.http.response import HttpResponse
from django.views.generic import TemplateView
from django.views.generic import View

from allegation.views.officer_allegation_api_view import (
    OfficerAllegationAPIView)
from api.models import Setting, InterfaceText
from common.constants import NO_DISCIPLINE_CODES
from common.json_serializer import JSONSerializer
from common.models import (
    Allegation, Area, AllegationCategory, Officer, OfficerAllegation)
from common.models import PoliceWitness
from share.models import Session

OFFICER_COMPLAINT_COUNT_RANGE = [
    [20, 0],  # x >= 9
    [9, 20],  # 3 <= x < 9
    [3, 9],  # 2 <= x < 3
    [2, 3],  # 1 <= x < 2
    [0, 2],  # x == 0
]
OFFICER_COMPLAINT_COUNT_RANGE = getattr(
    settings, 'OFFICER_COMPLAINT_COUNT_RANGE', OFFICER_COMPLAINT_COUNT_RANGE)

class AllegationListView(TemplateView):
    template_name = 'allegation/index.html'
    session = None

    def get_context_data(self, **kwargs):
        context = super(AllegationListView, self).get_context_data(**kwargs)
        admin_settings = Setting.objects.first()
        context.update({'admin_settings': admin_settings})

        interface_texts = {}
        for text in InterfaceText.objects.all():
            interface_texts[text.key] = text.text
        context.update({'interface_texts': json.dumps(interface_texts)})

        is_crawler = self.request.GET.get('is_crawler', False)
        show_disclaimer = (settings.DJANGO_ENV != 'test' or self.request.GET.get('with_disclaimer', False))\
            and not is_crawler
        context.update({'show_disclaimer': 'true' if show_disclaimer else 'false'})

        return context

    def get(self, request, hash_id=None, *args, **kwargs):
        return super(AllegationListView, self).get(request, *args, **kwargs)

class DataToolView(AllegationListView):
    def get_og_image(self, hash_id):
        return self.request.build_absolute_uri(reverse('allegation:sunburst-image', args=[hash_id]))

    def get_site_title(self, hash_id):
        try:
            session_id = Session.id_from_hash(hash_id)[0]
            session = Session.objects.filter(pk=session_id)[0]
            return session.title
        except IndexError:
            default_site_title = Setting.objects.first().default_site_title
            return default_site_title

    def get_context_data(self, **kwargs):
        context = super(DataToolView, self).get_context_data(**kwargs)
        context.update({
            'og_image': self.get_og_image(self.hash_id),
            'site_title': self.get_site_title(self.hash_id)})
        return context

    def get(self, request, hash_id, *args, **kwargs):
        self.hash_id = hash_id
        return super(DataToolView, self).get(request, *args, **kwargs)

class AreaAPIView(View):
    def get(self, request):
        areas = Area.objects.all()
        type_filter = request.GET.get('type')

        if type_filter:
            areas = areas.filter(type=type_filter)

        area_dict = {
            "type": "FeatureCollection",
            "features": [],
        }
        for area in areas:
            polygon = None
            if area.polygon:
                polygon = json.loads(area.polygon.geojson)
            if not polygon:
                continue
            area_json = {
                "type": "Feature",
                "properties": {
                    'id': area.id,
                    'name': area.name,
                    'type': area.type,
                },
                'geometry': polygon
            }

            area_dict['features'].append(area_json)
        content = json.dumps(area_dict)
        return HttpResponse(content)

class OfficerAllegationGISApiView(OfficerAllegationAPIView):
    def get(self, request):
        seen_crids = {}
        officer_allegations = self.get_officer_allegations(
            ignore_filters=['allegation__areas__id'])
        allegation_dict = {
            "type": "FeatureCollection",
            "features": [],
        }
        for officer_allegation in officer_allegations:
            allegation = officer_allegation.allegation
            if allegation.crid in seen_crids:
                continue
            seen_crids[allegation.crid] = True

            if allegation.point:
                point = json.loads(allegation.point.geojson)

                allegation_json = {
                    "type": "Feature",
                    "properties": {
                        "name": allegation.crid,
                        'id': allegation.id
                    },
                    'geometry': point
                }
                if officer_allegation.cat:
                    allegation_json['properties']['type'] = \
                        officer_allegation.cat.allegation_name
                allegation_dict['features'].append(allegation_json)

        content = json.dumps(allegation_dict)
        return HttpResponse(content)

class OfficerAllegationClusterApiView(OfficerAllegationAPIView):
    def get(self, request):
        areas = request.GET.getlist('allegation__areas__id')
        ignore_filters = ['allegation__areas__id']
        if areas:
            self.orig_query_dict = request.GET.copy()
            schools = Area.objects.filter(pk__in=areas, type='school-grounds')
            areas = list(schools.values_list('pk', flat=True))
            if areas:
                self.orig_query_dict.setlist('allegation__areas__id', areas)
                ignore_filters = []
        officer_allegations = self.get_officer_allegations(
            ignore_filters=ignore_filters)
        allegation_pks = list(
            officer_allegations.values_list('allegation__pk', flat=True))
        ret = {'features': [], 'type': 'FeatureCollection'}

        if len(allegation_pks) > 0:
            allegation_pks = ",".join(str(x) for x in allegation_pks)
            kursor = connection.cursor()

            grid_size = 0.0005

            kursor.execute('''
                SELECT
                    COUNT( point ) AS count,
                    ST_AsText( ST_Centroid(ST_Collect( point )) ) AS center
                FROM common_allegation WHERE point IS NOT NULL and id in (%s)
                GROUP BY
                    ST_SnapToGrid( ST_SetSRID(point, 4326), %s, %s)
                ''' % (allegation_pks, grid_size, grid_size)
                )
            kclusters = kursor.fetchall()

            for cluster in kclusters:
                point = fromstr(cluster[1])
                allegation_json = {
                    "type": "Feature",
                    "properties": {

                    },
                    'geometry': {
                        'coordinates': [point.x, point.y],
                        'type': 'Point'
                    }
                }
                ret['features'].append(allegation_json)

        content = json.dumps(ret)
        return HttpResponse(content)

class OfficerAllegationSummaryApiView(OfficerAllegationAPIView):
    def get(self, request):
        officer_allegations = self.get_officer_allegations(
            ignore_filters=['cat', 'cat__category'])

        count_query = officer_allegations.values_list('cat')\
            .annotate(dcount=Count('id'))
        count_by_category = dict(count_query)

        discipline_allegations = officer_allegations\
            .exclude(final_outcome__in=NO_DISCIPLINE_CODES)
        discipline_allegations = discipline_allegations\
            .exclude(final_outcome__isnull=True)
        discipline_count_query = discipline_allegations\
            .values_list('cat').annotate(dcount=Count('id'))
        discipline_count_by_category = dict(discipline_count_query)
        categories = AllegationCategory.objects.all().order_by('category')

        summary = []
        summary_map_by_name = {}

        for category in categories:
            if category.category in summary_map_by_name:
                summary_value = summary_map_by_name[category.category]
            else:
                summary_value = summary_map_by_name[category.category] = {
                    'name': category.category,
                    'total': 0,
                    'count': 0,
                    'subcategories': [],
                    'id': len(summary) + 1,
                }
                summary.append(summary_value)

            count = count_by_category.get(category.id, 0)
            if not count:
                continue
            summary_value['total'] += count
            summary_value['count'] += discipline_count_by_category.get(
                category.id, 0)
            summary_value['subcategories'].append({
                'name': category.allegation_name,
                'cat_id': category.cat_id,
                'id': category.id,
                'count': count,
                'id': category.id
            })

        summary = sorted(summary, key=lambda x: -x['total'])
        summary = [x for x in summary if x['total']]

        for summary_row in summary:
            summary_row['subcategories'] = sorted(summary_row['subcategories'], key=lambda x: -x['count'])

        if summary and summary[0]['total']:
            maximum = summary[0]['total']
        else:
            maximum = 1

        for value in summary:
            value['percentToMax'] = value['total'] * 100.0 / maximum

        content = JSONSerializer().serialize({
            'summary': summary
        })
        return HttpResponse(content, content_type="application/json")

class OfficerListAPIView(OfficerAllegationAPIView):
    def get(self, request):
        officer_allegations = self.get_officer_allegations()
        officers = Officer.objects.filter(officerallegation__in=officer_allegations)\
            .annotate(filtered_allegations_count=Count('officerallegation__allegation__pk'))\
            .order_by('-allegations_count')

        overview = []
        for r in OFFICER_COMPLAINT_COUNT_RANGE:
            range_officers = officers.filter(allegations_count__gte=r[0])
            if r[1]:
                range_officers = range_officers.filter(
                    allegations_count__lt=r[1])
            overview.append(range_officers.count())

        content = JSONSerializer().serialize({
            'officers': officers,
            'overview': overview,
        })
        return HttpResponse(content, content_type="application/json")

class PoliceWitnessAPIView(View):
    def get(self, request):
        try:
            allegation = Allegation.objects.get(crid=request.GET['crid'])
        except (KeyError, Allegation.DoesNotExist):
            return HttpResponse('{}', content_type="application/json")

        related_officers = Officer.objects.filter(
            officerallegation__allegation__pk=allegation.pk)
        witnesses = PoliceWitness.objects.filter(allegation__pk=allegation.pk)

        ret = {'police_witness': []}
        for witness in witnesses:
            witness.officer_name = "%s %s" % (
                witness.officer.officer_first, witness.officer.officer_last)

            officers = []
            for officer in related_officers:
                allegation_pks = OfficerAllegation.objects\
                    .filter(officer=officer)\
                    .values_list('allegation__pk', flat=True)
                total_witnesses = PoliceWitness.objects.filter(
                    officer__pk=witness.officer_id,
                    allegation_id=allegation_pks)
                related_officer_allegations = OfficerAllegation.objects\
                    .filter(allegation__pk__in=allegation_pks)\
                    .filter(
                        Q(officer__pk=witness.officer_id) |
                        Q(allegation__pk__in=total_witnesses
                            .values_list('allegation__pk', flat=True),
                          officer__pk=officer.pk))

                num_complaints = related_officer_allegations.count()
                no_action_taken_count = related_officer_allegations.filter(
                    Q(final_outcome__in=NO_DISCIPLINE_CODES) |
                    Q(final_outcome__isnull=True)).count()
                officers.append({
                    'num_complaints': num_complaints,
                    'no_action_taken': no_action_taken_count,
                    'officer': officer,
                })

            ret['police_witness'].append({
                'witness': witness,
                'witness_officer': witness.officer,
                'officers': officers,
            })

        content = JSONSerializer().serialize(ret)
        return HttpResponse(content, content_type="application/json")


from fastapi import FastAPI, UploadFile, File, Response, Form, BackgroundTasks
from fastapi.responses import FileResponse
from fastapi.middleware.cors import CORSMiddleware
from pathlib import Path
from fastapi.staticfiles import StaticFiles
import requests
from bs4 import BeautifulSoup
import caches
import os
from ExportPDF import export_pdf
from urllib.parse import urljoin
from CBIR_Multiprocess import Cbir_Color2
from CBIR_NonMultiprocess import Cbir_Color1
import caches2
from multiprocessing import Lock
from texture import Texture

UPLOAD_DIR_SEARCH = Path() / "uploads/search"
UPLOAD_DIR_DATA= Path() / "uploads/data-set"

app = FastAPI()

data_set_directory = Path(__file__).parent / "uploads/data-set"
app.mount("/uploads/data-set", StaticFiles(directory=data_set_directory), name="data-set")

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

cache2 = caches2.json_to_dict("./caches/data.json")
cache_lock = Lock()
cache1 = caches.csv_to_array()
cachetexture = caches2.json_to_dict("./caches/texture.json")

@app.post("/uploadfile/")
async def create_upload_file(file_upload: UploadFile):
    delete_search()
    contents = await file_upload.read()
    save_path = UPLOAD_DIR_SEARCH / "received_image.jpg"
    with open(save_path, "wb") as f:
        f.write(contents)

    return {"filename": file_upload.filename}

@app.post("/uploaddata/")
async def create_upload_file(file_uploads: list[UploadFile]):
    delete_dataset()
    for file_upload in file_uploads:
        contents = await file_upload.read()
        save_path = UPLOAD_DIR_DATA / file_upload.filename.split("/")[-1]
        with open(save_path, "wb") as f:
            f.write(contents)

    return {"filenames": [f.filename for f in file_uploads]}

@app.post("/capture_frame")
async def receive_image(file: UploadFile = File(...)):
    delete_search()
    content = await file.read()
    save_path = UPLOAD_DIR_SEARCH / "received_image.jpg"
    with open(save_path, "wb") as f:
        f.write(content)

    return {"message": "Image received and processed"}

@app.post("/scrape")
async def scrape_images(link: str = Form(...)):
    delete_dataset()
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36","Accept-Language": "en-US,en;q=0.5",}
    response = requests.get(url=link, headers=headers)
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html.parser')
        image_tags = soup.find_all('img')
        os.makedirs(UPLOAD_DIR_DATA, exist_ok=True)
        arr_responses = []
        responses = {}
        for index, img in enumerate(soup.find_all('img')):
            img_link = img.get('src')
            if img_link:
                if not img_link.startswith(('http://', 'https://')):
                    complete_url = urljoin(link, img_link)
                else:
                    complete_url = img_link

                img_response = requests.get(complete_url)

                if img_response.status_code == 200:
                    with open(os.path.join(UPLOAD_DIR_DATA, f"{index}.jpg"), 'wb') as f:
                        f.write(img_response.content)
                        arr_responses.append({"url": f"http://localhost:8000/uploads/data-set/{index}.jpg"})
                responses["data"] = arr_responses
    else:
        return {"message": "failed to scrape images"}

    return responses

@app.get("/get-result-color")
async def send_result_color(background_tasks: BackgroundTasks):

    if len(os.listdir(UPLOAD_DIR_DATA)) > 500:
        similarity_arr, time = Cbir_Color2(cache=cache2,cache_lock=cache_lock)
    else :
        similarity_arr, time = Cbir_Color1(cache=cache2)

    background_tasks.add_task(make_pdf, similarity_arr, time)

    image_objects = []
    for item in similarity_arr:
        image_objects.append({
            "url": f"http://127.0.0.1:8000/uploads/data-set/{item['url']}",
            "percentage": item["percentage"]
        })

    return {
        "data": image_objects,
        "time": time,
    }
@app.get("/get-result-texture")
async def send_resul_texture(background_tasks: BackgroundTasks):

    similarity_arr, time = Texture(cachetexture)

    background_tasks.add_task(make_pdf, similarity_arr, time)

    image_objects = []
    for item in similarity_arr:
        image_objects.append({
            "url": f"http://127.0.0.1:8000/uploads/data-set/{item['url']}",
            "percentage": item["percentage"]
        })

    return {
        "data": image_objects,
        "time": time,
    }

@app.get("/download_pdf")
async def download_pdf():

    if os.path.exists("./template-output.pdf"):
        return FileResponse("./template-output.pdf", headers={"Content-Disposition": "attachment; filename=pdfOutput.pdf"})
    else:
        return {"message": "PDF not found"}

def delete_dataset():
    for file in os.listdir(UPLOAD_DIR_DATA):
        os.remove(UPLOAD_DIR_DATA / file)
    return {"message": "Data-set deleted"}

def delete_search():
    for file in os.listdir(UPLOAD_DIR_SEARCH):
        os.remove(UPLOAD_DIR_SEARCH / file)
    return {"message": "Search deleted"}

async def make_pdf(similarity_arr, time):
    export_pdf(similarity_arr=similarity_arr, time=time)
    return {"message": "PDF made"}