from argparse import ArgumentParser
from script.oci import OCIManager, Citation
from requests import get
from json import loads, load
from re import sub, findall
from urllib.parse import unquote, quote
from datetime import datetime
from csv import DictReader, DictWriter
from os.path import isdir, exists
from os import walk, sep, makedirs

HTTP_HEADERS = {"User-Agent": "CROCI / Create New Citations (via OpenCitations - "
                              "http://opencitations.net; mailto:contact@opencitations.net)"}
BASE_URL = "http://dx.doi.org/"
CROCI_BASE = "https://w3id.org/oc/index/croci/"

class DOIManager(object):
    def __init__(self):
        self.api = "https://doi.org/api/handles/"

    def normalize(self, doi_entity):
        try:
            doi_string = sub("\s+", "", unquote(doi_entity[doi_entity.index("10."):]))
            return doi_string.lower().strip()
        except:  # Any error in processing the DOI will return None
            return None

    def call_doi(self, doi_entity):
        doi = self.normalize(doi_entity)
        r = get(self.api + quote(doi), headers=HTTP_HEADERS, timeout=30)
        if r.status_code == 200:
            r.encoding = "utf-8"
            json_res = loads(r.text)
            return json_res.get("responseCode")

    def is_valid(self, doi_entity):
        result = self.call_doi(doi_entity)
        return result == 1

class DataCiteManager(object):
    def __init__(self):
        self.date = {}
        self.api = "https://api.datacite.org/works/%s"
        self.dm = DOIManager()

    def call_datacite(self, doi_entity):
        doi = self.dm.normalize(doi_entity)
        r = get(self.api + quote(doi), headers=HTTP_HEADERS, timeout=30)
        if r.status_code == 200:
            r.encoding = "utf-8"
            json_res = loads(r.text)
            return json_res

    def __get_date(self, json_obj):
        return json_obj.get("published")

    def __retrive_all(self, doi):
        json_obj = self.call_datacite(doi)
        if json_obj.get("data") and json_obj["data"].get("attributes"):
            self.date[doi] = self.__get_date(json_obj["data"]["attributes"])

    def __get_item(self, doi_entity, c):
        doi = self.dm.normalize(doi_entity)
        if doi not in c:
            json_obj = self.call_datacite(doi)
            self.date[doi] = self.__get_date(json_obj["data"]["attributes"])
        return c.get(doi)

    def get_date(self, doi_entity):
        return self.__get_item(doi_entity, self.date)

class CrossrefManager(object):
    def __init__(self):
        self.issn = {}
        self.date = {}
        self.orcid = {}
        self.api = "https://api.crossref.org/works/%s"
        self.dm = DOIManager()

    def call_crossref(self, doi_entity):
        doi = self.dm.normalize(doi_entity)
        r = get(self.api + quote(doi), headers=HTTP_HEADERS, timeout=30)
        if r.status_code == 200:
            r.encoding = "utf-8"
            json_res = loads(r.text)
            return json_res.get("message")

    @staticmethod
    def contains(obj, key, value):
        field = None
        if obj:
            field = obj.get(key)
        return field and value in field

    def __get_orcid(self, json_obj):
        result = []
        if json_obj:
            authors = json_obj.get("author")
            if authors:
                for author in authors:
                    orcid = author.get("ORCID")
                    if orcid:
                        orcid = findall("....-....-....-....", orcid)
                        if orcid:
                            result.append(orcid[0])
        return result

    def __get_issn(self, json_obj):
        result = []
        if CrossrefManager.contains(json_obj, "type", "journal"):
            issns = json_obj["ISSN"]
            if issns:
                for issn in issns:
                    norm_issn = sub("\W", "", issn).upper()
                    result.append(norm_issn)
        return result

    def __get_date(self, json_obj):
        if json_obj:
            date = json_obj.get("issued")
            if date:
                date_list = date["date-parts"][0]
                if date_list is not None:
                    l_date_list = len(date_list)
                    if l_date_list != 0 and date_list[0] is not None:
                        if l_date_list == 3 and \
                                ((date_list[1] is not None and date_list[1] != 1) or
                                 (date_list[2] is not None and date_list[2] != 1)):
                            result = datetime(date_list[0], date_list[1], date_list[2], 0, 0).strftime('%Y-%m-%d')
                        elif l_date_list == 2 and date_list[1] is not None:
                            result = datetime(date_list[0], date_list[1], 1, 0, 0).strftime('%Y-%m')
                        else:
                            result = datetime(date_list[0], 1, 1, 0, 0).strftime('%Y')
                        return result

    def get_date(self, doi_entity):
        return self.__get_item(doi_entity, self.date)

    def get_issn(self, doi_entity):
        return self.__get_item(doi_entity, self.issn)

    def get_orcid(self, doi_entity):
        return self.__get_item(doi_entity, self.orcid)

    def __get_item(self, doi_entity, c):
        doi = self.dm.normalize(doi_entity)
        if doi not in c:
            json_obj = self.call_crossref(doi)
            self.issn[doi] = self.__get_issn(json_obj)
            self.date[doi] = self.__get_date(json_obj)
            self.orcid[doi] = self.__get_orcid(json_obj)
        return c.get(doi)

    def share_issn(self, doi_entity_1, doi_entity_2):
        result = False

        doi_entity_1_issns = self.get_issn(doi_entity_1)
        doi_entity_2_issns = self.get_issn(doi_entity_2)
        while not result and doi_entity_1_issns:
            result = doi_entity_1_issns.pop(0) in doi_entity_2_issns

        return result

    def share_orcid(self, doi_entity_1, doi_entity_2):
        result = False

        doi_entity_1_orcid = self.get_orcid(doi_entity_1)
        doi_entity_2_orcid = self.get_orcid(doi_entity_2)
        while not result and doi_entity_1_orcid:
            result = doi_entity_1_orcid.pop(0) in doi_entity_2_orcid

        return result

class ORCIDManager(object):
    def __init__(self, key, m_list=[]):
        self.orcid = {}
        self.api = "https://pub.orcid.org/v2.1/search?q="
        self.dm = DOIManager()
        self.header = {"Content-Type": "application/json"}
        if key:
            self.header["Authorization"] = "Bearer %s" % key
        self.header.update(HTTP_HEADERS)
        self.m_list = m_list

    def call_orcid(self, doi_entity):
        doi = self.dm.normalize(doi_entity)
        r = get(self.api + quote("doi-self:\"%s\" OR doi-self:\"%s\"" % (doi, doi.upper())),
                headers=self.header, timeout=30)
        if r.status_code == 200:
            r.encoding = "utf-8"
            json_res = loads(r.text)
            return json_res.get("result")

    def get_orcid(self, doi_entity):
        doi = self.dm.normalize(doi_entity)
        if doi not in self.orcid:
            json_obj = self.call_orcid(doi)
            result = []
            for item in json_obj:
                orcid = item.get("orcid-identifier")
                if orcid:
                    result.append(orcid["path"])
            self.orcid[doi] = result
        for m in self.m_list:
            if doi in m.orcid:
                for orcid in m.get_orcid(doi):
                    if orcid not in self.orcid[doi]:
                        self.orcid[doi].append(orcid)

        return self.orcid[doi]

    def share_orcid(self, doi_entity_1, doi_entity_2):
        result = False

        doi_entity_1_orcid = self.get_orcid(doi_entity_1)
        doi_entity_2_orcid = self.get_orcid(doi_entity_2)
        while not result and doi_entity_1_orcid:
            result = doi_entity_1_orcid.pop(0) in doi_entity_2_orcid

        return result

class CSVManager(object):
    @staticmethod
    def open_csv(fd_path, metadata=False, delimiter=","):
        result = []

        f_paths = set()
        if exists(fd_path):
            if isdir(fd_path):
                for cur_dir, cur_subdir, cur_files in walk(fd_path):
                    for cur_file in cur_files:
                        if cur_file.endswith(".csv"):
                            f_paths.add(cur_dir + sep + cur_file)
            else:
                if fd_path.endswith(".csv"):
                    f_paths.add(fd_path)

        for f_path in f_paths:
            meta = {}
            with open(f_path) as f:
                cur_citations = list(DictReader(f, delimiter=delimiter))
                if metadata:
                    with open(f_path.replace(".csv", ".json")) as mf:
                        meta = load(mf)
                result.append((cur_citations, meta))

        return result

    @staticmethod
    def list_citations(csv_result):
        result = []
        for citations, metadata in csv_result:
            result.extend(citations)
        return result

    @staticmethod
    def create_set_from_csv(csv_obj, key):
        result = set()

        for row in csv_obj:
            result.add(row[key])

        return result

    @staticmethod
    def store_row(o, t, csv_obj, rdf_graph, is_prov=False):
        d_path = o + sep + "csv" + sep + t[:7].replace("-", sep) + sep
        r_path = o + sep + "rdf" + sep + t[:7].replace("-", sep) + sep

        if is_prov:
            header = ["oci", "agent", "source", "datetime"]
            d_path = d_path.replace(o + sep, o + sep + ".." + sep + "prov" + sep)
            r_path = r_path.replace(o + sep, o + sep + ".." + sep + "prov" + sep)
        else:
            header = ["oci", "citing", "cited", "creation", "timespan", "journal_sc", "author_sc"]

        if not exists(d_path):
            makedirs(d_path)
        if not exists(r_path):
            makedirs(r_path)

        f_path = d_path + t + ".csv"
        f_exists = exists(f_path)
        with open(f_path, "a") as f:
            dw = DictWriter(f, header)
            if not f_exists:
                dw.writeheader()
            dw.writerow(csv_obj)

        t_path = r_path + t + ".ttl"
        with open(t_path, "a") as f:
            rdf_string = Citation.format_rdf(rdf_graph, "nt")
            f.write(rdf_string)

def get_date(doi, d, m_list):
    clean_d = None
    for y, m, d in findall("^([0-9][0-9][0-9][0-9])([0-9][0-9])?([0-9][0-9])?$", sub("[^\d]", "", d)):
        clean_d = y + ("-" + m if m else "") + ("-" + d if d else "")

    while not clean_d and m_list:
        clean_d = m_list.pop(0).get_date(doi)

    return clean_d

if __name__ == "__main__":
    arg_parser = ArgumentParser("cnc.py (Create New Citations",
                                description="This tool allows one to take a four column CSV file describing"
                                            "DOI-to-DOI citations, and to store it according to CSV used by"
                                            "the OpenCitations Indexes so as to be added to CROCI. It uses"
                                            "several online services to check several things to create the"
                                            "final CSV file.")

    arg_parser.add_argument("-i", "--input", required=True, nargs="+",
                            help="The input CSV with new citation data.")
    arg_parser.add_argument("-d", "--data", required=True,
                            help="The directory containing all the CSV files already added in CROCI.")
    arg_parser.add_argument("-o", "--orcid", default=None,
                            help="ORCID API key to be used to query them.")
    arg_parser.add_argument("-l", "--lookup", required=True,
                            help="The lookup table for producing OCIs.")

    args = arg_parser.parse_args()

    print("Retrieve new citation data")
    exi_citations = CSVManager.list_citations(CSVManager.open_csv(args.data))

    print("Retrieve existing citation data")
    exi_ocis = CSVManager.create_set_from_csv(exi_citations, "oci")

    print("Create the DOI Manager")
    doim = DOIManager()

    print("Create the Crossref Manager")
    cm = CrossrefManager()

    print("Create the DataCite Manager")
    dm = DataCiteManager()

    print("Create the ORCID Manager")
    om = ORCIDManager(args.orcid, [cm])

    print("Create the OCI Manager")
    ocim = OCIManager(lookup_file=args.lookup)
    cur_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')

    citations_already_present = 0
    new_citations_added = 0
    error_in_dois_syntax = 0
    error_in_dois_existence = 0
    all_citations = 0

    for f in args.input:
        try:
            if isdir(f):
                print("\nProcessing files in '%s'" % f)
            else:
                print("\nProcessing file '%s'" % f)
            all_new_citations = CSVManager.open_csv(f, metadata=True)
            for new_citations, new_meta in all_new_citations:
                all_citations += len(new_citations)
                for new_citation in new_citations:
                    citing_doi, cited_doi = \
                        doim.normalize(new_citation["citing_id"]), doim.normalize(new_citation["cited_id"])
                    if citing_doi and cited_doi:
                        oci = ocim.get_oci(citing_doi, cited_doi, "050").replace("oci:", "")
                        if oci not in exi_ocis:
                            exi_ocis.add(oci)
                            if doim.is_valid(citing_doi) and doim.is_valid(cited_doi):
                                print("Create citation data for 'oci:%s' between DOI '%s' and DOI '%s', from '%s'" %
                                      (oci, citing_doi, cited_doi, new_meta["source"]))
                                citing_pub_date, cited_pub_date = \
                                    get_date(citing_doi, new_citation["citing_publication_date"], [cm, dm]), \
                                    get_date(cited_doi, new_citation["cited_publication_date"], [cm, dm])
                                cit = Citation(oci,
                                               BASE_URL + quote(citing_doi), citing_pub_date,
                                               BASE_URL + quote(cited_doi), cited_pub_date,
                                               None, None,
                                               new_meta["agent"], new_meta["source"], cur_time,
                                               "CROCI", "doi", BASE_URL + "([[XXX__decode]])", "reference",
                                               cm.share_issn(citing_doi, cited_doi),
                                               om.share_orcid(citing_doi, cited_doi))

                                # Store in CSV and RDF
                                cit_json = loads(cit.get_citation_json())
                                cit_rdf = cit.get_citation_rdf(CROCI_BASE, False, False, False)
                                cit_json_prov = loads(cit.get_citation_json_prov())
                                cit_rdf_prov = cit.get_citation_prov_rdf(CROCI_BASE)
                                CSVManager.store_row(args.data, cur_time, cit_json, cit_rdf)
                                CSVManager.store_row(args.data, cur_time, cit_json_prov, cit_rdf_prov, True)
                                new_citations_added += 1
                            else:
                                print("WARNING: some DOIs, among '%s' and '%s', do not exist" % (citing_doi, cited_doi))
                                error_in_dois_existence += 1
                        else:
                            print("WARNING: the citation between DOI '%s' and DOI '%s' has been already processed" %
                                  (citing_doi, cited_doi))
                            citations_already_present += 1
                    else:
                        print("WARNING: some DOIs, among '%s' and '%s', is syntactically incorrect" %
                              (citing_doi, cited_doi))
                        error_in_dois_syntax += 1
        except Exception as e:
            print(e.message)

    print("\n# Summary\nNumber of new citations added: %s\nNumber of citations already present in CROCI: %s\nNumber "
          "of citations not added due to a wrong DOI specification: %s (syntax error) and %s (not found "
          "error)\nNumber of citations not processed due to an exception: %s" %
          (new_citations_added, citations_already_present, error_in_dois_syntax, error_in_dois_existence,
           (all_citations -
            (new_citations_added + citations_already_present + error_in_dois_syntax + error_in_dois_existence))))


import os
import sys

import pytest

sys.path.append(os.path.realpath(os.path.dirname(__file__) + "../.."))
from send_email_function import main

@pytest.fixture
def mock_env(monkeypatch):
    """Setting mock environment variables"""
    monkeypatch.setenv("SIGNED_URL", "True")
    monkeypatch.setenv("FROM_EMAIL", "sender@gmail.com")
    monkeypatch.setenv("TO_EMAILS", "recepient@gmail.com")
    monkeypatch.setenv("EMAIL_SUBJECT", "BigQuery email export")
    monkeypatch.setenv("SENDGRID_API_KEY", "SG.key")
    monkeypatch.setenv("SIGNED_URL_EXPIRATION", "24")

def test_raise_exception():
    """Tests that KeyError exception is raised when no env vars are set"""
    with pytest.raises(KeyError):
        main.get_env('SIGNED_URL')

@pytest.mark.parametrize("test_input,expected",
                         [("SIGNED_URL", "True"),
                          ("FROM_EMAIL", "sender@gmail.com"),
                          ("TO_EMAILS", "recepient@gmail.com"),
                          ("EMAIL_SUBJECT", "BigQuery email export"),
                          ("SENDGRID_API_KEY", "SG.key"),
                          ("SIGNED_URL_EXPIRATION", "24")])
def test_get_env(mock_env, test_input, expected):
    """Tests reading of env vars"""
    assert main.get_env(test_input) == expected

@pytest.mark.parametrize(
    "test_input,expected",
    [("gs://bucket/object.txt",
      "https://storage.cloud.google.com/bucket/object.txt"),
     ("gs://bucket/dir/object.txt",
      "https://storage.cloud.google.com/bucket/dir/object.txt"),
     ("gs://bucket/dir/subdir/object.json",
      "https://storage.cloud.google.com/bucket/dir/subdir/object.json")])
def test_get_auth_url(test_input, expected):
    """Tests creation of authenticated GCS URL"""
    assert main.get_auth_url(test_input) == expected


import os.path

import aiohttp
import asyncio

from pokeretriever.ability import Ability
from pokeretriever.custom_exceptions import InvalidFileTypeError, FileExtensions
from pokeretriever.move import Move
from pokeretriever.stat import Stat
from pokeretriever.pokemon import Pokemon
from pathlib import Path

def read_file(path: str) -> list:
    """
    Read the input file line by line and store the values in a list that will be used for a Pokemon API call.

    :param path: string, location of where the file exists
    :return: string list.
    """
    try:
        # check file extension
        extension_suffix = Path(path).suffix
        if extension_suffix not in [file.value for file in FileExtensions]:
            raise InvalidFileTypeError(f'Invalid file type')

        # check file exits
        if not os.path.exists(path):
            raise FileNotFoundError()

        # read file and append value from each line into data_list
        data_list = []
        with open(path, 'r') as file:
            for line_value in file:
                data_list.append(line_value)

    except FileNotFoundError:
        print(f'Error message: File not found. Please try again.')
        exit()
    except ValueError:
        print(f'Error message: Your file is not of proper format. Please try again')
        exit()
    except InvalidFileTypeError:
        print(f'Error message: Cannot read invalid file type. Please try again')
        exit()

    else:
        return data_list

async def get_single_request(url: str, session: aiohttp.ClientSession) -> dict | None:
    """
    Method used when the input data argument is provided in the command line.

    :param url: str
    :param session: aiohttp.client.ClientSession
    :return: json payload from the Pokemon's API request.
    """
    response = await session.request(method='GET', url=url)

    if response.status == 200:
        json_response = await response.json()
        return json_response

    if response.status == 404:
        return

async def get_requests(all_url: list):
    """
    Method used when the input file argument is provided in the command line.

    :param all_url: string list
    :return: json payload for all Pokemon's API request.
    """
    # This functions takes in a list of urls and makes batch async calls. Returns a list of all responses
    async with aiohttp.ClientSession() as session:
        async_coroutines = [get_single_request(url, session) for url in all_url]
        all_responses = await asyncio.gather(*async_coroutines)
        all_responses_filtered = [response for response in all_responses if response is not None]  # filter out None values
        return all_responses_filtered

def convert_json_into_ability_object(json_list) -> list:
    """
    Take the payload JSON result from API get request calls and use it to create ability objects.

    :param json_list: json list
    :return: List of ability objects
    """
    if len(json_list) == 0:
        return []

    ability_objects = []
    for ability in json_list:
        single_ability = Ability(
            ability['generation']['name'],
            ability['effect_entries'] if ability['effect_entries'] else None,
            ability['pokemon'],
            ability['name'],
            ability['id']
        )
        ability_objects.append(single_ability)

    return ability_objects

def convert_json_into_stat_object(json_list) -> list:
    """
    Take the payload JSON result from API get request calls and use it to create stat objects.

    :param json_list: json list
    :return: List of stat objects
    """
    if len(json_list) == 0:
        return []

    stat_objects = []
    for stat in json_list:
        single_stat = Stat(
            stat['is_battle_only'],
            stat['name'],
            stat['id']
        )
        stat_objects.append(single_stat)

    return stat_objects

def convert_json_into_move_object(json_list) -> list:
    """
    Take the payload JSON result from API get request calls and use it to create move objects.

    :param json_list: json list
    :return: List of move objects
    """
    if len(json_list) == 0:
        return []

    move_objects = []
    for move in json_list:
        single_move = Move(
            move['generation']['name'],
            move['accuracy'],
            move['pp'],
            move['power'],
            move['type']['name'],
            move['damage_class']['name'],
            move['effect_entries'][0]['short_effect'] if move['effect_entries'] and move['effect_entries'][0][
                'short_effect'] else None,
            move['name'],
            move['id']
        )
        move_objects.append(single_move)

    return move_objects

async def get_moves(value_list) -> list:
    """
    Method that is a wrapper to call other functions for move mode.

    :param value_list: string list of input data of pokemon's name or id.
    :return: List of move objects
    """
    moves_url = [f"https://pokeapi.co/api/v2/move/{value}/" for value in value_list]
    moves_api_response = await get_requests(moves_url)
    move_objects = convert_json_into_move_object(moves_api_response)
    return move_objects

async def get_abilities(value_list) -> list:
    """
    Method that is a wrapper to call other functions for ability mode.

    :param value_list: string list of input data of pokemon's name or id.
    :return: List of ability objects
    """
    abilities_url = [f'https://pokeapi.co/api/v2/ability/{value}/' for value in value_list]
    abilities_api_response = await get_requests(abilities_url)
    abilities_objects = convert_json_into_ability_object(abilities_api_response)
    return abilities_objects

async def get_all_pokemon_api_data(value_list):
    """
    Method used to send get API requests for pokemon mode.

    :param value_list: input values for pokemon mode used for multiple get requests
    :return: list of JSON payload data from the api response.
    """
    pokemons_url = [f'https://pokeapi.co/api/v2/pokemon/{value}' for value in value_list]
    pokemons_data_api_response = await get_requests(pokemons_url)
    return pokemons_data_api_response

async def get_pokemon_abilities(data) -> list:
    """
    Method that is a wrapper to call other functions for pokemon mode to extract abilities.

    :param data: string list of input data of pokemon's name or id.
    :return: List of ability objects that is contained in pokemon API result.
    """
    # This function calls get_requests api to get all the json data of abilities, convert to obj Ability, return list of obj Ability

    abilities = data['abilities']
    abilities_url = [ability['ability']['url'] for ability in abilities]
    abilities_api_response = await get_requests(abilities_url)  # all the abilities response from PokeAPI
    ability_objects = convert_json_into_ability_object(abilities_api_response)
    return ability_objects

async def get_pokemon_stats(data) -> list:
    """
    Method that is a wrapper to call other functions for pokemon mode to extract stat.

    :param data: string list of input data of pokemon's name or id.
    :return: List of stat objects that is contained in pokemon API result.
    """
    # This function calls get_requests and then make a list of Stats objects, returns Stats obj list

    stats = data['stats']
    stats_url = [stat['stat']['url'] for stat in stats]
    stats_api_response = await get_requests(stats_url)  # all the stats response from PokeAPI
    stat_objects = convert_json_into_stat_object(stats_api_response)
    return stat_objects

async def get_pokemon_moves(data) -> list:
    """
    Method that is a wrapper to call other functions for pokemon mode to extract moves.

    :param data: string list of input data of pokemon's name or id.
    :return: List of move objects that is contained in pokemon API result.
    """
    # This function calls get_requests and then make a list of Move objects, returns Move obj list

    moves = data['moves']
    moves_url = [move['move']['url'] for move in moves]
    moves_api_response = await get_requests(moves_url)
    move_objects = convert_json_into_move_object(moves_api_response)
    return move_objects

def get_expanded_pokemon(data, moves: list, abilities: list, stats: list) -> Pokemon:
    """
    Method that deals when the expanded argument is mentioned in the command line terminal

    :param data: json payload data return from get pokemon API request
    :param moves: list of move objects
    :param abilities: list of ability objects
    :param stats: list of stats objects
    :return: pokemon object
    """
    ptypes = data['types']
    ptype_name = ", ".join([ptype['type']['name'] for ptype in ptypes])

    single_pokemon = Pokemon(
        data['height'],
        data['weight'],
        stats,
        ptype_name,
        abilities,
        moves,
        data['name'],
        data['id']
    )

    return single_pokemon

async def get_non_expanded_pokemon(data) -> Pokemon:
    """
    Method that deals when the expanded argument is not mentioned in the command line terminal

    :param data: json payload data return from get pokemon API request
    :return: pokemon object
    """
    stats = data['stats']
    stats_info = [(stat['stat']['name']).title() + ': ' + str(stat['base_stat']) for stat in stats]

    abilities = data['abilities']
    abilities_info = ['Name: ' + ability['ability']['name'] for ability in abilities]

    ptypes = data['types']
    ptype_name = ", ".join([ptype['type']['name'] for ptype in ptypes])

    moves = data['moves']
    moves_info = ["Move name: " + move["move"]["name"] + " | Level acquired: " + str(move["version_group_details"][0]["level_learned_at"]) for move in moves]

    single_pokemon = Pokemon(
        data['height'],
        data['weight'],
        stats_info,
        ptype_name,
        abilities_info,
        moves_info,
        data['name'],
        data['id']
    )

    return single_pokemon


import json
import logging
import os

accuracy = 2

# Set up logging configuration
logging.basicConfig(level=logging.WARNING, format='%(levelname)s - %(message)s')

def delete_files_in_folder(folder_path):
    i = 0
    # Iterate over all the files in the folder
    for filename in os.listdir(folder_path):
        file_path = os.path.join(folder_path, filename)

        # Check if the path is a file (not a directory)
        if os.path.isfile(file_path):
            # Delete the file
            os.remove(file_path)
            i += 1
    print(f"Deleted {i} files")

def write_json(element, json_folder):
    guid = element.guid
    height = round(element.element_planes[0].height, accuracy)
    width = 0
    delivery_number = element.delivery_number
    for plane in element.element_planes:
        width += plane.width
    width = round(width, accuracy)
    plane_count = element.plane_count

    planes = []

    def get_children_data(opening):
        opening_dict = {}
        opening_dict["HEIGHT"] = round(opening.height, accuracy)
        opening_dict["WIDTH"] = round(opening.width, accuracy)
        opening_plane = opening.local_opening_plane
        opening_dict["LOCATION"] = {"ORIGIN": opening_plane.get_origin_str(),
                                    "X_VECTOR": opening_plane.get_x_vec_str(),
                                    "Y_VECTOR": opening_plane.get_y_vec_str()}

        opening_dict["LEVEL"] = opening.level

        if opening.top is not None:
            opening_dict["TOP"] = {"PROFILE": opening.top.profile, "GUID": opening.top.guid}
        if opening.bottom is not None:
            opening_dict["BOTTOM"] = {"PROFILE": opening.bottom.profile, "GUID": opening.bottom.guid}
        if opening.left is not None:
            opening_dict["LEFT"] = {"PROFILE": opening.left.profile, "GUID": opening.left.guid}
        if opening.right is not None:
            opening_dict["RIGHT"] = {"PROFILE": opening.right.profile, "GUID": opening.right.guid}

        opening_dict["SPLIT DIRECTION"] = opening.split_direction

        opening_dict["CHILDREN"] = []

        if len(opening.children) == 0:
            opening_dict["TYPE"] = opening.type

        for child_opening in opening.children:
            opening_dict["CHILDREN"].append(get_children_data(child_opening))

        return opening_dict

    for plane in element.element_planes:
        plane_dict = {}
        plane_dict["HEIGHT"] = round(plane.height, accuracy)
        plane_dict["WIDTH"] = round(plane.width, accuracy)

        opening = plane.opening

        opening_dict = {}

        opening_dict["HEIGHT"] = round(opening.height, accuracy)
        opening_dict["WIDTH"] = round(opening.width, accuracy)

        opening_plane = opening.plane
        opening_dict["LOCATION"] = {"ORIGIN": opening_plane.get_origin_str(),
                                    "X_VECTOR": opening_plane.get_x_vec_str(),
                                    "Y_VECTOR": opening_plane.get_y_vec_str()}

        opening_dict["LEVEL"] = opening.level

        if opening.top is not None:
            opening_dict["TOP"] = {"PROFILE": opening.top.profile, "GUID": opening.top.guid}
        if opening.bottom is not None:
            opening_dict["BOTTOM"] = {"PROFILE": opening.bottom.profile, "GUID": opening.bottom.guid}
        if opening.left is not None:
            opening_dict["LEFT"] = {"PROFILE": opening.left.profile, "GUID": opening.left.guid}
        if opening.right is not None:
            opening_dict["RIGHT"] = {"PROFILE": opening.right.profile, "GUID": opening.right.guid}

        opening_dict["SPLIT DIRECTION"] = opening.split_direction

        opening_dict["CHILDREN"] = []

        if len(opening.children) == 0:
            opening_dict["TYPE"] = opening.type

        for child_opening in opening.children:
            opening_dict["CHILDREN"].append(get_children_data(child_opening))

        plane_dict["OPENING"] = opening_dict

        planes.append(plane_dict)

    data = {
        "GUID": guid,
        "HEIGHT": height,
        "WIDTH": width,
        "PLANE COUNT": plane_count,
        "PLANES": planes,
        "DELIVERY_NUMBER": delivery_number
    }

    # serialize the data to JSON format

    json_data = json.dumps(data, indent=4)

    # create a new file and write the JSON data to it
    file_path = json_folder + '/' + guid + '.json'

    with open(file_path, "w") as file:
        file.write(json_data)


from pygame.math import Vector2 as V2
from pygame.math import Vector3 as V3
import math
import random

def get_stl_lines(filename):
    f = open(filename, "r")
    lines = f.readlines()
    f.close()
    return [line.replace("\n","") for line in lines]

def get_vertex(line):
    if "vertex" in line:
        line = line.split(" ")
        x, y, z = line[-3],line[-2],line[-1]
        return V3(float(x),float(y),float(z)), "v"
    elif "normal" in line:
        line = line.split(" ")
        x, y, z = line[-3],line[-2],line[-1]
        return V3(float(x),float(y),float(z)), "n"
    return False

def get_triangles(lines):
    triangles = []
    k = 0
    vertices = [None, None, None]
    normal = None
    for line in lines:
        v = get_vertex(line)
        if v:
            if v[1] == "v":
                vertices[k] = v[0]
                k += 1
            elif v[1] == "n":
                normal = v[0]
        if k == 3:
            t = Triangle(vertices[0],vertices[1],vertices[2])
            t.n = normal
            triangles.append(t)
            assert t.n is not None
            k = 0
    assert k == 0
    return triangles

#les triangles font des references a des points, qui eux subissent les transfos!

class Triangle:

    def __init__(self, v1, v2, v3, color=None):
        self.v1 = v1
        self.v2 = v2
        self.v3 = v3
        self.n = None
        self.c = None
        self.color = color
        if color is None:
            self.color = V3(random.randint(0,255),random.randint(0,255),random.randint(0,255))
        self.ecolor = self.color
        self.neighs = []
        self.pd = None

    def compute_normal(self):
        e1 = self.v2 - self.v1
        e2 = self.v3 - self.v1
##        e3 = self.p3 - self.p2
        n1 = e1.cross(e2)
##        n2 = e1.cross(e3)
##        n3 = e2.cross(e3)
        #n1,n2,n3 yield the same result
        return n1

    def refresh_normal(self):
        self.n = self.compute_normal().normalize()

    def vertices(self):
        yield self.v1
        yield self.v2
        yield self.v3

##    def refresh_cd(self):
##        self.c = (self.v1 + self.v2 + self.v3)/3.
##        self.d = self.c.length_squared()

    def refresh_cd(self):
        self.c = (self.v1 + self.v2 + self.v3)/3.
        self.d = self.c.length_squared()

    def refresh_pd(self):
        self.pd = self.d
        counter = 1
        for buddy in self.neighs:
            a = self.n.angle_to(buddy.n)
            if a == 0:
                counter += 1
                self.pd += buddy.d
        self.pd /= counter

class Object3D:

    def __init__(self, filename):
        self.filename = filename
        self.lines = get_stl_lines(filename)
        self.triangles = get_triangles(self.lines)
        self.from_center = V3()
        vset = self.get_vertices_set()
        self.vertices = {val:V3(val) for val in vset}
        self.compactize()

    def compute_neighbours(self):
        for t1 in self.triangles:
            for t2 in self.triangles:
                if t1 != t2:
                    counter = 0
                    for v in t1.vertices():
                        if v == t2.v1:
                            counter += 1
                        elif v == t2.v2:
                            counter += 1
                        elif v == t2.v3:
                            counter += 1
                    if counter > 1:
                        t1.neighs.append(t2)

    def compactize(self):
        for t in self.triangles:
            t.v1 = self.vertices[tuple(t.v1)]
            t.v2 = self.vertices[tuple(t.v2)]
            t.v3 = self.vertices[tuple(t.v3)]

    def move(self, delta):
        for v in self.vertices.values():
            v += delta
        self.from_center += delta

    def rotate_x(self, angle, refresh=True):
        for v in self.vertices.values():
            v.rotate_x_ip(angle)
        if refresh:
            self.refresh_normals()

    def rotate_around_center_x(self, angle, refresh=True):
        tmp = V3(self.from_center)
        self.move(-tmp)
        self.rotate_x(angle, refresh)
        self.move(tmp)

    def rotate_y(self, angle, refresh=True):
        for v in self.vertices.values():
            v.rotate_y_ip(angle)
        if refresh:
            self.refresh_normals()

    def rotate_around_center_y(self, angle, refresh=True):
        tmp = V3(self.from_center)
        self.move(-tmp)
        self.rotate_y(angle, refresh)
        self.move(tmp)

    def rotate_z(self, angle, refresh=True):
        for v in self.vertices.values():
            v.rotate_z_ip(angle)
        if refresh:
            self.refresh_normals()

    def rotate_around_center_z(self, angle, refresh=True):
        tmp = V3(self.from_center)
        self.move(-tmp)
        self.rotate_z(angle, refresh)
        self.move(tmp)

##    def rotate_around_center(self, x,y,z):
##        tmp = V3(self.from_center)
##        self.move(-tmp)
##        for v in self.vertices.values():
##            v.rotate_x_ip(x)
##            v.rotate_y_ip(y)
##            v.rotate_z_ip(z)
##        self.move(tmp)
##        self.refresh_normals()

    def refresh_normals(self):
        for t in self.triangles:
            t.refresh_normal()

    def scale(self, factor): #account for this for rotations!
        for v in self.vertices.values():
            v *= factor
        self.refresh_normals()

    def refresh(self):
        for t in self.triangles:
            t.refresh_cd()
        for t in self.triangles:
            t.refresh_pd()
        self.triangles.sort(key=lambda x:x.pd, reverse=True)
##        self.triangles.sort(key=lambda x:x.d,reverse=True)

    def get_vertices_set(self):
        vset = set()
        for t in self.triangles:
            for v in[t.v1,t.v2,t.v3]:
                vset.add(tuple(v))
        return vset

class ManualObject3D(Object3D):

    def __init__(self, triangles):
        self.triangles = triangles
        self.from_center = V3()
        vset = self.get_vertices_set()
        self.vertices = {val:V3(val) for val in vset}
        self.compactize()


from django.db import models
from mongoengine import connect,Document,StringField,DynamicField,DateTimeField
from cassandra.cqlengine import columns
from django_cassandra_engine.models import DjangoCassandraModel

connect(host="mongodb://192.168.57.119:27017/session")

class PgSession(models.Model):
    class Meta:
        db_table = 'session'
    token = models.CharField(max_length=512,primary_key=True)
    data = models.JSONField()
    expired_at = models.DateTimeField()

class MySession(models.Model):
    class Meta:
        db_table = 'my_session'

    token = models.CharField(max_length=512,primary_key=True)
    data = models.CharField(max_length=8192)
    expired_at = models.DateTimeField()

class MgSession(Document):
    token = StringField(primary_key=True)
    data = DynamicField()
    expired_at = DateTimeField()

class CaSession(DjangoCassandraModel):
    token = columns.Text(primary_key=True)
    data = columns.Text()
    expired_at = columns.DateTime()


from datetime import datetime

from mongoengine import (
    CASCADE,
    DateTimeField,
    DynamicField,
    EmbeddedDocument,
    EmbeddedDocumentListField,
    EnumField,
    GenericReferenceField,
    ListField,
    ReferenceField,
    StringField,
)

from core.audit.enum import ActivityTypesEnum
from core.auth.models.account import Account
from core.auth.models.role import Role
from core.auth.models.user import User
from core.basemodel import BaseModel

class DataRecord(EmbeddedDocument):
    field = StringField()
    old = DynamicField()
    new = DynamicField()

class Audit(BaseModel):
    meta = {
        "collection": "audits",
        "indexes": [
            {
                "fields": ["$message", "$source_user_agent", "$source_address"],
                "default_language": "english",
            }
        ],
    }
    created_by = ReferenceField(User, required=True, reverse_delete_rule=CASCADE)
    time = DateTimeField(default=datetime.utcnow)
    model = GenericReferenceField(
        choices=[User, Role, Account]
    )
    activity = EnumField(ActivityTypesEnum)
    model_name = StringField()
    source_address = StringField()
    source_user_agent = StringField()
    message = StringField()
    # the primitive parameters will be interpolated to the message directly as it is not
    # possible to define union types that might be scalar or object in GraphQL
    params = ListField(
        GenericReferenceField(
            choices=[User, Role, Account]
        )
    )
    data = EmbeddedDocumentListField(DataRecord)


import os
from loguru import logger
import torch
import torch.nn as nn

from decalib.utils.lossfunc import VGGFace2Loss
from .utils.config import cfg
from .utils import util

class ConvBlock(nn.Module):
    def __init__(self, in_channel, out_channel, kernel_size=3, stride=1, padding=1, dilation=1, bias=True):
        super(ConvBlock, self).__init__()
        self.conv = nn.Conv2d(in_channel, out_channel, kernel_size, stride, padding, dilation, bias)
        self.bn = nn.BatchNorm2d(out_channel)
        self.relu = nn.ReLU(inplace=True)
    def forward(self, x):
        x = self.conv(x)
        x = self.bn(x)
        x = self.relu(x)
        return x

class Generator(nn.Module):
    '''
    Simple UNet Structures like:
        encoder: [16, 32, 32, 32]
        decoder: [32, 32, 32, 32, 32, 16, 16]
    '''
    def __init__(self, model_config):
        super(Generator, self).__init__()
        in_channel = model_config.in_channel
        out_channel = model_config.out_channel
        encoder_nf = model_config.encoder_nf
        decoder_nf = model_config.decoder_nf[:len(encoder_nf)]
        final_nf = model_config.decoder_nf[len(encoder_nf):]

        ## Encoder
        self.pooling = [nn.MaxPool2d(2) for _ in range(len(encoder_nf))]
        self.encoder = nn.ModuleList()
        prev_nf = in_channel
        for i in range(len(encoder_nf)):
            blk = ConvBlock(prev_nf, encoder_nf[i], bias=False)
            self.encoder.append(blk)
            prev_nf = encoder_nf[i]
        ## Decoder
        self.upsampling = [nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False) for _ in range(len(decoder_nf))]
        self.decoder = nn.ModuleList()
        for i in range(len(decoder_nf)):
            blk = ConvBlock(prev_nf, decoder_nf[i], bias=False)
            self.decoder.append(blk)
            prev_nf = decoder_nf[i] + encoder_nf[-i-1]
        ## Final
        self.final = nn.ModuleList()
        for i in range(len(final_nf)):
            blk = ConvBlock(prev_nf, final_nf[i], bias=False)
            self.final.append(blk)
            prev_nf = final_nf[i]
        self.final.append(nn.Conv2d(prev_nf, out_channel, 1, 1, 0, 1, 1, False))

    def forward(self, x):
        x_hist = [x]
        for i in range(len(self.encoder)):
            x = self.encoder[i](x)
            x_hist.append(x)
            x = self.pooling[i](x)
        for i in range(len(self.decoder)):
            x = self.decoder[i](x)
            x = self.upsampling[i](x)
            x = torch.cat([x, x_hist.pop()], dim=1)
        for i in range(len(self.final)):
            x = self.final[i](x)
        return x

class Discriminator(nn.Module):
    '''
    Input: VGG features of size
    '''
    def __init__(self, model_config):
        super(Discriminator, self).__init__()
        in_channel = model_config.in_channel
        self.fc = nn.Linear(in_channel, 1)

    def forward(self, x):
        return self.fc(x)

class Generator1D(nn.Module):
    '''
    Input: 2 1D tensor
    Output: 1 1D tensor
    '''
    def __init__(self, model_config):
        super(Generator1D, self).__init__()
        ldm_len, shp_len = model_config.in_channel
        hidden_channel = model_config.hidden_channel
        out_channel = shp_len
        self.ldm_fc = nn.Sequential(
            nn.Linear(ldm_len, hidden_channel[0]//2),
            nn.LeakyReLU(0.2, inplace=True),
        )
        self.shp_fc = nn.Sequential(
            nn.Linear(shp_len, hidden_channel[0]//2),
            nn.LeakyReLU(0.2, inplace=True),
        )
        self.fc = nn.ModuleList()
        for i in range(len(hidden_channel)):
            if i == 0:
                continue
            else:
                self.fc.append(nn.Sequential(
                    nn.Linear(hidden_channel[i-1], hidden_channel[i]),
                    nn.LeakyReLU(0.2, inplace=True),
                ))
        self.out_fc = nn.Linear(hidden_channel[-1], out_channel)

    def forward(self, ldm, shp):
        ldm = self.ldm_fc(ldm)
        shp = self.shp_fc(shp)
        x = torch.cat([ldm, shp], dim=1)
        for i in range(len(self.fc)):
            x = self.fc[i](x)
        x = self.out_fc(x)
        return x

class GAN1D(nn.Module):
    def __init__(self, model_config, deca = None):
        super(GAN1D, self).__init__()
        self.vgg = VGGFace2Loss(pretrained_model=model_config.fr_model_path).eval()
        # freeze vgg
        for param in self.vgg.parameters():
            param.requires_grad = False
        self.generator = Generator1D(model_config.generator)
        self.discriminator = Discriminator(model_config.discriminator)
        self.deca = deca
        self.ldm_idx = model_config.lmk_idx
        self.real_img_ldm = model_config.real_img_ldm
        self.cfg = model_config

    def load_chkp(self, path):
        model_chkp = path
        chkp = torch.load(model_chkp)
        cur_dict = self.model_dict()
        print(f'trained model found. load {model_chkp}')
        for k in chkp.keys():
            if k in cur_dict.keys():
                print('load module dict', k, 'from checkpoint')
                util.copy_state_dict(cur_dict[k], chkp[k])

    def model_dict(self):
        d =  dict(
            generator = self.generator.state_dict(),
            discriminator = self.discriminator.state_dict(),
        )
        d.update(self.deca.model_dict())
        return d

    def forward(self, batches):
        # self.deca.eval()
        # # freeze deca
        # for param in self.deca.parameters():
        #     param.requires_grad = False

        images = batches['images']
        batch_size = images.shape[0]
        # get camera, light, pose, expression, shape, texture
        codedict_in = self.deca.encode(images, use_detail=False)
        shape_in = codedict_in['shape']
        # real landmarks?
        if self.real_img_ldm:
            # Option 1: calculate landmarks in real settings
            if not self.cfg.normal_settings:
                opdict_in = self.deca.decode(codedict_in, rendering=True, vis_lmk=False, return_vis=False, use_detail=False)
                landmarks_in = opdict_in['verts'][:, self.ldm_idx, :]
                codedict_del = {}
                for k in codedict_in.keys():
                    if k not in ['shape', 'tex']:
                        codedict_del[k] = codedict_in[k]
                    else:
                        codedict_del[k] = codedict_in[k][list(range(batch_size//2, batch_size))+list(range(batch_size//2))]
                opdict_del = self.deca.decode(codedict_del, only_verts=True)
                landmarks_del = opdict_del['verts'][:, self.ldm_idx, :]
                delta_landmarks = landmarks_in - landmarks_del
            # Option 2: normalize settings
            else:
                codedict_norm = util.normalize_codedict(codedict_in.copy())
                opdict_norm = self.deca.decode(codedict_norm, rendering=True, vis_lmk=False, return_vis=False, use_detail=False)
                landmarks_in = opdict_norm['verts'][:, self.ldm_idx, :]
                landmarks_del = landmarks_in[list(range(batch_size//2, batch_size))+list(range(batch_size//2))]
                delta_landmarks = landmarks_in - landmarks_del
                opdict_in = opdict_norm
                codedict_in = codedict_norm
        # not supported yet
        else: delta_landmarks = batches['landmarks']

        img_in = opdict_in['rendered_images']
        # generate fake shape params
        shape_out = self.generator(delta_landmarks.view(batch_size,-1), \
            shape_in) + shape_in
        logdict = {
            'shape_out change max': (shape_out/shape_in-1).abs().max().item(),
            'shape_out change mean': (shape_out/shape_in-1).abs().mean().item(),
            'landmark_init_error': delta_landmarks.norm(dim=-1).mean().item(),
        }
        codedict_out = codedict_in.copy()
        codedict_out['shape'] = shape_out
        # use same texture as target images
        if self.cfg.use_target_tex:
            codedict_out['tex'] = codedict_in['tex'][list(range(batch_size//2, batch_size))+list(range(batch_size//2))]
        opdict_out = self.deca.decode(codedict_out, rendering=True, vis_lmk=False, return_vis=False, use_detail=False)
        img_out = opdict_out['rendered_images']
        landmarks_out = opdict_out['verts'][:, self.ldm_idx, :]
        logdict.update({
            'landmark_dist': ((landmarks_in-landmarks_out-delta_landmarks)).norm(dim=-1).mean().item()
        })
        # render fake images (everything but the tex and shape are as original images)
        # codedict_out = {}
        # for k in codedict_in.keys():
        #     if k not in ['shape', 'tex']:
        #         codedict_out[k] = codedict_in[k].view(2, batch_size//2, -1)[[1,0]].flatten(0,1)
        # codedict_out['tex'] = codedict_in['tex']

        # vgg features
        imgs = torch.cat([img_in, img_out], dim=0)
        # F = self.vgg.forward_features(imgs)
        # F_real, F_gen = F[:batch_size], F[batch_size:]
        F_real, F_gen = self.vgg.forward_features(img_in), self.vgg.forward_features(img_out)
        visdict = {
            'images': imgs[:batch_size],
            'gen_images': imgs[batch_size:],
            'verts': opdict_in['verts'],
            'gen_verts': opdict_out['verts'],
            'trans_verts': opdict_in['trans_verts'],
            'gen_trans_verts': opdict_out['trans_verts'],
            'lmk': opdict_in['trans_verts'][:, self.ldm_idx, :],
            'lmk_gen': opdict_out['trans_verts'][:, self.ldm_idx, :],
        }

        return F_real, F_gen, landmarks_in, landmarks_out, delta_landmarks, visdict, logdict


import numpy as np
import cv2

#For more info about contours: https://docs.opencv.org/4.5.3/d3/d05/tutorial_py_table_of_contents_contours.html

im = np.zeros(shape =(400,400)).astype('uint8')
polygons = [np.array([[20,20],[120,50],[30,80]]),
            np.array([[200,200],[200,350],[350,350], [350,200] ]),
            np.array([[170, 350], [60, 170], [130, 140]])]
colors = [255,255,255]
cv2.fillPoly(im,polygons,colors)
im[250:340,250:340] = 0
im[270:300,290:320] = 255

contours,hierarchy = cv2.findContours(im, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
#try mode cv2.RETR_EXTERNAL and cv2.RETR_TREE
#try method cv2.CHAIN_APPROX_NONE and cv2.CHAIN_APPROX_SIMPLE

print('Types of contours:', type(contours))
print('Length of contours:', len(contours))

n = 0
print('Types of each contour:', type(contours[n]))
print('Shape of each contour:', contours[n].shape)
print('Each contour value (list of points):\n',contours[n])

out = np.zeros(shape =(400,400)).astype('uint8')
for pos in contours[n]:
    out[pos[0,1],pos[0,0]] = 255

#print('Hierarchy:\n', 'Left Right Child Parent\n', hierarchy)

cv2.imshow('im', im)
cv2.imshow('out', out)

cv2.waitKey()


"""
It is responsible for recommendation functionality, as also
providing the available filters that go along with the result

...

Methods
-------
get_documents(query_base_request, documents_deploy_request)
    Disables recommendation resource creation calls
"""
import cProfile
import pstats
import io
import os
import sys
import logging
from logging.handlers import TimedRotatingFileHandler
from time import perf_counter
from config import get_current_config

LEVEL = get_current_config().LOG_LEVEL

FORMATTER = logging.Formatter(
    "%(asctime)s - %(name)s - %(levelname)s - %(message)s")

def create_console_handler():
    console_handler = logging.StreamHandler(sys.stdout)
    console_handler.setFormatter(FORMATTER)
    return console_handler

def create_file_handler(file_name: str):
    log_file_name = f'{file_name}.log'
    log_file_path = os.path.join(get_current_config().LOGS_PATH, log_file_name)

    try:
        file = open(log_file_path, 'r+')
    except FileNotFoundError:
        if not os.path.exists(os.path.join(get_current_config().LOGS_PATH)):
            os.makedirs(os.path.join(get_current_config().LOGS_PATH))
        file = open(log_file_path, 'w+')

    file.close()
    file_handler = TimedRotatingFileHandler(
        log_file_path, when='midnight', encoding='UTF-8')
    file_handler.setFormatter(FORMATTER)
    return file_handler

def get_logger(name='default', file_name='log'):
    log = logging.getLogger(name)
    log.setLevel(LEVEL)
    log.addHandler(create_console_handler())
    log.addHandler(create_file_handler(file_name))
    log.propagate = False
    return log

class Logger:
    def __init__(self):
        self.log = get_logger()
        self.__start = 0

    def info(self, msg):
        self.log.info(msg)

    def exception(self, error):
        self.log.exception(error)

class Chronometer:

    def __init__(self):
        self.__start = 0

    def start(self):
        self.__start = perf_counter()

    def end(self):
        result = perf_counter() - self.__start
        self.__start = 0
        return result

def profile(fnc):
    def inner(*arg, **kwargs):
        pr = cProfile.Profile()
        pr.enable()
        retval = fnc(*arg, **kwargs)
        pr.disable()
        s = io.StringIO()
        sortby = 'cumulative'
        ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
        ps.print_stats()
        log = get_logger()
        log.info(s.getvalue())
        return retval
    return inner

chrono = Chronometer()
logger = Logger()


import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers import Conv2D, LeakyReLU, Flatten
from tensorflow.keras.layers import Dense, Conv2DTranspose
from tensorflow.keras.layers import Reshape, Activation
from tensorflow.keras import backend as K
from tensorflow.keras.callbacks import ModelCheckpoint

import os

# run params
SAVE_FOLDER = 'data'

if not os.path.exists(SAVE_FOLDER):
    os.mkdir(SAVE_FOLDER)
    os.mkdir(os.path.join(SAVE_FOLDER, 'weights'))

MODE =  'build' #'load' #
#MODE =  'load'

z_dim = 2

encoder_input = keras.Input(shape=(28, 28, 1), name='encoder_input')
x = Conv2D(32, 3, strides=1, padding="same", name='encoder_conv_0')(encoder_input)
x = LeakyReLU()(x)
x = Conv2D(64, 3, strides=2, padding="same", name='encoder_conv_1')(x)
x = LeakyReLU()(x)
x = Conv2D(64, 3, strides=2, padding="same", name='encoder_conv_2')(x)
x = LeakyReLU()(x)
x = Conv2D(64, 3, strides=1, padding="same", name='encoder_conv_3')(x)
x = LeakyReLU()(x)
shape_before_flattening = K.int_shape(x)[1:]
x = Flatten()(x)
encoder_output= Dense(z_dim, name='encoder_output')(x)
encoder = keras.Model(encoder_input, encoder_output, name='encoder')

decoder_input = keras.Input(shape=(z_dim,), name='decoder_input')
x = Dense(np.prod(shape_before_flattening))(decoder_input)
x = Reshape(shape_before_flattening)(x)
x = layers.Conv2DTranspose(64, 3, strides=1, padding="same", name='decoder_conv_t0')(x)
x = LeakyReLU()(x)
x = layers.Conv2DTranspose(64, 3, strides=2, padding="same", name='decoder_conv_t1')(x)
x = LeakyReLU()(x)
x = layers.Conv2DTranspose(32, 3, strides=2, padding="same", name='decoder_conv_t2')(x)
x = LeakyReLU()(x)
x = layers.Conv2DTranspose(1, 3, strides=1, padding="same", name='decoder_conv_t3')(x)
decoder_output = Activation('sigmoid')(x)
decoder = keras.Model(decoder_input, decoder_output, name="decoder")

class AutoEncoder(keras.Model):
    def __init__(self, encoder, decoder, **kwargs):
        super(AutoEncoder, self).__init__(**kwargs)
        self.encoder = encoder
        self.decoder = decoder
    def call(self,inputs):
        latent = self.encoder(inputs)
        return self.decoder(latent)

@tf.function
def r_loss(y_true, y_pred):
    return K.mean(K.square(y_true - y_pred), axis = [1,2,3])

LEARNING_RATE = 0.0005
BATCH_SIZE = 32
INITIAL_EPOCH = 0
EPOCHS = 10

save_folder = os.path.join(SAVE_FOLDER, 'weights')

ae = AutoEncoder(encoder, decoder)
ae.compile(optimizer=keras.optimizers.Adam(lr=LEARNING_RATE), loss=r_loss)

if MODE == 'build':
    x = np.random.rand(1,28,28,1)
    ae.predict(x)
    ae.save(save_folder)
else:
    #ae = keras.models.load_model(save_folder, custom_objects={'r_loss': r_loss})
    ae.load_weights(save_folder+'/'+'checkpoint')

# mnist 데이터 읽어오기
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
#mnist_digits = np.concatenate([x_train, x_test], axis=0)
#mnist_digits = x_train[:1000]
#print(mnist_digits.shape)
#print(mnist_digits.dtype)
# bad
#mnist_digits = np.expand_dims(mnist_digits, -1).astype("float32") / 255
# bad
#mnist_digits = np.expand_dims(mnist_digits, -1).astype("float64") / 255
#mnist_digits = mnist_digits/255.
#mnist_digits = np.expand_dims(mnist_digits, -1)
#mnist_digits = mnist_digits.reshape((-1, 28, 28, 1))
#print(mnist_digits.shape)
#print(mnist_digits.dtype)

print(x_train.shape)
print(x_train.dtype)
x_train = x_train/255.
x_test = x_test/255.
#x_train = x_train.reshape((60000, 28, 28, 1))
#x_train = x_train.reshape((-1, 28, 28, 1))
#x_train = x_train.reshape(x_train.shape+(1))
x_train = np.expand_dims(x_train, -1)
x_test = np.expand_dims(x_test, -1)
print(x_train.shape)
print(x_train.dtype)
import matplotlib.pyplot as plt
'''
plt.imshow(x_train[999].squeeze(), cmap='gray_r')
plt.colorbar()
plt.show()
plt.imshow(mnist_digits[999].squeeze(), cmap='gray_r')
plt.colorbar()
plt.show()
'''

# 매 epoch마다 모델 weights 저장
#lr_sched = step_decay_schedule(initial_lr=self.learning_rate, decay_factor=lr_decay, step_size=1)
#checkpoint = ModelCheckpoint(save_folder+'/'+'checkpoint', save_weights_only = False, verbose=1)
#callbacks_list = [checkpoint]

callbacks_list = []
ae.fit(x_train, x_train, epochs=EPOCHS, batch_size=BATCH_SIZE, callbacks=callbacks_list)
ae.save(save_folder)

import matplotlib.pyplot as plt
n_to_show = 10
example_idx = np.random.choice(range(len(x_test)), n_to_show)
example_images = x_test[example_idx]

z_points = ae.encoder.predict(example_images)

reconst_images = ae.decoder.predict(z_points)

fig = plt.figure(figsize=(15, 3))
fig.subplots_adjust(hspace=0.4, wspace=0.4)

for i in range(n_to_show):
    img = example_images[i].squeeze()
    ax = fig.add_subplot(2, n_to_show, i+1)
    ax.axis('off')
    ax.text(0.5, -0.35, str(np.round(z_points[i],1)), fontsize=10, ha='center', transform=ax.transAxes)
    ax.imshow(img, cmap='gray_r')

for i in range(n_to_show):
    img = reconst_images[i].squeeze()
    ax = fig.add_subplot(2, n_to_show, i+n_to_show+1)
    ax.axis('off')
    ax.imshow(img, cmap='binary')

plt.show()
'''
'''


from __future__ import print_function
from instantiate import gen_inst_task
from pddlstream.algorithms.downward import get_problem, task_from_domain_problem, get_cost_scale, \
    scale_cost, fd_from_fact, make_domain, make_predicate, evaluation_from_fd, fact_from_fd, \
    get_preimage_literals_with_idxes
from pddlstream.algorithms.instantiate_task import instantiate_task, sas_from_instantiated
from pddlstream.algorithms.scheduling.apply_fluents import convert_fluent_streams
from pddlstream.algorithms.scheduling.negative import convert_negative, recover_preimage_plan
from pddlstream.algorithms.scheduling.postprocess import postprocess_stream_plan
from pddlstream.algorithms.scheduling.recover_axioms import recover_axioms_plans
from pddlstream.algorithms.scheduling.recover_functions import compute_function_plan
from pddlstream.algorithms.scheduling.recover_streams import extract_stream_plan
from etamp.pddlstream.algorithms.scheduling.utils import get_instance_facts
from pddlstream.algorithms.search import solve_from_task

from pddlstream.language.constants import Not, get_prefix, EQ, pAction, pA_info, pAtom
from pddlstream.language.conversion import transform_plan_args, transform_action_args, pddl2obj
from pddlstream.language.conversion import objects_from_evaluations
from pddlstream.language.optimizer import UNSATISFIABLE

from pddlstream.utils import Verbose, flatten
from itertools import product
from copy import copy
from collections import namedtuple
from etamp.pddlstream.algorithms.instantiation import Instantiator
from pddlstream.algorithms.scheduling.recover_streams import get_achieving_streams, \
    expand_evals_by_results
from pddlstream.algorithms.constraints import PlanConstraints, WILD
from pddlstream.language.constants import FAILED_LEVEL, INFEASIBLE_EVAL, is_plan
from pddlstream.language.conversion import evaluation_from_fact, substitute_expression
from pddlstream.language.function import FunctionResult, Function
from pddlstream.language.stream import StreamResult
from pddlstream.language.statistics import check_effort, compute_stream_plan_effort
from pddlstream.language.object import Object, OptimisticObject
from pddlstream.utils import safe_zip, get_mapping, implies, INF
from pddlstream.algorithms.scheduling.skeleton_batch import SkeletonBatch

CONSTRAIN_STREAMS = True
CONSTRAIN_PLANS = True  # False  True
MAX_DEPTH = INF  # 1 | INF

PREDICATE_ORDER = "_applicable"
PARA_PROPO = '_p'
TYPE_PROPO = 'propo_action'

NT_Domain = namedtuple('NT_Domain',
                       ['name', 'requirements', 'types', 'type_to_constants',
                        'predicates', 'predicate_to_typeList', 'derived', 'action'])
NT_Problem = namedtuple('NT_Problem',
                        ['name', 'domain', 'type_to_objects', 'init', 'goal'])
NT_Action = namedtuple('NT_Action', ['name', 'parameters', 'precondition', 'effect'])
NT_Stream = namedtuple('NT_Stream', ['name', 'inputs', 'domain', 'outputs', 'certified'])

def add_stream_effort_to_action(node_from_atom, instantiated_task, effort_weight, **kwargs):
    """
    Count the estimated stream_cost on its corresponding actions.
    """
    if effort_weight is None:
        return

    for action in instantiated_task.fluent_actions:
        # TODO: prune stream actions here?
        # TODO: round each effort individually to penalize multiple streams
        facts = get_instance_facts(action, node_from_atom)
        # effort = COMBINE_OP([0] + [node_from_atom[fact].effort for fact in facts])
        stream_plan = []
        extract_stream_plan(node_from_atom, facts, stream_plan)

        estimated_effort = compute_stream_plan_effort(stream_plan, **kwargs)
        stream_cost = scale_cost(effort_weight * estimated_effort)
        action.cost += stream_cost

##################################################

def rename_instantiated_actions(instantiated):
    # TODO: rename SAS instead?
    actions = instantiated.fluent_actions[:]
    renamed_actions = []
    action_from_name = {}
    for i, action in enumerate(actions):
        renamed_actions.append(copy(action))
        renamed_name = 'a{}'.format(i)
        renamed_actions[-1].name = '({})'.format(renamed_name)
        action_from_name[renamed_name] = action  # Change reachable_action_params?
    instantiated.fluent_actions[:] = renamed_actions
    return action_from_name

# def pddl_from_instance(instance):
#     action = instance.action
#     args = [instance.var_mapping[para.name]
#             for para in action.parameters[:action.num_external_parameters]]
#     pa_info = pA_info(action.info.cost_fn, None)
#     return pAction(action.name, args, pa_info)

def create_pAtom(atom):
    return pAtom(atom.predicate, atom.args)

def propoAction2pddl(propo_action):
    """
    Transform a propositional action into a namedtuple Action.
    """
    action = propo_action.action
    full_args = [propo_action.var_mapping[para.name]
                 for para in action.parameters[:action.num_external_parameters]]

    pa_info = pA_info(action.info.cost_fn, [create_pAtom(e.effect) for e in propo_action.add_effects], propo_action)
    pa_info = pA_info(action.info.cost_fn, [create_pAtom(e.effect) for e in propo_action.add_effects], None)

    return pAction(action.name, full_args, pa_info)

##################################################

def get_plan_cost(action_plan, cost_from_action):
    """
    Sum up the cost of all actions in the plan.
    """
    if action_plan is None:
        return INF
    # return sum([0.] + [instance.cost for instance in action_plan])
    scaled_cost = sum([0.] + [cost_from_action[action]
                              for action in action_plan])
    return scaled_cost / get_cost_scale()

def instantiate_optimizer_axioms(instantiated, domain, results):
    # Needed for instantiating axioms before adding stream action effects
    # Otherwise, FastDownward will prune these unreachable axioms
    # TODO: compute this first and then apply the eager actions
    stream_init = {fd_from_fact(result.stream_fact)
                   for result in results if isinstance(result, StreamResult)}
    evaluations = list(map(evaluation_from_fd, stream_init | instantiated.atoms))
    temp_domain = make_domain(predicates=[make_predicate(UNSATISFIABLE, [])],
                              axioms=[ax for ax in domain.axioms if ax.name == UNSATISFIABLE])
    temp_problem = get_problem(evaluations, Not((UNSATISFIABLE,)), temp_domain)
    # TODO: UNSATISFIABLE might be in atoms making the goal always infeasible
    with Verbose():
        # TODO: the FastDownward instantiation will prune static preconditions
        new_instantiated = instantiate_task(task_from_domain_problem(temp_domain, temp_problem),
                                            check_infeasible=False, prune_static=False)
        assert new_instantiated is not None
    instantiated.axioms.extend(new_instantiated.axioms)
    instantiated.atoms.update(new_instantiated.atoms)

##################################################

def retrace_stream_plan(original_evaluation, evaluation, optms_evaluation, goal_expression, domain,
                        dict_fact_node, fluent_action_plan, list_fluent_axiom_plan,
                        negative_predicates, replan_step):
    original_task = task_from_domain_problem(domain, get_problem(original_evaluation, goal_expression, domain))
    real_task = task_from_domain_problem(domain, get_problem(evaluation, goal_expression, domain))
    optms_task = task_from_domain_problem(domain,
                                          get_problem(optms_evaluation, goal_expression, domain))  # for the solver

    negname_to_pred = {predicate.name: predicate
                       for predicate in negative_predicates}

    list_step_inits, preimage_plan = recover_preimage_plan(real_task, optms_task,
                                                           list_fluent_axiom_plan, fluent_action_plan,
                                                           negname_to_pred)

    function_plan = compute_function_plan(optms_evaluation, fluent_action_plan)

    literal_to_steps = get_preimage_literals_with_idxes(preimage_plan, [])  # of the full preimage.
    # Full preimage literals required by the plan.
    full_preimage = set(literal_to_steps)
    # Preimage literals to come from streams.
    stream_preimage = full_preimage - set(original_task.init)
    negative_preimage = set([literal for literal in stream_preimage
                             if literal.predicate in negname_to_pred])
    positive_preimage = stream_preimage - negative_preimage

    function_plan.update(
        convert_negative(negative_preimage, negname_to_pred, literal_to_steps, list_step_inits))

    # A mapping from a non-negative preimage fact (from Atom only) to positions where it arises.
    dict_fact_steps = {fact_from_fd(literal): literal_to_steps[literal]
                       for literal in positive_preimage
                       if not literal.negated}
    target_facts = {fact for fact in dict_fact_steps.keys()
                    if get_prefix(fact) != EQ}

    fact_to_minIdx = {fact: min(dict_fact_steps[fact])
                      for fact in target_facts}

    stream_to_minIdx = {}
    stream_plan = []  # empty container
    """stream_plan is recursively extracted to fulfill target facts according to dict_fact_node.
       Streams are basically used to fulfill the static facts of the (full) action_plan.
       ALL streams even for previously added facts are included."""
    extract_stream_plan(dict_fact_node, target_facts, stream_plan,
                        fact_to_minIdx, stream_to_minIdx)

    """evaluation:            steam_plan将仅包括与此次optimistic facts有关的streamResults;
       original_evaluation:   steam_plan将包括所有所依赖streamResults(包括之前生成的)"""
    post_stream_plan = postprocess_stream_plan(original_evaluation, domain, stream_plan, target_facts)

    applied_stream_plan = []
    step_to_actions = {}
    for result in (post_stream_plan + list(function_plan)):
        if (result.opt_index != 0) or (stream_to_minIdx.get(result, 0) < replan_step):
            """
            If the (optimistic) result is not unique, or its step index < replan_step,,
            then count it as an item of the stream plan.
            """
            applied_stream_plan.append(result)
        else:
            """
            If the result is unique, and its step index >= replan_step,
            then generate an action out of it. (Currently not used.)
            """
            step_to_actions.setdefault(stream_to_minIdx[result], []).append(result.get_action())
            assert len(step_to_actions) == 0

    # The stream plan with its streamResults grouped into static(s) and fluent(f)
    sf_stream_plan = convert_fluent_streams(applied_stream_plan, list_step_inits,
                                            fluent_action_plan, dict_fact_steps, dict_fact_node)
    # sf_stream_plan =applied_stream_plan

    """Generate an action plan, based on the fluent propositional action plan."""
    for index, fluent_action in enumerate(fluent_action_plan):
        paction = propoAction2pddl(fluent_action)
        paction_with_obj = transform_action_args(paction, pddl2obj)
        step_to_actions.setdefault(index, []).append(paction_with_obj)

    action_plan = list(flatten(step_to_actions[idx]
                               for idx in sorted(step_to_actions)))

    return sf_stream_plan, action_plan

##################################################

def solve_optimistic_sequential(instantiated_task, debug=False, max_cost=INF, **search_kwargs):
    action_from_name = rename_instantiated_actions(instantiated_task)
    with Verbose(debug):
        sas_task = sas_from_instantiated(instantiated_task)
        sas_task.metric = True

    # TODO: apply renaming to hierarchy as well
    # solve_from_task | serialized_solve_from_task | abstrips_solve_from_task | abstrips_solve_from_task_sequential
    renamed_plan, _ = solve_from_task(sas_task, debug=False,
                                      max_cost=max_cost, **search_kwargs)
    if renamed_plan is None:
        return None
    action_instances = [action_from_name[name]
                        for name, _, _ in renamed_plan]
    return action_instances

def is_refined(stream_plan):
    """
    This means that the results in stream_plan are refined,
    i.e., their input_objects are remapped to specifically adapt a existing plan.
    """
    # TODO: lazily expand the shared objects in some cases to prevent increase in size
    if not is_plan(stream_plan):
        return True
    # TODO: some of these opt_index equal None
    return all((result.opt_index is None) or (result.opt_index == 0)
               for result in stream_plan)

##################################################

def prune_high_effort_streams(streams, max_effort=INF, **effort_args):
    # TODO: convert streams to test streams with extremely high effort
    low_effort_streams = []
    for stream in streams:
        effort = stream.get_effort(**effort_args)
        if isinstance(stream, Function) or check_effort(effort, max_effort):
            low_effort_streams.append(stream)
    return low_effort_streams

def apply_streams(evaluations, streams, level_limit, **effort_args):
    pruned_streams = prune_high_effort_streams(streams, **effort_args)
    instantiator = Instantiator(pruned_streams)
    for evaluation, node in evaluations.items():
        if node.complexity <= level_limit:
            instantiator.add_atom(evaluation, node.complexity)
    optms_results = []

    while instantiator and (instantiator.min_level() <= level_limit):
        instance = instantiator.pop_stream()
        instance.opt_index = 0
        results = instance.next_optimistic()  # depending on if the instance is marked as refined, if it is, get an unique optimistic object

        for result in results:
            new_facts = False
            level = instantiator.compute_complexity(instance)
            result.level = level

            for fact in result.get_certified():
                new_facts |= instantiator.add_atom(evaluation_from_fact(fact), level)
            if isinstance(result, FunctionResult) or new_facts:
                # assert not isinstance(result, FunctionResult)
                optms_results.append(result)

        # TODO: instantiate and solve to avoid repeated work
    """Indicates nothing left in the queue.
       So higher level_limit will not enrich the optms results.
       Implies that the input evaluations are probably insufficient."""
    exhausted = not instantiator

    return optms_results, exhausted

##################################################

def is_plan(plan):
    """Return True
       if the plan is neither FAILED nor INFEASIBLE"""
    return plan not in [None, False]

def is_plan_certified(stream_plan):
    if stream_plan is None:
        return None

    # if all results in the stream plan is verified (optms_index=None or 0), then the plan is certified
    r = all((result.opt_index is None) or (result.opt_index == 0)
            for result in stream_plan)
    return r

def optimistic_stream_instantiation(instance, obj_mappings, evaluations, optms_evaluations, only_immediate=False):
    """
    Generating new instances by updating the input objects according to the sequence of stream_plan.
    For the new instance, reduce their optms_index and thereby make them more unique.
    They will be saved into their parent streams, and available for next Stream.get_instance()
    """
    new_instances = []
    for input_combo in product(*[obj_mappings.get(i, [i]) for i in instance.input_objects]):
        mapping = get_mapping(instance.input_objects, input_combo)
        domain_evaluations = set(map(evaluation_from_fact, substitute_expression(instance.get_domain(), mapping)))
        if domain_evaluations <= optms_evaluations:
            new_instance = instance.external.get_instance(input_combo)
            if (new_instance.opt_index != 0) and implies(only_immediate, domain_evaluations <= evaluations):
                # always -1
                new_instance.opt_index -= 1
            new_instances.append(new_instance)
    return new_instances

def optimistic_stream_evaluation(evaluations, stream_plan):
    """
    Generate new unique stream_results according to the new stream_instances from the given stream_plan.
    Those results are all related to the stream_plan.
    """
    evaluations = set(evaluations)  # For subset testing
    optms_evaluations = set(evaluations)
    related_results = []
    obj_mappings = {}  # the mapping from original outputs (optimistic) to stream-generated outputs
    active_streams = [s for s in stream_plan if not s.redundant]
    redundant_streams = [s for s in stream_plan if s.redundant]
    for result in active_streams:  # TODO: just refine the first step of the plan
        # unique instance
        new_instances = optimistic_stream_instantiation(result.instance, obj_mappings,
                                                        evaluations, optms_evaluations)
        for new_instance in new_instances:
            for new_result in new_instance.next_optimistic():
                optms_evaluations.update(map(evaluation_from_fact, new_result.get_certified()))
                new_result.level = result.level
                related_results.append(new_result)
                if isinstance(new_result, StreamResult):  # Could not add if same value
                    for old_obj, new_obj in safe_zip(result.output_objects, new_result.output_objects):
                        obj_mappings.setdefault(old_obj, []).append(new_obj)
    related_results.extend(redundant_streams)
    return related_results, obj_mappings

def compute_function_results(evaluations, related_results, externals, complexity_limit, **effort_args):
    # functions = list(filter(lambda s: type(s) is Function, externals))
    functions = [e for e in externals if type(e) is Function]
    extended_evaluations = expand_evals_by_results(evaluations, related_results)
    """Only functions are applied."""
    new_results, _ = apply_streams(extended_evaluations, functions, complexity_limit, **effort_args)
    return related_results + new_results

def compute_plan_constraints(action_plan, obj_mappings):
    """
    Generate an action plan with updated argument objects.
    And further generate a plan constraint.
    """
    new_plan = []
    groups = {old: news for old, news
              in obj_mappings.items() if len(news) != 1}
    for name, arg_tuple, _ in action_plan:
        new_arg_tuple = []
        for arg in arg_tuple:
            if isinstance(arg, Object):
                """For real object, arg remains."""
                new_arg_tuple.append(arg)
            elif isinstance(arg, OptimisticObject):
                assert obj_mappings.get(arg, [])
                if len(obj_mappings[arg]) == 1:
                    """For optm object with single mapping, arg updates."""
                    new_arg_tuple.append(obj_mappings[arg][0])
                else:
                    """For optm object with multiple mappings, arg remains/is marked as WILD."""
                    new_arg_tuple.append(WILD)
                    # new_arg_tuple.append(arg)
            else:
                raise ValueError(arg)
        new_plan.append((name, new_arg_tuple))
    # exact=False because we might need new actions
    return PlanConstraints(plans=[new_plan], groups=groups, exact=False, max_cost=INF)

def solve_and_refine(evaluations, externals, optms_results, optms_plan_solver, level_limit,
                     depth, constraints, **effort_args):
    """
    Plan refinement iteration
    """

    if MAX_DEPTH <= depth:
        return None, None, INF, depth

    stream_plan, action_plan, estimated_cost = optms_plan_solver(evaluations, optms_results,
                                                                 constraints)
    if estimated_cost == 0.514:
        return stream_plan, action_plan, estimated_cost, depth

    if not is_plan(action_plan):
        """Case1: None or False."""
        return stream_plan, action_plan, estimated_cost, depth

    if is_refined(stream_plan):
        """Case2: Already refined."""
        return stream_plan, action_plan, estimated_cost, depth

    related_results, obj_mappings = optimistic_stream_evaluation(evaluations, stream_plan)

    # next_results = compute_function_results(evaluations, related_results, externals, level_limit, **effort_args)

    next_constraints = compute_plan_constraints(action_plan, obj_mappings)

    """Re-run the plan solver with streams having refined stream instances."""
    return solve_and_refine(evaluations, externals, related_results, optms_plan_solver, level_limit,
                            depth + 1, next_constraints, **effort_args)

def plan_action_plan_stream(optms_plan_solver, evaluations, externals, level_limit, **effort_args):
    # externals = (streams + functions + optimizers)
    # Previously didn't have unique optimistic objects that could be constructed at arbitrary depths
    # evals = {e: n for e, n in evaluations.items() if n.complexity <= level_limit}
    evals = {e: n for e, n in evaluations.items()}
    assert len(evals) == len(evaluations)

    num_iterations = 0
    while True:
        num_iterations += 1

        optms_results, exhausted = apply_streams(evals, externals, level_limit, **effort_args)

        stream_plan, action_plan, cost, final_depth = solve_and_refine(evals, externals, optms_results.copy(),
                                                                       optms_plan_solver, level_limit,
                                                                       depth=0, constraints=None,
                                                                       **effort_args)
        print('  Attempt: {} | Results: {} | Depth: {} | Success: {}'.format(
            num_iterations, len(optms_results), final_depth, is_plan(action_plan)))

        if cost == 0.514:
            return stream_plan, action_plan, cost, exhausted

        if is_plan(action_plan):
            """
            Case2:
            """
            return stream_plan, action_plan, cost, exhausted
        """
        A: (final_depth==0 and not is_plan) concludes that at this level the solver cannot find any plausible plans.
           depth is the number of the refinement iteration.
        """
        if final_depth == 0:
            status = INFEASIBLE_EVAL if exhausted else FAILED_LEVEL
            """
            Case1:
            1) if exhausted, the search is failed due to the insufficiency of initial evaluations. -> INFEASIBLE
            2) if not, the search is failed due to the level limitation. -> FAILED
            """
            return status, status, cost, exhausted
        """
         B: Retry at current level
         """

class OptimisticPlanSolver(object):

    def __init__(self, goal_exp, domain, negative, search_kwargs, original_evaluation,
                 replan_actions=set(), max_effort=INF, effort_weight=None, dict_optms_cost={},
                 max_cost=INF):
        self.goal_exp = goal_exp
        self.domain = domain
        self.negative = negative  # negative predicates from the problem
        self.effort_weight = effort_weight
        self.max_effort = max_effort
        self.search_kwargs = search_kwargs
        self.replan_actions = replan_actions
        self.dict_optms_cost = dict_optms_cost
        self.original_evaluation = original_evaluation
        self.max_cost = max_cost

    def _planner_core(self, evaluations, optms_results, domain, goal_exp, max_cost):
        """
        Generate an action_plan with its stream_plan
        based on initial evaluations and all optimistic facts.
        """
        r_problem = get_problem(evaluations, goal_exp, domain)  # begin_metric
        original_init = r_problem.init
        original_goal = r_problem.goal

        dict_fact_node = get_achieving_streams(evaluations, optms_results,
                                               max_effort=self.max_effort)

        optms_evaluations = {evaluation_from_fact(f): n.result for f, n in dict_fact_node.items()}

        problem = get_problem(optms_evaluations, goal_exp, domain)  # begin_metric

        optms_init = problem.init

        if len(problem.init) > 300:
            sk_batch = SkeletonBatch('domain_p.pddl', original_init, original_goal, optms_results, optms_init)
            op_plan, results_for_step = sk_batch.get_next_operatorPlan()
            if op_plan:
                print('++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')
                return op_plan, results_for_step, 0.514

        pddl_task = task_from_domain_problem(domain, problem)

        # Mapping from string in pddl to Object/OptimiticObject
        pddl_to_obj = {obj.pddl: obj
                       for obj in objects_from_evaluations(optms_evaluations)}

        instantiated_task = gen_inst_task(pddl_task, pddl_to_obj)

        if instantiated_task is None:
            return None, None, INF

        # Estimated cost for actions.
        add_stream_effort_to_action(dict_fact_node, instantiated_task, self.effort_weight)
        action_to_EstmCost = {action: action.cost
                              for action in instantiated_task.fluent_actions}

        fluent_action_plan = solve_optimistic_sequential(instantiated_task,
                                                         max_cost=max_cost, **self.search_kwargs)

        # _plan = solve_optimistic_symk('found_plans/sas_plan.1')
        #
        # prop_action_plan = inst_task(_plan, pddl_task, pddl_to_obj)

        if fluent_action_plan is not None:
            print('')

        estimated_cost = get_plan_cost(fluent_action_plan, action_to_EstmCost)

        if fluent_action_plan is None:
            return None, None, estimated_cost

        list_fluent_axiom_plan = recover_axioms_plans(instantiated_task, fluent_action_plan)

        pddl_plan = transform_plan_args(map(propoAction2pddl, fluent_action_plan), pddl2obj)
        replan_step = min([step + 1 for step, action in enumerate(pddl_plan)
                           if action.name in self.replan_actions] or [len(pddl_plan)])

        stream_plan, action_plan = retrace_stream_plan(self.original_evaluation, evaluations, optms_evaluations,
                                                       goal_exp, domain, dict_fact_node, fluent_action_plan,
                                                       list_fluent_axiom_plan, self.negative, replan_step)

        return stream_plan, action_plan, estimated_cost

    def __call__(self, evaluations, optms_results, constraints):

        result = self._planner_core(evaluations, optms_results, self.domain, self.goal_exp, self.max_cost)

        return result


import numpy as np
import matplotlib.pyplot as plt

def tanh_test(x, x0, L):
    return np.tanh((x-x0)/L)

if __name__ == "__main__":
    x = np.linspace(0, 10, 1000)
    LL = [0.01, 0.05, 0.1, 1]
    x0 = 5
    for L in LL:
        y = tanh_test(x, x0, L)
        plt.plot(x, y, label=f"L={L}")
    plt.legend()
    plt.show()


import sys, time, argparse
import subprocess, os.path, pathlib
import math, shutil
from xml.etree.ElementInclude import default_loader

Description = """FASTQ file compression tool
For example
    {exe} example.txt

--------------------------
Command line options:
--------------------------
""".format(exe=sys.argv[0])

cmd7z = "7z a -mm=PPMd"
cmdbsc = "external/libbsc/bsc e"
cmdreorder = "reorder.py"
sap = "./sap"
lex = "./lex"

def main():
    parser = argparse.ArgumentParser(description=Description, formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('input',           help='input file name', type=str, nargs='+')
    parser.add_argument('-o', '--output',  help='output base name (def. input base name)', default="", type=str)
    parser.add_argument('--7z',            help='compression 7 zip: compression of the file with 7 zip', action='store_true', default = False)
    parser.add_argument('--bsc',           help='compression bsc: compression of the file with bsc', action='store_true', default = False)
    parser.add_argument('-1', '--c1',      help='compression 1: compression of the input file', action='store_true', default = False)
    parser.add_argument('-2', '--c2',      help='compression 2: compression of the file in partitions', action='store_true', default = False)
    parser.add_argument('-3', '--c3',      help='compression 3: compression of the file in 3-components and partitions', action='store_true', default = False)
    parser.add_argument('-r', '--reorder', help='reorder option: 1 (lexicographical order), 2 (colexicographical order) or 3(random order)', default=False, type=int)
    parser.add_argument('-p', '--part',    help='size of the part for file division in MB', default=False, type=int)
    parser.add_argument('-t', '--test',    help='compression test: original file', default="", type=str)
    parser.add_argument('-v',              help='verbose: extra info in the log file',action='store_true')
    args = parser.parse_args()

    define_basename(args)

    path = os.path.split(sys.argv[0])[0]
    path_inp = os.path.split(sys.argv[1])[0]
    inp = os.path.splitext(sys.argv[1])[0]

    if(args.bsc): #if the compression is with bsc
        op_comp = 0
    else: #if the compression is with 7zip
        op_comp = 1

    file_inp = sys.argv[1]
    if(args.reorder):
        op = args.reorder
        reorder(sys.argv[1], inp, op)
        if(op == 1):
            r = '_lex'
            file_inp = inp + "_lex.fastq"
        elif(op == 2):
            r = '_colex'
            file_inp = inp + "_colex.fastq"
        elif(op == 3):
            r = '_random'
            file_inp = inp + "_random.fastq"

    if(args.output): #all output files named args.argv[1]
        inp_aux = os.path.join(path_inp, args.output)
        if(args.reorder):
            inp_aux = inp_aux + r

    else: #all output files named --output
        inp_aux = inp
        if(args.reorder):
            inp_aux = inp + r

    if(op_comp == 1):
        logfile_name = inp_aux + ".zip.log"
        file_out = inp_aux + ".zip"
        file_outh = inp_aux + "_header.zip"
        file_outd = inp_aux + "_dna.zip"
        file_outq = inp_aux + "_qs.zip"
    else:
        logfile_name = inp_aux + ".bsc.log"
        file_out = inp_aux + ".bsc"
        file_outh = inp_aux + "_header.bsc"
        file_outd = inp_aux + "_dna.bsc"
        file_outq = inp_aux + "_qs.bsc"

    file_header = inp_aux + ".header"
    file_dna = inp_aux + ".dna"
    file_qs = inp_aux + ".qs"
    file_aux = inp_aux + "_"

    if(args.c1): #if the compression is total
        aux = 1
    if(args.c2): #if the compression is in partitions
        aux = 2
    if(args.c3): #if the compression is in 3-components + partitions
        aux = 3

    print("Sending logging messages to file:", logfile_name)

    with open(logfile_name,"a") as logfile:
        print(">>> Begin computation",file=logfile)
        show_command_line(logfile)
        logfile.flush()

        if(args.part):
            part = args.part*(2**20) # X MB
        else:
            part = 10*(2**20) # 10 MB

        sarq = os.path.getsize(file_inp)
        if(args.test):
            inp_original = os.path.splitext(args.test)[0]
            file_data = inp_original + "_" + str(aux) + "_data.txt" #create the document with the compression and time data
            f_data = open(file_data, 'a') #open file data

        ###### print data from the compression ######
        print("\n\n###### COMPRESSION ######\n")
        print("Size .fastq: {:.6f} bytes\n".format(sarq))

        if(args.c1): #if the compression is total

            t = compression(file_out, file_inp, logfile_name, op_comp ) #return of the time compression
            sarq_c = os.path.getsize(file_out)
            ratio = (sarq_c/sarq)*100

            print("Size .zip: {:.6f} bytes".format(sarq_c))
            print("Compression 1 (FASTQ): {:.6f}".format(sarq_c/sarq))
            print("Time 1: {:.6f}".format(t))
            if(args.test):
                f_data.write("{:.6f} {:.6f} {:.2f}\n".format(sarq_c/sarq, t, ratio))

        if(args.c2): #if the compression is in partitions (size set in the input command)

            parts = partition(file_aux, part, file_inp)
            cont = 1
            sarq_c = 0
            t = 0
            while(cont <= parts):
                file_part = file_aux + str(cont) + ".part"
                if(op_comp == 1):
                    file_outp = file_aux + str(cont) + ".zip"
                else:
                    file_outp = file_aux + str(cont) + ".bsc"

                t = t + compression(file_outp, file_part, logfile_name, op_comp) #return of the time compression, sum of the times
                sarq_c = sarq_c + os.path.getsize(file_outp)
                cont = cont + 1
            ratio = (sarq_c/sarq)*100

            print("Compression 3 (sum parts): {:.6f}".format(sarq_c/sarq))
            print("Time: {:.6f}".format(t))
            if(args.test):
                f_data.write("{:.6f} {:.6f} {:.2f}\n".format(sarq_c/sarq, t, ratio))

        if(args.c3): #if the compression is in 3-components (header, dna and qs)

            t = 0
            components(file_header, file_dna, file_qs, file_inp)

            file_h = os.path.splitext(file_outh)[0] + "_"
            file_d = os.path.splitext(file_outd)[0] + "_"
            file_q = os.path.splitext(file_outq)[0] + "_"

            parts_h = partition(file_h, part, file_header)
            parts_d = partition(file_d, part, file_dna)
            parts_q = partition(file_q, part, file_qs)

            cont = 1
            sarq_c = 0
            t = 0
            while(cont <= parts_h or cont <= parts_d or cont <= parts_q):

                if(cont <= parts_h):
                    file_part = file_h + str(cont) + ".part"
                    if(op_comp == 1):
                        file_outp = file_h + str(cont) + ".zip"
                    else:
                        file_outp = file_h + str(cont) + ".bsc"

                    t = t + compression(file_outp, file_part, logfile_name, op_comp) #return of the time compression, sum of the times
                    sarq_c = sarq_c + os.path.getsize(file_outp)

                if(cont <= parts_d):
                    file_part = file_d + str(cont) + ".part"
                    if(op_comp == 1):
                        file_outp = file_d + str(cont) + ".zip"
                    else:
                        file_outp = file_d + str(cont) + ".bsc"
                    t = t + compression(file_outp, file_part, logfile_name, op_comp) #return of the time compression, sum of the times
                    sarq_c = sarq_c + os.path.getsize(file_outp)

                if(cont <= parts_q):
                    file_part = file_q + str(cont) + ".part"
                    if(op_comp == 1):
                        file_outp = file_q + str(cont) + ".zip"
                    else:
                        file_outp = file_q + str(cont) + ".bsc"

                    t = t + compression(file_outp, file_part, logfile_name, op_comp) #return of the time compression, sum of the times
                    sarq_c = sarq_c + os.path.getsize(file_outp)
                cont = cont + 1
            ratio = (sarq_c/sarq)*100

            print("Compression 3-components with partitions (sum parts): {:.6f}".format(sarq_c/sarq))
            print("Time: {:.6f}".format(t))
            if(args.test):
                f_data.write("{:.6f} {:.6f} {:.2f}\n".format(sarq_c/sarq, t, ratio))

        if(args.test):
            f_data.close() #close data file

##########################################################################################################
###### function of the reordered ######
##########################################################################################################
def reorder(file_inp, file_output, op):
    command = "python3 {reorder} {input} {output} {op}".format(reorder=cmdreorder, input=file_inp, output=file_output, op=op)
    execute_command(command)

##########################################################################################################
###### function of the compression ######
##########################################################################################################
def compression(file_out, file_input, logfile, op_comp):
    if(op_comp == 1): #compression with 7zip
        cmd = cmd7z
        command =  cmd + " {zip} {input} >> {log}".format(zip=file_out, input=file_input, log=logfile)
    else: #compression with bsc
        cmd = cmdbsc
        command =  cmd + " {input} {zip} >> {log}".format(zip=file_out, input=file_input, log=logfile)

    #execution of the command
    t = execute_command(command)
    return t #return time de compression

##########################################################################################################
###### function of the 3-components division ######
##########################################################################################################
def components(file_header, file_dna, file_qs, file_inp):

    f = open(file_inp, 'r')
    f_header = open(file_header, 'w') #file with header
    f_dna = open(file_dna, 'w') #file with dna
    f_qs = open(file_qs, 'w') #file with qs

    s = None
    while(s != ""):
        s = f.readline()
        f_header.write(s)
        s = f.readline()
        f_dna.write(s)
        #+
        s = f.readline()
        s = f.readline()
        f_qs.write(s)

    f_header.close()
    f_dna.close()
    f_qs.close()
    f.close()

##########################################################################################################
###### function of the partition ######
##########################################################################################################
def partition(file_aux, part, file_inp):

    cont = 1 #count the number of parts

    file_part = file_aux + str(cont) + ".part"
    f_part = open(file_part, 'w')
    f = open(file_inp, 'r')
    sum = 0
    s = None
    while(s != ""):
        for i in range(4):
            s = f.readline()
            sum += len(s)
            f_part.write(s) #header, dna, +, qs

        if(s == "" and sum < part):
            cont = cont + 1
        elif(sum >= part):
            cont = cont + 1
            sum = 0
            f_part.close()
            file_part = file_aux + str(cont) + ".part"
            f_part = open(file_part, 'w')

    f_part.close()
    f.close()
    return cont-1

##########################################################################################################

def define_basename(args):
    if len(args.output)==0:
        args.basename = args.input[0]
    else:
        args.basename = args.output

##########################################################################################################
###### function of the command execution #######
##########################################################################################################
def execute_command(command):
    try:
        start = time.time()
        subprocess.call(command, shell = True)
        t = time.time()-start
    except subprocess.CalledProcessError:
        print("Error to executing command line: {command}".format(command = command))
        return False
    return t #return time of the execution

##########################################################################################################

def show_command_line(f):
    f.write("Python command line: ")
    for x in sys.argv:
        f.write(x+" ")
    f.write("\n")

##########################################################################################################

if __name__ == '__main__':
    main()