import logging
import numa
import bisect
import numpy as np
from resources.gpu import GPU
from resources.server_config import DefaultServerConfig
'''
Creates a server with the specified number of GPUs.
If unspecified, creates a server with 8 homogenous GPUs.
Assumes that GPUs in a server are always homogenous.
If CPU, DRAM, and disk_bw not specificed, uses the default values
of those objects.
'''

class Server():

    def __init__(self,
                 rack_id=None,
                 server_id=None,
                 server_config=None,
                 start_gpu=None,
                 ip=None,
                 port=None,
                 start_gpu_deploy=0,
                 start_cpu_deploy=0,
                 numa_aware=False):
        self.logger = logging.getLogger(__name__)
        self.server_id = 0
        self.rack_id = 0
        self.start_cpu_deploy = int(start_cpu_deploy)
        self.start_gpu_deploy = int(start_gpu_deploy)
        self.numa_aware = int(numa_aware)

        if server_config is not None:
            self.server_config = server_config
        else:
            self.server_config = DefaultServerConfig()

        if server_id is not None:
            self.server_id = server_id

        if rack_id is not None:
            self.rack_id = rack_id

        # If nopt simulation, the IP, port and rpc client of the worker
        self.ip = ip
        if port is not None:
            self.port = int(port)
        else:
            self.port = port
        self.rpc_client = None

        # Max resource availability in the server
        self.num_cpus = self.server_config.cpu
        self.memory = self.server_config.mem
        self.storage_speed = self.server_config.sspeed
        self.num_gpus = self.server_config.gpu
        self.network = self.server_config.net

        self.cpus_available = list(range(self.start_cpu_deploy + 0, self.start_cpu_deploy + self.num_cpus))
        self.cpu_map_numa = dict()
        self.cpu_map_numa_ref = dict()
        if self.numa_aware:
            self.init_numa()

        # Resources allocated in the server
        self.gpu_used = 0
        self.cpu_used = 0
        self.mem_used = 0
        self.sspeed_used = 0
        self.net_used = 0

        # Resources on hold during recursive allocation
        self.gpu_on_hold = 0
        self.cpu_on_hold = 0
        self.mem_on_hold = 0
        self.sspeed_on_hold = 0
        self.net_on_hold = 0

        # Resources actually utilized by the jobs its allocated to
        # This is updated by the running jobs
        self.cpu_true_utilization = 0
        self.mem_true_utilization = 0
        self.sspeed_true_utilization = 0
        self.net_true_utilization = 0

        # Resources demanded by the jobs
        # This is updated by the running jobs
        self.cpu_demand = 0
        self.mem_demand = 0
        self.sspeed_demand = 0
        self.net_demand = 0

        self.gpus = []
        self.allocated_gpus = []
        self.gpus = self.get_gpus(start_gpu)

    def init_numa(self):
        # Populate self.cpu_map_numa - one list entry for each numa node
        num_numa_nodes = numa.get_max_node() + 1
        for i in range(0, num_numa_nodes):
            self.cpu_map_numa[i] = list(numa.node_to_cpus(i))
            self.cpu_map_numa_ref[i] = list(numa.node_to_cpus(i))

        self.logger.info("Numa nodes = {}".format(num_numa_nodes))
        self.logger.info("Numa node CPUs = {}".format(self.cpu_map_numa))

    def get_cpus(self, num_cpus):
        if not self.numa_aware:
            if len(self.cpus_available)  < num_cpus:
                return None
            cpus = self.cpus_available[:num_cpus]
            del self.cpus_available[:num_cpus]
            self.logger.info("Get {} CPUs : {}".format(num_cpus, cpus))
            return cpus
        else:
            # FInd the numa node with fewest available CPUs
            # that can satisy the current demands
            candidate_nodes = [n for n, cpus in self.cpu_map_numa.items()\
                          if len(cpus) >= num_cpus]
            if len(candidate_nodes) > 0:
                target_numa = sorted(candidate_nodes, key = lambda n : len(self.cpu_map_numa[n]))[0]
                cpus = self.cpu_map_numa[target_numa][:num_cpus]
                del self.cpu_map_numa[target_numa][:num_cpus]
                self.logger.info("Get {} CPUs : {}".format(num_cpus, cpus))
                return cpus

            sorted_numa_list = sorted(self.cpu_map_numa.keys(), key = lambda n : len(self.cpu_map_numa[n]))
            cpus = []
            i = 0
            while len(cpus) < num_cpus:
                cpus_needed = num_cpus - len(cpus)
                numa_id = sorted_numa_list[i]
                if cpus_needed < len(self.cpu_map_numa[numa_id]):
                    cpu_ids = self.cpu_map_numa[numa_id][:cpus_needed]
                    del self.cpu_map_numa[numa_id][:cpus_needed]
                    cpus.extend(cpu_ids)
                else:
                    cpu_ids = self.cpu_map_numa[numa_id]
                    self.cpu_map_numa[numa_id] = []
                    cpus.extend(cpu_ids)
                i += 1
            return cpus

    def remove_cpus_available(self, cpu_ids):
        if self.numa_aware:
            for cpu in cpu_ids:
                for numa_id, cpu_list in self.cpu_map_numa.items():
                    if cpu in cpu_list:
                        cpu_list.remove(cpu)
                        self.cpu_map_numa[numa_id] = cpu_list

        else:
            for cpu in cpu_ids:
                self.cpus_available.remove(cpu)

    def add_cpus_available(self, cpu_ids):
        if self.numa_aware:
            for cpu in cpu_ids:
                for numa_id, cpu_list in self.cpu_map_numa_ref.items():
                    if cpu in cpu_list:
                        bisect.insort(self.cpu_map_numa[numa_id], cpu)

        else:
            for cpu in cpu_ids:
                bisect.insort(self.cpus_available, cpu)

    def print_cpus_available(self):
        if self.numa_aware:
            return self.cpu_map_numa
        else:
            return self.cpus_available

    def get_gpus(self, start_gpu):
        if start_gpu is None:
            start_gpu = 0
        job_id  = -1
        tenant_id = -1

        gpus = []

        for gpu_id in np.arange(start_gpu, start_gpu+self.num_gpus):
            gpu = GPU(self.rack_id, self.server_id, gpu_id, job_id,
                     tenant_id, server_handle=self)
            gpus.append(gpu)
        return gpus

    # Update allocation of cpu, gpu, mem in the server
    def allocate(self, res_map):
        #print(res_map)
        #print(type(res_map))
        if not isinstance(res_map, dict):
            raise ValueError("Resource map is invalid")

        # No strict allocation restriction on storage speed
        for res in res_map:
            if 'cpu' in res:
                assert(res_map[res] <= self.cpu_available)
                self.cpu_used += res_map[res]
                self.cpu_true_utilization += res_map[res]
                self.cpu_demand += res_map[res]
                assert(int(self.cpu_true_utilization) <= int(self.cpu_used))
                assert(int(self.cpu_true_utilization) >= 0)
            if 'mem' in res:
                assert(res_map[res] <= self.mem_available)
                self.mem_used += res_map[res]
                self.mem_true_utilization += res_map[res]
                self.mem_demand += res_map[res]
                assert(int(self.mem_true_utilization) <= int(self.mem_used))
                assert(int(self.mem_true_utilization) >= 0)
            if 'gpu' in res:
                assert(res_map[res] <= self.gpu_available)
                self.gpu_used += res_map[res]
            if 'sspeed' in res:
                self.sspeed_used += res_map[res]
                self.sspeed_true_utilization += res_map[res]
                self.sspeed_demand += res_map[res]
                assert(int(self.sspeed_true_utilization) <= int(self.sspeed_used))
                assert(int(self.sspeed_true_utilization) >= 0)
        self.logger.debug("Allocate [{}] : gpus:{}, cpus:{}, mem:{}, sspeed:{}".format(self.id, self.gpu_available, self.cpu_available, self.mem_available, self.sspeed_available))

    def deallocate(self, res_map):
        if not isinstance(res_map, dict):
            raise ValueError("Resource map is invalid")
        #print(res_map)
        for res in res_map:
            if 'cpu' in res:
                assert(self.cpu_used >= res_map[res])
                self.cpu_used -= res_map[res]
            if 'mem' in res:
                assert(res_map[res] <= self.mem_used)
                self.mem_used -= res_map[res]
            if 'gpu' in res:
                assert(res_map[res] <= self.gpu_used)
                self.gpu_used -= res_map[res]
            if 'sspeed' in res:
                self.sspeed_used -= res_map[res]
        self.logger.debug("Deallocate [{}] : gpus:{}, cpus:{}, mem:{}, sspeed:{}".format(self.id, self.gpu_available, self.cpu_available, self.mem_available, self.sspeed_available))

    @property
    def id(self):
        return self.server_id

    @property
    def cpu_available(self):
        return self.num_cpus - self.cpu_used

    @property
    def mem_available(self):
        return self.memory - self.mem_used

    @property
    def sspeed_available(self):
        return self.storage_speed - self.sspeed_used

    @property
    def gpu_available(self):
        return self.num_gpus - self.gpu_used

    @property
    def net_available(self):
        return self.network - self.net_used

    def alloc_stats(self, percent=False):
        # Gpu, cpu, mem, sspeed, net used
        if percent:
            return (self.gpu_used/self.num_gpus*100,
                self.cpu_used/self.num_cpus*100,
                self.mem_used/self.memory*100,
                self.sspeed_used/self.storage_speed*100,
                self.net_used/self.network*100)
        else:
            return (self.gpu_used,
                self.cpu_used,
                self.mem_used,
                self.sspeed_used,
                self.net_used)

    def utilization_stats(self, percent=False):
        # Gpu, cpu, mem, sspeed, net true utilization
        if percent:
            return (self.gpu_used/self.num_gpus*100,
                self.cpu_true_utilization/self.num_cpus*100,
                self.mem_true_utilization/self.memory*100,
                self.sspeed_true_utilization/self.storage_speed*100,
                self.net_true_utilization/self.network*100)
        else:
            return (self.gpu_used,
                self.cpu_true_utilization,
                self.mem_true_utilization,
                self.sspeed_true_utilization,
                self.net_true_utilization)

    def demand_stats(self, percent=False):
        # Gpu, cpu, mem, sspeed, net demand at this server
        if percent:
            return (self.gpu_used/self.num_gpus*100,
                self.cpu_demand/self.num_cpus*100,
                self.mem_demand/self.memory*100,
                self.sspeed_demand/self.storage_speed*100,
                self.net_demand/self.network*100)
        else:
            return (self.gpu_used,
                self.cpu_demand,
                self.mem_demand,
                self.sspeed_demand,
                self.net_demand)

    def availability_stats(self, percent=False):
        # Gpu, cpu, mem, sspeed, net demand at this server
        if percent:
            return (self.gpu_available/self.num_gpus*100,
                self.cpu_available/self.num_cpus*100,
                self.mem_available/self.memory*100,
                self.sspeed_available/self.storage_speed*100,
                self.net_available/self.network*100)
        else:
            return (self.gpu_available,
                self.cpu_available,
                self.mem_available,
                self.sspeed_available,
                self.net_available)

    def availability_stats_with_hold(self, percent=False):
        # Gpu, cpu, mem, sspeed, net demand at this server
        return (self.gpu_available - self.gpu_on_hold,
                self.cpu_available  - self.cpu_on_hold,
                self.mem_available - self.mem_on_hold,
                self.sspeed_available - self.sspeed_on_hold,
                self.net_available - self.net_on_hold)

    def release_held_resources(self):
        self.gpu_on_hold = 0
        self.cpu_on_hold = 0
        self.mem_on_hold = 0
        self.sspeed_on_hold = 0
        self.net_on_hold = 0

    def hold_resources(self, demand_vec):
        self.gpu_on_hold += demand_vec[0]
        self.cpu_on_hold += demand_vec[1]
        self.mem_on_hold += demand_vec[2]
        self.sspeed_on_hold += demand_vec[3]
        self.net_on_hold += demand_vec[4]

    def __getstate__(self):
        d = self.__dict__.copy()
        if 'logger' in d:
            d['logger'] = d['logger'].name
        return d

    def __setstate__(self, d):
        if 'logger' in d:
            d['logger'] = logging.getLogger(d['logger'])
        self.__dict__.update(d)

    @property
    def gpu_ids(self):
        gpu_ids = []
        for gpu in self.gpus:
            gpu_ids.append(gpu.gpu_id)

        return gpu_ids

    @property
    def gpu_list(self):
        return self.gpus

    @property
    def free_gpu_list(self):
        free = []
        for gpu in self.gpus:
            if gpu not in self.allocated_gpus:
                free.append(gpu)

        return free

    def __cmp__(self,other):
        return cmp(self.server_id, other.server_id)

    def __lt__(self,other):
        return self.server_id < other.server_id

    def print_server_stats(self):
        for gpu in self.gpus:
            print("{}".format(gpu))


import json
import yaml
import os
import shutil

import deepspeed
import torch
from accelerate import Accelerator
from beartype import beartype
from beartype.typing import Tuple
from einops import rearrange
from peft import get_peft_model, LoraConfig, TaskType
from torch.utils.data import DataLoader, Dataset
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
)

from chatllama.rlhf.config import ConfigActor
from chatllama.rlhf.model_list import (
    hf_models_causal_lm,
    llama_models,
    hf_models,
)

from chatllama.rlhf.model_loader import ModelLoader
from chatllama.rlhf.utils import TrainingStats

class ActorModel(torch.nn.Module):
    """Actor model that generates the augmented prompt from the initial
    user_input. The aim is to train this model to generate better prompts.

    Attributes:
        model: The model from LLaMA to be used
        tokenizer: The LLaMA tokenizer
        config (ConfigActor): Configuration for the actor model

    Methods:
        load: Load the model from a path
        save: Save the model to a path
        forward: Compute the action logits for a given sequence.
        generate: Generate a sequence from a given prompt
    """

    def __init__(self, config: ConfigActor) -> None:
        super().__init__()

        # save config
        self.config = config

        # initialize the self.model
        if config.model in llama_models:
            # llama module might not be present when HF models are used
            from chatllama.llama_model import (
                load_model,
                setup_model_parallel,
            )  # noqa

            local_rank, world_size = setup_model_parallel()

            # use load_model_test for testing
            self.model, self.tokenizer = load_model(
                ckpt_dir=config.model_folder,
                tokenizer_path=config.tokenizer_path,
                local_rank=local_rank,
                world_size=world_size,
                froze_embeddings=config.froze_embeddings,
                use_fairscale=config.use_fairscale,
                max_batch_size=config.batch_size,
            )
        elif config.model in hf_models_causal_lm:
            self.tokenizer = self.load_tokenizer(config)
            self.model = AutoModelForCausalLM.from_pretrained(
                config.model,
            )

            # Setup PEFT model
            if config.peft_enable:

                # check that the peft config exist
                if os.path.exists(config.peft_config_path):
                    # Read the peft config from yaml
                    with open(config.peft_config_path, "r") as c:
                        config_peft = yaml.safe_load(c)
                else:
                    raise ValueError(
                        f"PEFT config {config.peft_config_path} not found"
                    )

                print(config_peft)
                # define lora config for peft
                peft_config = LoraConfig(
                    task_type=TaskType.CAUSAL_LM, **config_peft
                )

                # create peft model
                self.model = get_peft_model(
                    model=self.model,
                    peft_config=peft_config,
                )

            self.model.to(config.device)

        else:
            raise ValueError(f"Model {config.model} not supported")

        # load the model from model_folder
        self.load()

    @beartype
    def load(self) -> None:
        """Load the model from the path"""
        # check if there is a model to load
        path = ModelLoader.check_model_path(
            config=self.config,
            is_checkpoint=False,
            current_epoch=None,
        )

        # if there is a model to load
        if path is not None:

            # load the model
            print("Loading ...")
            model_dict = torch.load(path)
            self.model.load_state_dict(model_dict.get("state_dict") or model_dict.get("model"))

    @beartype
    def save(self) -> None:
        """Save the model to the path"""
        # get the path to save the model
        model_folder, model_name, path = ModelLoader.get_model_path(
            config=self.config,
            is_checkpoint=False,
            current_epoch=None,
        )

        # save the model
        print(f"Saving model to {path} ...")
        torch.save(
            {"state_dict": self.model.state_dict()},
            path,
        )

    @staticmethod
    def load_tokenizer(config: ConfigActor):
        """Load the tokenizer from the model name"""
        if config.model in hf_models:
            # load the tokenizer from HF
            tokenizer = AutoTokenizer.from_pretrained(
                config.model,
                padding_side="left",
                padding=True,
                truncation=True,
                model_max_length=config.max_sequence_length,
            )

            # add eos token if not present
            if tokenizer.eos_token is None:
                tokenizer.eos_token = "</s>"
                tokenizer.eos_token_id = 2  # OPT eos-token-id

            # add pad token if not present
            if tokenizer.pad_token is None:
                tokenizer.pad_token = tokenizer.eos_token
                tokenizer.pad_token_id = tokenizer.eos_token_id
        elif config.model in llama_models:

            # llama module might not be present when HF models are used
            from chatllama.llama_model import (
                load_tokenizer,
            )  # noqa

            tokenizer = load_tokenizer(config.tokenizer_path)
        return tokenizer

    def parameters(self):
        """Return the parameters of the model"""
        return self.model.parameters()

    @beartype
    def forward(
        self, sequences: torch.Tensor, sequences_mask: torch.Tensor
    ) -> torch.Tensor:
        """Generate logits to have probability distribution over the vocabulary
            of the actions

        Args:
            sequences (torch.Tensor): Sequences of states and actions used to
                    compute token logits for the whole list of sequences
            attention_mask (torch.Tensor): Mask for the sequences attention

        Returns:
            logits (torch.Tensor): Logits for the actions taken
        """
        model_output = self.model.forward(
            sequences, attention_mask=sequences_mask
        )
        # need to return logits for the actions
        if self.config.model in hf_models_causal_lm:
            model_output = model_output.logits
        if self.config.debug:
            print("ActorModel.forward")
            print("model_output_logits shape", model_output.shape)
            print("model_output logits", model_output)
        return model_output

    @beartype
    @torch.no_grad()
    def generate(
        self, states: torch.Tensor, state_mask: torch.Tensor
    ) -> Tuple:
        """Generate actions and sequences=[states, actions] from state
            (i.e. input of the prompt generator model)

        Args:
            state (torch.Tensor): the input of the user
            state_mask (torch.Tensor): Mask for the state input (for padding)

        Returns:
            actions (torch.Tensor): Actions generated from the state
            sequences (torch.Tensor): Sequences generated from the
                state as [states, actions]
        """
        # temperature for the actor
        temperature = self.config.temperature

        # max sequence length for the actor (i.e. prompt + completion)
        max_sequence_length = self.config.max_sequence_length

        # max and min number of tokens to generate
        max_tokens = self.config.max_tokens
        min_tokens = self.config.min_tokens

        # max generation possible given the state and the max sequence length
        max_generation_possible = max_sequence_length - states.shape[1]
        if max_generation_possible < min_tokens:
            raise ValueError(
                f"The prompt is too long w.r.t the "
                f"model sequence length \n"
                f"max_sequence_length={max_sequence_length}\n"
                f"state_length={states.shape[1]}\n"
                f"min_tokens={min_tokens}\n"
                f"max_tokens={max_tokens}\n"
                f"max_generation_possible={max_generation_possible}\n"
            )

        # take the minimum the max_tokens and the max_generation_possible
        max_completion = min(max_tokens, max_generation_possible)

        sequences = self.model.generate(
            input_ids=states,
            attention_mask=state_mask,
            temperature=temperature,
            max_new_tokens=max_completion,
            no_repeat_ngram_size=3,
        )
        actions = sequences[:, states.shape[1] :]  # noqa E203
        if self.config.debug:
            print(
                f"input length {states.shape[1]} \n"
                f"max sequence length {max_sequence_length} \n"
                f"max completion {max_completion} \n"
                f"generated sequence {sequences.shape[1]} \n"
            )
            print("ActorModel.generate")
            print("state", states)
            print("state shape", states.shape)
            print("sequence shape", sequences.shape)
            print("sequence", sequences)
            print("actions shape", actions.shape)
            print("actions", actions)
        return actions, sequences

class ActorDataset(Dataset):
    """Dataset for the pretraining of the actor model
    read a json file with the following format:
    [
        {
            "user_input": "..."
            "completion": "..."
        },
        ...
    ]
    Where:
        user_input: the input of the user
        completion: the output of the user
    """

    def __init__(
        self,
        path: str,
    ) -> None:
        self.path = path
        with open(path, "r") as f:
            data = json.load(f)
        self.data = [d["user_input"] + d["completion"] for d in data]

    def __getitem__(self, idx):
        return self.data[idx]

    def __len__(
        self,
    ):
        return len(self.data)

class ActorTrainer:
    """Used to pre-train the actor model to generate better prompts.

    Args:
        config (ConfigActor): Configuration for the actor model

    Attributes:
        config (ConfigActor): Configuration for the actor model
        model (ActorModel): Actor model
        loss_function (torch.nn.CrossEntropyLoss): Loss function
        optimizer (torch.optim.Adam): Optimizer
        validation_flag (bool): Flag to indicate if the validation dataset
            is provided
        train_dataset (ActorDataset): Training dataset
        train_dataloader (DataLoader): Training dataloader
        validation_dataset (ActorDataset): Validation dataset
        validation_dataloader (DataLoader): Validation dataloader
        scheduler (torch.optim.lr_scheduler): Learning rate scheduler
        training_stats (TrainingStats): Training statistics
        model_engine (ModelEngine): Model engine for deepspeed training
        accelerator (Accelerator): Accelerator for accelerate training

    Methods:
        train: Train the actor model
        load_checkpoint: Load a checkpoint
        save_checkpoint: Save a checkpoint
    """

    def __init__(self, config: ConfigActor) -> None:

        # store config
        self.config = config

        # load the model
        self.actor = ActorModel(config)

        # define loss function
        self.loss_function = torch.nn.CrossEntropyLoss()

        # define optimizer
        self.optimizer = torch.optim.AdamW(
            self.actor.parameters(), lr=config.lr, weight_decay=1e-5
        )

        # check if validation dataset is provided
        self.validation_flag = False
        if config.validation_dataset_path is not None:
            self.validation_flag = True

        # create dataset and dataloaders
        self.train_dataset = ActorDataset(config.train_dataset_path)
        self.train_dataloader = DataLoader(
            self.train_dataset, batch_size=config.batch_size
        )
        if self.validation_flag:
            self.eval_dataset = ActorDataset(config.validation_dataset_path)
            self.validation_dataloader = DataLoader(
                self.eval_dataset, batch_size=config.batch_size
            )

        # define scheduler for the learning rate
        # learning rate is decreased until 10% of the initial value
        self.scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(
            self.optimizer,
            T_0=len(self.train_dataset) // config.batch_size,
            T_mult=1,
            eta_min=config.lr * 0.1,
        )

        # define training statistics
        stat_path = ModelLoader.get_training_stats_path(config)
        self.training_stats = TrainingStats(stat_path)

        # consistency check between accelerate and deepspeed
        if config.accelerate_enable and config.deepspeed_enable:
            raise ValueError(
                "Both DeepSpeed and Accelerate are enabled for the Actor."
                "Please choose one of them."
            )

        # initialize deepspeed
        self.model_engine = None
        if config.deepspeed_enable is True:
            if config.deepspeed_config_path is None:
                raise ValueError(
                    "DeepSpeed config path is None, but deepspeed is enabled"
                )
            if os.path.exists(config.deepspeed_config_path) is False:
                raise ValueError(
                    f"DeepSpeed config path {config.deepspeed_config_path}"
                    f"does not exist"
                )
            (
                self.model_engine,
                self.optimizer,
                self.train_dataloader,
                _,
            ) = deepspeed.initialize(
                args=None,
                model=self.actor,
                model_parameters=self.actor.parameters(),
                training_data=self.train_dataset,
                config=self.config.deepspeed_config_path,
            )
            print("Training with DeepSpeed")

        # initialize accelerate
        self.accelerator = None
        if config.accelerate_enable is True:
            self.accelerator = Accelerator()
            (
                self.actor,
                self.optimizer,
                self.train_dataloader,
                self.scheduler,
            ) = self.accelerator.prepare(
                self.actor,
                self.optimizer,
                self.train_dataloader,
                self.scheduler,
            )
            print("Training with Accelerate")

    @beartype
    def save_checkpoint(
        self,
        current_epoch: int,
        current_step: int,
        max_epochs: int,
        max_steps: int,
    ) -> None:
        """Save the current checkpoint

        Args:
            current_epoch (int): Current epoch
            current_step (int): Current step
            max_epochs (int): Maximum number of epochs
            max_steps (int): Maximum number of steps
        """

        print(
            f"Saving checkpoint for epoch {current_epoch + 1}, "
            f"step {current_step + 1} ..."
        )
        # look for path to save the checkpoint
        model_folder, model_name, path = ModelLoader.get_model_path(
            config=self.config,
            is_checkpoint=True,
            current_epoch=current_epoch,
            current_step=current_step,
            max_epochs=max_epochs,
            max_steps=max_steps,
        )

        # remove the checkpoint if it already exists
        if os.path.exists(path):
            if self.config.deepspeed_enable:
                shutil.rmtree(path)
            else:
                os.remove(path)

        if self.config.deepspeed_enable:
            client_state = {
                "epoch": current_epoch,
                "step": current_step,
            }
            self.model_engine.save_checkpoint(path, client_state=client_state)
        else:
            # save the checkpoint
            torch.save(
                {
                    "state_dict": self.actor.model.state_dict(),
                    "optim_state_dict": self.optimizer.state_dict(),
                    "training_stats": self.training_stats,
                    "epoch": current_epoch,
                    "step": current_step,
                },
                path,
            )

        # remove old checkpoints
        n_checkpoints_to_keep = self.config.n_checkpoints_to_keep
        ModelLoader.delete_old_checkpoints(
            model_folder, model_name, n_checkpoints_to_keep
        )

    @beartype
    def load_checkpoint(
        self,
    ) -> Tuple[int, int]:
        """Load a checkpoint from the model folder

        Returns:
            Tuple[int, int]: Current epoch and current step to resume
                training
        """

        print("Looking for checkpoints...")
        # look for a checkpoint
        path = ModelLoader.check_model_path(
            config=self.config,
            is_checkpoint=True,
            current_epoch=None,
        )

        # if there is a checkpoint
        if path is not None:
            print("Loading ...")

            if self.config.deepspeed_enable:
                # try to load the checkpoint
                try:
                    _, client_state = self.model_engine.load_checkpoint(path)
                except Exception:
                    print(
                        "Checkpoint corrupted!"
                        "Try to remove the last checkpoint."
                        "Now Starting from epoch 0, step 0"
                    )
                    return 0, 0
                # load epoch and step to resume loops
                epoch = client_state["epoch"]
                step = client_state["step"]
            else:
                # try to load the checkpoint
                try:
                    checkpoint = torch.load(path)
                except Exception:
                    print(
                        "Checkpoint corrupted!"
                        "Try to remove the last checkpoint."
                        "Now Starting from epoch 0, step 0"
                    )
                    return 0, 0

                # assing the checkpoint to the model
                epoch = checkpoint["epoch"]
                self.actor.model.load_state_dict(checkpoint["state_dict"])
                self.optimizer.load_state_dict(checkpoint["optim_state_dict"])
                self.trainign_stats = checkpoint["training_stats"]
                step = checkpoint["step"]
                return epoch, step + 1  # return the next episode to train
        return 0, 0

    def add_eos_token(
        self, tokens: torch.Tensor, mask: torch.Tensor
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        # given tokens and mask, add eos token to the end of each sequence
        # and update the mask
        batch_size, seq_len = tokens.shape
        eos_token = self.actor.tokenizer.eos_token_id

        # see if i can append 1 token
        n_tokens_to_append = min(self.config.max_sequence_length - seq_len, 1)
        n_tokens_to_append = max(n_tokens_to_append, 0)

        # concatenate eos to tokens and mask
        if n_tokens_to_append > 0:
            tokens = torch.cat(
                [
                    tokens,
                    torch.ones(batch_size, n_tokens_to_append)
                    .long()
                    .to(tokens.device)
                    * eos_token,
                ],
                dim=1,
            )
            mask = torch.cat(
                [
                    mask,
                    torch.ones(batch_size, n_tokens_to_append)
                    .long()
                    .to(mask.device),
                ],
                dim=1,
            )
        return tokens, mask

    def train(
        self,
    ) -> None:
        """Train the model"""
        print("Start Actor Model Pretraining")

        # get config parameters
        if self.config.deepspeed_enable:
            batch_size = self.train_dataloader.batch_size
        else:
            batch_size = self.config.batch_size
        epochs = self.config.epochs
        device = self.config.device
        checkpoint_steps = self.config.checkpoint_steps

        # compute the number of iterations
        n_iter = int(len(self.train_dataset) / batch_size)

        # load model_checkpoint
        start_epoch, start_step = self.load_checkpoint()

        if start_epoch == 0 and start_step == 0:
            self.training_stats.clear()

        # counter for the checkpoint
        cnt_checkpoint = 1

        # traing loop
        for epoch in range(start_epoch, epochs):
            self.actor.train()
            for i, input_text in enumerate(self.train_dataloader):

                # skip the first steps if we are resuming training
                if i < start_step:
                    continue

                # tokenize input
                with torch.no_grad():
                    input_tokenized = self.actor.tokenizer(
                        input_text,
                        return_tensors="pt",
                        truncation=True,
                        padding=True,
                    )

                    # split tokens and mask
                    input_tokenized_id = input_tokenized["input_ids"]
                    input_tokenized_mask = input_tokenized["attention_mask"]

                    # add eos token
                    (
                        input_tokenized_id,
                        input_tokenized_mask,
                    ) = self.add_eos_token(
                        input_tokenized_id,
                        input_tokenized_mask,
                    )

                    # split into input and output
                    training_output = input_tokenized_id[:, 1:]
                    training_input = input_tokenized_id[:, :-1]
                    attention_mask = input_tokenized_mask[:, :-1]

                    # move to device
                    training_output = training_output.to(device)
                    training_input = training_input.to(device)
                    attention_mask = attention_mask.to(device)

                # forward pass
                if self.config.deepspeed_enable:
                    est_output = self.model_engine(
                        training_input, attention_mask
                    )
                else:
                    est_output = self.actor(training_input, attention_mask)

                # compute loss
                est_output = rearrange(est_output, "b s v -> (b s) v")
                training_output = rearrange(training_output, "b s -> (b s)")
                loss = self.loss_function(est_output, training_output)
                self.training_stats.training_loss.append(loss.item())

                # backward pass
                if self.config.deepspeed_enable:
                    self.model_engine.backward(loss)
                    self.model_engine.step()
                elif self.config.accelerate_enable:
                    self.optimizer.zero_grad()
                    self.accelerator.backward(loss)
                    self.optimizer.step()
                    self.scheduler.step()
                else:
                    self.optimizer.zero_grad()
                    loss.backward()
                    self.optimizer.step()
                    self.scheduler.step()

                # print progress
                if i % self.config.iteration_per_print == 0:
                    print(
                        f"Epoch: {epoch+1}/{epochs}, "
                        f"Iteration: {i+1}/{n_iter}, "
                        f"Training Loss: {loss}"
                    )

                # save checkpoint periodically
                if cnt_checkpoint % checkpoint_steps == 0:
                    self.save_checkpoint(epoch, i, epochs, n_iter)
                    self.training_stats.save()
                    cnt_checkpoint = 1
                else:
                    cnt_checkpoint += 1

            # Validation
            if self.validation_flag:
                self.actor.eval()
                with torch.no_grad():
                    for i, input_text in enumerate(self.validation_dataloader):

                        # tokenize input
                        input_tokenized = self.actor.tokenizer(
                            input_text, return_tensors="pt", padding=True
                        )
                        validation_output = input_tokenized["input_ids"][:, 1:]
                        validation_input = input_tokenized["input_ids"][:, :-1]
                        attention_mask = input_tokenized["attention_mask"][
                            :, :-1
                        ]

                        # forward pass
                        est_output = self.actor.forward(
                            validation_input, attention_mask
                        )
                        validation_output = rearrange(
                            validation_output, "b s -> (b s)"
                        )

                        # compute loss
                        est_output = rearrange(est_output, "b s v -> (b s) v")
                        loss = self.loss_function(
                            est_output, validation_output
                        )
                        self.training_stats.validation_loss.append(loss.item())

                        # print progress
                        if i % self.config.iteration_per_print == 0:
                            print(
                                f"Epoch: {epoch+1}/{epochs}, "
                                f"Iteration: {i+1}/{n_iter}, "
                                f"Validation Loss: {loss}"
                            )
            # reset start_step after training is resumed
            start_step = 0

        # save the model
        self.actor.save()
        print("Training Finished ")

{'load_model': 'chatllama.llama_model.load_model', 'setup_model_parallel': 'chatllama.llama_model.setup_model_parallel', 'load_tokenizer': 'chatllama.llama_model.load_tokenizer'}


import streamlit as st
import pandas as pd
st.set_page_config(
    page_title="Ex-stream-ly Cool App",
    page_icon="ðŸ§Š",
    layout="wide"
)

st.title(" Welcome to the Smart Blood Report Generator ")

uploaded_file = st.file_uploader("Choose a file")
if uploaded_file is not None:
    # To read file as bytes:
    bytes_data = uploaded_file.getvalue()
    st.write(bytes_data)

    # To convert to a string based IO:
    stringio = StringIO(uploaded_file.getvalue().decode("utf-8"))
    st.write(stringio)

    # To read file as string:
    string_data = stringio.read()
    st.write(string_data)

    # Can be used wherever a "file-like" object is accepted:
    dataframe = pd.read_csv(uploaded_file)
    st.write(dataframe)

st.header("Dashboard")

df=pd.read_csv("./smart.csv")

for i in df.values:
    col1, col2 = st.columns(2)
    with col1:
        st.subheader(i[1])
        st.write(i[2])

    with col2:

        st.subheader(str(i[4])+str(i[3]) + "  -  " + str(i[6]))
        if(i[6]=="Normal"):
            st.success(i[7])
        elif(i[6]=="High"):
            st.error(i[7])
        else:
            st.info(i[7])


import xml.etree.ElementTree as ET

class Xml:
    def __init__(self, root):
        self._root = root

    @staticmethod
    def plain_text(tree: ET.Element):
        '''Extract plain text from the tag.'''
        txt = tree.text or ''

        for child in tree:
            txt += Xml.plain_text(child)
            txt += child.tail or ''

        return txt

    @staticmethod
    def plain_text_exclude(tree: ET.Element, exclude):
        '''Extract plain text from the tag exclude given tags.'''
        text = tree.text or ''

        for child in tree:
            if child.tag in exclude:
                pass
            else:
                text += Xml.plain_text_exclude(child, exclude)
            text += child.tail or ''

        return text.strip()

    @staticmethod
    def find_tag(tree: ET.Element, name: str):
        '''Return the tag with the given name. Recursive.'''
        if tree.tag == name:
            return tree
        for child in tree:
            found = Xml.find_tag(child, name)
            if found is not None:
                return found
        return None

    @staticmethod
    def find_tag_direct(tree: ET.Element, name: str):
        '''Return the child tag with the given name. NOT recursive.'''
        l = list(filter(lambda x: x.tag == name, tree))
        if len(l) == 0:
            return None
        return l[0]

    @staticmethod
    def filter_tags(tree: ET.Element, name: str, attribs=None):
        '''Return the list of the given name tag. Not recursive.'''
        if attribs is None:
            return list(filter(lambda x: x.tag == name, tree))
        else:
            ret = []
            filtered = list(filter(lambda x: x.tag == name, tree))
            for key in attribs:
                value = attribs[key]
                has_attrib_tags = filter(
                    lambda x: key in x.attrib.keys(),
                    filtered
                )
                match_tags = filter(
                    lambda x: x.attrib[key] == value,
                    has_attrib_tags
                )
                ret += list(match_tags)
            return ret


import ipywidgets as widgets
from IPython.display import display
from ipywidgets import *
from collections import Counter
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.feature_extraction import DictVectorizer
from pandas import DataFrame

data = pd.read_csv('cs-training.csv', sep=';').drop('Unnamed: 0', axis=1)
collections = 2
# min = pd.Series.min()
Cols = []
for i in range(len(data.columns)):
    Cols.append(data.columns[i].replace('-', ''))
data.columns = Cols
plt.figure(1)
df_data = {"age": 15, "NumberOfTime3059DaysPastDueNotWorse": 20}
df_data.age.plot.box()
Counter(df_data.age)
ind = np.where(df_data.age < 21)
df_data.age[ind[0]] = 21.
ind = np.where(df_data.age > 94)
df_data.age[ind[0]] = 94.
Counter(df_data.NumberOfTime3059DaysPastDueNotWorse)
# Set outlier values to median , that is 0.
ind = np.where(df_data.NumberOfTime3059DaysPastDueNotWorse > 95)
df_data.NumberOfTime3059DaysPastDueNotWorse[ind[0]] = 0.

plt.figure(figsize=(20, 15))
ax = plt.subplot(211)
# ax.set_ylim(0,20)
plt.plot(df_data.DebtRatio, 'bo', df_data.DebtRatio, 'k')
print('Median: %.7f \nMean: %.7f' %
      (np.median(df_data.DebtRatio), np.mean(df_data.DebtRatio)))
# ruoelLt2=len(df_data[df_data.RevolvingUtilizationOfUnsecuredLines < 2])
# ruoelACt=len(df_data.RevolvingUtilizationOfUnsecuredLines)
# print('Values less than 2 : %d in %d. Ratio: %.5f%%' %(ruoelLt2,ruoelACt,100*ruoelLt2/ruoelACt))
ax = sns.countplot(mad_based_outlier(df_data.DebtRatio))
plot_freq(l=len(df_data.DebtRatio))

minUpperBound = min([val for (val, out) in zip(
    df_data.DebtRatio, mad_based_outlier(df_data.DebtRatio)) if out == True])
# Set outlier values to upperbound, that is minUpperBound.
ind = np.where(df_data.DebtRatio > minUpperBound)
df_data.DebtRatio[ind[0]] = minUpperBound

plt.figure(figsize=(20, 15))
ax = plt.subplot(211)
plt.plot(df_data.DebtRatio, 'o')

df_data.DebtRatio.describe()

plt.figure(figsize=(20, 15))
ax = plt.subplot(211)
# ax.set_ylim(0,20)
plt.plot(df_data.MonthlyIncome, 'bo', df_data.MonthlyIncome, 'k')
print('Median: %.7f \nMean: %.7f' %
      (np.median(df_data.MonthlyIncome), np.mean(df_data.MonthlyIncome)))
maxUpperBound = min([val for (val, out) in zip(
    df_data.MonthlyIncome, mad_based_outlier(df_data.MonthlyIncome)) if out == True])
ind = np.where(df_data.MonthlyIncome > maxUpperBound)
df_data.MonthlyIncome[ind[0]] = maxUpperBound
ind = np.where(df_data.MonthlyIncome < 1500)
df_data.MonthlyIncome[ind[0]] = 1500
df_data.MonthlyIncome.describe()
Counter(df_data.NumberRealEstateLoansOrLines)
# Set outlier values to 16.
ind = np.where(df_data.NumberRealEstateLoansOrLines > 16)
df_data.NumberRealEstateLoansOrLines[ind[0]] = 16
Counter(df_data.NumberOfTime6089DaysPastDueNotWorse)
# Set outlier values to 0.
ind = np.where(df_data.NumberOfTime6089DaysPastDueNotWorse > 11)
df_data.NumberOfTime6089DaysPastDueNotWorse[ind[0]] = 0
Counter(df_data.NumberOfDependents)
ind = np.where(df_data.NumberOfDependents > 10)
df_data.NumberOfDependents[ind[0]] = 10

plt.style.use('ggplot')

form_item_layout = widgets.Layout(
    display='flex',
    flex_flow='row',
    justify_content='space-between'
)

# displaying the text widget
text = widgets.Text(
    placeholder='Enes',
    disabled=False
)
# display(text)
# add button that updates the graph based on the checkboxes
button = widgets.Button(description="Check credibility")
# display(button)
resultLabel = widgets.Label(
    value="",
    visible=False,
    disabled=True
)
# display(resultLabel)
revolve = widgets.FloatSlider(
    value=df_data.RevolvingUtilizationOfUnsecuredLines.median(),
    min=df_data.RevolvingUtilizationOfUnsecuredLines.min(),
    max=df_data.RevolvingUtilizationOfUnsecuredLines.max(),
    step=0.01,
    disabled=False,
    continuous_update=True,
    orientation='horizontal',
    readout=True,
    readout_format='.4f',
    slider_color='black'
)
age = widgets.IntSlider(
    value=df_data.age.median(),
    min=df_data.age.min(),
    max=df_data.age.max(),
    step=1,
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    slider_color='black'
)
income = widgets.FloatSlider(
    value=df_data.MonthlyIncome.median(),
    min=df_data.MonthlyIncome.min(),
    max=df_data.MonthlyIncome.max(),
    step=0.5,
    disabled=False,
    continuous_update=True,
    orientation='horizontal',
    readout=True,
    readout_format='.1f',
    slider_color='black'
)
debtRatio = widgets.FloatSlider(
    value=df_data.DebtRatio.median(),
    min=df_data.DebtRatio.min(),
    max=df_data.DebtRatio.max(),
    step=0.01,
    disabled=False,
    continuous_update=True,
    orientation='horizontal',
    readout=True,
    readout_format='.4f',
    slider_color='black'
)
NumberOfTime3059DaysPastDueNotWorse = widgets.IntSlider(
    value=df_data.NumberOfTime3059DaysPastDueNotWorse.median(),
    min=df_data.NumberOfTime3059DaysPastDueNotWorse.min(),
    max=df_data.NumberOfTime3059DaysPastDueNotWorse.max(),
    step=1,
    disabled=False,
    continuous_update=True,
    orientation='horizontal',
    readout=True,
    slider_color='black'
)
NumberOfTimes90DaysLate = widgets.IntSlider(
    value=df_data.NumberOfTimes90DaysLate.median(),
    min=df_data.NumberOfTimes90DaysLate.min(),
    max=df_data.NumberOfTimes90DaysLate.max(),
    step=1,
    disabled=False,
    continuous_update=True,
    orientation='horizontal',
    readout=True,
    slider_color='black'
)
NumberRealEstateLoansOrLines = widgets.IntSlider(
    value=df_data.NumberRealEstateLoansOrLines.median(),
    min=df_data.NumberRealEstateLoansOrLines.min(),
    max=df_data.NumberRealEstateLoansOrLines.max(),
    step=1,
    disabled=False,
    continuous_update=True,
    orientation='horizontal',
    readout=True,
    slider_color='black'
)
NumberOfTime6089DaysPastDueNotWorse = widgets.IntSlider(
    value=df_data.NumberOfTime6089DaysPastDueNotWorse.median(),
    min=df_data.NumberOfTime6089DaysPastDueNotWorse.min(),
    max=df_data.NumberOfTime6089DaysPastDueNotWorse.max(),
    step=1,
    disabled=False,
    continuous_update=True,
    orientation='horizontal',
    readout=True,
    slider_color='black'
)
NumberOfOpenCreditLinesAndLoans = widgets.IntSlider(
    value=df_data.NumberOfOpenCreditLinesAndLoans.median(),
    min=df_data.NumberOfOpenCreditLinesAndLoans.min(),
    max=df_data.NumberOfOpenCreditLinesAndLoans.max(),
    step=1,
    disabled=False,
    continuous_update=True,
    orientation='horizontal',
    readout=True,
    slider_color='black'
)

NumberOfDependents = widgets.IntSlider(
    value=df_data.NumberOfDependents.median(),
    min=df_data.NumberOfDependents.min(),
    max=df_data.NumberOfDependents.max(),
    step=1,
    disabled=False,
    continuous_update=True,
    orientation='horizontal',
    readout=True,
    slider_color='black'
)
algos = widgets.Dropdown(
    options=['Decision Tree', 'SVM', 'Logistic Regression', 'GaussianNB', 'MLP'])
# display(revolve)

form_items = [
    Box([Label(value='Please, enter the name:'), text], layout=form_item_layout),
    Box([Label(value='Revolving Util. of Unsecured Lines:'),
         revolve], layout=form_item_layout),
    Box([Label(value='Age:'), age], layout=form_item_layout),
    Box([Label(value='Monthly Income:'), income], layout=form_item_layout),
    Box([Label(value='Dept Ratio:'), debtRatio], layout=form_item_layout),
    Box([Label(value='NumberOfOpenCreditLinesAndLoans:'),
         NumberOfOpenCreditLinesAndLoans], layout=form_item_layout),
    Box([Label(value='NumberOfTime3059DaysPastDueNotWorse:'),
         NumberOfTime3059DaysPastDueNotWorse], layout=form_item_layout),
    Box([Label(value='NumberOfTimes90DaysLate:'),
         NumberOfTimes90DaysLate], layout=form_item_layout),
    Box([Label(value='NumberRealEstateLoansOrLines:'),
         NumberRealEstateLoansOrLines], layout=form_item_layout),
    Box([Label(value='NumberOfTime6089DaysPastDueNotWorse:'),
         NumberOfTime6089DaysPastDueNotWorse], layout=form_item_layout),
    Box([Label(value='NumberOfDependents:'),
         NumberOfDependents], layout=form_item_layout),
    Box([Label(value='Algorithm:'), algos], layout=form_item_layout),
    button,
    Box([Label(value='Result:'), resultLabel], layout=form_item_layout),
]

form = Box(form_items, layout=Layout(
    display='flex',
    flex_flow='column',
    border='dashed 2px',
    align_items='stretch',
    width='70%'
))

display(form)

# function to deal with the checkbox update button

def on_button_clicked(b):
    name = text.value
    roul = revolve.value
    ageV = age.value
    monI = income.value
    dratio = debtRatio.value
    noOpCL = NumberOfOpenCreditLinesAndLoans.value
    noT3059 = NumberOfTime3059DaysPastDueNotWorse.value
    noT90 = NumberOfTimes90DaysLate.value
    nreL = NumberRealEstateLoansOrLines.value
    noT6089 = NumberOfTime6089DaysPastDueNotWorse.value
    noD = NumberOfDependents.value
    algo = algos.value

    testARR = [[dratio, monI, noD, noOpCL,
                noT3059, noT6089, noT90, nreL, roul, ageV]]
    yGuess = []

    if algo == 'Decision Tree':
        yGuess = clf2.predict(testARR)
    elif algo == 'SVM':
        yGuess = clf3.predict(testARR)
    elif algo == 'Logistic Regression':
        yGuess = clf.predict(testARR)
    elif algo == 'GaussianNB':
        yGuess = gnb.predict(testARR)
    else:
        yGuess = clf4.predict(testARR)

    print(name, roul, ageV, monI, dratio, noOpCL, noT3059,
          noT90, nreL, noT6089, noD, algo, yGuess[0])

    if (resultLabel.visible == False and yGuess[0] == 0):
        resultLabel.value = text.value + ' can be provided with the loan.'
    else:
        resultLabel.value = text.value + ' should not be provided with the loan.'

button.on_click(on_button_clicked)
plt.show()

ind = df_data.SeriousDlqin2yrs[df_data.SeriousDlqin2yrs == 1]
ind = ind[1:100]
ind2 = df_data.SeriousDlqin2yrs[df_data.SeriousDlqin2yrs == 0]
ind2 = ind2[1:100]

plt.figure(figsize=(15, 15))
ax = plt.subplot(211)
# ax.set_ylim(0,20)
plt.plot(df_data.MonthlyIncome[ind.index], df_data.age[ind.index],
         'ko', df_data.MonthlyIncome[ind2.index], df_data.age)

# 🔵 Standard Compliance Report
# 🔹 Product Type: Credit Scoring
# 🔹 Code Block Name: credit_score_3.py
# 🔹 Analysis Result:
#   Standard => Credit Information Business Act B.E.2545
#   ⦿ Section: 3
#    ⁃ Count of standard compliance in the code block:
#      ◦ age: 14
#      ◦ NumberOfTime3059DaysPastDueNotWorse: 10
#      ◦ DebtRatio: 15
#      ◦ MonthlyIncome: 16
#      ◦ NumberRealEstateLoansOrLines: 10
#      ◦ NumberOfTime6089DaysPastDueNotWorse: 10
#      ◦ NumberOfDependents: 10
#      ◦ RevolvingUtilizationOfUnsecuredLines: 3
#      ◦ income: 3
#      ◦ debtRatio: 3
#      ◦ NumberOfOpenCreditLinesAndLoans: 7
#      ◦ Lines: 1
#      ◦ Age: 1
#      ◦ Income: 1
#      ◦ ageV: 3
#      ◦ loan: 2
#      Total: 109
#    ⁃ Count of standard violation in the code block: 0
# 🔹 Absent section in the code block:
#  ◦ Credit Information Business Act B.E.2545 at section 10
# 🔹 Reference Number: dbb813beb820f92dc84d74f5bee1b96ec320367e641e38cd8fa8c2ddd16bf9a6
# 🔵 Standard Compliance Report
# 🔹 Product Type: Credit Scoring
# 🔹 Code Block Name: credit_score_3.py
# 🔹 Analysis Result:
#   1. Standard => Credit Information Business Act B.E.2545
#   ⦿ Section: 3
#    ⁃ Count of standard compliance in the code block:
#      ◦ age: 14
#      ◦ NumberOfTime3059DaysPastDueNotWorse: 10
#      ◦ DebtRatio: 15
#      ◦ MonthlyIncome: 16
#      ◦ NumberRealEstateLoansOrLines: 10
#      ◦ NumberOfTime6089DaysPastDueNotWorse: 10
#      ◦ NumberOfDependents: 10
#      ◦ RevolvingUtilizationOfUnsecuredLines: 3
#      ◦ income: 3
#      ◦ debtRatio: 3
#      ◦ NumberOfOpenCreditLinesAndLoans: 7
#      ◦ Lines: 1
#      ◦ Age: 1
#      ◦ Income: 1
#      ◦ ageV: 3
#      ◦ loan: 2
#      Total: 109
#    ⁃ Count of standard violation in the code block: 0
# 🔹 Absent section in the code block:
#  ◦ Credit Information Business Act B.E.2545 at section 10
# 🔹 Reference Number: be52a8d6e1daf9104a1715475cefb4bcdad5b5bf483e9a0019470e58477fd31c
# 🔵 Standard Compliance Report
# 🔹 Product Type: Credit Scoring
# 🔹 Code Block Name: credit_score_3.py
# 🔹 Analysis Result:
#   1. Standard => ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices
#   ⦿ Section: Preparation of Credit Information no.8.2
#    ⁃ Count of standard compliance in the code block:
#      ◦ collections: 1
#      Total: 1
#    ⁃ Count of standard violation in the code block: 0
#   ⦿ Section: Credit Scoring Development
#    ⁃ Count of standard compliance in the code block:
#      ◦ data: 4
#      ◦ df_data: 79
#      Total: 83
#    ⁃ Count of standard violation in the code block: 0
#   ⦿ Section: Preparation of Credit Information no.8.6
#    ⁃ Count of standard compliance in the code block:
#      ◦ training: 1
#      Total: 1
#    ⁃ Count of standard violation in the code block: 0
#   ⦿ Section: Preparation of Credit Information no.8.1
#    ⁃ Count of standard compliance in the code block:
#      ◦ training: 1
#      Total: 1
#    ⁃ Count of standard violation in the code block: 0
#   ⦿ Section: Testing of the Reliability of Separation and/or Accuracy of Prediction Power of Credit Scoring
#    ⁃ Count of standard compliance in the code block:
#      ◦ testARR: 6
#      Total: 6
#    ⁃ Count of standard violation in the code block: 0
# 🔹 Absent section in the code block:
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Implementation of Credit Scoring System I
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Implementation of Credit Scoring System II
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Verification of the Applicant’s Information
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Preparation of Credit Information no.8.3
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Overrides in Credit Approval
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Construction of Credit Scoring
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Definition of Credit Scoring
# 🔹 Reference Number: dcdee9392d64ed729cc9eec3adf391899d18359c9b8529142cb09a1a506479ea
# 🔵 Standard Compliance Report
# 🔹 Product Type: Credit Scoring
# 🔹 Code Block Name: credit_score_3.py
# 🔹 Analysis Result:
#   1. Standard => ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices
#   ⦿ Section: Preparation of Credit Information no.8.2
#    ⁃ Count of standard compliance in the code block:
#      ◦ collections: 1
#      Total: 1
#    ⁃ Count of standard violation in the code block: 0
#   ⦿ Section: Credit Scoring Development
#    ⁃ Count of standard compliance in the code block:
#      ◦ data: 4
#      ◦ df_data: 79
#      Total: 83
#    ⁃ Count of standard violation in the code block: 0
#   ⦿ Section: Preparation of Credit Information no.8.6
#    ⁃ Count of standard compliance in the code block:
#      ◦ training: 1
#      Total: 1
#    ⁃ Count of standard violation in the code block: 0
#   ⦿ Section: Preparation of Credit Information no.8.1
#    ⁃ Count of standard compliance in the code block:
#      ◦ training: 1
#      Total: 1
#    ⁃ Count of standard violation in the code block: 0
#   ⦿ Section: Testing of the Reliability of Separation and/or Accuracy of Prediction Power of Credit Scoring
#    ⁃ Count of standard compliance in the code block:
#      ◦ testARR: 6
#      Total: 6
#    ⁃ Count of standard violation in the code block: 0
# 🔹 Absent section in the code block:
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Implementation of Credit Scoring System I
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Implementation of Credit Scoring System II
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Verification of the Applicant’s Information
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Preparation of Credit Information no.8.3
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Overrides in Credit Approval
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Construction of Credit Scoring
#  ◦ ThorPorTor. SorGorSor. (03) Wor. 277/2548 Re: Guidelines for Risk Management Practices at section Definition of Credit Scoring
# 🔹 Reference Number: cd130dc26b4cd2d2225ef14cccc66c30a6339b4ca95c272129d9d184e48185b5

# 🔵 Standard Compliance Report
# 🔹 Product Type: Credit Scoring
# 🔹 Code Block Name: credit_score_3.py
# 🔹 Analysis Result:
#   1. Standard => Credit Information Business Act B.E.2545
#   ⦿ Section: 3
#    ⁃ Count of standard compliance in the code block:
#      ◦ age: 15
#      ◦ NumberOfTime3059DaysPastDueNotWorse: 11
#      ◦ DebtRatio: 15
#      ◦ MonthlyIncome: 16
#      ◦ NumberRealEstateLoansOrLines: 10
#      ◦ NumberOfTime6089DaysPastDueNotWorse: 10
#      ◦ NumberOfDependents: 10
#      ◦ RevolvingUtilizationOfUnsecuredLines: 3
#      ◦ income: 3
#      ◦ debtRatio: 3
#      ◦ NumberOfOpenCreditLinesAndLoans: 7
#      ◦ Lines: 1
#      ◦ Age: 1
#      ◦ Income: 1
#      ◦ ageV: 3
#      ◦ loan: 2
#      Total: 111
#    ⁃ Count of standard violation in the code block: 0
# 🔹 Absent section in the code block:
#  ◦ Credit Information Business Act B.E.2545 at section 10
# 🔹 Reference Number: 6a8b835ae4287573549b7f96bcc2808de12e740254617b925132607e3abe4b79


from sqlite3.dbapi2 import Cursor

from numpy.core.fromnumeric import mean
from processOutput import initDB, closeAll
import math, numpy as np
import pylab as pl
import sys

def geomean(xs):
	return math.exp(math.fsum(math.log(x) if x > 0 else math.log(x+0.3) for x in xs) / len(xs))

def geosd(xs, mu): #geometric standard deviation
	return math.exp(math.sqrt(math.fsum((math.log(x/mu))**2 if x>0 else (math.log((x+0.3)/mu))**2 for x in xs)/len(xs)))

def getStrings(tottime, title):
	wapsTimeStr = None
	dmcTimeStr = None
	titleStr = None
	if tottime == True:
		wapsTimeStr = 'waps.total_t'
		dmcTimeStr = 'dmc.total_t'
	else:
		wapsTimeStr = 'waps.compile_t'
		dmcTimeStr = '(dmc.totpreaddcomp_t + dmc.addcomp_t)'
	if title == 'all': #not including enc1.. do it separately if needed
		titleStr = '((waps.type=\'bayes\' and dmc.type=\'bayes\') OR (waps.type=\'ps\' and dmc.type=\'ps\') OR (waps.type=\'wb\' and dmc.type=\'wb\'))'
	elif title == 'allunique':
		titleStr = '((waps.type=\'bayes\' and dmc.type=\'bayes\') OR (waps.type=\'ps\' and dmc.type=\'ps\') OR (waps.type=\'wb\' and dmc.type=\'wb\'AND (waps.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')) ))'
	elif title=='bayes':
		titleStr = '(waps.type=\'bayes\' and dmc.type=\'bayes\')'
	elif title == 'ps':
		titleStr = '(waps.type=\'ps\' and dmc.type=\'ps\')'
	elif title == 'db': #dpmc benchmarks
		titleStr = '((waps.type=\'bayes\' and dmc.type=\'bayes\') OR (waps.type=\'ps\' and dmc.type=\'ps\'))'
	elif title == 'wb':
		titleStr = '(waps.type=\'wb\' and dmc.type=\'wb\')'
	else:
		print("Title not recognized. Exiting..")
		sys.exit(1)
	return wapsTimeStr,dmcTimeStr,titleStr
# (waps.type='bayes' and cudd.type='bayes') AND

def calcSpeedups(cur:Cursor, tottime:bool, dmc_lib:str, title:str):
	wapsTimeStr,dmcTimeStr,titleStr = getStrings(tottime, title)
	rows = cur.execute("SELECT "+wapsTimeStr+"/"+dmcTimeStr+" from waps, "+dmc_lib+"_632ed69plus as dmc  WHERE "+titleStr+" AND (waps.name = dmc.name) AND (waps.type = dmc.type) AND ("+dmcTimeStr+" < 1000) AND ("+wapsTimeStr+" < 1000)").fetchall()
	mu = geomean(next(zip(*rows)))
	gsd = geosd(next(zip(*rows)), mu)
	print('geo-mean: ',mu,'  geo-sd:',gsd)

def calcFinished(cur:Cursor, tottime:bool, dmc_lib:str, title:str):
	wapsTimeStr,dmcTimeStr,titleStr = getStrings(tottime, title)
	both = cur.execute("SELECT COUNT(1) from waps, "+dmc_lib+"_632ed69plus as dmc  WHERE "+titleStr+" AND (waps.name = dmc.name) AND (waps.type = dmc.type) AND ("+dmcTimeStr+" < 1000) AND ("+wapsTimeStr+" < 1000)").fetchall()
	only_waps = cur.execute("SELECT COUNT(1) from waps, "+dmc_lib+"_632ed69plus as dmc  WHERE "+titleStr+" AND (waps.name = dmc.name) AND (waps.type = dmc.type) AND ("+dmcTimeStr+" IS NULL) AND ("+wapsTimeStr+" < 1000)").fetchall()
	only_dmc = cur.execute("SELECT COUNT(1) from waps, "+dmc_lib+"_632ed69plus as dmc  WHERE "+titleStr+" AND (waps.name = dmc.name) AND (waps.type = dmc.type) AND ("+dmcTimeStr+" < 1000) AND ("+wapsTimeStr+" IS NULL)").fetchall()
	print ("Both:",both[0][0]," only_waps:",only_waps[0][0]," only_dmc:",only_dmc[0][0])

def calcFastest(cur:Cursor, dmc_lib:str, title:str):
	# not computing for compile time because 'fastest' becomes blury.
	# for eg we only consider ddnnf compilation time as compilation for waps, and not include time for annotation
	# as that is done in python (slow) and we want to be fair. so while its okay for num_finished, not much sense
	# for num_fastest
	wapsTimeStr,dmcTimeStr,titleStr = getStrings(tottime=True, title=title)
	only_waps = cur.execute("SELECT COUNT(1) from waps, "+dmc_lib+"_632ed69plus as dmc  WHERE "+titleStr+" AND (waps.name = dmc.name) AND (waps.type = dmc.type) AND (( ("+dmcTimeStr+" < 1000) AND ("+wapsTimeStr+" < 1000) AND ("+dmcTimeStr+">"+wapsTimeStr+")) OR (("+wapsTimeStr+" < 1000) AND ("+dmcTimeStr+" IS NULL)))").fetchall()
	only_dmc = cur.execute("SELECT COUNT(1) from waps, "+dmc_lib+"_632ed69plus as dmc  WHERE "+titleStr+" AND (waps.name = dmc.name) AND (waps.type = dmc.type) AND (( ("+dmcTimeStr+" < 1000) AND ("+wapsTimeStr+" < 1000) AND ("+wapsTimeStr+">"+dmcTimeStr+")) OR (("+dmcTimeStr+" < 1000) AND ("+wapsTimeStr+" IS NULL)))").fetchall()
	print ("fastest_waps:",only_waps[0][0]," fastest_dmc:",only_dmc[0][0])

def plotSpeedupTWs(cur:Cursor, tottime:bool, dmc_lib:str, title:str):
	wapsTimeStr,dmcTimeStr,titleStr = getStrings(tottime, title)

	rows = cur.execute("SELECT "+wapsTimeStr+"/"+dmcTimeStr+", dmc.jt_width from waps, "+dmc_lib+"_632ed69plus as dmc  WHERE "+titleStr+" AND (waps.name = dmc.name) AND (waps.type = dmc.type) AND ("+dmcTimeStr+" < 1000) AND ("+wapsTimeStr+" < 1000) ORDER BY dmc.jt_width").fetchall()
	gms = []
	currJt = None
	currSUs = []
	for r in rows:
		newJt = r[1]
		if currJt == None: #first item
			currJt = newJt
			currSUs.append(r[0])
			continue
		if newJt != currJt:
			newGm = geomean(currSUs)
			gms.append((currJt,newGm, max(currSUs),min(currSUs)))
			currSUs = [r[0]]
			currJt = newJt
			continue
		assert newJt == currJt
		currSUs.append(r[0])
	colors=['green','blue','red','brown','orange']
	mkrs= ['o','^','+','x','D']
	pl.plot([g[0] for g in gms],[g[1] for g in gms],label='Geometric Means', linewidth=1.0, color=colors[0], marker='x')
	#pl.errorbar([g[0] for g in gms],[g[1] for g in gms],[[g[1]-g[3] for g in gms],[g[2]-g[1] for g in gms]])
	# for g in gms:
		# pl.errorbar(g[0],g[2]-g[3],bottom=g[3])
	pl.rcParams['pdf.fonttype'] = 42
	pl.rcParams['ps.fonttype'] = 42
	pl.legend(loc='lower right')
	pl.tick_params(labelsize=16)
	#pl.ylim(0,1010)
	#pl.xlim(0,1090)
	pl.xlabel('Treewidths', fontsize=20)
	pl.ylabel('Speedup', fontsize=20)
	pl.title(title+' '+dmc_lib+' '+wapsTimeStr.split('.')[1], fontsize=24)
	pl.axhline(y=1, color='r', linestyle='--')
	#pl.savefig('graphs/cactus_completed_'+config[1]+'.eps',bbox_inches='tight')
	pl.show()

def processPar2Rows(rows):
	res = {}
	for r in rows:
		jtw = r[4]
		wTime = r[1] if r[1] != None else 2000
		cTime = r[2] if r[2] != None else 2000
		sTime = r[3] if r[3] != None else 2000
		name = r[0]
		if jtw in res:
			res[jtw][0].append(name)
			res[jtw][1].append(wTime)
			res[jtw][2].append(cTime)
			res[jtw][3].append(sTime)
		else:
			res[jtw] = [[name],[wTime],[cTime],[sTime]]
	return res

def plotPar2TWs(cur:Cursor, title:str, windowSize, avg): #, tottime:bool, dmc_lib:str, title:str):
	if title == 'all': #not including enc1.. do it separately if needed
		titleStr = '((waps.type=\'bayes\' and cudd.type=\'bayes\') OR (waps.type=\'ps\' and cudd.type=\'ps\') OR (waps.type=\'wb\' and cudd.type=\'wb\'))'
	elif title == 'allunique':
		titleStr = '((waps.type=\'bayes\' and cudd.type=\'bayes\') OR (waps.type=\'ps\' and cudd.type=\'ps\') OR (waps.type=\'wb\' and cudd.type=\'wb\'AND (waps.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')) ))'
	elif title=='bayes':
		titleStr = '(waps.type=\'bayes\' and cudd.type=\'bayes\')'
	elif title == 'ps':
		titleStr = '(waps.type=\'ps\' and cudd.type=\'ps\')'
	elif title == 'db':
		titleStr = '((waps.type=\'bayes\' and cudd.type=\'bayes\') OR (waps.type=\'ps\' and cudd.type=\'ps\'))'
	elif title == 'wb':
		titleStr = '(waps.type=\'wb\' and cudd.type=\'wb\')'
	else:
		print("Title not recognized. Exiting..")
		sys.exit(1)
	rows = cur.execute("SELECT waps.name, waps.total_t, cudd.total_t, sylvan.total_t, cudd.jt_width from waps, cudd_632ed69plus as cudd, sylvan_632ed69plus as sylvan  WHERE  (waps.name = cudd.name) AND (waps.name = sylvan.name) AND (waps.type = cudd.type) AND (waps.type = sylvan.type) AND "+titleStr+" AND cudd.jt_width > 0 ORDER BY cudd.jt_width").fetchall()
	timemap = processPar2Rows(rows)

	jtws = sorted(timemap.keys())
	# print(jtws,timemap[jtws[10]])
	l = len(jtws)
	assert l > windowSize and windowSize>0
	res = []
	for i in range(windowSize,l):
		res.append([])
		windows = [[],[],[],[]] #jtws, wtimes, ctimes, stimes
		for j in range(i-windowSize,i):
			windows[0].append(jtws[j])
			times = timemap[jtws[j]]
			windows[1] += times[1]
			windows[2] += times[2]
			windows[3] += times[3]
		res[-1] += [avg(windows[0]),avg(windows[1]),avg(windows[2]),avg(windows[3])]
		for i,val in enumerate(res[-1]):
			if math.isnan(val):
				print(i,windows,res[-1][i])
				sys.exit(1)
	# print(res)
	colors=['green','blue','red','brown','orange']
	mkrs= ['o','^','+','x','D']
	for item in [[1,'WAPS'],[2,'DPSampler']]:#,[3,'DPSampler w/ Sylvan']]:
	 	pl.plot([g[0] for g in res],[g[item[0]] for g in res],label=item[1], linewidth=1.0, color=colors[item[0]])

	# pl.plot([g[0] for g in res],[g[1] for g in res],label='WAPS', linewidth=1.0, color=colors[0])

	pl.rcParams['pdf.fonttype'] = 42
	pl.rcParams['ps.fonttype'] = 42
	pl.legend(loc='lower right', fontsize=12)
	pl.tick_params(labelsize=14)
	#pl.ylim(0,1010)
	#pl.xlim(0,1090)
	pl.xlabel('Mean of '+str(windowSize)+' project-join treewidths',fontsize=16)#, fontsize=20)
	pl.ylabel('Mean PAR2 score of '+str(windowSize)+' widths',fontsize=16)#, fontsize=20)
	#pl.title(title+' '+dmc_lib+' '+wapsTimeStr.split('.')[1], fontsize=24)
	#pl.axhline(y=1, color='r', linestyle='--')
	pl.savefig('result_analysis/graphs/dmc_632ed69+/par2_'+title+'.eps',bbox_inches='tight')
	pl.show()

def plotCactus(cur:Cursor, title: str, caption:str, numBench): #, tottime:bool, dmc_lib:str, title:str):
	# assert title in ['bayes', 'ps']
	# not doing allunique because the graphs are plotted separately anyway
	if title=='all':
		title1 = 'bayes'
		title2 = 'ps'
		title3 = 'wb'
	elif title=='db':
		title1 = 'bayes'
		title2 = 'ps'
		title3 = 'ps'
	else:
		title1 = title
		title2 = title
		title3 = title
	rows_waps=cur.execute('SELECT t1.total_t, (SELECT COUNT(1) FROM waps t2 WHERE ((t2.type=\''+title1+'\') OR (t2.type=\''+title2+'\') OR (t2.type=\''+title3+'\'))  AND (t2.total_t <= t1.total_t)) AS num_complete FROM waps t1 WHERE ((t1.type=\''+title1+'\') OR (t1.type=\''+title2+'\') OR (t1.type=\''+title3+'\')) GROUP BY t1.total_t ORDER BY t1.total_t').fetchall()
	rows_cudd=cur.execute('SELECT t1.total_t, (SELECT COUNT(1) FROM cudd_632ed69plus t2 WHERE ((t2.type=\''+title1+'\') OR (t2.type=\''+title2+'\') OR (t2.type=\''+title3+'\')) AND (t2.total_t <= t1.total_t)) AS num_complete FROM cudd_632ed69plus t1 WHERE ((t1.type=\''+title1+'\') OR (t1.type=\''+title2+'\') OR (t1.type=\''+title3+'\')) GROUP BY t1.total_t ORDER BY t1.total_t').fetchall()
	# rows_sylvan=cur.execute('SELECT t1.total_t, (SELECT COUNT(1) FROM sylvan_632ed69plus t2 WHERE (t2.type=\''+title+'\') AND (t2.total_t <= t1.total_t)) AS num_complete FROM sylvan_632ed69plus t1 WHERE (t1.type=\''+title+'\') GROUP BY t1.total_t ORDER BY t1.total_t').fetchall()
	colors=['green','blue','red','brown','orange']
	mkrs= ['o','^','+','x','D']
	pl.plot([r[1] for r in rows_waps]+[rows_waps[-1][1]],[r[0] for r in rows_waps]+[1000],label='WAPS', linewidth=3.0, color=colors[2])
	pl.plot([r[1] for r in rows_cudd]+[rows_cudd[-1][1]],[r[0] for r in rows_cudd]+[1000],label='DPSampler', linewidth=3.0, color=colors[0])
	# pl.plot([r[1] for r in rows_sylvan],[r[0] for r in rows_sylvan],label='DPSampler w/ Sylvan', linewidth=3.0, color=colors[1])
	print([rows_waps[-1][1]], [rows_cudd[-1][1]])
	pl.rcParams['pdf.fonttype'] = 42
	pl.rcParams['ps.fonttype'] = 42
	pl.legend(loc='upper left',fontsize=12)
	pl.tick_params(labelsize=16)
	pl.ylim(0,1010)
	# pl.yscale('log')
	pl.xlim(0,numBench+10)
	pl.ylabel('Total Time (seconds)', fontsize=20)
	pl.xlabel('Number of benchmarks solved', fontsize=20)
	#pl.title(caption+' Benchmark Set', fontsize=24)
	pl.axhline(y=1000, color='r', linestyle='--')
	pl.savefig('result_analysis/graphs/dmc_632ed69+/cactus_'+title+'.eps',bbox_inches='tight')
	pl.show()

def plotSamplingComparison():
	# see /home/adi/Downloads/dpsampler/results/dmc_632ed69+/sampling_time_comparison/notes.txt for data source
	data_x = [319,679,1004,1612,2075]
	data_y_num = [3.779, 30.97, 31.57, 265.36, 771.02]
	data_y_den = [0.211, 1.2, 2.046, 3.827, 5.452]
	data_y = [data_y_num[i]/data_y_den[i] for i in range(len(data_y_den))]
	colors=['green','blue','red','brown','orange']
	mkrs= ['o','^','+','x','D']
	pl.scatter(data_x,data_y, color=colors[0], marker='x')
	pl.plot(np.unique(data_x), np.poly1d(np.polyfit(data_x, data_y, 1))(np.unique(data_x)), label='Best-Fit Line')
	#pl.errorbar([g[0] for g in gms],[g[1] for g in gms],[[g[1]-g[3] for g in gms],[g[2]-g[1] for g in gms]])
	# for g in gms:
		# pl.errorbar(g[0],g[2]-g[3],bottom=g[3])
	pl.rcParams['pdf.fonttype'] = 42
	pl.rcParams['ps.fonttype'] = 42
	pl.legend(loc='upper left',fontsize=12)
	pl.tick_params(labelsize=16)
	#pl.ylim(0,1010)
	#pl.xlim(0,1090)
	pl.xlabel('# nodes in Project-Join Tree', fontsize=20)
	pl.ylabel('Speedup', fontsize=20)
	#pl.title('Speedup of top-down sampling relative to bottom-up', fontsize=24)
	pl.axhline(y=1, color='r', linestyle='--')
	pl.savefig('result_analysis/graphs/dmc_632ed69+/sampling_comparison.eps',bbox_inches='tight')
	pl.show()

def plotSamplingComparison2(cur:Cursor, title: str):
	# bayes and ps comparison for iJCAI 22 (so no need for all unique)
	dmc1TimeStr = '(dmc1.sampgen_t)'
	dmc2TimeStr = '(dmc2.sampgen_t)'
	if title == 'all': #not including enc1.. do it separately if needed
		titleStr = '((dmc1.type=\'bayes\' and dmc2.type=\'bayes\') OR (dmc1.type=\'ps\' and dmc2.type=\'ps\'))'
	elif title=='bayes':
		titleStr = '(dmc1.type=\'bayes\' and dmc2.type=\'bayes\')'
	elif title == 'ps':
		titleStr = '(dmc1.type=\'ps\' and dmc2.type=\'ps\')'
	speedup_rows = cur.execute("SELECT "+dmc2TimeStr+"/"+dmc1TimeStr+" from cudd_632ed69plus dmc1, budmc dmc2  WHERE "+titleStr+" AND (dmc1.name = dmc2.name) AND (dmc1.type = dmc2.type) AND ("+dmc1TimeStr+" < 1000) AND ("+dmc2TimeStr+" < 1000)").fetchall()
	mu = geomean(next(zip(*speedup_rows)))
	gsd = geosd(next(zip(*speedup_rows)),mu)
	print ('geomean:',mu,' geosd:',gsd)
	rows = cur.execute("SELECT COUNT(*) from cudd_632ed69plus dmc WHERE (dmc.type='ps' OR dmc.type='bayes') AND (dmc.total_t < 1000)").fetchall()
	print('Total instances solved by top down',rows)
	rows = cur.execute("SELECT COUNT(*) from budmc dmc WHERE (dmc.type='ps' OR dmc.type='bayes') AND (dmc.total_t < 1000)").fetchall()
	print('Total instances solved by bottomup',rows)
	rows = cur.execute("SELECT "+dmc2TimeStr+"/"+dmc1TimeStr+",dmc2.declaredNodeCount from cudd_632ed69plus dmc1, budmc dmc2  WHERE "+titleStr+" AND (dmc1.name = dmc2.name) AND (dmc1.type = dmc2.type) AND ("+dmc1TimeStr+" < 1000) AND ("+dmc2TimeStr+" < 1000)").fetchall()
	from matplotlib import rc
	rc('text', usetex=True)
	colors=['green','blue','red','brown','orange']
	mkrs= ['o','^','+','x','D']
	speedups, nodecounts = zip(*rows)
	pl.scatter(nodecounts,speedups, color=colors[0], marker='x')
	pl.plot(np.unique(nodecounts), np.poly1d(np.polyfit(nodecounts,speedups, 1))(np.unique(nodecounts)), label='Best-Fit Line')
	#pl.errorbar([g[0] for g in gms],[g[1] for g in gms],[[g[1]-g[3] for g in gms],[g[2]-g[1] for g in gms]])
	# for g in gms:
		# pl.errorbar(g[0],g[2]-g[3],bottom=g[3])
	pl.rcParams['pdf.fonttype'] = 42
	pl.rcParams['ps.fonttype'] = 42
	pl.legend(loc='upper right',fontsize=12)
	pl.tick_params(labelsize=16)
	pl.ylim(-5,280)
	pl.xlim(0,13000)
	pl.ticklabel_format(style='sci', axis='x', scilimits=(-3,4))
	pl.xlabel('\# nodes in Project-Join Tree', fontsize=20)
	pl.ylabel('Speedup', fontsize=20)
	#pl.title('Speedup of top-down sampling relative to bottom-up', fontsize=24)
	pl.axhline(y=1, color='r', linestyle='--')
	pl.savefig('result_analysis/graphs/dmc_632ed69+/sampling_comparison2.png',bbox_inches='tight')
	pl.show()

def calcSylvanCuddSpeedup(cur: Cursor, title:str):
	dmc1TimeStr = '(dmc1.totpreaddcomp_t + dmc1.addcomp_t)'
	dmc2TimeStr = '(dmc2.totpreaddcomp_t + dmc2.addcomp_t)'
	if title == 'all': #not including enc1.. do it separately if needed
		titleStr = '((dmc1.type=\'bayes\' and dmc2.type=\'bayes\') OR (dmc1.type=\'ps\' and dmc2.type=\'ps\') OR (dmc1.type=\'wb\' and dmc2.type=\'wb\'))'
	elif title == 'allunique':
		titleStr = '((dmc1.type=\'bayes\' and dmc2.type=\'bayes\') OR (dmc1.type=\'ps\' and dmc2.type=\'ps\') OR (dmc1.type=\'wb\' and dmc2.type=\'wb\'AND (dmc2.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')) ))'
	elif title=='bayes':
		titleStr = '(dmc1.type=\'bayes\' and dmc2.type=\'bayes\')'
	elif title == 'ps':
		titleStr = '(dmc1.type=\'ps\' and dmc2.type=\'ps\')'
	elif title == 'wb':
		titleStr = '(dmc1.type=\'wb\' and dmc2.type=\'wb\')'
	elif title == 'db':
		titleStr = '((dmc1.type=\'bayes\' and dmc2.type=\'bayes\') OR (dmc1.type=\'ps\' and dmc2.type=\'ps\'))'
	rows = cur.execute("SELECT "+dmc1TimeStr+"/"+dmc2TimeStr+" from cudd_632ed69plus dmc1, sylvan_632ed69plus dmc2  WHERE "+titleStr+" AND (dmc1.name = dmc2.name) AND (dmc1.type = dmc2.type) AND ("+dmc1TimeStr+" < 1000) AND ("+dmc2TimeStr+" < 1000)").fetchall()
	print (geomean(next(zip(*rows))))

def failureAnalysis(cur):
	rows = cur.execute("SELECT COUNT(*) FROM cudd_632ed69plus dmc WHERE (dmc.type='bayes' OR dmc.type='ps' OR (dmc.type='wb'AND (dmc.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\'))))").fetchall()
	print('Total unique benchmarks:',rows)
	rows = cur.execute("SELECT COUNT(*) FROM cudd_632ed69plus dmc WHERE (dmc.type='bayes' OR dmc.type='ps' OR (dmc.type='wb'AND (dmc.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')))) AND (dmc.total_t IS NULL)").fetchall()
	print('Unique Benchmarks where dmc (cudd) failed to generate 5000 samples',rows)
	rows = cur.execute("SELECT COUNT(*) FROM cudd_632ed69plus dmc WHERE (dmc.type='bayes' OR dmc.type='ps' OR (dmc.type='wb'AND (dmc.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')))) AND (dmc.total_t IS NULL) AND (dmc.planner_t IS NULL)").fetchall()
	print('Unique Benchmarks where dmc (cudd) failed to generate 5000 samples and (because) no pj-tree was computed',rows)
	rows = cur.execute("SELECT COUNT(*) FROM cudd_632ed69plus dmc, waps WHERE (dmc.type='bayes' OR dmc.type='ps' OR (dmc.type='wb'AND (dmc.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')))) AND (waps.type=dmc.type) AND (waps.name=dmc.name) AND (waps.total_t <1000) AND (dmc.total_t IS NULL)").fetchall()
	print('Total unique benchmarks where Waps succeeded but dmc failed',rows)
	rows = cur.execute("SELECT COUNT(*) FROM cudd_632ed69plus dmc, waps WHERE (dmc.type='bayes' OR dmc.type='ps' OR (dmc.type='wb'AND (dmc.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')))) AND (waps.type=dmc.type) AND (waps.name=dmc.name) AND (waps.total_t <1000) AND (dmc.total_t IS NULL) AND (dmc.planner_t IS NULL)").fetchall()
	print('Total unique benchmarks where Waps succeeded but dmc failed and (because) pj-tree was not constructed',rows)
	rows = cur.execute("SELECT COUNT(*) FROM cudd_632ed69plus dmc WHERE (dmc.type='bayes' OR dmc.type='ps' OR (dmc.type='wb'AND (dmc.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')))) AND (dmc.planner_t < 1000)").fetchall()
	print('Total unique benchmarks where pj-tree was constructed',rows)
	denom = rows[0][0]
	rows = cur.execute("SELECT COUNT(*)/"+str(denom)+".0 FROM cudd_632ed69plus dmc WHERE (dmc.type='bayes' OR dmc.type='ps' OR (dmc.type='wb'AND (dmc.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')))) AND (dmc.planner_t < 1000) AND (dmc.addcomp_t < 1000)").fetchall()
	print('fraction of unique pj-tree constructed where add compilation succeeded',rows)
	rows = cur.execute("SELECT COUNT(*), COUNT(*)/"+str(denom)+".0 FROM cudd_632ed69plus dmc WHERE (dmc.type='bayes' OR dmc.type='ps' OR (dmc.type='wb'AND (dmc.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')))) AND (dmc.planner_t < 1000) AND (dmc.addcomp_t < 1000) AND (dmc.total_t < 1000)").fetchall()
	print('fraction of unique pj-tree constructed which were fully solved',rows)

def main():
	con, cur = initDB()
	#not including enc1 in all. Do it separately if needed.
	#rows = cur.execute("SELECT COUNT(*) from cudd_632ed69plus dmc, waps WHERE (dmc.name=waps.name) AND (dmc.type=waps.type) AND (waps.type='bayes' OR waps.type='ps' OR (waps.type=\'wb\' AND (waps.name NOT IN (SELECT waps2.name FROM waps waps2 WHERE waps2.type=\'bayes\' OR waps2.type=\'ps\')))) AND (dmc.total_t < 1000 AND waps.total_t < 1000)").fetchall()
	#print(rows)
	#sys.exit(1)
	print("Geometric Mean of Speedups relative to WAPS:")
	for t in ['totaltime', 'compiletime']:
		lib='cudd'
		for title in ['db','wb','all','allunique']:
			print("	"+t+" "+lib+" "+title+":",end=" ")
			calcSpeedups(cur,t=='totaltime',lib,title)
	print("Number of benchmarks finished:")
	for t in ['totaltime', 'compiletime']:
		lib='cudd'
		for title in ['db','wb','all','allunique']:
			print("	"+t+" "+lib+" "+title+":",end=" ")
			calcFinished(cur,t=='totaltime',lib,title)
	print("Number of benchmarks fastest on (total time):")
	lib='cudd'
	for title in ['db','wb','all','allunique']:
		print("	"+lib+" "+title+":",end=" ")
		calcFastest(cur,lib,title)

	for title in [['db','db',1945],['wb','wb', 773]]:
		plotCactus(cur,title[0],title[1],title[2])
	# plotSamplingComparison()

	print("Geometric Mean of Compile Time Sylvan Speedups relative to CUDD:")
	for title in ('db','wb','all','allunique'):
		print(title,end=': ')
		calcSylvanCuddSpeedup(cur, title)

	plotPar2TWs(cur,'allunique', 10,np.mean)

	print('Top vs bottom')
	plotSamplingComparison2(cur,'all')

	failureAnalysis(cur)
	#plotPar2TWs(cur,'bayes', 10,np.mean)
	#plotPar2TWs(cur,'ps', 10,np.mean)
	#plotPar2TWs(cur,'wb', 10,np.mean)

	# for t in ['totaltime', 'compiletime']:
	# 	for lib in ['cudd', 'sylvan']:
	# 		for title in ['bayes','ps','enc1','all']:
	# 			plotSpeedupTWs(cur,t=='totaltime',lib,title)

	#print('DPS(cudd) vs WAPS')
	#calcSpeedups(cur, False, 'cudd','wb')
	#calcFinished(cur, False, 'cudd','wb')
	#calcFastest(cur, 'cudd','wb')
	#print('CUDD vs Sylvan')
	#calcSylvanCuddSpeedup(cur, 'wb')

	closeAll(con, cur)

if __name__=="__main__":
   main()


"""Constants for Zadnego Ale integration."""
from __future__ import annotations

from datetime import timedelta
from typing import Final

from homeassistant.components.sensor import SensorEntityDescription

ATTRIBUTION: Final = (
    "Dane dostarczone przez Ośrodek Badania Alergenów Środowiskowych Sp. z o.o."
)

ATTR_LEVEL: Final = "level"
ATTR_TREND: Final = "trend"
ATTR_VALUE: Final = "value"

CONF_REGION: Final = "region"

DEFAULT_NAME: Final = "Żadnego Ale"
DEFAULT_UPDATE_INTERVAL = timedelta(minutes=60)
DOMAIN: Final = "zadnego_ale"

REGIONS: Final[tuple[str, ...]] = (
    "Wybrzeże",
    "Pomorze",
    "Warmia, Mazury i Podlasie",
    "Ziemia Lubuska",
    "Śląsk i Wielkopolska",
    "Mazowsze i Ziemia Łódzka",
    "Małopolska i Ziemia Lubelska",
    "Sudety",
    "Karpaty",
)

SENSORS: Final[tuple[SensorEntityDescription, ...]] = (
    SensorEntityDescription(
        key="alder",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Alder pollen concentration",
    ),
    SensorEntityDescription(
        key="alternaria",
        device_class="zadnego_ale__concentration",
        icon="mdi:mushroom-outline",
        name="Alternaria pollen concentration",
    ),
    SensorEntityDescription(
        key="ash_tree",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Ash Tree pollen concentration",
    ),
    SensorEntityDescription(
        key="beech",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Beech pollen concentration",
    ),
    SensorEntityDescription(
        key="birch_tree",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Birch Tree pollen concentration",
    ),
    SensorEntityDescription(
        key="cladosporium",
        device_class="zadnego_ale__concentration",
        icon="mdi:mushroom-outline",
        name="Cladosporium pollen concentration",
    ),
    SensorEntityDescription(
        key="elm",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Elm pollen concentration",
    ),
    SensorEntityDescription(
        key="goldenrod",
        device_class="zadnego_ale__concentration",
        icon="mdi:grass",
        name="Goldenrod pollen concentration",
    ),
    SensorEntityDescription(
        key="grass",
        device_class="zadnego_ale__concentration",
        icon="mdi:grass",
        name="Grass pollen concentration",
    ),
    SensorEntityDescription(
        key="hazel",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Hazel pollen concentration",
    ),
    SensorEntityDescription(
        key="hornbeam",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Hornbeam pollen concentration",
    ),
    SensorEntityDescription(
        key="maple",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Maple pollen concentration",
    ),
    SensorEntityDescription(
        key="mugwort",
        device_class="zadnego_ale__concentration",
        icon="mdi:sprout",
        name="Mugwort pollen concentration",
    ),
    SensorEntityDescription(
        key="nettle",
        device_class="zadnego_ale__concentration",
        icon="mdi:sprout",
        name="Nettle pollen concentration",
    ),
    SensorEntityDescription(
        key="oak",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Oak pollen concentration",
    ),
    SensorEntityDescription(
        key="pigweed",
        device_class="zadnego_ale__concentration",
        icon="mdi:sprout",
        name="Pigweed pollen concentration",
    ),
    SensorEntityDescription(
        key="pine",
        device_class="zadnego_ale__concentration",
        icon="mdi:pine-tree",
        name="Pine pollen concentration",
    ),
    SensorEntityDescription(
        key="plane_tree",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Plane Tree pollen concentration",
    ),
    SensorEntityDescription(
        key="plantain",
        device_class="zadnego_ale__concentration",
        icon="mdi:sprout",
        name="Plantain pollen concentration",
    ),
    SensorEntityDescription(
        key="poplar",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Poplar pollen concentration",
    ),
    SensorEntityDescription(
        key="ragweed",
        device_class="zadnego_ale__concentration",
        icon="mdi:sprout",
        name="Ragweed pollen concentration",
    ),
    SensorEntityDescription(
        key="sorrel",
        device_class="zadnego_ale__concentration",
        icon="mdi:sprout",
        name="Sorrel pollen concentration",
    ),
    SensorEntityDescription(
        key="willow",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Willow pollen concentration",
    ),
    SensorEntityDescription(
        key="yew",
        device_class="zadnego_ale__concentration",
        icon="mdi:tree",
        name="Yew pollen concentration",
    ),
)

SENSORS_MIGRATION: Final[list[tuple[str, str]]] = [
    ("ambrozja", "ragweed"),
    ("babka", "plantain"),
    ("brzoza", "birch_tree"),
    ("buk", "beech"),
    ("bylica", "mugwort"),
    ("cis", "yew"),
    ("dąb", "oak"),
    ("grab", "hornbeam"),
    ("jesion", "ash_tree"),
    ("klon", "maple"),
    ("komosa", "pigweed"),
    ("leszczyna", "hazel"),
    ("olsza", "alder"),
    ("platan", "plane_tree"),
    ("pokrzywa", "nettle"),
    ("sosna", "pine"),
    ("szczaw", "sorrel"),
    ("topola", "poplar"),
    ("trawy", "grass"),
    ("wierzba", "willow"),
    ("wiąz", "elm"),
]


import os.path
import yaml
import sys
import traceback
import luigi
import luigi.local_target
import pieshell
import traceback
import math
import time
import requests
import poltergust_luigi_utils # Add GCS luigi opener

DB_URL = os.environ.get("DB_URL")

def make_environment(envpath, environment, log):
    _ = pieshell.env(exports=dict(pieshell.env._exports))
    if not os.path.exists(envpath):
        envdir = os.path.dirname(envpath)
        if not os.path.exists(envdir):
            os.makedirs(envdir)
        +_.virtualenv(envpath, **environment.get("virtualenv", {}))
    +_.bashsource(envpath + "/bin/activate")
    for dep in environment["dependencies"]:
        for line in _.pip.install(dep):
            log(line)

def iter_to_pcnt(x, a=0.5):
    return 100 * a * math.log(1 + x) / (1 + a*math.log(1 + x))

class Logging(object):
    def on_failure(self, exception):
        traceback_string = traceback.format_exc()
        msg = "Runtime error:\n%s" % traceback_string
        self._log(msg)
        self.close_log()
        return msg

    def on_success(self):
        self._log("DONE")
        self.close_log()

    def close_log(self):
        with self.logfile().open("w") as f:
            f.write("\n".join(self.msgs) + "\n")

    def _log(self, msg):
        if not hasattr(self, "msgs"):
            self.msgs = []
        print(msg)
        self.msgs.append(msg)

    def log(self, msg):
        self._log(msg)
        self.set_status_message("\n".join(self.msgs))
        self.set_progress_percentage(iter_to_pcnt(len(self.msgs)))

class MakeEnvironment(Logging, luigi.Task):
    path = luigi.Parameter()
    hostname = luigi.Parameter()

    def run(self):
        with luigi.contrib.opener.OpenerTarget(self.path).open("r") as f:
            environment = yaml.load(f, Loader=yaml.SafeLoader)
        make_environment(self.envdir().path, environment, self.log)
        with self.output().open("w") as f:
            f.write("DONE")

    def envdir(self):
        return luigi.local_target.LocalTarget(
            os.path.join("/tmp/environments", self.path.replace("://", "/").lstrip("/")))

    def logfile(self):
        return luigi.contrib.opener.OpenerTarget("%s.%s.log.txt" % (self.path, self.hostname))

    def output(self):
        return luigi.local_target.LocalTarget(
            os.path.join("/tmp/environments", self.path.replace("://", "/").lstrip("/"), "done"))

class RunTask(Logging, luigi.Task):
    path = luigi.Parameter()
    hostname = luigi.Parameter()
    retry_on_error = luigi.Parameter(default=False)

    @property
    def scheduler(self):
        return self.set_progress_percentage.__self__._scheduler

    @property
    def scheduler_url(self):
        return self.scheduler._url

    def run(self):
        src = '%s.config.yaml' % (self.path,)
        fs = self.output().fs

        try:
            cfg = luigi.contrib.opener.OpenerTarget('%s.config.yaml' % (self.path,))
            try:
                with cfg.open("r") as f:
                    task = yaml.load(f, Loader=yaml.SafeLoader)
            except:
                if not cfg.fs.exists(cfg.path):
                    # The task has already been marked as done by another worker.
                    return
                raise

            with luigi.contrib.opener.OpenerTarget(task["environment"]).open("r") as f:
                environment = yaml.load(f, Loader=yaml.SafeLoader)

            env = MakeEnvironment(path=task["environment"], hostname=self.hostname)
            yield env
            envpath = env.envdir().path

            _ = pieshell.env(envpath, interactive=True)
            +_.bashsource(envpath + "/bin/activate")

            _._exports.update(environment.get("variables", {}))
            _._exports.update(task.get("variables", {}))

            command = task.get("command", None)

            task_args = dict(task.get("task", {}))
            task_name = task_args.pop("name", None)
            task_args["scheduler-url"] = self.scheduler_url
            task_args["retcode-already-running"] = "10"
            task_args["retcode-missing-data"] = "20"
            task_args["retcode-not-run"] = "25"
            task_args["retcode-task-failed"] = "30"
            task_args["retcode-scheduling-error"] = "35"
            task_args["retcode-unhandled-exception"] = "40"

            if command is None:
                command = "luigi(task_name, **task_args)"

            scope = pieshell.environ.EnvScope(env=_)
            scope["task_name"] = task_name
            scope["task_args"] = task_args

            # Rerun the task until it is actually being run (or is done!)
            # We need to do this, since luigi might exit because all
            # dependencies of a task are already being run by other
            # workers, and our task can't even start...
            while True:
                try:
                    for line in eval(command, scope):
                        self.log(line)
                except pieshell.PipelineFailed as e:
                    self.log(str(e))
                    if e.pipeline.exit_code <= 25:
                        time.sleep(5)
                        continue
                    raise
                break

        except Exception as e:
            self.on_failure(e)
            if self.retry_on_error:
                raise
            dst = '%s.error.yaml' % (self.path,)
            try:
                fs.move(src, dst)
            except:
                # Might already have been moved by another node...
                pass
        else:
            self.on_success()
            dst = '%s.done.yaml' % (self.path,)
            try:
                fs.move(src, dst)
            except:
                # Might already have been moved by another node...
                pass

        if DB_URL is not None:
            r = requests.get(DB_URL, params={"pipeline_url": self.path})
            if r.status_code != 200:
                print("Unable to update status", r.status_code, r.text)

    def logfile(self):
        return luigi.contrib.opener.OpenerTarget("%s.%s.log.txt" % (self.path, self.hostname))

    def output(self):
         return luigi.contrib.opener.OpenerTarget(
             '%s.done.yaml' % (self.path,))

class RunTasks(luigi.Task):
    path = luigi.Parameter()
    hostname = luigi.Parameter()

    def requires(self):
        return [RunTask(path=path.replace(".config.yaml", ""), hostname=self.hostname)
                for path in self.output().fs.list_wildcard('%s/*.config.yaml' % (self.path,))]

    def run(self):
        pass

    def output(self):
        # This target should never actually be created...
        return luigi.contrib.opener.OpenerTarget('%s/done' % (self.path,))

# luigi --module emerald_algorithms_evaluation.luigi Pipeline --param-evaluation_name=test-luigi-redhog-1


from .. import schemas, models
from src.base.services import BaseService
from tortoise.transactions import atomic

class AreasService(BaseService):
    model = models.Area
    create_schema = schemas.CreateArea
    update_schema = schemas.UpdateArea
    get_schema = schemas.GetArea
    get_detail = schemas.GetArea

    @atomic()
    async def update(self, unique_id: int, schema: schemas.UpdateArea) -> schemas.GetArea:
        await models.AreaPoint.filter(area_id=unique_id).delete()
        await models.Area.filter(id=unique_id).update(name=schema.name)
        for point in schema.points:
            await models.AreaPoint.create(**point.dict(exclude_unset=True), area_id=unique_id)
        return await self.get(pk=unique_id)

    async def get(self, **kwargs) -> schemas.GetArea:
        obj = await models.Area.get(**kwargs).prefetch_related('points')
        return schemas.GetArea(
            id=obj.id,
            name=obj.name,
            camera_id=obj.camera_id,
            points=[
                schemas.AreaPointSchema(
                    x1=p.x1, y1=p.y1,
                    x2=p.x2, y2=p.y2,
                    area_id=p.area_id
                ) for p in obj.points
            ]
        )

areas_s = AreasService()