from uuid import uuid4

def randId():
    return uuid4().hex

def loggedIn(session, LoggedIn):
    if ('user' in session) and (session['user'] is not None):
        userLoggedIn = LoggedIn.query.filter_by(rand_id=str(session['user'])).first()
        if userLoggedIn:
            return userLoggedIn.username
        return False
    else:
        return False

def logoutUser(session, LoggedIn, db):
    randID = session['user']
    user = LoggedIn.query.filter_by(rand_id=str(randID).encode('utf-8')).first()
    db.session.delete(user)
    db.session.commit()
    session.pop('user', None)

def split_skills(skills):
    result = skills.split(',')
    return result


"""
The main script that serves as the entry-point for all kinds of training experiments.
"""

from __future__ import annotations

import logging
from functools import partial
from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Sequence, Tuple, Union, cast

import torch
from al.core.data.collators import JointBatchToTensorDataCollator
from al.core.models.waal.xai_model import WAALXAIModel
from al.core.training.query_strategies.factory import QueryStrategyFactory
from al.core.training.trainer import DALTrainer
from ignite.contrib.handlers import TensorboardLogger
from xai_torch.core.args import Arguments
from xai_torch.core.constants import DataKeys
from xai_torch.core.models.utilities.data_collators import BatchToTensorDataCollator
from xai_torch.core.models.xai_model import XAIModel
from xai_torch.core.training.utilities import reset_random_seeds

if TYPE_CHECKING:
    from al.core.training.query_strategies.base import QueryStrategy
    from xai_torch.core.args import Arguments
    from xai_torch.core.data.data_modules.base import BaseDataModule
    from al.core.data.active_learning_datamodule import ActiveLearningDataModule
    from ignite.engine import Engine
    from torch.utils.data import DataLoader

from xai_torch.core.training.constants import TrainingStage

logging.basicConfig(level=logging.INFO)

class WAALTrainer(DALTrainer):
    @classmethod
    def configure_running_avg_logging(cls, args: Arguments, engine: Engine, stage: TrainingStage):
        from ignite.metrics import RunningAverage

        def output_transform(x: Any, index: int, name: str) -> Any:
            import numbers

            import torch

            if isinstance(x, Mapping):
                return x[name]
            elif isinstance(x, Sequence):
                return x[index]
            elif isinstance(x, (torch.Tensor, numbers.Number)):
                return x
            else:
                raise TypeError(
                    "Unhandled type of update_function's output. "
                    f"It should either mapping or sequence, but given {type(x)}"
                )

        # add loss as a running average metric
        if stage == TrainingStage.train:
            for i, n in enumerate([f"{step}_{DataKeys.LOSS}" for step in ["task", "dsc"]]):
                RunningAverage(
                    alpha=0.5, output_transform=partial(output_transform, index=i, name=n), epoch_bound=False
                ).attach(engine, f"{stage}/{n}")
        else:
            for i, n in enumerate([DataKeys.LOSS]):
                RunningAverage(
                    alpha=0.5, output_transform=partial(output_transform, index=i, name=n), epoch_bound=False
                ).attach(engine, f"{stage}/{n}")

    @classmethod
    def initialize_training_engine(
        cls,
        args: Arguments,
        model: WAALXAIModel,
        output_dir: str,
        device: Optional[Union[str, torch.device]] = torch.device("cpu"),
        scaler: Optional["torch.cuda.amp.GradScaler"] = None,
    ) -> Callable:

        if args.training_args.gradient_accumulation_steps <= 0:
            raise ValueError(
                "Gradient_accumulation_steps must be strictly positive. "
                "No gradient accumulation if the value set to one (default)."
            )

        from ignite.engine import Engine

        # get related arguments
        total_task_model_iterations = model.steps_per_epoch * args.training_args.max_epochs
        gradient_accumulation_steps = args.training_args.gradient_accumulation_steps
        non_blocking = args.training_args.non_blocking_tensor_conv

        if args.training_args.with_amp:
            try:
                from torch.cuda.amp import autocast
            except ImportError:
                raise ImportError("Please install torch>=1.6.0 to use amp_mode='amp'.")

            if scaler is None:
                from torch.cuda.amp.grad_scaler import GradScaler

                scaler = GradScaler(enabled=True)

        def update_model(engine, model, batch, step="task"):
            from xai_torch.core.constants import DataKeys

            # perform optimizers zero_grad() operation with gradient accumulation
            if (engine.state.iteration - 1) % gradient_accumulation_steps == 0:
                model.optimizers[step].zero_grad()

            with autocast(enabled=True):
                # forward pass
                model_output = model.torch_model.training_step(batch=batch, step=step)

                # make sure we get a dict from the model
                assert isinstance(model_output, dict), "Model must return an instance of dict."

                # get loss from the output dict
                loss = model_output[DataKeys.LOSS]

                # accumulate loss if required
                if gradient_accumulation_steps > 1:
                    loss = loss / gradient_accumulation_steps

            if scaler:
                scaler.scale(loss).backward()
                # perform optimizer update for correct gradient accumulation step
                if engine.state.iteration % gradient_accumulation_steps == 0:
                    scaler.step(model.optimizers[step])
                    scaler.update()
            else:
                # backward pass
                loss.backward()

                # perform optimizer update for correct gradient accumulation step
                if engine.state.iteration % gradient_accumulation_steps == 0:
                    model.optimizers[step].step()

            # if on the go training evaluation is required, detach data from the graph
            if args.training_args.eval_training and step == "task":
                return_dict = {}
                for key, value in model_output.items():
                    if key == DataKeys.LOSS:
                        return_dict[key] = value.item()
                    elif isinstance(value, torch.Tensor):
                        return_dict[key] = value.detach()
                return_dict[f"{step}_{DataKeys.LOSS}"] = return_dict[DataKeys.LOSS]
                del return_dict[DataKeys.LOSS]
                return return_dict

            return {f"{step}_{DataKeys.LOSS}": model_output[DataKeys.LOSS].item()}

        def training_step(engine: Engine, batch: Sequence[torch.Tensor]) -> Union[Any, Tuple[torch.Tensor]]:
            """
            Define the model training update step
            """

            from ignite.utils import convert_tensor

            # setup model for training
            model.torch_model.train()

            # put batch to device
            batch = convert_tensor(batch, device=device, non_blocking=non_blocking)

            # we store the task model after n epochs * n labeled training iterations so that its not overfitted
            # this is only for testing the performance
            # call task model update
            if engine.state.iteration == total_task_model_iterations:
                from ignite.engine import Events
                from ignite.handlers.checkpoint import BaseSaveHandler, DiskSaver

                checkpoint = {}
                model.on_save_checkpoint(checkpoint)

                # if only to save weights, remove all other keys
                if args.training_args.model_checkpoint_config.save_weights_only:
                    for k in list(checkpoint.keys()):
                        if k not in ["model"]:
                            checkpoint.pop(k)

                model_checkpoint_config = args.training_args.model_checkpoint_config
                checkpoint_dir = output_dir / model_checkpoint_config.dir
                save_handler = DiskSaver(
                    checkpoint_dir,
                    require_empty=False,
                )

                from ignite.handlers import Checkpoint

                checkpoint_handler = Checkpoint(
                    checkpoint,
                    cast(Union[Callable, BaseSaveHandler], save_handler),
                    filename_prefix="task",
                )
                checkpoint_handler(engine)

            task_output = update_model(engine, model, batch, step="task")

            # dsc output
            dsc_output = update_model(engine, model, batch, step="dsc")
            # print("Losses: ", task_output["task_loss"], dsc_output["dsc_loss"])

            return {**task_output, **dsc_output}

        return Engine(training_step)

    @classmethod
    def setup_model(
        cls,
        args: Arguments,
        datamodule: BaseDataModule,
        tb_logger: TensorboardLogger,
        summarize: bool = False,
        stage: TrainingStage = TrainingStage.train,
    ) -> XAIModel:
        """
        Initializes the model for training.
        """
        from xai_torch.core.models.factory import ModelFactory

        # setup model
        model = ModelFactory.create(args, datamodule, tb_logger=tb_logger, wrapper_class=WAALXAIModel)
        model.setup(stage=stage)

        # generate model summary
        if summarize:
            model.summarize()

        return model

    @classmethod
    def setup_training_engine(cls, args, model, train_dataloader, val_dataloader, output_dir, tb_logger, device):
        pass

        # setup training engine
        training_engine = cls.initialize_training_engine(args=args, model=model, output_dir=output_dir, device=device)

        validation_engine = None
        if args.general_args.do_val:
            # setup validation engine
            validation_engine = cls.initialize_validation_engine(args=args, model=model, device=device)

        # configure training and validation engines
        cls.configure_training_engine(
            args=args,
            training_engine=training_engine,
            model=model,
            output_dir=output_dir,
            tb_logger=tb_logger,
            train_dataloader=train_dataloader,
            validation_engine=validation_engine,
            val_dataloader=val_dataloader,
        )

        # add training hooks from the model
        model.add_training_hooks(training_engine)

        return training_engine, validation_engine

    @classmethod
    def setup_test_engine_task(cls, args, model, test_dataloader, output_dir, tb_logger, device):
        import glob

        from ignite.handlers import Checkpoint

        # setup training engine
        test_engine = cls.initialize_test_engine(args=args, model=model, device=device)

        # configure training and validation engines
        cls.configure_test_engine(
            args=args,
            test_engine=test_engine,
            model=model,
            output_dir=output_dir,
            tb_logger=tb_logger,
            load_checkpoint=False,
        )

        checkpoint = {}
        model_checkpoint_config = args.training_args.model_checkpoint_config
        checkpoint_dir = output_dir / model_checkpoint_config.dir
        list_checkpoints = glob.glob(str(checkpoint_dir) + "/*.pt")
        test_checkpoint_path = [c for c in list_checkpoints if "task" in c][0]
        model.on_save_checkpoint(checkpoint)
        test_checkpoint = torch.load(test_checkpoint_path, map_location="cpu")
        Checkpoint.load_objects(to_load=checkpoint, checkpoint=test_checkpoint)
        cls.configure_test_tb_logger(args=args, test_engine=test_engine, model=model, tb_logger=tb_logger, tag="task")
        return test_engine

    @classmethod
    def train(cls, local_rank, args: Arguments):
        """
        Initializes the training of a model given dataset, and their configurations.
        """

        import ignite.distributed as idist
        from xai_torch.core.training.utilities import initialize_training, setup_logging
        from xai_torch.utilities.logging_utils import DEFAULT_LOGGER_NAME, setup_logger

        # setup logging
        logger = setup_logger(DEFAULT_LOGGER_NAME, distributed_rank=local_rank, level=logging.INFO)

        # initialize training
        initialize_training(args)

        # initialize torch device (cpu or gpu)
        device = idist.device()

        # get device rank
        rank = idist.get_rank()

        # initialize logging directory and tensorboard logger
        output_dir, tb_logger = setup_logging(args)

        # setup datamodule
        datamodule: ActiveLearningDataModule = cls.setup_datamodule(args, rank=rank, stage=None)

        # setup model
        model = cls.setup_model(args, datamodule, tb_logger, summarize=True)

        # define active learning query strategy
        query_strategy: QueryStrategy = QueryStrategyFactory.create(
            datamodule=datamodule, model=model, device=device, args=args.al_args
        )

        # load active learning state
        al_state = DALTrainer.load_round_state(0, datamodule, output_dir=output_dir)
        curr_round = al_state["curr_round"]

        if curr_round == args.al_args.n_rounds:
            logger.warning(
                "Active learning rounds have already been finished! Either increase the number of "
                f"max rounds (current={args.al_args.n_rounds}) "
                "OR reset the training from start."
            )
            exit()

        # reset seeds for training. This allows multiple experiments with same seed for dataset initialization but
        # different seeds for the active learning training process.
        reset_random_seeds(args.al_args.al_seed)

        while curr_round < args.al_args.n_rounds:
            from al.core.training.query_strategies.impl.ceal import CEAL

            logger.info(f"============== Running round={curr_round} of active learning ===========")
            # update tblogger dir
            tb_logger = None
            if rank == 0:
                from ignite.contrib.handlers import TensorboardLogger

                tb_logger = TensorboardLogger(output_dir / str(curr_round))

            # print labels summary
            datamodule.print_label_summary()

            # Reset model for re-training
            if args.al_args.reset_model:
                model = cls.setup_model(args, datamodule, tb_logger, summarize=False)
            else:
                # Reset only optimizers and schedulers
                model._opt_sch_handler.setup_opt_sch()

            # get train dataloader for labelled data
            joint_dataloader = datamodule.get_joint_dataset_loader(
                collate_fn=JointBatchToTensorDataCollator(datamodule._collate_fns.train)
            )

            # setup gamma ratio
            args.al_args.training_args.gamma_ratio = len(datamodule.labeled_indices) / len(datamodule.unlabeled_indices)

            # get validation data loader
            val_dataloader = datamodule.val_dataloader()

            # setup training engine
            training_engine, _ = cls.setup_training_engine(
                args=args,
                model=model,
                train_dataloader=joint_dataloader,
                val_dataloader=val_dataloader,
                output_dir=output_dir / str(curr_round),  # append round number to output_dir
                tb_logger=tb_logger,
                device=device,
            )
            training_engine.logger = logger

            # NOTE: The test engine has already updated the model state with state of last/best
            # checkpoint which will be used for querying of the next round.
            def perform_query():
                import timeit

                # reset the querying strategy
                query_strategy.reset(model)

                # update the labeled pool
                start = timeit.default_timer()
                n_query_samples = int(args.al_args.n_query_ratio * datamodule.pool_size)
                if isinstance(query_strategy, CEAL):
                    query_indices = query_strategy.query(n_samples=n_query_samples, round=curr_round)
                else:
                    query_indices = query_strategy.query(n_samples=n_query_samples)
                stop = timeit.default_timer()
                tb_logger.writer.add_scalar("query_time", stop - start, curr_round)
                datamodule.update_dataset_labels(query_indices)

            def test_model():
                # after the training, the test engine automatically loads the 'best' model to continue the rounds.
                test_dataloader = datamodule.test_dataloader()

                # perform testing on task model which is trained for only set number of epochs on labeled dataset
                test_engine_task = cls.setup_test_engine_task(
                    args=args,
                    model=model,
                    test_dataloader=test_dataloader,
                    output_dir=output_dir / str(curr_round),
                    tb_logger=tb_logger,
                    device=device,
                )
                test_engine_task.logger = logger
                test_engine_task.run(test_dataloader)

                # perform testing on task model which is trained for complete iterations ran by joint dataloader
                test_engine = cls.setup_test_engine(
                    args=args,
                    model=model,
                    test_dataloader=test_dataloader,
                    output_dir=output_dir / str(curr_round),
                    tb_logger=tb_logger,
                    device=device,
                )
                test_engine.logger = logger
                test_engine.run(test_dataloader)

            resume_epoch = training_engine.state.epoch
            if not (training_engine._is_done(training_engine.state) or resume_epoch >= args.training_args.max_epochs):
                # run training
                training_engine.run(joint_dataloader, max_epochs=args.training_args.max_epochs)

                # perform query before testing
                perform_query()

                # test model
                test_model()
            else:
                # if we are resuming from last checkpoint and training is already finished
                logger.info(
                    "Training has already been finished! Either increase the number of "
                    f"epochs (current={args.training_args.max_epochs}) >= {resume_epoch} "
                    "OR reset the training from start."
                )

                # perform query before testing
                perform_query()

                # test model
                test_model()

            # save active learning query state for next round
            DALTrainer.save_round_state(curr_round + 1, datamodule, output_dir=output_dir)

            if rank == 0:
                # close tb logger
                tb_logger.close()

            curr_round += 1


import time
from flask import Blueprint, jsonify, request
import requests
import part2.health_check as health_check
from dbBrokerManager.config import async_session, engine, BaseBroker
from dbBrokerManager.AsyncDAL import DAL
import datetime
import asyncio

server = Blueprint("broker_manager_Read_Only", __name__)

brokers = list()                  # List of brokers in the network

topics_lock = True

class Broker:
    def __init__(self, address, broker_id, isReadOnly = False):
        self.address = address
        self.isAlive = True
        self.brokerID = broker_id
        self.isReadOnly = isReadOnly

    def declareDead(self):
        self.isAlive = False

    def declareAlive(self):
        self.isAlive = True

    async def checkHealth(self):
        try:
            r = requests.get(f"{self.address}/status")
            response = r.json()
            if response["status"] == "success" and response["message"] == "broker running":
                self.declareAlive()
            else:
                self.declareDead()
        except:
            self.declareDead()

lastBrokerCheckTime = datetime.datetime.utcnow()

async def checkForNewBrokers():
    brokers = list()
    ip, ports = health_check.doSearchJob(0)
    for port in ports:
        address = f"http://{ip}:{port}"
        brokers.append(Broker(address, len(brokers)))
    print(ports)
    return brokers

async def getServerAddress(broker_id):
    global topics_lock
    while topics_lock == False:
        time.sleep(1)
    topics_lock = False

    global brokers, lastBrokerCheckTime

    print(lastBrokerCheckTime)
    print(datetime.datetime.utcnow() - lastBrokerCheckTime)

    if datetime.datetime.utcnow() - lastBrokerCheckTime > datetime.timedelta(seconds=10):
        brokers = await checkForNewBrokers()
        lastBrokerCheckTime = datetime.datetime.utcnow()

    address = None

    await brokers[broker_id].checkHealth()
    if brokers[broker_id].isAlive:
        address = brokers[broker_id].address
    else:
        brokers[broker_id].declareDead()

    # if address is None:
    #     for replica in brokersReplica[broker_id]:
    #         replica.checkHealth()
    #         if replica.isAlive:
    #             address = replica.address
    #         else:
    #             replica.declareDead()

    topics_lock = True

    return address

@server.before_app_first_request
async def setUpBrokerManager():
    # This function sets up the broker manager by backing up things from server and setting up brokers in the network and read only copies of broker manager.
    global brokers, topics_lock
    topics_lock = True
    brokers = list()

    async with engine.begin() as conn:
        # await conn.run_sync(Base.metadata.drop_all)
        await conn.run_sync(BaseBroker.metadata.create_all, checkfirst=True)

    while len(brokers)==0:
        ip, ports = health_check.doSearchJob(0)
        print(ports)
        for id, port in enumerate(ports):
            brokers.append(Broker(f"http://{ip}:{port}", id))
    print(brokers)

@server.route("/")
def index():
    return "
Welcome to the Broker Manager Copy!
"

@server.route("/status")
def status():
    return jsonify({"status": "success", "message": "Broker Manager Copy running"})

@server.route("/topics", methods=["POST"])
async def create_topic():
    topic_name = request.json["topic_name"]

    brokers_list = list()

    tempVar = await getServerAddress(0)

    for broker_id in range(len(brokers)):
        address = await getServerAddress(broker_id)
        print(address)
        if address is None:
            return jsonify({"status": "failure", "message": f"Data Lost due to complete broker failure"})

        params = {"topic_name": topic_name, "partition_id": broker_id}
        r = requests.post(f"{address}/topics", json=params)
        response = r.json()
        if response["status"] == "failure":
            return jsonify({"status": "failure", "message": f"Topic '{topic_name}' could not be created"})
        brokers_list.append(broker_id)
    return jsonify({"status": "success", "message": f"Topic '{topic_name}' created successfully", "brokers_list": brokers_list})

@server.route("/producer/produce", methods=["POST"])
async def enqueue():
    topic_name = request.json["topic_name"]
    producer_id = request.json["producer_id"]
    log_message = request.json["log_message"]
    partition_id = request.json["partition_id"]

    address = await getServerAddress(partition_id)
    if address is None:
        return jsonify({"status": "failure", "message": f"Data Lost due to complete broker failure"})

    params = {"topic_name": topic_name, "producer_id": producer_id, "log_message": log_message, "partition_id": partition_id}
    r = requests.post(f"{address}/producer/produce", json=params)
    response = r.json()
    if response["status"] == "failure":
        return jsonify({"status": "failure", "message": f"Message production failed"})

    return jsonify({"status": "success"})

@server.route("/consumer/consume", methods=["GET"])
async def dequeue():
    topic_name = request.json["topic_name"]
    consumer_id = request.json["consumer_id"]
    partitions = request.json["partitions"]
    consumerFront = request.json["consumer_fronts"]

    log_message = None
    minTime = datetime.datetime.utcnow()
    minbrokerID = None

    for broker_id in partitions:
        address = await getServerAddress(broker_id)
        if address is None:
            return jsonify({"status": "failure", "message": f"Data Lost due to complete broker failure"})

        query = None
        params = {"topic_name": topic_name, "partition_id": broker_id, "consumer_front": consumerFront[str(broker_id)]}
        r = requests.get(f"{address}/consumer/consume", json=params)
        query = r.json()

        if query["status"] == "failure":
            continue

        if minTime > datetime.datetime.strptime(query["timestamp"], '%a, %d %b %Y %H:%M:%S %Z'):
            minTime = datetime.datetime.strptime(query["timestamp"], '%a, %d %b %Y %H:%M:%S %Z')
            log_message = query["log_message"]
            minbrokerID = broker_id

    if log_message is None:
        return jsonify({"status": "failure", "message": f"No message to consume"})

    return jsonify({"status": "success", "log_message": log_message, "broker_id": minbrokerID})

@server.route("/size", methods=["GET"])
async def size():
    topic_name = request.json["topic_name"]
    consumer_id = request.json["consumer_id"]

    partitions = request.json["partitions"]
    consumerFront = request.json["consumer_fronts"]

    consumer_size = 0

    for broker_id in partitions:
        address = await getServerAddress(broker_id)
        if address is None:
            return jsonify({"status": "failure", "message": f"Data Lost due to complete broker failure"})

        query = None
        params = {"topic_name": topic_name, "partition_id": broker_id, "consumer_front": consumerFront[str(broker_id)]}
        r = requests.get(f"{address}/size", json=params)
        query = r.json()
        if query["status"] == "failure":
            return jsonify({"status": "failure", "message": f"Size query failed"})
        consumer_size += query["size"]

    return jsonify({"status": "success", "size": consumer_size})


from selenium import webdriver
import time
import json
import os
from selenium.webdriver.common.by import By
import subprocess
import difflib
import re
from urllib.parse import unquote
from colorama import *

class Voltaire:

    def __init__(self):
        options = webdriver.ChromeOptions()
        options.add_experimental_option('excludeSwitches', ['enable-logging'])

        ch_output = subprocess.Popen(["chromedriver"])
        self.driver = webdriver.Chrome(options=options)
        self.driver.implicitly_wait(1)

    def Header(self):
        os.system('cls')

        print(f"""
{Fore.GREEN}__   __   _ _        {Fore.RED}_{Fore.GREEN}
\ \ / /__| | |_ __ _{Fore.RED}(_){Fore.GREEN}_ _ ___
 \ V / {Fore.WHITE}_{Fore.GREEN} \ |  _/ {Fore.WHITE}_{Fore.GREEN}` | | '_/ -_)
  \_/\___/_|\__\__,_|_|_| \___|{Fore.RESET} [{Fore.RED}solved{Fore.RESET}]
""")

        self.par = input(f' [{Fore.GREEN}*{Fore.RESET}] Supérieur / Excellence ? => ')
        self.nb = input(f' [{Fore.GREEN}*{Fore.RESET}] Test number => ')

        try:
            int(self.nb)
            pass
        except ValueError:
            print(f' [{Fore.RED}!{Fore.RESET}] INT is required !')
            time.sleep(3.5)
            self.Header()

    def solver(self):
        data_filename = f"Module{self.nb}.txt"

        directory = f"./src/{self.par}/"
        reponses = []

        for filename in os.listdir(directory): #Credit => https://github.com/sylvain-reynaud/projet-voltaire-solver
            if filename.endswith(data_filename):
                with open(directory + filename, 'r', encoding="utf-8") as f:
                    data = f.read()
                    try:
                        data = data[data.index("[\"java.util.ArrayList"):data.index("]")] + "]"
                        data = data.replace("\\", "\\\\")
                        reponses += json.loads(data)
                    except:
                        pass
        reponses = [x for x in reponses if "\\x3C" in x]

        input(f' [{Fore.GREEN}*{Fore.RESET}] Ready ? (Enter to start)')
        print(f' [{Fore.GREEN}*{Fore.RESET}] Test {self.nb} Started with 1 attempt every 5 seconds !')

        while True:
            try:
                time.sleep(5)
                items = self.driver.find_elements(By.XPATH, "/html/body/div[5]/div[2]/div[3]/div/div[1]/div/div/div[2]/div")
                for item in items:
                    phrase = item.text

                possibilites = difflib.get_close_matches(phrase, reponses)
                if len(possibilites) != 0:
                    toPrint = re.sub(r"<B>(.*)<\/B>", fr"{Fore.GREEN}\1{Fore.RESET}", unquote(possibilites[0].replace("\\x", "%")))
                    print(f' [{Fore.GREEN}*{Fore.RESET}]', toPrint)

                else:
                    print(f' [{Fore.GREEN}*{Fore.RESET}] Il n\'y a pas de faute')
                    self.driver.find_element(By.XPATH, "/html/body/div[5]/div[2]/div[3]/div/div[2]/div[1]/div[1]/div/button").click()
                    self.driver.find_element(By.XPATH, "/html/body/div[5]/div[2]/div[3]/div/div[3]/div[2]/button").click()

            except:
                print(f' [{Fore.RED}*{Fore.RESET}] Attempt Failed !')

Voltaire_solved = Voltaire()
Voltaire_solved.Header()
Voltaire_solved.solver()


from datetime import datetime

import pytz
import requests

from config import NO_IMG_URL, TIMEZONE

def convert_timezone(
    time=None, format="%Y-%m-%dT%H:%M:%SZ", ori_timezone=None, desire_timezone=TIMEZONE
):
    date_time = datetime.strptime(time, "%Y-%m-%dT%H:%M:%SZ")
    ori_timezone = pytz.timezone(ori_timezone)
    date_time = ori_timezone.localize(date_time)
    desired_timezone = pytz.timezone(desire_timezone)
    desire_time = pytz.utc.normalize(date_time).astimezone(desired_timezone)
    return desire_time

def check_img_access(img_url=""):

    response = requests.get(img_url)
    if not response.status_code == 200:
        img_url = NO_IMG_URL
    return img_url


from pathlib import Path
import csv
import random
from faker import Faker

fake=Faker()

p = Path('.')

# Find all files in folder
fileslist=list(p.glob('**/*.csv'))

# Set the folder for the anonymized files
outfolder = 'anon'

for file in fileslist:
    randid=random.randint(10000,99999)
    randid2=random.randint(10000,99999)
    a=fake.date_between(start_date='+10y',end_date='+14y')
    randdate=f'{str(a.day).zfill(2)}-{str(a.month).zfill(2)}-{a.year}'

    readFile=open(file,'r')
    writeFile=open(f'./{outfolder}/{randid2}.csv', 'w')
    print(file)
    for row in readFile:
        # Add for fields to be changed
        if 'Field1' in row:
            writeFile.write(f'Field1={randid}\n')
        elif 'Field2' in row:
            writeFile.write(f'Field2={randid2}\n')
        elif 'Date' in row:
            writeFile.write(f'Date={randdate}\n')
        else:
            writeFile.write(row)

    #Close and write new files
    readFile.close()
    writeFile.close()


#Before all this run the command pip install --user PyPDF2==1.26.0
#Shift + right click will bring up the powershell/cmd for the folder
#Ensure file is in the same folder and has the extension .pdf
import PyPDF2
import sys
import time
from tqdm import tqdm
pdfname = input("Enter the name of your file (example: Hello.pdf) :")
pdfname =pdfname.strip()
if pdfname.endswith(".pdf") == False:
    print("Enter a proper pdf name")
    print("Shutting down program...")
    time.sleep(5)
    sys.exit(0)
pdfFileObj = open(pdfname, 'rb') #If you get FileNotFoundError check if the file name you mentioned is in the same folder/directory
pdfReader = PyPDF2.PdfFileReader(pdfFileObj)
print("Number of pages: "+ str(pdfReader.numPages))
while True:
    ans = input("Do you want to extract from one page or all (o/a) :")
    if ans=="o":
        page = input("Which page number? :")
        pageObj = pdfReader.getPage(int(page)-1)
        print("Extracting text......")
        with open("extractedtext.txt","a") as f:
            f.write(pageObj.extractText() + "\n")
        for i in tqdm(range(5)):
            time.sleep(3)
        print("DONE!")
        break
    elif ans=="a":
        print("Extracting text......")
        for x in range(0,pdfReader.numPages):
            pageObj = pdfReader.getPage(x)
            with open("extractedtext.txt","a") as f:
                f.write(pageObj.extractText() + "\n")
        for i in tqdm(range(5)):
            time.sleep(3)
        print("DONE!")
        break
pdfFileObj.close()


from metrics import *
import numpy as np
import pandas as pd
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils.vis_utils import plot_model
from keras.utils import to_categorical
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import roc_auc_score
import random
import pickle
import json
import pathlib

MAX_NUM_WORDS = 200000

def evaluate_model(model, X_train, y_train, X_val, y_val):
    pred_train = model.predict(X_train)
    pred_val = model.predict(X_val)
    prec_train = precision(y_train, pred_train)
    prec_val = precision(y_val, pred_val)
    recall_train = recall(y_train, pred_train)
    recall_val = recall(y_val, pred_val)
    f1_train = fbeta_score(y_train, pred_train)
    f1_val = fbeta_score(y_val, pred_val)
    auc = roc_auc_score(y_val, pred_val)
    val_results = "Validation Results: Prec - {:.2f}  Recall - {:.2f}  F1 - {:.2f}  AUC - {:.2f}".format(prec_val,
                                                                                recall_val,
                                                                                f1_val,
                                                                                auc)
    train_results = "Train Results:      Prec - {:.2f}  Recall - {:.2f}  F1 - {:.2f}".format(prec_train,
                                                                                recall_train,
                                                                                f1_train)
    print("{}\n{}".format(train_results, val_results))
    return prec_val, recall_val, f1_val

def save_model(model, history, filename,
               prec=None, recall=None, f1=None,
               epochs=None, batch_size=None, random_seed=None,
               class_weights=None, sample_type=None):
    pathlib.Path("../models/{}".format(filename)).mkdir(parents=True, exist_ok=True)
    model.save("../models/{}/{}.h5".format(filename, filename))
    with open("../models/{}/{}.txt".format(filename, filename), 'wb') as f:
        pickle.dump(history.history, f)
    with open("../results/{}.txt".format(filename), 'w+') as f:
        if prec is not None:
            f.write("Validation Set Precision: {}\n".format(prec))
        if recall is not None:
            f.write("Validation Set Recall   : {}\n".format(recall))
        if f1 is not None:
            f.write("Validation Set F1 Score : {}\n".format(f1))
        if epochs is not None:
            f.write("Number epochs trained: {}\n".format(epochs))
        if batch_size is not None:
            f.write("Batch size: {}\n".format(batch_size))
        f.write("Random seed: {}\n".format(random_seed))
        if class_weights is not None:
            f.write(json.dumps(class_weights))
            f.write("\n")
        if sample_type is not None:
            f.write("Sample type: {}\n".format(sample_type))
    plot_model(model, to_file="../models/{}/{}.png".format(filename, filename), show_shapes=True)
    return

def shuffle_arrays(arr1, arr2, seed=None):
    if seed is None:
        seed = random.randint(0, 1000)
    indices = np.arange(arr1.shape[0])
    np.random.seed(seed)
    np.random.shuffle(indices)
    arr1_ret = arr1[indices]
    arr2_ret = arr2[indices]
    return arr1_ret, arr2_ret

def is_digit(string):
    #  Sourced from https://stackoverflow.com/questions/354038/how-do-i-check-if-a-string-is-a-number-float
    try:
        float(string)
        return True
    except ValueError:
        return False

def collapse_embedding_array(array):
    word = array[0]
    remaining_arr = []
    for i in range(1, len(array)):
        if not is_digit(array[i]):
            word += (" " + array[i])
        else:
            remaining_arr = array[i:]
            break
    return [word] + remaining_arr

def create_embedding_matrix(filepath, dictionary, embedding_dim):
    embeddings_index = {}
    with open(filepath, encoding="utf8") as f:
        for line in f:
            values = collapse_embedding_array(line.split())
            word = values[0]
            coefs = np.asarray(values[1:], dtype="float32")
            embeddings_index[word] = coefs
    embedding_matrix = np.zeros((len(dictionary)+1, embedding_dim))
    for word, i in dictionary.items():
        embedding_vector = embeddings_index.get(word)
        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector
    return embedding_matrix

def recover_queries(tokenizer, X, y):
    recovered_queries = []
    for i, query in enumerate(X):
        query_text = []
        for index in query:
            query_text.append(tokenizer.index_word.get(index, "UNKNOWN"))
        if y[i] == 1:
            recovered_queries.append(query_text)
    return recovered_queries

def read_data_embeddings(max_input_length=10, vocab_size=MAX_NUM_WORDS):
    X_train, y_train, X_val, y_val = read_train_val_data()
    X = pd.concat((X_train, X_val))
    tokenizer = Tokenizer(num_words=vocab_size, split=' ')
    tokenizer.fit_on_texts(X[0].values)
    X_train = tokenizer.texts_to_sequences(X_train[0].values)
    X_train = pad_sequences(X_train, maxlen=max_input_length)
    X_val = tokenizer.texts_to_sequences(X_val[0].values)
    X_val = pad_sequences(X_val, maxlen=max_input_length)
    y_train = y_train[0].values
    y_val = y_val[0].values
    return X_train, y_train, X_val, y_val, tokenizer

def read_bag_of_words(vocab_size=MAX_NUM_WORDS):
    X_train, y_train, X_val, y_val = read_train_val_data()
    X_total = pd.concat((X_train, X_val))
    tokenizer = Tokenizer(num_words=vocab_size, split=' ')
    tokenizer.fit_on_texts(X_total[0].values)
    X_train = tokenizer.texts_to_matrix(X_train[0].values)
    X_val = tokenizer.texts_to_matrix(X_val[0].values)
    y_train = y_train[0].values
    y_val = y_val[0].values
    return X_train, y_train, X_val, y_val, tokenizer

def validation_training_split(split=0.2):
    kdd_data = pd.read_csv("../data/KDD_Cup_2005_Data.csv")
    kdd_x = kdd_data[["Query"]]
    kdd_y = kdd_data[["Label"]]
    google_trends_data = pd.read_csv("../data/Google_Trends_Search_Queries.csv")
    trends_x = google_trends_data[["Query"]]
    trends_y = google_trends_data[["Label"]]
    kdd_data_8k = pd.read_csv("../data/KDD_Cup_2005_Data_8k.csv")
    kdd_8k_x = kdd_data_8k[["Query"]]
    kdd_8k_y = kdd_data_8k[["Label"]]
    X = pd.concat((kdd_x, kdd_8k_x, trends_x))
    y = pd.concat((kdd_y, kdd_8k_y, trends_y))
    X, y = shuffle_arrays(X.values, y.values)
    num_val_examples = int(split * X.shape[0])
    x_train = X[:-num_val_examples]
    y_train = y[:-num_val_examples]
    x_val = X[-num_val_examples:]
    y_val = y[-num_val_examples:]
    pd.DataFrame(x_train).to_csv("../data/X_train.csv", header=None, index_label=False, index=False)
    pd.DataFrame(y_train).to_csv("../data/y_train.csv", header=None, index_label=False, index=False)
    pd.DataFrame(x_val).to_csv("../data/X_val.csv", header=None, index_label=False, index=False)
    pd.DataFrame(y_val).to_csv("../data/y_val.csv", header=None, index_label=False, index=False)
    return

def read_train_val_data():
    x_train = pd.read_csv("../data/X_train.csv", header=None)
    y_train = pd.read_csv("../data/y_train.csv", header=None)
    x_val = pd.read_csv("../data/X_val.csv", header=None)
    y_val = pd.read_csv("../data/y_val.csv", header=None)
    return x_train, y_train, x_val, y_val


# 标准库
import io as _io
import os as _os
import sys as _sys
import imp as _imp
import codecs as _codecs
import traceback as _traceback
import pathlib as _pathlib

def enhance_init(work_dir=__file__, python_version_require=0, check_module_list=[]):
    """
    :param pythonVersionRequire(int): 最低python所需版本
    :param checkModuleList: 示例: ("requests", "requests_toolbelt", "urllib3", "bs4", "Crypto", "pyDes", "yaml", "lxml", "rsa")
    """
    # ==========检查python版本==========
    if not (
        _sys.version_info[0] == 3 and _sys.version_info[1] >= python_version_require
    ):
        raise Exception(
            "!!!!!!!!!!!!!!Python版本错误!!!!!!!!!!!!!!\n请使用python3.%d及以上版本，而不是[python %s]"
            % (python_version_require, _sys.version)
        )
    # ==========环境变量初始化==========
    try:
        print("==========开始初始化==========")
    except UnicodeEncodeError:
        # 设置默认输出编码为utf-8, 但是会影响腾讯云函数日志输出。
        _sys.stdout = _codecs.getwriter("utf-8")(_sys.stdout.detach())
        print("==========开始初始化(utf-8输出)==========")
    script_abs_path = _pathlib.Path(work_dir).absolute().parent
    _os.chdir(script_abs_path)  # 将工作路径设置为脚本位置
    if _os.name == "posix":
        # 如果是linux系统, 增加TZ环境变量
        _os.environ["TZ"] = "Asia/Shanghai"
    _sys.path.append(script_abs_path)  # 将脚本路径加入模块搜索路径
    # ==========检查第三方模块==========
    try:
        for i in check_module_list:
            _imp.find_module(i)
    except ImportError as e:  # 腾讯云函数在初始化过程中print运作不正常，所以将信息丢入异常中
        raise ImportError(
            f"""!!!!!!!!!!!!!!缺少第三方模块(依赖)!!!!!!!!!!!!!!
    请使用pip3命令安装或者手动将依赖拖入文件夹
    错误信息: [{e}]"""
        )

class Decorators:
    @staticmethod
    def catch_exception(func):
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception:
                print(_traceback.format_exc())
                return None

        return wrapper

class FileOut:
    """
    代替stdout和stderr, 使print同时输出到文件和终端中。
    start()方法可以直接用自身(self)替换stdout和stderr
    close()方法可以还原stdout和stderr
    """

    stdout = _sys.stdout
    stderr = _sys.stderr
    log: str = ""  # 同时将所有输出记录到log字符串中
    logFile: _io.TextIOWrapper = None

    @classmethod
    def set_file_out(cla, path: str = None):
        """
        设置日志输出文件
        :params path: 日志输出文件路径, 如果为空则取消日志文件输出
        """
        # 关闭旧文件
        if cla.logFile:
            cla.logFile.close()
            cla.logFile = None

        # 更新日志文件输出
        if path:
            try:
                path = _os.path.abspath(path)
                logDir = _os.path.dirname(path)
                if not _os.path.isdir(logDir):
                    _os.makedirs(logDir)
                cla.logFile = open(path, "w+", encoding="utf-8")
                cla.logFile.write(cla.log)
                cla.logFile.flush()
                return
            except Exception as e:
                print(2, f"设置日志文件输出失败, 错误信息: [{e}]")
                cla.logFile = None
                return
        else:
            cla.logFile = None
            return

    @classmethod
    def start(cla):
        """开始替换stdout和stderr"""
        if type(_sys.stdout) != cla and type(_sys.stderr) != cla:
            _sys.stdout = cla
            _sys.stderr = cla
        else:
            print("sysout/syserr已被替换为FileOut")

    @classmethod
    def write(cla, str_):
        r"""
        :params str: print传来的字符串
        :print(s)等价于sys.stdout.write(s+"\n")
        """
        str_ = str(str_)
        cla.log += str_
        if cla.logFile:
            cla.logFile.write(str_)
        cla.stdout.write(str_)
        cla.flush()

    @classmethod
    def flush(cla):
        """刷新缓冲区"""
        cla.stdout.flush()
        if cla.logFile:
            cla.logFile.flush()

    @classmethod
    def close(cla):
        """关闭"""
        if cla.logFile:
            cla.logFile.close()
        cla.log = ""
        _sys.stdout = cla.stdout
        _sys.stderr = cla.stderr


import logging
import multiprocessing
import random
import signal
import sys
import time
import traceback
from typing import Callable, Dict, List, Optional

from pebble import ProcessPool, sighandler

from .client import Client
from .util import constants as C
from .util import helper
from .util.enums import State

class Consumer:
    """
    Faktory Consumer (Worker).

    It fetches units of work (jobs) from the server and executes them.
    It retrieves the jobs from the queues, and decides how to execute them
    based on the jobtype.

    A handler must be attached to each jobtype before the consumer is launched.

    Parameters
    ----------
    client : pyfaktory.Client
        Faktory client. Client `role` must be either 'consumer' or 'both'.
    queues : List[str], default ['default']
        The queues from which the consumer will fetch jobs. If you provide
        no `queues`, the consumer will process `default` queue.
    priority : {{'strict', 'uniform', 'weighted'}}, default 'uniform'
        Priority indicates in which queue order the jobs should be fetched
        first. With `strict` priority, the worker always fetches from the first
        queue and will only try to fetch from the next once the previous queue
        is empty. With `uniform`, each queue has an equal chance of being
        fetched first. With `weighted`, queues have a different probability
        of being fetched. These probabilities are specified with `weights`
        argument.
    weights : Optional[List[float]], default None
        Probability of the queues to be fetched. This parameter is required
        when `priority` is `weighted` (and ignored in other cases), and must
        have the same number of elements as `queues`.
    concurrency : int, default 4
        Number of jobs to run at the same time.
    grace_period : int, default 25
        Grace period between the beginning of a shutdown and its end.
        This period is used to give the job some time to finish, to stop them
        properly and to notify the server. This period should never be longer
        than 30 seconds.
    sentry_capture_exception : bool, default False
        If `True` capture exceptions using Sentry before failling jobs.
    """

    def __init__(self,
                 client: Client,
                 queues: List[str] = C.DEFAULT_QUEUES,
                 priority: str = 'uniform',
                 weights: Optional[List[float]] = None,
                 concurrency: int = 4,
                 grace_period: int = C.DEFAULT_GRACE_PERIOD,
                 sentry_capture_exception: bool = False) -> None:
        self.logger = logging.getLogger(name='FaktoryConsumer')

        if client.role == 'producer':
            raise ValueError(
                "Provided client is exclusively producer and can't act as a consumer"
            )
        self.client = client

        self.queues = queues

        if priority not in ['strict', 'uniform', 'weighted']:
            raise ValueError(
                f"Unexpected priority ({priority}), priority should be 'strict', 'uniform' or 'weighted'"
            )
        self.priority = priority

        if self.priority == 'weighted':
            if weights is None:
                raise ValueError(
                    'Priority is weighted but weights are not provided')
            elif len(weights) != len(self.queues):
                raise ValueError('Weights and queues lengths mismatch')
            else:
                self.weights = weights

        self.concurrency = concurrency
        self.grace_period = grace_period
        self.sentry_capture_exception = sentry_capture_exception

        self.pool = ProcessPool(max_workers=self.concurrency)

        self.job_handlers: Dict[str, Callable] = {}

        self.pending_tasks_count = 0
        self.lock_pending_tasks_count = multiprocessing.Lock()

    @sighandler((signal.SIGTERM))
    def handle_sigterm(*_):
        raise KeyboardInterrupt

    def register(self, name: str, fn: Callable):
        """
        Register a handler for the given jobtype.

        It is expected that all jobtypes are registered upon process
        startup.
        """
        self.job_handlers[name] = fn
        self.logger.info(f"Registered handler for jobtype: {name}")

    def get_job_handler(self, name: str) -> Callable:
        try:
            return self.job_handlers[name]
        except KeyError:
            self.logger.error(f"'{name}' is not a registered job handler")
            # One could consider just failing the job and continue running,
            # but we are not doing it here because it is expected that all
            # jobtypes are registered upon process startup.
            raise ValueError(f"'{name}' has no registered handler")

    def get_queues(self) -> List[str]:
        if self.priority == 'strict':
            return self.queues
        elif self.priority == 'uniform':
            random.shuffle(self.queues)
            return self.queues
        else:
            return helper.weighted_shuffle(self.queues, self.weights)

    def task_done(self, future):
        self.logger.info(f'Task done callback called for job {future.job_id}')
        try:
            result = future.result()
            self.logger.info(f'Task (job {future.job_id}) returned {result}')
            self.client._ack(jid=future.job_id)
        except Exception as err:
            if self.sentry_capture_exception:
                import sentry_sdk
                sentry_sdk.capture_exception(err)

            err_type, err_value, err_traceback = sys.exc_info()
            self.logger.info(
                f'Task (job {future.job_id}) raised {err_type}: {err_value}')
            self.logger.debug(f'Task (job {future.job_id}) backtrace: ',
                              traceback.format_tb(err_traceback))
            self.client._fail(jid=future.job_id,
                              errtype=err_type.__name__,
                              message=str(err_value),
                              backtrace=traceback.format_tb(
                                  err_traceback, limit=future.backtrace))
        finally:
            with self.lock_pending_tasks_count:
                self.pending_tasks_count -= 1

    def run(self):
        """
        Start the consumer.

        When this method is called, the fetching and execution of the jobs
        starts. The job handlers must have been registered beforehand.

        This method is blocking, it only stops in the event of an error
        (only main loop errors, errors that occur in the handlers cause the job
        to fail and are reported to Faktory Server) or when a signal is received
        (Ctrl-C or from the Faktory Web UI).

        At the beginning of the shutdown, the worker gives itself a grace period
        to stop properly and notify the last information to the Faktory server.
        If a second signal is received, this causes an immediate shutdown.
        """
        self.logger.info('Entering run loop..')
        # TODO: check this
        while self.client.state in [State.IDENTIFIED, State.QUIET]:
            try:
                if self.client.state == State.QUIET:
                    self.logger.info(
                        f'State is {self.client.state}, not fetching further jobs'
                    )
                    time.sleep(self.client.beat_period)
                    continue

                if self.pending_tasks_count < self.concurrency:
                    queues_tmp = self.get_queues()
                    self.logger.info(f'Fetching from queues: {queues_tmp}')
                    # If no jobs are found, _fatch will block for up
                    # to 2 seconds on the first queue provided.
                    job = self.client._fetch(queues_tmp)

                    if job:
                        # TODO: check
                        # timeout=job.get('reserve_for', None)
                        job_handler = self.get_job_handler(job.get('jobtype'))

                        future = self.pool.schedule(job_handler,
                                                    args=job.get('args'))

                        with self.lock_pending_tasks_count:
                            self.pending_tasks_count += 1

                        future.job_id = job.get("jid")
                        future.backtrace = job.get("backtrace", 0)
                        future.add_done_callback(self.task_done)
                    else:
                        self.logger.debug('Queues are empty.')
                else:
                    # TODO: maybe use Event object instead of sleep
                    time.sleep(0.1)
            except KeyboardInterrupt:
                self.logger.info(
                    'First KeyboardInterrupt, stopping after grace period')
                break
            except Exception:
                self.logger.error(
                    f'Shutting down due to an error: {traceback.format_exc()}')
                break

        self.logger.info(f'Run loop exited, state is {self.client.state}')
        self.logger.info(f'Grace period of {self.grace_period} seconds...')
        self.logger.info(f'Press Ctrl-C again to stop immediately')

        try:
            self.pool.close()
            self.pool.join(timeout=self.grace_period)
        except KeyboardInterrupt:
            self.logger.info('Second KeyboardInterrupt, stopping immediately')

        self.logger.info(f'End of the grace period. Stopping.')
        sys.exit(1)

# TODO: set rss_kb (sys.getsizeof?)


import torch
from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
                              TensorDataset)
from tqdm import tqdm
from transformers import ElectraForTokenClassification, ElectraConfig, WEIGHTS_NAME, CONFIG_NAME
from transformers import ElectraTokenizer
import time
import pandas as pd
import spacy
import re
import _pickle as cPickle
import numpy as np
from extract_hpi import get_hpi_output
import argparse

#load pickle file which is a label to id mapping required for predictions
with open(r"./label2id.pkl", "rb") as input_file:
    label2id = cPickle.load(input_file)

_nlp = spacy.load('en_core_sci_lg', disable=['ner', 'parser', 'tagger', 'lemmatizer'])
_nlp.create_pipe('sentencizer')
_nlp.add_pipe('sentencizer')

tokenizer = ElectraTokenizer.from_pretrained('./electra/')
model = ElectraForTokenClassification.from_pretrained('./electra/')

def processing_hpi_output(data):
    """
    Clean the HPI section - text analysis

    :return: text
    :rtype: text

    """
    final = []
    for fileid, record in enumerate(data):

        text = re.sub('[\r\n]{2}', '. ', record['text'])
        text = re.sub('\r|\n|\W', ' ', text)
        text = re.sub('\\n', '', text)
        text = re.sub('\r|\n', ' ', text)
        esc = re.escape('.\\n\\n')
        text = re.sub(esc, '', text)
        esc2 = re.escape('\\n')
        text = re.sub(esc2, '', text)
        tokens = {tok.idx: tok.text for tok in _nlp.tokenizer(text)}

        out = ['-DOCSTART- -X- O O', '<s> _ _ O']
        sentence_sep = '</s> _ _ O'

        # get tokens and tags for documents
        for idx, txt in tokens.items():
            if re.search('\n+|\s+', txt):
                out.append('<blank> _ _ ')
            else:
                out.append(txt + ' _ _ ')

        # split documents into sentences and apply HPI sectioning if required
        text = ' '.join(o.split(' ')[0] for o in out)
        doc = _nlp(text)
        sentences = [sent.text.strip() for sent in doc.sents]

        start = 2
        for sent in sentences:
            start += len(sent.split())
            out.insert(start, sentence_sep)
            start += 1

        out.append(sentence_sep)
        final += out
    return final

def get_essentials(filepath):
    """
    Read train/valid/test data from input_dir and
    convert data to features (input_ids, label_ids, attention_masks)

    :return: label2id, num_labels, label2id[pad_token], id2label
    :rtype: dict, number, list, dict

    """

    pad_token = 0
    pad_token_label_id = -100
    with open(filepath, 'r') as f:
        lines = f.readlines()
        sentences, labels, idx, sent, lab, id = [], [], [], [], [], []
        tags = set()
        for line in lines:
            if '-DOCSTART-' in line or '</s>' in line or '<s>' in line or line.rstrip() == '':
                if sent and lab:
                    sentences.append(sent)
                    labels.append(lab)
                    idx.append(id)
                if '-DOCSTART-' in line:
                    sent, lab, id = ['D'], ['O'], ['D{}'.format(line.split()[-2])]
                else:
                    sent, lab, id = [], [], []
            else:
                sent.append(line.split()[0])
                id.append(line.split()[-2])
                lab.append(line.split()[-1])
                tags.add(line.split()[-1])

    # label_map
    label2id = {t: i for i, t in enumerate(list(tags))}
    num_labels = len(label2id)
    label2id[pad_token] = pad_token_label_id
    id2label = {v: k for k, v in label2id.items()}
    return label2id, num_labels, label2id[pad_token], id2label

def prepare_data(filepath):
    """
    Read train/valid/test data from input_dir and
    convert data to tokens

    :return: processed data, tokenized_token, tokenized_sentences
    :rtype: dataframe, list, list

    """

    pad_token = 0
    with open(filepath, 'r') as f:
        lines = f.readlines()
        sentences, sent = [], []
        for line in lines:
            if '-DOCSTART-' in line or '</s>' in line or '<s>' in line or line.rstrip() == '':
                if sent:
                    sentences.append(sent)
                if '-DOCSTART-' in line:
                    sent = ['D']
                else:
                    sent = []
            else:
                sent.append(line.split()[0])

    # tokenize the sentences and save the start offset of each subwords
    tokenized_sentences, tokenized_token = [], []
    for sent in sentences:
        tokenized_sent, tokenized_tok = [], []
        for word in sent:
            tokenized_word = tokenizer.tokenize(word)
            tokenized_sent.extend(tokenized_word)
            tokenized_tok.extend([word] * len(tokenized_word))
        # truncate the subword tokens longer than maxium sequence length
        if len(tokenized_sent) > 512:
            tokenized_sent = tokenized_sent[: 512]
            tokenized_tok = tokenized_tok[: 512]
        tokenized_sentences.append(tokenized_sent)
        tokenized_token.append(tokenized_tok)

    input_ids, attention_masks = [], []
    for sent in tokenized_sentences:
        # get token's id and label's id
        input_id = tokenizer.convert_tokens_to_ids(sent)
        # The mask has 1 for real tokens and 0 for padding tokens. Only real tokens are attended to
        input_mask = [1] * len(input_id)
        # Zero-pad up to the sequence length (pad on right)
        padding_length = 512 - len(input_id)
        input_id += [pad_token] * padding_length
        input_mask += [0] * padding_length
        input_ids.append(input_id)
        attention_masks.append(input_mask)

    data = TensorDataset(torch.tensor(input_ids), torch.tensor(attention_masks))
    return data, tokenized_token, tokenized_sentences

def format_tags(predictions,tokenized_token,id2label):
    """
    convert ids to original labels and create formatted output prediction

    :return: predicted tags, labels, indices
    :rtype: list, list, list

    """

    pred_tags, out = [], []
    docid =0
    start=0
    indices = []
    for prediction, token in zip(predictions,tokenized_token):
        i,j = False,False
        docid+=1
        if token == ['D']:
            i=True
        for index, (pred, tok) in enumerate(zip(prediction, token)):
            pred_tags.append(id2label[pred])
            text = '{} {}'.format(tok, id2label[pred])
            if text == 'D O':
                j=True
            out.append(text)
            if i==False and j==True:
                indices.append(start)
            j=False
            start+=1
        out.append('')
        start+=1
    return pred_tags, out,indices

def get_text_file(hpi_ip,col_name):
    """
    perform HPI processing

    """
    hpi_output = get_hpi_output(hpi_ip,col_name)
    processed_hpi_output = processing_hpi_output(hpi_output)
    with open('processed_file.txt', 'w') as f:
        f.writelines("%s\n" % o for o in processed_hpi_output)

def predict(file):
    """
    Load the saved model and perform prediction

    :return: labels dataframe
    :rtype: dataframe

    """
    hpi_ip = pd.read_csv(file)
    get_text_file(hpi_ip)
    te_data, tt, ts = prepare_data('./processed_file.txt')
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    test_dataloader = DataLoader(te_data, sampler=SequentialSampler(te_data), batch_size=32)
    model.to(device)
    model.eval()
    test_loss = 0
    test_steps = 0
    predictions, pr_label = [], []
    start = time.time()
    for batch in tqdm(test_dataloader, desc='Prediction'):
        # move batch to gpu
        b_input_ids, b_input_mask = tuple(t.to(device) for t in batch)
        with torch.no_grad():
            # Forward pass
            outputs = model(b_input_ids, attention_mask=b_input_mask)
            logits = outputs.logits
        logits = logits.detach().cpu().numpy()
        predictions.extend([list(p) for p in np.argmax(logits, axis=2)])

        test_steps += 1
    test_loss /= test_steps
    test_time = time.time() - start
    id2label = {v: k for k, v in label2id.items()}
    pred_tags, out, indices = format_tags(predictions, tt, id2label)
    index_pos_list = [i for i in range(len(out)) if out[i] == 'D O']
    for i in tqdm(indices, desc='indices'):
        index_pos_list.pop(index_pos_list.index(i))
    final = []
    for i in range(len(index_pos_list)):
        start = index_pos_list[i]
        if i < len(index_pos_list) - 1:
            end = index_pos_list[i + 1]
            final.append(out[start:end])
        else:
            final.append(out[start:])
    tags = []
    for i in tqdm(final):
        s = []
        for j in i:
            s.append(j.split(' ')[-1])
        tags.append(list(set(s)))
    labels = []
    for i in tqdm(tags, desc="Tags"):
        k = []
        for j in i:
            if j != '' or j != 'O':
                k.append(j.split('-')[-1])

        labels.append(k)
    hpi_ip['labels'] = pd.Series(labels)
    return hpi_ip

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Predict labels for clinical text')
    parser.add_argument('-ipf', '--input_file', dest='ipf', help='Path to input file', required=True)
    parser.add_argument('-cn', '--column_name', dest='cn', help='Name of the column containing the note', required=True)
    parser.add_argument('-opf', '--output_file', dest='opf', help='Path to output file', required=True)
    args = parser.parse_args()
    final_op = predict(args.ipf)
    final_op.to_csv(args.opf, index=False)


import torch
import json

speakers = torch.load('../models/vits_ca/speakers.pth')
print(type(speakers))
conv = [line.strip().split(',') for line in open('speakers_conversion.csv').readlines()]
new_speakers = {}
for source, target in conv:
    id = speakers.get(source)
    if id:
        new_speakers[target] = source
with open('speaker_ids.json', 'w') as out:
    json.dump(new_speakers, out)
print(new_speakers)


#!/usr/bin/python3
import matplotlib.pyplot as plt
import numpy as np

x = np.linspace(-10,10,100) # 创建一个包含-10到10之间100个等距点的数组作为x坐标
y= x ** 2 # 计算y坐标

plt.plot(x, y)  # 绘制曲线
plt.xlabel('x') # 设置x轴标签
plt.xlabel('y') # 设置y轴标签
plt.title('y = x^2') # 设置图标题
plt.grid(True) # 显示网格线

plt .show() #显示图形


from PIL import Image
import numpy as np
import cv2
import pdb
import os
import time
import signal
import argparse
import json
import shutil

def split_data(path_, set_):

    path_ = path_ + "/"
    file_array = [file for file in os.listdir(path_) if file.endswith('.txt')]
    file_array = sorted(file_array)

    if set_ == "train":
       split_save_dir = "train/"
       step = 1
    else:
       split_save_dir = "test/"
       step = 5

    if not os.path.exists(split_save_dir):
        os.makedirs(split_save_dir)

    for k in range(0, len(file_array), step):
        copy_path = path_ + file_array[k]
        save_path = split_save_dir + file_array[k]
        shutil.copyfile(copy_path, save_path)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', type=str, default='out_dir/Event_ReId/')
    params = parser.parse_args()

    for i in range(33):
            i_d = "{0:03}".format(i+1)
            path = params.data_dir + str(i_d) + '/'
            sub_path = os.listdir(path)

            for j in range(4):
                complete_path = os.path.join(path, sub_path[j])
                id_ = i+1
                if id_ not in [2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 32]:
                   set_ = "train"
                else:
                   set_ = "test"
                split_data(complete_path, set_)

if __name__ == "__main__":
    main()


# encoding: utf-8
"""
Core
"""
import json
import pytz

from typing import Any, Dict
from base64 import b64decode
from datetime import datetime
from google.cloud import datastore
from jsonschema import validate
from jsonschema.exceptions import ValidationError, SchemaError

import config
import log

def current_datetime():
    return datetime.now(pytz.timezone("America/Sao_Paulo")).isoformat()

def validate_schema(doc_value: json, schema_properties: json) -> bool:
    """
        Parameters
        ----------
        doc_value : json
            The document json to validate
        schema_properties : json
            Json schema with properties to validate doc_value
    import datetime
        Returns
        -------
        Boolean
            Return with the boolean value

    """
    try:
        validate(instance=doc_value, schema=schema_properties)
    except (ValidationError, SchemaError):
        return False
    return True

def validate_manifest(pubsub_message: Any) -> Dict[str, Any]:
    """
    Parameters
    ----------
    pubsub_message : str
        Pub/Sub Message to Validate
    Returns
    -------
    JSON Manifest, Dict
        Return JSON dict with Manifest Config

    """
    json_manifest = json.loads(b64decode(pubsub_message).decode("utf-8"))
    print(json_manifest)
    try:
        result = validate_schema(json_manifest, config.MANIFEST_SCHEMA)
        if result:
            return json_manifest
        return None
    except ValueError:  # Invalid JSON
        return None

def datastore_publish(json_manifest: json, id: str):
    """
    Parameters
    ----------
    json_manifest : json
        JSON with Manifest contract
    id : str
        Id of Manifest
    Returns
    -------
    None
    """
    # Instantiates a client
    datastore_client = datastore.Client()

    # The kind for the new entity
    kind = "schema-repository"
    # The Cloud Datastore key for the new entity
    task_key = datastore_client.key(kind, id)

    # Verify if Document Alredy Exists and
    # update `version`` and `last_update`` values
    #
    task_result = datastore_client.get(task_key)
    version = (
        int(task_result["manifest"]["meta"]["version"]) + 1
        if task_result is not None
        else 1
    )

    json_manifest["meta"]["version"] = version
    json_manifest["meta"]["last_update"] = current_datetime()

    # Prepares the new entity
    task = datastore.Entity(key=task_key)
    task["manifest"] = json_manifest

    # Saves the entity
    datastore_client.put(task)

    log.info(f"Saved {task.key.name}: {task['manifest']}")


import torch
from HDGCN import HDGCN
from utils import DatasetLoader, accuracy
from torch.utils.data import DataLoader
from adabelief_pytorch import AdaBelief
import torch.nn.functional as F

#
# Settings.
#

torch.cuda.set_device(4)
learning_rate = 0.001
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

#
# load datasets
#
batch_size = 32
bert_dim = 300
train_data = DatasetLoader('mr', set_name="train")
vocab = train_data.vocab
test_data = DatasetLoader('mr', set_name="test")
max_seq_len = train_data.nnodes
train_data_loader = DataLoader(dataset=train_data, batch_size=batch_size, shuffle=True)
test_data_loader = DataLoader(dataset=test_data, batch_size=batch_size, shuffle=True)
nhid = 300
vote_dim = 100
nclass = train_data.nclass()
input_cols = ['node_embeddings', 'dependency_graph', 'polarity']

#
# Create capsule network.
#

network = HDGCN(nnodes=max_seq_len,
                nfeat=bert_dim,
                nhid=nhid,
                nclass=nclass,
                max_seq_len=max_seq_len,
                device=device,
                batch_size=batch_size,
                vocab=vocab).to(device)

optimizer = AdaBelief(network.parameters(), lr=learning_rate, eps=1e-16, betas=(0.9, 0.999),
                      weight_decouple=True, rectify=False)

# Converts batches of class indices to classes of one-hot vectors.
def to_one_hot(x, length):
    batch_size = x.size(0)
    x_one_hot = torch.zeros(batch_size, length)
    for i in range(batch_size):
        x_one_hot[i, x[i]] = 1.0
    return x_one_hot

def test():
    network.eval()
    test_loss = 0
    correct = 0
    total_samples = 0
    with torch.no_grad():
        for i_batch, sample_batched in enumerate(test_data_loader):
            inputs = [sample_batched[col].to(device) for col in input_cols]
            sentences = sample_batched['sentence']
            output = network(inputs[0], inputs[1], sentences, batch=i_batch, vis=False)
            test_loss += float(F.nll_loss(output, inputs[2].max(1, keepdim=False)[1]).item())
            total_samples += len(output)
            correct += accuracy(output, inputs[2].max(1, keepdim=False)[1])

        test_loss /= (i_batch + 1)
        correct = correct.item()
        correct /= (i_batch + 1)
    return correct

def train(epoch):
    train_loss = 0.
    correct = 0.
    total_samples = 0
    network.train()
    for i_batch, sample_batched in enumerate(train_data_loader):
        inputs = [sample_batched[col].to(device) for col in input_cols]
        sentences = sample_batched['sentence']
        optimizer.zero_grad()
        if epoch % 5 == 0:
            visualize = True
        else:
            visualize = False
        output = network(inputs[0], inputs[1], sentences, batch=i_batch, vis=visualize)
        # loss = network.loss(output, inputs[2])
        loss = F.nll_loss(output, inputs[2].max(1, keepdim=False)[1])
        loss.backward()
        train_loss += float(loss.item())
        optimizer.step()
        total_samples += len(output)
        correct += accuracy(output, inputs[2].max(1, keepdim=False)[1])

    correct = correct.item()
    correct /= (i_batch + 1)

    train_loss /= (i_batch + 1)
    print('Train Epoch: {} \t Loss: {:.6f}, \t Accuracy: {:.6f}'.format(epoch, train_loss, correct))
    return train_loss

num_epochs = 1000
best_correct = 0.0
corrects = []
for epoch in range(1, num_epochs + 1):
    print('training epochs: {}'.format(epoch))
    train_loss = train(epoch)
    correct = test()
    corrects.append(correct)
    if correct > best_correct:
        best_correct = correct
    print("best correct: {:.6f} \n".format(best_correct))
    print('>' * 100)


#!/usr/bin/env python3

import multiprocessing
import time
import os

from src.model.load_json import load_json_to_dict
from src.model.load_script import load_script_file
from src.model.shunt import Shunt
from src.tools.engine.code.github.clone import CodeClone
from src.tools.log4py.log4py import print_log
from src.tools.log4py.log4py import LogLevel
from src.tools.engine.soft.install_soft import SoftTool
from src.config.config import configration

def main_process():

    print_log(msg=f"Main Process Started: PID {os.getpid()}", level=LogLevel.INFO)
    while True:
        print_log("Main Process is running...")
        folder_path = configration.get("script_path")
        script_type=[".json", ".py", ".shell", "sh"]
        scripts = load_script_file(folder_path=folder_path, script_type=script_type)

        print_log(msg="Begin execute scripts.", level=LogLevel.DEBUG)
        for item in scripts:

            print_log(msg=f"Begin run script: {item}.", level=LogLevel.DEBUG)

            script_shunt = Shunt(file_path=item, folder_path=folder_path)
            script_shunt.operation()
            print("|")
            print("|")
            print("|")

            print_log(msg=f"Run script: {item} over.", level=LogLevel.DEBUG)
            break

        time.sleep(configration.get("mian_process_sleep_time", 300))  # 模拟主进程工作

def monitor_process():
    while True:
        # 使用进程ID来检查主进程是否存活
        main_process_alive = False
        for process in multiprocessing.active_children():
            if process.name == 'MainProcess':
                main_process_alive = True
                break

        if not main_process_alive:
            print_log(msg="Main Process down! Restarting...", level=LogLevel.ERROR)
            # 重启主进程
            new_main_process = multiprocessing.Process(target=main_process, name='MainProcess')
            new_main_process.start()
        else:
            print_log(mgs=f"Main Process is alive.", level=LogLevel.INFO)

        time.sleep(30)  # 每隔3秒检查一次

if __name__ == "__main__":
    # 创建主进程
    main_proc = multiprocessing.Process(target=main_process, name='MainProcess')
    main_proc.start()

    # 创建监控进程
    monitor_proc = multiprocessing.Process(target=monitor_process)
    monitor_proc.start()

    main_proc.join()
    monitor_proc.join()


import os, sys, statistics as stats
tags = []

#Outliner v1.0
#Copyright Richard Gustafsson
#Release: Oct 15 2019

def spc(i=1337):
	if i != 1337:
		print(i)
	print("")

if os.path.isfile("tags.txt"):
	with open("tags.txt") as file:
		tags = [line.strip() for line in file]

	print("Tags loaded:")
	for index in range(0, len(tags)):
		if index == len(tags)-1:
			print(tags[index])
		else:
			print(tags[index], end=", ")
	spc()

else:
	print("ERROR: Could not find file tags.txt in this directory")
	print("Make a file called tags.txt and write one tag per line")
	input("Press any key to continue...")
	sys.exit()

def clear():
	if os.name == "nt":
		os.system("cls")
	else:
		os.system("clear")

def separate(file):
	with open(file) as f:
		tempFirstLine = f.readline()
		tempFirstLine = tempFirstLine.split()

		#reset the file pointer, otherwise the first row would
		#be ignored since it already has been read with f.readline()
		f.seek(0)

		#create a list containing each line, with any whitespace and
		#other weird chars like \n removed
		tempList = []
		for line in f.readlines():
			tempList.append(line.rstrip())

		return tempList, len(tempFirstLine)

def makeNumber(i):
	try:
		return int(i)
	except:
		return 0

def tryPrint(n):
	try:
		print(f"---{tags[n]}---")
	except:
		print("ERROR: UNKNOWN TAG")

def flattenData(fp):
	spc()
	spc()
	data, dataPerRow = separate(fp)

	lineCount = 0
	for line in data:
		lineCount += 1

	for i in range(1, dataPerRow+1):

		totalSum = 0
		medianList = []

		for line in data:

			tempLine = line.split()
			#if the list is still in range, continue
			try:
				totalSum += makeNumber(tempLine[i-1])
				medianList.append(makeNumber(tempLine[i-1]))
			except:
				pass

		#Calculate the average and the median
		avg = totalSum / lineCount

		med = stats.median(medianList)

		#print the content
		tryPrint(i-1)
		print(f"Sum: {totalSum}, Average: {round(avg, 1)}, Median {med}")
		spc()
	spc()
	print("Ready for another round")

while True:
	filePath = input("Name of data file (with extension)? ")
	if len(filePath) > 0:
		if os.path.isfile(filePath):
			flattenData(filePath)
		else:
			spc("ERROR: Not a file")
	else:
		spc("ERROR: Files must contain at least one character")


# import the libraries
from datetime import timedelta
# The DAG object; we'll need this to instantiate a DAG
from airflow import DAG
# Operators; we need this to write tasks!
from airflow.operators.bash_operator import BashOperator
# This makes scheduling easy
from airflow.utils.dates import days_ago
#defining DAG arguments

# You can override them on a per-task basis during operator initialization
default_args = {
    'owner': 'Amarigh',
    'start_date': days_ago(0),
    'email': ['amarigmustapha@gmail.com'],
    'email_on_failure': False,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=5),
}
# defining the DAG
# define the DAG
dag = DAG(
    'ETL_server_access_dag',
    default_args=default_args,
    description='ETL server access processing',
    schedule_interval=timedelta(days=1),
)

download= BashOperator(
    task_id='Download',
    bash_command='wget https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DB0250EN-SkillsNetwork/labs/Apache%20Airflow/Build%20a%20DAG%20using%20Airflow/web-server-access-log.txt' ,
    dag=dag,
)

extract= BashOperator(
  task_id='extract',
  bash_command=' cut -d"#" -f1,4 web-server-access-log.txt > extracted-data.txt ',
  dag=dag,
)

transform = BashOperator (
task_id="transform" ,
bash_command=' tr [--lower--] [--upper--] | tr "#" "," > extracted-data.csv',
dag=dag,
)

load = BashOperator(
task_id="load",
bash_command='gzip extracted-dat.csv',
dag=dag,
)

download >> extract >> transform >> load


#Run old_indexer first!

import sys

import whoosh.index as index
import whoosh.qparser as qparser
from whoosh.searching import Searcher

correct = 0
queries = 0

#Opens index
ix = index.open_dir("oldIndex")

#Opens test file
with open(sys.argv[1], 'r') as f:
    while True:
        #Reads next query/url pair
        line = f.readline()
        if not line:
            break

        #Loads query and expeced answer
        query = line.split(';')[0]
        expected = line.split(';')[1].rstrip('\n')
        retrieved = []
        found = False
        queries += 1

        #Handles query
        qp = qparser.QueryParser("content", schema=ix.schema)
        q = qp.parse(query)

        #Searches index for query, checks top 3 URLs for expected URL
        with ix.searcher() as searcher:
            results = searcher.search(q)
            #Correctness metric: Is our expected URL in the top 3?
            for i in range(3):
                retrieved.append(results[i]['title'])
                if expected == retrieved[i]:
                    correct += 1
                    found = True
            print("Query {}:".format(queries), query)
            print("URL:", expected)
            if found:
                print("PASSED")
            else:
                print("FAILED")
            #Uncomment to print top 3 URLs from IR
            #print("{}\n{}\n{}\n".format(retrieved[0], retrieved[1], retrieved[2]))

print("{} / {} queries matched.".format(correct, queries))
print("Accuracy: ", float(correct/queries))


# --------------------------------------------------------
# Licensed under The MIT License [see LICENSE for details]
# --------------------------------------------------------
import torch
from torch import nn
import numpy as np
from core.utils import *
import torch.nn.functional as F
from torch.distributions import Normal
from core.network_arch import *

class PointTrajLatentNet(nn.Module):
    def __init__(
        self,
        input_dim=3,
        pointnet_nclusters=32,
        pointnet_radius=0.02,
        model_scale=1,
        output_model_scale=1,
        extra_latent=0,
        large_feature=False,
        feature_option=0,
        extra_pred_dim=2,
        group_norm=True,
        **kwargs
    ):
        """
        pointnet++ backbone network
        """
        super(PointTrajLatentNet, self).__init__()
        self.input_dim = 3 + extra_latent
        self.model_scale = model_scale

        self.encoder = create_encoder(model_scale, pointnet_radius, pointnet_nclusters, self.input_dim, feature_option, group_norm=group_norm)
        self.feature_option = feature_option
        self.fc_embedding   = nn.Linear(int(self.model_scale * 512), int(output_model_scale * 512))

        self.extra_pred_dim = extra_pred_dim
        self.fc_extra  = get_fc_feat_head(int(self.model_scale * 512), [256, 64], extra_pred_dim)

    def forward(
        self,
        pc,
        grasp=None,
        feature_2=False,
        train=True,
        traj_state=None,
        traj_inbatch_index=None,
        traj_time_batch=None,
        traj_latent=None,
        encode=True
    ):

        input_features = pc
        extra = input_features
        if input_features.shape[-1] != 4096:
            input_features = input_features[...,6:]
        input_features =  input_features[:, :self.input_dim].contiguous()
        object_grasp_pc = input_features.transpose(1, -1)[..., :3].contiguous()
        point_feat = pointnet_encode(self.encoder, object_grasp_pc, input_features)
        z, extra = self.fc_embedding(point_feat), self.fc_extra(point_feat)

        extra = point_feat
        return z, extra

class PointNetFeature(nn.Module):
    def __init__(
        self,
        input_dim=3,
        pointnet_nclusters=32,
        pointnet_radius=0.02,
        model_scale=1,
        extra_latent=0,
        split_feature=False,
        policy_extra_latent=-1,
        critic_extra_latent=-1,
        action_concat=False,
        feature_option=0,
        group_norm=True,
        **kwargs ):
        """
        poinet++ feature network
        """
        super(PointNetFeature, self).__init__()
        self.input_dim = 3 + extra_latent
        input_dim = (3 + policy_extra_latent if policy_extra_latent > 0 else self.input_dim )
        self.policy_input_dim = input_dim
        self.model_scale = model_scale
        self.encoder = create_encoder(
            model_scale, pointnet_radius, pointnet_nclusters, self.policy_input_dim, feature_option, group_norm=group_norm )
        input_dim = 3 + critic_extra_latent if critic_extra_latent > 0 else input_dim
        self.critic_input_dim = input_dim
        if action_concat: self.critic_input_dim = input_dim + 6

        self.value_encoder = create_encoder(
            model_scale, pointnet_radius, pointnet_nclusters, self.critic_input_dim, feature_option, group_norm=group_norm )
        self.feature_option = feature_option

    def forward(
        self,
        pc,
        grasp=None,
        concat_option="channel_wise",
        feature_2=False,
        train=True,
        traj_state=None,
        traj_inbatch_index=None,
        traj_time_batch=None,
        traj_latent=None ):

        input_features = pc
        extra = input_features
        if input_features.shape[-1] != 4096:
            input_features = input_features[...,6:]
        input_features = (
            input_features[:, : self.critic_input_dim].contiguous()
            if feature_2
            else input_features[:, : self.policy_input_dim].contiguous()
        )
        object_grasp_pc = input_features.transpose(1, -1)[..., :3].contiguous()

        encoder = self.value_encoder if feature_2 else self.encoder
        z = pointnet_encode(encoder, object_grasp_pc, input_features)
        return z, extra

class STPointNetFeature(nn.Module):
    def __init__(
        self,
        input_dim=3,
        pointnet_nclusters=32,
        pointnet_radius=0.02,
        model_scale=1,
        extra_latent=0,
        feature_option=1,
        group_norm=True,
        **kwargs ):
        """
        spatiotemporal point network
        """
        super(STPointNetFeature, self).__init__()
        self.base_dim = 4 + extra_latent
        self.encoder, self.output_dim = create_encoder(
            model_scale, pointnet_radius, pointnet_nclusters,
            self.base_dim, feature_option,  traj_net=True, group_norm=group_norm )
        self.feature_option = feature_option

    def forward(
        self,
        pc,
        grasp=None,
        concat_option="channel_wise",
        feature_2=False,
        train=True,
        traj_state=None,
        traj_inbatch_index=None,
        traj_time_batch=None,
        traj_latent=None
    ):
        input_features = pc
        if input_features.shape[-1] != 4096:
            input_features = input_features[...,6:] # ignore hand points

        input_features_vis = input_features
        traj_time_batch = traj_time_batch[...,None].expand(-1, -1, input_features.shape[2])
        input_features  = torch.cat((input_features, traj_time_batch), dim=1)
        pc = input_features.transpose(1, -1)[..., :3].contiguous()

        global_feat = []
        for idx in torch.unique(traj_inbatch_index): # each traj pc separate process no speed up with batch
            index = torch.where(traj_inbatch_index == idx)
            size = len(index[0])
            global_pc = input_features[index].transpose(1, -1).contiguous().view(-1, self.base_dim).T.contiguous()[None]
            global_feat_i = self.encoder[0](global_pc)[0].expand(size, -1)
            global_feat.append(global_feat_i)

        global_feat = torch.cat(global_feat, dim=0)
        input_features = input_features[:,:self.base_dim].contiguous()
        local_feat_1 = pointnet_encode(self.encoder[-1], pc, input_features) # each timestep pc
        z = torch.cat((global_feat, local_feat_1), dim=1)
        return z, input_features_vis

class TrajSamplerNet(nn.Module):
    def __init__(
        self,
        num_inputs,
        num_actions,
        hidden_dim,
        action_space=None,
        extra_pred_dim=0,
        config=None,
        input_dim=3,
        **kwargs ):
        """
        latent plan sampler network
        """
        super(TrajSamplerNet, self).__init__()
        self.config = config
        self.setup_latent_sampler(**kwargs)

    def setup_latent_sampler(self, **kwargs):
        config = self.config
        input_dim = config.traj_latent_size
        self.curr_state_encoder = eval(config.traj_vae_feature_extractor_class)(**kwargs)
        self.sampler_bottleneck = create_bottleneck(config.policy_traj_latent_size, config.normal_vae_dim)
        self.cvae_encoder = get_fc_feat_head(input_dim + config.policy_traj_latent_size, [1024, 512, 512, 256, 256, 128], config.policy_traj_latent_size)
        self.cvae_decoder = get_fc_feat_head(input_dim + config.normal_vae_dim, [1024, 512, 512, 256, 256, 128], config.policy_traj_latent_size)
        self.apply(weights_init_)

    def forward(self,
                curr_point_state,
                exec_point_state=None,
                grasp=None,
                train=True,
                index_mask=None,
                extra_time=None,
                traj_latent=None,
                traj_time_batch=None,
                traj_inbatch_index=None,
                encode=True,
                vis=False):

        traj_sampler_latent, extra_feat_pred =  self.curr_state_encoder(curr_point_state,
                                                traj_latent=traj_latent,
                                                traj_time_batch=traj_time_batch,
                                                traj_inbatch_index=traj_inbatch_index,
                                                encode=encode)

        return traj_sampler_latent, None, None, extra_feat_pred

    def forward_bottleneck(self, traj_feat, traj_inbatch_index=None, prev_latent=None, traj_latent=None):
        sampler_mu, sampler_logsigma = self.sampler_bottleneck[0](traj_feat), self.sampler_bottleneck[1](traj_feat)
        if traj_inbatch_index is not None:
            sampler_mu_, sampler_logsigma_ = sampler_mu[traj_inbatch_index], sampler_logsigma[traj_inbatch_index]
            sample = reparameterize(sampler_mu_, sampler_logsigma_)
        else:
            sample = reparameterize(sampler_mu, sampler_logsigma)
        return sample, sampler_mu, sampler_logsigma

    def conditional_sampler_vae_head(self, traj_feat, traj_inbatch_index=None, conditional_latent=None):
        """
        conditional vae forward pass
        """
        sampler_mu, sampler_logsigma = None, None
        if conditional_latent is not None:
            encoded_latent = self.cvae_encoder(torch.cat((traj_feat[traj_inbatch_index][:len(conditional_latent)], conditional_latent), dim=-1))
            sampled_encoded_latent, sampler_mu, sampler_logsigma = self.forward_bottleneck(encoded_latent)
        else:
            sampled_encoded_latent = sample_gaussian((max(traj_feat.shape[0], len(traj_inbatch_index)), self.config.normal_vae_dim), truncate_std=self.config.test_log_sigma_clip).cuda()

        decoded_latent = self.cvae_decoder(torch.cat((traj_feat[traj_inbatch_index], sampled_encoded_latent), dim=-1))
        return decoded_latent, sampler_mu, sampler_logsigma, sampled_encoded_latent

class TrajEmbeddingNet(nn.Module):
    def __init__(
        self,
        feature_extractor_class,
        num_inputs,
        num_actions,
        hidden_dim,
        action_space=None,
        extra_pred_dim=0,
        config=None,
        input_dim=3,
        **kwargs
        ):
        """
        latent plan embedding network
        """
        super(TrajEmbeddingNet, self).__init__()
        config.num_inputs = num_inputs
        config.action_dim = num_actions
        config.action_space = PandaTaskSpace6D()
        self.config = config
        self.traj_encoder = eval(feature_extractor_class)(**kwargs)
        self.fc_embedding  = get_fc_feat_head(self.traj_encoder.output_dim, [512], config.traj_latent_size, end_with_act=True)
        self.traj_fc_embedding = nn.Linear(config.traj_latent_size, config.traj_latent_size)
        self.apply(weights_init_)

    def forward(self,
                traj_point_state=None,
                train=True,
                traj_state=None,
                traj_joint=None,
                traj_inbatch_index=None,
                traj_time=None,
                traj_goal=None,
                traj_action=None,
                traj_pose=None,
                vis=False,
                val=False,
                **kwargs):
        return  self.traj_encoder(   traj_point_state,
                                traj_state=traj_state,
                                traj_inbatch_index=traj_inbatch_index,
                                traj_time_batch=traj_time)

    def head(self, feat, traj_inbatch_index, val=False):
        """
        summarize local and global point features
        """
        feat_embedding = self.fc_embedding(feat)
        traj_feat = []
        for idx in torch.unique(traj_inbatch_index):
            traj_idx = torch.where(traj_inbatch_index == idx)
            global_feat_embedding = feat_embedding[traj_idx].max(0)
            traj_feat.append(global_feat_embedding[0])

        traj_feat = torch.stack(traj_feat, dim=0)
        traj_feat = self.traj_fc_embedding(traj_feat)
        return traj_feat, None


import os
import pandas as pd
import numpy as np
import os
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.pipeline import make_pipeline, Pipeline
import platform
import sys
import sklearn
import tensorflow as tf

file_path = 'H:/Study/Hackarthon/dacon/shopping/dataset/dataset'

train = pd.read_csv('H:/Study/Hackarthon/dacon/shopping/dataset/dataset/train.csv')
test = pd.read_csv('H:/Study/Hackarthon/dacon/shopping/dataset/dataset/test.csv')
sample_submission = pd.read_csv('H:/Study/Hackarthon/dacon/shopping/dataset/dataset/sample_submission.csv')

print(f"-os:{platform.platform()}")
print(f"-python:{sys.version}")
print(f"-pandas:{pd.__version__}")
print(f"-numpy:{np.__version__}")
print(f"-sklearn: {sklearn.__version__}")
print(f'-tensorflow: {tf.__version__}')

# print(train.shape) (6255, 13)
# print(test.shape) (180, 12)
# print(sample_submission.shape) (180, 2)

# print(train.info())

# #   Column        Non-Null Count  Dtype
# ---  ------        --------------  -----
#  0   id            6255 non-null   int64
#  1   Store         6255 non-null   int64
#  2   Date          6255 non-null   object
#  3   Temperature   6255 non-null   float64
#  4   Fuel_Price    6255 non-null   float64
#  5   Promotion1    2102 non-null   float64
#  6   Promotion2    1592 non-null   float64
#  7   Promotion3    1885 non-null   float64
#  8   Promotion4    1819 non-null   float64
#  9   Promotion5    2115 non-null   float64
#  10  Unemployment  6255 non-null   float64
#  11  IsHoliday     6255 non-null   bool
#  12  Weekly_Sales  6255 non-null   float64
# dtypes: bool(1), float64(9), int64(2), object(1)
# memory usage: 592.6+ KB
# None

train = train.fillna(0)

# print(train.shape) (6255, 13)

def date_encoder(date):
    day, month, year = map(int, date.split('/'))
    return day, month, year

train['Day'] = train['Date'].apply(lambda x: date_encoder(x)[0])
train['Month'] = train['Date'].apply(lambda x: date_encoder(x)[1])
train['Year'] = train['Date'].apply(lambda x: date_encoder(x)[2])

train = train.drop(columns=['Day', 'Date'])

scaler = StandardScaler()

scaler.fit(train[['Promotion1', 'Promotion2','Promotion3', 'Promotion4', 'Promotion5']])

scaled = scaler.transform(train[['Promotion1','Promotion2','Promotion3','Promotion4','Promotion5']])

train[['Scaled_Promotion1','Scaled_Promotion2',
        'Scaled_Promotion3','Scaled_promotion4',
        'Scaled_Promotion5']] = scaled

train = train.drop(columns=['Promotion1','Promotion2','Promotion3','Promotion4','Promotion5'])

test = test.fillna(0)

test['Month']=test['Date'].apply(lambda x: date_encoder(x)[1])
test['Year'] = test['Date'].apply(lambda x:date_encoder(x)[2])
test = test.drop(columns=['Date'])

scaled = scaler.transform(test[['Promotion1','Promotion2','Promotion3','Promotion4','Promotion5']])

test[['Scaled_Promotion1','Scaled_Promotion2','Scaled_Promotion3','Scaled_promotion4','Scaled_Promotion5']] = scaled

test = test.drop(columns=['Promotion1','Promotion2','Promotion3','Promotion4','Promotion5'])

from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from catboost import CatBoostRegressor
from sklearn.model_selection import train_test_split, KFold, cross_val_score, StratifiedKFold

model = make_pipeline(StandardScaler(), RandomForestRegressor())

# model = RandomForestRegressor()

train = train.drop(columns=['id'])
test = test.drop(columns=['id'])

x_train = train.drop(columns=['Weekly_Sales'])
y_train = train['Weekly_Sales']

model.fit(x_train, y_train)

prediction = model.predict(test)
print('---------------------------예측된 데이터의 상위 10개의 값 확인 --------------------------------- \n')
print(prediction[:10])

sample_submission['Weekly_Sales'] = prediction

# sample_submission.head()

sample_submission.to_csv('H:/Study/Hackarthon/dacon/shopping/dataset/dataset/sookook7.csv', index = False)