import errno
from tempfile import TemporaryDirectory
from unittest.mock import patch

import escapism
import pytest

import docker
from repo2docker.__main__ import make_r2d
from repo2docker.app import Repo2Docker
from repo2docker.utils import chdir

def test_find_image():
    images = [{"RepoTags": ["some-org/some-repo:latest"]}]

    with patch("repo2docker.docker.docker.APIClient") as FakeDockerClient:
        instance = FakeDockerClient.return_value
        instance.images.return_value = images

        r2d = Repo2Docker()
        r2d.output_image_spec = "some-org/some-repo"
        assert r2d.find_image()

        instance.images.assert_called_with()

def test_dont_find_image():
    images = [{"RepoTags": ["some-org/some-image-name:latest"]}]

    with patch("repo2docker.docker.docker.APIClient") as FakeDockerClient:
        instance = FakeDockerClient.return_value
        instance.images.return_value = images

        r2d = Repo2Docker()
        r2d.output_image_spec = "some-org/some-other-image-name"
        assert not r2d.find_image()

        instance.images.assert_called_with()

def test_image_name_remains_unchanged():
    # if we specify an image name, it should remain unmodified
    with TemporaryDirectory() as src:
        app = Repo2Docker()
        argv = ["--image-name", "a-special-name", "--no-build", src]
        app = make_r2d(argv)

        app.start()

        assert app.output_image_spec == "a-special-name"

def test_image_name_contains_sha1(repo_with_content):
    upstream, sha1 = repo_with_content
    app = Repo2Docker()
    # force selection of the git content provider by prefixing path with
    # file://. This is important as the Local content provider does not
    # store the SHA1 in the repo spec
    argv = ["--no-build", "file://" + upstream]
    app = make_r2d(argv)

    app.start()

    assert app.output_image_spec.endswith(sha1[:7])

def test_local_dir_image_name(repo_with_content):
    upstream, sha1 = repo_with_content
    app = Repo2Docker()
    argv = ["--no-build", upstream]
    app = make_r2d(argv)

    app.start()

    assert app.output_image_spec.startswith(
        "r2d" + escapism.escape(upstream, escape_char="-").lower()
    )

def test_build_kwargs(repo_with_content):
    upstream, sha1 = repo_with_content
    argv = [upstream]
    app = make_r2d(argv)
    app.extra_build_kwargs = {"somekey": "somevalue"}

    with patch.object(docker.APIClient, "build") as builds:
        builds.return_value = []
        app.build()
    builds.assert_called_once()
    args, kwargs = builds.call_args
    assert "somekey" in kwargs
    assert kwargs["somekey"] == "somevalue"

def test_run_kwargs(repo_with_content):
    upstream, sha1 = repo_with_content
    argv = [upstream]
    app = make_r2d(argv)
    app.extra_run_kwargs = {"somekey": "somevalue"}

    with patch.object(docker.DockerClient, "containers") as containers:
        app.start_container()
    containers.run.assert_called_once()
    args, kwargs = containers.run.call_args
    assert "somekey" in kwargs
    assert kwargs["somekey"] == "somevalue"

def test_root_not_allowed():
    with TemporaryDirectory() as src, patch("os.geteuid") as geteuid:
        geteuid.return_value = 0
        argv = [src]
        with pytest.raises(SystemExit) as exc:
            app = make_r2d(argv)
            assert exc.code == 1

        with pytest.raises(ValueError):
            app = Repo2Docker(repo=src, run=False)
            app.build()

        app = Repo2Docker(repo=src, user_id=1000, user_name="jovyan", run=False)
        app.initialize()
        with patch.object(docker.APIClient, "build") as builds:
            builds.return_value = []
            app.build()
        builds.assert_called_once()

def test_dryrun_works_without_docker(tmpdir, capsys):
    with chdir(tmpdir):
        with patch.object(docker, "APIClient") as client:
            client.side_effect = docker.errors.DockerException("Error: no Docker")
            app = Repo2Docker(dry_run=True)
            app.build()
            captured = capsys.readouterr()
            assert "Error: no Docker" not in captured.err

def test_error_log_without_docker(tmpdir, capsys):
    with chdir(tmpdir):
        with patch.object(docker, "APIClient") as client:
            client.side_effect = docker.errors.DockerException("Error: no Docker")
            app = Repo2Docker()

            with pytest.raises(SystemExit):
                app.build()
                captured = capsys.readouterr()
                assert "Error: no Docker" in captured.err


import torch
import torch.nn as nn

# Import the skrl components to build the RL system
from skrl.models.torch import Model, GaussianMixin, DeterministicMixin
from skrl.memories.torch import RandomMemory
from skrl.agents.torch.ppo import PPO, PPO_DEFAULT_CONFIG
from skrl.resources.schedulers.torch import KLAdaptiveRL
from skrl.resources.preprocessors.torch import RunningStandardScaler
from skrl.trainers.torch import SequentialTrainer
from skrl.envs.torch import wrap_env
from skrl.envs.torch import load_omniverse_isaacgym_env
from skrl.utils import set_seed
from learning.model import StochasticActorHeightmap, DeterministicHeightmap, ObserverationInfo, NetworkInfo
import hydra
from omegaconf import DictConfig
from hydra import compose, initialize
import wandb
import datetime
from omniisaacgymenvs.tasks.utils.camera.heightmap_distribution import Heightmap
from omniisaacgymenvs.tasks.utils.terrains.extract_terrain import extract_terrain
import os
#cfg_ppo = PPO_DEFAULT_CONFIG.copy()

# set the seed for reproducibility
set_seed(42)

class TrainerSKRL():
    def __init__(self):
        self._load_cfg()
        time_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        #self.wandb_group =f"Improved_{time_str}"
        self.wandb_group ="New"
        #self.wandb_name = f"run_{time_str}"
        self.wandb_name = "Run1"
       # self.start_simulation()
        #self.start_training()

    def _load_cfg(self):
        initialize(config_path="cfg", job_name="test_app")
        cfg = compose(config_name="config")

        self.cfg_ppo = PPO_DEFAULT_CONFIG.copy()
        self.cfg_network = cfg.trainSKRL.network
        self.cfg_experiment = cfg.trainSKRL.experiment
        self.cfg_config = cfg.trainSKRL.config
        self.sim_params = cfg.task
        # Set all parameters according to cfg file
        for param, value in (cfg.trainSKRL.config).items():
            self.cfg_ppo[param] = value

        print(self.cfg_ppo)
        hydra.core.global_hydra.GlobalHydra.instance().clear()

    def start_simulation(self):
        env = load_omniverse_isaacgym_env(task_name="Rover")
        self.env = wrap_env(env)

    def log_parameters(self):
        config = {                    "mlp layers": self.cfg_network.mlp.layers,
                    "mlp_activation": self.cfg_network.mlp.activation,
                    "encoder_layers": self.cfg_network.encoder.layers,
                    "encoder_activation": self.cfg_network.encoder.activation,
                    "hyperparameters": self.cfg_config,
                    "rewards": self.sim_params.rewards,
                    "sim parameters": {"environment": self.sim_params.env,
                                        "simulation": self.sim_params.sim,},
                    }

        # for key,value in (self.cfg_config).items():
        #     config[key] = value
        return config
    def train(self,sweep=False):
        env = self.env
        device = env.device
        if not sweep:
            self.log_parameters()

        # Instantiate a RandomMemory as rollout buffer (any memory can be used for this)
        memory = RandomMemory(memory_size=60, num_envs=self.env.num_envs, device=device)

        # Get values from cfg
        mlp_layers = self.cfg_network.mlp.layers
        encoder_layers = self.cfg_network.encoder.layers
        activation_function = self.cfg_network.mlp.activation
        #print(env.num_exteroceptive)
        #TODO fix
        heightmap_distribution = Heightmap()
        num_sparse = heightmap_distribution.get_num_sparse_vector()
        num_dense = heightmap_distribution.get_num_dense_vector()
        num_beneath = heightmap_distribution.get_num_beneath_vector()
        networkInfo = NetworkInfo([256,160,128],[80,60],[80,60],[80,60],"leakyrelu")
        observerationInfo = ObserverationInfo(4,num_sparse,num_dense,num_beneath)

        # Instantiate the agent's models (function approximators).
        models_ppo = {  "policy": StochasticActorHeightmap(env.observation_space, env.action_space, networkInfo, observerationInfo ),
                    "value": DeterministicHeightmap(env.observation_space, env.action_space, networkInfo,observerationInfo)}

        # print()

        # Instantiate parameters of the model
        # for model in models_ppo.values():
        #     model.init_parameters(method_name="normal_", mean=0.0, std=0.05)

        self.cfg_ppo["experiment"]["write_interval"] = 100
        # Define agent
        agent = PPO(models=models_ppo,
                memory=memory,
                cfg=self.cfg_ppo,
                observation_space=env.observation_space,
                action_space=env.action_space,
                device=device)
        #agent.migrate("best.pt")
        #agent.load("agent_219000.pt")
        #agent.load("agent_13000.pt")
        #agent.load("agent_939000.pt")
        # Configure and instantiate the RL trainer
        cfg_trainer = {"timesteps": 1000000, "headless": True}
        trainer = SequentialTrainer(cfg=cfg_trainer, env=env, agents=agent)

        # start training
        #trainer.eval()
        trainer.train()

    def start_training_sweep(self,n_sweeps):
        self.start_simulation()
       # Define sweep config
        sweep_configuration = {
            'method': 'bayes',
            'name': 'sweep',
            'metric': {'goal': 'maximize', 'name': 'Reward / Total reward (mean)'},
            'parameters':
            {
                'mini_batches': {'values': [4, 8]},
                'lr': {'max': 0.003, 'min': 0.00003}

            }
        }

        # Initialize sweep by passing in config. (Optional) Provide a name of the project.
        sweep_id = wandb.sweep(sweep=sweep_configuration, project='isaac-rover-2.0')
        wandb.agent(sweep_id, function=self.sweep, count=n_sweeps)
        # Start sweep job.

    def sweep(self):
        time_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
        #self.wandb_name = f"test-Anton_{time_str}"
        self.wandb_name = "no_dense_encoder"
        run = wandb.init(project='isaac-rover-2.0', sync_tensorboard=True,name=self.wandb_name,group=self.wandb_group, entity="aalborg-university")
        self.cfg_ppo["learning_rate"] = wandb.config.lr
        self.cfg_ppo["mini_batches"] = wandb.config.mini_batches
        self.train()
       # wandb.finish()

    def start_training(self):
        self.start_simulation()
        config = self.log_parameters()
        log=True
        if log:
            wandb.init(project='isaac-rover-2.0', config=config, sync_tensorboard=True,name=self.wandb_name,group=self.wandb_group, entity="aalborg-university")
        self.train()
        if log:
            wandb.finish()

    def start_training_sequential(self):
        for i in range(3):
            print(i)
            self.train()
        pass
if __name__ == '__main__':
    # Get hyperparameter config
    terrainExist = os.path.exists("tasks/utils/terrain/")
    if not terrainExist:
        extract_terrain()
    trainer = TrainerSKRL()
   # trainer.start_training_sequential()
    #trainer.start_training_sweep(4)
    trainer.start_training()
    #parse_hydra_configs()


import logging
import os
from datetime import datetime

from bson.objectid import ObjectId
from celery import shared_task as task
from celery.utils.log import get_task_logger

from products.models import Product
from utils.mongodb import mongo_db, mongo_update

logger = get_task_logger(__name__)

@task(name='jms_products_api')
def jms_import_products_api(mon_insert_id):
    import requests
    time_now = datetime.now()
    db = mongo_db()
    id_code = None
    if not mon_insert_id:
        id_code = db.product_logs.insert_one(
            {"user": None, "status": "Processing",
             "created_at": time_now, })
    response_next = None
    page = 1
    status = "done"
    while response_next or page == 1:
        response_next = None
        token = os.environ.get('3JMSTOKEN', '')
        headers = {'Authorization': "Token " + token}
        response = requests.get(
            "https://staging3jms.com/api/v1/inventory/?page=" + str(page),
            headers=headers
        )

        try:
            response_next = response.json()['next']
        except Exception as e:
            logging.exception(e)
            response_next = None
        try:
            response_result = response.json()['results']
        except Exception as e:
            logging.exception(e)
            response_result = []
            if page == 1:
                status = "error"
        page += 1
        for i in response_result:
            result = i
            try:
                Product.objects.update_or_create(
                    sku=result["vws_product_sku"],
                    defaults={
                        'name': result["name"],
                        'brand': result["brand"],
                        'weight': result["weight"],
                        'year': result["year"],
                        'price': result["price"],
                        'currency': result["currency"],
                        'bottle_size': result["bottle_size"],
                        'image_url': result["image_url"],
                        'category': result["category"],
                        'subcategory': result["subcategory"],
                        'upc': result["upc"]})
            except Exception as e:
                logging.exception(e)
                pass
    finish_time = datetime.now()
    mongo_update("product_logs",
                 {"_id": ObjectId(
                     mon_insert_id) if mon_insert_id else id_code.inserted_id},
                 {
                     "$set": {
                         "finished_at": finish_time,
                         "duration": (finish_time - time_now).total_seconds(),
                         "status": status}
                 }
                 )


from django.conf import settings
from django.utils.translation import ugettext_lazy as _

def robots(request):
    return {'ROBOTS_NOINDEX': getattr(settings, 'ROBOTS_NOINDEX', False)}

def google_analytics(request):
    key = 'GOOGLE_ANALYTICS_TRACKING_ID'
    return {key: getattr(settings, key, False)}

def ribbon(request):
    enabled = getattr(settings, 'RIBBON_ENABLED', True)
    if not enabled:
        return {'ribbon': False}

    url = 'http://github.com/colab/colab'
    text = _('Fork me!')

    return {
        'ribbon': {
            'text': getattr(settings, 'RIBBON_TEXT', text),
            'url': getattr(settings, 'RIBBON_URL', url),
        }
    }


import matplotlib.pyplot as plt
import torch.nn.functional as F
import argparse
import torch
import os

def calc_accuracy(model, data_loader, device):
    correct_pred = 0
    instance_count = 0

    with torch.no_grad():
        model.eval()
        for x, y in data_loader:
            x, y = x.to(device), y.to(device)

            pred = model(x)
            _, pred_labels = torch.max(F.softmax(pred, dim=1), 1)

            instance_count += y.size(0)
            correct_pred += (pred_labels == y).sum().item()

    acc = correct_pred / instance_count
    return acc

def parse_train_args():
    parser = argparse.ArgumentParser(description="Training argument parser")

    parser.add_argument("-ne", "--num_epoch", type=int, required=False, default=10, help="number of epochs")
    parser.add_argument("-bs", "--batch_size", type=int, required=False, default=50, help="batch size")
    parser.add_argument("-ms", "--manual_seed", type=int, required=False, default=0, help="random seed")
    parser.add_argument("-d", "--device", type=str, required=False, default="cuda", help="device: cuda/cpu")
    parser.add_argument("-lr", "--learning_rate", type=float, required=False, default=0.001, help="learning rate")
    parser.add_argument("-nc", "--num_classes", type=int, required=False, default=10, help="number of classes")
    parser.add_argument("-tr", "--training_set_ratio", type=float, required=False, default=0.8,
                        help="the ratio between training and validation set")
    parser.add_argument("-mp", "--model_dir", type=str, required=False, default="./checkpoints",
                        help="model path to save")
    parser.add_argument("-pf", "--plot_flag", type=bool, required=False, default=True, help="plot all test results")
    parser.add_argument("-sc", "--save_checkpoint", type=int, required=False, default=10,
                        help="saving checkpoint every save_checkpoint epochs")
    parser.add_argument("-lm", "--load_model", type=bool, required=False, default=False,
                        help="load pretrained model, on restart case")
    parser.add_argument("-lmp", "--load_model_path", type=str, required=False,
                        help="path of the pretrained model to load, on restart case")

    args = parser.parse_args()
    return args

def parse_test_args():
    parser = argparse.ArgumentParser(description="Testing argument parser")

    parser.add_argument("-d", "--device", type=str, required=False, default="cpu", help="device: cuda/cpu")
    parser.add_argument("-nc", "--num_classes", type=int, required=False, default=10, help="number of classes")
    parser.add_argument("-ms", "--manual_seed", type=int, required=False, default=0, help="random seed")
    parser.add_argument("-bs", "--batch_size", type=int, required=False, default=1, help="batch size")
    parser.add_argument("-mp", "--model_path", type=str, required=True, help="model path to load")
    parser.add_argument("-pf", "--plot_flag", type=bool, required=False, default=False, help="plot all test results")

    args = parser.parse_args()
    return args

def plot_loss_accuracy(train_loss_lst: list, valid_loss_lst: list, train_acc_lst: list, valid_acc_lst: list):
    fig = plt.figure()
    plt.plot(train_loss_lst, '-bo')
    plt.plot(valid_loss_lst, '-go')
    plt.xlabel('epoch')
    plt.ylabel('loss')
    plt.legend(['Training', 'Validation'])
    fig.suptitle('Loss')

    fig = plt.figure()
    plt.plot(train_acc_lst, '-bo')
    plt.plot(valid_acc_lst, '-go')
    plt.xlabel('epoch')
    plt.ylabel('accuracy')
    plt.legend(['Training', 'Validation'])
    fig.suptitle('Accuracy')

    plt.show()

def save_checkpoint(dir_path, epoch, model, optim, loss):
    if not os.path.isdir(dir_path):
        os.mkdir(dir_path)

    path = f"{dir_path}/checkpoint_{epoch}.pt"

    torch.save({'epoch': epoch,
                'model_state_dict': model.state_dict(),
                'optimizer_state_dict': optim.state_dict(),
                'loss': loss
                },
               path)

def load_checkpoint(path, model, optim):
    checkpoint = torch.load(path)
    model.load_state_dict(checkpoint['model_state_dict'])
    optim.load_state_dict(checkpoint['optimizer_state_dict'])
    epoch = checkpoint['epoch']
    loss = checkpoint['loss']
    return epoch, loss


import xml.etree.ElementTree as ET
from capas_proyecto.acceso_a_datos.comprobar_long_dict import contar_canciones_xml

def crear_dicc_nombre_ruta(RUTA_XML):
    try:
        arbol = ET.parse(RUTA_XML)
    except FileNotFoundError:
        exit("El nombre del archivo XML no es correcto.")
    except ET.ParseError:
        exit("El archivo XML está mal formado.")
    else:
        raiz = arbol.getroot()
        diccionario = {}
        for canciones in raiz:
            for cancion in canciones:
                nombre_y_ruta = {}
                nombre_y_ruta[cancion.find('name').text] = cancion.find("path").text
                diccionario.update(nombre_y_ruta)
        num_canciones = contar_canciones_xml(raiz)
        assert len(diccionario) == num_canciones
        return diccionario, num_canciones


from apple import Apple
from board import Board
from snake import Snake
from algorthim import DFS
import pygame
from constants import GAME,SNAKE,APPLE,COLOR,SCALE
import time
import random
from a import *

#GAMELOOP
class Game:

    def __init__(self, display):
        self.display = display #instance
        self.score = 0

    def loop(self):
        #TIME
        clock = pygame.time.Clock()

        #INSTANCE
        board = Board(self.display)
        apple = Apple(self.display)
        snake = Snake(self.display)

        #ACTUAL GAMELOOP
        while True:
            path = astar(board.returnBoard(), snake.snake_pos(),apple.apple_Position())
            print(path)
            for i in range(1,len(path)):

                #if next move is not orthogonal
                if snake.changeDirectionTo(snake.CoordinateToDirection(path[i])):
                    path = astar(board.returnBoard(), snake.snake_pos(),apple.apple_Position())

                for event in pygame.event.get():
                    if event.type == pygame.QUIT:
                        print("exit pressed!")
                        return 0

                # DRAW BACKGROUND
                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_UP:
                        snake.changeDirectionTo('UP')
                    elif event.key == pygame.K_DOWN:
                        snake.changeDirectionTo('DOWN')
                    elif event.key == pygame.K_LEFT:
                        snake.changeDirectionTo('LEFT')
                    elif event.key == pygame.K_RIGHT:
                        snake.changeDirectionTo('RIGHT')

                # MOVE SNAKE and record position of SNAKE
                snake.move()

                # Collision Detection
                if snake.collision():
                    return 0

                #eating
                if snake.ate(apple.apple_Position()):
                    self.score += 1
                    pygame.display.set_caption(GAME['CAPTION']+ str(self.score))
                    apple.randomize()

                #DRAW
                board.drawBoard(self.display,snake.snake_body(),apple.apple_Position())
                apple.draw()
                snake.draw_body()

                #DRAW UPDATE
                pygame.display.update()
                clock.tick(GAME['FPS'])

def main():
    display = pygame.display.set_mode((GAME['WIDTH'],GAME['HEIGHT']))
    pygame.display.set_caption(GAME['CAPTION'])

    game = Game(display)
    value = game.loop()

    #LINUX EXIT COMMANDS
    if value != 0:
        print("Game ended wrong: ", value)
        exit(1)

if __name__ == '__main__':
	main()


import os
import time
import cv2
import itertools
import numpy as np
from tqdm import tqdm

import matplotlib.pyplot as plt

from tensorflow.keras.optimizers import Adam
import tensorflow.keras.applications.inception_v3 as inception_v3
import tensorflow.keras.applications.inception_resnet_v2 as inception_resnet_v2
import tensorflow.keras.applications.densenet as densenet
import tensorflow.keras.applications.mobilenet_v2 as mobilenet_v2
import tensorflow.keras.applications.mobilenet as mobilenet
import tensorflow.keras.applications.resnet50 as resnet50
import tensorflow.keras.applications.vgg16 as vgg16
import tensorflow.keras.applications.vgg19 as vgg19
import tensorflow.keras.applications.xception as xception

from tensorflow.keras.models import Model
from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input
from tensorflow.keras.layers import Lambda

from tensorflow.keras.constraints import NonNeg

from tensorflow.keras.layers import Activation

from tensorflow.keras import backend as K
from tensorflow.keras.preprocessing import image

def loadimgs(path, shape_img=(595, 842)):
    '''
    path => Path of train directory or test directory
    '''

    X = []
    y = []

    for cartorio in os.listdir(path):
        print("Carregando Cartorio: " + cartorio)
        cartorio_path = os.path.join(path, cartorio)
        for filename in os.listdir(cartorio_path):
            image_path = os.path.join(cartorio_path, filename)
            img = image.load_img(image_path, target_size=shape_img)
            img = image.img_to_array(img)
            img = img[:int(img.shape[0] * 1/3.), :, :]
            try:
                X.append(img)
                y.append(cartorio)

            except Exception as e:
                print(e)

    y = np.vstack(y)
    X = np.stack(X)

    return X, y

def gaussian(x):
    return K.exp(-K.pow(x, 2))

def get_siamese_model(name=None, input_shape=(224, 224, 3),
                      embedding_vec_size=512, not_freeze_last=2):
    """
        Model architecture
    """

    if name == "InceptionV3":

        base_model = inception_v3.InceptionV3(

            weights='imagenet', include_top=False)

        model_preprocess_input = inception_v3.preprocess_input

    if name == "InceptionResNetV2":

        base_model = inception_resnet_v2.InceptionResNetV2(

            weights='imagenet', include_top=False)

        model_preprocess_input = inception_resnet_v2.preprocess_input

    if name == "DenseNet121":

        base_model = densenet.DenseNet121(

            weights='imagenet', include_top=False)

        model_preprocess_input = densenet.preprocess_input

    if name == "DenseNet169":

        base_model = densenet.DenseNet169(

            weights='imagenet', include_top=False)

        model_preprocess_input = densenet.preprocess_input

    if name == "DenseNet201":

        base_model = densenet.DenseNet201(

            weights='imagenet', include_top=False)

        model_preprocess_input = densenet.preprocess_input

    if name == "MobileNetV2":

        base_model = mobilenet_v2.MobileNetV2(

            weights='imagenet', include_top=False)

        model_preprocess_input = mobilenet_v2.preprocess_input

    if name == "MobileNet":

        base_model = mobilenet.MobileNet(

            weights='imagenet', include_top=False)

        model_preprocess_input = mobilenet.preprocess_input

    if name == "ResNet50":

        base_model = resnet50.ResNet50(

            weights='imagenet', include_top=False)

        model_preprocess_input = resnet50.preprocess_input

    if name == "VGG16":

        base_model = vgg16.VGG16(

            weights='imagenet', include_top=False)

        model_preprocess_input = vgg16.preprocess_input

    if name == "VGG19":

        base_model = vgg19.VGG19(

            weights='imagenet', include_top=False)

        model_preprocess_input = vgg19.preprocess_input

    if name == "Xception":

        base_model = xception.Xception(

            weights='imagenet', include_top=False)

        model_preprocess_input = xception.preprocess_input

    # Verifica se existe base_model

    if 'base_model' not in locals():

        return ["InceptionV3", "InceptionResNetV2",

                "DenseNet121", "DenseNet169", "DenseNet201",

                "MobileNetV2", "MobileNet",

                "ResNet50",

                "VGG16", "VGG19",

                "Xception"

                ]

    # desativando treinamento

    for layer in base_model.layers[:-not_freeze_last]:

        layer.trainable = False

    x = base_model.layers[-1].output

    x = GlobalAveragePooling2D()(x)

    x = Dense(

        embedding_vec_size,

        activation='linear',  # sigmoid? relu?

        name='embedding',

        use_bias=False

    )(x)

    model = Model(

        inputs=base_model.input,

        outputs=x

    )

    left_input = Input(input_shape)

    right_input = Input(input_shape)

    # Generate the encodings (feature vectors) for the two images

    encoded_l = model(left_input)

    encoded_r = model(right_input)

    # Add a customized layer to compute the absolute difference between the encodings

    L1_layer = Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))

    L1_distance = L1_layer([encoded_l, encoded_r])

    # Add a dense layer with a sigmoid unit to generate the similarity score

    prediction = Dense(

        1,

        activation=Activation(gaussian),

        use_bias=False,

        kernel_constraint=NonNeg()

    )(L1_distance)

    # Connect the inputs with the outputs

    siamese_net = Model(

        inputs=[left_input, right_input],

        outputs=prediction

    )

    return {

        "model": siamese_net,

        "preprocess_input": model_preprocess_input

    }

# In[5]:

train_folder = "dataset/train/"

val_folder = 'dataset/test/'

save_path = 'model_data/'

# In[6]:

X, y = loadimgs(train_folder)

# In[7]:

Xval, yval = loadimgs(val_folder)

# In[8]:

def show_img(img):

    plt.figure(figsize=(20, 7))

    plt.imshow(img/255, aspect='auto', interpolation='nearest')

# In[9]:

show_img(X[10])

# In[10]:

model_name = "MobileNetV2"

# In[11]:

model_dict = get_siamese_model(model_name, X[0].shape)

model = model_dict["model"]

preprocess_input = model_dict["preprocess_input"]

# In[12]:

model.summary()

# In[13]:

X = preprocess_input(X)

# In[14]:

X.shape

# In[15]:

def get_index(list1, list2):

    comb = list(

        itertools.product(

            enumerate(list1),

            enumerate(list2)

        )

    )

    y = np.array([int(c[0][1][0] == c[1][1][0]) for c in comb])

    idx_left = np.array([c[0][0] for c in comb])

    idx_right = np.array([c[1][0] for c in comb])

    return y, idx_left, idx_right

# In[16]:

def get_batch(X, y, batch_size, proportion=0.5):

    n_examples, width, height, depth = X.shape

    y_, idx_left, idx_right = get_index(y, y)

    idx_one = np.random.choice(

        np.where(y_ == 1)[0],

        int(batch_size * proportion)

    ).tolist()

    idx_zero = np.random.choice(

        np.where(y_ == 0)[0],

        int(batch_size * (1-proportion))

    ).tolist()

    sel_idx = idx_one + idx_zero

    np.random.shuffle(sel_idx)

    y_batch = y_[sel_idx]

    X_batch_l = X[idx_left[sel_idx]]

    X_batch_r = X[idx_right[sel_idx]]

    return [X_batch_l, X_batch_r], y_batch

# In[17]:

optimizer = Adam(lr=0.001)

model.compile(loss="binary_crossentropy", optimizer=optimizer)

# In[18]:

batch_size = 64

n_epochs = 20

proportion = 0.3

# In[19]:

# train model on each dataset

for epoch in tqdm(range(n_epochs)):

    X_train, y_train = get_batch(X, y, batch_size, proportion)

    model.fit(X_train, y_train, batch_size=batch_size, epochs=5)

# In[ ]:

# In[20]:

model.save(model_name + "_siamese.h5")

# In[26]:

model_embedding = model.layers[2]

# In[21]:

model.predict([Xval[0:4], Xval[15:19]])

# In[22]:

show_img(Xval[3])

# In[23]:

show_img(Xval[18])

# In[24]:

model.predict([Xval[0:4], Xval[0:4]])

# In[32]:

model.summary()

# In[36]:

model2 = Model(inputs=model.input, outputs=model.layers[-1].output)

# In[39]:

model2.predict([Xval[0:4], Xval[0:4]]) + 0.5


import numpy as np
from sklearn import metrics
import matplotlib.pyplot as plt
from kmeans import *
from minibatchkmeans import *
from gmm import *
from Dense_AutoEncoder import *
from CNN_AutoEncoder_TSNE import *

# load the data
data1 = np.load(r'kmnist-train-imgs.npz')
data2 = np.load(r'kmnist-train-labels.npz')
train_imgs = data1['arr_0']
train_labels = data2['arr_0']

NMI = []
ARI = []
n_clusters=10

#### Clustering with unprocessed data ####
train_imgs1=train_imgs.reshape(60000,28*28)

# kmeans
pred1 = kmeans(n_clusters,train_imgs1)
NMI1 = metrics.normalized_mutual_info_score(pred1, train_labels)
ARI1 = metrics.adjusted_rand_score(pred1, train_labels)
NMI.append(NMI1)
ARI.append(ARI1)

# gmm
pred2 = gmm(train_imgs1,n_clusters,1)
NMI2 = metrics.normalized_mutual_info_score(pred2, train_labels)
ARI2 = metrics.adjusted_rand_score(pred2, train_labels)
NMI.append(NMI2)
ARI.append(ARI2)

# minibatchkmeans
pred3 = minibatchkmeans(n_clusters,train_imgs1)
NMI3 = metrics.normalized_mutual_info_score(pred3, train_labels)
ARI3 = metrics.adjusted_rand_score(pred3, train_labels)
NMI.append(NMI3)
ARI.append(ARI3)

#### Clustering with data after dimensionality reduction using Dense Autoencoder ####
train_imgs2 = dense_AE(train_imgs,2)

# kmeans
pred4 = kmeans(n_clusters,train_imgs2)
NMI4 = metrics.normalized_mutual_info_score(pred4, train_labels)
ARI4 = metrics.adjusted_rand_score(pred4, train_labels)
NMI.append(NMI4)
ARI.append(ARI4)

# gmm
pred5 = gmm(train_imgs2,n_clusters,30)
NMI5 = metrics.normalized_mutual_info_score(pred5, train_labels)
ARI5 = metrics.adjusted_rand_score(pred5, train_labels)
NMI.append(NMI5)
ARI.append(ARI5)

# minibatchkmeans
pred6 = minibatchkmeans(n_clusters,train_imgs2)
NMI6 = metrics.normalized_mutual_info_score(pred6, train_labels)
ARI6 = metrics.adjusted_rand_score(pred6, train_labels)
NMI.append(NMI6)
ARI.append(ARI6)

#### Clustering with data after dimensionality reduction using Convolutional Autoencoder and T-sne ####
train_imgs3 = cnn_AE_Tsne(train_imgs)

# kmeans
pred7 = kmeans(n_clusters,train_imgs3)
NMI7 = metrics.normalized_mutual_info_score(pred7, train_labels)
ARI7 = metrics.adjusted_rand_score(pred7, train_labels)
NMI.append(NMI7)
ARI.append(ARI7)

# gmm
pred8 = gmm(train_imgs3,n_clusters,2)
NMI8 = metrics.normalized_mutual_info_score(pred8, train_labels)
ARI8 = metrics.adjusted_rand_score(pred8, train_labels)
NMI.append(NMI8)
ARI.append(ARI8)

# minibatchkmeans
pred9 = minibatchkmeans(n_clusters,train_imgs3)
NMI9 = metrics.normalized_mutual_info_score(pred9, train_labels)
ARI9 = metrics.adjusted_rand_score(pred9, train_labels)
NMI.append(NMI9)
ARI.append(ARI9)

#### Results Visualization ####

NMI_1=[round(x1,3) for x1 in NMI]
ARI_1=[round(x2,3) for x2 in ARI]

size = 9
x = np.arange(size)
total_width, n = 0.8, 3
width = total_width / n
x = x - (total_width - width) / 2

plt.figure(dpi=300,figsize=(13,6))
p1=plt.bar(x, NMI_1, width=width, label="NMI")
plt.bar_label(p1, label_type='edge',fontsize=8)
p2=plt.bar(x + width, ARI_1,  width=width, label="ARI")
plt.bar_label(p2, label_type='edge',fontsize=8)
x_labels = ['kmeans', 'GMM', 'Mkmeans', 'pre1_kmeans', 'pre1_GMM','pre1_Mkmeans', 'pre2_kmeans', 'pre2_GMM','pre2_Mkmeans']
plt.xticks(x, x_labels)
plt.title('NMI and ARI in different algorithms')
plt.legend()
plt.show()


from django.core.paginator import Paginator, EmptyPage
from django.core.serializers import serialize
from django.shortcuts import render, redirect, reverse, get_object_or_404
from django.views import View
from django.contrib.auth.mixins import LoginRequiredMixin
from .forms import CreateOfferForm
from companies.models import Companies, Countries
from .models import OffersImages, Offers
from django.views.generic import DeleteView
from users.models import User
from django.http import Http404, JsonResponse, HttpResponse, HttpResponseRedirect
from funcy import omit
from django.dispatch import receiver
import os
from django.db.models.signals import post_delete, pre_save
from users.views import get_profile_ph
from users.models import Profile
import json
from users.mixins import AccessForCompletesOnlyMixin, AccessForMembersOnlyMixin
# from users.views import is_allowed

class CreateOfferView(AccessForMembersOnlyMixin, View):
    template_name = 'offers/create_offer.html'

    def get(self, request, **kwargs):

        # try:
        #     is_allowed(request)
        # except PermissionError:
        #     return redirect('set-profile-info')

        context = {
            'form': CreateOfferForm(),
            'profile_ph': get_profile_ph(request),
        }
        return render(request, self.template_name, context)

    def post(self, request, **kwargs):

        # try:
        #     is_allowed(request)
        # except PermissionError:
        #     return redirect('set-profile-info')

        form = CreateOfferForm(request.POST, request.FILES)
        company_id = kwargs['pk']
        images = request.FILES.getlist('image')

        company = Companies.objects.filter(id=company_id, owner=request.user).first()
        if not company:
            raise Http404

        if form.is_valid():

            obj = form.save(commit=False)
            obj.company = company
            obj.save()
            for i in images:
                OffersImages(offer=obj, image=i).save()

            nexty = request.POST.get('next')
            if nexty:
                try:
                    return HttpResponseRedirect(nexty)
                except:
                    raise Http404
            else:
                return redirect('offers:my-offers')

        context = {
            'form': form,
            'profile_ph': get_profile_ph(request),
        }
        return render(request, self.template_name, context)

class DeleteOfferView(AccessForMembersOnlyMixin, View):
    def post(self, request, **kwargs):
        obj = get_object_or_404(Offers, id=request.POST.get('id'))

        if obj.company.owner != self.request.user:
            raise Http404

        obj.delete()
        data = {'deleted': True, 'success': True}
        return JsonResponse(data)

class UpdateOfferView(AccessForMembersOnlyMixin, View):
    template_name = 'offers/update_offer.html'

    def get(self, request, **kwargs):

        # try:
        #     is_allowed(request)
        # except PermissionError:
        #     return redirect('set-profile-info')

        obj = get_object_or_404(Offers, id=kwargs['pk'])

        if obj.company.owner != request.user:
            raise Http404

        context = {
            'form': CreateOfferForm(initial={
                'type': obj.type,
                'coupon_price': obj.coupon_price,
                'currency': obj.currency,
                'retail_price': obj.retail_price,
                'title': obj.title,
                'amount_min': obj.amount_min,
                'amount_max': obj.amount_max
            }),
            'images': OffersImages.objects.filter(offer=obj).all(),
            'profile_ph': get_profile_ph(request),
        }
        return render(request, self.template_name, context)

    def post(self, request, **kwargs):

        # try:
        #     is_allowed(request)
        # except PermissionError:
        #     return redirect('set-profile-info')

        instance = get_object_or_404(Offers, id=kwargs['pk'])

        if instance.company.owner != request.user:
            raise Http404

        form = CreateOfferForm(request.POST, request.FILES, instance=instance)
        images = request.FILES.getlist('image')

        if form.is_valid():
            obj = form.save()
            for i in images:
                OffersImages(offer=obj, image=i).save()

            nexty = request.POST.get('next')
            if nexty:
                try:
                    return HttpResponseRedirect(nexty)
                except:
                    raise Http404
            else:
                return redirect('offers:my-offers')

        context = {
            'form': form,
            'profile_ph': get_profile_ph(request),
        }
        return render(request, self.template_name, context)

class OfferImageDeleteView(AccessForMembersOnlyMixin, View):
    def get(self, request, **kwargs):
        post = get_object_or_404(OffersImages, id=request.GET.get('id'))
        if post.offer.company.owner == request.user:
            post.delete()
            return JsonResponse({'success': True})
        else:
            return Http404

@receiver(post_delete, sender=OffersImages)
def post_save_image(sender, instance, *args, **kwargs):
    """ Clean Old Image file """
    try:
        instance.image.delete(save=False)
    except:
        pass

class MyOffersPageView(AccessForMembersOnlyMixin, View):
    template_name = 'offers/my_offers.html'

    def get(self, request, **kwargs):

        # try:
        #     is_allowed(request)
        # except PermissionError:
        #     return redirect('set-profile-info')

        objects = Companies.objects.filter(owner=request.user).prefetch_related('offers_set',
                                                                                'owner__profile_set').all()
        context = {
            'objects': objects,
            'profile_ph': get_profile_ph(request),
        }
        return render(request, self.template_name, context)

class SendImagesView(AccessForMembersOnlyMixin, View):
    def get(self, request, **kwargs):
        images = OffersImages.objects.filter(offer=request.GET.get('id')).all()
        res = []
        for image in images:
            res.append(image.image.url)
        return JsonResponse({'images': res})

class CatalogPageView(View):
    catalog_template = 'offers/catalog.html'
    catalog_template_hidden = 'offers/catalog-hidden.html'

    def get_template(self):
        if self.request.user.is_authenticated:
            if self.request.user.role >= User.INVITED:
                return self.catalog_template
        return self.catalog_template_hidden

    def get(self, request, **kwargs):
        offers = Offers.objects.prefetch_related('offersimages_set', 'company__owner__profile_set').order_by(
            '-id').all()
        currencies = Offers.objects.values('currency').distinct()
        niches = Companies.objects.values('niche').distinct()

        # niche = request.GET.get('niche')
        # if niche:
        #     initial_offers = initial_offers.filter(company__niche=niche)
        #
        # amount_min = request.GET.get('amount_min')
        # amount_max = request.GET.get('amount_max')
        # if amount_min and amount_max:
        #     initial_offers = initial_offers.filter(amount_min__lte=amount_max, amount_max__gte=amount_min)
        # elif amount_min:
        #     initial_offers = initial_offers.filter(amount_max__gte=amount_min)
        # elif amount_max:
        #     initial_offers = initial_offers.filter(amount_min__lte=amount_max)
        #
        # currency = request.GET.get('currency')
        # if currency:
        #     initial_offers = initial_offers.filter(currency=currency)
        #
        # retail_price_min = request.GET.get('retail_price_min')
        # retail_price_max = request.GET.get('retail_price_max')
        # if retail_price_min and retail_price_max:
        #     initial_offers = initial_offers.filter(retail_price__gte=retail_price_min,
        #                                            retail_price__lte=retail_price_max)
        # elif retail_price_min:
        #     initial_offers = initial_offers.filter(retail_price__gte=retail_price_min)
        # elif retail_price_max:
        #     initial_offers = initial_offers.filter(retail_price__lte=retail_price_max)
        #
        # coupon_price_min = request.GET.get('coupon_price_min')
        # coupon_price_max = request.GET.get('retail_price_max')
        # if coupon_price_min and coupon_price_max:
        #     initial_offers = initial_offers.filter(retail_price__gte=coupon_price_min,
        #                                            retail_price__lte=coupon_price_max)
        # elif coupon_price_min:
        #     initial_offers = initial_offers.filter(retail_price__gte=coupon_price_min)
        # elif coupon_price_max:
        #     initial_offers = initial_offers.filter(retail_price__lte=coupon_price_max)
        #
        # country = request.GET.get('country')
        # if country:
        #     initial_offers = initial_offers.filter(company__country_of_res=country)
        #
        # phone_num_show = request.GET.get('phone_num_show')
        # #phone_num_show = 1
        # if phone_num_show:
        #     suitable_users = Profile.objects.filter(phone_num_show=True).values_list('user_id', flat=True)
        #     initial_offers = initial_offers.filter(company__owner__id__in=suitable_users)
        #
        # print(initial_offers)
        #
        # p = Paginator(initial_offers, 2)
        #
        # page = request.GET.get('page')
        #
        # try:
        #     res = p.page(page)
        # except EmptyPage:
        #     raise Http404
        #
        # serialized_data = serialize("json", res, use_natural_foreign_keys=True, use_natural_primary_keys=True)
        # print(serialized_data)
        #
        # if request.headers.get('X-Requested-With') == 'XMLHttpRequest':
        #     return JsonResponse(serialized_data, safe=False)
        try:
            profile_ph = get_profile_ph(request)
        except TypeError:
            profile_ph = None

        context = {
            'offers': offers,
            'profile_ph': profile_ph,
            'currencies': currencies,
            'niches': niches,
        }
        return render(request, self.get_template(), context)

@receiver(post_delete, sender=OffersImages)
def post_save_image(sender, instance, *args, **kwargs):
    """ Clean Old Image file """
    try:
        img = OffersImages.objects.filter(offer=instance.offer).first()
        obj = Offers.objects.get(id=instance.offer.id)
        if not img:
            obj.has_photo = False
            obj.save()
    except:
        pass

@receiver(pre_save, sender=OffersImages)
def pre_save_image(sender, instance, *args, **kwargs):
    """ instance old image file will delete from os """
    try:
        obj = Offers.objects.get(id=instance.offer.id)
        if not obj.has_photo:
            obj.has_photo = True
            obj.save()
    except:
        pass


from datetime import datetime, timezone
import json
import logging
from math import ceil
from slugify import slugify

from flask import Response, request
from flask_camp import current_api, allow
from sqlalchemy.sql.functions import func
from werkzeug.exceptions import BadRequest

from c2corg_api.search import DocumentSearch, DocumentLocaleSearch
from c2corg_api.models import USERPROFILE_TYPE, ROUTE_TYPE, models as document_types

log = logging.getLogger(__name__)

# Search engines accept not more than 50000 urls per sitemap,
# and the sitemap files may not exceed 10 MB. With 50000 urls the sitemaps
# are not bigger than 9MB, but to be safe we are using 45000 urls per sitemap.
# see http://www.sitemaps.org/protocol.html
PAGES_PER_SITEMAP = 45000

class _Sitemaps:
    @staticmethod
    def get_locales_per_type():
        return (
            current_api.database.session.query(DocumentSearch.document_type, func.count().label("count"))
            .join(DocumentLocaleSearch, DocumentSearch.id == DocumentLocaleSearch.id)
            .filter(DocumentSearch.document_type != USERPROFILE_TYPE)
            .group_by(DocumentSearch.document_type)
            .all()
        )

class _Sitemap:
    @staticmethod
    def get_locales(document_type, page):
        fields = [
            DocumentSearch.id,
            DocumentLocaleSearch.lang,
            DocumentLocaleSearch.title_prefix,
            DocumentLocaleSearch.title,
            DocumentSearch.timestamp,
        ]

        query = (
            current_api.database.session.query(*fields)
            .select_from(DocumentLocaleSearch)
            .join(DocumentSearch, DocumentSearch.id == DocumentLocaleSearch.id)
            .filter(DocumentSearch.document_type == document_type)
            .order_by(DocumentLocaleSearch.id, DocumentLocaleSearch.lang)
            .limit(PAGES_PER_SITEMAP)
            .offset(PAGES_PER_SITEMAP * page)
        )

        return query.all()

class SitemapsRest(_Sitemaps):
    rule = "/sitemaps"

    @allow("anonymous")
    def get(self):

        """Returns the information needed to generate a sitemap index file.
        See: http://www.sitemaps.org/protocol.html

        The response consists of a list of URLs to request the information
        needed to generate the sitemap linked from the sitemap index.

        E.g.

            {
                "sitemaps": [
                    "/sitemaps/w/0",
                    "/sitemaps/a/0",
                    "/sitemaps/i/0",
                    "/sitemaps/i/1",
                    "/sitemaps/i/2",
                    "/sitemaps/i/3",
                    "/sitemaps/i/4",
                    "/sitemaps/i/5",
                    ...
                ]
            }
        """

        document_locales_per_type = self.get_locales_per_type()

        sitemaps = []
        for doc_type, doc_count in document_locales_per_type:
            num_sitemaps = ceil(doc_count / PAGES_PER_SITEMAP)
            sitemaps += [
                {"url": f"/sitemaps/{doc_type}/{i}", "doc_type": doc_type, "i": i} for i in range(0, num_sitemaps)
            ]

        result = Response(response=json.dumps({"sitemaps": sitemaps}), content_type="application/json")
        result.add_etag()  # TODO : compute it only one time per day
        result.make_conditional(request)

        return result

class SitemapRest(_Sitemap):
    rule = "/sitemaps/<string:document_type>/<int:page>"

    @allow("anonymous")
    def get(self, document_type, page):
        """Returns the information needed to generate a sitemap for a given type and sitemap page number."""

        if document_type not in document_types:
            raise BadRequest("Invalid document type")

        document_locales = self.get_locales(document_type, page)

        # include `title_prefix` for routes
        is_route = document_type == ROUTE_TYPE
        data = {"pages": [self.format_page(is_route, *locale) for locale in document_locales]}

        result = Response(response=json.dumps(data), content_type="application/json")
        result.add_etag()  # TODO : compute it only one time per day
        result.make_conditional(request)

        return result

    @staticmethod
    def format_page(is_route, doc_id, lang, title, title_prefix, last_updated):
        page = {"document_id": doc_id, "lang": lang, "title": title, "lastmod": last_updated.isoformat()}

        if is_route:
            page["title_prefix"] = title_prefix

        return page

class SitemapsXml(_Sitemaps):
    rule = "/sitemaps.xml"

    @allow("anonymous")
    def get(self):
        """Returns a sitemap index file.
        See: http://www.sitemaps.org/protocol.html

        The response consists of a list of URLs of sitemaps.

        <?xml version="1.0" encoding="UTF-8"?>
            <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
                <sitemap>
                    <loc>https://api.camptocamp.org/sitemaps.xml/w/0.xml</loc>
                    <lastmod>2019-02-11T18:01:49.193770+00:00</lastmod>
                </sitemap>
                <sitemap>
                    <loc>https://api.camptocamp.org/sitemaps.xml/a/0.xml</loc>
                    <lastmod>2019-02-11T18:01:49.193770+00:00</lastmod>
                </sitemap>
                <sitemap>
                    <loc>https://api.camptocamp.org/sitemaps.xml/i/0.xml</loc>
                    <lastmod>2019-02-11T18:01:49.193770+00:00</lastmod>
                </sitemap>
                <sitemap>
                    <loc>https://api.camptocamp.org/sitemaps.xml/i/1.xml</loc>
                    <lastmod>2019-02-11T18:01:49.193770+00:00</lastmod>
                </sitemap>
            </sitemap>
        """

        document_locales_per_type = self.get_locales_per_type()

        sitemaps = []

        now = datetime.utcnow().replace(tzinfo=timezone.utc)
        lastmod = now.isoformat()

        template = """<sitemap>
        <loc>https://api.camptocamp.org/sitemaps.xml/{doc_type}/{i}.xml</loc>
        <lastmod>{lastmod}</lastmod>
        </sitemap>"""

        for doc_type, count in document_locales_per_type:
            num_sitemaps = ceil(count / PAGES_PER_SITEMAP)
            sitemaps_for_type = [
                template.format(doc_type=doc_type, i=i, lastmod=lastmod) for i in range(0, num_sitemaps)
            ]
            sitemaps.extend(sitemaps_for_type)

        body = """<?xml version="1.0" encoding="UTF-8"?>
        <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
        {}
        </sitemapindex>""".format(
            "\n".join(sitemaps)
        )

        result = Response(response=body, content_type="text/xml")
        result.add_etag()  # TODO : compute it only one time per day
        result.make_conditional(request)

        return result

class SitemapXml(_Sitemap):
    rule = "/sitemaps.xml/<string:document_type>/<int:page>.xml"

    @allow("anonymous")
    def get(self, document_type, page):
        """Returns a sitemap file for a given type and sitemap page number."""

        if document_type not in document_types:
            raise BadRequest("Invalid document type")

        document_locales = self.get_locales(document_type, page)

        body = """<?xml version="1.0" encoding="UTF-8"?>
        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
        {}
        </urlset>""".format(
            "\n".join([self.format_page(document_type, *locale) for locale in document_locales])
        )

        result = Response(response=body, content_type="text/xml")
        result.add_etag()  # TODO : compute it only one time per day
        result.make_conditional(request)

        return result

    @staticmethod
    def format_page(document_type, doc_id, lang, title_prefix, title, last_updated):

        page = {
            "document_id": doc_id,
            "lang": lang,
            "lastmod": last_updated.isoformat(),
            "document_type": document_type,
        }

        if title_prefix:
            page["title"] = slugify(f"{title_prefix} {title}")
        else:
            page["title"] = slugify(title)

        return """<url>
    <loc>https://www.camptocamp.org/{document_type}/{document_id}/{lang}/{title}</loc>
    <lastmod>{lastmod}</lastmod>
    <changefreq>weekly</changefreq>
    </url>""".format(
            **page
        )


from direct.gui.OnscreenImage import OnscreenImage
from pandac.PandaModules import TransparencyAttrib
from direct.gui.OnscreenText import OnscreenText
from direct.showbase.DirectObject import DirectObject
from pandac.PandaModules import TextNode

from gui.GUIOrder import GUIOrder
from event.InventoryEvent import AmmoChangeEvent, SelectedItemChangeEvent
import Globals

class HUDBottomRight(DirectObject):

    def __init__(self):
        self.node = base.a2dBottomRight.attachNewNode('hudBottomRight')#GUIOrder.ORDER[GUIOrder.HUD])
        self.node.setBin('fixed', GUIOrder.ORDER[GUIOrder.HUD])

        self.ammoIcon = OnscreenImage(image = 'Assets/Images/HUD/HUDBottomRight.png', scale = 512.0 / 1024, pos = (-0.5, 0, 0.5))
        self.ammoIcon.setTransparency(TransparencyAttrib.MAlpha)
        self.ammoIcon.reparentTo(self.node)

        self.ammoTextClip = OnscreenText(text = '30', pos = (-0.35, 0.09), scale = 0.12, fg = (1, 1, 1, 1), shadow = (0, 0, 0, 1), mayChange = True, align=TextNode.ARight, font = Globals.FONT_SAF)
        self.ammoTextClip.reparentTo(self.node)
        self.ammoTextClip.setBin('fixed', GUIOrder.ORDER[GUIOrder.HUD])

        self.ammoTextLeft = OnscreenText(text = '90', pos = (-0.23, 0.05), scale = 0.07, fg = (1, 1, 1, 1), shadow = (0, 0, 0, 1), mayChange = True, align=TextNode.ARight, font = Globals.FONT_SAF)
        self.ammoTextLeft.reparentTo(self.node)
        self.ammoTextLeft.setBin('fixed', GUIOrder.ORDER[GUIOrder.HUD])

        self.accept(AmmoChangeEvent.EventName, self.OnAmmoChangeEvent)
        self.accept(SelectedItemChangeEvent.EventName, self.OnSelectedItemChangeEvent)

    def OnAmmoChangeEvent(self, event):
        item = event.GetItem()
        if(item and item.GetCurrentClipAmmo() == ''):
            self.ChangeAmmoText('', '')
        else:
            self.ChangeAmmoText(str(item.GetCurrentClipAmmo()), str(item.GetTotalRemainingAmmo()))

    def ChangeAmmoText(self, clip, total):
        self.ammoTextClip.setText(clip)
        self.ammoTextLeft.setText(total)

    def OnSelectedItemChangeEvent(self, event):
        if(event.GetItemStack() and event.GetItemStack().GetItem()):
            self.OnAmmoChangeEvent(AmmoChangeEvent(None, event.GetItemStack().GetItem()))
        else:
            self.ChangeAmmoText('', '')

    def Destroy(self):
        self.ignoreAll()
        self.ammoIcon.removeNode()
        self.ammoTextClip.removeNode()
        self.ammoTextLeft.removeNode()
        self.node.removeNode()


from collections import deque
from sys import stdin

def bfs(x,n,visited,graph):

    queue = deque([x])

    visited[x] = 1

    while queue:

        x = queue.popleft()

        for v in graph[x]:

            if visited[v] == 0:

                visited[v] = 1
                queue.append(v)

    return True

n,m = map(int,stdin.readline().split())

graph = [[] for _ in range(n+1)]

for _ in range(m):

    u,v = map(int,stdin.readline().split())

    graph[u].append(v)
    graph[v].append(u)

visited = [0]*(n+1)

count = 0

for i in range(1,n+1):

    if visited[i] == 0:

        count += bfs(i,n,visited,graph)

print(count)


import sqlite3 as sql
import numpy as np
import pandas as pd
import pickle
import os
import joblib
import onnx
import onnxruntime as rt
import torch

filename = "./svm_iris.onnx"

PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))
rf_model_loaded = onnx.load(os.path.join(PROJECT_ROOT, "static/rf_model_init.onnx"))
sess = rt.InferenceSession(os.path.join(PROJECT_ROOT, "static/rf_model_init.onnx"))
input_name = sess.get_inputs()[0].name
label_name = sess.get_outputs()[0].name

# scaler_loaded = onnx.load(os.path.join(PROJECT_ROOT, "static/scaler_init.onnx"))
# onnx.checker.check_model(scaler_loaded)

fixed_acidity = 7.5
volatile_acidity = 0.75
citric_acid = 3.00
residual_sugar = 3.9
chlorides = 0.176
free_sulfur_dioxide = 12.0
total_sulfur_dioxide = 35.0
density = 0.91
ph = 3.9
sulphates = 0.56
alcohol = 9.4

X = np.array(
    [
        fixed_acidity,
        volatile_acidity,
        citric_acid,
        residual_sugar,
        chlorides,
        free_sulfur_dioxide,
        total_sulfur_dioxide,
        density,
        ph,
        sulphates,
        alcohol,
    ]
)
pred_onx = sess.run(None, {input_name: X.astype(np.float32)})[0]
print(pred_onx)

X_scaled = scaler_loaded.transform(X)
quality = rf_model_loaded.predict(X_scaled)[0]

res = "хорошее" if quality == 1 else "плохое"
msg = f"Вино {res}"


import numpy as np
import matplotlib.pyplot as plt

class differential:
    """Solver of differential equations
    using RK4. diffEq should be a function with the
    differential equation that returns acceleration.
    All variables inside diffEq must be global"""
    def __init__(self, diffEq, plot_str, dt=0.01, T=20, x0=0, v0=0):
        if callable(diffEq):
            self.diffEq = diffEq
        else:
            raise TypeError('diffEq must be a callable function')
        self.dt = float(dt)
        self.T = T
        self.n = int(self.T/self.dt) + 1
        self.x = np.zeros(self.n)
        self.v = np.zeros(self.n)
        self.t = np.linspace(0,self.T,self.n)
        self.x[0] = x0
        self.v[0] = v0
        self.plot_str = plot_str

    def solve(self):
        """ method that uses RK4 and diffEq
        to solve the entire motion"""
        for i in range(self.n - 1):
            self.x[i + 1],self.v[i + 1] = self.RK4(self.x[i],self.v[i],self.t[i])
        return self.x, self.v, self.t

    def RK4(self,xStart,vStart,tStart):
        """ Runge-Kutta method of 4th order"""
        a1 = self.diffEq(xStart,vStart,tStart)
        v1 = vStart
        xHalf1 = xStart + v1 * self.dt/2.0
        vHalf1 = vStart + a1 * self.dt/2.0
        a2 = self.diffEq(xHalf1,vHalf1,tStart+self.dt/2.0)
        v2 = vHalf1
        xHalf2 = xStart + v2 * self.dt/2.0
        vHalf2 = vStart + a2 * self.dt/2.0
        a3 = self.diffEq(xHalf2,vHalf2,tStart+self.dt/2.0)
        v3 = vHalf2
        xEnd = xStart + v3 * self.dt
        vEnd = vStart + a3 * self.dt
        a4 = self.diffEq(xEnd,vEnd,tStart + self.dt)
        v4 = vEnd
        aMiddle = 1.0/6.0 * (a1 + 2*a2 + 2*a3 + a4)
        vMiddle = 1.0/6.0 * (v1 + 2*v2 + 2*v3 + v4)
        xEnd = xStart + vMiddle * self.dt
        vEnd = vStart + aMiddle * self.dt
        return xEnd, vEnd

    def plot(self):
        """ plots phase space of motion,
        after solve() method is used"""
        plt.title(self.plot_str)
        plt.plot(self.x,self.v)
        plt.xlabel('position [m]')
        plt.ylabel('velocity [m/s]')
        plt.grid()
        plt.show()

if __name__ == '__main__':
    def diffEq(xNow,vNow,tNow):
        return - k*xNow/m
    oppgave1 = differential(diffEq,r'$m\ddot{x}(t) + kx(t) = 0$',x0=1.0)
    m = 0.500
    k = 1.0
    oppgave1.solve()
    oppgave1.plot()

"""
[Command: python -u /home/simen/github/university/4semester/fys2130/project.py]
[Finished in 2.405s]
"""


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('spirit_user', '0004_auto_20150731_2351'),
    ]

    operations = [
        migrations.AddField(
            model_name='userprofile',
            name='given_likes_count',
            field=models.PositiveIntegerField(default=0, verbose_name='given likes count'),
        ),
        migrations.AddField(
            model_name='userprofile',
            name='received_likes_count',
            field=models.PositiveIntegerField(default=0, verbose_name='received likes count'),
        ),
    ]


import yfinance as yf
import datetime
import pandas as pd

def get_dados(siglas, num_dias = 588, intervalo = '1wk', inicio = '',  fim = ''):

    """
        siglas -> []

        Retorna uma lista de DataFrames
        com os valores de fechamento das siglas passadas
    """

    if inicio == '':
        inicio = (datetime.date.today() - datetime.timedelta(num_dias))
    if fim == '':
        fim = datetime.date.today()

    dados = []
    for sigla in siglas:
        df = yf.download(sigla, start = inicio, end = fim, interval = intervalo)

        df.drop(['Open', 'High', 'Low', 'Adj Close', 'Volume'], axis = 1, inplace = True)
        df = df.transpose()
        df.dropna(axis = 1, inplace = True)
        df.index = [sigla]

        dados.append(df)

    dados = pd.concat(dados)

    return dados


#!/usr/bin/env python
# encoding: utf-8

from rdflib.serializer import Serializer
import configparser
import corpus
import csv
import glob
import json
import rdflib
import sys

CONFIG = configparser.ConfigParser()
CONFIG.read("rc.cfg")

PREAMBLE = """
@base <https://github.com/Coleridge-Initiative/adrf-onto/wiki/Vocabulary> .

@prefix cito:	<http://purl.org/spar/cito/> .
@prefix dct:	<http://purl.org/dc/terms/> .
@prefix foaf:	<http://xmlns.com/foaf/0.1/> .
@prefix rdf:	<http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix xsd:	<http://www.w3.org/2001/XMLSchema#> .
"""

TEMPLATE_DATASET = """
:{}
  rdf:type :Dataset ;
  foaf:page "{}"^^xsd:anyURI ;
  dct:publisher "{}" ;
  dct:title "{}" ;
"""

TEMPLATE_PUBLICATION = """
:{}
  rdf:type :ResearchPublication ;
  foaf:page "{}"^^xsd:anyURI ;
  dct:publisher "{}" ;
  dct:title "{}" ;
  dct:identifier "{}" ;
  :openAccess "{}"^^xsd:anyURI ;
"""

if __name__ == "__main__":
    out_buf = [ PREAMBLE.lstrip() ]

    ## load the datasets
    dataset_path = CONFIG["DEFAULT"]["dataset_path"]

    known_datasets = {}

    with open(dataset_path, "r") as f:
        for elem in json.load(f):
            dat_id = elem["id"]
            id_list = [elem["provider"], elem["title"]]
            known_datasets[dat_id] = corpus.get_hash(id_list, prefix="dataset-")

            if "url" in elem:
                url = elem["url"]
            else:
                url = "http://example.com"

            out_buf.append(
                TEMPLATE_DATASET.format(
                    known_datasets[dat_id],
                    url,
                    elem["provider"],
                    elem["title"]
                    ).strip()
                )

            if "alt_title" in elem:
                for alt_title in elem["alt_title"]:
                    out_buf.append("  dct:alternative \"{}\" ;".format(alt_title))

            out_buf.append(".\n")

    ## load the publications
    for filename in glob.glob("corpus/pub/*.json"):
        with open(filename) as f:
            for elem in json.load(f):
                link_map = elem["datasets"]

                if len(link_map) > 0:
                    id_list = [elem["publisher"], elem["title"]]
                    pub_id = corpus.get_hash(id_list, prefix="publication-")

                    out_buf.append(
                        TEMPLATE_PUBLICATION.format(
                            pub_id,
                            elem["url"],
                            elem["publisher"],
                            elem["title"],
                            elem["doi"],
                            elem["pdf"]
                            ).strip()
                        )

                    dat_list = [ ":{}".format(known_datasets[dat_id]) for dat_id in link_map ]
                    out_buf.append("  cito:citesAsDataSource {} ;".format(", ".join(dat_list)))
                    out_buf.append(".\n")

    ## write the TTL output
    filename = "tmp.ttl"

    with open(filename, "w") as f:
        for text in out_buf:
            f.write(text)
            f.write("\n")

    ## load the TTL output as a graph
    graph = rdflib.Graph()
    graph.parse(filename, format="n3")

    ## transform graph into JSON-LD
    with open("corpus/vocab.json", "r") as f:
        context = json.load(f)

    with open("tmp.jsonld", "wb") as f:
        f.write(graph.serialize(format="json-ld", context=context, indent=2))

    ## read back
    graph = rdflib.Graph()
    graph.parse("tmp.jsonld", format="json-ld")


import tvm
import tvm.relay as relay
import tvm.relay.testing as testing

from graphviz import Digraph
import os
from collage.utils import get_backend_from_backend_pattern_annotation

def _traverse_expr(node, node_dict):
    if node in node_dict:
        return
    if isinstance(node, tvm.ir.op.Op):
        return

    node_dict[node] = len(node_dict)

def get_node_color(node):
    backend_name = get_backend_from_backend_pattern_annotation(node.backend)

    # If this is default (no backend op assignment)
    color = "greenyellow"

    if backend_name == "tensorrt":
        color = "orange"
    elif backend_name[:3] == "tvm":
        color = "greenyellow"
    elif backend_name[:5] == "cudnn":
        color = "yellow"
    elif backend_name[:6] == "cublas":
        color = "grey60"

    return color

def visualize_backend_placement(expr, file_name, expr2node=None):

    dot = Digraph(format='pdf')
    dot.attr(rankdir='BT')

    node_dict = {}
    relay.analysis.post_order_visit(expr, lambda node: _traverse_expr(node, node_dict))

    for node, node_idx in node_dict.items():
        if not isinstance(node, relay.Let):
            node_idx_backend_str = f"[{node_idx}, {node.backend}]"
        else:
            node_idx_backend_str = f"[{node_idx}, NO_BACKEND]"

        # Debug for DP: print node_dfs_order
        if expr2node is not None and hash(node) in expr2node:
            node_dfs_order = expr2node[hash(node)]._topological_order
            node_idx_backend_str = f"[{node_dfs_order}, {node_idx}, {node.backend}]"

        node_color = get_node_color(node)

        if isinstance(node, relay.Function):
            dot.node(str(node_idx), f'Function ({node_idx})', shape='doubleoctagon')
            dot.edge(str(node_dict[node.body]), str(node_idx))

        elif isinstance(node, relay.expr.Var):
            if isinstance(node.type_annotation, tvm.ir.type.TupleType):
                type_info = node.type_annotation.fields
                tensor_info = f'Tensor[TupleType{tuple(type_info)}]'
            elif not hasattr(node.type_annotation, 'shape'):
                tensor_info = f'NoType'
            else:
                type_info = node.type_annotation.shape
                tensor_info = f'Tensor[{tuple(type_info)}, {node.type_annotation.dtype}]'

            dot.node(str(node_idx), \
                     f'{node.name_hint} {node_idx_backend_str}:\n{tensor_info}', \
                     shape='rectangle'
                     )
        elif isinstance(node, relay.expr.GlobalVar):
            dot.node(str(node_idx), \
                     f'{node.name_hint} {node_idx_backend_str}', \
                     shape='rectangle'
                     )

        elif isinstance(node, relay.Constant):
            dot.node(str(node_idx), \
                     f'Constant {node_idx_backend_str}:\nTensor[{tuple(node.data.shape)}, {node.data.dtype}]', \
                     shape='rectangle'
                     )

        elif isinstance(node, relay.expr.Call):
            args = [node_dict[arg] for arg in node.args]
            if isinstance(node.op, tvm.relay.Function):
                dot.node(str(node_idx), f'Call {node_idx_backend_str}(Function({node_dict[node.op.body]}))', shape='ellipse',
                         style='filled', color=node_color)
            else:
                if isinstance(node.op, relay.expr.GlobalVar):
                    dot.node(str(node_idx), f'Call{node_idx_backend_str}(GlobalVar={node.op.name_hint})', shape='ellipse', style='filled', color=node_color)
                elif isinstance(node.op, relay.Var):
                    dot.node(str(node_idx), f'Call {node_idx_backend_str}(Var={node.op.name_hint})', shape='ellipse', style='filled', color=node_color)
                else:
                    dot.node(str(node_idx), f'Call {node_idx_backend_str}(op={node.op.name})', shape='ellipse', style='filled', color=node_color)

            for arg in args:
                dot.edge(str(arg), str(node_idx))
        elif isinstance(node, relay.expr.TupleGetItem):
            dot.node(str(node_idx), f'TupleGetItem {node_idx_backend_str}(idx={node.index})', shape='ellipse', style='filled', color=node_color)
            dot.edge(str(node_dict[node.tuple_value]), str(node_idx))
        elif isinstance(node, relay.expr.Tuple):
            args = [node_dict[field] for field in node.fields]
            dot.node(str(node_idx), f'Tuple {node_idx_backend_str}(fileds=none)', shape='ellipse', style='filled', color=node_color)
            for arg in args:
                dot.edge(str(arg), str(node_idx))
        else:
            raise RuntimeError(f'Unknown node type. node_idx: {node_idx}, node: {type(node)}')

    dot.render(f'{file_name}.gv')
    os.remove(f'{file_name}.gv')


# -*- coding: cp1252 -*-

import io
import os
import sys
import time

import misctools
import stringtools

class Apho:
    def __init__( self ):
        self.thous = [] # list of pair (sentence, author)
        self.aCountSaid = [] # for each sentence, number of said time
        self.aLastSaid = [] # time of last said

        self.embed = None # when using it

    def load( self ):
        """
        Charge les pensees: un fichier avec des pensées, puis sur la derniere ligne l'auteur. séparé par des lignes vides.
        eg:
Je ne pense pas à toute la misère, je pense à la beauté qui reste.
Anne Frank

Fais de ta vie un rêve et d’un rêve une réalité.
Antoine de Saint-Exupery
        """
        """
        TODO a l'occasion: prendre un gros roman puis chercher des phrases assez courte sans prénom et les définir comme des pensee avec nom de l'auteur, livre et année.
        cf D:\books avec des pdfs
        """
        f = io.open(misctools.getThisFilePath()+"datas/pensee.txt","r",encoding='cp1252')
        blob = [] # un bloc de ligne de texte séparé par une ligne vide
        bContinue = 1
        while bContinue:
            line = f.readline()
            if len(line)<1:
                bContinue = 0
            if bContinue and line[-1] == '\n': line = line[:-1]
            if len(line)<1:
                if len(blob)>1:
                    # end of blob
                    citation = " ".join(blob[:-1])
                    auth = blob[-1]
                    self.thous.append( (citation,auth) )
                    self.aCountSaid.append(0)
                    self.aLastSaid.append(0)
                blob = []
            else:
                blob.append(line)
        #~ print("DBG: load: blob: %s" % str(blob))
        #~ print(self.thous)
        print("INF: Apho.load: %d loaded apho(s))" % len(self.thous))

    def getThoughts( self, sentence ):
        """
        find a thoughts not said a lot, relative to sentence.
        return a pair, (thought,author) or None if none
        """
        bVerbose = 1
        #~ bVerbose = 0

        bMatchShort = 0
        bMatchShort = 1

        bUseWordMatching = 1
        #~ bUseWordMatching = 0 # use camembert

        if bUseWordMatching:
            if 0:
                sentence = sentence.replace('.', ' ').replace(',', ' ')
                words = sentence.split()
                words = stringtools.lowerList(words)
            else:
                import usual_words
                words = usual_words.filterSentences(sentence,bVerbose=0)
                words = stringtools.lowerList(words)

            # add also words without '
            i = 0
            while i < len(words):
                if "'" in words[i]:
                    words.extend(words[i].split("'"))
                i += 1
            if bVerbose: print("DBG: getThoughts: words: %s" % words)

            # find radical style
            i = 0
            while i < len(words):
                if len(words[i])<3:
                    del words[i]
                    continue
                # on le fera plus tard pour essayer de matcher sur le mot reel
                #~ if len(words[i])>5:
                    #~ words[i] = words[i][:-3] # travailler => travail
                i += 1

            # remove usual words
            if 0:
                import usual_words
                i = 0
                while i < len(words):
                    if usual_words.isUsualWord(words[i]):
                        del words[i]
                        continue
                    i += 1

            print("match word: %s" % words)
            match = []
            for t in self.thous:
                cit = t[0]
                cit = cit.lower()
                n = 0
                for w in words:
                    #~ print("compare with cit: %s" % cit)
                    if w in cit:
                        if bVerbose or 0: print( "match: '%s' in '%s'" % (w,cit) )
                        #~ n += 1
                        n += len(w) # count more point if word is long!
                    if bMatchShort and len(w)>5:
                        # lemmatisation du pauvre
                        if "er" == w[-2:]:
                            ws = w[:-3]
                        else:
                            ws = w[:-2]
                        if ws in cit:
                            # pb: ecoute => eco can match with recommencer
                            if bVerbose: print( "match short: '%s' in '%s'" % (ws,cit) )
                            n += len(ws)

                match.append(n*30/len(cit))
        else:
            # camembert
            import numpy as np
            sys.path.append("../camembert")
            import sentence_embedding

            # generating for all apho
            if self.embed == None:
                timeBegin = time.time()
                if 0:
                    # 6s
                    listEmb = []
                    for t in self.thous:
                        cit = t[0]
                        v = sentence_embedding.camEmbedList([cit])[0]
                        listEmb.append(v)
                else:
                    # once precomputed: 0.12s
                    list_s = [x[0] for x in self.thous]
                    listEmb = sentence_embedding.precomputeList(list_s,"apho_embed.txt")
                    # le 7ieme chiffre après la virgule est différent quand on sauve dans un fichier
                print("listEmb takes %.2fs" % (time.time()-timeBegin))
                #~ print(listEmb[0])
                #~ print(listEmb[1])
                #~ print(listEmb[2])
                self.embed = listEmb

            e = sentence_embedding.camEmbedList(sentence)
            match = []
            for v in self.embed:
                simi = np.dot(e,v)/0.4 # 0.4 is a good threshold for simi
                 #reglage pour ne pas prendre ceux qui sont trop loin (on a ensuite un filtre >=1)
                #~ simi *= 2.35 # un peu trop sympa, 19 hits: laisse passer 'c'est bien fait peur. tu es un bon petit gars. eh mon petit gnocchi est-ce qu'il a." => Gaïa, c'est l'heure d'aller te coucher.
                simi *= 2.2 # plus limité: 14 hits
                match.append(simi)

        # at this point we have a list of match for each thou (the greater the better)

        print("match: %s" % match)
        #~ [x for _, x in sorted(zip(Y, X))]
        # both are working, but second seems faster, todolater: measures
        #~ index_order = [x for _, x in sorted(zip(match, range(len(match))),reverse=True)]
        index_order = sorted(range(len(match)), key=lambda k: match[k],reverse=True)
        #~ print("index_order: %s" % index_order)
        # etais ce vraiment la peine de les trier, alors qu'on va les parcourir ensuite ?
        less_said_idx = index_order[0]
        for idx in index_order[1:]:
            if match[idx]<1:
                break
            if time.time()-self.aLastSaid[idx]<5*60:
                continue
            if self.aCountSaid[less_said_idx] > self.aCountSaid[idx]:
                less_said_idx = idx
        print("less_said_idx: %d" % less_said_idx )

        if match[less_said_idx] < 1:
            return None

        if self.aCountSaid[less_said_idx] > 0 and 0:
            # decide to say already said or not ?
            return None

        # first sentence of the list can be selected by default
        if time.time()-self.aLastSaid[less_said_idx]<5*60:
            return None

        self.aCountSaid[less_said_idx] += 1
        self.aLastSaid[less_said_idx] = time.time()
        print("match: %.2f" % match[less_said_idx] )
        return self.thous[less_said_idx]

# class Apho - end

apho = Apho()
apho.load()

global_tts = None
def say(txt):
    global global_tts
    if global_tts == None:
        import pyttsx3
        global_tts = pyttsx3.init()
    if 1:
        txt = txt.replace("Gaia", "Gaïa")
    print("INF: say: '%s'" % txt)
    global_tts.say(txt)
    global_tts.runAndWait()

def sayGoogle(txt):
    sys.path.append("../scripts")
    import tts_say
    tts_say.say(txt)

def wordsCallback(words,confidence):
    if confidence<0.6:
        return
    if len(words)<4:
        return
    print("INF: heard: '%s'" % words)
    #~ say(phrase)
    ret = apho.getThoughts(words)
    if ret != None:
        saymethod = say
        if 1:
            saymethod = sayGoogle
        saymethod(ret[0])
        saymethod(ret[1])

def test_loop_asr():

    if 0:
        from pocketsphinx import LiveSpeech, get_model_path
        import os
        model_path = get_model_path()
        print("model_path: %s" % model_path )
        # good model path:
        model_path = "C:\\Python39\\Lib\\site-packages\\speech_recognition\\pocketsphinx-data\\"
        strAnsiLang = "fr-FR"
        for phrase in LiveSpeech(
            hmm=(os.path.join(model_path, strAnsiLang)+"\\acoustic-model\\"),
            lm=os.path.join(model_path, strAnsiLang+'\\language-model.lm.bin'),
            dic=os.path.join(model_path, strAnsiLang+'\\pronounciation-dictionary.dict')
        ):
            phrase = str(phrase)
            wordsCallback( phrase, 0.5)

    else:
        # my own one
        import microphone
        microphone.loopProcess(wordsCallback)

"""
# probleme actuel, les mots banals hits trop:
+ mettre malus sur longueur de la phrase!

INF: heard: 'c'est cool'
less_said_idx: 3
INF: say: 'Le plus difficile, ce n'est pas de sortir de Polytechnique, c'est de sortir de l'ordinaire.'

INF: heard: 'je suis bien d'accord'
less_said_idx: 115
INF: say: 'Je ne comprends pas qu'on achète du vin sans l'avoir goûté au préalable. Il ne viendrait à personne l'idée d'acheter un pantalon sans l'essayer avant. Alors, Dieu me tire-bouchonne, ne refusez pas à votre bouche ce que vous accordez à vos fesses.'

INF: heard: 'un bon petit vin rouge'
less_said_idx: 8
INF: say: 'Le bonheur est une petite chose que l'on grignote, assis par terre, au soleil.'
"""

def strToPrint(s):
    if sys.version_info[0] >= 3:
        return s

    o = ""
    for c in s:
        #~ print( ord(c) )
        if ord(c) <= 127:
            o += c
    return o

global_testApho_nbr_hit = 0
def testApho(s):
    global global_testApho_nbr_hit
    ret = apho.getThoughts(s)
    print("\n%s" % strToPrint(s))
    print("=>")

    if ret != None:
        s = ret[0]
        #~ print(str(s)) # UnicodeEncodeError: 'ascii' codec can't encode character u'\xe0'
        print(strToPrint(s)) # cette ligne bloque en python 2.7,  LookupError: unknown encoding: cp65001 # corriger en faisant dans le shell: set PYTHONIOENCODING=UTF-8
        global_testApho_nbr_hit += 1
    print("")

def autoTest():
    testApho("j'aime pas travailler")
    testApho("j'aime pas travailler")
    testApho("j'aime pas travailler")
    testApho("j'aime pas travailler")
    print("")

    #~ testApho("j'ai la volonté de t'aider"))
    #~ testApho("j'ai la volonté de t'aider"))
    #~ testApho("j'ai la volonté de t'aider"))
    testApho("j'ai la volonté de t'aider")
    testApho("j'ai la volonté de t'aider")
    testApho("j'ai la volonté de t'aider")
    print("")

    testApho("Il me faudrait du courage")
    testApho("Il me faudrait du courage")
    testApho("J'aime le ChamPagne.")
    testApho("J'aime le ChamPagne.")
    testApho("J'aime le vin.")
    testApho("J'aime le vin.")
    testApho("d'attendre la pluie")
    testApho("d'attendre la pluie")
    testApho("attendre la pluie")
    print("")

    testApho("Dis moi une phrase")
    testApho("Ecoute moi")
    testApho("Dis moi un truc intelligent!")
    testApho("Dis moi un truc intelligent!")
    print("")

    testApho("Dis moi, tu connais des gens célèbres?")
    testApho("Dis moi, tu connais des gens célèbres?")
    testApho("travailler moins c'est cool ou pas ?")
    testApho("travailler moins c'est cool ou pas ?")
    testApho("Consommer moins c'est cool ou pas ?")

    testApho("c'est bien fait peur. tu es un bon petit gars. eh mon petit gnocchi est-ce qu'il a.")
    # => "Gourmandise : source inépuisable de bonheur. a cause de bonheur" bon match bonheur, c'est moche.
    # => maintenant donne, Quand vous êtes à Rome, faites comme les Romains. fait => faites
    # => Le seul fait d’exister est un véritable bonheur
    testApho("ambulance") # => Ben par exemple, c'est un mec qui va a une soirée genre bien habillé
    print("")

    print("global_testApho_nbr_hit: %s" % global_testApho_nbr_hit )
    assert(global_testApho_nbr_hit >= 20)

    if 0:
        # test sur python 2.7
        print(stringtools.accentToHtml("un élève"))
        for i,a in enumerate(apho.thous):
            print(i)
            #~ print(stringtools.accentToHtml(a[0]))
            s1 = stringtools.cp1252ToHtml(a[0])
            s2 = stringtools.cp1252ToHtml(a[1])
            print(s1)
            print(s2)
            print(stringtools.transformAccentToUtf8(a[0]))
            print(stringtools.transformAccentToUtf8(a[1]))
            #~ if i>80:
                #~ break

if __name__ == "__main__":
    #~ autoTest()
    test_loop_asr()


"""SimPhoNy-wrapper for celery-workflows"""

import logging
from typing import TYPE_CHECKING

from osp.core.namespaces import emmo
from osp.core.session import SimWrapperSession

from .celery_workflow_engine import CeleryWorkflowEngine

if TYPE_CHECKING:
    from typing import UUID, Any, Dict, List, Optional

    from pydantic import BaseSettings

    from osp.core.cuds import Cuds

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

class CeleryWorkflowSession(SimWrapperSession):
    """SimPhoNy-wrapper session for Celery-Chains and Worklows"""

    def __init__(
        self,
        input_uuid: "UUID",
        engine: CeleryWorkflowEngine = None,
        logging_id: str = None,
    ) -> None:
        """Initalite the session."""
        if not engine:
            engine = CeleryWorkflowEngine(input_uuid, logging_id=logging_id)
        super().__init__(engine=engine)

    # OVERRIDE
    def __str__(self):
        return "CeleryWorkflowSession"

    # OVERRIDE
    def _run(self, root_cuds_object) -> str:
        """Run the wrapper session."""
        return self._engine.run()

    # OVERRIDE
    def _apply_added(self, root_obj, buffer) -> None:
        """Apply scans of added cuds in buffer."""
        for obj in buffer.values():
            if obj.is_a(emmo.Workflow):
                message = """Found %s. Will search for workflow steps
                    and related workers on the platform."""
                logger.info(message, obj)
                self._scan_for_neighbours(obj)
        if not self._engine.tasks:
            message = """Did not find any workflow steps with
            complementary workers. Will scan for single
            object of type %s."""
            logger.info(message, emmo.Calculation)
            self._scan_for_single_calc(buffer)
        else:
            message = """Scan for workflow steps complete.
            Identified the following chain of workers: %s"""
            logger.info(message, self._engine.tasks)

    def _scan_for_neighbours(self, obj: "Cuds", step: str = "first") -> None:
        """Scan the input cuds for neighbour-tasks"""
        neighbour = obj.get(rel=emmo[f"hasSpatial{step.title()}"])
        if not neighbour:
            message = f"""Did not find {step} task in the chain of calculations in {obj}.
            Workflow chain has ended here."""
            logger.info(message)
        else:
            neighbour = neighbour.pop()
            self._get_worker_mapping(neighbour)
            self._scan_for_neighbours(neighbour, step="next")

    def _scan_for_single_calc(self, buffer: "Dict[Any, Cuds]") -> None:
        """Find single calculations to be run in the buffer."""
        for obj in buffer.values():
            if obj.is_a(emmo.Calculation):
                self._get_worker_mapping(obj)
        if not self._engine.tasks:
            message = """Did not find any calculations with
            complementary workers."""
            raise TypeError(message)
        message = """Found additional workers %s in the buffer,
        but will ignored because not part of a workflow chain."""
        logger.info(message, self._engine.tasks)

    def _get_worker_mapping(self, calculation: "Cuds") -> None:
        mapping = self._scan_worker_mapping(calculation)
        if not mapping:
            message = f"""Task in the chain of calculations is not properly
            mapped to an existing worker {calculation}."""
            logger.info(message)
        else:
            self._engine.add_task((calculation, mapping))

    def _scan_worker_mapping(self, calculation: "Cuds") -> "Optional[str]":
        response = []
        for superclass, mapping in self.worker_mapping.items():
            if calculation.is_a(superclass):
                response.append(mapping)
        if len(response) > 1:
            raise ValueError(
                f"More than 1 {calculation.oclass} found in worker mapping!"
            )
        if response:
            response = response.pop()
        return response

    # OVERRIDE
    def _apply_deleted(self, root_obj, buffer) -> None:
        """Apply functions for updated-buffer."""

    def _apply_updated(self, root_obj, buffer) -> None:
        """Apply functions for deleted-buffer."""

    # OVERRIDE
    def _load_from_backend(self, uids, expired=None) -> "Cuds":
        """Load a cuds from backend."""
        for uid in uids:
            if uid in self._registry:
                yield self._registry.get(uid)
            else:
                yield None

    @property
    def settings(cls) -> "BaseSettings":
        """Return the settings from the engine."""
        return cls._engine.settings

    @property
    def result(cls) -> "Dict[str, Any]":
        """Return the final result of the workflow"""
        return cls._engine.result

    @property
    def worker_mapping(cls) -> "Dict[str, Any]":
        """Return the mappings of the ontology class and worker name"""
        return cls._engine.worker_mapping

    @property
    def workers(cls) -> "List[str]":
        """Return the list of workers to pass the knowledge graph in a chain."""
        return cls._engine.workers


# coding:utf8
# author:winton

import logging
import os
import datetime
import argparse

from conf import Config
from util import Util
from ConsumerManager import ConsumerManager

class Lams:
    '''
    控制数据收集的主要流程
    '''

    def init(self):
        '''
        读取配置并完成初始化
        '''
        loggerConfig = Config.logger
        # 检查日志文件夹是否存在
        logDir = os.path.split(loggerConfig['filename'])[0]
        if not os.path.exists(logDir):
            os.makedirs(logDir)
        # 日志配置
        logging.basicConfig(
            level=loggerConfig['level'],
            format=loggerConfig['format'],
            filename=loggerConfig['filename'],
            encoding=loggerConfig['encoding']
        )

        # 载入consumer
        self.cm = ConsumerManager(Config)
        logging.info('%d Consumers loaded' % len(self.cm.consumers))
        self.allFile = 0
        self.successFile = 0

        logging.info('Lams starting...')

    def startForNew(self, move_after_dispatch=True):
        '''
        开始处理指定目录下的数据
        '''
        dataDir = Config.datapool_new
        dataList = os.listdir(dataDir)
        if len(dataList) == 0:
            logging.info('New data not found, exiting...')
            os.system('exit 0')
        logging.info('New data found, dispatching...')
        for filename in dataList:
            self.dispatch(filename, dataDir, move_after_dispatch)
        logging.info('Dispatching finish, %d success, %d fail' % (self.successFile, self.allFile - self.successFile))
        os.system('exit 0')

    def startForAll(self, classInfo=None):
        '''
        重新处理所有数据
        '''
        if classInfo is not None:
            logging.info('dispatch for class [%s]' % classInfo)
        logging.info('dispatch all data...')
        for parent, dirnames, filenames in os.walk(Config.datapool):
            for filename in filenames:
                self.dispatch(filename, parent, False, classInfo=classInfo)
        logging.info('Dispatching finish, %d success, %d fail' % (self.successFile, self.allFile - self.successFile))
        os.system('exit 0')

    def dispatch(self, filename, dataDir, move_after_dispatch=True, classInfo=None):
        '''
        分发对应的文件列表
        '''
        filePath = os.path.join(dataDir, filename)
        try:
            event = Util.loadJsonFile(filePath)
            consumers = self.cm.getMapConsumer(event, classInfo)
            for csm in consumers:
                logging.info('event "%s" is sending to consumer "%s"' % (filePath, csm))
            self.cm.emitEvent(event, consumers)
        except Exception as e:
            logging.exception('Error when dispatching "%s" [%s]' % (filePath, str(e)))
        else:
            self.successFile += 1
            # 将已处理的文件移动到指定文件夹
            if move_after_dispatch:
                t = datetime.datetime.now()
                today = t.strftime('%Y-%m-%d')
                newDir = '%s/%s' % (Config.datapool, today)
                if not os.path.exists(newDir):
                    os.makedirs(newDir)
                newFilePath = '%s/%s' % (newDir, filename)
                logging.debug('moving [src=%s] [dst=%s]' % (filePath, newFilePath))
                os.rename(filePath, newFilePath)
        finally:
            self.allFile += 1

if __name__ == '__main__':
    ap = argparse.ArgumentParser(description='do dispatching jobs')
    ap.add_argument('-A', '--all', action='store_true', help='dispatch all history data and new data')
    ap.add_argument('-c', help='dispatch all but just dispatch to one class, use it in this form [moduleName:className]')
    args = ap.parse_args()

    test = Lams()
    test.init()
    if args.all:
        test.startForAll()
    elif args.c is not None:
        test.startForAll(args.c.split(':'))
    else:
        test.startForNew()


from django.urls import path
from .views import MovieList,MovieDetail,MovieCategory,MovieLanguage,MovieSearch,MovieYear,MostWatch
from django.conf.urls import url

app_name='movie'
urlpatterns = [
    path('', MovieList.as_view(),name='Movie_List'),
    path('category/<str:category>', MovieCategory.as_view(),name='MovieCategory'),
    path('language/<str:lang>', MovieLanguage.as_view(),name='MovieLanguage'),
    path('search/', MovieSearch.as_view(),name='MovieSearch'),
    path('<slug:slug>', MovieDetail.as_view(),name='Movie_Detail'),
    path('year/<int:year>', MovieYear.as_view(),name='MovieYear'),
    path('mostwatch/',MostWatch.as_view(),name='MostWatch')
]