from django.urls import path
from . import views
from django.conf import settings
from django.conf.urls.static import static

urlpatterns = [
    path('', views.home, name='home'),
    path('login/', views.VRegistro.as_view(), name="autenticacion"),
    path('autenticar/', views.autenticar, name="autenticar"),
    path('logout/', views.LogOut, name="LogOut"),

    path('perfil/', views.act_info, name="act_info"),
    path('cambiar_contraseña/', views.cambiar_contrasena, name="cambiar_contraseña"),

    path('añadir_categoria/', views.categoria_add, name="categoria_add"),
    path('ver_categoria/', views.categoria_list, name="categoria_list"),
    path('delete_categoria/<id>', views.delete_categoria, name='categoria_delete'),
    path('update_categoria/<id>', views.update_categoria, name='categoria_update'),

    path('añadir_area_especialidad/', views.area_especialidad_add, name="area_especialidad_add"),
    path('ver_area_especialidad/', views.area_especialidad_list, name="area_especialidad_list"),
    path('delete_area_especialidad/<id>', views.delete_area_especialidad, name='area_especialidad_delete'),
    path('update_area_especialidad/<id>', views.update_area_especialidad, name='area_especialidad_update'),

    path('añadir_rol_trabajo/', views.rol_trabajo_add, name="rol_trabajo_add"),
    path('ver_rol_trabajo/', views.rol_trabajo_list, name="rol_trabajo_list"),
    path('ver_rol_trabajo/edit_rol_trabajo/<id>', views.update_rol_trabajo, name='rol_trabajo_edit'),
    path('ver_rol_trabajo/rol_trabajo/<id>', views.rol_trabajo, name='rol_trabajo_ver'),
    path('delete_rol_trabajo/<id>', views.delete_rol_trabajo, name='rol_trabajo_delete'),

    path('añadir_conocimiento/', views.conocimiento_add, name="conocimiento_add"),
    path('ver_conocimiento/', views.conocimiento_list, name="conocimiento_list"),
    path('delete_conocimiento/<id>', views.delete_conocimiento, name='conocimiento_delete'),
    path('update_conocimiento/<id>', views.update_conocimiento, name='conocimiento_update'),

    path('añadir_habilidad/', views.habilidad_add, name="habilidad_add"),
    path('ver_habilidad/', views.habilidad_list, name="habilidad_list"),
    path('delete_habilidad/<id>', views.delete_habilidad, name='habilidad_delete'),
    path('update_habilidad/<id>', views.update_habilidad, name='habilidad_update'),

    path('añadir_tarea/', views.tarea_add, name="tarea_add"),
    path('ver_tarea/', views.tarea_list, name="tarea_list"),
    path('delete_tarea/<id>', views.delete_tarea, name='tarea_delete'),
    path('update_tarea/<id>', views.update_tarea, name='tarea_update'),

    path('añadir_herramienta/', views.herramienta_add, name="herramienta_add"),
    path('ver_herramienta/', views.herramienta_list, name="herramienta_list"),
    path('delete_herramienta/<id>', views.delete_herramienta, name='herramienta_delete'),
    path('update_herramienta/<id>', views.update_herramienta, name='herramienta_update'),

    path('añadir_centro_trabajo/', views.centro_trabajo_add, name="centro_trabajo_add"),
    path('ver_centro_trabajo/', views.centro_trabajo_list, name="centro_trabajo_list"),
    path('delete_centro_trabajo/<id>', views.delete_centro_trabajo, name='centro_trabajo_delete'),
    path('update_centro_trabajo/<id>', views.update_centro_trabajo, name='centro_trabajo_update'),

    path('añadir_genero/', views.genero_add, name="genero_add"),
    path('ver_genero/', views.genero_list, name="genero_list"),
    path('delete_genero/<id>', views.delete_genero, name='genero_delete'),
    path('update_genero/<id>', views.update_genero, name='genero_update'),

    path('añadir_municipio/', views.municipio_add, name="municipio_add"),
    path('ver_municipio/', views.municipio_list, name="municipio_list"),
    path('delete_municipio/<id>', views.delete_municipio, name='municipio_delete'),
    path('update_municipio/<id>', views.update_municipio, name='municipio_update'),

    path('añadir_provincia/', views.provincia_add, name="provincia_add"),
    path('ver_provincia/', views.provincia_list, name="provincia_list"),
    path('delete_provincia/<id>', views.delete_provincia, name='provincia_delete'),
    path('update_provincia/<id>', views.update_provincia, name='provincia_update'),

    path('añadir_admin/', views.AdminAdd.as_view(), name="admin_add"),
    path('ver_admin/', views.admin_list, name="admin_list"),
    path('delete_admin/<id>', views.delete_admin, name='admin_delete'),

    path('añadir_especialista/', views.EspecialistaAdd.as_view(), name="especialista_add"),
    path('ver_especialista/', views.especialista_list, name="especialista_list"),
    path('delete_especialista/<id>', views.delete_especialista, name='especialista_delete'),

    path('evaluar_especialista/<rt_id>', views.evaluar_especialista, name='especialista_evaluar'),

]

urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


# need to revisit this script
# this does not do much other than list all the available ips
# no longer shows available ip's errors out

import subprocess
import ipaddress

network_address = input('Enter a network address in CIDR format(example 192.168.1.0/24): ')

ip_network = ipaddress.ip_network(network_address)
for all_hosts in ip_network.hosts():
    print(all_hosts)

    status, result = subprocess.getstatusoutput('ping -c1 -w2 ' + all_hosts)

if status == 0:
    print('Host ' + all_hosts + ' is UP')
else:
    print('Host ' + all_hosts + ' is DOWN')


import datetime

from oslo.config import cfg
import webob.exc

from nova.api.openstack import extensions
from nova import compute
from nova import utils

CONF = cfg.CONF
CONF.import_opt('compute_topic', 'nova.compute.rpcapi')

authorize = extensions.extension_authorizer('compute',
                           'instance_usage_audit_log')

class InstanceUsageAuditLogController(object):
    def __init__(self):
        self.host_api = compute.HostAPI()

    def index(self, req):
        context = req.environ['nova.context']
        authorize(context)
        task_log = self._get_audit_task_logs(context)
        return {'instance_usage_audit_logs': task_log}

    def show(self, req, id):
        context = req.environ['nova.context']
        authorize(context)
        try:
            if '.' in id:
                before_date = datetime.datetime.strptime(str(id),
                                                "%Y-%m-%d %H:%M:%S.%f")
            else:
                before_date = datetime.datetime.strptime(str(id),
                                                "%Y-%m-%d %H:%M:%S")
        except ValueError:
            msg = _("Invalid timestamp for date %s") % id
            raise webob.exc.HTTPBadRequest(explanation=msg)
        task_log = self._get_audit_task_logs(context,
                                                     before=before_date)
        return {'instance_usage_audit_log': task_log}

    def _get_audit_task_logs(self, context, begin=None, end=None,
                             before=None):
        """Returns a full log for all instance usage audit tasks on all
           computes.

        :param begin: datetime beginning of audit period to get logs for,
            Defaults to the beginning of the most recently completed
            audit period prior to the 'before' date.
        :param end: datetime ending of audit period to get logs for,
            Defaults to the ending of the most recently completed
            audit period prior to the 'before' date.
        :param before: By default we look for the audit period most recently
            completed before this datetime. Has no effect if both begin and end
            are specified.
        """
        defbegin, defend = utils.last_completed_audit_period(before=before)
        if begin is None:
            begin = defbegin
        if end is None:
            end = defend
        task_logs = self.host_api.task_log_get_all(context,
                                                   "instance_usage_audit",
                                                   begin, end)
        # We do this this way to include disabled compute services,
        # which can have instances on them. (mdragon)
        filters = {'topic': CONF.compute_topic}
        services = self.host_api.service_get_all(context, filters=filters)
        hosts = set(serv['host'] for serv in services)
        seen_hosts = set()
        done_hosts = set()
        running_hosts = set()
        total_errors = 0
        total_items = 0
        for tlog in task_logs:
            seen_hosts.add(tlog['host'])
            if tlog['state'] == "DONE":
                done_hosts.add(tlog['host'])
            if tlog['state'] == "RUNNING":
                running_hosts.add(tlog['host'])
            total_errors += tlog['errors']
            total_items += tlog['task_items']
        log = dict((tl['host'], dict(state=tl['state'],
                                  instances=tl['task_items'],
                                  errors=tl['errors'],
                                  message=tl['message']))
                  for tl in task_logs)
        missing_hosts = hosts - seen_hosts
        overall_status = "%s hosts done. %s errors." % (
                    'ALL' if len(done_hosts) == len(hosts)
                    else "%s of %s" % (len(done_hosts), len(hosts)),
                    total_errors)
        return dict(period_beginning=str(begin),
                    period_ending=str(end),
                    num_hosts=len(hosts),
                    num_hosts_done=len(done_hosts),
                    num_hosts_running=len(running_hosts),
                    num_hosts_not_run=len(missing_hosts),
                    hosts_not_run=list(missing_hosts),
                    total_instances=total_items,
                    total_errors=total_errors,
                    overall_status=overall_status,
                    log=log)

class Instance_usage_audit_log(extensions.ExtensionDescriptor):
    """Admin-only Task Log Monitoring."""
    name = "OSInstanceUsageAuditLog"
    alias = "os-instance_usage_audit_log"
    namespace = "http://docs.openstack.org/ext/services/api/v1.1"
    updated = "2012-07-06T01:00:00+00:00"

    def get_resources(self):
        ext = extensions.ResourceExtension('os-instance_usage_audit_log',
                                           InstanceUsageAuditLogController())
        return [ext]


import pytest
import sys
from selenium import webdriver
from webdriver_manager.firefox import GeckoDriverManager
from webdriver_manager.chrome import ChromeDriverManager
from webdriver_manager.microsoft import EdgeChromiumDriverManager
from webdriver_manager.microsoft import IEDriverManager
from webdriver_manager.utils import ChromeType

sys.path.append('pageobjects')
sys.dont_write_bytecode = True

# add cli options
def pytest_addoption(parser):
    parser.addoption('--url', action='store', default='https://jisho.org/', help='Specify the base URL for testing')
    parser.addoption('--driver', action='store', default='chrome', help='Supported browsers: Firefox, Chrome, Chromium, Edge, and IE')

# driver fixture passed to all tests
@pytest.fixture(scope='session')
def driver(request):
    browserName = request.config.getoption('--driver')
    if browserName == "firefox":
        driver = webdriver.Firefox(executable_path=GeckoDriverManager().install())
    elif browserName == "chrome":
        driver = webdriver.Chrome(ChromeDriverManager().install())
    elif browserName == "chromium":
        driver = webdriver.Chrome(ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install())
    elif browserName == "edge":
        driver = webdriver.Edge(EdgeChromiumDriverManager().install())
    elif browserName == "ie":
        driver = webdriver.Ie(IEDriverManager().install())
    else:
        raise ValueError('Invalid browser name: ' + browserName)
    driver.set_window_size(1280, 800)
    driver.base_url = request.config.getoption('--url')
    yield driver
    driver.quit()


from keras.applications import *
from keras.layers.pooling import GlobalAveragePooling2D, GlobalMaxPooling2D
from keras.layers import Dense
from keras.models import Model
from keras.callbacks import ModelCheckpoint,EarlyStopping, ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import *
import os

nb_classes = 5
batch_size = 32
img_w = 299
img_h = 299
img_channels = 3
rotation_degrees = 15
transformation_ratio = .05
nb_epoch = 150
keep_frozen = 100

base_model = Xception(include_top=False, weights='imagenet', input_shape=(img_w,img_h,img_channels))

x = base_model.output
x = GlobalAveragePooling2D()(x)

predictions = Dense(nb_classes, activation='softmax')(x)

model = Model(base_model.input, predictions)

for layer in model.layers[:keep_frozen]:
    layer.trainable = False
for layer in model.layers[keep_frozen:]:
    layer.trainable = True

train_datagen = ImageDataGenerator(rotation_range=10,
    width_shift_range=0.2,
    height_shift_range=0.2,
    zoom_range=0.2,
    fill_mode='constant',
    horizontal_flip=True)

validation_datagen = ImageDataGenerator(rotation_range=10,
    width_shift_range=0.2,
    height_shift_range=0.2,
    zoom_range=0.2,
    fill_mode='constant',
    horizontal_flip=True)

train_generator = train_datagen.flow_from_directory("images/training_set/",
                                                        target_size=(img_w, img_h),
                                                        batch_size=batch_size,
                                                        class_mode='categorical')

validation_generator = validation_datagen.flow_from_directory("images/validation_set/",
                                           target_size=(img_w, img_h),
                                           batch_size=batch_size,
                                           class_mode='categorical')

opt = SGD(lr=1e-3)

model.compile(optimizer=opt,
                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

top_weights_path = os.path.join("model", 'top_model_weights_xception.h5')

callbacks_list = [
    ModelCheckpoint(top_weights_path, monitor='val_acc', verbose=1, save_best_only=True),
    EarlyStopping(monitor='val_acc', patience=60, verbose=0),
    ReduceLROnPlateau(monitor='val_loss', factor=0.3, patience=40, verbose=1, mode='auto', epsilon=1e-10, cooldown=0, min_lr=1e-10)]
print(model.summary())

model.fit_generator(train_generator,
                    steps_per_epoch=(train_generator.n/batch_size)/3,
                    epochs=1,#nb_epoch*3,
                    callbacks=callbacks_list,
                    validation_data=validation_generator,
                    validation_steps=(validation_generator.n / batch_size)/5)

model_json = model.to_json()
with open(os.path.join("model/", 'model_xception.json'), 'w') as json_file:
    json_file.write(model_json)


import pandas as pd
import os
import sys
import json

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')))
import common.bert.constants as constants

def get_sentiment(x):
    if x['opos'] == 0 and x['oneg'] == 1:
        sentiment = 0  # negative
    elif x['opos'] == 1 and x['oneg'] == 0:
        sentiment = 1  # positive
    elif x['opos'] == 1 and x['oneg'] == 1:
        sentiment = 2  # mixed
    else:
        sentiment = 3  # neutral

    return sentiment

def convert(dataset, basedir=os.path.join('data', 'classification')):

    path_data = os.path.join(basedir, dataset)
    data = pd.read_csv(path_data)

    data['sentiment'] = data.apply(lambda x: get_sentiment(x), axis=1)
    data['sentiment'].value_counts()
    data.to_csv(os.path.join(path_data))

    index2category = {0: 'negative', 1: 'positive', 2: 'mixed', 3: 'neutral'}
    with open(os.path.join(basedir, constants.LABELS_FILE), 'w') as outfile:
        json.dump(index2category, outfile)

if __name__ == '__main__':
    convert(dataset=sys.argv[1], basedir=sys.argv[2])


# -*- coding: utf-8 -*-
"""
Created on Fri Feb 23 09:16:30 2018

@author: arhen
"""

import matplotlib.pyplot as plt
x = range(0,5)
y1 = [0.1, 0.2, 0.5, 0.3, 0.3]
y2 = [0.3, 0.6, 0.8, 0.7, 0.5]
plt.close()
plt.plot(x, y1)
plt.plot(x, y2)
plt.ylim([0,1])
plt.show()


from ...utils.logger import logger as log

def get_drawdowns(
    data: list[float | int],
    as_percentage: bool = True,
) -> list[float | int]:
    """Gets a list of all drawdown values of a univariate timeseries.

    Args:
        data (list[float | int]): The timeseries data for which to obtain drawdowns.

    Returns:
        dict[str, Any]: Dictionary describing the max drawdown value.
    """
    log.info("Calling get_drawdowns")

    drawdowns = []
    previous_datum = -1  # data values are assumed to be positive semi-definite
    for i, datum in enumerate(data):
        if i + 1 < len(data) and datum >= previous_datum:
            drawdown = 0
            proceeding = data[i + 1 :]  # all data after the current point
            for p in proceeding:
                if p < datum and (datum - p) > drawdown:
                    drawdown = datum - p
                else:
                    break
            if as_percentage:
                drawdown = drawdown / datum * 100
            drawdowns.append(drawdown)
            previous_datum = datum

    return drawdowns


from dataclasses import dataclass
from pathlib import Path
from typing import Any, List, Optional, Tuple
import tensorflow as tf

from trainer.metrics import MeanIoU

@dataclass
class TrainerParams:
    num_classes: int
    batch_size: int
    learning_rate: float
    epochs: int
    save_checkpoint: bool = False
    checkpoint_path: Optional[Path] = None
    load_best_checkpoint_after_fit: bool = False
    fit_verbosity: int = 0

class Trainer:
    def __init__(self, params: TrainerParams,
                 dataset_loader: Any,
                 model: tf.keras.Model,
                 input_dim: Tuple[int, int, int],
                 extra_callbacks: List[tf.keras.callbacks.Callback] = []):
        self.params = params
        self.input_dim = input_dim
        self.loader = dataset_loader
        self.model = model
        self.callbacks = extra_callbacks

        self._build_optimizer()
        self._build_loss()
        self._build_metrics()
        self._prepare_model()
        if self.params.save_checkpoint:
            assert self.params.checkpoint_path is not None
            self._build_checkpoint_callback()

    def _build_optimizer(self):
        self.optimizer = tf.keras.optimizers.Adam(
            learning_rate=self.params.learning_rate,
        )

    def _build_loss(self):
        self.loss = tf.keras.losses.SparseCategoricalCrossentropy()

    def _build_metrics(self):
        self.metrics = [
            MeanIoU(self.params.num_classes, name='mean_iou')
        ]

    def _prepare_model(self):
        self.model.compile(
            optimizer=self.optimizer,
            loss=self.loss,
            metrics=self.metrics,
        )
        self.model.build((None,) + self.input_dim)
        print(self.model.summary())

    def _build_checkpoint_callback(self):
        self.callbacks.append(
            tf.keras.callbacks.ModelCheckpoint(
                filepath=self.params.checkpoint_path / 'ckpt',
                save_best_only=True,
                save_weights_only=True,
                save_freq='epoch',
            )
        )

    def fit(self):
        if self.params.checkpoint_path is not None and self.params.checkpoint_path.exists():
            print(f'Loading from checkpoint at {self.params.checkpoint_path}')
            checkpoint = tf.train.Checkpoint(self.model)
            checkpoint.restore(
                str(self.params.checkpoint_path / 'ckpt')).expect_partial()
        train_ds = self.loader.train\
            .prefetch(tf.data.AUTOTUNE)\
            .shuffle(self.params.batch_size * 512)\
            .batch(self.params.batch_size)
        val_ds = self.loader.val.prefetch(
            tf.data.AUTOTUNE).batch(self.params.batch_size)
        self.model.fit(
            train_ds,
            validation_data=val_ds,
            epochs=self.params.epochs,
            callbacks=self.callbacks,
            verbose=max(0, self.params.fit_verbosity),
        )
        if self.params.save_checkpoint and self.params.load_best_checkpoint_after_fit:
            checkpoint = tf.train.Checkpoint(self.model)
            checkpoint.restore(
                str(self.params.checkpoint_path / 'ckpt')).expect_partial()

    def save_model(self, model_path: Path):
        self.model.save(model_path / '1',
                        overwrite=True)


from __future__ import print_function

from pyglet import window
from pyglet import clock
from pyglet import event

import game_manager

class GameWindow(window.Window):
    def __init__(self, *args, **kwargs):
        super(GameWindow, self).__init__(*args, **kwargs)
        self.has_exit = False
        self.playerManager = None
        self.gameManager = game_manager.GameManager(self)

wm = """
class WindowDrawManager(event.EventDispatcher):
    def __init__(self, parent):
        super(WindowDrawManager, self).__init__()
        self.parent = parent

        self.activate()

    def activate(self):
        self.parent.push_handlers(self.on_draw)

    def deactivate(self):
        self.parent.pop_handlers()

    def on_draw(self):
        self.parent.clear()
        self.dispatch_event('window_draw')
        self.parent.flip()

WindowDrawManager.register_event_type('window_draw')
"""


#%%
import torch
from torch_geometric.nn import MessagePassing
from torch_geometric.utils import add_self_loops, degree
import torch.nn.functional as F
from torch_geometric.data import Data
from torch.utils.data import random_split
import matplotlib.pyplot as plt
import pickle
import pandas as pd
from scipy.spatial import Delaunay
import numpy as np
from torch_geometric.loader import DataLoader
from tqdm import tqdm
import torch.optim as optim
import math
from tqdm import tqdm
from itertools import combinations

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
class EdgeConv(MessagePassing):
    def __init__(self, num_node_features, num_edge_features, out_channels):
        super(EdgeConv, self).__init__(aggr='add')  # "Add" aggregation.
        self.lin = torch.nn.Linear(num_node_features * 2 + num_edge_features, out_channels)

    def forward(self, x, edge_index, edge_attr):
        # x has shape [N, in_channels]
        # edge_index has shape [2, E]
        # edge_attr has shape [E, edge_attr_dim]

        # Add self-loops to the adjacency matrix.
        edge_index, _ = add_self_loops(edge_index, num_nodes=x.size(0))

        # Add edge attributes for self-loops.
        loop_attr = torch.zeros((x.size(0), edge_attr.size(1)), device=edge_attr.device)
        edge_attr = torch.cat([edge_attr, loop_attr], dim=0)

        return self.propagate(edge_index, size=(x.size(0), x.size(0)), x=x, edge_attr=edge_attr)

    def message(self, x_i, x_j, edge_attr):
        # x_i has shape [E, in_channels]
        # x_j has shape [E, in_channels]
        # edge_attr has shape [E, edge_attr_dim, 1]
        #print(f'x_i shape (in model): {x_i.shape}')
        #print(f'x_j shape (in model): {x_j.shape}')
        #print(f'edge_attr shape (in model): {edge_attr.shape}')
        edge_attr = edge_attr.squeeze(-1)  # Remove the singleton dimension
        edge_input = torch.cat([x_i, x_j, edge_attr], dim=1)  # shape [E, in_channels * 2 + edge_attr_dim]
        return self.lin(edge_input)

class GCN(torch.nn.Module):
    def __init__(self, num_node_features, num_edge_features, num_classes):
        super(GCN, self).__init__()
        self.conv1 = EdgeConv(num_node_features, num_edge_features, 512)
        self.conv2 = EdgeConv(512, num_edge_features, 256)
        self.conv3 = EdgeConv(256, num_edge_features, 128)
        self.conv4 = EdgeConv(128, num_edge_features, 64)
        self.conv5 = EdgeConv(64, num_edge_features, 32)
        self.batch_norm = torch.nn.BatchNorm1d(32)
        self.classifier = torch.nn.Linear(32, num_classes)

    def forward(self, data):
        x, edge_index, edge_attr = data.x, data.edge_index, data.edge_attr

        x = self.conv1(x, edge_index, edge_attr)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)

        x = self.conv2(x, edge_index, edge_attr)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)

        x = self.conv3(x, edge_index, edge_attr)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)

        x = self.conv4(x, edge_index, edge_attr)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)

        x = self.conv5(x, edge_index, edge_attr)
        x = self.batch_norm(x)
        x = F.dropout(x, training=self.training)

        x = self.classifier(x)

        return F.log_softmax(x, dim=1)
#%%
with open('dataset_hist.pkl', 'rb') as f:
    data = pickle.load(f)

point_sets = data['dataframes']
print(len(point_sets))

def edge_features(points, edge):
    # Edge length
    length = np.linalg.norm(points[edge[0]] - points[edge[1]])

    if length == 0:
        print(f"Zero length encountered. Points: {points[edge[0]], points[edge[1]]}")

    # Calculate the edge vector, taking the point with the largest y value minus the other
    if points[edge[0], 1] > points[edge[1], 1]:
        vec = points[edge[0]] - points[edge[1]]
    else:
        vec = points[edge[1]] - points[edge[0]]

    # Cosine of the angle
    cosine = vec[0] / length
    # Sine of the angle
    sine = vec[1] / length

    return np.array([length, cosine, sine])

def calculate_angle(p1, p2, p3):
    """
    Calculates the angle between three points.
    """
    v1 = p1 - p2
    v2 = p3 - p2

    cos_theta = np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))
    cos_theta = np.clip(cos_theta, -1, 1)  # ensure the value is within [-1, 1]
    return np.arccos(cos_theta)

data_list = []
for idx, df in enumerate(point_sets):
    points = df[['x', 'y']].values
    tri = Delaunay(points)

    edges = set()
    for simplex in tri.simplices:
        for i in range(3):
            edge = tuple(sorted([simplex[i], simplex[(i+1)%3]]))
            edges.add(edge)

    adjacency_matrix = np.zeros((len(points), len(points)))

    num_nodes = len(points)
    print(f"Number of nodes in dataframe {idx}: {num_nodes}")

    edge_index = torch.tensor(list(edges), dtype=torch.long).t().contiguous()

    num_edges = edge_index.shape[1]
    print(f"Number of edges in dataframe {idx}: {num_edges}")

    for edge in edge_index.T:
        #print(f"Edge: {edge}, Edge[0]: {edge[0]}, Edge[1]: {edge[1]}")
        # Since it's an undirected graph, we add 1 for both (i, j) and (j, i)
        adjacency_matrix[edge[0], edge[1]] = 1
        adjacency_matrix[edge[1], edge[0]] = 1

    edge_attr = []
    for edge in edge_index.t().numpy():
        edge_attr.append(edge_features(points, edge))
    edge_attr = np.array(edge_attr)

    edge_attr = torch.tensor(edge_attr, dtype=torch.float)

    degree = np.sum(adjacency_matrix, axis=1)
    df['degree'] = degree

    df['x'] = (df['x'] - df['x'].min()) / (df['x'].max() - df['x'].min())
    df['y'] = (df['y'] - df['y'].min()) / (df['y'].max() - df['y'].min())

    centroid_x = df['x'].mean()
    centroid_y = df['y'].mean()

    df['relative_x'] = df['x'] - centroid_x
    df['relative_y'] = df['y'] - centroid_y

    # Initialize average_neighbor_distance
    df['average_neighbor_distance'] = 0

    # Iterate over edges
    for edge in edge_index.T:
        # Compute distance between nodes
        distance = np.linalg.norm(points[edge[0]] - points[edge[1]])

        # Convert tensor to numpy for indexing
        edge_numpy = edge.numpy()

        # Add the distance to both nodes, we will average it later
        df.loc[edge_numpy[0], 'average_neighbor_distance'] += distance
        df.loc[edge_numpy[1], 'average_neighbor_distance'] += distance

    # Get the degrees of the nodes
    degrees = np.sum(adjacency_matrix, axis=1)

    # Average the distances
    df['average_neighbor_distance'] /= degrees

    df['min_n_dist'] = 0
    df['max_n_dist'] = 0

    # Compute pairwise distance matrix
    distance_matrix = np.linalg.norm(points[:, None, :] - points[None, :, :], axis=-1)

    for i in range(len(points)):
        # Get indices of neighbors
        neighbors = np.where(adjacency_matrix[i, :] > 0)[0]

        if len(neighbors) > 0:  # Check if node has any neighbors
            # Get distances to neighbors
            neighbor_distances = distance_matrix[i, neighbors]

            # Compute minimum and maximum neighbor distance
            min_n_dist = np.min(neighbor_distances)
            max_n_dist = np.max(neighbor_distances)

            # Update the dataframe
            df.loc[i, 'min_n_dist'] = min_n_dist
            df.loc[i, 'max_n_dist'] = max_n_dist
        else:  # For isolated nodes, we'll set the distance to inf
            df.loc[i, 'min_n_dist'] = np.inf
            df.loc[i, 'max_n_dist'] = np.inf

    df['min_relative_angle'] = 0
    df['max_relative_angle'] = 0

    for i in range(len(points)):
        # Get indices of neighbors
        neighbors = np.where(adjacency_matrix[i, :] > 0)[0]

        if len(neighbors) > 1:  # Check if node has more than one neighbor
            # Calculate all angles between neighbors
            angles = [calculate_angle(points[i], points[n1], points[n2]) for n1, n2 in combinations(neighbors, 2)]

            # Compute minimum and maximum neighbor angle
            min_angle = np.min(angles)
            max_angle = np.max(angles)

            # Update the dataframe
            df.loc[i, 'min_relative_angle'] = min_angle
            df.loc[i, 'max_relative_angle'] = max_angle
        else:  # For nodes with one or no neighbors, we'll set the angles to 0
            df.loc[i, 'min_relative_angle'] = 0
            df.loc[i, 'max_relative_angle'] = 0

    #print(df['average_neighbor_distance'])
    #print("degree",df['degree'])
    node_features = torch.tensor(df[['relative_x', 'relative_y', 'degree','average_neighbor_distance','min_n_dist','max_n_dist', 'min_relative_angle', 'max_relative_angle']].values, dtype=torch.float)

    labels = torch.tensor(df['label'].values.astype(int), dtype=torch.long)
    data = Data(x=node_features, edge_index=edge_index, edge_attr=edge_attr, y=labels)

    data_list.append(data)

#%%
# Split your data into a training set and a validation set
train_size = int(0.75 * len(data_list))  # Assuming you want 80% of the data for training
val_size = len(data_list) - train_size
train_dataset, val_dataset = random_split(data_list, [train_size, val_size])

# Create a DataLoader for the training set and the validation set
train_loader = DataLoader(train_dataset, batch_size=32)
val_loader = DataLoader(val_dataset, batch_size=32)

# Define your model, loss function and optimizer
model = GCN(num_node_features=8, num_edge_features=3, num_classes=2)
loss_fn = torch.nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

# Keep track of losses for each epoch
train_losses = []
val_losses = []

# Training function
def train(epoch):
    model.train()
    total_loss = 0
    for batch in tqdm(train_loader, desc=f"Training batches (Epoch {epoch+1})"):
        out = model(batch)
        loss = loss_fn(out, batch.y)
        total_loss += loss.item()
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    avg_loss = total_loss / len(train_loader)
    train_losses.append(avg_loss)  # Append to list
    print(f"Train Epoch {epoch+1}, Loss: {avg_loss:.4f}")

# Validation function
def validate(epoch):
    model.eval()
    total_loss = 0
    with torch.no_grad():
        for batch in val_loader:
            out = model(batch)
            loss = loss_fn(out, batch.y)
            total_loss += loss.item()
    avg_loss = total_loss / len(val_loader)
    val_losses.append(avg_loss)  # Append to list
    print(f"Val Epoch {epoch+1}, Loss: {avg_loss:.4f}")

# Initialize best validation loss with a high value
best_val_loss = float('inf')

# Training and validation function
for epoch in range(300):  # Number of epochs
    train(epoch)
    validate(epoch)

    # Save the model weights at the best epoch
    if val_losses[-1] < best_val_loss:
        best_val_loss = val_losses[-1]
        torch.save(model.state_dict(), 'best_model_weights.pt')

# Load the best model weights
model.load_state_dict(torch.load('best_model_weights.pt'))

# Plotting losses
plt.plot(range(1, len(train_losses)+1), train_losses, label='Train')
plt.plot(range(1, len(val_losses)+1), val_losses, label='Validation')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()

# %%
from torch_geometric.data import Batch
# Select the point_set you want to plot
point_set_idx = 94 # or whichever point_set you want to plot
point_set = val_dataset[point_set_idx]

# Convert point_set to a batch for model input
point_set = point_set.to('cpu')  # Assuming you are using CPU for inference
point_set = Batch.from_data_list([point_set])

# Run the point_set through the model to get the predicted labels
model.eval()  # Ensure model is in evaluation mode
with torch.no_grad():
    out = model(point_set)
predicted_labels = torch.argmax(out, dim=1).numpy()

# Extract the points and plot
points = point_set.x.numpy()
plt.scatter(points[:, 0], points[:, 1], c=predicted_labels)
plt.title(f'Point Set {point_set_idx} Predicted Labels')
plt.axis("off")
plt.axis("equal")
plt.show()
# %%


import dataclasses

import torch

from ...base import DeviceArg, LearnableConfig, register_learnable
from ...constants import ActionSpace
from ...models.builders import (
    create_continuous_q_function,
    create_normal_policy,
)
from ...models.encoders import EncoderFactory, make_encoder_field
from ...models.optimizers import OptimizerFactory, make_optimizer_field
from ...models.q_functions import QFunctionFactory, make_q_func_field
from ...models.torch import Parameter
from ...types import Shape
from .base import QLearningAlgoBase
from .torch.awac_impl import AWACImpl
from .torch.sac_impl import SACModules

__all__ = ["AWACConfig", "AWAC"]

@dataclasses.dataclass()
class AWACConfig(LearnableConfig):
    r"""Config of Advantage Weighted Actor-Critic algorithm.

    AWAC is a TD3-based actor-critic algorithm that enables efficient
    fine-tuning where the policy is trained with offline datasets and is
    deployed to online training.

    The policy is trained as a supervised regression.

    .. math::

        J(\phi) = \mathbb{E}_{s_t, a_t \sim D}
            [\log \pi_\phi(a_t|s_t)
                \exp(\frac{1}{\lambda} A^\pi (s_t, a_t))]

    where :math:`A^\pi (s_t, a_t) = Q_\theta(s_t, a_t) -
    Q_\theta(s_t, a'_t)` and :math:`a'_t \sim \pi_\phi(\cdot|s_t)`

    The key difference from AWR is that AWAC uses Q-function trained via TD
    learning for the better sample-efficiency.

    References:
        * `Nair et al., Accelerating Online Reinforcement Learning with Offline
          Datasets. <https://arxiv.org/abs/2006.09359>`_

    Args:
        observation_scaler (d3rlpy.preprocessing.ObservationScaler):
            Observation preprocessor.
        action_scaler (d3rlpy.preprocessing.ActionScaler): Action preprocessor.
        reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
        actor_learning_rate (float): Learning rate for policy function.
        critic_learning_rate (float): Learning rate for Q functions.
        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
            Optimizer factory for the actor.
        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
            Optimizer factory for the critic.
        actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
            Encoder factory for the actor.
        critic_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
            Encoder factory for the critic.
        q_func_factory (d3rlpy.models.q_functions.QFunctionFactory):
            Q function factory.
        batch_size (int): Mini-batch size.
        gamma (float): Discount factor.
        tau (float): Target network synchronization coefficiency.
        lam (float): :math:`\lambda` for weight calculation.
        n_action_samples (int): Number of sampled actions to calculate
            :math:`A^\pi(s_t, a_t)`.
        n_critics (int): Number of Q functions for ensemble.
    """
    actor_learning_rate: float = 3e-4
    critic_learning_rate: float = 3e-4
    actor_optim_factory: OptimizerFactory = make_optimizer_field()
    critic_optim_factory: OptimizerFactory = make_optimizer_field()
    actor_encoder_factory: EncoderFactory = make_encoder_field()
    critic_encoder_factory: EncoderFactory = make_encoder_field()
    q_func_factory: QFunctionFactory = make_q_func_field()
    batch_size: int = 1024
    gamma: float = 0.99
    tau: float = 0.005
    lam: float = 1.0
    n_action_samples: int = 1
    n_critics: int = 2

    def create(self, device: DeviceArg = False) -> "AWAC":
        return AWAC(self, device)

    @staticmethod
    def get_type() -> str:
        return "awac"

class AWAC(QLearningAlgoBase[AWACImpl, AWACConfig]):
    def inner_create_impl(
        self, observation_shape: Shape, action_size: int
    ) -> None:
        policy = create_normal_policy(
            observation_shape,
            action_size,
            self._config.actor_encoder_factory,
            min_logstd=-6.0,
            max_logstd=0.0,
            use_std_parameter=True,
            device=self._device,
        )
        q_funcs, q_func_forwarder = create_continuous_q_function(
            observation_shape,
            action_size,
            self._config.critic_encoder_factory,
            self._config.q_func_factory,
            n_ensembles=self._config.n_critics,
            device=self._device,
        )
        targ_q_funcs, targ_q_func_forwarder = create_continuous_q_function(
            observation_shape,
            action_size,
            self._config.critic_encoder_factory,
            self._config.q_func_factory,
            n_ensembles=self._config.n_critics,
            device=self._device,
        )

        actor_optim = self._config.actor_optim_factory.create(
            policy.named_modules(), lr=self._config.actor_learning_rate
        )
        critic_optim = self._config.critic_optim_factory.create(
            q_funcs.named_modules(), lr=self._config.critic_learning_rate
        )

        dummy_log_temp = Parameter(torch.zeros(1, 1))
        dummy_log_temp.to(self._device)
        modules = SACModules(
            policy=policy,
            q_funcs=q_funcs,
            targ_q_funcs=targ_q_funcs,
            log_temp=dummy_log_temp,
            actor_optim=actor_optim,
            critic_optim=critic_optim,
            temp_optim=None,
        )

        self._impl = AWACImpl(
            observation_shape=observation_shape,
            action_size=action_size,
            modules=modules,
            q_func_forwarder=q_func_forwarder,
            targ_q_func_forwarder=targ_q_func_forwarder,
            gamma=self._config.gamma,
            tau=self._config.tau,
            lam=self._config.lam,
            n_action_samples=self._config.n_action_samples,
            device=self._device,
        )

    def get_action_type(self) -> ActionSpace:
        return ActionSpace.CONTINUOUS

register_learnable(AWACConfig)


"""Парсит 'https://smart-lab.ru/q/shares/' при помощи библиотек
requests и BeautifulSoup. Из скаченного материала выбираются названия эмитентов,
очищаются от "мусора" и записываются в базу MySQL запросом UPDATE."""

import requests
from bs4 import BeautifulSoup as bs
import re
from updt_cmpn_names import update_names

"""Получаем данные с сайта 'https://smart-lab.ru/q/shares/' и формируем из них
bs-объект"""
url = 'https://smart-lab.ru/q/shares/'
req = requests.get(url)
soup = bs(req.text, features = 'lxml')
soup_str = str(soup)

"""Записываем bs-объект в файл для последующего писка нужной информации"""
with open('text_sl.txt', 'w') as file:
    file.write(soup_str)

"""Находим все ссылки, содержащие названия эмитентов"""
res_link_lst = []
with open('text_sl.txt', 'r') as file_prs:
    for line in file_prs:
        item = re.findall('/forum.+?\s', line)
        res_link_lst.append(item)

"""Очищаем список имен эмитентов от пустых значений"""
clean_lst = [value for value in res_link_lst if value != []]

"""Формируем список имен эмитентов для занесения в базу данных MySQL"""
name_lst = []
for item in clean_lst:
    name = re.findall('>[А-Яа-яa-zA-Z0-9-.+]+.<?', item[0])
    name_lst.append(name)
clean_name_lst0 = [value for value in name_lst if value != []]
clean_name_lst1 = []
for item in clean_name_lst0:
    name = item[0]
    clean_name = name[1:-1]
    clean_name_lst1.append(clean_name)
del clean_name_lst1[:2]
del clean_name_lst1[-3:]

"""При помощи функции 'update_names', содержащейся в файле 'updt_cmpn_names.py'
обновляем названия имен в таблице MySQL"""
update_names(clean_name_lst1)


from django.urls import path

from .views import (
    GeneralView, AvtoCourseDetailView, TeacherDetailView, CourseFormView,
    FeedbackFormView, AvtoSchoolDetailView,
)

urlpatterns = [
    path('', GeneralView.as_view(), name="index"),
    path('avtocourse/<int:pk>/', AvtoCourseDetailView.as_view(),
         name="avto_course_detail"),
    path('avtoschool/<int:pk>/', AvtoSchoolDetailView.as_view(),
         name="avto_school_detail"),
    path('teacher/<int:pk>/', TeacherDetailView.as_view(),
         name="teacher_detail"),
    path('feedback-form', FeedbackFormView.as_view(),
         name="feedback_form"),
    path('course-form', CourseFormView.as_view(),
         name="course_form"),
]


import json
import string
from threading import Lock

import requests
import time
from zeroconf import ServiceBrowser, ServiceStateChange
from zeroconf import Zeroconf

IDENTIFIER = "restsysmon"
HA_URL = "http://192.168.1.8:8123"
HA_TOKEN = "..."

class HASensorPusher:
    def __init__(self, identifier=IDENTIFIER):
        self.zero = None
        self.browser = None
        self.nodes = {}
        self.running = False
        self.identifier = identifier.encode("utf-8")
        self.ttl = 10  # seconds between rounds of sensor readings
        self.sleep_time = 0.3  # seconds between individual sensor readings
        self.lock = Lock()

    @staticmethod
    def ha_binary_sensor_update(device_id, state="on", attrs=None):
        device_id = device_id.replace(" ", "_")
        for s in string.punctuation:
            device_id = device_id.replace(s, "_")
        attrs = attrs or {"friendly_name": device_id,
                          "state_color": True,
                          "device_class": "presence"}

        print("updating binary sensor - ", device_id)
        try:
            response = requests.post(
                f"{HA_URL}/api/states/binary_sensor.{IDENTIFIER}_{device_id}",
                headers={
                    "Authorization": f"Bearer {HA_TOKEN}",
                    "content-type": "application/json",
                },
                data=json.dumps({"state": state, "attributes": attrs}),
            )
            print(response.text)
        except:
            print("failed to push data to HA")

    @staticmethod
    def ha_sensor_update(device_id, state="on", attrs=None):
        attrs = attrs or {"friendly_name": device_id,
                          "state_color": True}
        print("updating sensor - ", device_id)
        try:
            response = requests.post(
                f"{HA_URL}/api/states/sensor.{IDENTIFIER}_{device_id}",
                headers={
                    "Authorization": f"Bearer {HA_TOKEN}",
                    "content-type": "application/json",
                },
                data=json.dumps({"state": state, "attributes": attrs}),
            )
            print(response.text)
        except:
            print("failed to push data to HA")

    def on_new_node(self, node):
        device_id = node["name"]
        print("found", node)
        self.on_node_update(node)
        self.get_readings()
        self.get_info(device_id)  # one time sensor readings

    def on_lost_node(self, node):
        print("lost", node)
        self.on_node_update(node)

    def on_node_update(self, node):
        device_id = node["name"]
        self.nodes[device_id] = node
        self.ha_binary_sensor_update(device_id, state=node["state"],
                                     attrs={"friendly_name": node["name"],
                                            "state_color": True,
                                            "device_class": "presence"})

    def on_service_state_change(self, zeroconf, service_type, name,
                                state_change):
        info = zeroconf.get_service_info(service_type, name)
        if info and info.properties:
            for key, value in info.properties.items():
                if key == b"type" and value == self.identifier:
                    host = info._properties[b"host"].decode("utf-8")
                    port = info._properties[b"port"].decode("utf-8")
                    name = info._properties[b"name"].decode("utf-8")
                    node = {"host": host, "port": port, "name": name}
                    node["last_seen"] = time.time()
                    if state_change is ServiceStateChange.Added:
                        node["state"] = "on"
                        self.on_new_node(node)
                    elif ServiceStateChange.Removed:
                        node["state"] = "off"
                        self.on_lost_node(node)
                    else:
                        node["state"] = "on"
                        self.on_node_update(node)

    def start(self):
        self.zero = Zeroconf()
        self.browser = ServiceBrowser(self.zero, "_http._tcp.local.",
                                      handlers=[self.on_service_state_change])
        self.running = True
        while self.running:
            try:
                self.get_readings()
                time.sleep(self.ttl)
            except KeyboardInterrupt:
                break
        self.stop()

    def stop(self):
        if self.zero:
            self.zero.close()
        self.zero = None
        self.browser = None
        self.running = False

    def get_info(self, device_id):
        data = self.nodes[device_id]
        url = f"{data['host']}:{data['port']}"
        readings = {
            "os": f"{url}/os_name",
            "boot_time": f"{url}/boot_time",
            "system": f"{url}/system",
            "release": f"{url}/release",
            "machine": f"{url}/machine",
            "architecture": f"{url}/architecture",
            "cpu_count": f"{url}/cpu_count",
            "fans": f"{url}/fans",
            "network_interfaces": f"{url}/network_interfaces",
            "external_ip": f"{url}/external_ip",
            "pulseaudio_version": f"{url}/pulseaudio/version",
            "is_systemd": f"{url}/is_systemd",
            "is_dbus": f"{url}/is_dbus",
            "cpu_freq_max": f"{url}/cpu_freq_max",
            "cpu_freq_min": f"{url}/cpu_freq_min",
            "disk_total": f"{url}/disk_total",
            "pulseaudio_hostname": f"{url}/pulseaudio/hostname",
            "memory_total": f"{url}/memory_total"
        }
        for sensor_name, url in readings.items():
            self.read_sensor(device_id, sensor_name, url)
            time.sleep(self.sleep_time)

    def read_sensor(self, device_id, sensor_name, url):
        sensor_id = f"{device_id}_{sensor_name}"
        try:
            res = requests.get("http://" + url)
            try:
                res = res.json()
            except:
                res = res.text
            try:  # numeric
                if "/is_" in url:
                    res = str(res) == "1"
                    self.ha_binary_sensor_update(sensor_id, state="on" if res else "off",
                                                 attrs={"friendly_name": sensor_name,
                                                        "state_color": True,
                                                        "device_class": "presence"})
                elif url.endswith("_count"):
                    res = int(res)
                    a = {"friendly_name": sensor_name,
                         "state_color": True}
                    self.ha_sensor_update(sensor_id, state=str(res), attrs=a)
                else:
                    res = float(res)  # numeric reading
                    a = {"friendly_name": sensor_name,
                         "state_color": True}
                    if "temperature" in url:
                        a["unit_of_measurement"] = "°C"
                    self.ha_sensor_update(sensor_id, state=str(res), attrs=a)

            except Exception as e:
                # text
                if isinstance(res, list):
                    if sensor_id.endswith("pulseaudio_list_cards"):
                        for e in res:
                            sensor_id = f"{device_id}_pulseaudio_card_{e['index']}"
                            f = e.get("card_name") or e.get("name") or sensor_id
                            self.ha_binary_sensor_update(sensor_id, state="on",
                                                         attrs={"friendly_name": f.lower().strip(),
                                                                "state_color": True,
                                                                "device_class": "presence"})
                    elif sensor_id.endswith("pulseaudio_list_sources"):
                        for e in res:
                            sensor_id = f"{device_id}_pulseaudio_source_{e['index']}"
                            self.ha_binary_sensor_update(sensor_id, state="on",
                                                         attrs={"friendly_name": e.get("name") or sensor_id,
                                                                "state_color": True,
                                                                "device_class": "presence"})
                    elif sensor_id.endswith("pulseaudio_list_sinks"):
                        for e in res:
                            sensor_id = f"{device_id}_pulseaudio_sink_{e['index']}"
                            self.ha_binary_sensor_update(sensor_id, state="on",
                                                         attrs={"friendly_name": e.get("name") or sensor_id,
                                                                "state_color": True,
                                                                "device_class": "presence"})
                    elif sensor_id.endswith("pulseaudio_list_input_sinks"):
                        # streams of interest, even if not currently playing will report a status
                        APPS = [
                            "netflix",
                            "spotify"
                        ]
                        for app in APPS:

                            if any(app == _["application"] for _ in res):
                                # app names handled below, we just want streams here
                                # eg, detect netflix in firefox via pulseaudio playback
                                continue

                            if any(app == _.get('media_name', "").lower() for _ in res):
                                # detected application running via pulseaudio playback
                                state = "on"
                                mute_state = "off"  # TODO - check the playback info
                            else:
                                state = mute_state = "off"

                            sensor_id = f"{device_id}_pulseaudio_stream_{app}"
                            self.ha_binary_sensor_update(sensor_id, state=state,
                                                         attrs={"friendly_name": app,
                                                                "state_color": True,
                                                                "device_class": "presence"})
                            sensor_id = f"{device_id}_pulseaudio_stream_{app}_muted"
                            self.ha_binary_sensor_update(sensor_id, state=mute_state,
                                                         attrs={"friendly_name": app + " muted",
                                                                "state_color": True,
                                                                "device_class": "presence"})

                        for e in res:
                            sensor_id = f"{device_id}_pulseaudio_stream_{e['application']}"
                            f = e.get('media_name') or e.get('name') or e.get('application')
                            self.ha_binary_sensor_update(sensor_id, state="on" if e.get("playing") else "off",
                                                         attrs={"friendly_name": f.lower().strip(),
                                                                "state_color": True,
                                                                "device_class": "presence"})

                            sensor_id = f"{device_id}_pulseaudio_stream_{e['application']}_muted"
                            f = e.get('media_name') or e.get('name') or e.get('application')
                            self.ha_binary_sensor_update(sensor_id, state="on" if e.get("mute") else "off",
                                                         attrs={"friendly_name": f.lower().strip() + " muted",
                                                                "state_color": True,
                                                                "device_class": "presence"})

                            sensor_id = f"{device_id}_pulseaudio_stream_{e['application']}_sink"
                            f = e.get('media_name') or e.get('name') or e.get('application')
                            self.ha_sensor_update(sensor_id, state=str(e["sink"]),
                                                  attrs={"friendly_name": f.lower().strip() + " sink",
                                                         "state_color": True,
                                                         "device_class": "presence"})
                    elif sensor_id.endswith("network_interfaces"):
                        for e in res:
                            sensor_id = f"{device_id}_network_if_{e}"
                            self.ha_binary_sensor_update(sensor_id, state="on",
                                                         attrs={"friendly_name": e,
                                                                "state_color": True,
                                                                "device_class": "presence"})
                    else:
                        print("## TODO", sensor_id, res)
                    pass
                elif isinstance(res, dict):
                    # print("## TODO", sensor_id, res)
                    pass
                else:
                    a = {"friendly_name": sensor_name,
                         "state_color": True}
                    self.ha_sensor_update(sensor_id, state=str(res), attrs=a)
        except:
            return False
        return True

    def get_readings(self):
        with self.lock:
            for device_id, data in dict(self.nodes).items():
                alive = False
                if data["state"] == "on":
                    url = f"{data['host']}:{data['port']}"
                    readings = {
                        "swap_usage": f"{url}/swap_usage",
                        "swap_total": f"{url}/swap_total",
                        "disk_usage": f"{url}/disk_usage",
                        "external_ip": f"{url}/external_ip",
                        "is_kdeconnect": f"{url}/is_kdeconnect",
                        "is_pulseaudio": f"{url}/is_pulseaudio",
                        "is_pipewire": f"{url}/is_pipewire",
                        "is_plasma": f"{url}/is_plasma",
                        "is_hivemind": f"{url}/is_hivemind",
                        "is_ovos": f"{url}/is_ovos",
                        "is_spotify": f"{url}/is_spotify",
                        "is_firefox": f"{url}/is_firefox",
                        "is_upmpdcli": f"{url}/is_upmpdcli",
                        "is_minidlna": f"{url}/is_minidlna",
                        "pulseaudio_channel_count": f"{url}/pulseaudio/channel_count",
                        "pulseaudio_default_sink": f"{url}/pulseaudio/default_sink",
                        "pulseaudio_default_source": f"{url}/pulseaudio/default_source",
                        "pulseaudio_list_cards": f"{url}/pulseaudio/list_cards",
                        "pulseaudio_list_sources": f"{url}/pulseaudio/list_sources",
                        "pulseaudio_list_sinks": f"{url}/pulseaudio/list_sinks",
                        "pulseaudio_list_input_sinks": f"{url}/pulseaudio/list_input_sinks",
                        "cpu_usage": f"{url}/cpu_usage",
                        "cpu_freq": f"{url}/cpu_freq",
                        "cpu_temperature": f"{url}/cpu_temperature",
                        "memory_usage": f"{url}/memory_usage",
                        "disk_percent": f"{url}/disk_percent",
                        "battery_percent": f"{url}/battery_percent",
                        "pulseaudio_now_playing": f"{url}/pulseaudio/now_playing",
                        "pulseaudio_is_playing": f"{url}/pulseaudio/is_playing",
                        "pulseaudio_bluez_active": f"{url}/pulseaudio/is_bluez_active",
                        "pulseaudio_bluez_connected": f"{url}/pulseaudio/is_bluez_connected",
                        "brightness": f"{url}/brightness"
                    }
                    for sensor_name, url in readings.items():
                        alive = self.read_sensor(device_id, sensor_name, url) or alive
                        time.sleep(self.sleep_time)

                data["state"] = alive
                self.on_node_update(data)

if __name__ == "__main__":
    ha = HASensorPusher()
    ha.start()
    while True:
        try:
            time.sleep(1)
        except KeyboardInterrupt:
            break
    ha.stop()


import numpy as np
from util import load_data
from util import create_logfile
import csv
import pandas as pd
from sklearn.decomposition import PCA
from vanilla_ann import DataGenerator, Prediction_Model, train_model
import keras
from imblearn.keras import BalancedBatchGenerator
from imblearn.under_sampling import RandomUnderSampler

if __name__ == '__main__':

    (X_train, y_train), (X_test, y_test), (X_val, y_val) = load_data()
    y_train_onehot = keras.utils.to_categorical(y_train, len(np.unique(y_train.values.astype('float32'))))

    num_features = 6
    layers = [64, 64, 64]
    model_name = 'ann-pca'
    num_outputs = len(np.unique(y_train.astype(int)))
    batch_size = 128

    pca = PCA(n_components=num_features)
    pca.fit(X_train)
    comp_columns = [f'comp {i}' for i in range(len(pca.components_))]

    X_train = pd.DataFrame(pca.transform(X_train), columns=comp_columns)
    X_val = pd.DataFrame(pca.transform(X_val), columns=comp_columns)
    X_test = pd.DataFrame(pca.transform(X_test), columns=comp_columns)

    model = Prediction_Model((num_features, layers, num_outputs, model_name))

    # Parameters
    params = {'num_features': num_features,
              'batch_size': batch_size,
              'shuffle': True,
              'num_outputs': num_outputs}

    # training_generator = DataGenerator(X_train, y_train, **params)
    sampler = RandomUnderSampler(random_state=42)
    training_generator = BalancedBatchGenerator(
        X_train, y_train_onehot, sampler=sampler, batch_size=batch_size, random_state=42)
    validation_generator = DataGenerator(X_val, y_val, **params)
    test_generator = DataGenerator(X_test, y_test, **params)

    train_model(
        model,
        training_generator,
        validation_generator,
        model_name,
        num_epochs=30
    )

    model.load_weights(f'log/{model_name}/weights.hdf5')
    log_fname = f'log/repres_losses.csv'

    create_logfile(log_fname)

    scores = model.evaluate_generator(test_generator)

    with open(log_fname, 'a') as f:
        writer = csv.writer(f)
        writer.writerow([model_name] + scores)

    # print('test loss', model_name, scores)


from django.shortcuts import render, redirect
import datetime
from bs4 import BeautifulSoup
import requests
# Create your views here.

# time on now
time_now = datetime.datetime.now()

alert_message = False

# for crawling
req = requests.get(
    'https://search.naver.com/search.naver?sm=top_hty&fbm=1&ie=utf8&query=%EC%97%84%EB%B3%B5%EB%8F%99+%EA%B4%80%EA%B0%9D%EC%88%98')
html = req.text
soup = BeautifulSoup(html, 'html.parser')
info = soup.find("div", class_="movie_info section").find("div", class_="info_main").find(
    "dl", class_="desc_detail").find("span", class_="property").text
noa = info.replace("명", "", 1).replace(",", "", 1)

def forPrint(array):
    for item in array:
        return item.text

def home(request):

    # for crawling
    req = requests.get(
        'https://search.naver.com/search.naver?where=nexearch&sm=tab_etc&query=%EB%B0%95%EC%8A%A4%EC%98%A4%ED%94%BC%EC%8A%A4')
    html = req.text
    soup = BeautifulSoup(html, 'html.parser')
    my_titles_soup = soup.select(
        '#main_pack > div.content_search.section._cs_movie_box_office > div > div.contents03_sub > div > div.movie_rank_wrap > div.movie_audience_ranking._main_panel.v2 > div:nth-child(1) > ul > li > div.movie_info > dl > dd:nth-child(2)'
    )

    for i in range(len(my_titles_soup)):
        my_titles_soup[i] = my_titles_soup[i].text

    my_titles2_soup = soup.select(
        '#main_pack > div.content_search.section._cs_movie_box_office > div > div.contents03_sub > div > div.movie_rank_wrap > div.movie_audience_ranking._main_panel.v2 > div:nth-child(1) > ul > li > div > a > div > strong'
    )

    for i in range(len(my_titles2_soup)):
        my_titles2_soup[i] = my_titles2_soup[i].text

    my_titlesday_soup = soup.select(
        '#main_pack > div.content_search.section._cs_movie_box_office > div > div.contents03_sub > div > div.movie_rank_wrap > div.movie_audience_ranking._main_panel.v2 > div:nth-child(1) > ul > li > div.movie_info > dl > dd:nth-child(4)'
    )
    my_titlesday = []
    for i in range(len(my_titlesday_soup)):
        my_titlesday.append(round(float(my_titlesday_soup[i].text.replace(
            "명", "").replace(",", "")) / int(noa), 2))

    my_titlesall_soup = soup.select(
        '#main_pack > div.content_search.section._cs_movie_box_office > div > div.contents03_sub > div > div.movie_rank_wrap > div.movie_audience_ranking._main_panel.v2 > div:nth-child(1) > ul > li > div.movie_info > dl > dd:nth-child(6)'
    )
    my_titlesall = []
    for i in range(len(my_titlesall_soup)):
        my_titlesall.append(round(float(my_titlesall_soup[i].text.replace(
            "명", "").replace(",", "")) / int(noa), 2))

    my_image = soup.select(
        '#main_pack > div.content_search.section._cs_movie_box_office > div > div.contents03_sub > div > div.movie_rank_wrap > div.movie_audience_ranking._main_panel.v2 > div:nth-child(1) > ul > li > div.thumb > a > img'
    )

    my_link = soup.select(
        '#main_pack > div.content_search.section._cs_movie_box_office > div > div.contents03_sub > div > div.movie_rank_wrap > div.movie_audience_ranking._main_panel.v2 > div:nth-child(1) > ul > li > div.thumb > a'
    )

    movies = []
    for i in range(len(my_image)):
        movies.append({'movie_title': my_titles2_soup[i], 'infor': my_titles_soup[i], 'day_ubd': "일간 UBD : " + str(my_titlesday[i]), 'total_ubd': " 누적 UBD : " + str(my_titlesall[i]),
                       'image': my_image[i].get("src"), 'link': my_link[i].get("href")})
    # return render(request, 'index.html', {'now': time_now, 'my_titles_res': my_titles_result, 'my_image': my_image})
    return render(request, 'index.html', {'now': time_now, 'movies': movies, 'message': alert_message})

def ubdresult(request):

    if request.GET['ubdint'] == "":
        alert_message = True
        return redirect('home')

    ubd_int = float(request.GET['ubdint']) / int(noa)
    ubd_rep = round(ubd_int, 3)

    return render(request, 'ubdresult.html', {'ubdrep': ubd_rep, 'now': time_now})

def ubdresult_rev(request):
    if request.GET['ubdint_rev'] == "":
        alert_message = True
        return redirect('home')

    ubd_rev = float(request.GET['ubdint_rev']) * int(noa)
    ubd_revrep = round(ubd_rev, 3)

    return render(request, 'ubdresult_rev.html', {'ubdrev': ubd_revrep, 'now': time_now})

def base_layout(request):
    template = 'base.html'
    return render(request, template)


import logbook
from report import Report
from utils.money import decimal_to_string

class ReceiptsReport(Report):
    report_type = "receipts"

    def aggregate(self, report_id):
        logbook.info("Get receipts aggregation for {}", report_id)
        from model import AccountHistory, User

        result = []
        users = {user.user_id: user.name for user in User.query}
        users[None] = ""
        for row in AccountHistory.report(report_id.start, report_id.end):
            r = {"email": row[0].email,
                 "date": row[1].date,
                 "amount": decimal_to_string(row[1].delta),
                 "currency": row[2].currency,
                 "comment": row[1].comment,
                 "user": users[row[1].user_id]}
            result.append(r)
        return result


import datetime
import functools
import inspect
import os
import random
import re
import shlex
import shutil
import warnings
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Set, Union

import nemo_launcher.utils.job_utils as job_utils
import yaml
from nemo_launcher.core.logger import logger
from omegaconf import DictConfig, OmegaConf

NEMO_LAUNCHER_CI = os.getenv("NEMO_LAUNCHER_CI", "False").lower() in ("true", "t", "1")
NEMO_LAUNCHER_DEBUG = os.getenv("NEMO_LAUNCHER_DEBUG", "False").lower() in (
    "true",
    "t",
    "1",
)
NEMO_LAUNCHER_MEMORY_MEASURE = os.getenv(
    "NEMO_LAUNCHER_MEMORY_MEASURE", "False"
).lower() in ("true", "t", "1")

class AutoLauncher:
    """
    Automatic launcher class. It will create a launcher based on input cluster name.
    """

    def __init__(
        self,
        folder: Union[str, Path],
        job_name: str,
        cluster: Optional[str] = None,
        **kwargs: Any,
    ) -> None:
        self.cluster = cluster or self.which()
        self.cluster = self.cluster.lower()

        launchers = self.get_launchers()
        if self.cluster not in launchers:
            raise ValueError(
                f"AutoLauncher doesn't know any cluster named {self.cluster}"
            )

        self._launcher = launchers[self.cluster](folder, job_name, **kwargs)

    def launch(self, command_groups: List[List[str]]) -> str:
        """
        Use the launcher to launch the command groups.

        :param List[List[str]] command_groups: Command groups to launch with
        :return: job id on slurm based system otherwise empty string
        :rtype: str
        """
        job_id = self._launcher.launch(command_groups)
        return job_id

    @staticmethod
    def which() -> str:
        """Returns what the detected cluster is"""
        raise NotImplementedError

    @staticmethod
    def get_launchers():
        """Returns supported launchers as a dictionary from launcher name to launcher class"""
        return {
            "bcm": SlurmLauncher,
            "bcp": BCPLauncher,
            "interactive": InteractiveLauncher,
            "k8s": K8SLauncher,
        }

class Launcher:
    """Base launcher class"""

    def __init__(self, folder: Union[Path, str], job_name: str):
        self.folder = folder
        self.job_name = job_name

    def launch(self, command_groups: List[List[str]]) -> str:
        """
        Use the launcher to launch the command groups.

        :param List[List[str]] command_groups: Command groups to launch with
        :return: job id on slurm based system otherwise empty string
        :rtype: str
        """
        submission_file_path = self._make_submission_file(command_groups)
        logger.info(
            f"Job {self.job_name} submission file created at '{submission_file_path}'"
        )

        job_id = ""
        if not NEMO_LAUNCHER_DEBUG:
            job_id = self._submit_command(submission_file_path)
            if job_id:
                logger.info(f"Job {self.job_name} submitted with Job ID {job_id}")
                with open(self.folder / "launcher.log", "w") as f:
                    f.write(f"Submitted batch job {job_id}")
        else:
            job_id = str(random.randint(10000, 99999))
            logger.info(
                f"[DEBUG] Job {self.job_name} submitted with FAKE Job ID {job_id}"
            )

        return job_id

    def _submit_command(self, submission_file_path: Path) -> str:
        """Submits a set of command groups to the cluster"""
        raise NotImplementedError

    def _make_submission_file(self, command_groups: List[List[str]]) -> Path:
        """
        Make a submission script file, as following
            on interactive cluster, it's a bash file, trigger with bash.
            on slurm cluster, it's a slurm script file, trigger with sbatch.
            on BCP cluster, it's a BCP script file, trigger with bash.
            on k8s cluster, it's a Helm chart, triggered with helm.

        :param List[List[str]] command_groups: Command groups to launch with
        :return: job id on slurm based system otherwise empty string
        :rtype: str
        """
        job_paths = job_utils.JobPaths(folder=self.folder, job_name=self.job_name)
        folder = job_paths.folder
        folder.mkdir(parents=True, exist_ok=True)

        submission_file_path = job_paths.submission_file
        with submission_file_path.open("w") as f:
            f.write(self._make_submission_file_text(command_groups))
        return submission_file_path

class InteractiveLauncher(Launcher):
    """
    Interactive job launcher
    This class is used to hold the parameters to run a job on an interactive node (single node only).
    In practice, it will create a batch file in the specified directory for the job and
    trigger the job with `bash` command.

    :param Union[Path, str] folder: folder for storing job submission/output and logs.
    :param str job_name: Name of the job, used as job folder name
    :param Any **kwargs: Parse other cluster parameters required for interactive running
    """

    def __init__(self, folder: Union[Path, str], job_name: str, **kwargs: Any) -> None:
        super().__init__(folder, job_name)
        self.parameters = kwargs

    def _submit_command(self, submission_file_path: Path) -> str:
        """Launch the submission command"""
        command_list = self._make_submission_command(submission_file_path)
        # run
        job_utils.CommandFunction(
            command_list, ret_stdout=False, verbose=False
        )()  # explicit errors
        return ""

    @staticmethod
    def _make_submission_command(submission_file_path: Path) -> List[str]:
        """Make a command to trigger submission script. On interactive cluster, the script is triggerred with bash"""
        return ["bash", str(submission_file_path)]

    def _make_submission_file_text(self, command_groups: List[List[str]]) -> str:
        """
        Given the command groups, generate submission script file's text.
        Command groups is a list of command group. A command group is defined as:
              0. Command group is a list of command strings
              1. Each command group occupies one bcprun, srun or bash
              2. Each command group eventually has multiple commands connected by ";"
        On interactive cluster, multi-gpu python scripts are launched with `torchrun --nproc_per_node=??`

        :param List[List[str]] command_groups: Command groups to launch with
        :return: submission script file's text
        :rtype: str
        """
        nodes = self.parameters.get("nodes", 1)
        ntasks_per_node = self.parameters.get("ntasks_per_node", 1)
        assert nodes == 1, "Multi-node is not supported in interactive mode."

        paths = job_utils.JobPaths(folder=self.folder, job_name=self.job_name)
        time_tag = datetime.datetime.now().strftime("%m%d_%H%M%S")
        stdout = str(paths.stdout).replace("_%j", f"_{time_tag}")

        # now create
        lines = ["#!/bin/bash", ""]

        # environment setup:
        setup = self.parameters.get("setup", None)
        if setup is not None:
            lines += ["", "# setup"] + setup

        for group_ind, command_group in enumerate(command_groups):
            command = ";\n  ".join(command_group)
            command = command.replace(
                "python3 -u", f"torchrun --nproc_per_node={ntasks_per_node}"
            )

            lines += [
                "",
                f"# command {group_ind + 1}",
                f'bash -c "',
                f'  {command} " 2>&1 | tee -a {stdout}',
                "",
            ]
        return "\n".join(lines)

class BCPLauncher(Launcher):
    """
    BCP job launcher
    This class is used to hold the parameters to run a job on BCP platform.
    In practice, it will create a batch file in the specified directory for the job
    and trigger the job with `bash` command.

    :param Union[Path, str] folder: folder for storing job submission/output and logs.
    :param str job_name: Name of the job, used as job folder name
    :param Any **kwargs: Parse other cluster parameters required for BCP running,
        including `nodes`, `ntasks_pernode`, `bcp_launcher`, etc.
    """

    def __init__(self, folder: Union[Path, str], job_name: str, **kwargs: Any) -> None:
        super().__init__(folder, job_name)
        self.parameters = kwargs
        self.parameters = self._convert_parameters(self.parameters)

    @classmethod
    def _equivalence_dict(cls):
        return {
            "name": "job_name",
            "nodes": "nnodes",
            "tasks_per_node": "npernode",
            "ntasks_per_node": "npernode",
            "bcp_launcher": "launcher",
        }

    def _convert_parameters(self, params: Dict[str, Any]) -> Dict[str, Any]:
        """translate bcp parameter names"""
        # replace type in some cases
        eq_dict = self._equivalence_dict()
        if eq_dict is not None:
            params = {eq_dict.get(k, k): v for k, v in params.items()}
        return params

    def _submit_command(self, submission_file_path: Path) -> str:
        """Launch the submission command"""
        command_list = self._make_submission_command(submission_file_path)
        # run
        job_utils.CommandFunction(
            command_list, ret_stdout=False, verbose=False
        )()  # explicit errors
        return ""

    @staticmethod
    def _make_submission_command(submission_file_path: Path) -> List[str]:
        """Make a command to trigger submission script. On BCP cluster, the script is triggerred with bash"""
        return ["bash", str(submission_file_path)]

    def _make_submission_file_text(self, command_groups: List[List[str]]) -> str:
        """
        Given the command groups, generate submission script file's text.
        Command groups is a list of command group. A command group is defined as:
              0. Command group is a list of command strings
              1. Each command group occupies one bcprun, srun or bash
              2. Each command group eventually has multiple commands connected by ";"
        On BCP cluster, multi-gpu python scripts are launched with `bcprun --nnodes ? --npernode ?`

        :param List[List[str]] command_groups: Command groups to launch with
        :return: submission script file's text
        :rtype: str
        """
        paths = job_utils.JobPaths(folder=self.folder, job_name=self.job_name)
        time_tag = datetime.datetime.now().strftime("%m%d_%H%M%S")
        stdout = str(paths.stdout).replace("_%j", f"_{time_tag}")

        nnodes = self.parameters.get("nnodes", 1)
        npernode = self.parameters.get("npernode", 1)
        launcher = self.parameters.get("launcher")
        launcher_flags = ""
        if launcher is not None:
            launcher_flags = f"--launcher {launcher}"
        env_vars = self.parameters.get("env_vars")
        env_flags = ""
        if env_vars is not None:
            env_flags = [f"--env '{k}={v}'" for k, v in env_vars.items()]
            env_flags = " ".join(env_flags)

        # now create
        lines = ["#!/bin/bash", ""]

        # environment setup:
        setup = self.parameters.get("setup", None)
        if setup is not None:
            lines += ["", "# setup"] + setup

        # Add pause_and_prime_dns_connection to command groups on BCP
        launcher_scripts_path = Path(
            "/opt/NeMo-Megatron-Launcher/launcher_scripts"
        )  # Hard code path on BCP
        pause_and_prime_dns_connection_command = f"python3 -u {launcher_scripts_path / 'nemo_launcher/collections/pause_and_prime_dns_connections.py'}"
        _nemo_code_path = "/opt/NeMo"
        for ind in range(len(command_groups)):
            # TODO: Find a better way to insert pause_and_prime_dns_connection_command
            if _nemo_code_path in command_groups[ind]:
                command_groups[ind] = [
                    pause_and_prime_dns_connection_command
                ] + command_groups[ind]

        for group_ind, command_group in enumerate(command_groups):
            command = ";\n  ".join(command_group)
            if group_ind + 1 == len(command_groups):
                bcprun_cmd = f"bcprun --nnodes {nnodes} --npernode {npernode}"
            else:
                bcprun_cmd = f"bcprun --nnodes 1 --npernode 1"
            lines += [
                "",
                f"# command {group_ind + 1}",
                f"{bcprun_cmd} " f'{launcher_flags} {env_flags} --cmd "',
                f'  {command} " 2>&1 | tee -a {stdout}',
                "",
            ]
        return "\n".join(lines)

class SlurmLauncher(Launcher):
    """
    Slurm job launcher
    This class is used to hold the parameters to run a job on slurm.
    In practice, it will create a batch file in the specified directory for the job,
    trigger the job with `sbatch` command and return a job id.

    :param Union[Path, str] folder: folder for storing job submission/output and logs.
    :param str job_name: Name of the job, used as job folder name
    :param Any **kwargs: See slurm documentation for most parameters.
            Most useful parameters are: time, mem, gpus_per_node, cpus_per_task, partition
            Below are the parameters that differ from slurm documentation:
                setup: a list of command to run in sbatch before running srun
    """

    def __init__(self, folder: Union[Path, str], job_name: str, **kwargs: Any) -> None:
        super().__init__(folder, job_name)
        self.parameters = {}
        self._update_parameters(job_name=job_name, **kwargs)

        if shutil.which("srun") is None and not NEMO_LAUNCHER_DEBUG:
            raise RuntimeError(
                'Could not detect "srun", are you indeed on a slurm cluster?'
            )

    @classmethod
    def _equivalence_dict(cls):
        return {
            "name": "job_name",
            "timeout_min": "time",
            "mem_gb": "mem",
            "nodes": "nodes",
            "cpus_per_task": "cpus_per_task",
            "gpus_per_node": "gpus_per_node",
            "tasks_per_node": "ntasks_per_node",
        }

    @classmethod
    def _valid_parameters(cls) -> Set[str]:
        """Parameters that can be set through update_parameters"""
        return set(_get_default_parameters())

    def _convert_parameters(self, params: Dict[str, Any]) -> Dict[str, Any]:
        """translate slurm parameter names"""
        # replace type in some cases
        eq_dict = self._equivalence_dict()
        if eq_dict is not None:
            params = {eq_dict.get(k, k): v for k, v in params.items()}
        if "mem" in params:
            params["mem"] = _convert_mem(params["mem"])
        return params

    def _update_parameters(self, **kwargs: Any) -> None:
        """
        Updates sbatch submission file parameters
        Raises ValueError:
            In case an erroneous keyword argument is added, a list of all eligible parameters
            is printed, with their default values

        :param Any **kwargs: See slurm documentation for most parameters.
            Most useful parameters are: time, mem, gpus_per_node, cpus_per_task, partition
            Below are the parameters that differ from slurm documentation:
                setup: a list of command to run in sbatch before running srun
        """
        defaults = _get_default_parameters()
        in_valid_parameters = sorted(set(kwargs) - set(defaults))
        if in_valid_parameters:
            string = "\n  - ".join(
                f"{x} (default: {repr(y)})" for x, y in sorted(defaults.items())
            )
            logger.warning(
                f"Unrecognized sbatch parameter(s): {in_valid_parameters}. Use at your own risk.\n\nValid parameters are:\n  - {string}"
            )

        self.parameters.update(
            {k: v for k, v in kwargs.items() if k not in in_valid_parameters}
        )
        self.parameters.update(
            {"additional_parameters": {k: kwargs[k] for k in in_valid_parameters}},
        )
        self.parameters = self._convert_parameters(self.parameters)

    def _submit_command(self, submission_file_path: Path) -> str:
        """Launch the submission command"""
        command_list = self._make_submission_command(submission_file_path)
        # run
        output = job_utils.CommandFunction(
            command_list, verbose=False
        )()  # explicit errors

        job_id = ""
        if output:
            job_id = self._get_job_id_from_submission_command(output)
        return job_id

    def _make_submission_file_text(self, command_groups: List[List[str]]) -> str:
        """
        Given the command groups, generate submission script file's text.
        Command groups is a list of command group. A command group is defined as:
              0. Command group is a list of command strings
              1. Each command group occupies one bcprun, srun or bash
              2. Each command group eventually has multiple commands connected by ";"

        :param List[List[str]] command_groups: Command groups to launch with
        :return: submission script file's text
        :rtype: str
        """
        return _make_sbatch_string(
            command_groups=command_groups, folder=self.folder, **self.parameters
        )

    @staticmethod
    def _make_submission_command(submission_file_path: Path) -> List[str]:
        """Make a command to trigger submission script. On slurm cluster, the script is triggerred with sbatch"""
        return ["sbatch", str(submission_file_path)]

    @staticmethod
    def _get_job_id_from_submission_command(string: Union[bytes, str]) -> str:
        """Returns the job ID from the output of sbatch string"""
        if not isinstance(string, str):
            string = string.decode()
        output = re.search(r"job (?P<id>[0-9]+)", string)
        if output is None:
            raise utils.FailedSubmissionError(
                f'Could not make sense of sbatch output "{string}"\n'
                "Job instance will not be able to fetch status\n"
                "(you may however set the job job_id manually if needed)"
            )
        return output.group("id")

class K8SLauncher(Launcher):
    """
    K8s job launcher
    This class is used to hold the parameters to run a job on kubernetes.
    In practice, it will create a Helm chart in the specified directory for the job
    and trigger the job with `bash` command.

    :param Union[Path, str] folder: folder for storing job submission/output and logs.
    :param str job_name: Name of the job, used as job folder name
    :param Any **kwargs: Parse other cluster parameters required for k8s running,
        including `nodes`, `ntasks_pernode`, `bcp_launcher`, etc.
    """

    def __init__(self, folder: Union[Path, str], job_name: str, **kwargs: Any) -> None:
        super().__init__(folder, job_name)
        self.parameters = kwargs
        self.parameters = self._convert_parameters(self.parameters)

    @classmethod
    def _equivalence_dict(cls):
        return {
            "name": "job_name",
            "nodes": "nnodes",
            "tasks_per_node": "npernode",
            "ntasks_per_node": "npernode",
        }

    def _convert_parameters(self, params: Dict[str, Any]) -> Dict[str, Any]:
        """translate k8s parameter names"""
        # replace type in some cases
        eq_dict = self._equivalence_dict()
        if eq_dict is not None:
            params = {eq_dict.get(k, k): v for k, v in params.items()}
        return params

    def _submit_command(self, submission_file_path: Path) -> str:
        """Launch the submission command"""
        command_list = self._make_submission_command(submission_file_path)
        # run
        job_utils.CommandFunction(
            command_list, ret_stdout=False, verbose=False
        )()  # explicit errors
        return ""

    @staticmethod
    def _make_submission_command(submission_file_path: Path) -> List[str]:
        """Make a command to trigger submission script. On a k8s cluster, the script is triggerred with Helm"""
        return ["bash", str(submission_file_path)]

    def _make_submission_file_text(self, command_groups: List[List[str]]) -> str:
        """
        Generate the script to launch the Helm chart.
        A very simple bash script is generated which runs `helm install` for the
        Helm chart that was generated.

        :param List[List[str]] command_groups: Command groups to launch with
        :return: submission script file's text
        :rtype: str
        """
        paths = job_utils.JobPaths(folder=self.folder, job_name=self.job_name)
        helm_charts = paths.folder / "k8s_template"
        job_name = self.job_name.replace("_", "-")

        return f"#!/bin/bash\nhelm install {job_name} {helm_charts}\n"

@functools.lru_cache()
def _get_default_parameters() -> Dict[str, Any]:
    """Parameters that can be set through update_parameters"""
    specs = inspect.getfullargspec(_make_sbatch_string)
    zipped = zip(specs.args[-len(specs.defaults) :], specs.defaults)  # type: ignore
    return {key: val for key, val in zipped if key not in {"command_groups", "folder"}}

# pylint: disable=too-many-arguments,unused-argument, too-many-locals
def _make_sbatch_string(
    command_groups: List[List[str]],
    folder: Union[str, Path],
    job_name: str = "nemo_launcher",
    partition: Optional[str] = None,
    time: int = 5,
    nodes: Union[int, List[int]] = 1,
    ntasks_per_node: Optional[Union[int, List[int]]] = None,
    cpus_per_task: Optional[int] = None,
    cpus_per_gpu: Optional[int] = None,
    num_gpus: Optional[int] = None,  # legacy
    gpus_per_node: Optional[int] = None,
    gpus_per_task: Optional[int] = None,
    qos: Optional[str] = None,  # quality of service
    setup: Optional[List[str]] = None,
    mem: Optional[str] = None,
    mem_per_gpu: Optional[str] = None,
    mem_per_cpu: Optional[str] = None,
    dependency: Optional[str] = None,
    comment: Optional[str] = None,
    constraint: Optional[str] = None,
    exclude: Optional[str] = None,
    account: Optional[str] = None,
    gres: Optional[str] = None,
    exclusive: Optional[Union[bool, str]] = None,
    array: Optional[str] = None,
    stderr_to_stdout: bool = False,
    container_image: Optional[str] = None,
    container_mounts: Optional[str] = None,
    additional_parameters: Optional[Dict[str, Any]] = None,
    srun_args: Optional[Iterable[str]] = None,
    heterogeneous: bool = False,
) -> str:
    """Creates the content of an sbatch file with provided parameters

    Parameters
    ----------
    See slurm sbatch documentation for most parameters:
    https://slurm.schedmd.com/sbatch.html

    Below are the parameters that differ from slurm documentation:

    command_groups:
        each command group will be assigned one srun
    folder: str/Path
        folder where print logs and error logs will be written
    setup: list
        a list of command to run in sbatch before running srun
    additional_parameters: dict
        Forces any parameter to a given value in sbatch. This can be useful
        to add parameters which are not currently available in nemo_launcher.
        Eg: {"mail-user": "blublu@nvidia.com", "mail-type": "BEGIN"}
    srun_args: List[str]
        Add each argument in the list to the srun call

    Raises
    ------
    ValueError
        In case an erroneous keyword argument is added, a list of all eligible parameters
        is printed, with their default values
    """
    nonslurm = [
        "nonslurm",
        "folder",
        "command_groups",
        "additional_parameters",
        "setup",
        "stderr_to_stdout",
        "container_image",
        "container_mounts",
        "srun_args",
        "heterogeneous",
    ]
    parameters = {
        k: v for k, v in locals().items() if v is not None and k not in nonslurm
    }
    # rename and reformat parameters

    if num_gpus is not None:
        warnings.warn(
            '"num_gpus" is deprecated, please use "gpus_per_node" instead (overwritting with num_gpus)'
        )
        parameters["gpus_per_node"] = parameters.pop("num_gpus", 0)
    if "cpus_per_gpu" in parameters and "gpus_per_task" not in parameters:
        warnings.warn(
            '"cpus_per_gpu" requires to set "gpus_per_task" to work (and not "gpus_per_node")'
        )
    # add necessary parameters
    job_name = parameters.get("job_name")
    paths = job_utils.JobPaths(folder=folder, job_name=job_name)
    stdout = str(paths.stdout)
    stderr = str(paths.stderr)

    if array is not None:
        stdout = stdout.replace("%j", "%A_%a")
        stderr = stderr.replace("%j", "%A_%a")
    parameters["output"] = stdout.replace("%t", "0")

    if not stderr_to_stdout:
        parameters["error"] = stderr.replace("%t", "0")

    if NEMO_LAUNCHER_CI:  # Override output file for slurm
        parameters["output"] = parameters["error"] = str(paths.folder / "slurm_%j.out")
        stdout = stderr = parameters["output"]

    if additional_parameters is not None:
        parameters.update(additional_parameters)
    # now create
    lines = ["#!/bin/bash", "", "# Parameters"]
    if heterogeneous:
        for i in range(len(nodes)):
            het_parameters = parameters.copy()
            het_parameters["output"] = parameters["output"].replace("_%j", f"_{i}_%j")
            if "error" in parameters:
                het_parameters["error"] = parameters["error"].replace("_%j", f"_{i}_%j")
            het_parameters.update(
                {
                    "job_name": f"{job_name}_{i}",
                    "nodes": nodes[i],
                    "ntasks_per_node": ntasks_per_node[i],
                }
            )
            for k in sorted(parameters):
                lines.append(_as_sbatch_flag(k, het_parameters[k]))
            if i != len(nodes) - 1:
                lines.append(f"#SBATCH hetjob")
    else:
        for k in sorted(parameters):
            lines.append(_as_sbatch_flag(k, parameters[k]))
    # environment setup:
    if setup is not None:
        lines += ["", "# setup"] + setup

    # commandline (this will run the function and args specified in the file provided as argument)
    # We pass --output and --error here, because the SBATCH command doesn't work as expected with a filename pattern
    stderr_flags = [] if stderr_to_stdout else ["--error", stderr]
    container_flags = ["--container-image", container_image] if container_image else []
    container_flags += (
        ["--container-mounts", container_mounts] if container_mounts else []
    )
    if srun_args is None:
        srun_args = []

    if NEMO_LAUNCHER_MEMORY_MEASURE:
        srun_args += ["--overlap"]

        mem_stdout = stdout.replace("_%j", "_mem_%j")
        mem_stdout = mem_stdout.replace("_%A_%a", "_mem_%A_%a")
        mem_srun_cmd = shlex.join(
            [
                "srun",
                "--ntasks=1",
                "--ntasks-per-node=1",
                "--output",
                mem_stdout,
                *container_flags,
                *srun_args,
            ]
        )
        lines += [
            "",
            "# run memory measure",
            f"{mem_srun_cmd} \\",
            f"  nvidia-smi --query-gpu=timestamp,index,,memory.total,memory.free,memory.used --format=csv -l 1 & ",
            "",
        ]

    for group_ind, command_group in enumerate(command_groups):
        if heterogeneous:
            het_group = f"--het-group={group_ind}"
            het_stdout = stdout.replace("_%j", f"_{group_ind}_%j")
            het_stderr = stderr_flags.copy()
            if het_stderr:
                het_stderr[-1] = het_stderr[-1].replace("_%j", f"_{group_ind}_%j")
            srun_cmd = shlex.join(
                [
                    "srun",
                    "--output",
                    het_stdout,
                    *het_stderr,
                    *container_flags,
                    *srun_args,
                    het_group,
                ]
            )
            command = ";\n  ".join(command_group)
            lines += [
                "",
                f"# command {group_ind + 1}",
                f'{srun_cmd} bash -c "',
                f'  {command} " &',
                "",
            ]
            if group_ind == len(nodes) - 1:
                lines += ["wait"]
            else:
                lines += ["sleep 30"]
        else:
            srun_cmd = shlex.join(
                [
                    "srun",
                    "--output",
                    stdout,
                    *stderr_flags,
                    *container_flags,
                    *srun_args,
                ]
            )
            command = ";\n  ".join(command_group)
            lines += [
                "",
                f"# command {group_ind + 1}",
                f'{srun_cmd} bash -c "',
                f'  {command} "',
                "",
            ]
    return "\n".join(lines)

def _convert_mem(mem_gb: float) -> str:
    """Convert non-integer mem_gb to unit MB"""
    if mem_gb == int(mem_gb):
        if int(mem_gb) == 0:
            return "0"
        return f"{int(mem_gb)}GB"
    return f"{int(mem_gb * 1024)}MB"

def _as_sbatch_flag(key: str, value: Any) -> str:
    """Convert key value pairs to `#SBATCH --{key}={value}` flags"""
    key = key.replace("_", "-")
    if value is True:
        return f"#SBATCH --{key}"

    value = shlex.quote(str(value))
    return f"#SBATCH --{key}={value}"


from collections import namedtuple
from enum import Enum
from itertools import product, permutations
from math import ceil, floor, sqrt
from typing import List

from gurobipy import GRB, GenExpr, LinExpr, Model, tupledict, abs_, and_, max_, min_, or_, QuadExpr, GurobiError

from .classes import Layout, Element

def preserve_relationships(m: Model, elements: List[Element], x0, x1, y0, y1):
    for element, other in permutations(elements, 2):
        # If elements overlap, don’t force relationship
        if not element.does_overlap(other):
            # If an element is above or to the left of the other, it should stay there
            if element.y1 <= other.y0: # element is above the other
                m.addConstr(y1[element.id] <= y0[other.id])
            if element.x1 <= other.x0: # element is on the left side of the other
                m.addConstr(x1[element.id] <= x0[other.id])

            if True:
                # Preferences for overlap in a single dimension
                if element.x0 <= other.x0: # Element begins before the other
                    m.addConstr(x0[element.id] <= x1[other.id])
                if element.x1 >= other.x1: # Element ends after the other
                    m.addConstr(x1[element.id] >= x0[other.id])
                if element.y0 <= other.y0: # Element begins before the other
                    m.addConstr(y0[element.id] <= y1[other.id])
                if element.y1 >= other.y1: # Element ends after the other
                    m.addConstr(y1[element.id] >= y0[other.id])

def prevent_overlap(m: Model, elem_ids: List[str], x0x1diff: tupledict, y0y1diff: tupledict, min_distance: int = 0):

    distance = m.addVars(permutations(elem_ids, 2), lb=-GRB.INFINITY, vtype=GRB.INTEGER)

    for i1, i2 in permutations(elem_ids, 2):
        m.addConstr(distance[i1, i2] == max_(x0x1diff[i1, i2], x0x1diff[i2, i1], y0y1diff[i1, i2], y0y1diff[i2, i1]))
        m.addConstr(distance[i1, i2] >= min_distance)

def add_pairwise_diff(m: Model, ids: List, var1: tupledict, var2: tupledict=None):
    if var2 is None:
        var2 = var1
    diff = m.addVars(permutations(ids, 2), lb=-GRB.INFINITY, vtype=GRB.INTEGER)
    for i1, i2 in permutations(ids, 2):
        m.addConstr(diff[i1, i2] == var1[i1] - var2[i2])
    return diff

def add_coord_vars(m: Model, elem_ids, available_width, available_height):
    x0 = m.addVars(elem_ids, lb=0, vtype=GRB.INTEGER)
    y0 = m.addVars(elem_ids, lb=0, vtype=GRB.INTEGER)
    x1 = m.addVars(elem_ids, lb=1, vtype=GRB.INTEGER)
    y1 = m.addVars(elem_ids, lb=1, vtype=GRB.INTEGER)

    for i in elem_ids:
        m.addConstr(x0[i] <= x1[i] - 1)         # x0 < x1 sanity
        m.addConstr(y0[i] <= y1[i] - 1)         # y0 < y1 sanity
        m.addConstr(x1[i] <= available_width)   # contain to available width
        m.addConstr(y1[i] <= available_height)  # contain to available height

    return x0, y0, x1, y1

def add_less_than_vars(m: Model, ids: List, diff: tupledict):
    less_than = m.addVars(permutations(ids, 2), vtype=GRB.BINARY)
    for i1, i2 in permutations(ids, 2):
        m.addConstr((less_than[i1, i2] == 1) >> (diff[i1, i2] <= -1))
        m.addConstr((less_than[i1, i2] == 0) >> (diff[i1, i2] >= 0))
    return less_than

def add_equals_var(m: Model, var1: tupledict, var2: tupledict):
    var_max = m.addVar(vtype=GRB.INTEGER)
    var_min = m.addVar(vtype=GRB.INTEGER)
    m.addConstr(var_max == max_(var1, var2))
    m.addConstr(var_min == min_(var1, var2))

    equals = m.addVar(vtype=GRB.BINARY)
    m.addConstr((equals == 1) >> (var_max == var_min))
    m.addConstr((equals == 0) >> (var_max >= 1 + var_min))

    return equals