# Dataset:
# wget
# https://github.com/anujshah1003/own_data_cnn_implementation_keras/blob/master/data.zip?raw=true
# -O data.zip

# IMPORTS
from __future__ import print_function

import os
import cv2                  # load and preprocess dataset
import numpy as np              # reshape images, ...
import matplotlib.pyplot as plt

from sklearn.utils import shuffle
from sklearn.cross_validation import train_test_split
from sklearn import preprocessing

from keras import backend as K
K.set_image_dim_ordering('tf')
# tensorflow (N, height or rows , width or cols     , channels)
# theano     (N, channels       , height or rows    , width or cols)

from keras.utils import np_utils
from keras.models import Sequential, Model
from keras.models import model_from_json
from keras.models import load_model
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.optimizers import SGD, RMSprop, adam
from keras.callbacks import ModelCheckpoint, CSVLogger, TensorBoard, EarlyStopping
from keras.preprocessing.image import ImageDataGenerator

from keras import applications
from keras.datasets import cifar10

import logging
FORMAT = "[%(lineno)4s : %(funcName)-30s ] %(message)s"
logging.basicConfig(level=logging.DEBUG, format=FORMAT)

import keras
import tensorflow as tf
logging.debug('keras {}'.format(keras.__version__))
logging.debug('tensorflow {}'.format(tf.__version__))

# GLOBALS
# 32 for cifar
img_height = 150         # rows
img_width = 150          # cols
# img_channels = 1
img_channels=3

# 408 samples from 4 classes [cat, dog, horse, human]
dataset_path = 'dataset'
model_saved_path = 'output/model.h5'

data_augmentation = True
num_classes = 2
#batch_size = 32
batch_size = 16
epochs = 2

# FUNCTIONS

if not os.path.exists('output'):
    os.makedirs('output')

if not os.path.exists('logs'):
    os.makedirs('logs')

def load_and_preprocess_data_1():

    img_dataset_list = []

    classes_list = os.listdir(dataset_path)
    logging.debug('classes_list {}'.format(classes_list))

    for class_name in classes_list:

        img_list = os.listdir(os.path.join(dataset_path, class_name))

        for img_name in img_list:
            img_path_name = os.path.join(dataset_path, class_name, img_name)
            # logging.debug('img_path {}'.format(img_path_name))
            # # (379, 499, 3

            img_input = cv2.imread(img_path_name)
            # logging.debug('img_input.shape {}'.format(img_input.shape))

            img_input_gray = cv2.cvtColor(img_input, cv2.COLOR_BGR2GRAY)

            # cv2.resize(src, dsize[, dst[, fx[, fy[, interpolation]]]]) -> dst
            # where fx is scale factor along the horizontal axis
            img_input_resize = cv2.resize(img_input_gray, (img_height, img_width))
            # logging.debug('img_input_resize.shape
            # {}'.format(img_input_resize.shape))             # (128, 128)

            # Add all images in the dataset to img_dataset_list
            # img_dataset_list is a list of 808 elements{images} each of which is a
            # 128x128{i.e 2D list}
            img_dataset_list.append(img_input_resize)

    img_dataset_arr = np.array(img_dataset_list)
    logging.debug('img_dataset_arr {}'.format(img_dataset_arr.shape)
                    )                               # (808, 128, 128)
    # (808, 128, 128, 3) for RGB

    img_dataset_arr = img_dataset_arr.astype('float32')

    # Method 1
    # Normalize (between 0 and 1)
    img_dataset_arr /= 255

    # Subtract mean

    # Method 2
    # 0 mean and unit variance
    img_dataset_arr_scaled = preprocessing.scale(img_dataset_arr)

    # Dimension ordering

    if img_channels is 1:
        if K.image_dim_ordering() is 'th':
            img_dataset_arr = np.expand_dims(img_dataset_arr, axis=1)
            logging.debug('img_dataset_arr.shape {}'.format(
                img_dataset_arr.shape))                 # (808, 1, 128, 128)
        else:
            img_dataset_arr = np.expand_dims(img_dataset_arr, axis=3)
            logging.debug('img_dataset_arr.shape {}'.format(
                img_dataset_arr.shape))  # (808, 128, 128, 1)

    else:
        # RGB
        if K.image_dim_ordering is 'th':
            img_dataset_arr = np.rollaxis(img_dataset_arr, 3, 1)
            logging.debug('img_dataset_arr.shape {}'.format(
                img_dataset_arr.shape))  # (808, 3, 128, 128)

    # TODO

    return img_dataset_arr

def load_and_preprocess_data_2():

    img_dataset_list = []

    classes_list = os.listdir(dataset_path)
    logging.debug('classes_list {}'.format(classes_list))

    for class_name in classes_list:

        img_list = os.listdir(os.path.join(dataset_path, class_name))

        for img_name in img_list:
            img_path_name = os.path.join(dataset_path, class_name, img_name)
            # logging.debug('img_path {}'.format(img_path_name))
            # # (379, 499, 3

            img_input = cv2.imread(img_path_name)
            # logging.debug('img_input.shape {}'.format(img_input.shape))

            img_input_gray = cv2.cvtColor(img_input, cv2.COLOR_BGR2GRAY)
            img_input_gray = img_input

            # cv2.resize(src, dsize[, dst[, fx[, fy[, interpolation]]]]) -> dst
            # where fx is scale factor along the horizontal axis
            # img_input_resize = cv2.resize(img_input_gray, (img_height, img_width))
            # logging.debug('img_input_resize.shape
            # {}'.format(img_input_resize.shape))             # (128, 128)

            img_input_flatten = cv2.resize(
                img_input_gray, (img_height, img_width)).flatten()
            # logging.debug('img_input_flatten.shape
            # {}'.format(img_input_flatten.shape))           # (16384,)

            # Add all images in the dataset to img_dataset_list
            # img_dataset_list is a list of 808 elements{images} each of which is a
            # 16384
            img_dataset_list.append(img_input_flatten)

    img_dataset_arr = np.array(img_dataset_list)
    # (808, 128x128) i.e. (808, 16384)
    logging.debug('img_dataset_arr {}'.format(img_dataset_arr.shape))
    # (808, 128x128x3) for RGB

    img_dataset_arr_scaled = preprocessing.scale(img_dataset_arr)
    logging.debug('img_dataset_arr_scaled.shape {}'.format(
        img_dataset_arr_scaled.shape))           # (808, 16384)

    logging.debug('img_dataset_arr_scaled mean {}'.format(
        np.mean(img_dataset_arr_scaled)))         # -3.69616328743e-17
    logging.debug('img_dataset_arr_scaled std {}'.format(
        np.std(img_dataset_arr_scaled)))           # 1.0

    # [  1.50869416e-16   1.42487782e-16  -1.27785571e-16 ...,   1.42899993e-16 1.69006723e-16   5.66103819e-17]
    logging.debug('img_dataset_arr_scaled mean {}'.format(img_dataset_arr_scaled.mean(axis=0)))
    logging.debug(
        'img_dataset_arr_scaled std {}'.format(
            img_dataset_arr_scaled.std(axis=0)))       # [ 1.  1.  1. ...,  1.  1.  1.]

    # Dimension ordering

    if K.image_dim_ordering() is 'th':
        img_data = img_dataset_arr_scaled.reshape(img_dataset_arr.shape[0], img_channels, img_height, img_width)
        logging.debug('img_data.shape {}'.format(img_data.shape))
    else:
        img_data = img_dataset_arr_scaled.reshape(img_dataset_arr.shape[0], img_height, img_width, img_channels)
        logging.debug('img_data.shape {}'.format(img_data.shape)
                            )                                   # (808, 128, 128, c)

    input_shape = img_data[0].shape
    logging.debug('input_shape {}'.format(input_shape))

    # Labels to dataset
    num_of_samples = img_data.shape[0]
    logging.debug('num_of_samples {}'.format(num_of_samples))

    labels = np.ones((num_of_samples), dtype='int64')
    labels[0:102] = 0
    labels[102:204] = 1
    labels[204:606] = 2
    labels[606:] = 3

    class_names = ['cats', 'dogs', 'horses', 'humans']

    # One hot encoding
    Y = keras.utils.to_categorical(labels, num_classes)
    #Y = np_utils.to_categorical(labels, num_classes)

    # Shuffle and split dataset
    x, y = shuffle(img_data, Y, random_state=2)
    X_train, X_test, y_train, y_test = train_test_split(
        x, y, test_size=0.2, random_state=4)  # Same split every time as random state is set

    return X_train, X_test, y_train, y_test, input_shape

def load_and_preprocess_data_3():
    # The data, shuffled and split between train and test sets:
    (X_train, y_train), (x_test, y_test) = cifar10.load_data()
    logging.debug('X_train shape: {}'.format(X_train.shape))
    logging.debug('train samples: {}'.format(X_train.shape[0]))
    logging.debug('test samples: {}'.format(x_test.shape[0]))

    # Convert class vectors to binary class matrices.
    y_train = keras.utils.to_categorical(y_train, num_classes)
    y_test = keras.utils.to_categorical(y_test, num_classes)

    X_train = X_train.astype('float32')
    x_test = x_test.astype('float32')
    X_train /= 255
    x_test /= 255

    input_shape = X_train[0].shape
    logging.debug('input_shape {}'.format(input_shape))
    input_shape = X_train.shape[1:]
    logging.debug('input_shape {}'.format(input_shape))

    return X_train, x_test, y_train, y_test, input_shape

def load_and_preprocess_data_generator():

        # TBD
        train_data_dir = "dataset2/train"
        validation_data_dir = "dataset2/validation"

        # Initiate the train and test generators with data Augumentation
        train_datagen = ImageDataGenerator(rescale = 1./255, shear_range=0.2, horizontal_flip = True, fill_mode = "nearest",
                                           zoom_range = 0.3, width_shift_range = 0.3, height_shift_range=0.3,
                                           rotation_range=30)

        test_datagen = ImageDataGenerator(rescale = 1./255, shear_range=0.2, horizontal_flip = True, fill_mode = "nearest",
                                          zoom_range = 0.3, width_shift_range = 0.3, height_shift_range=0.3,
                                          rotation_range=30)

        train_generator = train_datagen.flow_from_directory(train_data_dir, target_size = (img_height, img_width),
                                                            batch_size = batch_size, class_mode = "categorical")

        validation_generator = test_datagen.flow_from_directory(validation_data_dir, target_size = (img_height, img_width),
                                                                class_mode = "categorical")

        # HARDCODING
        input_shape = (img_width, img_height, img_channels)
        logging.debug('input_shape {}'.format(input_shape))

        return train_generator, validation_generator, input_shape

def model_create(input_shape, num_classes):
        logging.debug('input_shape {}'.format(input_shape))

        model = Sequential()

        model.add(Conv2D(32, (3, 3), border_mode='same', input_shape=input_shape))
        model.add(Activation('relu'))

        model.add(Conv2D(32, (3, 3)))
        model.add(Activation('relu'))
        model.add(MaxPooling2D(pool_size=(2, 2)))
        model.add(Dropout(0.5))

        model.add(Flatten())
        model.add(Dense(128))
        model.add(Activation('relu'))
        model.add(Dropout(0.5))

        model.add(Dense(num_classes))
        model.add(Activation('softmax'))

        # use binary_crossentropy if has just 2 prediction yes or no
        model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])

        return model

def model_vgg_create(input_shape, num_classes):

    logging.debug('input_shape {}'.format(input_shape))
    #model = applications.VGG19(weights = "imagenet", include_top=False, input_shape = (256, 256, 3))
    model = applications.VGG19(weights = "imagenet", include_top=False, input_shape = (input_shape))        #  input_shape (128, 128, 1)
                                                                                #  input_shape (128, 128, 3)

    # Freeze the layers which you don't want to train. Freezing the first 5 layers.
    for layer in model.layers[:5]:
        layer.trainable = False

    # Adding custom Layers
    x = model.output
    x = Flatten()(x)
    x = Dense(1024, activation="relu")(x)
    x = Dropout(0.5)(x)
    x = Dense(1024, activation="relu")(x)
    predictions = Dense(num_classes, activation="softmax")(x)

    # Creating the final model
    model_final = Model(inputs = model.input, outputs = predictions)

    # Compile the model
    # opt = RMSprop(lr=0.0001, decay=1e-6)
    opt = SGD(lr=0.0001, momentum=0.9)
    model_final.compile(loss = "categorical_crossentropy", optimizer = opt, metrics=["accuracy"])

    return model_final

def model_train(model, X_train, y_train, X_test, y_test, train_generator=None, validation_generator=None):

        # Callbacks
        filename = 'output/model_train.csv'
        csv_log = CSVLogger(filename, separator=' ', append=False)

        early_stopping = EarlyStopping(
            monitor='val_loss', patience=50, verbose=1, mode='min')

        filepath = "output/best-weights-{epoch:03d}-{loss:.4f}-{acc:.4f}.hdf5"
        checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1,
            save_best_only=True, save_weights_only=False, mode='min', period=1)                     # min because we are monitoring val_loss that should decrease

        tensorboard = TensorBoard(log_dir='./logs', histogram_freq=0, write_graph=True, write_images=True)

        callbacks_list = [csv_log, early_stopping, checkpoint, tensorboard]

        if train_generator is None:

            # Train
            if not data_augmentation:
                logging.debug('Not using data augmentation.')
                model.fit(X_train, y_train,
                        batch_size=batch_size,
                        epochs=epochs,
                        validation_data=(x_test, y_test),
                        verbose=1,
                        shuffle=True,
                        callbacks=callbacks_list)
                #hist = model.fit(X_train, y_train, batch_size=32, epochs=20, verbose=1, validation_split=0.2, callbacks=callbacks_list)
            else:
                logging.debug('Using real-time data augmentation.')
                # This will do preprocessing and realtime data augmentation:
                datagen = ImageDataGenerator(
                    featurewise_center=False,  # set input mean to 0 over the dataset
                    samplewise_center=False,  # set each sample mean to 0
                    featurewise_std_normalization=False,  # divide inputs by std of the dataset
                    samplewise_std_normalization=False,  # divide each input by its std
                    zca_whitening=False,  # apply ZCA whitening
                    rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)
                    width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
                    height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
                    horizontal_flip=True,  # randomly flip images
                    vertical_flip=False,  # randomly flip images
                    rescale=0)

                logging.debug('X_train.shape {}'.format(X_train.shape))

                # Compute quantities required for feature-wise normalization
                # (std, mean, and principal components if ZCA whitening is applied).
                datagen.fit(X_train)

                # Fit the model on the batches generated by datagen.flow().
                model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),
                                    steps_per_epoch=X_train.shape[0] // batch_size,
                                    epochs=epochs,
                                    validation_data=(X_test, y_test),
                                    callbacks = callbacks_list)

        else:

            model.fit_generator(
                        train_generator,
                        steps_per_epoch=2000// batch_size,
                        epochs=epochs,
                        validation_data=validation_generator,
                        validation_steps=800// batch_size)

def model_evaluate(model, X_train, y_train, X_test, y_test):

        score = model.evaluate(X_test, y_test, verbose=1)

        logging.debug('Test Loss: {}'.format(score[0]))
        logging.debug('Test Accuracy: {}'.format(score[1]))

        img_test = X_test[0:1]
        logging.debug('X_test.shape {}'.format(X_test.shape))                                           # (162, 128, 128, 1)
        logging.debug('img_test.shape {}'.format(img_test.shape))                                       # (1, 128, 128, 1)

        logging.debug('Prediction: {}'.format(model.predict(img_test)))
        logging.debug('Prediction: {}'.format(model.predict_classes(img_test)))
        logging.debug('Actual: {}'.format(y_test[0:1]))

def test_image():

        image_path_name = 'dataset/Humans/rider-8.jpg'

        # Testing a new image
        img_test = cv2.imread(image_path_name)
        img_test = cv2.cvtColor(img_test, cv2.COLOR_BGR2GRAY)
        img_test = cv2.resize(img_test, (img_height, img_width))
        img_test = np.array(img_test)
        img_test = img_test.astype('float32')
        img_test /= 255
        logging.debug('img_test.shape {}'.format(img_test.shape))                                       # (128, 128)

        if img_channels is 1:
                if K.image_dim_ordering() is 'th':
                        img_test = np.expand_dims(img_test, axis=0)
                        img_test = np.expand_dims(img_test, axis=0)
                        logging.debug('img_test.shape {}'.format(img_test.shape))
                else:
                        img_test = np.expand_dims(img_test, axis=3)
                        img_test = np.expand_dims(img_test, axis=0)
                        logging.debug('img_test.shape {}'.format(img_test.shape)
                                      )                               # (1, 128, 128, 1)

        else:
                if K.image_dim_ordering() == 'th':
                        img_test = np.rollaxis(img_test, 2, 0)
                        img_test = np.expand_dims(img_test, axis=0)
                        logging.debug('img_test.shape {}'.format(img_test.shape))
                else:
                        img_test = np.expand_dims(img_test, axis=0)
                        logging.debug('img_test.shape {}'.format(img_test.shape))

        # Predicting the test image
        logging.debug('Prediction: {}'.format(model.predict(img_test)))
        logging.debug('Predictied class: {}'.format(model.predict_classes(img_test)))

def model_save(model):

        model.save(model_saved_path)

        # Save Model and Weights
        # # Serialize model to JSON
        # model_json = model.to_json()

        # if not os.path.exists('output'):
        #     os.makedirs('output')

        # with open('output/model.json', 'w') as json_file:
        #     json_file.write(model_json)

        # model.save_weights('output/model.h5')
        # logging.debug('Model saved to disk')

def model_load(model_saved_path):

        model = load_model(model_saved_path)

        return model

def model_summary(model):
        model.summary()
        model.get_config()
        model.layers[0].get_config()
        model.layers[0].input_shape
        model.layers[0].output_shape
        model.layers[0].get_weights()
        model.layers[0].trainable

        # Each layer in Keras is a callable object
        logging.debug('Model Input Tensors: {}'.format(model.input))
        for layer in model.layers:
            logging.debug('layer.name {} layer.trainable {}'.format(layer.name, layer.trainable))
            logging.debug('layer config {}'.format(layer.get_config()))
        logging.debug('Model Output Tensors: {}'.format(model.output))

### MAIN ###

# X_train, X_test, y_train, y_test, input_shape = load_and_preprocess_data_3()

train_generator, validation_generator, input_shape = load_and_preprocess_data_generator()

if os.path.exists(model_saved_path):
    logging.debug('Loading saved model...')
    model = model_load(model_saved_path)
else:
    logging.debug('Creating model...')
    # model = model_create(input_shape, num_classes)
    model = model_vgg_create(input_shape, num_classes)

# model_train(model, X_train, y_train, X_test, y_test, None, None)
model_train(model, None, None, None, None, train_generator, validation_generator)

model_save(model)

# model_evaluate(model, X_train, y_train, X_test, y_test)

# model_summary(model)
# test_image()

# intermediate_layer_summary()
# plot_graphs()
# visualize_intermediate_cnn_layers()
# plot_confusion_matrix()


from flask import Flask,request,redirect,jsonify
from modules.Cabos import Cabo
import pandas as pd
import json
import pytest
from flask_cors import CORS, cross_origin

app = Flask(__name__)
app_version = 'pre-alpha 0.1'

app.config['vao']=100
app.config['tag']=None
app.config['t01']=300
app.config['t_min']=16
app.config['t_eds']=22
app.config['t_ope']=75

condutor = Cabo()

df = condutor.df

@app.get('/cabo') #Lista todos os condutores
@cross_origin()
def listar_cabos():
    json_data=[]
    for idx,row in df.iterrows():
        # json_item = {"id":idx,"cabo":row['Tag']}
        json_item = {"id":idx,
                 "cabo":row['Tag'],
                 "area":row['ÁREA'],
                 "diametro":row['DIÂMETRO'],
                 "massa":row['MASSA_LINEAR'],
                 "elasticidade":row['MÓDULO_DE_ELASTICIDADE'],
                 "dilatacao": row['COEFICIENTE_DE_DILATAÇÃO_TÉRMICA'],
                 "ruptura": row["Carga_de_ruptura"]
                }
        json_data.append(json_item)
    cables = pd.io.json.dumps(json_data)
    return cables

@app.get('/cabo/<int:index>') #/cabo/3
@cross_origin()
def select_cabo(index):
    index-=1
    row = df.iloc[index]
    print(row)
    row_dict = row.to_dict()
    row_json = json.dumps(row_dict)
    return row_json

@app.get('/set_cabo/<int:index>') #/set_cabo/2
def set_cabo(index):
    # condutor.set_cabo(index)
    app.config['tag']=condutor.set_cabo(index)
    return condutor.cabo

@app.get('/set_vao') #/set_vao?vao=50.20
def set_vao(vao=0):
    #define a variavel global "vao" como entrada
    if vao == 0:
        app.config['vao'] = request.args.get('vao', default=0.0, type=float)
    else:
        app.config['vao'] = vao

    # variavel de teste
    meu_vao = app.config['vao']

    # retorna a variavel de teste meu_vao
    return f'Vao: {meu_vao}'

@app.get('/get_vao') #funcao de teste que retorna o valor global de "vao"
def get_vao():
    vao_global = app.config['vao']
    return vao_global

@app.get('/set_t01') #/set_t01?t01=230
def set_t01(t01=0):
      #define a variavel global "t01" como entrada
    if t01 == 0:
        app.config['t01'] = request.args.get('t01', default=0.0, type=float)
    else:
        app.config['t01'] = t01

    # variavel de teste
    t01 = app.config['t01']

    # retorna a variavel de teste meu_vao
    return t01

@app.get('/get_t01') #funcao de teste que retorna o valor global de "t01"
def get_t01():
    t01_global = app.config['t01']
    return t01_global

@app.get('/set_tmin') #/set_tmin?tmin=50.20
def set_tmin(tmin=0):
    #define a variavel global "tmin" como entrada
    if tmin == 0:
        app.config['t_min'] = request.args.get('tmin', default=app.config['t_min'], type=float)
    else:
        app.config['t_min'] = tmin

    # app.config['t_min'] = request.args.get('tmin', default=0.0, type=float)
    t_min = app.config['t_min']
    return t_min

@app.get('/set_teds') #/set_teds?teds=50.20
def set_teds(teds=0):
    #define a variavel global "teds" como entrada
    if teds == 0:
        app.config['t_eds'] = request.args.get('teds', default=app.config['t_eds'], type=float)
    else:
        app.config['t_eds'] = teds

    t_eds = app.config['t_eds']
    return t_eds

@app.get('/set_tope') #/set_tope?tope=75
def set_tope(tope=0):
    #define a variavel global "tope" como entrada
    if tope == 0:
        app.config['t_ope'] = request.args.get('tope', default=app.config['t_ope'], type=float)
    else:
        app.config['t_ope'] = tope

    t_ope = app.config['t_ope']
    return t_ope

@app.get('/load_temp') #carrega todas as temperaturas
def load_temp():
    t_min=app.config['t_min']
    t_eds=app.config['t_eds']
    t_ope=app.config['t_ope']
    condutor.set_temperaturas(t_min,t_eds,t_ope)
    return condutor.get_temperaturas()

@app.get('/cabo/<int:index>/load') #/cabo/2/load?vao=100&t01=200&tmin=16&teds=22&tope=75
@cross_origin()
def load(index):
    set_cabo(index)
    set_vao(request.args.get('vao',default=0.0, type=float))
    set_t01(request.args.get('t01',default=0.0, type=float))
    set_tmin(request.args.get('tmin',default=app.config['t_min'],type=float))
    set_teds(request.args.get('teds',default=app.config['t_eds'],type=float))
    set_tope(request.args.get('tope',default=app.config['t_ope'],type=float))

    load_temp()
    vao = get_vao()
    t01 = get_t01()

    condutor.dados_esticamento(t01,vao)
    esticamento = condutor.esticamento.to_json(orient='records')

    return esticamento

@app.get('/cabo/esticamento')
def esticamento():
    load_temp()
    vao=app.config['vao']
    t01=app.config['t01']
    print(t01)
    condutor.dados_esticamento(t01,vao)
    esticamento = condutor.esticamento.to_json(orient='records')

    # var_class = condutor.__dict__
    # variaveis = {k: v for k,v in var_class.items() if not isinstance(v, pd.DataFrame)}

    return esticamento

if __name__ =="__main__":
    app.run()


import time
import random
import requests
from assetapi import *
from teleNotice import *
import datetime

url = 'https://api.knightwar.io/item/listItem/'
body = {
    'optionClass': {
        'sword': True,
        'bow': True,
        'staff': True
    },
    'optionEnchant': {
        'enchant': 0
    },
    'optionPrice': {
        'minPrice': 0,
        'highPrice': 7000
    },
    'optionStar': {
        'star1': False,
        'star2': True,
        'star3': True,
        'star4': True,
        'star5': True,
        'star6': True
    },
    'pageIndex': 1,
    'pageSize': 300,
    'sort_column': 'sellPrice',
    'sort_type': 1
}

# Global Variable
idList = []
priceDict = {}

def toFile_owner():
    global response

    weapons = response.json()['data']['result']

    with open('F:/Python/Knightwar/owner.txt', 'w') as file1:
        for weapon in weapons:
            file1.write(str(weapon['id']) + ' ' + weapon['user']['walletId'] + '\n')

def toFile_price():
    global response

    weapons = response.json()['data']['result']

    with open('F:/Python/Knightwar/price.txt', 'w') as file1:
        for weapon in weapons:
            file1.write(str(weapon['id']) + ' ' + weapon['sellPrice'][:-18] + '\n')

def craw_priceDict():
    global response
    global priceDict
    weapons = response.json()['data']['result']

    for weapon in weapons:
        priceDict[str(weapon['id'])] = weapon['sellPrice'][:-18]

    return priceDict

def craw_idList():
    global response
    global idList
    idList.clear()
    weapons = response.json()['data']['result']

    for weapon in weapons:
        idList.append(weapon['id'])

    return idList

def toFilte_idList():
    global idList

    with open('F:/Python/Knightwar/id.txt', 'w') as file1:
        for id in idList:
            file1.write(str(id) + '\n')

def compare_OldOwner():
    global idList
    global priceDict
    global response
    weapons = response.json()['data']['result']

    # Old owner
    with open('F:/Python/Knightwar/owner.txt', 'r') as file1:
        oldOwnerList = file1.readlines()

    oldOwner = {}

    for waepon in oldOwnerList:
        oldOwner[waepon.split(' ')[0]] = waepon.split(' ')[1].split('\n')[0]

    # Old price

    with open('F:/Python/Knightwar/price.txt', 'r') as file1:
        oldPriceList = file1.readlines()

    oldPrice = {}

    for waepon in oldPriceList:
        oldPrice[waepon.split(' ')[0]] = waepon.split(' ')[1].split('\n')[0]

    # Old id
    with open('F:/Python/Knightwar/id.txt', 'r') as file1:
        oldIDList = file1.readlines()

    oldId = []

    for id in oldIDList:
        oldId.append(id.split('\n')[0])

    # # new Data and compare
    # url = 'https://nft.knightwar.io/asset/'
    marketplace_url = 'https://marketplace.knightwar.io/marketplace/nft/'

    # Check new weapon
    for id in idList:

        # if str(id) not in oldOwner:
        #     send_test_message(str(id) + ' new to de marketplace \nLink here: ' + marketplace_url+str(id))

        # compare time

        # dps[0] baseDPS
        # dps[1] type
        # dps[2] star
        # dps[3] level
        # dps[4] info
        # dps[5] baseHP

        #  New one, not int old id
        if str(id) not in oldId:
            dps = dpsCalc(str(id))
            time.sleep(6)
            print(str(id) + ' ' + dps[2][0])
            # if dps[2][0] == '1':
            #     if dps[0] >= 333:
            #         send_test_message('1 NEW ' + dps[2] + '\n'
            #                           + dps[1] + '\n'
            #                           + str(dps[7]) + 'dps -->' + str(dps[8]) + 'dps\n'
            #                           + dps[3] + ', ' + dps[4] + '\n'
            #                           + str(priceDict[str(id)]) + 'KWS\n'
            #                           + marketplace_url + str(id))
            #     if dps[5] >= 1600 and dps[1] == 'sword':
            #         send_test_message('1* Basehp ' + dps[2] + '\n'
            #                           + dps[1] + '\n'
            #                           + str(dps[7]) + 'dps -->' + str(dps[8]) + 'dps\n'
            #                           + dps[3] + ', ' + dps[4] + '\n'
            #                           + str(priceDict[str(id)]) + 'KWS\n'
            #                           + marketplace_url + str(id))

            if dps[8] >= 3900:
                send_test_message('NEW    ' + dps[2] + '\n'
                                  + dps[1] + '\n'
                                  + str(dps[7]) + ' dps      -->     ' + str(dps[8]) + ' dps\n'
                                  + dps[5] + '\n'
                                  + dps[6] + '\n'
                                  + dps[3] + ', ' + dps[4] + '\n'
                                  + str(priceDict[str(id)]) + 'KWS\n'
                                  + marketplace_url + str(id))

            if int(priceDict[str(id)]) <= 400:
                send_test_message('SUPERRRRRRR DEALLLLLL\n' + marketplace_url + str(id))
        else:

            try:
                if int(priceDict[str(id)]) != int(oldPrice[str(id)]):
                    dps = dpsCalc(str(id))
                    time.sleep(6)
                    if dps[8] >= 3900:
                        send_test_message('Change Price NEW ' + dps[2] + '\n'
                                          + dps[1] + '\n'
                                          + str(dps[7]) + 'dps -->' + str(dps[8]) + 'dps\n'
                                          + dps[5] + '\n'
                                          + dps[6] + '\n'
                                          + dps[3] + ', ' + dps[4] + '\n'
                                          + str(oldPrice[str(id)]) + 'KWS ->' + str(priceDict[str(id)]) + '\n'
                                          + marketplace_url + str(id))

            except:
                pass

def countdown(t):
    while t:
        mins, secs = divmod(t, 60)
        timer = '{:02d}:{:02d}'.format(mins, secs)
        print(timer, end='\r')
        time.sleep(1)
        t -= 1

def run():
    # # Lay ve idlist
    idList = craw_idList()
    priceDict = craw_priceDict()

    # # So Khop owner va thong bao
    compare_OldOwner()
    # #
    # #
    # #
    # # # Update id and owner and price
    toFilte_idList()
    toFile_owner()
    toFile_price()

preTotal = -1
curTotal = 0
i = 0
response = requests.post(url, json=body)
while 1:
    i += 1

    try:

        # curTotal = response.json()['data']['total']
        # if curTotal != preTotal:
        #     send_test_message('Total from ' + str(preTotal) + ' to ' + str(curTotal))
        #     preTotal = curTotal

        run()

        print('Done' + str(i) + ' ' + str(datetime.datetime.now().time()))

        if i == 1:
            send_test_message('Start!')

    except Exception as ex:
        print(ex)
        print('error in run')
        time.sleep(180)
        pass
    if i % 30 == 0:
        time.sleep(200)
    time.sleep(50)

    # Change user agent
    ua_list = [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 "
        "Safari/537.36 Edge/18.19577 "
        , "Mozilla/5.0 (X11) AppleWebKit/62.41 (KHTML, like Gecko) Edge/17.10859 Safari/452.6",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2656.18 "
        "Safari/537.36 "
        ,
        "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36",
        "Mozilla/5.0 (Linux; U; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13",
        "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 "
        "Safari/533.20.27 "
        ,
        "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) "
        "Version/5.0.4 Safari/533.20.27",
        "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1"
        ,
        "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5370a Safari/604.1"
        ,
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246"
        ,
        "Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36"
        ,
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9"
        ,
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36"
        ,
        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1"
    ]
    ua = random.choice(ua_list)
    headers1 = {'User-Agent': ua}
    try:
        response = requests.post(url, json=body, headers=headers1, timeout=10)
        response.close()
        print(response.json()['statusCode'])
    except:
        time.sleep(180)
        pass

    if response.json()['statusCode'] != 201:
        send_test_message('Break dude to api')
        break

    time.sleep(50)


from enum import Enum

import wx
from wx.lib.dialogs import MultiMessageDialog
from pubsub import pub

from pyxenoverse.gui import get_first_item, get_next_item
from pyxenoverse.bcs.color_selector import ColorSelector
from yabcs.dlg.find import FindDialog
from yabcs.utils import FIND_ITEM_TYPES, color_db

class Replace(Enum):
    NOT_REPLACED = 0
    REPLACED = 1
    SKIPPED = 2

class ReplaceDialog(FindDialog):
    def __init__(self, parent, entry_list, *args, **kw):
        super().__init__(parent, entry_list, *args, **kw)
        self.SetTitle("Replace")

        self.replace_ctrl = wx.TextCtrl(self, -1, '', style=wx.TE_PROCESS_ENTER)
        self.replace_ctrl.Bind(wx.EVT_TEXT_ENTER, self.on_replace)
        self.replace_ctrl.MoveAfterInTabOrder(self.find_ctrl)

        self.grid_sizer.Add(wx.StaticText(self, -1, 'Replace: '))
        self.grid_sizer.Add(self.replace_ctrl, 0, wx.EXPAND)

        self.replace_button = wx.Button(self, -1, "Replace")
        self.replace_button.Bind(wx.EVT_BUTTON, self.on_replace)
        self.replace_button.MoveAfterInTabOrder(self.find_button)
        self.replace_all_button = wx.Button(self, -1, "Replace All")
        self.replace_all_button.Bind(wx.EVT_BUTTON, self.on_replace_all)
        self.replace_all_button.MoveAfterInTabOrder(self.replace_button)

        self.button_sizer.Insert(1, self.replace_button, 0, wx.ALL, 2)
        self.button_sizer.Insert(2, self.replace_all_button, 0, wx.ALL, 2)

        self.sizer.Fit(self)
        self.Layout()

    def on_replace(self, _):
        if not color_db.bcs:
            self.status_bar.SetStatusText("BCS Not Loaded")
            return
        item_type, fields = FIND_ITEM_TYPES[self.items.GetSelection()]
        entry_type = fields[self.entry.GetSelection()]
        find = self.find_ctrl.GetValue()
        replace = self.replace_ctrl.GetValue()
        if "name" not in entry_type:
            try:
                find = int(find, 0)
                replace = int(replace, 0)
            except ValueError:
                self.status_bar.SetStatusText("Invalid Value")
                return
        selected = self.part_sets_list.GetSelections()

        # Only do this if we have don't have one selected item
        if len(selected) != 1:
            item, _ = get_first_item(self.part_sets_list)
            self.find(item, item_type, entry_type, find)
            return
        selected = selected[0]
        data = self.part_sets_list.GetItemData(selected)

        # Check to see if current entry is not one we're looking for
        res = self.replace_item(data, item_type, entry_type, find, replace)

        # Reload if replaced
        if res == Replace.REPLACED:
            self.main_panel.pages["Part Sets"].on_select(None)

        # Find next item to replace
        self.find(selected, item_type, entry_type, find)
        if res == Replace.REPLACED:
            self.status_bar.SetStatusText(f"Replaced 1 entry")
        elif res == Replace.SKIPPED:
            self.status_bar.SetStatusText(f"Skipped 1 entry. Check your part colors")

    def on_replace_all(self, _):
        if not color_db.bcs:
            self.status_bar.SetStatusText("BCS Not Loaded")
            return
        item_type, fields = FIND_ITEM_TYPES[self.items.GetSelection()]
        entry_type = fields[self.entry.GetSelection()]
        find = self.find_ctrl.GetValue()
        replace = self.replace_ctrl.GetValue()
        if "name" not in entry_type:
            try:
                find = int(self.find_ctrl.GetValue(), 0)
                replace = int(self.replace_ctrl.GetValue(), 0)
            except ValueError:
                self.status_bar.SetStatusText("Invalid Value")
                return
        count = 0
        skipped = 0
        skipped_entries = set()
        item, _ = get_first_item(self.part_sets_list)
        while item.IsOk():
            data = self.part_sets_list.GetItemData(item)
            res = self.replace_item(data, item_type, entry_type, find, replace, skipped_entries)
            if res == Replace.REPLACED:
                count += 1
            elif res == Replace.SKIPPED:
                skipped += 1
            item = get_next_item(self.part_sets_list, item)

        self.main_panel.pages["Part Sets"].on_select(None)
        pub.sendMessage('reindex_part_sets')
        msg = f'Replaced {count} entry(s) (skipped {skipped}). '
        if skipped:
            msg += "Check your part colors"
        self.status_bar.SetStatusText(msg)

        if item_type == ColorSelector and skipped_entries:
            if entry_type == "part_colors":
                msg = "\n".join(f" * Color Selector ({cs[0]}, {cs[1]}) -> ({replace}, {cs[1]})"
                                for cs in sorted(skipped_entries))
            else:
                msg = "\n".join(f" * Color Selector ({cs[0]}, {cs[1]}) -> ({cs[0]}, {replace})"
                                for cs in sorted(skipped_entries))
            with MultiMessageDialog(self, f"The following Color Selectors were skipped.\n"
                                          f"Please check your part colors.", "Warning", msg, wx.OK) as dlg:
                dlg.ShowModal()

    @staticmethod
    def get_color_selector_indexes(data, entry_type, replace):
        if entry_type == "part_colors":
            part_colors_index = replace
            color_index = data.color
        else:
            part_colors_index = data.part_colors
            color_index = replace
        return part_colors_index, color_index

    def replace_item(self, data, item_type, entry_type, find, replace, skipped=None):
        if type(data) == item_type:
            if item_type == ColorSelector and data[entry_type] == find:
                part_colors_index, color_index = self.get_color_selector_indexes(data, entry_type, replace)
                try:
                    value = color_db[part_colors_index][color_index]
                    data[entry_type] = replace
                except IndexError:
                    if skipped is not None:
                        skipped.add((data.part_colors, data.color))
                    return Replace.SKIPPED
            else:
                if isinstance(find, int) and data[entry_type] == find:
                    data[entry_type] = replace
                    return Replace.REPLACED
                elif isinstance(find, str) and find in data[entry_type]:
                    data[entry_type] = data[entry_type].replace(find, replace)
                    return Replace.REPLACED
        return Replace.NOT_REPLACED


from django.utils.translation import gettext_lazy as _
from rest_framework_simplejwt.authentication import JWTAuthentication
from rest_framework_simplejwt.exceptions import (
    AuthenticationFailed,
    InvalidToken,
    TokenError,
)
from rest_framework_simplejwt.state import User
from rest_framework_simplejwt.settings import api_settings

class APIAuthentication(JWTAuthentication):
    def get_user(self, validated_token):
        """
        Attempts to find and return a user using the given validated token.
        """
        try:
            user_id = validated_token[api_settings.USER_ID_CLAIM]
        except KeyError:
            raise InvalidToken(_("Token contained no recognizable user identification"))

        try:
            user = User.objects.get(**{api_settings.USER_ID_FIELD: user_id})
        except User.DoesNotExist:
            raise AuthenticationFailed(_("User not found"), code="user_not_found")

        if not user.is_active:
            raise AuthenticationFailed(_("User is inactive"), code="user_inactive")

        if not "access_token_secret" in validated_token:
            raise AuthenticationFailed(
                _("Bad Token used. Get new access Token"), code="get_new_access_token"
            )

        if (
            not user.profile.access_token_secret
            == validated_token["access_token_secret"]
        ):
            raise AuthenticationFailed(
                _("Bad Token used. Get new access Token"), code="get_new_access_token"
            )

        return user


import argparse, subprocess, os, glob

def main(args):

    outfile = os.path.join(args.wdir,'gt',str(args.conf),args.o+"_3dt.npz")
    if(os.path.isfile(outfile) and not args.overwrite):
        print("exists!")
        return

    # extract features from mpu
    command = [args.sure_dir + "/feat",
               "-w", str(args.wdir),
               "-i", str(args.i),
               "-o", str(args.o),
               "-g", str(args.g),
               "-s", "npz",
               "-e",""]
    print("run command: ", command)
    p = subprocess.Popen(command)
    p.wait()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='reconbench evaluation')

    parser.add_argument('-d', '--dataset_dir', type=str, default="/mnt/raphael/ModelNet10/",
                        help='working directory which should include the different scene folders.')
    parser.add_argument('--overwrite', type=int, default=0,
                        help='overwrite existing files')
    parser.add_argument('--sure_dir', type=str, default="/home/raphael/cpp/surfaceReconstruction/build/release",
                        help='Indicate the sure build directory, pointing to .../build/release folder starting from user_dir')
    parser.add_argument('--conf', type=int, default=42,
                        help='The scan conf')

    parser.add_argument('--category', type=str, default=None,
                        help='Indicate the category class')

    args = parser.parse_args()

    if args.category is not None:
        categories = [args.category]
    else:
        categories = os.listdir(args.dataset_dir)
    if 'x' in categories:
        categories.remove('x')
    if 'real' in categories:
        categories.remove('real')

    # scan all training data with random configuration from 0,1,2
    # and test data with 0,1,2

    ### scanner confs
    # 0 (easy) --cameras 15 --points 12000 --noise 0.000 --outliers 0.0
    # 1 (medium) --cameras 15 --points 3000 --noise 0.0025 --outliers 0.0
    # 2 (hard) --cameras 15 --points 12000 --noise 0.005 --outliers 0.33
    # 3 (convonet) --cameras 50 --points 3000 --noise 0.005 --outliers 0.0

    for idx,c in enumerate(categories):
        if c.startswith('.'):
            continue
        print("\n############## Processing {}/{} ############\n".format(idx+1,len(categories)))

        ### train
        args.input = os.listdir(os.path.join(args.dataset_dir, c, "3_scan", str(args.conf)))

        conf_dir = os.path.join(args.dataset_dir,c,"gt",str(args.conf))
        if not os.path.exists(conf_dir):
            os.makedirs(conf_dir)

        for i in args.input:
            i=i[:-4]
            args.wdir = os.path.join(args.dataset_dir, c)
            args.i = os.path.join("3_scan", str(args.conf),i)
            name = i.split('_')[1]
            args.o = os.path.join(name,i)
            args.g = "2_watertight/"+i+".off"

            try:
                main(args)

                # move
                in_dir =  os.path.join(args.dataset_dir,c,"gt",i+"*")
                out_dir = os.path.join(conf_dir,name)
                if not os.path.exists(out_dir):
                    os.makedirs(out_dir)
                for f in glob.glob(in_dir):
                    p = subprocess.Popen(["mv",f,out_dir])
                    p.wait()
            except:
                print("Problem with shape {}-{}".format(c,i))

        # # night_stand to nightstand
        # in_dir =  os.path.join(args.dataset_dir,c,"train")
        # fl = os.listdir(in_dir)
        # for f in fl:
        #     inf = os.path.join(in_dir,f)
        #     fn = f.split('/')[-1][11:]
        #     out = os.path.join(in_dir,"nightstand"+fn)
        #     p = subprocess.Popen(["mv",inf,out])
        #     p.wait()


from bika.lims.interfaces import IAnalysis
from bika.lims.interfaces import IReferenceAnalysis
from bika.lims.interfaces import IResultOutOfRange
from senaite import api
from senaite.api import _marker
from zope.component._api import getAdapters

def is_out_of_range(brain_or_object, result=_marker):
    """Checks if the result for the analysis passed in is out of range and/or
    out of shoulders range.

            min                                                   max
            warn            min                   max             warn
    ·········|---------------|=====================|---------------|·········
    ----- out-of-range -----><----- in-range ------><----- out-of-range -----
             <-- shoulder --><----- in-range ------><-- shoulder -->

    :param brain_or_object: A single catalog brain or content object
    :param result: Tentative result. If None, use the analysis result
    :type brain_or_object: ATContentType/DexterityContentType/CatalogBrain
    :returns: Tuple of two elements. The first value is `True` if the result is
    out of range and `False` if it is in range. The second value is `True` if
    the result is out of shoulder range and `False` if it is in shoulder range
    :rtype: (bool, bool)
    """
    analysis = api.get_object(brain_or_object)
    if not IAnalysis.providedBy(analysis) and \
            not IReferenceAnalysis.providedBy(analysis):
        api.fail("{} is not supported. Needs to be IAnalysis or "
                 "IReferenceAnalysis".format(repr(analysis)))

    if result is _marker:
        result = api.safe_getattr(analysis, "getResult", None)
    if not api.is_floatable(result):
        # Result is empty/None or not a valid number
        return False, False

    result = api.to_float(result)

    # Note that routine analyses, duplicates and reference analyses all them
    # implement the function getResultRange:
    # - For routine analyses, the function returns the valid range based on the
    #   specs assigned during the creation process.
    # - For duplicates, the valid range is the result of the analysis the
    #   the duplicate was generated from +/- the duplicate variation.
    # - For reference analyses, getResultRange returns the valid range as
    #   indicated in the Reference Sample from which the analysis was created.
    result_range = api.safe_getattr(analysis, "getResultsRange", None)
    if not result_range:
        # No result range defined or the passed in object does not suit
        return False, False

    # Maybe there is a custom adapter
    adapters = getAdapters((analysis,), IResultOutOfRange)
    for name, adapter in adapters:
        ret = adapter(result=result, specification=result_range)
        if not ret or not ret.get('out_of_range', False):
            continue
        if not ret.get('acceptable', True):
            # Out of range + out of shoulders
            return True, True
        # Out of range, but in shoulders
        return True, False

    # The assignment of result as default fallback for min and max guarantees
    # the result will be in range also if no min/max values are defined
    specs_min = api.to_float(result_range.get('min', result), result)
    specs_max = api.to_float(result_range.get('max', result), result)
    if specs_min <= result <= specs_max:
        # In range, no need to check shoulders
        return False, False

    # Out of range, check shoulders. If no explicit warn_min or warn_max have
    # been defined, no shoulders must be considered for this analysis. Thus, use
    # specs' min and max as default fallback values
    warn_min = api.to_float(result_range.get('warn_min', specs_min), specs_min)
    warn_max = api.to_float(result_range.get('warn_max', specs_max), specs_max)
    in_shoulder = warn_min <= result <= warn_max
    return True, not in_shoulder


import ipaddress
import subprocess
import time

import pylxd.exceptions

from lib.base_module import LXDBaseModule
import lib.validate as validate

INSTANCE_TEMPLATE = "nethop-{}"
PULL_SERVER = "https://images.linuxcontainers.org"
TEMPLATE_NAME = "hop_alpine_313"
PRETEMPLATE_NAME = "alpine_313"

class NetReservation(LXDBaseModule):

    def __init__(self, mm):
        self.mm = mm

    __FUNCS__ = {
        "list": {
            "_desc": "View network hops"
        },
        "add_network_hop": {
            "_desc": "Add a network hop",
            "front_ip": "IP",
            "fqdn": "TEXT",
            "net_addr": "IP_NETWORK",
            "description": "TEXT",
            "switch": "TEXT"
        },
        "remove_network_hop": {
            "_desc": "Remove a network hop",
            'id': "INTEGER"
        },
        "start_hop": {
            "_desc": "Start a hop router between networks",
            'id': "INTEGER"
        },
        "stop_hop": {
            "_desc": "Stop a hop router between networks. Will cut off communications.",
            'id': "INTEGER"
        },
    }

    __SHORTNAME__  = "nethop"
    __DESC__ = "Manages creating networks that are behind a next hop"
    __AUTHOR__ = "Jacob Hartman"

    def _vtysh_exec(self, commands):
        vtyshPath = subprocess.check_output(["/bin/sh", '-c', 'which vtysh']).strip().decode()
        full_command = [vtyshPath]
        for command in commands:
            full_command += ['-c', command]

        try:
            return subprocess.check_output(full_command).strip().decode()
        except subprocess.CalledProcessError as e:
            print(e)
            return None

    def run(self, func, **kwargs) :
        dbc = self.mm.db.cursor()
        if func == "list":
            dbc.execute("SELECT * FROM nethop;")
            results = dbc.fetchall()
            return None, {
                "rows": results,
                "columns": ["ID", "Range", "Description", "Switch"]
            }
        elif func == "add_network_hop":
            perror, _ = self.validate_params(self.__FUNCS__[func], kwargs)
            if perror is not None:
                return perror, None

            front_ip = kwargs['front_ip']
            fqdn = kwargs['fqdn']
            new_network = kwargs['net_addr']
            new_switch = kwargs['switch']
            description = kwargs['description']

            dbc.execute("SELECT hop_id FROM nethop WHERE fqdn=? OR front_ip=?", (fqdn,front_ip))
            if dbc.fetchone():
                return "A hop server already exists of that FQDN or IP", None

            # See if the switch for the front ip exists
            error, front_switch = self.mm['netreserve'].run("get_ip_switch", ip_addr=front_ip)

            if error is not None:
                return error, None

            # Allocate the front the IP address
            error, _ = self.mm['ipreserve'].run("add_ip", ip_addr=front_ip, description="Hop for network {}".format(new_network))
            if error is not None:
                return error, None

            # Configure the DNS name
            err, _ = self.mm['dns'].run("add_host", fqdn=fqdn, ip_addr=front_ip)
            if err is not None:
                return err, None

            # Add the hop network in netreserve
            error, _ = self.mm['netreserve'].run("add_hop_network", net_addr=new_network, description=description, switch=new_switch)
            if error is not None:
                return error, None

            # Add the hop to the database
            dbc.execute('INSERT INTO nethop (fqdn, front_ip, switch_name) VALUES (?, ?, ?)', (fqdn, front_ip, new_switch))
            self.mm.db.commit()

            # Our new hop id
            hop_id = dbc.lastrowid
            container_name = INSTANCE_TEMPLATE.format(hop_id)

            # Create the hop container
            try:
                container = self.mm.lxd.containers.create({
                    'name': container_name,
                    'source': {'type': 'image', 'alias': TEMPLATE_NAME},
                    'config': {

                    },
                    "devices": {
                        "eth0": {
                            "type": "nic",
                            "nictype": "bridged",
                            "parent": front_switch,
                            "name": "eth0"
                        },
                        "eth1": {
                            "type": "nic",
                            "nictype": "bridged",
                            "parent": new_switch,
                            "name": "eth1"
                        }
                    },
                }, wait=True)
            except pylxd.exceptions.LXDAPIException as e:
                return str(e), None

            error, _ =  self.run("start_hop", id=hop_id)
            if error != None:
                return error, None

            time.sleep(10)
            self.lxd_execute(container_name, "echo '' >> /etc/quagga/zebra.conf")
            self.lxd_execute(container_name, "echo 'router rip' >> /etc/quagga/ripd.conf")
            self.lxd_execute(container_name, "echo ' version 2' >> /etc/quagga/ripd.conf")
            self.lxd_execute(container_name, "echo ' network eth0' >> /etc/quagga/ripd.conf")
            self.lxd_execute(container_name, "echo ' network eth1' >> /etc/quagga/ripd.conf")
            self.lxd_execute(container_name, "echo ' redistribute connected' >> /etc/quagga/ripd.conf")
            self.lxd_execute(container_name, "service zebra restart")
            self.lxd_execute(container_name, "service ripd restart")
            self.lxd_execute(container_name, "service zebra restart")
            self.lxd_execute(container_name, "service ripd restart")

            nerror, is_hop = self.mm['netreserve'].run("is_hop_network_by_switch", switch=front_switch)
            if nerror is not None:
                return nerror, None

            if not is_hop:
                self.print("Not a hop, configuring local quagga...")

                self._vtysh_exec([
                    'config t',
                    'router rip',
                    'version 2',
                    'network {}'.format(front_switch),
                ])

                self._vtysh_exec([
                    'write mem'
                ])

            return None, hop_id

        elif func == "remove_network_hop":
            perror, _ = self.validate_params(self.__FUNCS__[func], kwargs)
            if perror is not None:
                return perror, None

            hop_id = kwargs['id']
            container_name = INSTANCE_TEMPLATE.format(hop_id)

            dbc.execute("SELECT front_ip, fqdn, switch_name FROM nethop WHERE hop_id=?", (hop_id,))
            result = dbc.fetchone()
            if not result:
                return "The hop does not exist", None

            front_ip = result[0]
            fqdn = result[1]
            switch_name = result[2]

            # Deallocate our IP address
            error, _ = self.mm['ipreserve'].run("remove_ip", ip_addr=front_ip)

            # Remove the host from the DNS server
            err, _ = self.mm['dns'].run("remove_host", fqdn=fqdn, ip_addr=front_ip)

            # Delete the hop container itself
            err, _ = self.lxd_delete(container_name)

            time.sleep(2)

            # Remove the container from the database
            dbc.execute("DELETE FROM nethop WHERE hop_id=?", (hop_id,))
            self.mm.db.commit()

            # Delete the switch if possible
            err, network_data = self.mm['netreserve'].run("get_network_by_switch", switch=switch_name)
            if err is not None:
                return err, None

            return self.mm['netreserve'].run("remove_network", id=network_data['net_id'])

        elif func == "start_hop":
            perror, _ = self.validate_params(self.__FUNCS__[func], kwargs)
            if perror is not None:
                return perror, None

            hop_id = kwargs['id']
            container_name = INSTANCE_TEMPLATE.format(hop_id)

            dbc.execute("SELECT front_ip, fqdn, switch_name FROM nethop WHERE hop_id=?", (hop_id,))
            result = dbc.fetchone()
            if not result:
                return "The hop does not exist", None

            front_ip = result[0]
            switch_name = result[2]

            serror, _ = self.lxd_start(container_name, front_ip)
            if serror is not None:
                return serror, None

            # Add second switch interface
            err, network_data = self.mm['netreserve'].run("get_network_by_switch", switch=switch_name)
            if err is not None:
                return err, None

            network_obj = ipaddress.ip_network(network_data['net_address'])

            err, _ = self.lxd_execute(container_name, "ip addr add {}/{} dev eth1".format(str(list(network_obj.hosts())[0]), str(network_obj.prefixlen)))
            if err is not None:
                return err, None

            return None, True
        elif func == "stop_hop":
            perror, _ = self.validate_params(self.__FUNCS__['stop_hop'], kwargs)
            if perror is not None:
                return perror, None

            hop_id = kwargs['id']
            container_name = INSTANCE_TEMPLATE.format(hop_id)

            return self.lxd_stop(container_name)
        else:
            return "Invalid function '{}.{}'".format(self.__SHORTNAME__, func), None

    def check(self):
        dbc = self.mm.db.cursor()

        dbc.execute("SELECT * FROM sqlite_master WHERE type='table' AND name='nethop';")
        if dbc.fetchone() is None:
            dbc.execute("CREATE TABLE nethop (hop_id INTEGER PRIMARY KEY, front_ip TEXT, fqdn TEXT, switch_name TEXT);")
            self.mm.db.commit()

    def build(self):

        self.print("Pulling down LXC image for hop...")

        remote_name = "alpine/3.13"

        try:
            self.mm.lxd.images.get_by_alias(PRETEMPLATE_NAME)
        except pylxd.exceptions.NotFound:

            self.print("Pulling in {} - {}".format(PULL_SERVER, remote_name))
            image = self.mm.lxd.images.create_from_simplestreams(PULL_SERVER, remote_name)

            found = False
            for alias in image.aliases:
                if alias['name'] == TEMPLATE_NAME:
                    found = True
            if not found:
                image.add_alias(name=PRETEMPLATE_NAME, description=PRETEMPLATE_NAME)

        lxd_net_config = {
            'ipv4.address': "10.100.10.1/24",
            'ipv4.nat': 'true',
            'ipv4.dhcp': 'true'
        }

        BUILD_SWITCH = "buildnet0"
        self.mm.lxd.networks.create(BUILD_SWITCH, config=lxd_net_config)

        hop_commands = [
            "apk add quagga openssh-server",
            "echo '' > /etc/network/interfaces",
            "echo 'auto eth0' >> /etc/network/interfaces",
            "echo 'iface eth0 inet manual' >> /etc/network/interfaces",
            "echo '' >> /etc/network/interfaces",
            "echo 'auto eth1' >> /etc/network/interfaces",
            "echo 'iface eth1 inet manual' >> /etc/network/interfaces",
            "echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config",
            "rc-update add zebra",
            "rc-update add ripd"
        ]

        ok = self.lxd_build_image(TEMPLATE_NAME, PRETEMPLATE_NAME, BUILD_SWITCH, hop_commands)
        if not ok:
            print(ok)

        lxd_network = self.mm.lxd.networks.get(BUILD_SWITCH)
        lxd_network.delete()

    def get_list(self):
        dbc = self.mm.db.cursor()

        dbc.execute("SELECT hop_id, front_ip, fqdn FROM nethop;")

        results = dbc.fetchall()
        new_list = []
        for container in results:
            new_data = ["lxd"]
            new_data += [container[0], container[1], container[2]]
            container_name = container[2].split(".")[0]
            _, status = self.lxd_get_status(container_name)
            new_data += status[1]
            new_list.append(new_data)
        return new_list

__MODULE__ = NetReservation


import sys
import random
import pygame
from entity.Entity import Entity
from entity.Tube import Tube
from util.Util import load_image

class DebugBall(Entity):
	"""
	A debug entity that bounces around and jumps through network tubes.
	"""
	def __init__(self):
		Entity.__init__(self)
		self.vx = 32
		self.vy = 128
		self.dummy = "DUMMY"
		self.texture = load_image("ball64.png")
		self.texture = pygame.transform.smoothscale(self.texture, (16,16))
		self.rect = pygame.Rect(0, 0, 16, 16)

	def serialize(self):
		e = Entity.serialize(self)
		e.update({
			"module":"entity.dummy.DebugBall",
			"class":"DebugBall",
			"dummy":self.dummy
		})
		return e

	@staticmethod
	def deserialize(serial):
		e = DebugBall()
		return DebugBall.rebuild(e, serial)

	@staticmethod
	def rebuild(e, serial):
		Entity.rebuild(e, serial)
		e.dummy = serial["dummy"]
		return e

	def update(self, delta):
		Entity.update(self, delta)
		self.x += self.vx * delta
		self.y += self.vy * delta
		if self.x < 0 or self.x > self.fishtank.size[0]:
			self.vx = -self.vx
			self.x = 0 if self.x < 0 else self.fishtank.size[0] if self.x > self.fishtank.size[0] else self.x
		if self.y < 0 or self.y > self.fishtank.size[1]:
			self.vy = -self.vy
			self.y = 0 if self.y < 0 else self.fishtank.size[1] if self.y > self.fishtank.size[1] else self.y

		for entity in self.fishtank.entities:
			if type(entity) is Tube and abs(self.x - entity.x) < 64 and abs(self.y - entity.y) < 64:
				entity.accept(self)
				break

	def draw(self, screen):
		Entity.draw(self, screen)
		#rect = self.texture.get_rect()
		self.rect.center = (int(self.x), int(self.y))
		screen.blit(self.texture, self.rect)


import requests

city = input('enter a city:')

url = "http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword"
data = {
    'cname': '',
    'pid': '',
    'keyword': city,
    'pageIndex': '1',
    'pageSize': '10'
}

headers = {
    'User-Agent': "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
}

response = requests.post(url, data, headers=headers)
json_text = response.text

print(json_text)


#!/usr/bin/env python3

import cv2
from matplotlib import pyplot as plt
import numpy as np
from sklearn import datasets, svm, metrics
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
import os
from PIL import Image
from joblib import dump, load
# from sklearn_porter import port, save, make, test

dataset_count = {}
imgs = []
labels = []
for subdir, dirs, files in os.walk("./dataset3"):
    for file in files:
      if file != ".DS_Store":
        img = Image.open(os.path.join(subdir, file)).resize(
            (32, 32), Image.LANCZOS).convert("1")
        # print(np.array(img).reshape((-1, 64)).flatten())
        imgs.append(np.array(img).reshape((1024, -1)).flatten())
        labels.append(subdir[-1])
        if subdir[-1] in dataset_count:
          dataset_count[subdir[-1]] += 1
        else:
          dataset_count[subdir[-1]] = 1

print(dataset_count)

# _, axes = plt.subplots(nrows=1, ncols=4, figsize=(10, 3))
# for ax, image, label in zip(axes, imgs, labels):
#     ax.set_axis_off()
#     ax.imshow(image, cmap=plt.cm.gray_r, interpolation="nearest")
#     ax.set_title("Training: %i" % label)

# clf = LogisticRegression(max_iter=400)
clf = KNeighborsClassifier(n_neighbors=1)
# clf = SGDClassifier(random_state=42, max_iter=1000, tol=1e-3)
X_train, X_test, y_train, y_test = train_test_split(
    np.array(imgs), np.array(labels), test_size=0.1, random_state=42, stratify=np.array(labels))

#--------- Initiate kNN, train it on the training data, then test it with the test data with k=1

# x = np.array(imgs)

# train = x[:500,:].reshape(-1, 64).astype(np.float32)  # Size = (2500,400)
# test = x[500:1000, :].reshape(-1, 64).astype(np.float32)  # Size = (2500,400)
# # Create labels for train and test data
# k = np.arange(10)
# train_labels = np.repeat(k, 250)[:, np.newaxis]
# test_labels = train_labels.copy()

# knn = cv2.ml.KNearest_create()
# knn.train(X_train, cv2.ml.ROW_SAMPLE, y_train)
# ret, result, neighbours, dist = knn.findNearest(X_test, k=5)

# matches = result == y_test
# correct = np.count_nonzero(matches)
# accuracy = correct*100.0/result.size
# print(accuracy)

#----------

print(X_train)

clf.fit(X_train, y_train)

predicted = clf.predict(X_test)

print(
    f"Classification report for classifier {clf}:\n"
    f"{metrics.classification_report(y_test, predicted)}\n"
)

disp = metrics.ConfusionMatrixDisplay.from_predictions(y_test, predicted)
disp.figure_.suptitle("Confusion Matrix")
print(f"Confusion matrix:\n{disp.confusion_matrix}")

dump(clf, "edu.pkl")
# # 2. Port or transpile an estimator:
# output = port(clf, language='js', template='attached')
# print(output)

# # 3. Save the ported estimator:
# src_path, json_path = save(
#     clf, language='js', template='exported', directory='/tmp')
# print(src_path, json_path)

# # 4. Make predictions with the ported estimator:
# y_classes, y_probas = make(clf, X_test, language='js', template='exported')
# print(y_classes, y_probas)

# # 5. Test always the ported estimator by making an integrity check:
# score = test(clf, X_test, language='js', template='exported')
# print(score)

plt.show()


import openai

def askPrompt(prompt):
    response = openai.Completion.create(
        engine="text-davinci-003",
        prompt=prompt,
        temperature=0.7,
        max_tokens=1500,
        top_p=1,
        frequency_penalty=0,
        presence_penalty=0,
    )

    return response.choices[0].text


from django.shortcuts import render, redirect,reverse
from create.models import Create
from django_summernote.widgets import SummernoteInplaceWidget
from django.views.generic import ListView, DetailView, CreateView, DeleteView
from create.forms import Title, BodyFrom
from create.models import Tag
from django.views import View
from django.shortcuts import get_object_or_404
from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
from django.utils import timezone
from django.contrib.auth.models import User
from django.contrib.auth.decorators import login_required
# Create your views here.

# class PostCreateView(CreateView):
#     model = Create
#     fields = ['text', 'blog']
#
#     def get_form(self, form_class=BodyFrom):
#         form = super(PostCreateView, self).get_form(BodyFrom)
#         form.fields['body'].widget = SummernoteInplaceWidget()
#         return form
#
#     def form_valid(self,form):
#         form.instance.author = self.request.user
#         keywords = self.request.POST['hidden-tags-value']
#
#         response = super().form_valid(form)
#
#         self.object.keywords = keywords
#         self.object.save()
#
#         keys = keywords.split(',')
#
#         for key in keys:
#             keyword = Tag.objects.create(post=self.object,tag=key,tag_upper_case=key.upper())
#
#
#         # return response
#         return redirect('index', pk=self.object.pk)

# def PostCreateView(requests):
#     return render(requests, 'create/create.html')

from tinymce.widgets import TinyMCE

class PostCreateView(CreateView):
    model = Create
    fields = ['text', 'blog', 'rank', 'category', 'metadata']

    def get_form(self, form_class=BodyFrom):
        form = super(PostCreateView, self).get_form(BodyFrom)
        form.fields['body'].widget = TinyMCE()
        return form

    def form_valid(self, form):

        form.instance.author = self.request.user

        while Create.objects.filter(author=self.request.user, rank=form.instance.rank):
            form.instance.rank = form.instance.rank + 1

        response = super().form_valid(form)

        keywords = self.request.POST['hidden-category-value']
        self.object.category = keywords
        self.object.save()

        keys = keywords.split(', ')

        for key in keys:
            keyword = Tag.objects.create(post=self.object,tag=key,tag_upper_case=key.upper())

        return response

    # def get_success_url(self):
    #     return reverse('index')

    def form_invalid(self,form):
        errors = form.errors
        for error in errors:
            print(error)

def postcreateview(request):
    if request.method == 'POST':
        form = BodyFrom(request.POST, instance=request.user.users)

        if form.is_valid():
            rank = form.cleaned_data['rank']
            body = form.cleaned_data['body']
            title = form.cleaned_data['title']
            return redirect('index')
    form = BodyFrom()
    context = {
        'form': form
    }
    return render(request, 'create/create.html', context)

class PostDetailView(DetailView):
    model = Create

class PostUpdateView(View):
    template_name = 'create/create_form_edit.html'

    def get(self, request, post_id):
        post = get_object_or_404(Create, pk=post_id, author=request.user)

        form = BodyFrom(initial= {'title': post.title, 'body': post.body, 'rank': post.rank, 'category': post.category, 'metadata': post.metadata})

        form.fields['body'].widget = TinyMCE()

        context = {
            'post': post,
            'form': form
        }

        return render(request, self.template_name, context)

    def post(self, request, post_id):
        ## Only Authors can edit their own posts
        post = get_object_or_404(Create, pk=post_id, author=request.user)
        form = BodyFrom(request.POST)

        if form.is_valid():
            post.title = form.cleaned_data['title']
            post.body = form.cleaned_data['body']
            post.rank = form.cleaned_data['rank']
            # post.category = form.cleaned_data['category']
            post.metadata = form.cleaned_data['metadata']
            post.keywords = self.request.POST['hidden-category-value']
            post.modified_date = timezone.now()

            post.save()

            ## Saving the kewwords
            keywords = self.request.POST['hidden-category-value']
            ## The best way is to re create them
            keys = Tag.objects.filter(post=post)
            for key in keys:
                key.delete()

            keys = keywords.split(', ')

            for key in keys:
                keyword = Tag.objects.create(post=post,tag=key,tag_upper_case=key.upper())

        return redirect('index')

class PostDeleteView( LoginRequiredMixin, UserPassesTestMixin, DeleteView):
    model = Create
    success_url = '/favorites/'

    def test_func(self):
        post = self.get_object()
        if self.request.user == post.author:
            return True
        return False


import struct
from typing import Any, cast
from collections.abc import Callable

from xpra.util.str_fn import ellipsizer
from xpra.os_util import hexstr, bytestostr
from xpra.log import Logger
log = Logger("x11")

class Unmanageable(Exception):
    pass

REPR_FUNCTIONS : dict[type, Callable] = {}

# Just to make it easier to pass around and have a helpful debug logging.
# Really, just a python objects where we can stick random bags of attributes.
class X11Event:
    def __init__(self, name: str):
        self.name: str = name

    def __repr__(self):
        d = {}
        for k, v in self.__dict__.items():
            if k in ("name", "display", "type"):
                continue
            if k in ("serial", "window", "delivered_to"):
                d[k] = f"{v:x}"
            elif k == "send_event" and v is False:
                continue
            else:
                fn = REPR_FUNCTIONS.get(type(v), str)
                d[k] = fn(v)
        return f"<X11:{self.name} {d!r}>"

# we duplicate some of the code found in gtk_x11.prop ...
# which is still better than having dependencies on that GTK here
def get_X11_window_property(xid: int, name: str, req_type: str):
    try:
        from xpra.x11.bindings.window import X11WindowBindings, PropertyError
        try:
            prop = X11WindowBindings().XGetWindowProperty(xid, name, req_type)
            log("get_X11_window_property(%#x, %s, %s)=%s, len=%s", xid, name, req_type, type(prop), len(prop or ()))
            return prop
        except PropertyError as e:
            log("get_X11_window_property(%#x, %s, %s): %s", xid, name, req_type, e)
    except Exception as e:
        log.warn(f"Warning: failed to get X11 window property {name!r} on window {xid:x}: {e}")
        log("get_X11_window_property%s", (xid, name, req_type), exc_info=True)
    return None

def get_X11_root_property(name: str, req_type: str):
    try:
        from xpra.x11.bindings.window import X11WindowBindings
        return get_X11_window_property(X11WindowBindings().get_root_xid(), name, req_type)
    except Exception as e:
        log("get_X11_root_property(%s, %s)", name, req_type, exc_info=True)
        log.warn(f"Warning: failed to get X11 root property {name!r}")
        log.warn(" %s", e)
    return None

def get_wm_name() -> str:
    try:
        wm_check = get_X11_root_property("_NET_SUPPORTING_WM_CHECK", "WINDOW")
        if wm_check:
            xid = struct.unpack(b"@L", wm_check)[0]
            log("_NET_SUPPORTING_WM_CHECK window=%#x", xid)
            wm_name = get_X11_window_property(xid, "_NET_WM_NAME", "UTF8_STRING")
            log("_NET_WM_NAME=%s", wm_name)
            if wm_name:
                return wm_name.decode("utf8")
    except Exception as e:
        log("get_wm_name()", exc_info=True)
        log.error("Error accessing window manager information:")
        log.estr(e)
    return ""

def get_icc_data() -> dict[str, Any]:
    icc : dict[str, Any] = {}
    try:
        data = get_X11_root_property("_ICC_PROFILE", "CARDINAL")
        if data:
            log("_ICC_PROFILE=%s (%s)", type(data), len(data))
            version = get_X11_root_property("_ICC_PROFILE_IN_X_VERSION", "CARDINAL")
            log("get_icc_info() found _ICC_PROFILE_IN_X_VERSION=%s, _ICC_PROFILE=%s",
                hexstr(version or ""), hexstr(data))
            icc |= {
                "source"    : "_ICC_PROFILE",
                "data"      : data,
            }
            if version:
                try:
                    version = ord(version)
                except TypeError:
                    pass
                icc["version"] = version
    except Exception as e:
        log.error("Error: cannot access `_ICC_PROFILE` X11 window property")
        log.estr(e)
        log("get_icc_info()", exc_info=True)
    log("get_x11_icc_data()=%s", icc)
    return icc

def get_current_desktop() -> int:
    v = -1
    d = None
    try:
        d = get_X11_root_property("_NET_CURRENT_DESKTOP", "CARDINAL")
        if d:
            v = struct.unpack(b"@L", d)[0]
    except Exception as e:
        log("get_current_desktop()", exc_info=True)
        log.error("Error: accessing `_NET_CURRENT_DESKTOP`:")
        log.estr(e)
    log("get_current_desktop() %s=%s", hexstr(d or ""), v)
    return v

def get_number_of_desktops() -> int:
    v = 0
    d = None
    try:
        d = get_X11_root_property("_NET_NUMBER_OF_DESKTOPS", "CARDINAL")
        if d:
            v = struct.unpack(b"@L", d)[0]
    except Exception as e:
        log("get_number_of_desktops()", exc_info=True)
        log.error("Error: accessing `_NET_NUMBER_OF_DESKTOPS`:")
        log.estr(e)
    v = max(1, v)
    log("get_number_of_desktops() %s=%s", hexstr(d or ""), v)
    return v

def get_workarea() -> tuple[int, int, int, int] | None:
    try:
        d = get_current_desktop()
        if d < 0:
            return None
        workarea = get_X11_root_property("_NET_WORKAREA", "CARDINAL")
        if not workarea:
            return None
        log("get_workarea() _NET_WORKAREA=%s (%s), len=%s",
            ellipsizer(workarea), type(workarea), len(workarea))
        # workarea comes as a list of 4 CARDINAL dimensions (x,y,w,h), one for each desktop
        sizeof_long = struct.calcsize(b"@L")
        if len(workarea) < (d+1)*4*sizeof_long:
            log.warn(f"Warning: invalid `_NET_WORKAREA` value length: {workarea!r}")
        else:
            cur_workarea = workarea[d*4*sizeof_long:(d+1)*4*sizeof_long]
            v = cast(tuple[int, int, int, int], struct.unpack(b"@LLLL", cur_workarea))
            log("get_workarea() %s=%s", hexstr(cur_workarea), v)
            return v
    except Exception as e:
        log("get_workarea()", exc_info=True)
        log.error("Error: querying the x11 workarea:")
        log.estr(e)
    return None

def get_desktop_names() -> tuple[str, ...]:
    v : tuple[str, ...] = ("Main", )
    d = None
    try:
        d = get_X11_root_property("_NET_DESKTOP_NAMES", "UTF8_STRING")
        if d:
            d = d.split(b"\0")
            if len(d) > 1 and d[-1] == b"":
                d = d[:-1]
            return tuple(x.decode("utf8") for x in d)
    except Exception as e:
        log.error("Error querying `_NET_DESKTOP_NAMES`:")
        log.estr(e)
    log("get_desktop_names() %s=%s", hexstr(d or ""), v)
    return v

def get_vrefresh() -> int:
    v = -1
    try:
        from xpra.x11.bindings.randr import RandRBindings
        randr = RandRBindings()
        if randr.has_randr():
            v = randr.get_vrefresh()
    except Exception as e:
        log("get_vrefresh()", exc_info=True)
        log.error("Error querying the display vertical refresh rate:")
        log.estr(e)
    log("get_vrefresh()=%s", v)
    return v

def send_client_message(window, message_type, *values) -> None:
    try:
        from xpra.x11.bindings.window import constants, X11WindowBindings  # @UnresolvedImport
        X11Window = X11WindowBindings()
        root_xid = X11Window.get_root_xid()
        if window:
            xid = window.get_xid()
        else:
            xid = root_xid
        SubstructureNotifyMask = constants["SubstructureNotifyMask"]
        SubstructureRedirectMask = constants["SubstructureRedirectMask"]
        event_mask = SubstructureNotifyMask | SubstructureRedirectMask
        X11Window.sendClientMessage(root_xid, xid, False, event_mask, message_type, *values)
    except Exception as e:
        log.warn(f"Warning: failed to send client message {message_type!r} with values={values}: {e}")

device_bell = None
def system_bell(window, device, percent:int, _pitch, _duration:int, bell_class, bell_id, bell_name:str) -> bool:
    global device_bell
    if device_bell is False:
        # failed already
        return False
    if device_bell is None:
        # try to load it:
        try:
            from xpra.x11.bindings.keyboard import X11KeyboardBindings
            device_bell = X11KeyboardBindings().device_bell
        except ImportError:
            log("x11_bell()", exc_info=True)
            log.warn("Warning: cannot use X11 bell device without the X11 bindings")
            return False
    device_bell(window.get_xid(), device, bell_class, bell_id, percent, bell_name)
    return True

def get_xsettings():
    from xpra.x11.bindings.window import X11WindowBindings  # @UnresolvedImport
    X11Window = X11WindowBindings()
    selection = "_XSETTINGS_S0"
    owner = X11Window.XGetSelectionOwner(selection)
    if not owner:
        return None
    XSETTINGS = "_XSETTINGS_SETTINGS"
    data = X11Window.XGetWindowProperty(owner, XSETTINGS, XSETTINGS)
    if not data:
        return None
    from xpra.x11.xsettings_prop import bytes_to_xsettings
    return bytes_to_xsettings(data)

def xsettings_to_dict(v) -> dict[str, tuple[int, Any]]:
    d : dict[str, tuple[int, Any]] = {}
    if v:
        _, values = v
        for setting_type, prop_name, value, _ in values:
            d[bytestostr(prop_name)] = (setting_type, value)
    return d

def get_randr_dpi() -> tuple[int, int]:
    from xpra.x11.bindings.randr import RandRBindings  # @UnresolvedImport
    randr_bindings = RandRBindings()
    if randr_bindings and randr_bindings.has_randr():
        wmm, hmm = randr_bindings.get_screen_size_mm()
        if wmm > 0 and hmm > 0:
            w, h = randr_bindings.get_screen_size()
            dpix = round(w * 25.4 / wmm)
            dpiy = round(h * 25.4 / hmm)
            log("xdpi=%s, ydpi=%s - size-mm=%ix%i, size=%ix%i", dpix, dpiy, wmm, hmm, w, h)
            return dpix, dpiy
    return -1, -1

def get_xresources() -> dict[str, str] | None:
    try:
        value = get_X11_root_property("RESOURCE_MANAGER", "STRING")
        log(f"RESOURCE_MANAGER={value}")
        if value is None:
            return None
        # parse the resources into a dict:
        values : dict[str, str] = {}
        options = bytestostr(value).split("\n")
        for option in options:
            if not option:
                continue
            parts = option.split(":\t", 1)
            if len(parts) != 2:
                log(f"skipped invalid option: {option!r}")
                continue
            values[parts[0]] = parts[1]
        return values
    except Exception as e:
        log(f"_get_xresources error: {e!r}")
    return None

def get_cursor_size() -> int:
    d = get_xresources() or {}
    try:
        return int(d.get("Xcursor.size", 0))
    except ValueError:
        return -1


import os

from jax import *
from jax.config import config
import pandas as pd

from hessians import outer_prod, loss_hessian
from architectures import fully_connected
from dataloader import DatasetTorch
from torch.utils.data import DataLoader
from jax.experimental.stax import softmax, logsoftmax
from initializers import *
import argparse
from data import get_dataset

parser = argparse.ArgumentParser()
parser.add_argument('--units', default='10,10', type=str)
parser.add_argument('--loss', default='mse', type=str)
parser.add_argument('--dataset', default='MNIST', type=str)
parser.add_argument('--batch_size', default=10, type=int)
parser.add_argument('--init', default='glorot', type=str)
parser.add_argument('--dim', default=64, type=int)
parser.add_argument('--K', default=10, type=int)

args = parser.parse_args()

os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
os.environ["CUDA_VISIBLE_DEVICES"] = ''

config.update("jax_enable_x64", True)
config.update('jax_platform_name', 'cpu')
key = random.PRNGKey(1)

n_train_max = 400

m = args.units.split(',')
depth = len(m)
K = args.K
m = [int(m[i]) for i in range(depth)]
d = args.dim
in_d = args.dim

init = get_init(args.init)

unit_string = str(d) + 'x'
for i in range(depth):
    unit_string += str(m[i]) + 'x'
unit_string += str(K)

bs = args.batch_size
iters = n_train_max // bs

n_trains = [bs + i * bs for i in range(iters)]

# Prepare dictionaries to store results
preds = {'rank_L': [], 'rank_F': [], 'rank_outer': []}
ranks = {'rank_L': [], 'rank_F': [], 'rank_outer': []}
ranks_cov = []

dir_path = os.path.dirname(os.path.realpath(__file__))
store_path = dir_path + '/results/store/samplesize/'
store_path += args.loss + '/' + args.dataset + '/' + unit_string + '/'
temp_path = dir_path + '/results/temporary/samplesize/'
temp_path += args.loss + '/' + args.dataset + '/' + unit_string + '/'

try:
    # Create directory to store results
    os.mkdir(store_path)
    print("Directory ", store_path,  " Created ")
except FileExistsError:
    print("Directory ", store_path,  " already exists")

try:
    # Create directory to store intermediate results
    os.mkdir(temp_path)
    print("Directory ", temp_path,  " Created ")
except FileExistsError:
    print("Directory ", temp_path,  " already exists")

# Set parameters in case we don't use cross entropy
K_form = K
cross = False

# Choose loss function and adapt parameters in case of cross entropy
if args.loss == 'mse':
    def loss(preds, targets):
        return 1 / 2 * jnp.sum((preds - targets) ** 2)

    def loss_params(params, inputs, targets):
        preds = apply_fn(params, inputs)

        return 1 / 2 * jnp.sum((preds - targets) ** 2)

if args.loss == 'cosh':
    def loss(preds, targets):
        return jnp.sum(jnp.log(jnp.cosh(preds - targets)))

    def loss_params(params, inputs, targets):
        preds = apply_fn(params, inputs)

        return loss(preds, targets)

if args.loss == 'cross':
    def loss(preds, targets):
        return -jnp.sum(logsoftmax(preds) * targets)

    def loss_params(params, inputs, targets):
        preds = apply_fn(params, inputs)

        return -jnp.sum(logsoftmax(preds) * targets)

    K_form = K - 1
    cross = True

# Define neural architecture
init_fn, apply_fn = fully_connected(units=m, classes=K, activation='linear', init=init)

# Initialize the parameters
_, params = init_fn(key, (-1, in_d))

# Make sure that parameters are float64, calculations are not precise enough otherwise
params = [jnp.double(param) for param in params]

if K == 1:
    # If we only have one output, we use two classes
    data = get_dataset(args.dataset, n_train=n_train_max, n_test=2, dim=d, classes=2)
else:
    data = get_dataset(args.dataset, n_train=n_train_max, n_test=2, dim=d, classes=K)

# Define the trainloader to perform computation in batches
train_loader = DataLoader(DatasetTorch(data.x_train, data.y_train), batch_size=bs, shuffle=False)

cov = jnp.zeros(shape=(in_d, in_d))

# Initialize the functions to compute the Hessians

def loss_hessian_input(batch): return loss_hessian(loss_params, params, batch[0], batch[1])

# Jit them for faster, repeated calculations
loss_hessian_input_jitted = jit(loss_hessian_input)

def outer_hessian_input(batch): return outer_prod(loss, apply_fn, params, batch[0], batch[1],  cross=cross)

outer_hessian_input_jitted = jit(outer_hessian_input)

i = 0
for batch_input, batch_label in train_loader:
    batch = (batch_input.numpy(), batch_label.numpy())
    H_L_batched = loss_hessian_input_jitted(batch)

    if i == 0:
        H_L = H_L_batched
    else:
        H_L = jnp.load(temp_path + 'H_L.npy')
        H_L += H_L_batched

    # Calculate the rank
    rank_L = jnp.linalg.matrix_rank(H_L)
    ranks['rank_L'].append(rank_L)

    # Store it temporarily
    jnp.save(temp_path + 'H_L', H_L)
    # Free memory
    del H_L
    # Calculate outer product
    outer_batched = outer_hessian_input_jitted(batch)
    if i == 0:
        outer = outer_batched
    else:
        outer = jnp.load(temp_path + 'outer.npy')
        outer += outer_batched

    # Calculate the rank
    rank_outer = jnp.linalg.matrix_rank(outer)
    ranks['rank_outer'].append(rank_outer)

    # Calculate the functional hessian, call it H_F to do in-place assignments
    H_F = jnp.load(temp_path + 'H_L.npy')
    H_F = H_F - outer

    # Store it temporarily
    jnp.save(temp_path + 'outer', outer)
    del outer

    # Calculate the rank
    rank_F = jnp.linalg.matrix_rank(H_F)
    ranks['rank_F'].append(rank_F)

    # Update the covariance matrix
    cov_new = batch_input.T @ batch_input
    cov += cov_new.numpy()

    # Calculate the rank
    rank_cov = jnp.linalg.matrix_rank(cov)
    ranks_cov.append(rank_cov)

    # Calculate the corresponding formulas from the paper
    q_rk = jnp.min(jnp.array([rank_cov, K_form]))
    q_all = jnp.min(jnp.array([rank_cov, K_form] + m))
    pred_F = 2 * q_all * sum(m) + 2 * q_all * q_rk - (len(m)+1) * q_all**2
    pred_outer = (rank_cov + K_form - q_all) * q_all
    pred_L = pred_F + pred_outer + q_all * (q_all - 2 * q_rk)

    preds['rank_L'].append(pred_L)
    preds['rank_F'].append(pred_F)
    preds['rank_outer'].append(pred_outer)

    print('Iteration ' + str(i) + ' out of ' + str(iters))
    i += 1

# Save the ranks to the directory
rank_L_frame = pd.DataFrame({'n':  n_trains, 'Rank': jnp.array(ranks['rank_L'])}, dtype=float)
rank_L_frame.to_pickle(path=store_path + 'rank_L')

preds_F_frame = pd.DataFrame({'n': n_trains, 'Pred': jnp.array(preds['rank_F'])}, dtype=float)
preds_F_frame.to_pickle(path=store_path + 'pred_F')

preds_L_frame = pd.DataFrame({'n': n_trains, 'Pred': jnp.array(preds['rank_L'])}, dtype=float)
preds_L_frame.to_pickle(path=store_path + 'pred_L')

preds_outer_frame = pd.DataFrame({'n': n_trains, 'Pred': jnp.array(preds['rank_outer'])}, dtype=float)
preds_outer_frame.to_pickle(path=store_path + 'pred_outer')

rank_F_frame = pd.DataFrame({'n': n_trains, 'Rank': jnp.array(ranks['rank_F'])}, dtype=float)
rank_F_frame.to_pickle(path=store_path + 'rank_F')

rank_outer_frame = pd.DataFrame({'n': n_trains, 'Rank': jnp.array(ranks['rank_outer'])}, dtype=float)
rank_outer_frame.to_pickle(path=store_path + 'rank_outer')