# -*- coding: utf-8 -*-

import logging

import numpy as np
from keras import backend as K
from keras.engine import Layer, InputSpec
from keras.layers import Flatten
from keras import initializers, regularizers, constraints
import tensorflow as tf

logger = logging.getLogger(__name__)

def _softmax(x, axis):
    ''' Computes softmax along a specified axis. '''
    if K.backend() == 'tensorflow':
        import tensorflow as tf
        return tf.nn.softmax(x, axis)
    elif K.backend() == 'cntk':
        import cntk
        return cntk.softmax(x, axis)
    elif K.backend() == 'theano':
        # Theano cannot softmax along an arbitrary dim.
        # So, we will shuffle `dim` to -1 and un-shuffle after softmax.
        perm = np.arange(K.ndim(x))
        perm[axis], perm[-1] = perm[-1], perm[axis]
        x_perm = K.permute_dimensions(x, perm)
        output = K.softmax(x_perm)

        # Permute back
        perm[axis], perm[-1] = perm[-1], perm[axis]
        output = K.permute_dimensions(x, output)
        return output
    else:
        raise ValueError("Backend '{}' not supported".format(K.backend()))

class AttentionLayer(Layer):
    ''' Attention layer that computes a learned attention over input sequence.

    For details, see papers:
    - https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf
    - http://colinraffel.com/publications/iclr2016feed.pdf (fig 1)

    Input:
      x: Input tensor for shape `(..., timesteps, features)` where `features`
      must be static (known).

    Output:
      2D tensor of shape `(..., features)`, i.e., `timesteps` axis is attended
      over and reduced.
    '''

    def __init__(self,
                 kernel_initializer='he_normal',
                 kernel_regularizer=None,
                 kernel_constraint=None,
                 use_bias=True,
                 bias_initializer='zeros',
                 bias_regularizer=None,
                 bias_constraint=None,
                 use_context=True,
                 context_initializer='he_normal',
                 context_regularizer=None,
                 context_constraint=None,
                 attention_dim=None,
                 **kwargs):
        '''
        Args:
          attention_dim: The dimensionality of the inner attention calculating
            neural network.
            For input `(32, 10, 300)`, with `attention_dim` of 100, the output
            is `(32, 10, 100)`, i.e., the attended words are 100 dimensional.
            This is then collapsed via summation to `(32, 10, 1)` to indicate
            the attention weights for 10 words.
            If set to None, `features` dims are used as `attention_dim`.
            (Default value: None)
        '''
        if 'input_shape' not in kwargs and 'input_dim' in kwargs:
            kwargs['input_shape'] = (kwargs.pop('input_dim'),)

        super(AttentionLayer, self).__init__(**kwargs)
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)

        self.use_bias = use_bias
        self.bias_initializer = initializers.get(bias_initializer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.bias_constraint = constraints.get(bias_constraint)

        self.use_context = use_context
        self.context_initializer = initializers.get(context_initializer)
        self.context_regularizer = regularizers.get(context_regularizer)
        self.context_constraint = constraints.get(context_constraint)

        self.attention_dim = attention_dim
        self.supports_masking = True

    def build(self, input_shape):
        # if len(input_shape) < 3:
        if len(input_shape) != 3:
            raise ValueError("Expected input shape of "
                             "`(..., timesteps, features)`, found `{}`"
                             .format(input_shape))

        attention_dim = input_shape[-1] if self.attention_dim is None \
            else self.attention_dim
        self.kernel = self.add_weight(name='kernel',
                                      shape=(input_shape[-1], attention_dim),
                                      initializer=self.kernel_initializer,
                                      regularizer=self.kernel_regularizer,
                                      constraint=self.kernel_constraint)
        if self.use_bias:
            self.bias = self.add_weight(name='bias',
                                        shape=(attention_dim,),
                                        initializer=self.bias_initializer,
                                        regularizer=self.bias_regularizer,
                                        constraint=self.bias_constraint)
        else:
            self.bias = None
        if self.use_context:
            self.context_kernel = self.add_weight(
                name='context_kernel',
                shape=(attention_dim,),
                initializer=self.context_initializer,
                regularizer=self.context_regularizer,
                constraint=self.context_constraint)
        else:
            self.context_kernel = None
        super(AttentionLayer, self).build(input_shape)

    def call(self, inputs, mask=None):
        # inputs: [..., timesteps, features]
        # ut : [..., timesteps, attention_dim]
        ut = K.dot(inputs, self.kernel)
        if self.use_bias:
            ut = K.bias_add(ut, self.bias)
        ut = K.tanh(ut)
        if self.use_context:
            ut = ut * self.context_kernel

        # Collapse `attention_dim` to 1. This indicates the weight for each
        # timestep.
        ut = K.sum(ut, axis=-1, keepdims=True)
        # ut = K.sum(ut, axis=-1)

        # Convert those weights into a distribution along timestep axis.
        # i.e., sum of alphas along `timesteps` axis should be 1.
        self.at = _softmax(ut, axis=1)
        # self.at = K.expand_dims(K.softmax(ut), -1)
        if mask is not None:
            self.at *= K.cast(K.expand_dims(mask, -1), K.floatx())

        # Weighted sum along `timesteps` axis.
        return K.sum(inputs * self.at, axis=-2)

    def comput_mask(self, inputs, input_mask=None):
        # Do not pass the mask to the next layers
        return None

    def compute_output_shape(self, input_shape):
        return input_shape[0], input_shape[-1]
        # input_shape = list(input_shape)
        # input_shape[-2] = input_shape[-1]
        # return tuple(input_shape[:-1])

    def get_attention_tensor(self):
        if not hasattr(self, 'at'):
            raise ValueError('Attention tensor is available after calling this'
                             ' layer with an input')
        return self.at

    def get_config(self):
        config = {
            'kernel_initializer':
            initializers.serialize(self.kernel_initializer),
            'kernel_regularizer':
            regularizers.serialize(self.kernel_regularizer),
            'kernel_constraint':
            constraints.serialize(self.kernel_constraint),
            'bias_initializer':
            initializers.serialize(self.bias_initializer),
            'bias_regularizer':
            regularizers.serialize(self.bias_regularizer),
            'bias_constraint':
            constraints.serialize(self.bias_constraint),
            'context_initializer':
            initializers.serialize(self.context_initializer),
            'context_regularizer':
            regularizers.serialize(self.context_regularizer),
            'context_constraint':
            constraints.serialize(self.context_constraint)
        }
        base_config = super(AttentionLayer, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

class ConsumeMask(Layer):
    ''' Layer that prevents mask propagation. '''

    def compute_mask(self, inputs, input_mask=None):
        # Do not pass the mask to the next layer.
        return None

    def call(self, inputs, mask=None):
        return inputs

class GlobalKMaxPooling1D(Layer):
    ''' Global k-max pooling operartion for temporal data.

    # Input shape
      3D tensor with shape: `(batch_size, steps, features)`.

    # Output shape
      2D tensor with shape: `(batch_size, k * features).
    '''

    def __init__(self, k=1, **kwargs):
        super(GlobalKMaxPooling1D, self).__init__(**kwargs)
        self.k = k
        self.input_spec = InputSpec(ndim=3)

    def compute_output_shape(self, input_shape):
        return (input_shape[0], input_shape[2] * self.k)

    def call(self, inputs):

        # Swap last two dimensions since top_k will be applied along the last
        # dimension.
        shifted_inputs = tf.transpose(inputs, [0, 2, 1])

        # Extract top_k, returns two tensors [values, indices]
        top_k = tf.nn.top_k(shifted_inputs, k=self.k, sorted=False,
                            name=None)[0]
        return K.batch_flatten(top_k)

    def get_config(self):
        config = {'k': self.k}
        base_config = super(GlobalKMaxPooling1D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


import os
import time
import argparse
import inspect
import json
import torch
from pathlib import Path

from torch.nn import functional as F

from dataclasses import dataclass
from utils.registry import Model, Dataset
from models import GPT2, ButterflyGPT
from contextlib import nullcontext

@dataclass
class TrainingConfig:
    log_interval: int = 10
    validation_interval: int = 200
    validation_iters: int = 100
    batch_size: int = 64
    gradient_accumulation_steps: int = 1
    model: str = 'GPT2'
    name: str = 'athena'
    dataset: str = None
    learning_rate: float = 6e-4
    rate_decay: float = 0.99
    decay_iters: int = 600000                   # should be ~= max_iters per Chinchilla
    min_rate: float = 6e-5                      # minimum learning rate, should be ~= learning_rate/10 per Chinchilla
    weight_decay: float = 1e-1
    beta1: float = 0.9
    beta2: float = 0.95
    grad_clip: float = 1.0
    seed: int = 1337
    max_iters: int = 5000

@dataclass
class TrainingState:
    iter: int = 0
    validation_loss: float = 1e10

def LoadCheckpoint(file, device='cuda'):
    checkpoint = torch.load(file, map_location=device)
    config = checkpoint['config']
    model = CreateModel(config.model)
    model.load_state_dict(checkpoint['model'])

    return model, config

def CreateModel(modelname):
    if modelname not in Model.registry:
        raise Exception(f'Unknown model "{modelname}"')
    return Model.registry[modelname]()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Load language model and generate text')
    parser.add_argument('--checkpoint', type=argparse.FileType('rb'), help='Path to a checkpoint to resume training from')
    parser.add_argument('--seed', type=int, default=1337, help="Random seed to init torch with")
    parser.add_argument('--device', type=str, default='cuda', help='Device to train model on')
    parser.add_argument('--compile', type=bool, default=True, help='Compile model before training or not')
    parser.add_argument('--dtype', type=str, default='bfloat16', choices=['bfloat16', 'float16', 'float32'])
    parser.add_argument('--prompt', type=str, default='\n')
    parser.add_argument('--length', default=128, type=int)
    parser.add_argument('--num_samples', default=1, type=int)
    parser.add_argument('--temperature', default=0.8, type=float)

    arguments = parser.parse_args()

    model, config = LoadCheckpoint(arguments.checkpoint)
    config.seed = arguments.seed

    n_params = sum(p.numel() for p in model.parameters())
    n_params -= model.emb.weight.numel()
    print(f'Model with {n_params/1e6:.2f} M parameters')

    device = arguments.device
    dtype = arguments.dtype
    device_type = 'cuda' if 'cuda' in device else 'cpu'

    torch.manual_seed(config.seed)
    torch.backends.cuda.matmul.allow_tf32 = True    # allow tf32 on matmul
    torch.backends.cudnn.allow_tf32 = True          # allow tf32 on cudnn

    ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]
    scaler = torch.cuda.amp.GradScaler(enabled=(dtype == 'float16'))
    ctx = nullcontext() if device_type == 'cpu' else torch.amp.autocast(device_type=device_type, dtype=ptdtype)

    model.to(device)

    if arguments.compile:
        try:
            model = torch.compile(model)
        except Exception as e:
            print(f'Model compilation failed - {str(e)}')
            pass

    x = torch.ByteTensor(list(bytes(arguments.prompt, 'utf8'))).type(torch.int64).to(device)
    x = x.unsqueeze(0)
    decode = lambda l: ''.join([chr(i) for i in l.tolist()[0]])

    with torch.no_grad():
        with ctx:
            for _ in range(arguments.num_samples):
                sample = x
                for _ in range(arguments.length):
                    logits, _ = model(sample)
                    logits = logits[:, -1, :] / arguments.temperature
                    probs = F.softmax(logits, dim=-1)
                    s = torch.multinomial(probs, num_samples=1)
                    sample = torch.cat((sample, s), dim=1)
                print('------')
                print(decode(sample))


import os
import os.path as osp

import torch
import numpy as np
from PIL import Image
from torch.utils.data import Dataset
from torchvision import transforms

class DatasetLoader(Dataset):

    def __init__(self, setname, args=None):
        DATASET_DIR = os.path.join(args.data_dir, 'FC100/')
        if setname == 'train':
            THE_PATH = osp.join(DATASET_DIR, 'train')
            label_list = os.listdir(THE_PATH)
        elif setname == 'test':
            THE_PATH = osp.join(DATASET_DIR, 'test')
            label_list = os.listdir(THE_PATH)
        elif setname == 'val':
            THE_PATH = osp.join(DATASET_DIR, 'val')
            label_list = os.listdir(THE_PATH)
        else:
            raise ValueError('Wrong setname.')
        self.wnids = label_list

        data = []
        label = []

        if 'num_patch' not in vars(args).keys():
            print('no num_patch parameter, set as default: 9')
            self.num_patch = 9
        else:
            self.num_patch = args.num_patch

        folders = [osp.join(THE_PATH, label) for label in label_list if os.path.isdir(osp.join(THE_PATH, label))]

        for idx, this_folder in enumerate(folders):
            this_folder_images = os.listdir(this_folder)
            for image_path in this_folder_images:
                data.append(osp.join(this_folder, image_path))
                label.append(idx)
        self.data = data
        self.label = label
        self.num_class = len(set(label))

        image_size = 84
        self.transform = transforms.Compose([
            transforms.RandomResizedCrop(image_size),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            transforms.Normalize(np.array([x / 255.0 for x in [125.3, 123.0, 113.9]]),
                                 np.array([x / 255.0 for x in [63.0, 62.1, 66.7]]))
        ])

    def __len__(self):
        return len(self.data)

    def __getitem__(self, i):
        path, label = self.data[i], self.label[i]
        patch_list = []
        for _ in range(self.num_patch):
            patch_list.append(self.transform(Image.open(path).convert('RGB')))
        patch_list = torch.stack(patch_list, dim=0)
        return patch_list, label

if __name__ == '__main__':
    pass


import time
from kafka import KafkaConsumer

SERVER = ['121.196.16.204:9091','121.196.16.204:9092','121.196.16.204:9093']
TOPIC = 'rpa-message'
print(("kafka server is {},topic is [{}]".format(SERVER, TOPIC)))
consumer = KafkaConsumer(
    TOPIC,
    bootstrap_servers=SERVER,
    group_id='rpa-worker',
    auto_offset_reset='earliest',
    enable_auto_commit=False,
    consumer_timeout_ms=3 * 1000,
    # max_poll_interval_ms= 50*60*1000,
    # max_poll_records = 100

)

while 1:
    try:
        message = next(consumer)
        consumer.commit()
        print(message)
    except StopIteration:
        time.sleep(.5)

kafka消费太多重复信息.py


from time import time
import hashlib
import json
from urllib.parse import urlparse
import requests

class Blockchain:
    # Constructor
    def __init__(self):
        self.chain = []  # Normally a linked list, but we're lazy. So.
        self.nodes = set()  # Address book of neighbor nodes
        self.current_transactions = []  # Transactions that haven't been posted onto a block

        # Create the first (genesis) block of the chain
        self.new_block(previous_hash='1', proof=100)

    # Function to register a neighboring node (other user) in personal address book
    def register_node(self, address):
        parsed_url = urlparse(address)  # Parse value into usable format

        if parsed_url.netloc:  # If there is a specific net domain, e.g. "example.com"
            self.nodes.add(parsed_url.netloc)  # Add the net location
        elif parsed_url.path:  # If URL has 'IP address' schema, e.g. "192.168.1.1:3000"
            self.nodes.add(parsed_url.path)  # Add the IP path
        else:  # If no URL provided
            raise ValueError("Invalid address")

    # Function to check if a chain is valid
    def valid_chain(self, chain):
        last_block = chain[0]  # We'll iterate through all blocks
        current_index = 1
        # Normally, we should be checking all of the proofs of work.
        # But instead we're going to check the hash values lining up

        while current_index < len(chain):
            block = chain[current_index]  # Temp. store current block
            last_block_hash = self.hash(last_block)  # Store last hash

            # If hash pointer is wrong, it's not a valid chain.
            if block["previous_hash"] != last_block_hash:
                return False

            last_block = block  # Iterate to next block in chain
            current_index += 1

        # At this point, all of the blocks have been checked and are NOT INvalid
        return True

    # Function to resolve conflicts between neighbors' chains
    def resolve_conflicts(self):
        neighbors = self.nodes  # Load address book into mutable temp. store
        new_chain = None  # Temp. store for neighbors' chains

        max_length = len(self.chain)

        for node in neighbors:  # Get all chains from all neighbor servers
            response = requests.get(f"http://{node}/chain")

            if response.status_code == 200:  # OK status
                length = response.json()["length"]
                chain = response.json()["chain"]

                # The longest chain is going to be the correct one, for this example
                if length > max_length and self.valid_chain(chain):
                    max_length = length
                    new_chain = chain

        # At this point, if a new chain has not been found, the new_chain will still have None from initialization
        # None always coerces to False. But any other value will coerce to True
        if new_chain:
            self.chain = new_chain  # Replace the chain
            return True

        return False  # The chain wasn't replaced

    # Function to create a new block and add it to the chain
    def new_block(self, proof, previous_hash):
        block = {
            "index": len(self.chain) - 1,  # Point in chain
            "timestamp": time(),  # Uses date/time library
            "transactions": self.current_transactions,
            "proof": proof,
            "previous_hash": previous_hash or self.hash(self.last_block)
        }

        # Reset the list of our un-tracked transactions
        self.current_transactions = []
        # Append the new block to our chain
        self.chain.append(block)
        # Return our block object (for other uses later)
        return block

    # Function to create a new sale/transaction of money
    def new_transaction(self, sender, recipient, amount):
        self.current_transactions.append({  # Skipping a step and appending straight
            "sender": sender,
            "recipient": recipient,
            "amount": amount
        })

        # Return the index of the new block w/ this transaction
        return self.last_block["index"] + 1

    # Function to get the last block of a chain (think peek from linked list)
    @property
    def last_block(self):
        return self.chain[-1]

    # Function to create a cryptographic hash
    @staticmethod  # This function won't be re-made every time we make a new Blockchain
    def hash(block):
        # Put all JSON metadata into encoded string
        strBlock = json.dumps(block, sort_keys=True).encode()
        # Return SHA256 hash in hex
        return hashlib.sha256(strBlock).hexdigest()

    # Function to create the proof of work of a block
    def proof_of_work(self, last_block):
        last_proof = last_block["proof"]  # Last proof of work
        last_hash = self.hash(last_block)  # Recalc last hash

        proof = 0
        # Don't actually do this... This is for simplification purposes.
        while self.valid_proof(last_proof, proof, last_hash) is False:
            proof += 1  # Iterates proof values instead of calculating
            # Will keep iterating until a proof works and is valid
            # Which means this could theoretically take infinite time

        return proof

    # Function to check if a proof of work is valid
    @staticmethod
    def valid_proof(last_proof, proof, last_hash):
        # Create a proof based on these encoded values
        guess = f"{last_proof}{proof}{last_hash}".encode()
        guess_hash = hashlib.sha256(guess).hexdigest()

        # A valid proof will have first four bits zeroes, and return True
        return guess_hash[:4] == "0000"


from sklearn.model_selection import train_test_split
from data_ingestion import data_loader
from data_preprocessing import preprocessing,clustering
from best_model_finder import tuner
from file_operations import file_methods
from application_logging import logger

class trainModel:
    def __init(self):
        self.log_writer = logger.App_logger()
        self.file_object = open('Training_logs/ModelTraining.txt')

    def trainingModel(self):

        #Logging start train
        self.log_writer.log(self.file_object,"start training....")
        try:
            # getting the data from the source.
            data_getter = data_loader.Data_Getter(self.file_object,self.log_writer)
            data = data_getter.get_data()

            # data preprocessing....
            X,Y = preprocesseur.is_null_present()
            is_null_present= preprocessor.is_null_present(X)

            # if missing values are ther we have to replace them
            if (is_null_present):
                X = preprocesseur.input_missing_values(X)
            # check the columns that not contribut to predictions, if std(column)==0 (i.e) the amount of variation is null (i.e) do not contribute to predictions.
            # so this columns have to be droped
            columns_to_drop = preprocessor.get_columns_with_zero_std(X)

            #drop
            X = preprocessor.remove_columns(X,columns_to_drop)

            # Clustring
            #A fundamental step for any unsupervised algorithm is to determine the optimal number
            # of clusters into which the data may be clustered.
            # Since we do not have any predefined number of clusters in unsupervised learning.
            k_means = clustering.KMeansClustring(self.file_object,self.log_writer)
            number_of_clusters = k_means.elbow_plot(X) # In cluster analysis, the elbow method is a heuristic used
                                                        #in determining the number of clusters in a data set. The method consists of plotting
                                                        #  the explained variation as a function of the number of clusters and picking the elbow of
                                                        #  the curve as the number of clusters to use [[ pt fr flexion de la courbe]]

            # devide data into "k_means" clusters
            X = k_means.create_clusters(X,number_of_clusters)
            X['Labels'] = Y
            list_of_clusters = X['Cluster'].unique()

            # choose the best ML model to fit on individual cluster #

            for i in list_of_clusters:
                cluster_data=X[X['Cluster']==i] # filter the data for one cluster

                # Prepare the feature and Label columns
                cluster_features=cluster_data.drop(['Labels','Cluster'],axis=1)
                cluster_label= cluster_data['Labels']

                # splitting the data into training and test set for each cluster one by one
                x_train, x_test, y_train, y_test = train_test_split(cluster_features, cluster_label, test_size=1 / 3, random_state=355)

                model_finder=tuner.Model_Finder(self.file_object,self.log_writer) # object initialization

                #getting the best model for each of the clusters
                best_model_name,best_model=model_finder.get_best_model(x_train,y_train,x_test,y_test)

                #saving the best model to the directory.
                file_op = file_methods.File_Operation(self.file_object,self.log_writer)
                save_model=file_op.save_model(best_model,best_model_name+str(i))

            # logging the successful Training
            self.log_writer.log(self.file_object, 'Successful End of Training')
            self.file_object.close()

        except Exception:
            # logging the unsuccessful Training
            self.log_writer.log(self.file_object, 'Unsuccessful End of Training')
            self.file_object.close()
            raise Exception


from ..utils import inch_to_base_unit, create_polygon, create_base_object
from ..coord import min_x_coord, max_x_coord, min_y_coord, max_y_coord

MINIMUM_BAY = inch_to_base_unit(12.0)
WIRE_SPACE = inch_to_base_unit(1.0)

WIRE_WIDTH = inch_to_base_unit(0.1)

# BUILDING_LAYER is a constant for the input layer name
BUILDING_LAYER = "joist"

# ANNOTATIONS_LAYER is a constant for the output layer name
ANNOTATIONS_LAYER = "annotations/joistBayLines"

class JoistBayLinesGenerator:
    def __init__(self, variation, params={}):
        self.variation = variation
        self.params = params
        self.generated_layers = self.params["layers"]
        self.changes = set()

    def generate(self):
        interior_objects = self.generated_layers[BUILDING_LAYER]["objects"]
        new_layer = self.variation.add_layer(ANNOTATIONS_LAYER)
        self.changes.add(ANNOTATIONS_LAYER)

        for i in range(len(interior_objects) - 1):
            obj = interior_objects[i]
            next_obj = interior_objects[i+1]

            min_x = max_x_coord(obj)
            max_x = min_x_coord(next_obj)
            min_y = min(min_y_coord(obj), min_y_coord(next_obj))
            max_y = max(max_y_coord(obj), max_y_coord(next_obj))

            # if bay is at least MINIMUM_BAY, create joist bay polygon
            # vertical bay
            width = max_x - min_x
            if width >= MINIMUM_BAY and width <= max_y - min_y:
                # add the wires
                curr = min_x + WIRE_SPACE
                while curr + WIRE_SPACE + WIRE_WIDTH <= max_x:
                    # render the curr wire
                    wire_coords = [
                        curr, min_y,
                        curr + WIRE_WIDTH, min_y,
                        curr + WIRE_WIDTH, max_y,
                        curr, max_y
                    ]
                    bay_line = create_polygon(wire_coords)

                    wire_base_object = create_base_object(
                        bay_line, properties={"color": "red"}
                    )
                    self.variation.add_object(
                        ANNOTATIONS_LAYER, wire_base_object)
                    curr = curr + WIRE_SPACE + WIRE_WIDTH

                # add the bay
                base_coords = [
                    min_x, min_y,
                    max_x, min_y,
                    max_x, max_y,
                    min_x, max_y,
                ]
                bay_polygon = create_polygon(base_coords)
                base_object = create_base_object(
                    bay_polygon, properties={"color": "white"}
                )
                self.variation.add_object(ANNOTATIONS_LAYER, base_object)
                continue

            min_x = min(min_x_coord(obj), min_x_coord(next_obj))
            max_x = max(max_x_coord(obj), max_x_coord(next_obj))
            min_y = max_y_coord(next_obj)
            max_y = min_y_coord(obj)

            # horizontal bay
            width = max_y - min_y
            if width >= MINIMUM_BAY and max_x - min_x > width:
                # add the wires
                curr = max_y - WIRE_SPACE

                while curr - WIRE_SPACE - WIRE_WIDTH >= min_y:
                    # render the curr wire
                    wire_coords = [
                        min_x, curr - WIRE_WIDTH,
                        max_x, curr - WIRE_WIDTH,
                        max_x, curr,
                        min_x, curr
                    ]
                    bay_line = create_polygon(wire_coords)

                    wire_base_object = create_base_object(
                        bay_line, properties={"color": "red"}
                    )
                    self.variation.add_object(
                        ANNOTATIONS_LAYER, wire_base_object)
                    curr = curr - WIRE_SPACE - WIRE_WIDTH

                # add the bay
                base_coords = [
                    min_x, min_y,
                    max_x, min_y,
                    max_x, max_y,
                    min_x, max_y,
                ]
                bay_polygon = create_polygon(base_coords)
                base_object = create_base_object(
                    bay_polygon, properties={"color": "white"}
                )
                self.variation.add_object(ANNOTATIONS_LAYER, base_object)


import cv2
import os

def start_capture():
    cwd = os.getcwd()

    cam = cv2.VideoCapture(0)

    cv2.namedWindow("Calibration Window")

    img_counter = 0

    while True:
        ret, frame = cam.read()
        cv2.imshow("Calibration Window", frame)
        if not ret:
            break
        k = cv2.waitKey(1)

        if k%256 == 27:
            # ESC pressed
            break
        elif k%256 == 32:
            # SPACE pressed
            img_name = "calib_{}.png".format(img_counter)
            data_path = cwd + "/data"
            fullpath = os.path.join(data_path, img_name)
            cv2.imwrite(fullpath, frame)

            img_counter += 1

    cam.release()

    cv2.destroyAllWindows()


# -*- coding: utf-8 -*-

import pytest

import os

from betfair import betfair
from tests.utils import response_fixture_factory

@pytest.fixture
def client():
    return betfair.Betfair(app_key='test', cert_file='path/to/cert')

@pytest.fixture
def logged_in_client(client):
    client = betfair.Betfair(app_key='test', cert_file='path/to/cert')
    client.session_token = 'secret'
    return client

login_success = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'certlogin'),
    {
        'loginStatus': 'SUCCESS',
        'sessionToken': 'secret',
    },
)

login_failure = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'certlogin'),
    {'loginStatus': 'INVALID_USERNAME_OR_PASSWORD'},
)

login_bad_code = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'certlogin'),
    status=422,
)

keepalive_success = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'keepAlive'),
    {'status': 'SUCCESS'},
)

keepalive_failure = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'keepAlive'),
    {
        'status': 'FAIL',
        'error': 'NO_SESSION',
    },
)

logout_success = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'logout'),
    {'status': 'SUCCESS'},
)

logout_failure = response_fixture_factory(
    os.path.join(betfair.IDENTITY_URLS[None], 'logout'),
    {
        'status': 'FAIL',
        'error': 'NO_SESSION',
    },
)

login_required_methods = [
    'keep_alive',
    'logout',
    'list_event_types',
    'list_competitions',
    'list_time_ranges',
    'list_events',
    'list_market_types',
    'list_countries',
    'list_venues',
    'list_market_catalogue',
    'list_market_book',
    'list_market_profit_and_loss',
    'list_current_orders',
    'list_cleared_orders',
    'place_orders',
    'cancel_orders',
    'replace_orders',
    'update_orders',
]


from collections import OrderedDict

from ydk.types import Entity, EntityPath, Identity, Enum, YType, YLeaf, YLeafList, YList, LeafDataList, Bits, Empty, Decimal64
from ydk.filters import YFilter
from ydk.errors import YError, YModelError
from ydk.errors.error_handler import handle_type_error as _handle_type_error

class Asr9kFabMode(Enum):
    """
    Asr9kFabMode (Enum Class)

    Asr9k fab mode

    .. data:: a99_highbandwidth = 2

    	A99 High bandwidth mode

    """

    a99_highbandwidth = Enum.YLeaf(2, "a99-highbandwidth")

class FabVqiConfig(Entity):
    """
    Configure Fabric Operation Mode

    .. attribute:: mode

    	Mode Type
    	**type**\:  :py:class:`Mode <ydk.models.cisco_ios_xr.Cisco_IOS_XR_asr9k_fab_cfg.FabVqiConfig.Mode>`

    """

    _prefix = 'asr9k-fab-cfg'
    _revision = '2015-11-09'

    def __init__(self):
        super(FabVqiConfig, self).__init__()
        self._top_entity = None

        self.yang_name = "fab-vqi-config"
        self.yang_parent_name = "Cisco-IOS-XR-asr9k-fab-cfg"
        self.is_top_level_class = True
        self.has_list_ancestor = False
        self.ylist_key_names = []
        self._child_container_classes = OrderedDict([("mode", ("mode", FabVqiConfig.Mode))])
        self._child_list_classes = OrderedDict([])
        self._leafs = OrderedDict()

        self.mode = FabVqiConfig.Mode()
        self.mode.parent = self
        self._children_name_map["mode"] = "mode"
        self._children_yang_names.add("mode")
        self._segment_path = lambda: "Cisco-IOS-XR-asr9k-fab-cfg:fab-vqi-config"

    class Mode(Entity):
        """
        Mode Type

        .. attribute:: fab_mode_type_xr

        	Mode Type
        	**type**\:  :py:class:`Asr9kFabMode <ydk.models.cisco_ios_xr.Cisco_IOS_XR_asr9k_fab_cfg.Asr9kFabMode>`

        .. attribute:: fab_mode_type

        	Mode Type
        	**type**\:  :py:class:`Asr9kFabMode <ydk.models.cisco_ios_xr.Cisco_IOS_XR_asr9k_fab_cfg.Asr9kFabMode>`

        """

        _prefix = 'asr9k-fab-cfg'
        _revision = '2015-11-09'

        def __init__(self):
            super(FabVqiConfig.Mode, self).__init__()

            self.yang_name = "mode"
            self.yang_parent_name = "fab-vqi-config"
            self.is_top_level_class = False
            self.has_list_ancestor = False
            self.ylist_key_names = []
            self._child_container_classes = OrderedDict([])
            self._child_list_classes = OrderedDict([])
            self._leafs = OrderedDict([
                ('fab_mode_type_xr', YLeaf(YType.enumeration, 'fab-mode-type-xr')),
                ('fab_mode_type', YLeaf(YType.enumeration, 'fab-mode-type')),
            ])
            self.fab_mode_type_xr = None
            self.fab_mode_type = None
            self._segment_path = lambda: "mode"
            self._absolute_path = lambda: "Cisco-IOS-XR-asr9k-fab-cfg:fab-vqi-config/%s" % self._segment_path()

        def __setattr__(self, name, value):
            self._perform_setattr(FabVqiConfig.Mode, ['fab_mode_type_xr', 'fab_mode_type'], name, value)

    def clone_ptr(self):
        self._top_entity = FabVqiConfig()
        return self._top_entity


import numpy as np
import matplotlib
import matplotlib.pyplot as plt

import azureml
from azureml.core import Workspace, Run
from azureml.core import Workspace
from azureml.core.model import Model
import os
import pickle
from sklearn.externals import joblib
from utils import load_data

from sklearn.metrics import confusion_matrix
# display the core SDK version number
print("Azure ML SDK Version: ", azureml.core.VERSION)

ws = Workspace.from_config()
model=Model(ws, 'sklearn_mnist')

model.download(target_dir=os.getcwd(), exist_ok=True)

# verify the downloaded model file
file_path = os.path.join(os.getcwd(), "sklearn_mnist_model.pkl")

os.stat(file_path)
data_folder = os.path.join(os.getcwd(), 'data')
# note we also shrink the intensity values (X) from 0-255 to 0-1. This helps the neural network converge faster
X_test = load_data(os.path.join(data_folder, 'test-images.gz'), False) / 255.0
y_test = load_data(os.path.join(data_folder, 'test-labels.gz'), True).reshape(-1)

clf = joblib.load( os.path.join(os.getcwd(), 'sklearn_mnist_model.pkl'))
y_hat = clf.predict(X_test)

conf_mx = confusion_matrix(y_test, y_hat)
print(conf_mx)
print('Overall accuracy:', np.average(y_hat == y_test))
# normalize the diagonal cells so that they don't overpower the rest of the cells when visualized
row_sums = conf_mx.sum(axis=1, keepdims=True)
norm_conf_mx = conf_mx / row_sums
np.fill_diagonal(norm_conf_mx, 0)

fig = plt.figure(figsize=(8,5))
ax = fig.add_subplot(111)
cax = ax.matshow(norm_conf_mx, cmap=plt.cm.bone)
ticks = np.arange(0, 10, 1)
ax.set_xticks(ticks)
ax.set_yticks(ticks)
ax.set_xticklabels(ticks)
ax.set_yticklabels(ticks)
fig.colorbar(cax)
plt.ylabel('true labels', fontsize=14)
plt.xlabel('predicted values', fontsize=14)
plt.savefig('conf.png')
plt.show()


from typing import List
import numpy as np

def sample_array(arr, rate):
    arr = np.array(arr)
    if rate < 1:
        rate = int(1 / rate)
        num_samples = rate - 1
        sampled = np.hstack(
            [
                np.linspace(arr[i], arr[i + 1], num_samples + 2, endpoint=False)[1:]
                for i in range(len(arr) - 1)
            ]
        )
        sampled = np.hstack([arr, sampled])
        sampled.sort()
    else:
        sampled = arr[::rate]
    return sampled

def as_beats(frames):
    frames = [f"{frame}:({mag})" for frame, mag in frames]
    return ",".join(frames)

class BeatVisuals:

    def __init__(self, beats, times, fps=24):
        self.fps = fps
        self.beats = times[beats]
        self.beats = np.insert(self.beats, 0, 0)
        self.max_frames = int(max(times) * fps)

    # times are in sec
    # adsr_times = decay, release
    # adsr_mags = on_value, decay_value, off_value
    def beats_deforum(
        self, decay: float, adsr_mags: List[float], every_n: int = 1
    ):
        on_value, off_value = adsr_mags
        frames = []
        beats = sample_array(self.beats, every_n)
        for i in range(0, len(beats)):
            time = round(beats[i], 3)
            keyframes = np.transpose(
                (
                    [time, time + decay],
                    [on_value, off_value],
                )
            )
            frames.extend([(int(k[0] * self.fps), k[1]) for k in keyframes])

        frames = sorted(frames, key=lambda x: x[0])

        # remove duplicate time entries
        found_keyframes = set()
        for kf, val in frames:
            if kf in found_keyframes:
                frames.remove((kf, val))
            else:
                found_keyframes.add(kf)
        return frames

    def as_xy(self, frames):
        frames = np.array(frames)
        x, y = frames[:, 0], frames[:, 1]
        x = x / self.fps
        return x, y

    def beats_prompts(self, prompts):
        prompt_idx = 0
        res = {0: prompts[0]}
        for i in range(len(self.beats)):
            time = round(self.beats[i], 3)
            frame = int(time * self.fps)
            prompt_idx = 0 if prompt_idx == len(prompts) - 1 else prompt_idx + 1
            res[frame] = prompts[prompt_idx]
        return res


import os
import re
from pathlib import Path

import bson
import pandas as pd
from bs4 import BeautifulSoup as bs
from git import Repo
from loguru import logger
from markdownify import markdownify
from tqdm import tqdm

def main():
    # https://podatki.gov.si/dataset/osnovni-podatki-o-predpisih-rs
    vpliva_na = pd.read_csv("data/vplivana.csv")
    osnovni_raw = pd.read_csv("data/osnovni.csv")
    osnovni = osnovni_raw.dropna(subset=["D_SPREJEMA"]).copy()
    osnovni["date_accepted"] = pd.to_datetime(osnovni["D_SPREJEMA"], format="%d.%m.%y")
    # https://podatki.gov.si/dataset/neuradna-preciscena-besedila-predpisov
    with open("data/vsebina.bson/pisrs/vsebina.bson", "rb") as f:
        laws = bson.decode_all(f.read())

    laws_list = [law["idPredpisa"] for law in laws]
    laws_changes = [law["idPredpisaChng"] for law in laws]

    law_id = "ZAKO4697"
    law_code = get_law_code(law_id, osnovni)
    affected_ids = vpliva_na[vpliva_na.VPLIVA_NA == law_id]
    affected = osnovni[
        ((osnovni.ID.isin(affected_ids.ID)) & (osnovni.KRATICA.str.contains(law_code)))
        | (osnovni.ID == law_id)
    ]
    affected_laws = affected[affected.ID.str.startswith("ZAKO")].sort_values(
        "date_accepted"
    )

    data_location = "/tmp/slovenian_laws/"
    os.mkdir(data_location)
    repo = Repo.init(data_location)
    for _, affected_law_row in tqdm(affected_laws.iterrows()):
        #  affected_law_id = affected_laws["ID"].iloc[0]
        affected_law_id = affected_law_row["ID"]
        affected_law_title = affected_law_row["NASLOV"]
        affected_law_code = affected_law_row["KRATICA"]
        commit_msg = (
            affected_law_code + " - " + affected_law_id + " - " + affected_law_title
        )
        affected_law_date = affected_law_row["date_accepted"]
        affected_law = get_law(affected_law_id, laws)
        if not affected_law:
            continue
        vsebina = affected_law["vsebina"]
        vsebina_clean = re.sub(r"( |\n|\r)+", " ", vsebina)

        soup = bs(vsebina_clean)  # make BeautifulSoup
        prettyHTML = soup.prettify()
        #  vsebina_md = markdownify(vsebina, convert=["style"])
        #  vsebina_md = re.sub(
        #      "/(<!--.*?-->)|(<!--[\S\s]+?-->)|(<!--[\S\s]*?$)/g", "", vsebina_md
        #  )
        #  Path(data_location + law_code + ".md").write_text(vsebina_md)
        Path(data_location + law_code + ".html").write_text(prettyHTML)
        repo.git.add(all=True)
        repo.git.commit(date=affected_law_date, m=commit_msg)

def get_law(affected_law_id, laws):
    laws_list = [law["idPredpisa"] for law in laws]
    if affected_law_id in laws_list:
        law_number = laws_list.index(affected_law_id)
        law = laws[law_number]
    else:
        logger.warning(f"No law available for {affected_law_id}")
        law = ""
    return law

def get_law_code(law_id, osnovni):
    law_code = osnovni[osnovni.ID == law_id].iloc[0].KRATICA
    return law_code

def get_law_id(law_code, osnovni):
    law_id = osnovni[osnovni.KRATICA == law_code].iloc[0].ID
    return law_id

def get_markdown(vsebina):
    vsebina_md = markdownify(vsebina)
    return vsebina_md

if __name__ == "__main__":
    main()


from enum import IntEnum
from itertools import takewhile
import datetime

class Part(IntEnum):
    X = 1
    A = 2
    B = 3
    C = 4
    D = 5

    def max_num(self):
        if self == Part.X:
            return 11
        else:
            return 4

    def to_x_num(self):
        if (self == Part.X):
            raise ValueError('Not supported for X.')
        return 2 * self - 1

    def to_x_pos(self):
        return Pos.build(Part.X, self.to_x_num())

    def all_positions_asc(self):
        if self == Part.A:
            return [Pos.A1, Pos.A2, Pos.A3, Pos.A4]
        if self == Part.B:
            return [Pos.B1, Pos.B2, Pos.B3, Pos.B4]
        if self == Part.C:
            return [Pos.C1, Pos.C2, Pos.C3, Pos.C4]
        if self == Part.D:
            return [Pos.D1, Pos.D2, Pos.D3, Pos.D4]

        return [Pos.X1, Pos.X2, Pos.X3, Pos.X4, Pos.X5, Pos.X6, Pos.X7, Pos.X8, Pos.X9, Pos.X10, Pos.X11]

    def to_path(self, start, end, include_start):
        start_excluded = start - 1 if include_start else start
        if start_excluded == end:
            return []
        nums = reversed(range(end, start_excluded, -1 if start_excluded < end else 1))
        return list(map(lambda n: Pos.build(self, n), nums))

    def multiplier(self):
        return [0, 1, 10, 100, 1000][self - 1]

    def __repr__(self):
        return self.name

class Pos(IntEnum):
    X1 = 1
    X2 = 2
    X3 = 3 # Does this position even matter?
    X4 = 4
    X5 = 5 # Does this position even matter?
    X6 = 6
    X7 = 7 # Does this position even matter?
    X8 = 8
    X9 = 9 # Does this position even matter?
    X10 = 10
    X11 = 11
    A1 = 21
    A2 = 22
    A3 = 23
    A4 = 24
    B1 = 31
    B2 = 32
    B3 = 33
    B4 = 34
    C1 = 41
    C2 = 42
    C3 = 43
    C4 = 44
    D1 = 51
    D2 = 52
    D3 = 53
    D4 = 54

    def __repr__(self):
        return self.name

    def num(self):
        enum_int = int(self)
        if enum_int >= 20:
            return enum_int % 10
        else:
            return enum_int

    def part(self):
        divide = int(int(self) / 10)
        if divide == 0:
            return Part(1)
        else:
            return Part(divide)

    def dist(self, pos2):
        part1 = self.part()
        part2 = pos2.part()
        num1 = self.num()
        num2 = pos2.num()

        if part1 == part2:
            return abs(num1 - num2)
        if part1 != Part.X and part2 != Part.X:
            corridor = 2 * abs(part1 - part2)
            dist_from_corr_1 = 5 - num1
            dist_from_corr_2 = 5 - num2
            return corridor + dist_from_corr_1 + dist_from_corr_2
        if part1 == Part.X:
            dist_from_corr = 5 - num2
            dist_to_room = abs(num1 - part2.to_x_num())
            return dist_from_corr + dist_to_room
        dist_from_corr = 5 - num1
        dist_to_room = abs(num2 - part1.to_x_num())
        return dist_from_corr + dist_to_room

    @classmethod
    def build(cls, part: Part, num):
        int_part = int(part)
        if int_part == 1:
            return Pos(num)
        else:
            return Pos(int_part * 10 + num)

class Pawn:
    def __init__(self, start_pos: Pos, dest_part: Part):
        self.start_pos = start_pos
        self.curr_pos = start_pos
        self.dest_part = dest_part
        self.total_dist = 0
        self.is_finished_cached = False

    def copy_pawn(self):
        new_pawn = Pawn(self.start_pos, self.dest_part)
        new_pawn.curr_pos = self.curr_pos
        new_pawn.total_dist = self.total_dist
        new_pawn.is_finished_cached = self.is_finished_cached
        return new_pawn

    def num(self):
        return self.curr_pos.num()

    def didnt_move(self):
        return self.moves == 0

    def corridor_entry_pos(self):
        start_part = self.start_pos.part()
        corr_pos = start_part.to_x_num()
        return Pos.build(Part.X, corr_pos)

    def move_to(self, pos: Pos):
        old_pos = self.curr_pos
        self.curr_pos = pos
        self.total_dist += old_pos.dist(pos)

    def curr_part(self):
        return self.curr_pos.part()

    def __repr__(self):
        return "Pawn at %s, dest = %s" % (self.curr_pos.name, self.dest_part)

class Board:
    def __init__(self, pawns):
        self.pawns: list[Pawn] = pawns
        self.path_dict = {}
        self.occupied_dict = {}
        self.pawn_dict = {}
        self.finish_pos_dict = {}
        self.moves = []
        self.occupied = [[False] * 11, [False] * 4, [False] * 4, [False] * 4, [False] * 4]
        for pawn in self.pawns:
            part = pawn.curr_part()
            num = pawn.num()
            self.occupied[part - 1][num - 1] = True

    def is_occupied(self, pos: Pos):
        return self.is_occupied_impl2(pos)

    def is_occupied_impl1(self, pos: Pos):
        occupied_value = self.occupied_dict.get(pos)
        if occupied_value != None:
            return occupied_value

        is_occupied = pos in map(lambda p: p.curr_pos, self.pawns)
        self.occupied_dict[pos] = is_occupied
        return is_occupied

    def is_occupied_impl2(self, pos: Pos):
        part = pos.part()
        num = pos.num()
        return self.occupied[part - 1][num - 1]

    def path_free(self, part: Part, start: int, end: int, start_included):
        real_start = start if start_included else start + 1
        dict_value = self.get_from_dict(part, real_start, end)

        if dict_value != None:
            return dict_value

        path = part.to_path(start, end, start_included)
        for pos in path:
            if self.is_occupied(pos):
                self.store_in_dict(part, real_start, end, False)
                return False

        self.store_in_dict(part, real_start, end, False)
        return True

    def store_in_dict(self, part: Part, real_start: int, end: int, result: bool):
        if real_start < end:
            self.path_dict[(part, real_start, end)] = result
        else:
            self.path_dict[(part, end, real_start)] = result

    def get_from_dict(self, part: Part, real_start: int, end: int):
        if real_start < end:
            return self.path_dict.get((part, real_start, end))
        else:
            return self.path_dict.get((part, end, real_start))

    def is_allowed(self, pos: Pos):
        return not pos in [Pos.X3, Pos.X5, Pos.X7, Pos.X9]

    def pawn_at(self, pos: Pos):
        if pos in self.pawn_dict:
            return self.pawn_dict[pos]

        pawn = next((p for p in self.pawns if p.curr_pos == pos), None)
        self.pawn_dict[pos] = pawn
        return pawn

    def is_finished(self, pawn: Pawn):
        if pawn.is_finished_cached:
            return True

        curr_pos = pawn.curr_pos
        curr_part = curr_pos.part()
        if curr_pos.part() == pawn.dest_part:
            nums_lower = range(1, pawn.num())

            nums_lower = range(1, curr_pos.num())
            for num in nums_lower:
                pawn_lower = self.pawn_at(Pos.build(curr_part, num))
                if pawn_lower == None or pawn_lower.dest_part != curr_part:
                    return False

            pawn.is_finished_cached = True
            return True

        return False

    def finish_pos(self, part: Part):
        if part in self.finish_pos_dict:
            return self.finish_pos_dict[part]

        all_positions_desc = list(reversed(part.all_positions_asc()))
        all_pawns = list(map(lambda p: self.pawn_at(p), all_positions_desc))

        all_open = list(takewhile(lambda p: p == None, all_pawns))
        all_open_len = len(all_open)

        if all_open_len > 0 and (all_open_len == 4 or self.is_finished(all_pawns[all_open_len])):
            finish = all_positions_desc[all_open_len - 1]
            self.finish_pos_dict[part] = finish
            return finish

        self.finish_pos_dict[part] = None
        return None

    def next_positions(self, pawn: Pawn):
        if self.is_finished(pawn):
            return []

        finish_pos = self.finish_pos(pawn.dest_part)
        if finish_pos != None and self.can_reach(pawn.curr_pos, finish_pos):
            return [finish_pos]
        else:
            if pawn.curr_part() == Part.X:
                return [] # Pawn is already at X and cannot finish

        x_entry_pos = pawn.curr_part().to_x_pos()
        if self.can_reach(pawn.curr_pos, x_entry_pos):
            accessible_xs = self.accessible_in_x_from_num(x_entry_pos.num())
            allowed_xs = list(filter(lambda p: self.is_allowed(p), accessible_xs))
            return allowed_xs
        return []

    def can_reach(self, start: Pos, end: Pos):
        start_part = start.part()
        end_part = end.part()
        if start_part == end_part:
            return self.path_free(start_part, start.num(), end.num(), False)

        if end_part == Part.X:
            return self.path_free(start_part, start.num(), 4, False) and \
                self.path_free(Part.X, start_part.to_x_num(), end.num(), False)

        if start_part == Part.X:
            return self.path_free(Part.X, start.num(), end_part.to_x_num(), False) and \
                self.path_free(end_part, 4, end.num(), True)

        return self.path_free(start_part, start.num(), 4, False) and \
            self.path_free(Part.X, start_part.to_x_num(), end_part.to_x_num(), False) and \
            self.path_free(end_part, 4, end.num(), True)

    def accessible_in_x(self, curr_pos: Pos):
        num = curr_pos.num()
        part = curr_pos.part()
        if part == Part.X:
            return self.accessible_in_x_from_num(num)
        route_to_x = part.to_path(num, 4, False)
        if all(lambda p: not self.is_occupied(p), route_to_x):
            entry_x_num = part.to_x_num()
            return self.accessible_in_x_from_num(entry_x_num)
        return []

    def accessible_in_x_from_num(self, x_num):
        lower = Part.X.to_path(x_num, 1, False)
        higher = Part.X.to_path(x_num, 11, False)
        lower_free = list(reversed(list(takewhile(lambda p: not self.is_occupied(p), lower))))
        higher_free = list(takewhile(lambda p: not self.is_occupied(p), higher))
        return lower_free + higher_free

    def move_new_board(self, start: Pos, end: Pos):
        new_pawns = list(map(lambda p: p.copy_pawn(), self.pawns))
        pawn = next(filter(lambda p: p.curr_pos == start, new_pawns), None)
        pawn.move_to(end)
        new_board = Board(new_pawns)
        existing_moves = self.moves.copy()
        existing_moves.append(f"({pawn}) moved {start.name} âž¡ {end.name}")
        new_moves = existing_moves
        new_board.moves = new_moves
        return new_board

    # Good that this does not use pawn_at, because that uses cached results
    # and can be outdated after a move
    def pawn_char(self, pos: Pos):
        result = next(filter(lambda p: p.curr_pos == pos, self.pawns), None)
        return " " if result == None else result.dest_part.name

    def print_board(self):
        print()
        print("#############")
        xs_chars = ''.join(list(map(lambda p: self.pawn_char(p), Part.X.all_positions_asc())))
        print("#" + xs_chars + "#")
        print("###" + self.pawn_char(Pos.A4) + "#" + self.pawn_char(Pos.B4) + "#" \
              + self.pawn_char(Pos.C4) + "#" + self.pawn_char(Pos.D4) + "###")
        print("  #" + self.pawn_char(Pos.A3) + "#" + self.pawn_char(Pos.B3) + "#" \
              + self.pawn_char(Pos.C3) + "#" + self.pawn_char(Pos.D3) + "#  ")
        print("  #" + self.pawn_char(Pos.A2) + "#" + self.pawn_char(Pos.B2) + "#" \
              + self.pawn_char(Pos.C2) + "#" + self.pawn_char(Pos.D2) + "#  ")
        print("  #" + self.pawn_char(Pos.A1) + "#" + self.pawn_char(Pos.B1) + "#" \
              + self.pawn_char(Pos.C1) + "#" + self.pawn_char(Pos.D1) + "#  ")
        print("  #########  ")
        print()

    def all_moves(self):
        moves = []
        for pawn in self.pawns:
            next_for_pawn = self.next_positions(pawn)
            moves_for_pawn = list(map(lambda p: Move(pawn.curr_pos, p), next_for_pawn))
            moves.extend(moves_for_pawn)

        return moves

    def end_reached(self):
        for part in [Part.A, Part.B, Part.C, Part.D]:
            for pos in part.all_positions_asc():
                pawn_on_pos = self.pawn_at(pos)
                if pawn_on_pos == None or pawn_on_pos.dest_part != part:
                    return False

        return True

    def calc_score(self):
        sum = 0
        for pawn in self.pawns:
            sum += pawn.total_dist * pawn.dest_part.multiplier()

        return sum

    def __repr__(self):
        return f"Board with {len(self.moves)} moves"

class Move:
    def __init__(self, start: Pos, end: Pos):
        self.start = start
        self.end = end

    def apply_move(self, board: Board):
        return board.move_new_board(self.start, self.end)

    def __repr__(self):
        return "Move %s âž¡ %s" % (self.start.name, self.end.name)

pawns_init_part_2 = [
    Pawn(Pos.A4, Part.B), Pawn(Pos.B4, Part.C), Pawn(Pos.C4, Part.A), Pawn(Pos.D4, Part.D),
    Pawn(Pos.A3, Part.D), Pawn(Pos.B3, Part.C), Pawn(Pos.C3, Part.B), Pawn(Pos.D3, Part.A),
    Pawn(Pos.A2, Part.D), Pawn(Pos.B2, Part.B), Pawn(Pos.C2, Part.A), Pawn(Pos.D2, Part.C),
    Pawn(Pos.A1, Part.B), Pawn(Pos.B1, Part.C), Pawn(Pos.C1, Part.D), Pawn(Pos.D1, Part.A)]

pawns_init_part_1 = [
    Pawn(Pos.A4, Part.B), Pawn(Pos.B4, Part.C), Pawn(Pos.C4, Part.A), Pawn(Pos.D4, Part.D),
    Pawn(Pos.A3, Part.B), Pawn(Pos.B3, Part.C), Pawn(Pos.C3, Part.D), Pawn(Pos.D3, Part.A),
    Pawn(Pos.A2, Part.A), Pawn(Pos.B2, Part.B), Pawn(Pos.C2, Part.C), Pawn(Pos.D2, Part.D),
    Pawn(Pos.A1, Part.A), Pawn(Pos.B1, Part.B), Pawn(Pos.C1, Part.C), Pawn(Pos.D1, Part.D)]

pawns_init_experiment = [
    Pawn(Pos.A4, Part.B), Pawn(Pos.B4, Part.A), Pawn(Pos.C4, Part.C), Pawn(Pos.D4, Part.D),
    Pawn(Pos.A3, Part.A), Pawn(Pos.B3, Part.B), Pawn(Pos.C3, Part.C), Pawn(Pos.D3, Part.D),
    Pawn(Pos.A2, Part.A), Pawn(Pos.B2, Part.B), Pawn(Pos.C2, Part.C), Pawn(Pos.D2, Part.D),
    Pawn(Pos.A1, Part.A), Pawn(Pos.B1, Part.B), Pawn(Pos.C1, Part.C), Pawn(Pos.D1, Part.D)]

lowest_score = 1000000
cycle = 0
move_to_test = 1
winning_moves = []

def check_moves(board_start: Board):
    global cycle
    global move_to_test
    global lowest_score
    global winning_moves

    print("Checking boards started")

    boards_to_check = [board_start]

    while len(boards_to_check) > 0:
        curr_board = boards_to_check.pop(0)
        moves = curr_board.all_moves()

        if len(moves) > 0:
            boards_to_insert = []
            for move in moves:
                new_board = move.apply_move(curr_board)
                boards_to_insert.append(new_board)
            for b in boards_to_insert:
                boards_to_check.insert(0, b)

            cycle += 1
            if cycle % 20000 == 0:
                time_format = "%H:%M:%S"
                print(f"> cycle {cycle}, time: {datetime.datetime.now().strftime(time_format)}, boards active: {len(boards_to_check)}")
        else:
            if curr_board.end_reached():
                score = curr_board.calc_score()
                if score < lowest_score:
                    lowest_score = score
                    winning_moves = curr_board.moves.copy()
                    print(f"Lowest score is now {lowest_score}")

check_moves(Board(pawns_init_part_2))

print("-- All options checked --")
print(f"Lowest score = {lowest_score}")

for move in winning_moves:
    print(">  " + move)

# 166 too low
# 42752 too low
# 42732 too low (ðŸ˜„)
# In total 2.280.000 cycles


from django.contrib.auth.decorators import login_required
from django.core.exceptions import ObjectDoesNotExist
from django.shortcuts import render, redirect, get_object_or_404
# from django.http import HttpResponse

from store.models import Product, Variation

from .models import Cart, CartItem

# Create your views here.

def _cart_id(request):
    # Use the session id as cart id.
    cart = request.session.session_key
    if not cart:
        cart = request.session.create()
    return cart

def _add_user_to_cart(request, cart=None):
    try:
        if not cart:
            cart = Cart.objects.get(cart_id=_cart_id(request))
        cart.user = request.user
        cart.save()
    except Exception as e:
        print(e)
        cart = None
    return cart

def _update_carts(request, old_cart_id):
    """Update anonymous and users previous cart id to match"""

    users_cart = Cart.objects.filter(user__pk=request.user.id).first()
    anonymous_cart = Cart.objects.filter(cart_id=old_cart_id).first()

    try:
        if users_cart and anonymous_cart:
            # ideally all of this will be atomic
            # TODO: on update to postgresql use: https://docs.djangoproject.com/en/4.1/ref/models/querysets/#select-for-update
            users_cart.cart_id = _cart_id(request)
            users_cart.save()

            # move anonymous cart items to users previous cart. First check whether
            # the users previous cart contains the unique product, varient combination.
            # If so, increment the already instanciated cart_item quantity by the
            # New varient quantity.
            # Otherwise, move the product-varient combo to the users cart and add the
            # variation to the existing_product_varients list
            users_cart_items = CartItem.objects.filter(cart=users_cart)

            existing_product_varients = [[item.product.id, *list(item.variations.all())]
                                            for item in users_cart_items]

            anonymous_cart_items = CartItem.objects.filter(cart=anonymous_cart)

            for a_item in anonymous_cart_items:
                varient = [a_item.product.id, *list(a_item.variations.all())]
                if varient in existing_product_varients:
                    # Increment the cart_item quantity and save
                    index = existing_product_varients.index(varient)
                    existing_varient = users_cart_items[index]
                    existing_varient.quantity += a_item.quantity
                    existing_varient.save()
                else:
                    # Move the item over to the cart
                    a_item.cart = users_cart
                    a_item.save()
                    existing_product_varients.append(varient)
            # Declutter the database and get rid of the anonymous cart
            anonymous_cart.delete()

        elif anonymous_cart:
            # This block executes only if
            # users_cart is None and anonymous_cart is not None
            #
            # update anonymous cart id to current session and add
            # authenticated user to anonymous cart
            anonymous_cart.cart_id = _cart_id(request)
            _add_user_to_cart(request, anonymous_cart)
            users_cart = anonymous_cart
        elif users_cart:
            # This block executes only if
            # users_cart is not None and anonymous_cart is None
            #
            # update users previous cart id only
            users_cart.cart_id = _cart_id(request)
            users_cart.save()
    except Exception as e:
        print(e)
        users_cart = None
    finally:
        return users_cart

def _get_post_request_variation(product_id, varients):
    """Parse a dict for variations"""
    product = Product.objects.get(id=product_id)
    product_variations = []
    if isinstance(varients, dict):
        for key, value in varients.items():
            try:
                # Filter out only the relevant information
                variation = Variation.objects.get(product=product,
                                                variation_category__iexact=key,
                                                variation_value__iexact=value)
                # add the product variation
                product_variations.append(variation)
            except:
                # Not a varient
                pass
    return product_variations

def add_item(request, product_id):
    """Increment cart item by one"""
    # Get the product variations from Post request
    product_variation = []
    if request.method == 'POST':
        product_variation = _get_post_request_variation(product_id, request.POST)

    # Add the product to the cart
    cart, _ = Cart.objects.get_or_create(cart_id=_cart_id(request))
    product = Product.objects.get(id=product_id)
    cart_items = CartItem.objects.filter(product=product, cart=cart)

    # If the variation already exists in the cart, increment the quantity
    added = False
    for ci in cart_items:
        if product_variation == list(ci.variations.all()):
            ci.quantity += 1
            added = True
            ci.save()
            break

    # Otherwise, create a new item
    if not added:
        ci = CartItem.objects.create(product=product, cart=cart, quantity=1)
        ci.variations.add(*product_variation)
        ci.save()

    return redirect('cart')

def remove_item(request, product_id, cart_item_id):
    """Decerement cart item by one"""
    product = get_object_or_404(Product, id=product_id)
    cart, _ = Cart.objects.get_or_create(cart_id=_cart_id(request))
    try:
        # Decrement or remove the item varient from the cart
        cart_item = CartItem.objects.get(product=product, cart=cart, id=cart_item_id)
        if cart_item.quantity > 1:
                cart_item.quantity -= 1
                cart_item.save()
        else:
            cart_item.delete()
    except:
        pass
    return redirect('cart')

def remove_cart_item(request, product_id, cart_item_id):
    """Remove all items of the varient from the cart"""
    product = get_object_or_404(Product, id=product_id)
    cart = Cart.objects.get(cart_id=_cart_id(request))
    try:
        cart_item = CartItem.objects.filter(product=product, cart=cart, id=cart_item_id)
        cart_item.delete()
    except:
        pass
    return redirect('cart')

def cart(request, total=0, quantity=0, cart_items=None):
    try:
        cart = Cart.objects.get(cart_id=_cart_id(request))
        cart_items = CartItem.objects.filter(cart=cart, is_active=True)
        for cart_item in cart_items:
            total += (cart_item.product.price * cart_item.quantity)
            quantity += cart_item.quantity
        tax = 0.095 * total
        grand_total = total + tax
    except ObjectDoesNotExist:
        #TODO: Fix this Make handling better
        tax=0
        grand_total=0

    context = {
        'total': total,
        'tax': tax,
        'grand_total': grand_total,
        'quantity': quantity,
        'cart_items': cart_items,
    }

    return render(request, 'cart.html', context=context)

@login_required(login_url='login')
def checkout(request, total=0, quantity=0, cart_items=None):
    try:
        cart = Cart.objects.get(cart_id=_cart_id(request))
        cart_items = CartItem.objects.filter(cart=cart, is_active=True)
        for cart_item in cart_items:
            total += (cart_item.product.price * cart_item.quantity)
            quantity += cart_item.quantity
        tax = 0.095 * total
        grand_total = total + tax
    except ObjectDoesNotExist:
        #TODO: Fix this Make handling better
        tax=0
        grand_total=0

    context = {
        'total': total,
        'tax': tax,
        'grand_total': grand_total,
        'quantity': quantity,
        'cart_items': cart_items,
    }

    return render(request, 'checkout.html', context=context)


import pdb, dateutil, random
from datetime import datetime, timedelta
from models.activity import Activity
from models.location import Location
from models.member import Member

import repositories.activity_repo as act_repo
import repositories.location_repo as loc_repo
import repositories.member_repo as member_repo

# delete all submitted values and start fresh
loc_repo.delete_all()
member_repo.delete_all()
act_repo.delete_all()

# add some members
allen = Member("Allen", "Kelly", "allen@kelly.com", True)
dave = Member("David", "Taylor", "dt@taylor.com", False)

member_repo.save(allen)
member_repo.save(dave)

# add some locations
spin_room = Location("Spin room", 12)
hall1 = Location("Gym hall 1", 20)
hall2 = Location("Gym hall 2", 10)
small_room = Location("Small gym", 5)

loc_repo.save(spin_room)
loc_repo.save(hall1)
loc_repo.save(hall2)
loc_repo.save(small_room)

# return all members
all_members = member_repo.select_all()

# return a specific member
one_member = member_repo.select(allen.id)

# return all locations
all_locs = loc_repo.select_all()

# return specific location
one_loc = loc_repo.select(hall1.id)

# set up some activities for demonstration
today = datetime.now().replace(hour=8, minute=0, second=0, microsecond=0)

# generate a list of timedelta objects of increasing size
times = []
for hour in range(0, 14, 1):
    times.append(timedelta(minutes=60 * hour))

act1 = Activity("Yoga", today, hall2)
act2 = Activity("Boxfit", today, hall1)
act3 = Activity("Spin", today, spin_room)
act4 = Activity("Cross-fit", today, small_room)
act5 = Activity("Step", today, hall1)
act6 = Activity("BodyPump", today, hall2)
act7 = Activity("Spin", today, spin_room)
act8 = Activity("BodyPump", today, hall1)
act9 = Activity("Yoga", today, small_room)
act10 = Activity("Salsa", today, hall2)
act11 = Activity("Spin", today, spin_room)

all_activities = [act1, act2, act3, act4, act5,
                 act6, act7, act8, act9, act10,
                 act11]

days = 0

while days < 7:
    for activity in all_activities:
        # assign a random time to an activity
        activity.start = today + random.choice(times)
        act_repo.save(activity)

    days += 1
    today += timedelta(days=1)

pdb.set_trace()


'''
Author : Na Rae Baek
Author email : naris27@dgu.ac.kr
Github : https://github.com/ban2aru
License : MIT license
Modified Year-Month : 2022-04

Description : test.py
The main code for testing ResNet using Pytorch
'''
from .utils import get_test_dataloader
import torch
import time

def test(args, model):
    test_loader = get_test_dataloader(args)

    checkpoint = torch.load(args.chk_dir)
    model = checkpoint['model']

    if args.gpu :
        model = model.cuda()

    model.eval()

    test_correct_top1, test_correct_top5 = 0.0, 0.0

    with torch.no_grad():
        for batch_index, (imgs, labels) in enumerate(test_loader):

            if args.gpu :
                imgs = imgs.cuda()
                labels = labels.cuda()

            outputs = model(imgs)
            _, predicted = outputs.topk(5, 1, largest=True, sorted=True)

            labels = labels.view(labels.size(0), -1).expand_as(predicted)
            correct = predicted.eq(labels).float()

            if args.num_classes > 5 :
                test_correct_top5 += correct[:, :5].sum()

            test_correct_top1 += correct[:, :1].sum()

        print('||'+time.strftime('%Y-%m-%d-%H:%M:%S')+'||Test Step')
        print('||'+time.strftime('%Y-%m-%d-%H:%M:%S')+'||Test Acc@1 %.4f' %(test_correct_top1/len(test_loader.dataset)))
        if args.num_classes > 5 :
            print('||'+time.strftime('%Y-%m-%d-%H:%M:%S')+'||Test Acc@5 %.4f' %(test_correct_top5/len(test_loader.dataset)))
        print('||'+time.strftime('%Y-%m-%d-%H:%M:%S')+'||Parameter numbers %d' %(sum(p.numel() for p in model.parameters())))


#!/usr/bin/env python3
import sqlite3
import math
import pprint
import logging

import typing as t

import sys, getopt
import re
import os
import json
from enum import IntEnum, auto

def die(msg):
	print(msg, file=sys.stderr)
	sys.exit(1)

class sqltype(IntEnum):
	INTEGER = auto()
	REAL = auto()
	TEXT = auto()
	def __str__(self) -> str:
		return self.name

def merge_sqltypes(obja, objb):
	return max(obja, objb)

def make_sqltype(obj: t.Any) -> sqltype:
	try:
		int(obj)
		return sqltype.INTEGER
	except ValueError:
		pass
	except TypeError:
		pass
	try:
		float(obj)
		return sqltype.REAL
	except ValueError:
		pass
	except TypeError:
		pass
	logging.debug(f"type of {obj} is TEXT")
	return sqltype.TEXT

def split_keyvalueline(line: str) -> t.Mapping[str,str]:
	""" read a line of 'key=value' pairs separated by whitespace(s) into a dict """
	attrs=dict()
	while line.find('=') != -1:
		key = line[:line.find('=')]
		valuematch = re.match(r'^\S+', line[line.find('=')+1:])
		assert valuematch, 'invalid key/value line: %s' % line
		value = valuematch.group(0)
		line=line[len(key)+len(value)+1:]
		key = key.strip()
		value = value.strip()
		attrs[key] = value
	return attrs

def split_resultline(line: str) -> t.Mapping[str,str]:
	""" read a RESULT line and put the keyvalue pairs into a dict """
	return split_keyvalueline(line[len('RESULT '):].strip())

def create_json_table(tablename: str, tablefilename: str):
	import json
	with open(tablefilename,'r') as tablefile:
		json_data = json.loads(tablefile.read())
	keys = dict()
	for entry in json_data:
		for key in entry.keys():
			if not key in keys:
				keys[key] = make_sqltype(entry[key])
			else:
				keys[key] = merge_sqltypes(make_sqltype(entry[key]), keys[key])

	columns=[]
	for key in keys:
		columns.append('"%s" %s' % (key, str(keys[key])))
	sqlexecute('CREATE TABLE "%s" (%s);' % (tablename, ', '.join(columns)))

	for entry in json_data:
		sqlexecute('INSERT INTO "%s" (%s) VALUES (%s);' % (tablename
			, ', '.join(map(lambda key : '"' + str(key) + '"' , entry.keys()))
			, ', '.join(map(lambda value : '\'' + str(value) + '\'', entry.values()))))
	conn.commit()

def create_table(tablename: str, tablefilename: str):
	""" read the types of the used keys """
	keys = dict()
	with open(tablefilename,'r') as tablefile:
		for tableLine in tablefile.readlines():
			if tableLine.startswith('RESULT '):
				attrs = split_resultline(tableLine)
				for key in attrs:
					if not key in keys:
						keys[key] = make_sqltype(attrs[key])
					else:
						keys[key] = merge_sqltypes(make_sqltype(attrs[key]), keys[key])
	columns=[]
	for key in keys:
		columns.append('"%s" %s' % (key, str(keys[key])))
	sqlexecute('CREATE TABLE "%s" (%s);' % (tablename, ', '.join(columns)))

	# read the values
	with open(tablefilename, 'r') as tablefile:
		for tableLine in tablefile.readlines():
			if tableLine.startswith('RESULT '):
				attrs = split_resultline(tableLine)
				sqlexecute('INSERT INTO "%s" (%s) VALUES (%s);' % (tablename
					, ', '.join(map(lambda key : '"' + key + '"' , attrs.keys()))
					, ', '.join(map(lambda value : '\'' + value + '\'', attrs.values()))))
		conn.commit()

class ReadStatus(IntEnum):
	NONE = auto()
	MULTIPLOT = auto()
	SINGLEPLOT = auto()
	TABULAR = auto()
	MATRIX = auto()
	MACRO = auto()
	ERASE = auto() #! used for updating a document in-place by removing the old insertions (which is done until finding a newline)

keyword_to_status : t.Mapping[str, ReadStatus] = {
		"MULTIPLOT"  : ReadStatus.MULTIPLOT,
		"SINGLEPLOT" : ReadStatus.SINGLEPLOT,
		"TABULAR"    : ReadStatus.TABULAR,
		"MATRIX"     : ReadStatus.MATRIX,
		"DEFINE"     : ReadStatus.MACRO
		}

def apply_macros(sqlbuffer: str) -> str:
	match = re.search(r'\$(\w+)', sqlbuffer)
	while match:
		macroname = match.group(1)
		assert macroname in macros, 'macro not defined: "%s". used in the sql expression: %s' % (macroname, sqlbuffer)
		sqlbuffer = macros[macroname].apply(sqlbuffer)
		match = re.search(r'\\$(\w+)', sqlbuffer)
	return sqlbuffer

def multiplot(sqlbuffer : str, multiplot_columns : t.List[str]) -> t.Mapping[str, t.List[t.Tuple[str,str]]]:
	""" reads a MULTIPLOT statement and returns a dictionary mapping a MULTIPLOT instance to a list of coordinates """
	sqlbuffer = apply_macros(sqlbuffer)
	logging.info("macros-expanded SQL : " + sqlbuffer);

	""" if a column is is `a`.`b`, then we have to alias it to `a.b` """
	group_query = re.sub('MULTIPLOT', ','.join(map(lambda col: ".".join(map(lambda el: '"%s"' % el, col.split('.'))) + ' AS "%s"' % col, multiplot_columns)), sqlbuffer, 1)

	group_query = re.sub('MULTIPLOT', ','.join(map(lambda col: '"%s"' % col, multiplot_columns)), group_query)

	""" query for all possible values the variable MULTIPLOT can have """
	sqlexecute(group_query + ';')
	multiplot_value_tuples=set()
	# for row in cursor.fetchall():
	# 	multiplot_value_tuples.add(tuple(row[x] for x in multiplot_columns))

	multiplot_value_tuples = list(map(lambda row: tuple(row[x] for x in multiplot_columns), cursor.fetchall()))
	coordinates=dict()
	for multiplot_values in multiplot_value_tuples:
		#! this is a heuristic: we assume that if there is a where clause than this is for the outmost select command (and not for a subquery)
		if re.search(' where ', group_query, re.IGNORECASE):
			query = re.sub('(.*) where ', '\\1 WHERE %s AND ' %
					' AND '.join(map(lambda x: '"%s" = \'%s\'' % (x[0],x[1]), zip(multiplot_columns, multiplot_values)))
					, group_query, flags=re.IGNORECASE)
		else:
			query = re.sub('(.*) group by ', '\\1 WHERE %s GROUP BY ' %
					' AND '.join(map(lambda x: '"%s" = \'%s\'' % (x[0],x[1]), zip(multiplot_columns, multiplot_values)))
					, group_query, flags=re.IGNORECASE)
		sqlexecute(query + ';')
		rows = cursor.fetchall()
		#if(len(rows) > 0):
			# try:
			# 	list(map(lambda row: (float(row['x']), float(row['y'])), rows))
			# except ValueError:
			# 	die('Values of the MULTIPLOT tuple %s are not floats/not defined. SQL-Statement: %s ' % (str(multiplot_values),sqlbuffer) )
            #
		coordinates[multiplot_values] = list(map(lambda row: (row['x'], row['y']), rows))
	return coordinates

class Filetype(IntEnum):
	TEX = auto()
	PYTHON = auto()
	JS = auto()
	CSV = auto()
	GNUPLOT = auto()
	UNKNOWN = auto()

	def comment(self):
		if(self == Filetype.CSV):
			return '##'
		if(self == Filetype.PYTHON or self == Filetype.GNUPLOT):
			return '##'
		if(self == Filetype.JS):
			return '///'
		else:
			return '%%'

	@staticmethod
	def fromString(text: str) -> "Filetype":
		if(text == 'csv'):
			return Filetype.CSV
		if(text == 'tex'):
			return Filetype.TEX
		if(text == 'csv'):
			return Filetype.JS
		if(text == 'py'):
			return Filetype.PYTHON
		if(text == 'plt' or text == 'gnuplot'):
			return Filetype.GNUPLOT
		return Filetype.UNKNOWN

## MAIN
color_entries = dict()
try:
	from ast import literal_eval
	with open('pgf_color_entries.txt','r') as txtfile:
		for line in txtfile.readlines():
			if line.startswith('#'):
				continue
			cols = line.split("\t")
			assert len(cols) == 2, "Invalid Line : " + line
			if len(cols) == 2:
				try:
					color_entries[literal_eval(cols[0])] = int(cols[1])
					""" we use literal_eval to deserialize a tuple as keys are tuples """
				except ValueError:
					print('could not parse the line `%s` in pgf_color_entries.txt' % line, file=sys.stderr)
					sys.exit(1)

except IOError:
	print('file pgf_color_entries.txt does not exist -> will create it.', file=sys.stderr)

def sqlexecute(sqlcommand: str):
	try:
		logging.debug("SQL query: " + sqlbuffer);
		cursor.execute(sqlcommand)
	except sqlite3.Error as e:
		print("Error while executing the SQL statement: ", sqlcommand, file=sys.stderr)
		raise e

class Macro:
	name : str
	arguments : t.List[str]
	body : str

	def __init__(self, name: str, arguments: t.List[str], body: str):
		self.name = name
		self.arguments = arguments
		self.body = body

	def apply(self, s: str) -> str:
		""" applies the macro with a given list of parameters """
		regex = r'\$' + self.name + r'\s*\(([^)]+)\)'
		match = re.search(regex, s)
		while match:
			parameters = match.group(1).split(',')
			assert len(parameters) == len(self.arguments), "length of arguments and parameters for macro %s mismatch: " % (self.name, parameters)
			body = self.body
			for rule in zip(self.arguments, parameters):
				body = body.replace('$' + rule[0], rule[1])
			s = re.sub(regex, body, s, 1)
			match = re.search(regex, s)
		return s

""" writes the output of MULTIPLOT or SINGLEPLOT, where coordinates is a dict mapping an entryname to a list of coordinates. Adds to `previous_entries` the number of written entries """
def plot_coordinates(sqlbuffer: str,
		outfilename: str,
		coordinates: t.Mapping[str, t.List[t.Tuple[str,str]]],
		outfile: t.IO,
		outfiletype: Filetype,
		previous_entries: int) -> int:
	entrynames = list(coordinates.keys())
	entrynames.sort()
	sqlbuffer = sqlbuffer.replace('\n', ' ')

	if outfiletype == Filetype.PYTHON:
		pprint.pprint(coordinates, outfile)
	elif outfiletype == Filetype.GNUPLOT:
		if outfilename not in gnuplot_line_index:
			gnuplot_line_index[outfilename] = 0
		print('# ' + sqlbuffer, file=outfile)
		print('', file=outfile)
		for entry_id in range(len(entrynames)):
			entryname = entrynames[entry_id]
			print('#index %d with parameter %s' % (gnuplot_line_index[outfilename], entryname[0] if len(entryname) == 1 else str(entryname).replace(',',';')), file=outfile)
			for coordinate in coordinates[entryname]:
				print('%s\t%s' % (coordinate[0], coordinate[1]), file=outfile)
			""" gnuplot needs two newlines for marking the values of the next entry """
			print('', file=outfile)
			print('', file=outfile)
			gnuplot_line_index[outfilename] += 1

		print('# plot \\', file=outfile)
		for entry_id in range(len(entrynames)):
			entryname = entrynames[entry_id]
			index = gnuplot_line_index[outfilename] - len(entrynames) + entry_id
			title = entryname[0] if len(entryname) == 1 else str(entryname).replace(',',';')
			print('# \'%s\' index %d title "%s" with linespoints ls %d, \\' % (outfilename, index, title, index+1), file=outfile)
		print('# ', file=outfile)
		print('\n', file=outfile)

	elif outfiletype == Filetype.CSV:
		print('title,x,y', file=outfile)
		for entry_id in range(len(entrynames)):
			entryname = entrynames[entry_id]
			for coordinate in coordinates[entryname]:
				print('%s,%f,%f' % (entryname[0] if len(entryname) == 1 else str(entryname).replace(',',';'), coordinate[0], coordinate[1]), file=outfile)
	elif outfiletype == Filetype.JS:
		jsonoutput=dict()
		jsonoutput['query'] = sqlbuffer
		j=[]
		for entry_id in range(len(entrynames)):
			entryname = entrynames[entry_id]
			for coordinate in coordinates[entryname]:
				entry = dict()
				entry['name'] = entryname
				entry['x'] = coordinate[0]
				entry['y'] = coordinate[1]
				j.append(entry)
		jsonoutput['result'] = j
		json.dump(jsonoutput, outfile, indent=1)
	else: # default: latex
		if outfilename != 'stdout':
			print('% ' + sqlbuffer, file=outfile)

		for entry_id in range(len(entrynames)):
			entry = entrynames[entry_id]
			if not 'colorcache' in config_args or config_args['colorcache'] != 'none':
				if entry not in color_entries:
					color_entries[entry] = len(color_entries)+1
				shift = color_entries[entry]-(entry_id+previous_entries)
				print('\\pgfplotsset{cycle list shift=%d} %% %s' % (shift, str(color_entries[entry])), file=outfile)
			print('\\addplot coordinates{%s};' % ' '.join(map(lambda coord: '(%s, %s)' % (coord[0], coord[1]), coordinates[entry])), file=outfile)
			print('\\addlegendentry{%s};' % (str(entry) if len(entry) > 1 else entry[0]), file=outfile)
		previous_entries = previous_entries + len(entrynames) # number of previous entries -> needed for a subsequent plot call to determine the cycle list correctly
	return previous_entries

def print_tablentry(entry: t.Any) -> str:
	if entry == None:
		return 'NONE'
	typ = make_sqltype(entry)
	if typ == sqltype.INTEGER or typ == sqltype.REAL:
		return "\\num{%s}" % entry
	return str(entry)

if __name__ == "__main__":

	databasename=':memory:'
	loging_level_parameter='warning'
	filename=''
	filetype = Filetype.TEX

	try:
		opts, args = getopt.getopt(sys.argv[1:],"D:l:i:",["database=","log="])
	except getopt.GetoptError:
		print (sys.argv[0] + ' -D <databasename> -l <logginglevel> -i <infile>')
		sys.exit(2)
	for opt, arg in opts:
		if opt in ('-D', '--database'):
			databasename = arg
		elif opt in ('-l', '--log'):
			loging_level_parameter = arg
		elif opt in ('-i', '--infile'):
			filename = arg
		else:
			assert False, "unhandled option: %s" % arg
	if filename == '':
		assert False, "need at least the inputfile (-i) as parameter"

	logging_level = getattr(logging, loging_level_parameter.upper(), None)
	if not isinstance(logging_level, int):
		raise ValueError('Invalid log level: %s' % loging_level_parameter)
	logging.basicConfig(level=logging_level)

	readstatus = ReadStatus.NONE
	sqlbuffer = ''

	assert os.access(filename, os.R_OK), 'cannot read file %s' % filename
	filetype = Filetype.fromString(os.path.splitext(filename)[1][1:])
	if filetype == Filetype.UNKNOWN:
		die("unknown file type of file %s" % filename)

	conn = sqlite3.connect(databasename)
	if logging_level <= logging.DEBUG:
		sqlite3.enable_callback_tracebacks(True)
		conn.set_trace_callback(print)
	conn.row_factory = sqlite3.Row
	conn.create_function("pow", 2, lambda base,exp: math.pow(base, exp))
	conn.create_function("log", 2, lambda base,x: math.log(x, base))
	conn.create_function("basename", 1, lambda filepath: os.path.basename(filepath))
	cursor = conn.cursor()

	config_args : t.Mapping[str,str] = dict()
	outfile = sys.stdout
	outfilename = None
	outfiletype = Filetype.TEX
	previous_entries = -1

	""" mapping names to macros """
	macros : t.Mapping[str, Macro] = dict()

	""" storing the last index of the written gnuplot data for each file """
	gnuplot_line_index : t.Dict[str, int] = dict()

	with open(filename) as texfile:
		for texLine in texfile.readlines():
			if readstatus == ReadStatus.MACRO:
				if texLine.startswith(filetype.comment()):
					print(texLine, end='')
					sqlbuffer+=' ' + texLine[len(filetype.comment()):].rstrip()
					continue
				readstatus = ReadStatus.NONE
				match = re.match(r'\s*DEFINE\s+(\w+)\s*\(([^)]+)\)\s*(.*)', sqlbuffer)
				assert match, "no valid MACRO: " + sqlbuffer
				name = match.group(1)
				arguments = list(map(lambda x: x.strip(), match.group(2).split(',')))
				body = match.group(3)
				for argument in arguments:
					assert body.find('$' + argument) != -1, "argument %s not found in body: %s" % (argument, body)
				macros[name] = Macro(name, arguments, body)

			if readstatus in [ReadStatus.MULTIPLOT, ReadStatus.TABULAR, ReadStatus.SINGLEPLOT, ReadStatus.MATRIX]:
				if texLine.startswith(filetype.comment()):
					print(texLine, end='')
					if texLine.startswith('%s CONFIG' % filetype.comment()):
						config_args = split_keyvalueline(texLine[len('%s CONFIG' % filetype.comment()):])
					else:
						sqlbuffer+=' ' + texLine[len(filetype.comment()):].rstrip()
					continue
				else:
					if readstatus in [ReadStatus.MULTIPLOT, ReadStatus.SINGLEPLOT]:
						outfiletype = filetype
						""" if mode=a we use the previous_entries for the cycle list """
						if not 'mode' in config_args or config_args['mode'].find('a') == -1:
							previous_entries = 0
						if 'type' in config_args:
							outfiletype = Filetype.fromString(config_args['type'])
						if 'file' in config_args:
							if outfiletype == Filetype.GNUPLOT:
								assert ('mode' in config_args and config_args['mode'].find('a') != -1) or config_args['file'] not in gnuplot_line_index, 'overwriting a .dat file created within this execution without append mode is prohibited'

							os.makedirs(os.path.dirname(config_args['file']), exist_ok=True)
							outfile = open(config_args['file'], 'w' if not 'mode' in config_args else config_args['mode'])
							if outfiletype == Filetype.TEX:
								print('\\input{%s}' % config_args['file'])
							elif outfiletype == Filetype.PYTHON:
								print('# read comments in file %s for the plot command' % config_args['file'])

						else:
							outfile = sys.stdout
							assert outfiletype != Filetype.GNUPLOT, "need CONFIG file={outfile} parameter to know where to write the data"
						# try:
						# 	previous_entries
						# except NameError:
						if previous_entries == -1:
							die('mode is set to append, but there is no previous content!')

					if readstatus == ReadStatus.TABULAR:
						readstatus = ReadStatus.ERASE
						sqlbuffer = apply_macros(sqlbuffer[sqlbuffer.find('TABULAR')+len('TABULAR'):])
						sqlexecute(sqlbuffer + ';')

						if 'file' in config_args:
							outfile = open(config_args['file'], 'w' if not 'mode' in config_args else config_args['mode'])
							print('\\input{%s}' % config_args['file'])
						else:
							outfile = sys.stdout

						for row in cursor.fetchall():
							print(" & ".join(map(print_tablentry, row)) + ' \\\\', file=outfile)
					elif readstatus == ReadStatus.SINGLEPLOT:
						readstatus = ReadStatus.ERASE
						match = re.match(r'\s*SINGLEPLOT\(([^)]+)\)', sqlbuffer)
						assert match, "no singleplot argument given: " + sqlbuffer
						singleplot_name = match.group(1)
						sqlbuffer = sqlbuffer[match.span()[1]:] #remove 'MULTIPLOT(...) directive
						sqlexecute(sqlbuffer + ';')
						rows = cursor.fetchall()
						coordinates=dict()
						coordinates[(singleplot_name,)] = list(map(lambda row: (row['x'], row['y']), rows))
						previous_entries = plot_coordinates(sqlbuffer, config_args['file'] if 'file' in config_args else 'stdout', coordinates, outfile, outfiletype, previous_entries)
					elif readstatus == ReadStatus.MATRIX:
						readstatus = ReadStatus.ERASE
						sqlbuffer = apply_macros(sqlbuffer[sqlbuffer.find('MATRIX')+len('MATRIX'):])
						sqlexecute(sqlbuffer + ';')

						if 'file' in config_args:
							outfile = open(config_args['file'], 'w' if not 'mode' in config_args else config_args['mode'])
							print('\\input{%s}' % config_args['file'])
						else:
							outfile = sys.stdout

						column_names=set()
						row_names=set()
						matrix=dict()
						for row in cursor.fetchall():
							print(row)
							column_names.add(row['x'])
							row_names.add(row['y'])
							matrix[(row['x'], row['y'])] = row['val']
						print(" & ".join(map(str, column_names)) + ' \\\\', file=outfile)
						for row in row_names:
							print(row + " & " + " & ".join(map(print_tablentry, map(lambda x: matrix[(x, row)], column_names))) + ' \\\\', file=outfile)
					else:
						assert readstatus == ReadStatus.MULTIPLOT
						readstatus = ReadStatus.ERASE
						match = re.match(r'\s*MULTIPLOT\(([^)]+)\)', sqlbuffer)
						assert match, "no multiplot argument given: " + sqlbuffer
						multiplot_columns = match.group(1)
						sqlbuffer_rest = sqlbuffer[match.span()[1]:] #remove 'MULTIPLOT(...) directive
						coordinates = multiplot(sqlbuffer_rest, list(map(lambda col: col.strip(), multiplot_columns.split(','))))
						previous_entries = plot_coordinates(sqlbuffer, config_args['file'] if 'file' in config_args else 'stdout', coordinates, outfile, outfiletype, previous_entries)
					#cleanup
					if 'file' in config_args:
						outfile.close()
					config_args=dict()

			if readstatus == ReadStatus.ERASE:
				if len(texLine.strip()) == 0 or texLine.startswith(filetype.comment()):
					readstatus = ReadStatus.NONE
				else:
					continue

			#! check for a multiline command stored in keyword_to_status
			for key in keyword_to_status:
				if texLine.startswith('%s %s' % (filetype.comment(), key) ):
					config_args=dict()
					sqlbuffer = texLine[len(filetype.comment()):].rstrip()
					readstatus = keyword_to_status[key]
					break

			if texLine.startswith('%s UNDEF ' % filetype.comment()):
				sqlbuffer = texLine[len(filetype.comment()):].strip()
				match = re.match(r'UNDEF\s+(\w+)\s*', sqlbuffer)
				assert match, 'invalid UNDEF syntax : ' + sqlbuffer
				name = match.group(1).strip()
				assert name in macros, 'cannot UNDEF undefined macro: ' + name
				del macros[name]

			#! read a log file with '^RESULT .*' statements into a sql table
			if texLine.startswith('%s IMPORT-DATA ' % filetype.comment()):
				match = re.match('%s IMPORT-DATA ([^ ]+) (.+)' % filetype.comment(), texLine)
				assert match, 'invalid texLine ' + texLine
				create_table(match.group(1), match.group(2))

			#! read a list of JSON log statements into a sql table
			if texLine.startswith('%s IMPORT-JSON-DATA ' % filetype.comment()):
				match = re.match('%s IMPORT-JSON-DATA ([^ ]+) (.+)' % filetype.comment(), texLine)
				assert match, 'invalid texLine ' + texLine
				create_json_table(match.group(1), match.group(2))

			print(texLine, end='')

	conn.close()

	if filetype == Filetype.TEX:
		with open('pgf_color_entries.txt','w') as txtfile:
			print('# this file is automatically created by sqlplot.py to ensure the same legend symbol for each entry in all plots generated by sqlplot.py', file=txtfile)
			for key in color_entries:
				txtfile.write('%s\t%d\n' % (key, color_entries[key]))

# vim: ts=2


"""Check prediction entropy and dice score correlation"""

# %%
import os
import yaml

import matplotlib.pyplot as plt
import seaborn as sns

import torch

# enable lib loading even if not installed as a pip package or in PYTHONPATH
# also convenient for relative paths in example config files
from pathlib import Path
os.chdir(Path(__file__).resolve().parent.parent)

from adpkd_segmentation.config.config_utils import get_object_instance # noqa
from adpkd_segmentation.evaluate import validate # noqa
from adpkd_segmentation.utils.train_utils import load_model_data # noqa

sns.set()

# %%
CONFIG = "experiments/september02/random_split_new_data_less_albu_10_more/val/val.yaml" # noqa

with open(CONFIG, "r") as f:
    config = yaml.load(f, Loader=yaml.FullLoader)

# %%
model_config = config["_MODEL_CONFIG"]
dataloader_config = config["_VAL_DATALOADER_CONFIG"]
losses_config = config["_LOSSES_METRICS_CONFIG"]
saved_checkpoint = config["_MODEL_CHECKPOINT"]
# override
dataloader_config["batchsize"] = 1

# %%
model = get_object_instance(model_config)()
load_model_data(saved_checkpoint, model, True)
dataloader = get_object_instance(dataloader_config)()
loss_metric = get_object_instance(losses_config)()

# %%
device = torch.device("cuda:0")
model = model.to(device)
model.eval()

# %%
averaged, all_losses_and_metrics = validate(
    dataloader, model, loss_metric, device, output_losses_list=True)

# %%
dice_scores = all_losses_and_metrics["dice_metric"]
entropy = all_losses_and_metrics["prediction_entropy"]

plt.xlabel("Prediction entropy")
plt.ylabel("Dice score")
sns.scatterplot(x=entropy, y=dice_scores)

# %%