#!/bin/python3

from exceptions import EdgeDoesNotExists, NoCompatibleConnectionFound
from itertools import combinations

import pprint

class Weight:

    def __init__(self, w):
       self.vector = w

    def __add__(self, other):
        sum = self.vector + other.vector
        return Weight(sum)

    def __sub__(self, other):
        sum = (self.vector[0] - other.vector[0], self.vector[1] - other.vector[1])
        return Weight(sum)

    def __rmul__(self, other):
        return Weight([other*entry for entry in self.vector])

    def __repr__(self) -> str:
        return f"({self.vector[0]}, {self.vector[1]})"

def GKMCondition(w1: Weight, w2: Weight, w3: Weight) -> bool:
    """
    Consider w3 = nabla_w1 w2 = w2 + c*w1
    (overloaded notation, w are weights and edges...)
    Return True if such a c exists, otherwise False

    AT THE MOMENT, IT WORKS ONLY FOR WEIGHTS IN Z^2
    """

    w = w3 - w2
    det = w.vector[0]*w1.vector[1] - w.vector[1]*w1.vector[0]

    if not det == 0: return False

    try:
        if w.vector[0] % w1.vector[0] == 0:
            return True
        else:
            return False
    except ZeroDivisionError:
        if w.vector[1] % w1.vector[1] == 0:
            return True
        else:
            return False

class Edge:

    def __init__(self, v1, v2, w = None) -> None:
        self.v1 = v1
        self.v2 = v2
        self.weight = w
        self.name = (v1,v2)
        self.hash = f"{v1}{v2}"

    def setWeight(self, w : Weight):
        self.weight = w
        return self

    def __repr__(self) -> str:
        r = ""

        return f"({self.v1},{self.v2}){r}"

    def __eq__(self, other) -> bool:
        if (self.v1 == other.v1 and self.v2 == other.v2):
            return True
        else:
            return False

    def __hash__(self):
        return hash(self.hash)

def findEdge(edges: list, v1, v2)->Edge:

    for e  in edges:
        if (e.v1 == v1 and e.v2 == v2) or (e.v1 == v2 and e.v2 == v1):
            return e

    raise EdgeDoesNotExists("Edge does not exist")

class Connection:

    def __init__(self) -> None:
        self.con = {}

    def setConnection(self, e1: Edge, e2: Edge, e3: Edge):
        self.con[e1] = { e2 : e3 }
        return self

class Graph:

    """
    In the configuration file the first column represents a vertex und the other
    columns indicate the vertex to which the former one is connected.
    """

    def __init__(self, configuration, connection = None) -> None:
        self.vertices = []
        self.edges = []
        self.connection = {}

        with open(configuration , "r") as file:
            for line in file.readlines():
                items = line.strip().split(" ")
                for item in items:
                    self.vertices.append(item) if not item in self.vertices else self.vertices
                    if not item == items[0]:
                        self.edges.append(Edge(items[0],item))

        if connection:
            with open(connection, "r") as file:
                for line in file.readlines():
                    i1 = line.split(";")[0].split(",")

                    for edge in self.edges:
                        if edge.v1 == i1[0] and edge.v2 == i1[1]:
                            e1 = edge

                    if not e1 in self.connection:
                        self.connection[e1] = {}

                    i2 = line.split(";")[1].split(",")
                    i3 = line.split(";")[2].split(",")

                    for edge in self.edges:
                        if edge.v1 == i2[0] and edge.v2 == i2[1]:
                            e2 = edge
                        if edge.v1 == i3[0] and edge.v2 == i3[1].strip():
                            e3 = edge

                    self.connection[e1][e2] = e3
                    self.connection[e1][e3] = e2

    def findEdge(self, v1, v2) -> Edge:
        e = Edge(str(v1),str(v2))
        for edge in self.edges:
            if edge == e:
                return edge
        else:
            raise EdgeDoesNotExists(f"Edge: ({e.v1},{e.v2}) does not exists!")

    def returnEdges(self, name):
        with open(f"edges_{name}.gkm", "w") as file:
            for edge in self.edges:
                file.write(f"{edge.v1},{edge.v2};\n")

    def loadWeights(self, name):
        with open(f"edges_{name}.gkm", "r") as file:
            for line in file.readlines():
                items = line.strip().split(";")
                e1 = items[0].split(",")[0]
                e2 = items[0].split(",")[1]
                w1 = items[1].split(",")[0]
                w2 = items[1].split(",")[1]

                e = self.findEdge(e1, e2)
                e.setWeight(Weight((int(w1),int(w2))))

        return self

    def connectionPath(self, e1: Edge, e2: Edge):
        """
        We start with nabla_e1 (e2)
        """
        path = []
        start = e1
        end = e2
        next = self.connection[e1][e2]
        prev = e1
        path.append(e1)

        while True:

            if next == end and self.connection[next][prev] == start:
                path.append(end)
                break

            path.append(next)
            p = next
            # nabla_next (prev)
            next = self.connection[next][prev]
            prev = p

        return path

    def emanatingEdges(self, vertex):
        edges = []

        for edge in self.edges:
            if edge.v1 == vertex or edge.v2 == vertex:
                edges.append(edge)

        return edges

    def getEpsilon(self, e1: Edge, e2: Edge):
        e3 = self.connection[e1][e2]

        w1 = e1.weight
        w2 = e2.weight
        w3 = e3.weight

        if GKMCondition(w1,w2,w3):
            return 1
        if GKMCondition(w1,(-1)*w2,w3):
            return -1

    def computeEta(self, vertex, pathEdge: Edge):
        edges = self.emanatingEdges(vertex)
        eta = -1

        for edge in edges:
            if not edge == pathEdge:
                eps = self.getEpsilon(pathEdge, edge)
                eta = eta * eps

        return eta

    def computeOrientationPath(self, path: list):
        r = 1
        for e in path:
            eta = self.computeEta(e.v1, e)
            r = r * eta
        return r

    def computeAllConnectionPaths(self):
        paths = []
        set_paths = []

        for vertex in self.vertices:
            edges = self.emanatingEdges(vertex)
            pairs = list(combinations(edges, 2))
            for pair in pairs:
                new = self.connectionPath(pair[0],pair[1])
                if not set(new) in set_paths:
                    paths.append(new)
                    set_paths.append(set(new))

        return paths

    def createConnection(self):
        for edge in self.edges:
            self.connection[edge] = {}
            initial_edges = self.emanatingEdges(edge.v1)
            terminal_edges = self.emanatingEdges(edge.v2)

            initial_edges.remove(edge)
            terminal_edges.remove(edge)

            for initial in initial_edges:
                for terminal in terminal_edges:
                    if GKMCondition(edge.weight, initial.weight,
                                    terminal.weight):
                        #print(edge, initial, terminal)
                        self.connection[edge][initial] = terminal
                        self.connection[edge][terminal] = initial
                        terminal_edges.remove(terminal)
                        break
                    elif GKMCondition(edge.weight, (-1)*initial.weight,
                                    terminal.weight):
                        #print(edge, initial, terminal)
                        self.connection[edge][initial] = terminal
                        self.connection[edge][terminal] = initial
                        terminal_edges.remove(terminal)
                        break
                    else:
                        #print(f"nabla_{edge}{initial}={terminal} violates GKM condition")
                        pass

            if self.connection[edge] == {}:
                raise NoCompatibleConnectionFound(f"No compatible connection found for edge{edge}")

def vgraph():
    vg = Graph("vgraph.gkm")

    vg.loadWeights("vgraph")
    p = [ (1,3), (3,4), (4,6), (6,7), (7,8), (8,11), (11,12), (1,12) ]
    #p = [(4,5), (5,6), (6,7), (7,9), (8,9), (7,8), (6,7), (4,6)]
    outer = [ vg.findEdge(e[0],e[1]) for e in p]
    vg.createConnection()
    print(p, vg.computeOrientationPath(outer))
    #print(vg.computeOrientationPath(outer))
    for path in vg.computeAllConnectionPaths():
        print(path, vg.computeOrientationPath(path))

def cp3():
    cp3 = Graph("cp3.gkm")
    cp3.loadWeights("cp3")
    cp3.createConnection()
    paths = cp3.computeAllConnectionPaths()
    for path in paths:
        print(path, cp3.computeOrientationPath(path))
    #connection_paths = cp3.computeAllConnectionPaths()
    #for path in connection_paths:
    #    print(cp3.computeOrientationPath(path))

def tolman():
    tolman = Graph("tolman.gkm")
    tolman.loadWeights("tolman")
    tolman.createConnection()
    pprint.pprint(tolman.connection)
    for path in tolman.computeAllConnectionPaths():
        print(path, tolman.computeOrientationPath(path))

def main():
    #cp3()
    #tolman()
    vgraph()

if __name__ == "__main__":
  main()


from django import forms

from .models import Profile
from .models import ProfileSettings

class ProfileForm(forms.ModelForm):
    class Meta:
        model=Profile
        fields=(
            'external_id',
            'name',
        )
        widgets={
            'name':forms.TextInput,
        }
class ProfileSettingsForm(forms.ModelForm):
    class Meta:
        model=ProfileSettings
        fields=(
            'external',
            'api_key',
            'secret_key',
            'subaccount_email',
            'pool_username',
        )
        widgets={
            'api_key':forms.TextInput,
            'secret_key': forms.TextInput,
            'subaccount_email': forms.TextInput,
            'pool_username': forms.TextInput,
        }


# Imports python modules
from time import time, sleep

import matplotlib.pyplot as plt
import seaborn as sns

from torchvision import datasets, transforms

import torch
import torchvision
import torchvision.transforms as transforms
import torch.optim as optim
#import time
import torch.nn.functional as F
import torch.nn as nn
import matplotlib.pyplot as plt
from torchvision import models
#import helper
#import fc_model
import numpy as np
from torch import nn

from PIL import Image
import glob, os
from torch.autograd import Variable

import matplotlib.pyplot as plt
import os, random
import json
from collections import OrderedDict
#import time
# Imports functions created for this program
from get_input_args import get_input_args

#Load a pre-trained network (If you need a starting point, the VGG networks work great and are straightforward to use)
# validation function
def validate(model, test_dataloader, device, criterion):
    """
    Validate
    Parameters:
     model - model to be used
     test_dataloader - test set dataloader
     device - CPU or GPU
     criterion - criterion to be used
    Returns:
     None
    """
    model.eval()
    val_running_loss = 0.0
    val_running_correct = 0
    for int, data in enumerate(test_dataloader):
        data, target = data[0].to(device), data[1].to(device)
        output = model(data)
        loss = criterion(output, target)

        val_running_loss += loss.item()
        _, preds = torch.max(output.data, 1)
        val_running_correct += (preds == target).sum().item()

    val_loss = val_running_loss/len(test_dataloader.dataset)
    val_accuracy = 100. * val_running_correct/len(test_dataloader.dataset)

    return val_loss, val_accuracy

#Load a pre-trained network (If you need a starting point, the VGG networks work great and are straightforward to use)
# training function
def fit(model, train_dataloader, device, optimizer, criterion):
    """
    Fit
    Parameters:
     model - model to be used
     train_dataloader - training set dataloader
     device - CPU or GPU
     optimizer -  optimizer to be used
     criterion - criterion to be used
    Returns:
     None
    """
    model.train()
    train_running_loss = 0.0
    train_running_correct = 0
    for i, data in enumerate(train_dataloader):
        data, target = data[0].to(device), data[1].to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = criterion(output, target)
        train_running_loss += loss.item()
        _, preds = torch.max(output.data, 1)
        train_running_correct += (preds == target).sum().item()
        loss.backward()
        optimizer.step()
    train_loss = train_running_loss/len(train_dataloader.dataset)
    train_accuracy = 100. * train_running_correct/len(train_dataloader.dataset)
    print(f'Train Loss: {train_loss:.4f}, Train Acc: {train_accuracy:.2f}')

    return train_loss, train_accuracy

def load_data(data_dir):
    """
    Load data from data directory
    Parameters:
     data_dir - data directory
    Returns:
     Dataloaders for training, testing and validation
    """
    train_dir = data_dir + '/train'
    valid_dir = data_dir + '/valid'
    test_dir = data_dir + '/test'
    # TODO: Define your transforms for the training, validation, and testing sets
    data_transforms = {
         'training' : transforms.Compose([transforms.RandomResizedCrop(224),
                                    transforms.RandomHorizontalFlip(),transforms.RandomRotation(30),
                                    transforms.ToTensor(),
                                    transforms.Normalize([0.485, 0.456, 0.406],
                                                         [0.229, 0.224, 0.225])]),

        'validation' : transforms.Compose([transforms.Resize(256),
                                      transforms.CenterCrop(224),
                                      transforms.ToTensor(),
                                      transforms.Normalize([0.485, 0.456, 0.406],
                                                           [0.229, 0.224, 0.225])]),

        'testing' : transforms.Compose([transforms.Resize(256),
                                      transforms.CenterCrop(224),
                                      transforms.ToTensor(),
                                      transforms.Normalize([0.485, 0.456, 0.406],
                                                           [0.229, 0.224, 0.225])])
    }
    print("Loading from {}".format(train_dir))
    print("Loading from {}".format(test_dir))
    print("Loading from {}".format(valid_dir))

    # TODO: Load the datasets with ImageFolder
    image_datasets = {
        'training' : datasets.ImageFolder(train_dir, transform=data_transforms['training']),
        'testing' : datasets.ImageFolder(test_dir, transform=data_transforms['testing']),
        'validation' : datasets.ImageFolder(valid_dir, transform=data_transforms['validation'])
    }

    # TODO: Using the image datasets and the trainforms, define the dataloaders
    dataloaders = {
        'training' : torch.utils.data.DataLoader(image_datasets['training'], batch_size=32, shuffle=True),
        'testing' : torch.utils.data.DataLoader(image_datasets['testing'], batch_size=32, shuffle=False),
        'validation' : torch.utils.data.DataLoader(image_datasets['validation'], batch_size=32, shuffle=True)
    }
    print("Loaded from {}".format(train_dir))
    print("Loaded from {}".format(test_dir))
    print("Loaded from {}".format(valid_dir))
    return image_datasets, dataloaders

def load_model(device, learning_rate):
    """
    Load model
    Parameters:
     device - cpu or gpu
     learning_rate - learning rate to be set
    Returns:
     Model, optimizer and criterion
    """
    #Load a pre-trained network (If you need a starting point, the VGG networks work great and are straightforward to use)
    model = models.vgg19(pretrained=True)

    classifier = nn.Sequential(OrderedDict([
                          ('fc1', nn.Linear(25088, 4096)), # First layer
                          ('relu', nn.ReLU()), # Apply activation function
                          ('fc2', nn.Linear(4096, 102)), # Output layer
                          ('output', nn.LogSoftmax(dim=1)) # Apply loss function
                          ]))
    model.classifier = classifier

    # freeze convolution weights
    for param in model.features.parameters():
        param.requires_grad = False

    model.to(device)

    # optimizer
    print("learning_rate = {}".format(learning_rate))

    optimizer = optim.SGD(model.classifier.parameters(), lr=learning_rate, momentum=0.9)
    # loss function
    criterion = nn.CrossEntropyLoss()
    return model, optimizer, criterion

def train(model, dataloaders, epoch_to_train, device, optimizer, criterion):
    """
    Train the model
    Parameters:
     model - model loaded to be trained
     dataloaders - dataloaders to get datasets
     epoch -  number of epoch to run training
     device - cpu or gpu
     optimizer - optimizer
     criterion - criterion
    Returns:
     Model
    """
    #Load a pre-trained network (If you need a starting point, the VGG networks work great and are straightforward to use)
    train_loss , train_accuracy = [], []
    val_loss , val_accuracy = [], []
    start = time()

    print("epoch_to_train = {}".format(epoch_to_train))
    for epoch in range(epoch_to_train):
        print("Starting epoch = {}".format(epoch))
        print("Fitting")
        train_epoch_loss, train_epoch_accuracy = fit(model, dataloaders['training'], device, optimizer, criterion)
        print("Validating")
        val_epoch_loss, val_epoch_accuracy = validate(model, dataloaders['testing'], device, criterion)
        print("val_epoch_accuracy = {}".format(val_epoch_accuracy))
        train_loss.append(train_epoch_loss)
        train_accuracy.append(train_epoch_accuracy)
        val_loss.append(val_epoch_loss)
        val_accuracy.append(val_epoch_accuracy)
        #stop training when accuracy exceeds 85
        if val_epoch_accuracy > 85:
            print("Exiting at epoch = {}".format(epoch))
            break
    end = time()
    print('Training time: ', (end-start)/60, ' minutes')
    return model

def validate_model(model, validation_dataloader):
    #run against validation dataset
    model.to('cpu')
    images, labels = next(iter(validation_dataloader))
    # Get the class probabilities
    ps = torch.exp(model(images))
    print(ps.shape)
    top_p, top_class = ps.topk(1, dim=1)
    # Look at the most likely classes for the first 10 examples
    print(top_class[:10,:])
    matches = top_class == labels.view(*top_class.shape)
    accuracy = torch.mean(matches.type(torch.FloatTensor))
    return accuracy

def save_checkpoint(model, image_datasets):
    model.class_to_idx = image_datasets['training'].class_to_idx
    torch.save({'arch': 'vgg19',
            'input_size': 25088,
            'output_size': 102,
            'state_dict': model.state_dict(), # Holds all the weights and biases
            'class_to_idx': model.class_to_idx},
            'model.pth')
    print("Model saved as 'model.pth'")

# Main program function defined below
def main():
    # TODO 0: Measures total program runtime by collecting start time
    start_time = time()
    # check GPU availability
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    print(device)

    in_arg = get_input_args()
    print(in_arg)
#    check_command_line_arguments(in_arg)
    image_datasets, dataloaders = load_data(in_arg.data_directory)
    #model, optimizer, criterion = load_model(device, 0.001)
    model, optimizer, criterion = load_model(device, in_arg.learning_rate)
    print("Model loaded")
    print(model)
    #model = train(model, dataloaders, 9, device, optimizer, criterion)
    model = train(model, dataloaders, in_arg.epochs, device, optimizer, criterion)
    print("Model trained")
    print(model)
    print("Validating model")
    accuracy = validate_model(model, dataloaders['validation'])
    print(f'Accuracy: {accuracy.item()*100}%')
    print("Saving checkpoint")
    save_checkpoint(model, image_datasets)

    end_time = time()

    tot_time = end_time - start_time #calculate difference between end time and start time
    print("\n** Total Elapsed Runtime:",
          str(int((tot_time/3600)))+":"+str(int((tot_time%3600)/60))+":"
          +str(int((tot_time%3600)%60)) )

# Call to main function to run the program
if __name__ == "__main__":
    main()


import json

from google.cloud import pubsub_v1
from constans import BUCKET_KEY_GCP
from core.format_converte import ExportMusic
from app import app
# from modelos import db

class PubSubConverter:

    def __init__(self):

        self.json_key = f'{BUCKET_KEY_GCP}/key.json'

    def subscriber_message(self):
        subscriber = pubsub_v1.SubscriberClient.from_service_account_json(self.json_key)
        subscription_path = subscriber.subscription_path("flask-test-366421", "pruebas-flask-sub")

        def callback(message: pubsub_v1.subscriber.message.Message) -> None:
            print("message:")
            print(message.data)
            value = json.loads(message.data)
            print(value)
            export = ExportMusic()
            export.export_file(value)
            message.ack()

        streaming_pull_future = subscriber.subscribe(subscription_path, callback=callback)
        print(f"Listening for messages on {subscription_path}..\n")
        with subscriber:
            try:
                # When `timeout` is not set, result() will block indefinitely,
                streaming_pull_future.result()
            except TimeoutError:
                streaming_pull_future.cancel()  # Trigger the shutdown.
                streaming_pull_future.result()


from __future__ import absolute_import, division, print_function

import decimal
from decimal import *
import logging
import typing
from builtins import *

import past.builtins
import xlrd
import xlwt

import canmatrix
import canmatrix.formats.xls_common

logger = logging.getLogger(__name__)
default_float_factory = decimal.Decimal

# Font Size : 8pt * 20 = 160
# font = 'font: name Arial Narrow, height 160'
font = 'font: name Verdana, height 160'

if xlwt is not None:
    sty_header = xlwt.easyxf(font + ', bold on; align: vertical center, horizontal center',
                             'pattern: pattern solid, fore-colour rose')
    sty_norm = xlwt.easyxf(font + ', colour black')
    sty_first_frame = xlwt.easyxf(font + ', colour black; borders: top thin')
    sty_white = xlwt.easyxf(font + ', colour white')

    # ECU Matrix-Styles
    sty_green = xlwt.easyxf('pattern: pattern solid, fore-colour light_green')
    sty_green_first_frame = xlwt.easyxf('pattern: pattern solid, fore-colour light_green; borders: top thin')
    sty_sender = xlwt.easyxf('pattern: pattern 0x04, fore-colour gray25')
    sty_sender_first_frame = xlwt.easyxf('pattern: pattern 0x04, fore-colour gray25; borders: top thin')
    sty_sender_green = xlwt.easyxf('pattern: pattern 0x04, fore-colour gray25, back-colour light_green')
    sty_sender_green_first_frame = xlwt.easyxf(
        'pattern: pattern 0x04, fore-colour gray25, back-colour light_green; borders: top thin')

def write_ecu_matrix(ecus, sig, frame, worksheet, row, col, first_frame):

    print("def : format - xls - write_ecu_matrix")

    # type: (typing.Sequence[str], typing.Optional[canmatrix.Signal], canmatrix.Frame, xlwt.Worksheet, int, int, xlwt.XFStyle) -> int
    # first-frame - style with borders:
    if first_frame == sty_first_frame:
        norm = sty_first_frame
        sender = sty_sender_first_frame
        norm_green = sty_green_first_frame
        sender_green = sty_sender_green_first_frame
    # consecutive-frame - style without borders:
    else:
        norm = sty_norm
        sender = sty_sender
        norm_green = sty_green
        sender_green = sty_sender_green

    # iterate over ECUs:
    for ecu_name in ecus:
        # every second ECU with other style
        if col % 2 == 0:
            loc_style = norm
            loc_style_sender = sender
        # every second ECU with other style
        else:
            loc_style = norm_green
            loc_style_sender = sender_green

        # write "s" "r" "r/s" if signal is sent, received or send and received by ECU
        if sig and ecu_name in sig.receivers and ecu_name in frame.transmitters:
            worksheet.write(row, col, label="r/s", style=loc_style_sender)
        elif sig and ecu_name in sig.receivers:
            worksheet.write(row, col, label="r", style=loc_style)
        elif ecu_name in frame.transmitters:
            worksheet.write(row, col, label="s", style=loc_style_sender)
        else:
            worksheet.write(row, col, label="", style=loc_style)
        col += 1
    # loop over ECUs ends here
    return col

def write_excel_line(worksheet, row, col, row_array, style):

    print("def : format - xls - write_excel_line")

    # type: (xlwt.Worksheet, int, int, typing.Sequence, xlwt.XFStyle) -> int
    for item in row_array:
        worksheet.write(row, col, label=item, style=style)
        col += 1
    return col

def dump(db, file, **options):

    print("def : format - xls - dump")

    # type: (canmatrix.CanMatrix, typing.IO, **typing.Any) -> None
    head_top = ['ID', 'Frame Name', 'Cycle Time [ms]', 'Launch Type', 'Launch Parameter', 'Signal Byte No.',
                'Signal Bit No.', 'Signal Name', 'Signal Function', 'Signal Length [Bit]', 'Signal Default',
                ' Signal Not Available', 'Byteorder']
    head_tail = ['Value', 'Name / Phys. Range', 'Function / Increment Unit']

    if len(options.get("additionalSignalAttributes", "")) > 0:
        additional_signal_columns = options.get("additionalSignalAttributes").split(",")  # type: typing.List[str]
    else:
        additional_signal_columns = []  # ["attributes['DisplayDecimalPlaces']"]

    if len(options.get("additionalFrameAttributes", "")) > 0:
        additional_frame_columns = options.get("additionalFrameAttributes").split(",")  # type: typing.List[str]
    else:
        additional_frame_columns = []  # ["attributes['DisplayDecimalPlaces']"]

    motorola_bit_format = options.get("xlsMotorolaBitFormat", "msbreverse")

    workbook = xlwt.Workbook(encoding='utf8')
    #    ws_name = os.path.basename(filename).replace('.xls', '')
    #    worksheet = workbook.add_sheet('K-Matrix ' + ws_name[0:22])
    worksheet = workbook.add_sheet('K-Matrix ')

    row_array = []  # type: typing.List[str]
    col = 0

    # write ECUs in first row:
    ecu_list = [ecu.name for ecu in db.ecus]

    row_array += head_top
    head_start = len(row_array)

    row_array += ecu_list
    for col in range(len(row_array)):
        worksheet.col(col).width = 1111
    tail_start = len(row_array)
    row_array += head_tail

    additional_frame_start = len(row_array)

    for col in range(tail_start, len(row_array)):
        worksheet.col(col).width = 3333

    for additionalCol in additional_frame_columns:
        row_array.append("frame." + additionalCol)
        col += 1

    for additionalCol in additional_signal_columns:
        row_array.append("signal." + additionalCol)
        col += 1

    write_excel_line(worksheet, 0, 0, row_array, sty_header)

    # set width of selected Cols
    worksheet.col(1).width = 5555
    worksheet.col(3).width = 3333
    worksheet.col(7).width = 5555
    worksheet.col(8).width = 7777
    worksheet.col(head_start).width = 1111
    worksheet.col(head_start + 1).width = 5555

    frame_hash = {}
    if db.type == canmatrix.matrix_class.CAN:
        logger.debug("Length of db.frames is %d", len(db.frames))
        for frame in db.frames:
            if frame.is_complex_multiplexed:
                logger.error("export complex multiplexers is not supported - ignoring frame %s", frame.name)
                continue
            frame_hash[int(frame.arbitration_id.id)] = frame
    else:
        frame_hash = {a.name: a for a in db.frames}

    # set row to first Frame (row = 0 is header)
    row = 1

    # iterate over the frames
    for idx in sorted(frame_hash.keys()):

        frame = frame_hash[idx]
        frame_style = sty_first_frame

        # sort signals:
        sig_hash = {"{:02d}{}".format(sig.get_startbit(), sig.name): sig for sig in frame.signals}

        # set style for first line with border
        sig_style = sty_first_frame

        additional_frame_info = [frame.attribute(frameInfo, default="") for frameInfo in additional_frame_columns]

        # iterate over signals
        row_array = []
        if len(sig_hash) == 0:  # Frames without signals
            row_array += canmatrix.formats.xls_common.get_frame_info(db, frame)
            for _ in range(5, head_start):
                row_array.append("")
            temp_col = write_excel_line(worksheet, row, 0, row_array, frame_style)
            temp_col = write_ecu_matrix(ecu_list, None, frame, worksheet, row, temp_col, frame_style)

            row_array = []
            for col in range(temp_col, additional_frame_start):
                row_array.append("")
            row_array += additional_frame_info
            for _ in additional_signal_columns:
                row_array.append("")
            write_excel_line(worksheet, row, temp_col, row_array, frame_style)
            row += 1
            continue

        # iterate over signals
        for sig_idx in sorted(sig_hash.keys()):
            sig = sig_hash[sig_idx]

            # if not first Signal in Frame, set style
            if sig_style != sty_first_frame:
                sig_style = sty_norm

            if sig.values.__len__() > 0:  # signals with value table
                val_style = sig_style
                # iterate over values in value table
                for val in sorted(sig.values.keys()):
                    row_array = canmatrix.formats.xls_common.get_frame_info(db, frame)
                    front_col = write_excel_line(worksheet, row, 0, row_array, frame_style)
                    if frame_style != sty_first_frame:
                        worksheet.row(row).level = 1

                    col = head_start
                    col = write_ecu_matrix(ecu_list, sig, frame, worksheet, row, col, frame_style)

                    # write Value
                    (frontRow, backRow) = canmatrix.formats.xls_common.get_signal(db, frame, sig, motorola_bit_format)
                    write_excel_line(worksheet, row, front_col, frontRow, sig_style)
                    backRow += additional_frame_info
                    for item in additional_signal_columns:
                        temp = getattr(sig, item, "")
                        backRow.append(temp)

                    write_excel_line(worksheet, row, col + 2, backRow, sig_style)
                    write_excel_line(worksheet, row, col, [val, sig.values[val]], val_style)

                    # no min/max here, because min/max has same col as values...
                    # next row
                    row += 1
                    # set style to normal - without border
                    sig_style = sty_white
                    frame_style = sty_white
                    val_style = sty_norm
                # loop over values ends here
            # no value table available
            else:
                row_array = canmatrix.formats.xls_common.get_frame_info(db, frame)
                front_col = write_excel_line(worksheet, row, 0, row_array, frame_style)
                if frame_style != sty_first_frame:
                    worksheet.row(row).level = 1

                col = head_start
                col = write_ecu_matrix(
                    ecu_list, sig, frame, worksheet, row, col, frame_style)
                (frontRow, backRow) = canmatrix.formats.xls_common.get_signal(db, frame, sig, motorola_bit_format)
                write_excel_line(worksheet, row, front_col, frontRow, sig_style)

                if float(sig.min) != 0 or float(sig.max) != 1.0:
                    backRow.insert(0, str("%g..%g" % (sig.min, sig.max)))  # type: ignore
                else:
                    backRow.insert(0, "")
                backRow.insert(0, "")

                backRow += additional_frame_info
                for item in additional_signal_columns:
                    temp = getattr(sig, item, "")
                    backRow.append(temp)

                write_excel_line(worksheet, row, col, backRow, sig_style)

                # next row
                row += 1
                # set style to normal - without border
                sig_style = sty_white
                frame_style = sty_white
        # loop over signals ends here
    # loop over frames ends here

    # frozen headings instead of split panes
    worksheet.set_panes_frozen(True)
    # in general, freeze after last heading row
    worksheet.set_horz_split_pos(1)
    worksheet.set_remove_splits(True)
    # save file
    workbook.save(file)

# ########################### load ###############################

def parse_value_name_column(value_name, value_str, signal_size, float_factory):

    print("def : format - xls - parse_value_name_column")

    # type: (str, str, int, typing.Callable) -> typing.Tuple
    mini = maxi = offset = None  # type: typing.Any
    value_table = dict()
    if ".." in value_name:
        (mini, maxi) = value_name.strip().split("..")
        mini = float_factory(mini)
        maxi = float_factory(maxi)
        offset = mini

    elif len(value_name) > 0:
        if len(value_str.strip()) > 0:
            # Value Table
            value = int(float(value_str))
            value_table[value] = value_name
        maxi = pow(2, signal_size) - 1
        maxi = float_factory(maxi)
        mini = 0
        offset = 0
    return mini, maxi, offset, value_table

def read_additional_signal_attributes(signal, attribute_name, attribute_value):

    print("def : format - xls - read_additional_signal_attributes")

    if not attribute_name.startswith("signal"):
        return
    if attribute_name.replace("signal.", "") in vars(signal):
        command_str = attribute_name + "="
        command_str += str(attribute_value)
        if len(str(attribute_value)) > 0:
            exec(command_str)
    else:
        pass

def load(file, **options):

    print("def : format - xls - load")

    # type: (typing.IO, **typing.Any) -> canmatrix.CanMatrix
    wb = xlrd.open_workbook(file_contents=file.read())
    sh = wb.sheet_by_index(0)
    db = canmatrix.CanMatrix()

    # Defines not imported... add by users
    # add_ecu_defines BA_DEF_ BU_
    #db.add_ecu_defines("NWM-Stationsadresse", 'HEX 0 63')
    #db.add_ecu_defines("NWM-Knoten", 'ENUM  "nein","ja"')

    # add_frame_defines BA_DEF_ BO_
    #db.add_frame_defines("GenMsgDelayTime", 'INT 0 65535')
    #db.add_frame_defines("GenMsgCycleTimeActive", 'INT 0 65535')
    #db.add_frame_defines("GenMsgNrOfRepetitions", 'INT 0 65535')
    #db.add_frame_defines("GenMsgStartValue", 'STRING')

    # add_signal_defines BA_DEF_ SG_
    #db.add_signal_defines("GenSigSNA", 'STRING')

    SignalGroupTemp = []
    FrameTemp = []

    FrameWithSignal = []

    # eval search for correct columns:
    index = {}
    for i in range(sh.ncols):
        value = sh.cell(0, i).value
        if value == "Transmitter":
            index['Transmitter'] = i
        elif "Receiver" in value:
            index['Receiver'] = i
        elif "Signal Name" in value:
            index['SignalName'] = i
        elif "Message Name" in value:
            index['MessageName'] = i
        elif "Message ID" in value:
            index['MessageID'] = i
        elif "Message Type" in value:
            index['MessageType'] = i
        elif "Period [ms]" in value:
            index['Period'] = i
        elif "DLC [Byte]" in value:
            index['DLC'] = i
        elif "MSB" in value:
            index['MSB'] = i
        elif "LSB" in value:
            index['LSB'] = i
        elif "Size [Bit]" in value:
            index['Size'] = i
        elif "Byte Order" in value:
            print("def : format - xls - load - BYTE ORDER INDEX: {} ".format(i))
            index['ByteOrder'] = i
        elif "Datatype" in value:
            index['DataType'] = i
        elif "Default Initialized Value" in value:
            index['DefaultInitializedValue'] = i
        elif "Factor" in value:
            index['Factor'] = i
        elif "Offset" in value:
            index['Offset'] = i
        elif "P-Minimum" in value:
            index['Minimum'] = i
        elif "P-Maximum" in value:
            index['Maximum'] = i
        elif "Unit" == value:
            print("def : format - xls - load - UNIT INDEX: {} ".format(i))
            index['Unit'] = i
        elif "Coding" in value:
            index['Coding'] = i
        elif "Signal Group" in value:
            index['Signal Group'] = i
        elif "Comment" in value:
            index['Comment'] = i
        # elif "Multiplexer" in value:
        #     print("def : format - xls - load - MULTIPLEXER INDEX: {} ".format(i))
        #     index['Multiplexer'] = i

    index['ECUstart'] = index['Comment'] + 1
    index['ECUend'] = sh.ncols

    # ECUs:
    for x in range(index['ECUstart'], index['ECUend']):
        print("def : format - xls - load - ECU : {}".format(sh.cell(0, x).value))
        db.add_ecu(canmatrix.Ecu(sh.cell(0, x).value))

    # initialize:
    frame_id = None
    signal_name = None

    # read xls/xlsx row by row
    for row_num in range(1, sh.nrows):
        # new signal detected
        if sh.cell(row_num, index['SignalName']).value.strip() != signal_name or \
                sh.cell(row_num, index['MessageID']).value.strip() != frame_id:

            # receiver
            receiver = [sh.cell(row_num, index['Receiver']).value]

            # signal name
            signal_name = sh.cell(row_num, index['SignalName']).value.strip()

            # MSB
            msb = int(sh.cell(row_num, index['MSB']).value)

            # LSB
            lsb = int(sh.cell(row_num, index['LSB']).value)

            # size
            signal_length = int(sh.cell(row_num, index['Size']).value)

            # cycle_time
            cycle_time = 0

            # byte order
            if index.get("ByteOrder", False):
                signal_byte_order = sh.cell(row_num, index['ByteOrder']).value

                if 'intel' in signal_byte_order:
                    is_little_endian = True
                else:
                    is_little_endian = False
            else:
                is_little_endian = True  # Default Intel

            # datatype
            is_signed = sh.cell(row_num, index['DataType']).value

            # default initialized value (not use in dbc file)
            signal_default = sh.cell(row_num, index['DefaultInitializedValue']).value

            # factor
            factor = str(sh.cell(row_num, index['Factor']).value)
            factor = decimal.Decimal(factor)

            # offset
            offset = str(sh.cell(row_num, index['Offset']).value)
            offset = decimal.Decimal(offset)

            # p-minimum
            pminimum = str(sh.cell(row_num, index['Minimum']).value)
            pminimum = decimal.Decimal(pminimum)

            # p-maximum
            pmaximum = str(sh.cell(row_num, index['Maximum']).value)
            pmaximum = decimal.Decimal(pmaximum)

            # unit
            unit = sh.cell(row_num, index['Unit']).value
            print("def : format - xls - load - unit : {} - {} - {}".format(row_num,unit,index['Unit']))

            # coding
            coding = sh.cell(row_num, index['Coding']).value
            # eval coding (value table)
            value_table = dict()
            coding = coding.replace('\n', ':').split(':')

            for i in range(len(coding)):
                if coding[i].isdigit():
                    value_table[coding[i]] = coding[i + 1]
                else:
                    pass

            # # message name
            # frame_name = sh.cell(row_num, index['MessageName']).value
            #
            # signal_group = sh.cell(row_num, index['Signal Group']).value
            # signal_group_find = False
            # # Frame Name = 0 , Signal Group Name = 1 , Signal Group ID = 2 , Signals = 3
            # if signal_group != '/':
            #     for each in SignalGroupTemp:
            #         if each[1] == signal_group:
            #             signal_group_find = True
            #             if len(each[3]) == 0:
            #                 each[3] = []
            #                 each[3].append(signal_name)
            #             else:
            #                 each[3].append(signal_name)
            #     if signal_group_find == False:
            #         SignalGroupTemp.append([frame_name,signal_group,1,[signal_name]])

            # comment
            signal_comment = sh.cell(row_num, index['Comment']).value.strip()

            # signal_multiplexer = sh.cell(row_num, index['Multiplexer']).value

            signal_multiplexer = '/'
            if signal_multiplexer == '/':

                # create Canmatrix signal and add to db
                new_signal = canmatrix.Signal(
                    name=signal_name,
                    msb=msb,
                    lsb=lsb,
                    size=int(signal_length),
                    cycle_time=cycle_time,
                    is_little_endian=is_little_endian,
                    is_signed=is_signed,
                    offset=offset,
                    factor=factor,
                    receivers=receiver,
                    unit=unit,
                    min=pminimum,
                    max=pmaximum,
                    values=value_table)

                new_signal.add_comment(signal_comment)

                print("def : format - xls - load - SIGNAL - {} - {}".format(new_signal.name,new_signal.values))
                db.add_signal(new_signal)

            else:

                if signal_multiplexer != "Multiplexor":
                    # create Canmatrix signal and add to db
                    new_signal = canmatrix.Signal(
                        name=signal_name,
                        msb=msb,
                        lsb=lsb,
                        size=int(signal_length),
                        cycle_time=cycle_time,
                        is_little_endian=is_little_endian,
                        is_signed=is_signed,
                        offset=offset,
                        factor=factor,
                        receivers=receiver,
                        unit=unit,
                        min=pminimum,
                        max=pmaximum,
                        multiplex=int(signal_multiplexer),
                        values=value_table)

                    new_signal.is_multiplexer = True
                    new_signal.add_comment(signal_comment)

                    print("def : format - xls - load - SIGNAL - {} - {}".format(new_signal.name, new_signal.values))
                    db.add_signal(new_signal)
                else:
                    # create Canmatrix signal and add to db
                    new_signal = canmatrix.Signal(
                        name=signal_name,
                        msb=msb,
                        lsb=lsb,
                        size=int(signal_length),
                        cycle_time=cycle_time,
                        is_little_endian=is_little_endian,
                        is_signed=is_signed,
                        offset=offset,
                        factor=factor,
                        receivers=receiver,
                        unit=unit,
                        min=pminimum,
                        max=pmaximum,
                        values=value_table)

                    new_signal.is_multiplexer = True
                    new_signal.multiplex = 'Multiplexor'
                    new_signal.add_comment(signal_comment)

                    print("def : format - xls - load - SIGNAL - {} - {}".format(new_signal.name, new_signal.values))
                    db.add_signal(new_signal)

            # Frame information for new signal

            # message name
            frame_name = sh.cell(row_num, index['MessageName']).value

            signal_group = sh.cell(row_num, index['Signal Group']).value
            signal_group_find = False
            # Frame Name = 0 , Signal Group Name = 1 , Signal Group ID = 2 , Signals = 3
            if signal_group != '/':
                for each in SignalGroupTemp:
                    if each[1] == signal_group:
                        signal_group_find = True
                        if len(each[3]) == 0:
                            each[3] = []
                            each[3].append(new_signal)
                        else:
                            each[3].append(new_signal)
                if signal_group_find == False:
                    SignalGroupTemp.append([frame_name,signal_group,1,[new_signal]])

            # message id
            frame_id = sh.cell(row_num, index['MessageID']).value

            # period (follow signal period)
            cycle_time = int(sh.cell(row_num, index['Period']).value)
            # dlc
            dlc = int(sh.cell(row_num, index['DLC']).value)

            # transmitter
            transmitter = [sh.cell(row_num, index['Transmitter']).value]

            # create Canmatrix frame
            new_frame: object = canmatrix.Frame(frame_name, size=dlc)

            new_frame.cycle_time = cycle_time

            # eval transmitter
            new_frame.transmitters = transmitter

            # eval message ID
            if frame_id.endswith("xh"):
                new_frame.arbitration_id = canmatrix.ArbitrationId(int(frame_id[:-2], 16), extended=True)
            else:
                new_frame.arbitration_id = canmatrix.ArbitrationId(int(frame_id[2:], 16), extended=False)

            if sh.cell(row_num, index['MessageType']).value == "EXT":
                #frame.arbitration_id.id = (int(temp.group(1)) - 0x80000000)
                new_frame.arbitration_id.extended = True

            # add frame information to signal
            new_signal.frames = new_frame
            FrameTemp.append(new_frame)

            if signal_name != '/':
                FrameWithSignal.append(new_frame)

    # read rows values loop end

    # add vframeformat
    for signal in db.signals:
        signal.frames.set_fd_type()

    SkipUsedFrame = []
    for each in  SignalGroupTemp:
        #print("def : format - xls - load - SIGNAL GROUP - START - {}".format(each[0]))
        #print("def : format - xls - load - SIGNAL GROUP - START - ",db.signals)
        for each_frame in FrameTemp:
            #print("def : format - xls - load - SIGNAL GROUP - SCAN FRAME - {} - {}".format(each[0],each_frame.name))
            if each_frame.name == each[0] and each_frame.name not in SkipUsedFrame:
                SkipUsedFrame.append(each_frame.name)
                print("def : format - xls - load - SIGNAL GROUP - SCAN FRAME - {} - {}".format(each[1], each[0]))
                each_frame.add_signal_group(each[1],1,each[3])
                print("def : format - xls - load - SIGNAL GROUP - ", each[3])

            #print("def : format - xls - load - SIGNAL GROUP - {}".format(each_frame))
            #each_frame.add_signal_group("A",1,["Hello"])

    #WRITE DEF
    # sh_def = wb.sheet_by_index(1)
    #
    # # eval search for correct columns:
    # index = {}
    # for i in range(sh_def.ncols):
    #     value = sh_def.cell(0, i).value
    #     if value == "DEF TYPE":
    #         index['DEFTYPE'] = i
    #     elif "VALUE" in value:
    #         index['VALUE'] = i
    #     elif "TYPE" in value:
    #         index['TYPE'] = i
    #     elif "MIN" in value:
    #         index['MIN'] = i
    #     elif "MAX" in value:
    #         index['MAX'] = i
    #     elif "DEFAULT" in value:
    #         index['DEFAULT'] = i
    #
    # for row_num in range(1, sh_def.nrows):
    #
    #     #print("DEF ROW : {} - {} - {}".format(sh_def.cell(row_num, index['DEFTYPE']).value,sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['TYPE']).value))
    #
    #     if sh_def.cell(row_num, index['DEFTYPE']).value == "GLO DEF":
    #         #print("DEF ROW - G : {} {} ".format(sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['TYPE']).value))
    #
    #         if sh_def.cell(row_num, index['TYPE']).value == "HEX":
    #
    #             #print("DEF ROW - G : {} {} {}".format(sh_def.cell(row_num, index['VALUE']).value,int(sh_def.cell(row_num, index['MIN']).value),int(sh_def.cell(row_num, index['MAX']).value)))
    #             db.add_global_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {} {}".format(sh_def.cell(row_num, index['TYPE']).value,str(int(sh_def.cell(row_num, index['MIN']).value)),str(int(sh_def.cell(row_num, index['MAX']).value))))
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "INT":
    #             #print("DEF ROW - G - INT : {} - {}".format(sh_def.cell(row_num, index['VALUE']).value,int(sh_def.cell(row_num, index['DEFAULT']).value)))
    #             db.add_global_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {} {}".format(sh_def.cell(row_num, index['TYPE']).value,str(int(sh_def.cell(row_num, index['MIN']).value)),str(int(sh_def.cell(row_num, index['MAX']).value))))
    #         elif sh_def.cell(row_num, index['TYPE']).value == "STRING":
    #             print("DEF ROW - G - STRING : {} - {}".format(sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['DEFAULT']).value))
    #             db.add_global_defines(sh_def.cell(row_num, index['VALUE']).value, sh_def.cell(row_num, index['TYPE']).value)
    #
    #         if sh_def.cell(row_num, index['DEFAULT']).value != '/':
    #             print("DEF ROW - TRACE : {} - {} ".format(sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['DEFAULT']).value))
    #             #db.add_define_default(sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['DEFAULT']).value)
    #         else:
    #             db.add_define_default(sh_def.cell(row_num, index['VALUE']).value,"")
    #
    #
    #     elif sh_def.cell(row_num, index['DEFTYPE']).value == "ECU DEF":
    #
    #         if sh_def.cell(row_num, index['TYPE']).value == "INT":
    #             db.add_ecu_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {} {}".format(sh_def.cell(row_num, index['TYPE']).value,int(sh_def.cell(row_num, index['MIN']).value),int(sh_def.cell(row_num, index['MAX']).value)))
    #             # if sh_def.cell(row_num, index['DEFAULT']).value != '/':
    #             #     db.add_define_default(sh_def.cell(row_num, index['VALUE']).value,"0")
    #             # else:
    #             #     db.add_define_default(sh_def.cell(row_num, index['VALUE']).value,"")
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "HEX":
    #
    #             db.add_ecu_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {} {}".format(sh_def.cell(row_num, index['TYPE']).value,int(sh_def.cell(row_num, index['MIN']).value),int(sh_def.cell(row_num, index['MAX']).value)))
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "ENUM":
    #
    #             db.add_ecu_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {}".format(sh_def.cell(row_num, index['TYPE']).value,sh_def.cell(row_num, index['MIN']).value))
    #
    #         else:
    #             db.add_ecu_defines(sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['TYPE']).value)
    #
    #
    #         if sh_def.cell(row_num, index['DEFAULT']).value != '/':
    #
    #             #print("DEBUG : {}".format(sh_def.cell(row_num, index['DEFAULT']).value))
    #             db.add_define_default(sh_def.cell(row_num, index['VALUE']).value,str(int(sh_def.cell(row_num, index['DEFAULT']).value)))
    #         else:
    #             db.add_define_default(sh_def.cell(row_num, index['VALUE']).value,"")
    #
    #     elif sh_def.cell(row_num, index['DEFTYPE']).value == "ENV DEF":
    #
    #         if sh_def.cell(row_num, index['TYPE']).value == "ENUM":
    #
    #             db.add_env_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {}".format(sh_def.cell(row_num, index['TYPE']).value,sh_def.cell(row_num, index['MIN']).value))
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "INT":
    #
    #             db.add_env_defines(sh_def.cell(row_num, index['VALUE']).value,
    #                                "{} {} {}".format(sh_def.cell(row_num, index['TYPE']).value,
    #                                               int(sh_def.cell(row_num, index['MIN']).value),
    #                                                  int(sh_def.cell(row_num, index['MAX']).value)))
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "STRING":
    #
    #             db.add_env_defines(sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['TYPE']).value)
    #
    #     elif sh_def.cell(row_num, index['DEFTYPE']).value == "FRM DEF":
    #
    #         if sh_def.cell(row_num, index['TYPE']).value == "INT":
    #
    #             db.add_frame_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {} {}".format(sh_def.cell(row_num, index['TYPE']).value,int(sh_def.cell(row_num, index['MIN']).value),int(sh_def.cell(row_num, index['MAX']).value)))
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "HEX":
    #             pass
    #         elif sh_def.cell(row_num, index['TYPE']).value == "STRING":
    #
    #             db.add_frame_defines(sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['TYPE']).value)
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "FLOAT":
    #             db.add_frame_defines(sh_def.cell(row_num, index['VALUE']).value,
    #                                  "{} {} {}".format(sh_def.cell(row_num, index['TYPE']).value,
    #                                                    sh_def.cell(row_num, index['MIN']).value,
    #                                                    sh_def.cell(row_num, index['MAX']).value))
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "ENUM":
    #
    #             db.add_frame_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {}".format(sh_def.cell(row_num, index['TYPE']).value,sh_def.cell(row_num, index['MIN']).value))
    #
    #     elif sh_def.cell(row_num, index['DEFTYPE']).value == "SIG DEF":
    #
    #         if sh_def.cell(row_num, index['TYPE']).value == "INT":
    #
    #             db.add_signal_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {} {}".format(sh_def.cell(row_num, index['TYPE']).value,int(sh_def.cell(row_num, index['MIN']).value),int(sh_def.cell(row_num, index['MAX']).value)))
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "HEX":
    #             pass
    #         elif sh_def.cell(row_num, index['TYPE']).value == "STRING":
    #
    #             db.add_signal_defines(sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['TYPE']).value)
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "FLOAT":
    #             db.add_signal_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {} {}".format(sh_def.cell(row_num, index['TYPE']).value,int(sh_def.cell(row_num, index['MIN']).value),int(sh_def.cell(row_num, index['MAX']).value)))
    #
    #         elif sh_def.cell(row_num, index['TYPE']).value == "ENUM":
    #             #print("DEBUG : SIG STRING ENUM : {}".format(sh_def.cell(row_num, index['MIN']).value))
    #             db.add_signal_defines(sh_def.cell(row_num, index['VALUE']).value,"{} {}".format(sh_def.cell(row_num, index['TYPE']).value,sh_def.cell(row_num, index['MIN']).value))
    #
    # #WRITE BA_DEF_DEF_
    #
    # for row_num in range(1, sh_def.nrows):
    #     if sh_def.cell(row_num, index['DEFTYPE']).value == "BA_DEF_DEF_":
    #         print("def : format - xls - load - BA_DEF_DEF_ : {} - {} ".format(sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['DEFAULT']).value))
    #         if sh_def.cell(row_num, index['DEFAULT']).value == "/":
    #             #print("def : format - dbc - load - BA_DEF_DEF_ : {} - {} ".format(sh_def.cell(row_num, index['VALUE']).value,''))
    #             db.add_define_default(sh_def.cell(row_num, index['VALUE']).value,"")
    #         else:
    #             db.add_define_default(sh_def.cell(row_num, index['VALUE']).value,sh_def.cell(row_num, index['DEFAULT']).value)
    #
    # #WRITE ATTR
    # sh_attr = wb.sheet_by_index(2)
    #
    # # eval search for correct columns:
    # index = {}
    # for i in range(sh_attr.ncols):
    #     value = sh_attr.cell(0, i).value
    #     if value == "P1":
    #         index['P1'] = i
    #     elif "P2" in value:
    #         index['P2'] = i
    #     elif "P3" in value:
    #         index['P3'] = i
    #     elif "P4" in value:
    #         index['P4'] = i
    #     elif "P5" in value:
    #         index['P5'] = i
    #
    # for row_num in range(1, sh_attr.nrows):
    #
    #     if sh_attr.cell(row_num, index['P2']).value == "BU_":
    #         db.ecu_by_name(sh_attr.cell(row_num, index['P3']).value).add_attribute(
    #             sh_attr.cell(row_num, index['P1']).value,
    #             sh_attr.cell(row_num, index['P4']).value)
    #
    #     elif sh_attr.cell(row_num, index['P2']).value == "BO_":
    #
    #         #for each in FrameTemp:
    #         #for each in FrameWithSignal:
    #         for each in FrameTemp:
    #             # print("DEBUG : ADD FRAME ATTR - {} - {}".format(each.arbitration_id.id,
    #             #
    #             #
    #             #                                                sh_attr.cell(row_num, index['P3']).value))
    #             if each.arbitration_id.extended == False:
    #                 if str(each.arbitration_id.id) == sh_attr.cell(row_num, index['P3']).value:
    #                     # if sh_attr.cell(row_num, index['P1']).value != "GenMsgILSupport" or sh_attr.cell(row_num, index['P1']).value != "Diag_Response":
    #
    #                     print("DEBUG : ADD STD FRAME ATTR - {} - {} - {}".format(each.name,sh_attr.cell(row_num, index['P1']).value,sh_attr.cell(row_num, index['P4']).value))
    #                     if sh_attr.cell(row_num, index['P4']).value == "/":
    #                         each.add_attribute(sh_attr.cell(row_num, index['P1']).value,"")
    #                     else:
    #                         each.add_attribute(sh_attr.cell(row_num, index['P1']).value,sh_attr.cell(row_num, index['P4']).value)
    #             else:
    #                 if str(each.arbitration_id.id + 0x80000000) == sh_attr.cell(row_num, index['P3']).value:
    #                     # if sh_attr.cell(row_num, index['P1']).value != "GenMsgILSupport" or sh_attr.cell(row_num, index['P1']).value != "Diag_Response":
    #
    #                     print("DEBUG : ADD EXT FRAME ATTR - {} - {} - {}".format(each.name,
    #                                                                          sh_attr.cell(row_num, index['P1']).value,
    #                                                                          sh_attr.cell(row_num, index['P4']).value))
    #                     if sh_attr.cell(row_num, index['P4']).value == "/":
    #                         each.add_attribute(sh_attr.cell(row_num, index['P1']).value, "")
    #                     else:
    #                         each.add_attribute(sh_attr.cell(row_num, index['P1']).value,
    #                                            sh_attr.cell(row_num, index['P4']).value)
    #
    #         # get_frame_by_id(canmatrix.ArbitrationId.from_compound_integer(sh_attr.cell(row_num, index['P3']).value)).add_attribute(
    #         #     sh_attr.cell(row_num, index['P1']).value,
    #         #     sh_attr.cell(row_num, index['P4']).value)
    #
    #     elif sh_attr.cell(row_num, index['P2']).value == "SG_":
    #         FrameIdandSignal = (sh_attr.cell(row_num, index['P3']).value).split(" ")
    #         print("DEBUG : ADD SIGNAL ATTR - BA_ SG_ - {} - {}".format(FrameIdandSignal[0],FrameIdandSignal[1]))
    #         for signal in db.signals:
    #             if signal.name == FrameIdandSignal[1] and signal.frames.arbitration_id.id == int(FrameIdandSignal[0]):
    #                 print("DEBUG : ADD SIGNAL ATTR - BA_ SG_ - {} ; {}".format(sh_attr.cell(row_num, index['P1']).value,sh_attr.cell(row_num, index['P4']).value))
    #                 signal.add_attribute(sh_attr.cell(row_num, index['P1']).value,sh_attr.cell(row_num, index['P4']).value)
    #
    #     elif sh_attr.cell(row_num, index['P2']).value == "GLO":
    #         db.add_attribute(sh_attr.cell(row_num, index['P1']).value, sh_attr.cell(row_num, index['P4']).value)
    #
    #     elif sh_attr.cell(row_num, index['P2']).value == "SIG_VALTYPE_":
    #         for signal in db.signals:
    #             if signal.name == sh_attr.cell(row_num, index['P4']).value and signal.frames.arbitration_id.id == int(sh_attr.cell(row_num, index['P3']).value):
    #                 signal.is_float = True

    # ecu-attributes:
    # for ecu in db.ecus:
    #     for attrib, val in sorted(ecu.attributes.items()):
    #
    #         print("def : format - dbc - dump - ECU ATTRIBUTE : [{} , {}]".format(attrib, val))

            # f.write(
            #     create_attribute_string(attrib, "BU_", ecu.name, val, db.ecu_defines[attrib].type == "STRING").encode(
            #         dbc_export_encoding, ignore_encoding_errors))

    # # global-attributes:
    # for attrib, val in sorted(db.attributes.items()):
    #
    #     print("def : format - dbc - dump - GLOBAL ATTRIBUTE : [{} , {}]".format(attrib, val))
    #
    #     f.write(create_attribute_string(attrib, "", "", val, db.global_defines[attrib].type == "STRING").encode(
    #         dbc_export_encoding, ignore_encoding_errors))
    #
    # # messages-attributes:
    # curr_frame = None
    # for signal in db.signals:
    #     frame = signal.frames
    #     if frame.name != curr_frame:
    #         for attrib, val in sorted(frame.attributes.items()):
    #
    #             print("def : format - dbc - dump - MESSAGE ATTRIBUTE : [{} , {}]".format(attrib,val))
    #
    #             f.write(create_attribute_string(attrib, "BO_", str(frame.arbitration_id.to_compound_integer()), val,
    #                                             db.frame_defines[attrib].type == "STRING").encode(dbc_export_encoding,
    #                                                                                ignore_encoding_errors))
    #         curr_frame = frame.name
    #
    # # signal-attributes:
    # for signal in db.signals:
    #     frame = signal.frames
    #     for attrib, val in sorted(signal.attributes.items()):
    #         name = output_names[frame][signal]
    #         if isinstance(val, float):
    #             val = format_float(val)
    #         if attrib in db.signal_defines:
    #
    #             print("def : format - dbc - dump - SIGNAL ATTRIBUTE : {}".format(attrib))
    #
    #             f.write(create_attribute_string(
    #                 attrib, "SG_", '%d ' % frame.arbitration_id.to_compound_integer() + name, val,
    #                                db.signal_defines[attrib].type == "STRING").encode(dbc_export_encoding,
    #                                                                                ignore_encoding_errors))
    #
    # # environment-attributes:
    # for env_var_name, env_var in db.env_vars.items():
    #     if "attributes" in env_var:
    #         for attribute, value in env_var["attributes"].items():
    #
    #             print("def : format - dbc - dump - ENVIRONMENT ATTRIBUTE : [{} , {}]".format(attrib, val))
    #
    #             f.write(create_attribute_string(attribute, "EV_", "", value,
    #                                             db.env_defines[attribute].type == "STRING")
    #                     .encode(dbc_export_encoding, ignore_encoding_errors))

    return db


# ========================== Third Party libs ========================
from typing import List
import torch
from transformers import LogitsProcessor
import scipy.stats
from math import sqrt
from itertools import chain, tee
import collections
# ============================ My packages ============================
from .hash_functions import prf_lookup, seeding_scheme_lookup

class WatermarkBase:
    def __init__(
            self,
            vocab: List[int] = None,
            gamma: float = 0.5,
            delta: float = 2.0,
            seeding_scheme: str = "simple_1",
            # simple default, find more schemes in alternative_prf_schemes.py
            select_green_tokens: bool = True,
            # should always be the default if not running in legacy mode
    ):
        # patch now that None could now maybe be passed as seeding_scheme
        if seeding_scheme is None:
            seeding_scheme = "simple_1"

        self.seeding_scheme = seeding_scheme
        # Vocabulary setup
        self.vocab = vocab
        self.vocab_size = len(vocab)

        # Watermark behavior:
        self.gamma = gamma
        self.delta = delta
        self.rng = None
        self._initialize_seeding_scheme(seeding_scheme)
        # Legacy behavior:
        self.select_green_tokens = select_green_tokens

    def _initialize_seeding_scheme(self, seeding_scheme: str) -> None:
        """Initialize all internal settings of the seeding strategy from a colloquial,
        "public" name for the scheme."""
        self.prf_type, self.context_width, self.self_salt, self.hash_key = seeding_scheme_lookup(
            seeding_scheme)

    def _seed_rng(self, input_ids: torch.LongTensor) -> None:
        """Seed RNG from local context. Not batched, because the generators we use
        (like cuda.random) are not batched."""
        # Need to have enough context for seed generation
        if input_ids.shape[-1] < self.context_width:
            raise ValueError(
                f"seeding_scheme requires at least a {self.context_width} token "
                f"prefix to seed the RNG."
            )

        prf_key = prf_lookup[self.prf_type](
            input_ids[-self.context_width:], salt_key=self.hash_key
        )
        # enable for long, interesting streams of pseudorandom numbers: print(prf_key)
        self.rng.manual_seed(prf_key % (2 ** 64 - 1))  # safeguard against overflow from long

    def _get_greenlist_ids(self, input_ids: torch.LongTensor) -> torch.LongTensor:
        """Seed rng based on local context width and use this information to
        generate ids on the green list."""
        self._seed_rng(input_ids)

        greenlist_size = int(self.vocab_size * self.gamma)
        vocab_permutation = torch.randperm(
            self.vocab_size, device=input_ids.device, generator=self.rng
        )
        if self.select_green_tokens:  # directly
            greenlist_ids = vocab_permutation[:greenlist_size]  # new
        else:  # select green via red
            greenlist_ids = vocab_permutation[
                            (self.vocab_size - greenlist_size):
                            ]  # legacy behavior
        return greenlist_ids

class WatermarkLogitsProcessor(WatermarkBase, LogitsProcessor):
    """LogitsProcessor modifying model output scores in a pipe. Can be used in any HF pipeline
    to modify scores to fit the watermark, but can also be used as a standalone tool inserted
    for any model producing scores in between model outputs and next token sampler.
    """

    def __init__(self, *args, store_spike_ents: bool = False, **kwargs):
        super().__init__(*args, **kwargs)

        self.store_spike_ents = store_spike_ents
        self.spike_entropies = None
        if self.store_spike_ents:
            self._init_spike_entropies()

    def _init_spike_entropies(self):
        alpha = torch.exp(torch.tensor(self.delta)).item()
        gamma = self.gamma

        self.z_value = ((1 - gamma) * (alpha - 1)) / (1 - gamma + (alpha * gamma))
        self.expected_gl_coef = (gamma * alpha) / (1 - gamma + (alpha * gamma))

        # catch for overflow when bias is "infinite"
        if alpha == torch.inf:
            self.z_value = 1.0
            self.expected_gl_coef = 1.0

    def _get_spike_entropies(self):
        spike_ents = [[] for _ in range(len(self.spike_entropies))]
        for b_idx, ent_tensor_list in enumerate(self.spike_entropies):
            for ent_tensor in ent_tensor_list:
                spike_ents[b_idx].append(ent_tensor.item())
        return spike_ents

    def _get_and_clear_stored_spike_ents(self):
        spike_ents = self._get_spike_entropies()
        self.spike_entropies = None
        return spike_ents

    def _compute_spike_entropy(self, scores):
        # precomputed z value in init
        probs = scores.softmax(dim=-1)
        denoms = 1 + (self.z_value * probs)
        renormed_probs = probs / denoms
        sum_renormed_probs = renormed_probs.sum()
        return sum_renormed_probs

    def _calc_greenlist_mask(
            self, scores: torch.FloatTensor, greenlist_token_ids
    ) -> torch.BoolTensor:
        # Cannot lose loop, greenlists might have different lengths
        green_tokens_mask = torch.zeros_like(scores, dtype=torch.bool)
        for b_idx, greenlist in enumerate(greenlist_token_ids):
            if len(greenlist) > 0:
                green_tokens_mask[b_idx][greenlist] = True
        return green_tokens_mask

    def _bias_greenlist_logits(
            self, scores: torch.Tensor, greenlist_mask: torch.Tensor, greenlist_bias: float
    ) -> torch.Tensor:
        scores[greenlist_mask] = scores[greenlist_mask] + greenlist_bias
        return scores

    def _score_rejection_sampling(
            self, input_ids: torch.LongTensor, scores: torch.FloatTensor, tail_rule="fixed_compute"
    ) -> List[int]:
        """Generate greenlist based on current candidate next token. Reject and move on if
        necessary. Method not batched.
        This is only a partial version of Alg.3 "Robust Private Watermarking", as it always
        assumes greedy sampling. It will still (kinda)
        work for all types of sampling, but less effectively.
        To work efficiently, this function can switch between a number of rules for handling the
        distribution tail.
        These are not exposed by default.
        """
        sorted_scores, greedy_predictions = scores.sort(dim=-1, descending=True)

        final_greenlist = []
        for idx, prediction_candidate in enumerate(greedy_predictions):
            greenlist_ids = self._get_greenlist_ids(
                torch.cat([input_ids, prediction_candidate[None]], dim=0)
            )  # add candidate to prefix
            if prediction_candidate in greenlist_ids:  # test for consistency
                final_greenlist.append(prediction_candidate)

            # What follows below are optional early-stopping rules for efficiency
            if tail_rule == "fixed_score":
                if sorted_scores[0] - sorted_scores[idx + 1] > self.delta:
                    break
            elif tail_rule == "fixed_list_length":
                if len(final_greenlist) == 10:
                    break
            elif tail_rule == "fixed_compute":
                if idx == 40:
                    break
            else:
                pass  # do not break early
        return torch.as_tensor(final_greenlist, device=input_ids.device)

    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
        """Call with previous context as input_ids, and scores for next token."""

        # this is lazy to allow us to co-locate on the watermarked model's device
        self.rng = torch.Generator(device=input_ids.device) if self.rng is None else self.rng

        # NOTE, it would be nice to get rid of this batch loop, but currently,
        # the seed and partition operations are not tensor/vectorized, thus
        # each sequence in the batch needs to be treated separately.

        list_of_greenlist_ids = [None for _ in input_ids]  # Greenlists could differ in length
        for b_idx, input_seq in enumerate(input_ids):
            if self.self_salt:
                greenlist_ids = self._score_rejection_sampling(input_seq, scores[b_idx])
            else:
                greenlist_ids = self._get_greenlist_ids(input_seq)
            list_of_greenlist_ids[b_idx] = greenlist_ids

            # logic for computing and storing spike entropies for analysis
            if self.store_spike_ents:
                if self.spike_entropies is None:
                    self.spike_entropies = [[] for _ in range(input_ids.shape[0])]
                self.spike_entropies[b_idx].append(self._compute_spike_entropy(scores[b_idx]))

        green_tokens_mask = self._calc_greenlist_mask(
            scores=scores, greenlist_token_ids=list_of_greenlist_ids
        )
        scores = self._bias_greenlist_logits(
            scores=scores, greenlist_mask=green_tokens_mask, greenlist_bias=self.delta
        )

        return scores

class WatermarkDetector(WatermarkBase):
    def __init__(
            self,
            *args,
            device: torch.device = None,
            tokenizer=None,
            z_threshold: float = 4.0,
            ignore_repeated_bigrams: bool = False,
            **kwargs,
    ):
        super().__init__(*args, **kwargs)
        # also configure the metrics returned/preprocessing options
        assert device, "Must pass device"
        assert tokenizer, "Need an instance of the generating tokenizer to perform detection"

        self.tokenizer = tokenizer
        self.device = device
        self.z_threshold = z_threshold
        self.rng = torch.Generator(device=self.device)

        if self.seeding_scheme == "simple_1":
            self.min_prefix_len = 1
        else:
            raise NotImplementedError(f"Unexpected seeding_scheme: {self.seeding_scheme}")

        self.ignore_repeated_bigrams = ignore_repeated_bigrams
        if self.ignore_repeated_bigrams:
            assert self.seeding_scheme == "simple_1", "No repeated bigram credit variant assumes " \
                                                      "the single token seeding scheme."

    def _compute_z_score(self, observed_count, T):
        # count refers to number of green tokens, T is total number of tokens
        expected_count = self.gamma
        numer = observed_count - expected_count * T
        denom = sqrt(T * expected_count * (1 - expected_count))
        z = numer / denom
        return z

    def _compute_p_value(self, z):
        p_value = scipy.stats.norm.sf(z)
        return p_value

    def _score_sequence(
            self,
            input_ids: torch.Tensor,
            return_num_tokens_scored: bool = True,
            return_num_green_tokens: bool = True,
            return_green_fraction: bool = True,
            return_green_token_mask: bool = False,
            return_z_score: bool = True,
            return_p_value: bool = True,
    ):
        if self.ignore_repeated_bigrams:
            # Method that only counts a green/red hit once per unique bigram.
            # New num total tokens scored (T) becomes the number unique bigrams.
            # We iterate over all unqiue token bigrams in the input, computing the greenlist
            # induced by the first token in each, and then checking whether the second
            # token falls in that greenlist.
            assert return_green_token_mask == False, "Can't return the green/red mask when " \
                                                     "ignoring repeats."
            bigram_table = {}
            token_bigram_generator = ngrams(input_ids.cpu().tolist(), 2)
            freq = collections.Counter(token_bigram_generator)
            num_tokens_scored = len(freq.keys())
            for idx, bigram in enumerate(freq.keys()):
                prefix = torch.tensor([bigram[0]],
                                      device=self.device)  # expects a 1-d prefix tensor on
                # the randperm device
                greenlist_ids = self._get_greenlist_ids(prefix)
                bigram_table[bigram] = True if bigram[1] in greenlist_ids else False
            green_token_count = sum(bigram_table.values())
        else:
            num_tokens_scored = len(input_ids) - self.min_prefix_len
            if num_tokens_scored < 1:
                raise ValueError((f"Must have at least {1} token to score after "
                                  f"the first min_prefix_len={self.min_prefix_len} tokens required "
                                  f"by the seeding scheme."))
            # Standard method.
            # Since we generally need at least 1 token (for the simplest scheme)
            # we start the iteration over the token sequence with a minimum
            # num tokens as the first prefix for the seeding scheme,
            # and at each step, compute the greenlist induced by the
            # current prefix and check if the current token falls in the greenlist.
            green_token_count, green_token_mask = 0, []
            for idx in range(self.min_prefix_len, len(input_ids)):
                curr_token = input_ids[idx]
                greenlist_ids = self._get_greenlist_ids(input_ids[:idx])
                if curr_token in greenlist_ids:
                    green_token_count += 1
                    green_token_mask.append(True)
                else:
                    green_token_mask.append(False)

        score_dict = dict()
        if return_num_tokens_scored:
            score_dict.update(dict(num_tokens_scored=num_tokens_scored))
        if return_num_green_tokens:
            score_dict.update(dict(num_green_tokens=green_token_count))
        if return_green_fraction:
            score_dict.update(dict(green_fraction=(green_token_count / num_tokens_scored)))
        if return_z_score:
            score_dict.update(
                dict(z_score=self._compute_z_score(green_token_count, num_tokens_scored)))
        if return_p_value:
            z_score = score_dict.get("z_score")
            if z_score is None:
                z_score = self._compute_z_score(green_token_count, num_tokens_scored)
            score_dict.update(dict(p_value=self._compute_p_value(z_score)))
        if return_green_token_mask:
            score_dict.update(dict(green_token_mask=green_token_mask))

        return score_dict

    def detect(
            self,
            text: str = None,
            tokenized_text: List[int] = None,
            return_prediction: bool = True,
            return_scores: bool = True,
            z_threshold: float = None,
            **kwargs,
    ) -> dict:

        assert (text is not None) ^ (
                tokenized_text is not None), "Must pass either the raw or tokenized string"
        if return_prediction:
            kwargs[
                "return_p_value"] = True  # to return the "confidence":=1-p of positive detections

        if tokenized_text is None:
            assert self.tokenizer is not None, (
                "Watermark detection on raw string ",
                "requires an instance of the tokenizer ",
                "that was used at generation time.",
            )
            tokenized_text = \
                self.tokenizer(text, return_tensors="pt", add_special_tokens=False)["input_ids"][
                    0].to(
                    self.device)
            if tokenized_text[0] == self.tokenizer.bos_token_id:
                tokenized_text = tokenized_text[1:]
        else:
            # try to remove the bos_tok at beginning if it's there
            if (self.tokenizer is not None) and (tokenized_text[0] == self.tokenizer.bos_token_id):
                tokenized_text = tokenized_text[1:]

        # call score method
        output_dict = {}
        score_dict = self._score_sequence(tokenized_text, **kwargs)
        if return_scores:
            output_dict.update(score_dict)
        # if passed return_prediction then perform the hypothesis test and return the outcome
        if return_prediction:
            z_threshold = z_threshold if z_threshold else self.z_threshold
            assert z_threshold is not None, "Need a threshold in order to decide outcome of detection test"
            output_dict["prediction"] = score_dict["z_score"] > z_threshold
            if output_dict["prediction"]:
                output_dict["confidence"] = 1 - score_dict["p_value"]

        return output_dict

def ngrams(sequence, n, pad_left=False, pad_right=False, pad_symbol=None):
    sequence = iter(sequence)
    if pad_left:
        sequence = chain((pad_symbol,) * (n - 1), sequence)
    if pad_right:
        sequence = chain(sequence, (pad_symbol,) * (n - 1))
    iterables = tee(sequence, n)

    for i, sub_iterable in enumerate(iterables):  # For each window,
        for _ in range(i):  # iterate through every order of ngrams
            next(sub_iterable, None)  # generate the ngrams within the window.
    return zip(*iterables)  # Unpack and flattens the iterables.

def load_detector(tokenizer, args):
    # device = "cuda" if (args.use_gpu and torch.cuda.is_available()) else "cpu"

    watermark_detector = WatermarkDetector(
        vocab=list(tokenizer.get_vocab().values()),
        gamma=args.gamma,
        seeding_scheme=args.seeding_scheme,
        device=args.device,
        tokenizer=tokenizer,
        z_threshold=args.detection_z_threshold)

    return watermark_detector


import os
import time
import sys
import traceback

import cv2
import torch
import timm
import numpy as np

from models.build_model import build_model
from collections import OrderedDict

def normalize(img, mean, std, max_pixel_value=255.0):
    mean = np.array(mean, dtype=np.float32)
    mean *= max_pixel_value

    std = np.array(std, dtype=np.float32)
    std *= max_pixel_value

    denominator = np.reciprocal(std, dtype=np.float32)

    img = img.astype(np.float32)
    img -= mean[:, None, None]
    img *= denominator[:, None, None]
    return img

def transforms_cv2(image, resize=(224, 224)):
    image = cv2.resize(image, resize, interpolation=cv2.INTER_LINEAR)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = image.transpose(2, 0, 1)
    image = normalize(image, (0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
    image = torch.from_numpy(image)
    image = torch.unsqueeze(image, 0)

    return image

def main(args):
    # only use single gpu or cpu
    device = torch.device('cuda:0') if args.device=='cuda' else torch.device(args.device)
    classes = torch.load(args.checkpoint, map_location=torch.device(device))['classes']
    if args.hub == 'tv':
        model = build_model(args.net, pretrained=False, fine_tune=False, num_classes=len(classes))
    elif args.hub == 'timm':
        #print(timm.list_models(pretrained=True))
        model = timm.create_model(args.net, pretrained=False, num_classes=len(classes))
    else:
        raise NameError('Model hub only support tv or timm')
    print('Loading trained model weightes...')
    model.load_state_dict({
        k.replace('module.', ''): v for k, v in
        torch.load(args.checkpoint, map_location=torch.device(device))['model_state_dict'].items()})

    model = model.to(device)

    model.eval()

    cnt = 0
    total_used_time = 0.0
    for image_name in os.listdir(args.test_path):
        try:
            image_ = args.test_path + '/' + image_name
            image = cv2.imread(image_)
            cnt += 1
            if cnt >= 5: # warmup for gpu
                bg_time = time.time()
            image_tensor = transforms_cv2(image, resize=(args.resize, args.resize))
            input = image_tensor.to(device)
            output = model(input)
            if cnt >= 5:
                total_used_time += (time.time() - bg_time)

            index = output.detach().cpu().numpy().argmax()
            #print(output.data.cpu().numpy())
            print('{}\tpredict: {}\t{}'.format(image_, classes[index], index))
            sys.stdout.flush()

        except:
            #print(image)
            traceback.print_exc()

    print('Total test num: {}, Avg used time: {}'.format(cnt, total_used_time/(cnt-4)))

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='PyTorch Classification Inference')

    parser.add_argument('--test-path', default='./data/beauty', help='dataset')
    parser.add_argument('--hub', default='tv', help='model hub, from torchvision(tv) or timm')
    parser.add_argument('--net', default='resnet50', help='model name')
    parser.add_argument('--device', default='cpu', choices=['cuda', 'cpu'], help='device, cpu or cuda')
    parser.add_argument('--checkpoint', default='./checkpoints/model.pth', help='checkpoint')
    parser.add_argument('--resize', default=224, type=int, help='size of resize')

    args = parser.parse_args()

    print(args)
    main(args)


# -*- coding: utf-8 -*-
from maya import cmds
import os
import json

def maya_export():
    selection = cmds.ls(sl=True)
    long_names = cmds.ls(sl=True, l=True)
    if selection:
        cmds.inViewMessage( amg='<hl>Go Maya</hl> : Export Selected Objects', pos='midCenterTop', fade=True, ta=0.75, a=0.5)
        temp = __name__.split('.')#nameは自分自身のモジュール名。splitでピリオドごとに3分割。
        folderPath = os.path.join(
            os.getenv('MAYA_APP_DIR'),#Mayaのディレクトリ環境変数を取得
            'Scripting_Files',
            temp[-1]
            )
        if not os.path.exists(folderPath):
            os.makedirs(os.path.dirname(folderPath+os.sep))  # 末尾\\が必要なので注意
        print(folderPath)
        files = os.listdir(folderPath)
        sel_dict = dict()
        if files is not None:
            for file in files:
                os.remove(folderPath + os.sep + file)
        for sel, long_name in zip(selection, long_names):
            cmds.select(sel, r=True)
            name = sel.replace('|', '__Pipe__')
            cmds.file(folderPath+os.sep+name+'.ma', force=True, options="v=0", typ="mayaAscii", pr=True, es=True)
            sel_dict[name+'.ma'] = long_name
        cmds.select(selection, r=True)
        #選択ノード名を保存
        fine_name = folderPath+os.sep +'go_maya_selection_node.json'
        with open(fine_name, 'w') as f:
            json.dump(sel_dict, f)

def maya_import():
    temp = __name__.split('.')#nameは自分自身のモジュール名。splitでピリオドごとに3分割。
    folderPath = os.path.join(os.getenv('MAYA_APP_DIR'),'Scripting_Files','go')
    if not os.path.exists(folderPath):
        os.makedirs(os.path.dirname(folderPath+os.sep))  # 末尾\\が必要なので注意
    #print(folderPath)
    files = os.listdir(folderPath)
    if files is not None:
        for file in files:
            print(file)
            nameSpace = file.replace('.ma', '')
            cmds.file(folderPath+os.sep+file, i=True, typ="mayaAscii", iv=True, mnc=False, options="v=0;", pr=True)
            #重複マテリアルにファイル名が頭に付与されてしまうのを修正
            allMat = cmds.ls(mat=True)
            fileName = file.split('.')[0]
            for mat in allMat:
                if mat.startswith(fileName+'_'):
                    cmds.rename(mat, mat.replace(fileName+'_', ''))
        cmds.inViewMessage( amg='<hl>Go Maya</hl> : Imoprt objects', pos='midCenterTop', fade=True, ta=0.75, a=0.5)
    else:
        cmds.inViewMessage( amg='<hl>Go Maya</hl> : There is no exported object', pos='midCenterTop', fade=True, ta=0.75, a=0.5)


import numpy as np
import h5py
from tqdm.auto import trange
import torch
from torch_geometric.utils import dense_to_sparse

def load_data(dpath):
    X, y = [],[]
    with h5py.File(dpath, "r") as handle:
        input = handle['input']
        Hs = input["channel_to_noise_matched"]
        Plin = 10**(np.asarray(input['PdB'][...], dtype=float)/10)
        xopt = handle['xopt'][...].astype(float)
        objval_opt = handle['objval'][...].astype(float) # or wsee

        ns, nu, _ = Hs.shape # eg:(1000,4,4)

        for hidx in range(ns):
            edge_index, h = dense_to_sparse(torch.from_numpy(Hs[hidx].astype(float)))

            x1 = np.hstack([(h.reshape((-1,1))*Plin).T, # -->(h1p1, h2p1, h3p1, ...)
                             Plin.reshape(-1,1)])
            X.append( x1 )
            y.append( xopt[hidx] )

        cinfo = {'mu': input['PA inefficency'][...].item(),
                 'Pc': input['Pc'][...].item(),
                 'edge_index': edge_index,
                 'objval_opt': objval_opt} # or wsee

    y = np.concatenate((y))
    X = np.concatenate((X))
    y = y[~np.any(np.isnan(X),-1)]
    X = X[~np.any(np.isnan(X),-1)]

    return X, y, cinfo

def load_data_unsup(dpath, **kwargs):
    # parameters; ref: https://github.com/bmatthiesen/deep-EE-opt/blob/062093fde6b3c6edbb8aa83462165265deefce1a/src/globalOpt/run_wsee.py#L30
    extract_args = lambda a, k: a if not k in kwargs else kwargs[k]
    PdB = extract_args(np.array(range(-40,10+1,1)), 'PdB')

    mu = extract_args(4.0, 'mu')
    Pc = extract_args(1.0, 'Pc')
    hxp = extract_args(True, 'hxp')
    num_stab = extract_args(0., 'num_stab')

    Plin = 10**(PdB/10)
    if hxp:
        Ph = Plin
    else:
        Ph = torch.empty(Plin.shape).fill_(1)

    X = []
    with h5py.File(dpath, "r") as handle:
        Hs = handle['input']["channel_to_noise_matched"]

        ns, nu, _ = Hs.shape # eg:(1000,4,4)

        for hidx in range(ns):
            edge_index, h = dense_to_sparse(torch.from_numpy(Hs[hidx].astype(float)))
            h += num_stab

            x1 = np.hstack([(h.reshape((-1,1))*Ph).T, # -->(h1p1, h2p1, h3p1, ...)
                             Plin.reshape(-1,1)])
            X.append( x1 )

        cinfo = {'mu': mu,
                 'Pc': Pc,
                 'edge_index': edge_index} # or wsee

    X = np.concatenate((X))
    X = X[~np.any(np.isnan(X),-1)]
    y = np.full([X.shape[0], nu], np.nan)

    return X, y, cinfo


import pickle
import platform
import subprocess

from django.conf import settings
from django.core.files import File

def get_system_config():
    sysconfig = {}

    os = platform.system()
    if os == "Linux":
        command = "cat /proc/cpuinfo"
        cpu_info = subprocess.check_output(command, shell=True).strip()

        if not type(cpu_info) == str:
            cpu_info = cpu_info.decode()

        clock_speed = cpu_info.split('@')[1].split(' ')[1].split('\n')[0]
        sysconfig['clock_speed'] = clock_speed

        cores = cpu_info.split('\n')[12].split(':')[1].strip()
        sysconfig['cpu'] = cores

        command = "cat /proc/meminfo"
        ram_info = subprocess.check_output(command, shell=True).strip()

        if not type(ram_info) == str:
            ram_info = ram_info.decode()

        ram_size = int(ram_info.split('\n')[0].split(':')[1].strip(
        ).split(' ')[0]) / 1024.0**2
        sysconfig['ram'] = ram_size

        sysconfig['architecture'] = platform.architecture()[0]

        return sysconfig

def save_encrypted_file(algorithm, filename, content):
    '''
    saves the encrypted file to database.
    '''

    filename = settings.MEDIA_ROOT + '/files/' + algorithm + "_" + filename
    with open(filename, 'wb+') as f:
        encrypted_file = File(f)

        if isinstance(content, list):
            pickle.dump(content, encrypted_file)
        else:
            encrypted_file.write(content)

    return filename


import json
from collections import defaultdict

import pandas as pd
from responsibly.dataset import COMPASDataset, GermanDataset
from sklearn.preprocessing import LabelEncoder

'''class DatasetType(Enum):
    """
    Enumeration of the Datasets under consideration.
    Values represent the sensitive attributes for each of the dataset types
    """
    COMPAS = ['age', 'sex']
    GERMAN_CREDIT = ['credit']
'''

class Dataset:
    """
    The Dataset Interface for all fairness analysis
    """

    def __init__(self, config_file) :
        f = open(config_file, 'r')
        config = json.loads(f.read())
        self.type = config['dataset']
        _dataset = COMPASDataset() if self.type == 'COMPAS' else GermanDataset()
        self._sensitive_attributes = config["sensitive_attributes"]
        self._non_numeric_attributes = config["non_numeric_attributes"]

        self._n_attributes_dict = config["numeric_attributes"]
        self._numeric_attributes = list(self._n_attributes_dict.keys()) if self._n_attributes_dict else []
        self._date_attributes_dict = config["date_attributes"]
        self._date_attributes = list(self._date_attributes_dict.keys()) if self._date_attributes_dict else []

        self._target = config['ground_truth']
        self._predictions = config['predictions']

        self._data_x_readable = pd.DataFrame(_dataset.df
                                             [
                                                 self._sensitive_attributes+self._non_numeric_attributes+self._numeric_attributes+self._date_attributes])
        self._data_y_readable = pd.DataFrame(_dataset.df[self._target])
        self._encoder_dict_x = defaultdict(LabelEncoder)
        self._encoder_dict_y = defaultdict(LabelEncoder)
        self._data_x, self._data_y = self._preprocess_data()

    def _preprocess_data(self) :
        """
        Private method to process data by converting all text features to numeric
        :return: Tuple containing encoded features and targets
        """

        if self.type == 'COMPAS' :
            self._data_x_readable.fillna('', inplace=True)

        self._data_x_readable = self._data_x_readable.loc[:, ~self._data_x_readable.columns.duplicated()]

        # write a date attribute function

        if self._numeric_attributes :
            for attribute in self._numeric_attributes :
                self._data_x_readable[attribute+'_bins'] = pd.cut(x=self._data_x_readable[attribute],
                                                                  bins=self._n_attributes_dict[attribute])
                self._data_x_readable = self._data_x_readable.drop(columns=[attribute])
                self._data_x_readable = self._data_x_readable.rename(columns={attribute+'_bins' : attribute})

        return self._data_x_readable.apply(lambda x : self._encoder_dict_x[x.name].fit_transform(x)), \
               self._data_y_readable.apply(lambda y : self._encoder_dict_y[y.name].fit_transform(y))

    def get_sensitive_attributes(self) :
        return self._sensitive_attributes

    def get_data(self, readable=False) :
        """
        Method to get the data
        :param readable: Boolean flag to set if the returned data is human-readable
        :return: Tuple of data features and targets
        """
        if readable :
            return self._data_x_readable, self._data_y_readable
        return self._data_x, self._data_y

    # this as sort of useless. The init function ensures readable and unreadable forms are available as private vars,
    # accessible by get_data - skv2109
    def make_data_readable(self, x_df, y_df=None) :
        """
        Method to convert encoded data to their human readable form
        :param x_df: The encoded feature dataframe
        :param y_df: Optional dataframe with targets
        :return: Decoded features and targets
        """
        if self.type == 'COMPAS' :
            x_df.fillna('', inplace=True)
        x_readable = x_df.loc[:, ~self._data_x_readable.columns.duplicated()]
        x_readable = x_readable.apply(
            lambda x : self._encoder_dict_x[x.name].inverse_transform(x))
        if y_df is None:
            return x_readable
        y_readable = y_df.apply(lambda y: self._encoder_dict_y[y.name].inverse_transform(y))
        return x_readable, y_readable

    def encode_data(self, x_df, y_df=None):
        """
        Method to encode data to be processed by a computational model
        :param x_df: The human readable feature dataframe
        :param y_df: Optional dataframe with human readable targets
        :return: Encoded features (and targets)
        """
        # if self.type == 'COMPAS':
        # x_df.fillna('', inplace=True)
        x_encoded = x_df.loc[:, ~self._data_x_readable.columns.duplicated()]
        x_encoded = x_encoded.apply(
            lambda x : self._encoder_dict_x[x.name].transform(x))
        if y_df is None :
            return x_encoded
        y_encoded = y_df.apply(lambda y : self._encoder_dict_y[y.name].transform(y))
        return x_encoded, y_encoded

if __name__ == '__main__' :
    dataset = Dataset('config_compas.json')
    x, y = dataset.get_data(readable=True)


"""
Definition of events that can be sent on the game
"""

import enum

__author__ = "Benjamin Schubert <ben.c.schubert@gmail.com>"

@enum.unique
class Event(enum.IntEnum):
    """
    Enumeration of all events valid in the game
    """
    ERROR = 0
    TOKEN = 1
    GAME_INFO = 2
    STATE = 3
    FOOD = 4
    DEATH = 5
    BULLETS = 6
    BONUS = 7
    HOOK = 8
    ALIVE = 9
    FINISHED = 10

@enum.unique
class Error(enum.IntEnum):
    """
    Enumeration of all errors that the server can throw to a player
    """
    TOKEN_INVALID = 0
    MAX_CAPACITY = 1
    NO_TOKEN = 2
    DUPLICATE_USERNAME = 3