from collections import OrderedDict
import sys

import torch
from torch import nn
import torch.nn.init as nn_init
import torch.nn.functional as F
from torch.autograd import Variable

# CNN BASED MULTIDECODERS

# Simpler multidecoder design with convolutional encoder/decoder layers
class CNNMultidecoder(nn.Module):
    def __init__(self, freq_dim=40,
                       splicing=[5,5],
                       enc_channel_sizes=[],
                       enc_kernel_sizes=[],
                       enc_downsample_sizes=[],
                       enc_fc_sizes=[],
                       latent_dim=512,
                       dec_fc_sizes=[],
                       dec_channel_sizes=[],
                       dec_kernel_sizes=[],
                       dec_upsample_sizes=[],
                       activation="ReLU",
                       decoder_classes=[""],
                       use_batch_norm=False,
                       weight_init="xavier_uniform"):
        super(CNNMultidecoder, self).__init__()

        # Store initial parameters
        self.freq_dim = freq_dim
        self.splicing = splicing    # [left, right]
        self.time_dim = (sum(splicing) + 1)

        self.enc_channel_sizes = enc_channel_sizes
        self.enc_kernel_sizes = enc_kernel_sizes
        self.enc_downsample_sizes = enc_downsample_sizes
        self.enc_fc_sizes = enc_fc_sizes

        self.latent_dim = latent_dim

        self.dec_fc_sizes = dec_fc_sizes
        self.dec_channel_sizes = dec_channel_sizes
        self.dec_kernel_sizes = dec_kernel_sizes
        self.dec_upsample_sizes = dec_upsample_sizes

        self.activation = activation
        self.decoder_classes = decoder_classes
        self.use_batch_norm = use_batch_norm
        self.weight_init = weight_init

        # STEP 1: Construct encoder

        current_channels = 1
        current_height = self.time_dim
        current_width = self.freq_dim

        # Convolutional stage
        self.encoder_conv_layers = OrderedDict()
        for idx in range(len(enc_channel_sizes)):
            enc_channel_size = enc_channel_sizes[idx]
            enc_kernel_size = enc_kernel_sizes[idx]
            enc_downsample_size = enc_downsample_sizes[idx]

            # Pad signal to avoid edge effects in output
            padding = enc_kernel_size - 1
            self.encoder_conv_layers["conv2d_%d" % idx] = nn.Conv2d(current_channels,
                                                                    enc_channel_size,
                                                                    enc_kernel_size,
                                                                    padding=padding)
            self.init_weights(self.encoder_conv_layers["conv2d_%d" % idx], "conv2d")
            current_channels = enc_channel_size

            # Formula from http://pytorch.org/docs/master/nn.html#conv2d
            # Assumes stride = 1, dilation = 1
            current_height = (current_height + (2 * padding) - (enc_kernel_size - 1) - 1) + 1
            current_width = (current_width + (2 * padding) - (enc_kernel_size - 1) - 1) + 1

            if self.use_batch_norm:
                self.encoder_conv_layers["batchnorm2d_%d" % idx] = nn.BatchNorm2d(enc_channel_size)

            self.encoder_conv_layers["%s_%d" % (self.activation, idx)] = getattr(nn, self.activation)()

            if enc_downsample_size > 0:
                # Pool only in frequency direction (i.e. kernel and stride 1 in time dimension)
                # Return indices as well (useful for unpooling: see
                #   http://pytorch.org/docs/master/nn.html#maxunpool2d)
                self.encoder_conv_layers["maxpool2d_%d" % idx] = nn.MaxPool2d((1, enc_downsample_size),
                                                                              return_indices=True)

                # Formula from http://pytorch.org/docs/master/nn.html#maxpool2d
                # Assumes stride = enc_downsample_size (default), padding = 0, dilation = 1
                current_height = current_height     # No change in time dimension!
                current_width = int((current_width - (enc_downsample_size - 1) - 1) / enc_downsample_size) + 1

        self.encoder_conv = nn.Sequential(self.encoder_conv_layers)

        # Fully-connected stage
        self.encoder_fc_layers = OrderedDict()
        current_fc_dim = current_channels * current_height * current_width
        for idx in range(len(enc_fc_sizes)):
            enc_fc_size = enc_fc_sizes[idx]

            self.encoder_fc_layers["lin_%d" % idx] = nn.Linear(current_fc_dim, enc_fc_size)
            self.init_weights(self.encoder_fc_layers["lin_%d" % idx], self.activation)
            current_fc_dim = enc_fc_size

            if self.use_batch_norm:
                self.encoder_fc_layers["batchnorm1d_%d" % idx] = nn.BatchNorm1d(current_fc_dim)
            self.encoder_fc_layers["%s_%d" % (self.activation, idx)] = getattr(nn, self.activation)()

        self.encoder_fc_layers["lin_final"] = nn.Linear(current_fc_dim, self.latent_dim)
        self.init_weights(self.encoder_fc_layers["lin_final"], self.activation)
        # self.encoder_fc_layers["%s_final" % self.activation] = getattr(nn, self.activation)()
        self.encoder_fc = nn.Sequential(self.encoder_fc_layers)

        # STEP 2: Construct decoders

        self.decoder_fc = dict()
        self.decoder_fc_layers = dict()
        self.decoder_deconv = dict()
        self.decoder_deconv_layers = dict()

        # Save values from encoder stage
        input_channels = current_channels
        input_height = current_height
        input_width = current_width

        for decoder_class in self.decoder_classes:
            # Fully-connected stage
            self.decoder_fc_layers[decoder_class] = OrderedDict()
            current_fc_dim = self.latent_dim
            for idx in range(len(dec_fc_sizes)):
                dec_fc_size = dec_fc_sizes[idx]

                self.decoder_fc_layers[decoder_class]["%s_%d" % (self.activation, idx)] = getattr(nn, self.activation)()

                self.decoder_fc_layers[decoder_class]["lin_%d" % idx] = nn.Linear(current_fc_dim, dec_fc_size)
                self.init_weights(self.decoder_fc_layers[decoder_class]["lin_%d" % idx], self.activation)
                current_fc_dim = dec_fc_size

                if self.use_batch_norm:
                    self.decoder_fc_layers[decoder_class]["batchnorm1d_%d" % idx] = nn.BatchNorm1d(current_fc_dim)

            self.decoder_fc_layers[decoder_class]["%s_final" % self.activation] = getattr(nn, self.activation)()
            self.decoder_fc_layers[decoder_class]["lin_final"] = nn.Linear(current_fc_dim,
                                                                           input_channels * input_height * input_width)
            self.init_weights(self.decoder_fc_layers[decoder_class]["lin_final"], self.activation)

            self.decoder_fc[decoder_class] = nn.Sequential(self.decoder_fc_layers[decoder_class])
            self.add_module("decoder_fc_%s" % decoder_class, self.decoder_fc[decoder_class])

            # Deconvolution stage
            current_height = input_height
            current_width = input_width

            self.decoder_deconv_layers[decoder_class] = OrderedDict()
            for idx in range(len(dec_channel_sizes)):
                dec_channel_size = dec_channel_sizes[idx]
                dec_kernel_size = dec_kernel_sizes[idx]
                dec_upsample_size = dec_upsample_sizes[idx]

                self.decoder_deconv_layers[decoder_class]["%s_%d" % (self.activation, idx)] = getattr(nn, self.activation)()

                if dec_upsample_size > 0:
                    # Un-pool only in frequency direction (i.e. kernel and stride 1 in time dimension)
                    self.decoder_deconv_layers[decoder_class]["maxunpool2d_%d" % idx] = nn.MaxUnpool2d((1, dec_upsample_size))

                    # Formula from http://pytorch.org/docs/master/nn.html#maxunpool2d
                    # Assumes stride = dec_upsample_size (default), padding = 0, dilation = 1
                    current_height = current_height     # No change in time dimension!
                    current_width = current_width * dec_upsample_size

                output_channels = 1 if idx == len(dec_channel_sizes) - 1 else dec_channel_sizes[idx + 1]

                self.decoder_deconv_layers[decoder_class]["conv2d_%d" % idx] = nn.Conv2d(dec_channel_size,
                                                                        output_channels,
                                                                        dec_kernel_size)

                # Formula for length from http://pytorch.org/docs/master/nn.html#conv2d
                # Assumes stride = 1, dilation = 1
                current_height = current_height - (dec_kernel_size - 1)
                current_width = current_width - (dec_kernel_size - 1)

                if self.use_batch_norm and idx != len(dec_channel_sizes) - 1:
                    # Don't normalize if it's the output layer!
                    self.decoder_deconv_layers[decoder_class]["batchnorm2d_%d" % idx] = nn.BatchNorm2d(output_channels)

            self.decoder_deconv[decoder_class] = nn.Sequential(self.decoder_deconv_layers[decoder_class])
            self.add_module("decoder_deconv_%s" % decoder_class, self.decoder_deconv[decoder_class])

    def init_weights(self, layer, layer_name):
        if "xavier" in self.weight_init:
            try:
                gain = nn_init.calculate_gain(layer_name.lower())
                getattr(nn_init, self.weight_init)(layer.weight, gain=gain)
            except ValueError:
                # Happens if layer type isn't supported yet -- just use default gain
                getattr(nn_init, self.weight_init)(layer.weight)
        else:
            getattr(nn_init, self.weight_init)(layer.weight)

    def decoder_parameters(self, decoder_class):
        # Get parameters for just a specific decoder
        if decoder_class not in self.decoder_classes:
            print("Decoder class \"%s\" not found in decoders: %s" % (decoder_class, self.decoder_classes),
                  flush=True)
            sys.exit(1)
        decoder_deconv_parameters = self.decoder_deconv[decoder_class].parameters()
        for param in decoder_deconv_parameters:
            yield param
        decoder_fc_parameters = self.decoder_fc[decoder_class].parameters()
        for param in decoder_fc_parameters:
            yield param

    def encoder_parameters(self):
        # Get parameters for just the encoder
        encoder_fc_parameters = self.encoder_fc.parameters()
        for param in encoder_fc_parameters:
            yield param
        encoder_conv_parameters = self.encoder_conv.parameters()
        for param in encoder_conv_parameters:
            yield param

    def generator_parameters(self):
        # Get parameters for encoder and decoders (i.e., no adversaries)
        for decoder_class in self.decoder_classes:
            for param in self.decoder_parameters(decoder_class):
                yield param
        for param in self.encoder_parameters():
            yield param

    def encode(self, feats):
        # Need to go layer-by-layer to get pooling indices
        pooling_indices = []
        unpool_sizes = []
        conv_encoded = feats
        for i, (encoder_conv_layer_name, encoder_conv_layer) in enumerate(self.encoder_conv_layers.items()):
            if "maxpool2d" in encoder_conv_layer_name:
                unpool_sizes.append(conv_encoded.size())
                conv_encoded, new_pooling_indices = encoder_conv_layer(conv_encoded)
                pooling_indices.append(new_pooling_indices)
            else:
                conv_encoded = encoder_conv_layer(conv_encoded)
        fc_input_size = conv_encoded.size()
        conv_encoded_vec = conv_encoded.view(conv_encoded.size()[0], -1)

        return (self.encoder_fc(conv_encoded_vec), fc_input_size, unpool_sizes, pooling_indices)

    def decode(self, z, decoder_class, fc_input_size, conv_input_sizes=None, unpool_sizes=None, pooling_indices=None):
        fc_decoded = self.decoder_fc[decoder_class](z)
        fc_decoded_mat = fc_decoded.view(fc_input_size)

        # Need to go layer-by-layer to insert pooling indices into unpooling layers
        output = fc_decoded_mat
        for i, (decoder_deconv_layer_name, decoder_deconv_layer) in enumerate(self.decoder_deconv_layers[decoder_class].items()):
            if "maxunpool2d" in decoder_deconv_layer_name:
                current_pooling_indices = pooling_indices.pop()
                current_unpool_size = unpool_sizes.pop()
                output = decoder_deconv_layer(output,
                                              current_pooling_indices,
                                              output_size=current_unpool_size)
            else:
                output = decoder_deconv_layer(output)

        return output

    def forward_decoder(self, feats, decoder_class):
        latent, fc_input_size, unpool_sizes, pooling_indices = self.encode(feats.view(-1,
                                                                           1,
                                                                           self.time_dim,
                                                                           self.freq_dim))
        return self.decode(latent, decoder_class, fc_input_size, unpool_sizes=unpool_sizes, pooling_indices=pooling_indices)

# Includes an adversarial classifier for picking clean vs. dirty
class CNNDomainAdversarialMultidecoder(CNNMultidecoder):
    def __init__(self, freq_dim=40,
                       splicing=[5,5],
                       enc_channel_sizes=[],
                       enc_kernel_sizes=[],
                       enc_downsample_sizes=[],
                       enc_fc_sizes=[],
                       latent_dim=512,
                       dec_fc_sizes=[],
                       dec_channel_sizes=[],
                       dec_kernel_sizes=[],
                       dec_upsample_sizes=[],
                       activation="ReLU",
                       decoder_classes=[""],
                       use_batch_norm=False,
                       weight_init="xavier_uniform",
                       domain_adv_fc_sizes=[],
                       domain_adv_activation="Sigmoid"):
        super(CNNDomainAdversarialMultidecoder, self).__init__(freq_dim=freq_dim,
                                                         splicing=splicing,
                                                         enc_channel_sizes=enc_channel_sizes,
                                                         enc_kernel_sizes=enc_kernel_sizes,
                                                         enc_downsample_sizes=enc_downsample_sizes,
                                                         enc_fc_sizes=enc_fc_sizes,
                                                         latent_dim=latent_dim,
                                                         dec_fc_sizes=dec_fc_sizes,
                                                         dec_channel_sizes=dec_channel_sizes,
                                                         dec_kernel_sizes=dec_kernel_sizes,
                                                         dec_upsample_sizes=dec_upsample_sizes,
                                                         activation=activation,
                                                         decoder_classes=decoder_classes,
                                                         use_batch_norm=use_batch_norm,
                                                         weight_init=weight_init)

        self.domain_adv_fc_sizes = domain_adv_fc_sizes
        self.domain_adv_activation = domain_adv_activation

        # Construct domain_adversary
        # Simple linear classifier
        self.domain_adversary_layers = OrderedDict()
        current_dim = self.latent_dim
        for i in range(len(self.domain_adv_fc_sizes)):
            next_dim = self.domain_adv_fc_sizes[i]
            self.domain_adversary_layers["lin_%d" % i] = nn.Linear(current_dim, next_dim)
            self.domain_adversary_layers["%s_%d" % (self.domain_adv_activation, i)] = getattr(nn, self.domain_adv_activation)()
            current_dim = next_dim
        self.domain_adversary_layers["lin_final"] = nn.Linear(current_dim, 1)
        self.domain_adversary_layers["%s_final" % self.domain_adv_activation] = getattr(nn, self.domain_adv_activation)()
        self.domain_adversary = nn.Sequential(self.domain_adversary_layers)

    def domain_adversary_parameters(self):
        # Get parameters for just the domain_adversary
        domain_adversary_parameters = self.domain_adversary.parameters()
        for param in domain_adversary_parameters:
            yield param

# Includes an adversarial classifier for detecting real vs. fake utterances in each domain
class CNNGANMultidecoder(CNNMultidecoder):
    def __init__(self, freq_dim=40,
                       splicing=[5,5],
                       enc_channel_sizes=[],
                       enc_kernel_sizes=[],
                       enc_downsample_sizes=[],
                       enc_fc_sizes=[],
                       latent_dim=512,
                       dec_fc_sizes=[],
                       dec_channel_sizes=[],
                       dec_kernel_sizes=[],
                       dec_upsample_sizes=[],
                       activation="ReLU",
                       decoder_classes=[""],
                       use_batch_norm=False,
                       weight_init="xavier_uniform",
                       gan_fc_sizes=[],
                       gan_activation="Sigmoid"):
        super(CNNGANMultidecoder, self).__init__(freq_dim=freq_dim,
                                                 splicing=splicing,
                                                 enc_channel_sizes=enc_channel_sizes,
                                                 enc_kernel_sizes=enc_kernel_sizes,
                                                 enc_downsample_sizes=enc_downsample_sizes,
                                                 enc_fc_sizes=enc_fc_sizes,
                                                 latent_dim=latent_dim,
                                                 dec_fc_sizes=dec_fc_sizes,
                                                 dec_channel_sizes=dec_channel_sizes,
                                                 dec_kernel_sizes=dec_kernel_sizes,
                                                 dec_upsample_sizes=dec_upsample_sizes,
                                                 activation=activation,
                                                 decoder_classes=decoder_classes,
                                                 use_batch_norm=use_batch_norm,
                                                 weight_init=weight_init)

        self.gan_fc_sizes = gan_fc_sizes
        self.gan_activation = gan_activation

        # Construct adversary for each decoder class
        # Simple linear classifier
        self.gan_layers = dict()
        self.gans = dict()
        for decoder_class in self.decoder_classes:
            self.gan_layers[decoder_class] = OrderedDict()
            current_dim = self.freq_dim * self.time_dim
            for i in range(len(self.gan_fc_sizes)):
                next_dim = self.gan_fc_sizes[i]
                self.gan_layers[decoder_class]["lin_%d" % i] = nn.Linear(current_dim, next_dim)
                self.gan_layers[decoder_class]["%s_%d" % (self.gan_activation, i)] = getattr(nn, self.gan_activation)()
                current_dim = next_dim
            self.gan_layers[decoder_class]["lin_final"] = nn.Linear(current_dim, 1)
            # Skip in favor of more numerically stable BCEWithLogitsLoss
            # self.gan_layers[decoder_class]["Sigmoid_final"] = nn.Sigmoid()
            self.gans[decoder_class] = nn.Sequential(self.gan_layers[decoder_class])
            self.add_module("gan_%s" % decoder_class, self.gans[decoder_class])

    def gan_parameters(self, decoder_class):
        # Get parameters for just the adversary
        gan_parameters = self.gans[decoder_class].parameters()
        for param in gan_parameters:
            yield param

    def forward_gan(self, feats, decoder_class):
        return self.gans[decoder_class](feats.view((-1, self.time_dim * self.freq_dim)))

# Includes a phone classifier at the latent space
class CNNPhoneMultidecoder(CNNMultidecoder):
    def __init__(self, freq_dim=40,
                       splicing=[5,5],
                       enc_channel_sizes=[],
                       enc_kernel_sizes=[],
                       enc_downsample_sizes=[],
                       enc_fc_sizes=[],
                       latent_dim=512,
                       dec_fc_sizes=[],
                       dec_channel_sizes=[],
                       dec_kernel_sizes=[],
                       dec_upsample_sizes=[],
                       activation="ReLU",
                       decoder_classes=[""],
                       use_batch_norm=False,
                       weight_init="xavier_uniform",
                       phone_fc_sizes=[],
                       phone_activation="Sigmoid",
                       num_phones=1):
        super(CNNPhoneMultidecoder, self).__init__(freq_dim=freq_dim,
                                                splicing=splicing,
                                                enc_channel_sizes=enc_channel_sizes,
                                                enc_kernel_sizes=enc_kernel_sizes,
                                                enc_downsample_sizes=enc_downsample_sizes,
                                                enc_fc_sizes=enc_fc_sizes,
                                                latent_dim=latent_dim,
                                                dec_fc_sizes=dec_fc_sizes,
                                                dec_channel_sizes=dec_channel_sizes,
                                                dec_kernel_sizes=dec_kernel_sizes,
                                                dec_upsample_sizes=dec_upsample_sizes,
                                                activation=activation,
                                                decoder_classes=decoder_classes,
                                                use_batch_norm=use_batch_norm,
                                                weight_init=weight_init)

        self.phone_fc_sizes = phone_fc_sizes
        self.phone_activation = phone_activation
        self.num_phones = num_phones

        # Construct simple linear phone classifier
        self.phone_layers = OrderedDict()
        current_dim = self.latent_dim
        for i in range(len(self.phone_fc_sizes)):
            next_dim = self.phone_fc_sizes[i]
            self.phone_layers["lin_%d" % i] = nn.Linear(current_dim, next_dim)
            self.phone_layers["%s_%d" % (self.phone_activation, i)] = getattr(nn, self.phone_activation)()
            current_dim = next_dim
        self.phone_layers["lin_final"] = nn.Linear(current_dim, self.num_phones)
        self.phone_layers["LogSoftmax_final"] = nn.LogSoftmax(dim=1)
        self.phone_classifier = nn.Sequential(self.phone_layers)

    def generator_parameters(self):
        # Get parameters for generator and phone classifier
        for param in self.phone_classifier.parameters():
            yield param
        for decoder_class in self.decoder_classes:
            for param in self.decoder_parameters(decoder_class):
                yield param
        for param in self.encoder_parameters():
            yield param

    def forward_phones(self, feats):
        latent, fc_input_size, unpool_sizes, pooling_indices = self.encode(feats.view(-1,
                                                                           1,
                                                                           self.time_dim,
                                                                           self.freq_dim))

        return self.phone_classifier(latent)

# Includes a phone classifier at the output features -- trained end-to-end
class CNNEnd2EndMultidecoder(CNNMultidecoder):
    def __init__(self, freq_dim=40,
                       splicing=[5,5],
                       enc_channel_sizes=[],
                       enc_kernel_sizes=[],
                       enc_downsample_sizes=[],
                       enc_fc_sizes=[],
                       latent_dim=512,
                       dec_fc_sizes=[],
                       dec_channel_sizes=[],
                       dec_kernel_sizes=[],
                       dec_upsample_sizes=[],
                       activation="ReLU",
                       decoder_classes=[""],
                       use_batch_norm=False,
                       weight_init="xavier_uniform",
                       e2e_fc_sizes=[],
                       e2e_activation="Sigmoid",
                       num_phones=1):
        if len(decoder_classes) > 1 or decoder_classes[0] != "clean":
            raise RuntimeError("CNNEnd2EndMultidecoder only uses one decoder class (clean)")

        super(CNNEnd2EndMultidecoder, self).__init__(freq_dim=freq_dim,
                                                splicing=splicing,
                                                enc_channel_sizes=enc_channel_sizes,
                                                enc_kernel_sizes=enc_kernel_sizes,
                                                enc_downsample_sizes=enc_downsample_sizes,
                                                enc_fc_sizes=enc_fc_sizes,
                                                latent_dim=latent_dim,
                                                dec_fc_sizes=dec_fc_sizes,
                                                dec_channel_sizes=dec_channel_sizes,
                                                dec_kernel_sizes=dec_kernel_sizes,
                                                dec_upsample_sizes=dec_upsample_sizes,
                                                activation=activation,
                                                decoder_classes=decoder_classes,
                                                use_batch_norm=use_batch_norm,
                                                weight_init=weight_init)

        self.e2e_fc_sizes = e2e_fc_sizes
        self.e2e_activation = e2e_activation
        self.num_phones = num_phones

        # Construct simple linear phone classifier on top of reconstruction layer
        self.phone_layers = OrderedDict()
        current_dim = self.time_dim * self.freq_dim
        for i in range(len(self.e2e_fc_sizes)):
            next_dim = self.e2e_fc_sizes[i]
            self.phone_layers["lin_%d" % i] = nn.Linear(current_dim, next_dim)
            self.phone_layers["%s_%d" % (self.e2e_activation, i)] = getattr(nn, self.e2e_activation)()
            current_dim = next_dim
        self.phone_layers["lin_final"] = nn.Linear(current_dim, self.num_phones)
        self.phone_layers["LogSoftmax_final"] = nn.LogSoftmax(dim=1)
        self.phone_classifier = nn.Sequential(self.phone_layers)

    def generator_parameters(self):
        # Get parameters for generator and phone classifier
        for param in self.phone_classifier.parameters():
            yield param
        for param in self.decoder_parameters("clean"):
            yield param
        for param in self.encoder_parameters():
            yield param

    def forward_phones(self, feats):
        enhanced_feats = self.forward_decoder(feats, "clean")
        return self.phone_classifier(enhanced_feats.view((-1, self.time_dim * self.freq_dim)))


import json

def save(data: dict) -> None:
    """Save data to json file."""
    print("== Save to json file ==")
    with open('tree.json', 'w') as f:
        json.dump(data, f)

def load() -> None:
    """Load data from json file."""
    print("== Load data from json file ==")
    with open('tree.json', 'r', encoding='utf-8') as f:
        json_file = json.load(f)

    for line in json_file:
        print(f"{line['name']} {line['parent']} {line['type']}"
              f" {line['size']}")


import requests
import csv
import json
import pandas as pd

filename = "connections.csv"
platforms = {}

datareader = csv.reader(open(filename))
domains_count= len(list(datareader))
print(f"found {domains_count} domains, processing...")
step = 0

with open(filename, "r") as csvfile:
    datareader = csv.reader(csvfile)
    next(datareader) # skip the header

    for row in datareader:
        step += 1
        domain = row[0]
        url = f"https://{domain}/.well-known/nodeinfo"

        try:
            response = requests.get(url)
            response_json = response.json()
            nodeinfo_url = response_json["links"][0]["href"]

            response = requests.get(nodeinfo_url)
            response_json = response.json()
            software_name = response_json["software"]["name"]

            print(f"{step}/{domains_count}: {domain} uses {software_name}")

            if software_name in platforms:
                platforms[software_name] += 1
            else:
                platforms[software_name] = 1
        except requests.exceptions.RequestException as e:
            print(f"error accessing {domain}")

platforms_formatted = json.dumps(platforms, indent=2)
print(platforms_formatted)

df = pd.DataFrame(platforms.items(), columns=['platform', 'servers'])
df.to_csv('platforms.csv', encoding='utf-8', index=False)


#!/usr/bin/env python3
import requests
from bs4 import BeautifulSoup
import json
from slugify import slugify

# https://github.com/un33k/python-slugify

URLS = [
    "https://www.swm.de/baeder/schwimmen-sauna/auslastung"
]

def main():
    orgs = []
    for url in URLS:
        orgs += crawl_url(url)

    orgs.sort(key=lambda x: x["display_name"])

    with open("organizations.json", "w") as file:
        json.dump(orgs, file, indent=2)

def crawl_url(url):
    orgs = []
    req = requests.get(url)
    soup = BeautifulSoup(req.content, 'html.parser')
    for ticos_el in soup.select("multiple-ticos-bar-counter"):
        ids = json.loads(ticos_el["organizations"])
        displays = json.loads(ticos_el["displays"])
        orgs += [dict(id=id, name=slugify(name), display_name=name) for (id, name) in zip(ids, displays)]
    return orgs

if __name__ == "__main__":
    main()


#!/usr/bin/env python

import argparse
import csv
from Bio import SeqIO
import time

parser = argparse.ArgumentParser(description='writes a table of repeat matches content in a genome given an assembly a gff file containing repeats.')
parser.add_argument("-s","--seqfile",
					type=str,
					default='',
					help="Sequence file in fasta format")
parser.add_argument("-g","--gff",
					type=str,
					default='',
					help="gff3 file containing repeats as a type")
parser.add_argument("-wsi","--windowsize",
					type=int,
					default=10000,
					help="window size in basepairs")
parser.add_argument("-wst","--windowstep",
					type=int,
					default=10000,
					help="window step length in basepairs")
parser.add_argument("-o","--output",
					type=str,
					default='',
					help="name of cytoband output file")
args=parser.parse_args()

seqfile = args.seqfile
gff = args.gff
windowsize = args.windowsize
windowstep = args.windowstep
output = args.output

#########################################################################################

class GFF:
	def __init__(self, gff_entry):
		self.seqid = gff_entry[0]
		self.source = gff_entry[1]
		self.type = gff_entry[2]
		self.start = int(gff_entry[3])
		self.end = int(gff_entry[4])
		self.score = gff_entry[5]
		self.strand = gff_entry[6]
		self.phase = gff_entry[7]
		self.attributes = gff_entry[8]

def GFF_parse(gff) :
	gff_list = []
	with open(gff) as OF:
		reader = csv.reader(OF, delimiter='\t')
		for row in reader :
			if (row[0][0] != '#') and (len(row) > 1) :
				gff_entry = GFF(row)
				gff_list.append(gff_entry)
	return(gff_list)

def Remove_encompassed_repeats(reduced_gff):
	if len(reduced_gff) < 2 :
		return(reduced_gff)
	else:
		#start_time = time.time()
		reduced_gff.sort(key=lambda x: (x.end),reverse=True)
		reduced_gff.sort(key=lambda x: (x.start))
		reduced_gff = reduced_gff + ['end']
		entry_to_keep = []
		focal_entry = reduced_gff[0]
		if len(reduced_gff) > 1:
			test_entry = reduced_gff[1]
			#print(focal_entry.start, focal_entry.end)
			while test_entry != 'end':
				#print(test_entry.start, test_entry.end)
				if test_entry.end <= focal_entry.end :
					index = reduced_gff.index(test_entry)
					test_entry = reduced_gff[(index + 1)]
				else:
					entry_to_keep.append(focal_entry)
					focal_entry = test_entry
					#print(reduced_gff.index(focal_entry))
					index = reduced_gff.index(test_entry)
					test_entry = reduced_gff[(index + 1)]
		else:
			entry_to_keep.append(focal_entry)
		#end_time = time.time()
		#print((end_time - start_time))
		return(entry_to_keep)

def Remove_overlapping_repeats(reduced_gff):
	if len(reduced_gff) < 2 :
		return(reduced_gff)
	else:
		#start_time = time.time()
		#if len(reduced_gff) == 1:
		#	print(reduced_gff)
		reduced_gff.sort(key=lambda x: (x.end),reverse=True)
		reduced_gff.sort(key=lambda x: (x.start))
		reduced_gff = reduced_gff + ['end']
		entry_to_keep = []
		focal_entry = reduced_gff[0]
		if len(reduced_gff) > 1:
			i = 1
			#entry_to_keep.append(focal_entry)
			while reduced_gff[i] != 'end':
				if reduced_gff[i].start < focal_entry.end:
					gff_entry = GFF([reduced_gff[i].seqid, reduced_gff[i].source, reduced_gff[i].type, focal_entry.start, reduced_gff[i].end, '','','',''])
					focal_entry = gff_entry
					i = i + 1
				else:
					entry_to_keep.append(focal_entry)
					focal_entry = reduced_gff[i]
					i = i + 1
		entry_to_keep.append(focal_entry)
		#end_time = time.time()
		#print((end_time - start_time))
		return(entry_to_keep)

def Calc_repeat_content(sequence_list, gff_list, windowsize, windowstep) :
	start_time = time.time()
	content_list = []
	for seq in sequence_list:
		print(seq.id)
		reduced_gff = [repeat for repeat in gff_list if repeat.seqid == seq.id]
		endtime1 = time.time()
		print('time to reduce list = ' + str(endtime1 - start_time))
		#print(len(reduced_gff))
		reduced_gff = Remove_encompassed_repeats(reduced_gff)
		endtime2 = time.time()
		print('time to remove encompassing = ' + str(endtime2 - endtime1))
		#print(len(reduced_gff))
		#print(reduced_gff[-1])
		reduced_gff = Remove_overlapping_repeats(reduced_gff)
		endtime3 = time.time()
		print('time to remove overlap = ' + str(endtime3 - endtime2))
		if len(seq.seq) < windowsize:
			sum_repeat_length = 0
			for repeat in reduced_gff :
				repeat_length = abs(repeat.end - repeat.start)
				sum_repeat_length = sum_repeat_length + repeat_length
			repeat_content = sum_repeat_length / len(seq.seq)
			start = 1
			end = len(seq.seq)
			entry = [seq.id, start, end, repeat_content]
			print(repeat_content, start, end, seq.id)
			content_list.append(entry)
		else:
			start = 1
			end = windowsize
			while end <= len(seq.seq) :
				sum_repeat_length = 0
				for repeat in reduced_gff:
					if repeat.start >= start and repeat.end <= end :
						repeat_length = abs((repeat.end + 1) - repeat.start)
						sum_repeat_length = sum_repeat_length + repeat_length
					elif repeat.start < start and repeat.end > start and repeat.end <= end :
						repeat_length = abs((repeat.end + 1) - start)
						sum_repeat_length = sum_repeat_length + repeat_length
					elif repeat.start >= start and repeat.start < end and repeat.end > end :
						repeat_length = abs((end + 1) - repeat.start)
						sum_repeat_length = sum_repeat_length + repeat_length
				repeat_content = sum_repeat_length / windowsize
				entry = [seq.id, start, end, repeat_content]
				print(repeat_content,start,end, seq.id)
				content_list.append(entry)
				start = start + windowstep
				end = end + windowstep
		print()
	end_time = time.time()
	print((end_time - start_time))
	return(content_list)

#########################################################################################

gff_list = GFF_parse(gff)

gff_list = [ entry for entry in gff_list if entry.source == 'repeat_gff:repeatmasker' and entry.type == 'match']

sequences = list(SeqIO.parse(seqfile,"fasta"))

repeat_list = Calc_repeat_content(sequences, gff_list, windowsize, windowstep)
#test = Calc_repeat_content(reduced_sequences, gff_list, windowsize, windowstep)

with open(output,'w') as outFile:
	outfile_writer = csv.writer(outFile, delimiter = '\t')
	outfile_writer.writerow(['scaffold', 'start', 'end', 'repeat_content'])
	for entry in repeat_list:
		outfile_writer.writerow(entry)

outFile.close()

#########################################################################################


#!/usr/bin/python

import rospy
import rospkg

from std_msgs.msg import String

from moveit_commander import (
	MoveGroupCommandInterpreter,
	MoveGroupInfoLevel,
)

import os
import subprocess

class bcolors:
	HEADER = "\033[95m"
	OKBLUE = "\033[94m"
	OKGREEN = "\033[92m"
	WARNING = "\033[93m"
	FAIL = "\033[91m"
	ENDC = "\033[0m"

def print_message(level, msg):
	if level == MoveGroupInfoLevel.FAIL:
		print(bcolors.FAIL + msg + bcolors.ENDC)
	elif level == MoveGroupInfoLevel.WARN:
		print(bcolors.WARNING + msg + bcolors.ENDC)
	elif level == MoveGroupInfoLevel.SUCCESS:
		print(bcolors.OKGREEN + msg + bcolors.ENDC)
	elif level == MoveGroupInfoLevel.DEBUG:
		print(bcolors.OKBLUE + msg + bcolors.ENDC)
	else:
		print(msg)

class RoutinesServer:
	def __init__(self):
		rospack = rospkg.RosPack()
		pkg_dir = rospack.get_path('s4a_teleop')
		self.routines_dir = os.path.join(pkg_dir, 'routines/')

		self.run_routine_sub = rospy.Subscriber(
			"run_routine",
			String,
			self.on_run_routine
		)

	'''
		Recursive function
	'''
	def exec_routine_script(self, routine_basename):
		fn = os.path.join(self.routines_dir, routine_basename)
		with open(fn) as f:
			for line in f.readlines():
				if line.strip() == '':
					# Skip empty line.
					continue
				elif line.lstrip().startswith('#'):
					# Skip comment.
					continue
				elif line.lstrip().startswith('%include'):
					# Include recusively routine.
					bn2 = line.strip().split()[1]
					self.exec_routine_script(bn2)
				else:
					# Input.
					name = ''
					ag = self.cmd_interp.get_active_group()
					if ag != None:
						name = ag.get_name()
					print(
						bcolors.OKBLUE + name + '> ' + bcolors.ENDC + line,
						end = ''
					)

					# Execute.
					(level, msg) = self.cmd_interp.execute(line)

					# Output.
					print_message(level, msg)

	def go_routine_mode_and_exec(self, routine_name):
		rospy.loginfo('Running routine: {}'.format(routine_name))

		# Change mode.
		def change_mode(mode):
			cmd = 'rosrun s4a_teleop change_controller.py --silent'.split()
			cmd.append(mode)
			subprocess.run(cmd)

		change_mode('traj')

		self.cmd_interp = MoveGroupCommandInterpreter()

		self.exec_routine_script(routine_name + '.moveitcmd')

		# Return back.
		change_mode('servo')

	def on_run_routine(self, msg):
		routine_name = msg.data
		self.go_routine_mode_and_exec(routine_name)

if __name__ == '__main__':
	rospy.init_node('routines_server')
	controller = RoutinesServer()
	rospy.spin()


from django.core.urlresolvers import reverse

from core.tests.base import LoginWithUserTestCase

class CalculatorTestCase(LoginWithUserTestCase):
    def test_index_smoke(self):
        response = self.client.get(reverse('calculator:index'))
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, 'calculator/calculator.html')


from scipy.spatial.qhull import Voronoi

from .base import mse
from .layers import *

class LayerVoronoiImage(Image):
    def __init__(self, layers, original_image, pts_number, size):
        super().__init__(original_image)
        self.a_im = None
        self.layers = layers
        self.im = None
        self.pts_number = pts_number
        self.radius = 4  # size[0] // int(np.sqrt(pts_number)) // 2
        self.size = size
        self.rgb = [None] * 3
        self.polys = [None] * 3
        self.ch_l_point = None

    def __deepcopy__(self, memo):
        copy = type(self)(None, self.original_image, self.pts_number, self.size)
        memo[id(self)] = copy
        copy.layers = deepcopy(self.layers, memo)
        copy.rgb = deepcopy(self.rgb)
        copy.polys = [self.polys[i] for i in range(len(self.polys))]
        copy.ch_l_point = None
        return copy

    def draw(self, canvas=None, make_beautiful=False):
        if self.im is None:
            layers = [np.zeros(self.size, dtype=np.uint8) for _ in range(3)] if canvas is None else cv2.split(
                np.copy(canvas))
            for i, layer in enumerate(self.layers):
                # if there is no change in color
                if self.ch_l_point is None or self.ch_l_point[0] != i or self.rgb[i] is None:

                    # if change is in structure
                    if self.rgb[i] is None or make_beautiful:
                        if self.polys[i] is None:
                            vor = Voronoi(layer[0], incremental=False)
                            self.polys[i] = vor
                        else:
                            vor = self.polys[i]

                        # draw from scratch
                        ver = np.int32(vor.vertices)
                        for j in range(0, len(vor.point_region)):
                            ri = vor.point_region[j]
                            if len(vor.regions[ri]) != 0 and -1 not in vor.regions[ri]:
                                cv2.fillPoly(layers[i], [ver[vor.regions[ri]].reshape((-1, 1, 2))],
                                             int(self.layers[i][1][j]), cv2.LINE_AA if make_beautiful else cv2.LINE_8)

                    # nothing changed this time
                    else:
                        layers[i] = self.rgb[i]
                else:
                    # change only color
                    vor = self.polys[i]
                    layers[i] = self.rgb[i]
                    j = self.ch_l_point[1]
                    ri = vor.point_region[j]
                    ver = np.int32(vor.vertices)
                    if len(vor.regions[ri]) != 0 and -1 not in vor.regions[ri]:
                        cv2.fillPoly(layers[i], [ver[vor.regions[ri]].reshape((-1, 1, 2))],
                                     int(self.layers[i][1][j]))
            self.rgb = layers
            self.im = cv2.merge(layers)
        return self.im

    # draw centers
    def draw_approximated(self):
        if self.a_im is None:
            self.a_im = LayerVoronoiImage.approximated(self.layers, self.radius, self.size)
        return self.a_im

    # fast method of drawing
    @staticmethod
    def approximated(pts, r, size):
        layers = np.zeros((3, size[0], size[1]), dtype=np.uint8)
        for i in range(len(pts)):
            layer = pts[i]
            for j in range(len(layer[0])):
                x, y = pts[i][0][j]
                c = pts[i][1][j]
                layers[i, y - r: y + r, x - r: x + r] = int(c)
        return cv2.merge(layers)

    def fitness(self, image=None, gen=0, **kwargs):
        if self.fit is None:
            if image is None:
                image = self.original_image
            self.fit = mse(image, self.draw())
            # self.draw_approximated() if gen < 3000 else self.draw())  # 500 / np.linalg.norm(self.draw() - image)
        return self.fit

    def mutate(self, gen):
        mutated = deepcopy(self)
        mutated.im = None
        self._mutate(mutated, gen)
        return mutated

    def _chance(self, gen):
        if gen < 500_000:
            return 0.5
        else:
            return 0.5 + (gen - 500_000) * 0.5 / 500_000

    def _mutate(self, mutated, gen):
        changed_layer = rd.randint(0, 3)
        layer = mutated.layers[changed_layer]
        point = rd.randint(0, len(layer[0]))

        if rd.rand() > 0.5:#self._chance(gen):
            mutated.rgb[changed_layer] = None
            mutated.polys[changed_layer] = None
            xy = rd.randint(0, 2)
            if gen < 50000:
                layer[0][point][xy] = rd.randint(0, self.size[xy])
            else:
                layer[0][point][xy] = layer[0][point][xy] + rd.randn() * 10
        else:
            layer[1][point] = rd.randint(0, 256)
            mutated.ch_l_point = (changed_layer, point)

    def draw_bfl(self):
        layers = [np.zeros(self.size, dtype=np.uint8) for _ in range(3)]
        for i, layer in enumerate(self.layers):
            vor = Voronoi(layer[0], incremental=False)
            # draw from scratch
            ver = np.int32(vor.vertices)
            for j in range(0, len(vor.point_region)):
                ri = vor.point_region[j]
                if len(vor.regions[ri]) != 0 and -1 not in vor.regions[ri]:
                    cv2.fillPoly(layers[i], [ver[vor.regions[ri]].reshape((-1, 1, 2))],
                                 int(self.layers[i][1][j]), cv2.LINE_AA)

        return cv2.merge(layers)

    def save(self, name):
        cv2.imwrite(name, self.draw_bfl())

class VoronoiLayerImageGenerator(Generator):
    def __init__(self, original_image):
        super().__init__(original_image)

    def generate(self, n=1, pts_numbers=35*35):
        ims = []
        for _ in range(n):
            pts = [[[], []] for _ in range(3)]
            for i in range(3):
                for _ in range(3):
                    pts[i][0] = rd.randint(0, self._original_image.shape[0], size=(pts_numbers, 2))
                    pts[i][1] = rd.randint(0, 256, (pts_numbers))
            ims.append(LayerVoronoiImage(pts, self._original_image, pts_numbers, self._original_image.shape[:2]))
        return np.array(ims)


import mmw
import typing

class TextField(mmw.Drawable):
    """A Textfield (WIP)"""
    def __repr__(self) -> str:
        if self.useRelativePos:
            return 'mmw.TextField(type=mmw.{}, name={}) at ({}/{}, {}/{})'.\
                format(self._type, repr(self.name), self.xRel, self.x,
                       self.yRel, self.y)
        else:
            return 'mmw.TextField(type=mmw.{}, name={}) at ({}, {})'.format(
                self._type, repr(self.name), self.x, self.y)

    def __init__(self, type=mmw.consts.TF_ONELINE, name=''):
        super().__init__(name)
        self.text = ['']
        self._type = None
        self.field_type = type

    @property
    def field_type(self) -> str:
        return self._type

    @field_type.setter
    def field_type(self, value):
        if value in mmw.consts.TF_TYPES:
            self._type = value
        else:
            raise ValueError(repr(value)+' is not part of mmw.consts.TF_TYPES '
                             'and cannot be used as a type. '
                             'See mmw.consts.TF_TYPES')

    def draw(self) -> typing.List[str]:
        output = ''
        if self.name != '':
            output += self.name+' '
        if self.field_type == mmw.consts.TF_ONELINE:
            if self.forcedWidth <= 0:  # Brak ustawionej wielkosci
                if self.autoResize:  # Automagiczna wielkosc
                    output += self.style['TextLeftBorder']
                    if isinstance(self.text, list):
                        output += self.text[0]
                    elif isinstance(self.text, str):
                        output += self.text
                    else:
                        output += repr(self.text)
                    output += self.style['TextRightBorder']
                    self.width = len(output)
                else:
                    raise mmw.errors.InvalidStateError('self.forcedWidth '
                                                       'is <= 0 '
                                                       'but self.autoResize '
                                                       'is not set.')
            else:
                if self.autoResize:
                    raise mmw.errors.InvalidStateError('self.autoResize '
                                                       'is set and '
                                                       'self.forcedWidth '
                                                       'is set.')
                else:
                    text = ''
                    if isinstance(self.text, list):
                        text = self.text[0][:self.forcedWidth-2]
                    elif isinstance(self.text, str):
                        text = self.text[:self.forcedWidth-2]
                    else:
                        text = repr(self.text)[:self.forcedWidth-2]
                    output += self.style['TextLeftBorder']
                    output += text
                    output += self.style['TextEmpty']\
                        * (self.forcedWidth - len(text) - 2)
                    output += self.style['TextRightBorder']
        return output.split('\n')


from django.urls import path
from .views import *
from django.contrib.auth.views import LogoutView

app_name = 'blogjmv'

urlpatterns = [

    path("", Index.as_view(), name="index"),

    path("login/", login_request, name="login"),
    path("register/", register, name="register"),
    path("logout/", LogoutView.as_view(template_name="logout.html"), name="logout"),

    path("about/", About.as_view(), name="about"),
    path("post_detail/<pk>", PostDetail.as_view(), name="post_detail"),
    path("post_like/<id>", post_like, name="post_like"),
    path("comment_create/<id>", comment_create, name="comment_create"),
    path("post_by_subject/<pk>", PostBySubject.as_view(), name="post_by_subject"),
    path("post_by_author/<pk>", PostByAuthor.as_view(), name="post_by_author"),

]


import asyncio

from sqlalchemy import select

from database import async_session_maker
from auth.models import Role

async def db_init():
    async with async_session_maker() as session:
        roles = await session.scalars(select(Role))
        if len(roles.all()) == 0:
            session.add(Role(name='user'))
            session.add(Role(name='admin'))
            await session.commit()
            print("`admin` and `user` roles was added")

if __name__ == '__main__':
    asyncio.run(db_init())


"""
    代码作用为合并DOT的TLSA记录和对应证书链，用于验证DANE
    输入依次为TLSA文件，cert文件，输出文件。
    输出为json文件
"""

__author__ = "lrx"

import argparse
import sys
import json

def getTLSA(tlsafile, merge_dict):
    f = open(tlsafile, "r")

    while True:
        line = f.readline()
        if not line: break
        data = json.loads(line)

        if data["tlsa"] != "":
            key = data["domain"] + ":" + data["ip"]

            merge_dict[key] = {"domain": data["domain"], "ip": data["ip"], "tlsa": data["tlsa"],
                               "dnssec": data["tlsa_dnssec"]}

    return merge_dict

def getCert(certfile, merge_dict):
    f = open(certfile, "r")
    keys = merge_dict.keys()

    while True:
        line = f.readline()
        if not line: break
        data = json.loads(line)

        key = data["domain"] + ":" + data["ip"]
        if key in keys:
            merge_dict[key]["cert_chain"] = data["raw_cert_chain"]

    return merge_dict

def main(args):
    global transport_type
    parser = argparse.ArgumentParser(description="Merge TLSA records and certificates to validate DANE")
    parser.add_argument('tlsa_file', help="Input file containing a list of TLSA records")
    parser.add_argument('cert_file', help="Input file containing a list of certificate")
    parser.add_argument('output', help="Output dir to write results to")

    args = parser.parse_args(args)

    merge_dict = {}
    merge_dict = getTLSA(args.tlsa_file, merge_dict)
    merge_dict = getCert(args.cert_file, merge_dict)

    fOut = open(args.output, "w")

    for key in merge_dict.keys():
        if merge_dict[key]["cert_chain"] != "":
            json.dump(merge_dict[key], fOut)
            fOut.write("\n")

if __name__ == "__main__":
    main(sys.argv[1:])


import csv
import re
import docx
from docx.shared import Inches, Pt, RGBColor
from docx.oxml.ns import nsdecls
from docx.oxml import parse_xml
from docx.enum.text import WD_COLOR_INDEX

# Global Variables
# test suite header
TEST_SUITE_HEADER = r"\d{1,2}\.\d{1,2}"
# Input Variables
# Name of CSV to use for test output
CSVFILES = [
    "../Arista Agora Test Plan V9.3 - EOS Test Items.csv",
    "../Arista Agora Test Plan V9.3 - CVA_CVP Test Items.csv",
]
DOCFILE = "../test_summary_results.docx"
# EOS version
EOS = "4.29.1F-DPE"
# Column name of interesting values
TITLE_ROW = 0
HEADER_ROW = 1
TC_ID = r".*Test Case ID.*"
CASE_NAME = r".*Test Case Description.*"
PROCEDURE = r".*Test Steps.*"
EXPECTED_RESULT = r".*Criteria.*"
PASS_FAIL = r"Pass/Fail"
OBSERVATION = r".*Observations.*"
TC_TYPE = r".*Test Case Type.*"
# Column Width
WIDTHS = (
    Inches(0.5),
    Inches(1.17),
    Inches(2.50),
    Inches(1.25),
    Inches(0.75),
    Inches(1.50),
)

def parse_csv():
    """Iterate through a list of CSV files.  Open and parse CSV data
       and find test data

    Returns:
        list: List of interesting test data
    """
    test_data = []

    # iterate through CSV files
    for csvfile in CSVFILES:
        print(f"Parsing CSV file: {csvfile} for test data...")

        # open csv
        with open(csvfile, "r", encoding="utf-8") as file:
            csv_reader = csv.reader(file)
            # regex for matching a test suite header
            inspect_csv_data(test_data, csv_reader)

    return test_data

def set_header_columns(row):
    """Use regex to find a column index for interesting data

    Args:
        row (list): CSV row

    Returns:
        dict: key is column name and value is column index
    """

    header_columns = {}

    # iterate through cells to find column headers
    for count, _ in enumerate(row):
        if re.match(TC_ID, row[count]):
            header_columns["tc_id"] = count
        elif re.match(CASE_NAME, row[count]):
            header_columns["case_name"] = count
        elif re.match(PROCEDURE, row[count]):
            header_columns["procedure"] = count
        elif re.match(EXPECTED_RESULT, row[count]):
            header_columns["expected_result"] = count
        elif re.match(PASS_FAIL, row[count]):
            header_columns["pass_fail"] = count
        elif re.match(OBSERVATION, row[count]):
            header_columns["observation"] = count
        elif re.match(TC_TYPE, row[count]):
            header_columns["tc_type"] = count

    return header_columns

def inspect_csv_data(test_data, csv_reader):
    """Inspect each row of CSV data and determine if its interesting

    Args:
        test_data (list): Use CSV file to update list with interesting test data
        csv_reader (obj): CSV object to iterate through
    """

    for count, row in enumerate(csv_reader):
        test_data_entry = []

        # skip title row
        if count == TITLE_ROW:
            pass
        # inspect header and find correct column numbers
        elif count == HEADER_ROW:
            header_columns = set_header_columns(row)
        # parse csv row for a test case ids
        elif "TN" in row[header_columns["tc_id"]]:
            test_data_entry.append(row[header_columns["tc_id"]])
            test_data_entry.append(row[header_columns["case_name"]])
            test_data_entry.append(row[header_columns["procedure"]])
            test_data_entry.append(row[header_columns["expected_result"]])
            test_data_entry.append(row[header_columns["pass_fail"]])
            test_data_entry.append(row[header_columns["observation"]])
            test_data.append(test_data_entry)
        # parse csv row for header
        elif re.match(TEST_SUITE_HEADER, row[header_columns["case_name"]]):
            test_data_entry.append(row[header_columns["case_name"]])
            test_data.append(test_data_entry)
        # parse csv row for sub-section
        elif (not row[header_columns["tc_id"]] and not row[header_columns["tc_type"]]) and row[
            header_columns["case_name"]
        ] != "":
            test_data_entry.append("")
            test_data_entry.append(row[header_columns["case_name"]])
            test_data.append(test_data_entry)

def write_report(test_data):
    """Write word doc with test data collected from CSV

    Args:
        test_data (list): List of interesting test data
    """
    print(f"Writing report doc: {DOCFILE}")

    # Open doc
    document = docx.Document()
    # Change doc margins
    section = document.sections[0]
    section.left_margin = Inches(0.5)
    section.right_margin = Inches(0.5)

    write_header(document)
    write_results_table(document, test_data)

    doc_text = (
        "\n\n\nPlease note – for more details on test outputs please "
        "reference the test case output appendix docs.\n"
    )
    write_text(document, doc_text)

    # Close doc
    document.save(DOCFILE)

def write_header(document):
    """Write test section header to word doc

    Args:
        document (obj): Test report Word doc
        test_data (list): List of interesting test data
    """

    # Write summary header
    heading = document.add_heading(level=2)
    run = heading.add_run("4	Test Summary Result")
    run.font.size = Pt(16)
    run.font.color.rgb = RGBColor(204, 0, 0)

    # Header text
    doc_text = (
        f"\nTest Cases for Arista BAU Hardware & EOS {EOS} Certification. "
        "Please note – for more details on test outputs please reference "
        "the test case output appendix docs.\n\n\n"
    )
    write_text(document, doc_text)

def write_text(document, doc_text, font_pt=10, font="Arial"):
    """Write a paragraph to Word Doc

    Args:
        document (obj): Word doc object
        doc_text (str): Text to write to doc
        pt (int): Word font size
        font (string): Word font style
    """
    para = document.add_paragraph()
    run = para.add_run(doc_text)
    run.font.size = Pt(font_pt)
    run.font.name = font

def write_header_row(table):
    """Write the header row for the result table

    Args:
        table (obj): Word doc table object
    """

    headers = [
        "Test Case",
        "Case Name",
        "Procedure",
        "Expected result",
        "Pass/Fail",
        "Observation",
    ]

    # Write header row in table
    for count, header in enumerate(headers):
        set_cell_widths(table, 0)
        para = table.rows[0].cells[count].paragraphs[0]

        run = para.add_run(header)
        run.font.name = "Arial"
        run.font.size = Pt(9)
        run.font.bold = True

        shade_cell(count, table, 0, "00FFFF")

def write_results_table(document, test_data):
    """Write results table to Word doc

    Args:
        document (obj): Word doc object
        test_data (list): List of interesting test data
    """

    # create table
    table = document.add_table(rows=1, cols=6, style="Table Grid")
    table.autofit = False

    # create table rows
    write_header_row(table)
    write_data_row(table, test_data)

def write_data_row(table, test_data):
    """Write a test case result row in table

    Args:
        table (obj): Word doc table object
        test_data (list): List of interesting test data
    """

    # Iterate through test data and write table rows
    for count, test_case_entry in enumerate(test_data):
        count += 1
        row_cells = table.add_row().cells
        set_cell_widths(table, count)

        # Identify test suite sub-header row
        if re.match(TEST_SUITE_HEADER, test_case_entry[0]):
            row_cells[0].merge(row_cells[5])
            row_cells[0].text = ""
            run = row_cells[0].paragraphs[0].add_run(test_case_entry)
            run.font.name = "Arial"
            run.font.size = Pt(9)
            run.bold = True
            shade_cell(0, table, count, "CCFFFF")

        # Identify test case result data row
        elif "TN" in test_case_entry[0]:
            for counter, test_data_entry in enumerate(test_case_entry):
                row_cells[counter].text = ""
                run.font.name = "Arial"
                run.font.size = Pt(9)

                # Test Case Identifier cell
                if counter == 0:
                    test_data_entry = format_test_id(test_data_entry)
                    run = row_cells[counter].paragraphs[0].add_run(test_data_entry)
                # Test Case Pass/Fail cell
                elif counter == 4:
                    format_pass_fail(row_cells, counter, test_data_entry)
                # Cells not needing special formatting
                else:
                    run = row_cells[counter].paragraphs[0].add_run(test_data_entry)

                run.font.name = "Arial"
                run.font.size = Pt(9)

        # identify test suite sub section
        elif test_case_entry[0] == "":
            row_cells[2].merge(row_cells[5])

            run = row_cells[1].paragraphs[0].add_run(test_case_entry[1])
            run.font.name = "Arial"
            run.font.size = Pt(9)

            for cell_idx in [0, 1, 2]:
                shade_cell(cell_idx, table, count, "CCFFFF")

def shade_cell(cell_idx, table, count, shade):
    """Shade a cell in word doc table

    Args:
        cell_idx (int): Column index for cell to shade
        table (obj): Word doc table object
        count (int): Row index for cell to shade
        shade (str): hexadecimal color representation
    """
    cell = table.cell(count, cell_idx)
    color = parse_xml(
        # pylint: disable-next=consider-using-f-string
        r'<w:shd {} w:fill="{}"/>'.format(nsdecls("w"), shade)
    )
    # pylint: disable-next=protected-access
    cell._tc.get_or_add_tcPr().append(color)

def format_test_id(test_data_entry):
    """Format the test case id

    Args:
        test_data_entry (str): Text for table cell

    Returns:
        str: Formatted text for table cell
    """
    if "TN" in test_data_entry:
        test_data_entry = test_data_entry.split("TN")[1]

    return test_data_entry

def format_pass_fail(row_cells, counter, test_data_entry):
    """Format pass fail cell

    Args:
        row_cells (obj): Table cell
        counter (int): Table cell number
        test_data_entry (str): Text for table cell
    """
    run = row_cells[counter].paragraphs[0].add_run(test_data_entry.upper())

    if test_data_entry.upper() == "PASS":
        # pylint: disable-next=no-member
        run.font.highlight_color = WD_COLOR_INDEX.BRIGHT_GREEN
        run.font.bold = True
    elif test_data_entry.upper() == "FAIL":
        # pylint: disable-next=no-member
        run.font.highlight_color = WD_COLOR_INDEX.RED
        run.font.bold = True
    else:
        # pylint: disable-next=no-member
        run.font.highlight_color = WD_COLOR_INDEX.YELLOW
        run.font.bold = True

def set_cell_widths(table, row):
    """Set width of Word table cell

    Args:
        table (obj): Word doc table object
        row (int): Table row number for operations
    """

    for column, width in enumerate(WIDTHS):
        table.rows[row].cells[column].width = width

def main():
    """Main Python function"""

    print("Starting...")
    test_data = parse_csv()
    write_report(test_data)
    print("Complete")

if __name__ == "__main__":
    main()


import http
from datetime import datetime, timedelta
import unittest
from department_app import app, db
from department_app import service
from config import Config, TestingConfig
from department_app.models import Department, Employee
from department_app.tests.tests import BaseCase

class EmployeeViewTestCase(BaseCase):
    def test_employees_view(self):
        response = self.client.get('/employees')
        self.assertEqual(response.status_code, http.HTTPStatus.OK)
        response = self.client.post('/employees', data=dict(name='test'), follow_redirects=True)
        self.assertEqual(response.status_code, http.HTTPStatus.OK)

    def test_employee_view(self):
        response = self.client.get('/employees/1')
        self.assertNotEqual(response.status_code, http.HTTPStatus.OK)
        employee = Employee(name="test",
                            date_of_birth=datetime.strptime('1999-01-01', '%Y-%m-%d'),
                            salary=123,
                            department_id=1,
                            )

        db.session.add(employee)
        db.session.commit()
        response = self.client.get('/employees/1')
        self.assertEqual(response.status_code, http.HTTPStatus.OK)

    def test_employee_delete_view(self):
        response = self.client.get('/employees/1/delete')
        self.assertEqual(response.status_code, http.HTTPStatus.NOT_FOUND)
        employee = Employee(name="test",
                            date_of_birth=datetime.strptime('1999-01-01', '%Y-%m-%d'),
                            salary=123,
                            department_id=1,
                            )

        db.session.add(employee)
        db.session.commit()
        response = self.client.get('/employees/1/delete')
        self.assertEqual(response.status_code, http.HTTPStatus.FOUND)


import typing as t
from fastapi import APIRouter, Depends, Query, status
from fermerce.app.order import schemas, services
from fermerce.app.staff.dependency import (
    require_super_admin_or_admin,
    require_dispatcher,
)
from fermerce.core.enum.sort_type import SortOrder

router = APIRouter(prefix="/orders", tags=["orders"])

@router.get(
    "/",
    status_code=status.HTTP_200_OK,
    dependencies=[Depends(require_super_admin_or_admin)],
)
async def get_all_orders(
    user=None,
    filter_string: t.Optional[str] = Query(
        default="", alias="filter", description="filter all shipping_address"
    ),
    per_page: int = 10,
    page: int = 1,
    sort_by: t.Optional[SortOrder] = Query(
        default=SortOrder.desc, description="order by attribute, e.g. id"
    ),
    order_by: t.Optional[str] = Query(
        default="id", description="order by attribute, e.g. id"
    ),
):
    return await services.filter(
        filter_string=filter_string,
        page=page,
        per_page=per_page,
        sort_by=sort_by,
        order_by=order_by,
    )

@router.put(
    "/",
    status_code=status.HTTP_200_OK,
    # dependencies=[Depends(require_dispatcher)],
)
async def update_order_status(data_in: schemas.IOrderUpdate):
    return await services.update_order_status(data_in=data_in)


import h5py
import os
import numpy as np

from keras.models import Sequential
from keras.layers import Dense
from dotenv import load_dotenv

from dlgo.agent.predict import DeepLearningAgent, load_prediction_agent
from dlgo.dataprocess.parallel_processor import GoDataProcessor
from dlgo.encoders.sevenplane import SevenPlaneEncoder
from dlgo.httpfront import get_web_app
from dlgo.neuralnet import large, small
from dlgo.agent.naive import RandomBot

def end_to_end():
    go_board_rows, go_board_cols = 19, 19
    nb_classes = go_board_rows * go_board_cols
    encoder = SevenPlaneEncoder((go_board_rows, go_board_cols))
    processor = GoDataProcessor(encoder=encoder.name())

    X, y = processor.load_go_data(num_samples=10)

    input_shape = (encoder.num_planes, go_board_rows, go_board_cols)
    model = Sequential()
    network_layers = small.layers(input_shape)
    for layer in network_layers:
        model.add(layer)
    model.add(Dense(nb_classes, activation="softmax"))
    model.compile(loss="categorical_crossentropy", optimizer="adadelta", metrics=["accuracy"])
    model.summary()

    model.fit(X, y, batch_size=128, epochs=1, verbose=1)

    load_dotenv(verbose=True)
    AGENT_DIR = os.getenv("AGENT_DIR")
    end_bot_name = AGENT_DIR + "/end_to_end.h5"

    deep_learning_bot = DeepLearningAgent(model, encoder)
    deep_learning_bot.serialize(h5py.File(end_bot_name, "w"))

    model_file = h5py.File(end_bot_name, "r")
    bot_from_file = load_prediction_agent(model_file)
    bot_from_file.model.summary()

    x_pred = X[100]
    x_pred = x_pred.reshape((1, encoder.num_planes, go_board_rows, go_board_cols))

    y_pred = bot_from_file.model.predict(x_pred, verbose=1)
    print(y_pred)

    web_app = get_web_app({"predict": bot_from_file})
    web_app.run()

if __name__ == "__main__":
    end_to_end()


import pygame
from pygame.locals import *
from gui_form import Form
from gui_button import Button
from gui_textbox import TextBox
from gui_progressbar import ProgressBar
from constantes import *
from gui_label import Label

import sqlite3

class FormMostrarDatos(Form):
    def __init__(self, name, master_surface, x, y, w, h, color_background, color_border, active):
        super().__init__(name, master_surface, x, y, w, h, color_background, color_border, active)
        #self.tablero = TextBox(master=self, x=0, y=0, w=w/2, h=h/2, color_background=color_background, color_border=None, text=text, font="Arial", font_size=14, font_color=C_BLACK)
        self.lista_widget  = []
        self.boton = Button(master=self, x=40, y=300, w=90, h=80, color_background=None, color_border=None, image_background="images/gui/set_gui_01/Pixel_Border/Buttons/arrowLeft.png",
                             on_click=self.on_click_boton_active, on_click_param="form_menu_A", text="", font="Verdana", font_size=22, font_color=C_BLACK)
        self.label_score = Label(master=self, x=150, y=10, w=200, h=50, color_background=(53,57,69), color_border=None, text="Rating", font="Arial", font_size=20, font_color=C_BLACK)
        self.lista_widget.append(self.label_score)
        self.lista_widget.append(self.boton)

    def update(self, lista_eventos,keys,delta_ms,tiempo_actual, tiempo_restante):
          for aux_widget in self.lista_widget:
            aux_widget.update(lista_eventos)

    def on_show(self, datos):
        lineas = datos.splitlines()

        for i, linea in enumerate(lineas):
            label = Label(master=self, x=150, y=70*i, w=250, h=50, color_background=(53,57,69), color_border=C_BLACK, text=linea, font="Arial", font_size=16, font_color=C_BLACK)
            self.lista_widget.append(label)

    def on_click_boton_active(self, parametro):
        print("Hola")
        self.set_active(parametro)

    def draw(self):
        super().draw()
        for aux_widget in self.lista_widget:
            aux_widget.draw()

class FormMenuB(Form):
    def __init__(self,name,master_surface,x,y,w,h,color_background,color_border,active):
        super().__init__(name,master_surface,x,y,w,h,color_background,color_border,active)
        self.text_score=""
        self.boton1 = Button(master=self,x=150,y=150,w=200,h=40,color_background=None,color_border=None,image_background="images/gui/set_gui_01/Comic_Border/Buttons/Button_XL_08.png",on_click=self.on_click_boton1,on_click_param="form_game_L1",text="BACK",font="Verdana",font_size=30,font_color=C_BLACK)
        self.boton2 = Button(master=self,x=150,y=200,w=200,h=40,color_background=None,color_border=None,image_background="images/gui/set_gui_01/Comic_Border/Buttons/Button_XL_08.png",on_click=self.on_click_boton2,on_click_param="",text="AGREGAR",font="Verdana",font_size=30,font_color=C_BLACK)
        self.boton3 = Button(master=self,x=150,y=250,w=200,h=40,color_background=None,color_border=None,image_background="images/gui/set_gui_01/Comic_Border/Buttons/Button_XL_08.png",on_click=self.on_click_boton3,on_click_param="",text="CREAR",font="Verdana",font_size=30,font_color=C_BLACK)
        self.boton4 = Button(master=self,x=150,y=300,w=200,h=40,color_background=None,color_border=None,image_background="images/gui/set_gui_01/Comic_Border/Buttons/Button_XL_08.png",on_click=self.on_click_boton4,on_click_param="",text="MOSTRAR",font="Verdana",font_size=30,font_color=C_BLACK)
        self.boton5 = Button(master=self,x=150,y=350,w=200,h=40,color_background=None,color_border=None,image_background="images/gui/set_gui_01/Comic_Border/Buttons/Button_XL_08.png",on_click=self.on_click_boton5,on_click_param="",text="Borrar datos",font="Verdana",font_size=30,font_color=C_BLACK)

        self.txt1 = TextBox(master=self,x=130,y=50,w=240,h=40,color_background=None,color_border=None,image_background="images/gui/set_gui_01/Comic_Border/Buttons/Button_XL_08.png",text="Ingrese su nombre",font="Verdana",font_size=20,font_color=C_BLACK)
        #self.label = Label(master=self, x=0, y=100, w=250, h=40, color_background=(53,57,69), color_border=C_BLACK, text="Score: {0}".format(self.text_score), font="Arial", font_size=20, font_color=C_BLACK)

        self.lista_widget = [self.boton1,self.boton2,self.boton3,self.boton4,self.txt1, self.boton5]

    def on_click_boton1(self, parametro):
        self.set_active(parametro)

    def on_click_boton2(self, parametro):
        import sqlite3
        with sqlite3.connect("db/db_score.db") as conexion:
            try:
                conexion.execute("insert into score (nombre,value) values (?,?)", (self.txt1._text, self.text_score))
                conexion.commit()# Actualiza los datos realmente en la tabla
            except:
                print("Error")

    def on_click_boton3(self, parametro):

        with sqlite3.connect("db/db_score.db") as conexion:
            try:
                sentencia = ''' create  table score
                                (
                                        id integer primary key autoincrement,
                                        nombre text,
                                        value real
                                )
                            '''
                conexion.execute(sentencia)
                print("Se creo la tabla personajes")
            except sqlite3.OperationalError:
                print("La tabla ya existe")

    def on_click_boton4(self, parametro):
        with sqlite3.connect("db/db_score.db") as conexion:
            cursor=conexion.execute("SELECT * FROM score")
            datos=""
            lista_datos=[]
            for fila in cursor:
                datos += f"Nombre: {fila[1]}, Score: {fila[2]}\n"
                lista_datos.append(fila)

            form_mostrar = FormMostrarDatos("MostrarDatos", self.master_surface,300,200, self.w, self.h, (53,57,69), (53,57,69), False)
            form_mostrar.on_show(datos)  # Configurar los datos en la etiqueta del formulario mostrado
            self.set_active("MostrarDatos")  # Mostrar el formulario FormMostrarDatos

    def on_click_boton5(self, parametro):
        with sqlite3.connect("db/db_score.db") as conexion:
            cursor = conexion.cursor()
            cursor.execute("DROP TABLE IF EXISTS score;")
            conexion.commit()

    def update(self, lista_eventos,keys,delta_ms,tiempo_actual, tiempo_restante):
        self.text_score = str(self.score)
        label = Label(master=self, x=130, y=100, w=250, h=40, color_background=(53,57,69), color_border=C_BLACK, text="Score: {0}".format(self.text_score), font="Arial", font_size=30, font_color=C_BLACK)
        self.lista_widget.append(label)

        for aux_widget in self.lista_widget:
            aux_widget.update(lista_eventos)

    def draw(self):
        super().draw()
        for aux_widget in self.lista_widget:
            aux_widget.draw()


# coding: utf-8
"""
This module is based off of the below article:
http://fjavieralba.com/basic-sentiment-analysis-with-python.html

I took what Javiar was doing and ran with it. A 1000 thanks mate :-)
"""

import nltk
import yaml

def split(text):
    """
    input format: a paragraph of text
    output format: a list of lists of words.
        e.g.: [['this', 'is', 'a', 'sentence'], ['this', 'is', 'another', 'one']]
    """

    nltk_splitter = nltk.data.load('tokenizers/punkt/english.pickle')
    nltk_tokenizer = nltk.tokenize.TreebankWordTokenizer()
    return [nltk_tokenizer.tokenize(sent) for sent in nltk_splitter.tokenize(text)]

def pos_tag(sentences):
    """
    input format: list of lists of words
        e.g.: [['this', 'is', 'a', 'sentence'], ['this', 'is', 'another', 'one']]
    output format: list of lists of tagged tokens. Each tagged tokens has a
    form, a lemma, and a list of tags
        e.g: [[('this', 'this', ['DT']), ('is', 'be', ['VB']), ('a', 'a', ['DT']), ('sentence', 'sentence', ['NN'])],
                [('this', 'this', ['DT']), ('is', 'be', ['VB']), ('another', 'another', ['DT']),
                ('one', 'one', ['CARD'])]]
    """

    pos = [nltk.pos_tag(sentence) for sentence in sentences]
    # adapt format
    pos = [[(word, word, [postag]) for (word, postag) in sentence] for sentence in pos]
    return pos

class DictionaryTagger(object):
    """
        Class to tag sentences
    """
    def __init__(self, dictionary_paths):
        """
            Opens YAML files and instantiates object of type DictionaryTagger
        :param dictionary_paths:
        """
        files = [open(path, 'r') for path in dictionary_paths]
        dictionaries = [yaml.load(dict_file) for dict_file in files]
        map(lambda x: x.close(), files)
        self.dictionary = {}
        self.max_key_size = 0
        for curr_dict in dictionaries:
            for key in curr_dict:
                if key in self.dictionary:
                    self.dictionary[key].extend(curr_dict[key])
                else:
                    self.dictionary[key] = curr_dict[key]
                    self.max_key_size = max(self.max_key_size, len(key))

    def tag(self, postagged_sentences):
        """
            Tags sentences with internal tags for positive and negative
        :param postagged_sentences:
        :return: tagged sentences
        """
        return [self.tag_sentence(sentence) for sentence in postagged_sentences]

    def tag_sentence(self, sentence, tag_with_lemmas=False):
        """
        the result is only one tagging of all the possible ones.
        The resulting tagging is determined by these two priority rules:
            - longest matches have higher priority
            - search is made from left to right
        """

        tag_sentence = []
        N = len(sentence)

        if self.max_key_size == 0:
            self.max_key_size = N
        i = 0

        while i < N:
            j = min(i + self.max_key_size, N)  # avoid overflow
            tagged = False

            while j > i:
                expression_form = ' '.join([word[0] for word in sentence[i:j]]).lower()
                expression_lemma = ' '.join([word[1] for word in sentence[i:j]]).lower()

                if tag_with_lemmas:
                    literal = expression_lemma
                else:
                    literal = expression_form
                if literal in self.dictionary:
                    # self.logger.debug("found: %s" % literal)
                    is_single_token = j - i == 1
                    original_position = i
                    i = j
                    taggings = [tag for tag in self.dictionary[literal]]
                    tagged_expression = (expression_form, expression_lemma, taggings)
                    if is_single_token:  # if the tagged literal is a single token, conserve its previous taggings:
                        original_token_tagging = sentence[original_position][2]
                        tagged_expression[2].extend(original_token_tagging)
                    tag_sentence.append(tagged_expression)
                    tagged = True
                else:
                    j = j - 1
            if not tagged:
                tag_sentence.append(sentence[i])
                i += 1
        return tag_sentence

def __value_of(sentiment):
    """
        Internal method to quantify polarity
    :param sentiment:
    :return:
    """
    if sentiment == 'positive': return 1
    if sentiment == 'negative': return -1
    return 0

def run_sentiment_analysis(network, add_polarity=True, flag_changes=True):
    """
    Runs Sentiment analysis on a network and returns an analyzed and scored network.
    :param network:
    :param add_polarity:
    :param flag_changes:
    :return: scored network
    """
    score = []
    data = [col for col in network.columns if ('msrc' or 'Data') in col]
    for chunk in network[data]:
        split_sentences = split(chunk)
        pos_sentences = pos_tag(split_sentences)
        tagged_sentences = DictionaryTagger(
        ['/Users/markrice/Documents/GitHub/RGH_NLP/DataTools/positive.yml',
        '/Users/markrice/Documents/GitHub/RGH_NLP/DataTools/negative.yml',
        './inv.yml']).tag(pos_sentences)
        relationship_score = sentiment_score(tagged_sentences)

        score.append(relationship_score)

    network['Sentiment Score'] = score

    if add_polarity:
        polarity = []
        for sc in score:
            if sc > 0:
                polarity.append('positive')
            elif sc < 0:
                polarity.append('negative')
            else:
                polarity.append('unknown')

        network['Polarity'] = polarity

    if flag_changes:
        network.loc[network['Polarity'] != network['PS_Polarity'], 'Polarity_Changed'] = True
        network.loc[network['Polarity'] == network['PS_Polarity'], 'Polarity_Changed'] = False

    return network

def sentence_score(sentence_tokens, previous_token, acum_score):
    """
        scores sentences thats been tagged.
    :param sentence_tokens:
    :param previous_token:
    :param acum_score:
    :return:
    """
    if not sentence_tokens:
        return acum_score
    else:
        current_token = sentence_tokens[0]
        tags = current_token[2]
        token_score = sum([__value_of(tag) for tag in tags])
        if previous_token is not None:
            previous_tags = previous_token[2]
            # if 'inc' in previous_tags:
            #    token_score *= 2.0
            # elif 'dec' in previous_tags:
            #    token_score /= 2.0
            if 'inv' in previous_tags:
                token_score *= -1.0
        return sentence_score(sentence_tokens[1:], current_token, acum_score + token_score)

def sentiment_score(review):
    """
    Returns cumulative score of scored sentences
    :param review:
    :return:
    """
    return sum([sentence_score(sentence, None, 0.0) for sentence in review])

def weighted_sentence_score(sentence_tokens):
    """
    Weighted score...
    :param sentence_tokens:
    :return: Totally arbitrary weight based on hokey pokey
    """
    tags = []
    for x in range(len(sentence_tokens)):
        tags.append(sentence_tokens[x][2])
    return sum([__value_of(tag[0]) / len(sentence_tokens) for tag in tags])

def weighted_sentiment_score(review, matrix_flag):
    """
        Totally arbitrary weight based on hokey pokey
    :param review:
    :param matrix_flag:
    :return: Totally arbitrary weight based on hokey pokey
    """
    if matrix_flag:
        return [weighted_sentence_score(sentence) for sentence in review]
    else:
        return sum([weighted_sentence_score(sentence) for sentence in review])


import pytest

from django.core.exceptions import ValidationError

from mce_django_app.models import common as models

CURRENT_MODEL = models.Region

def test_create_success(mce_app_provider_aws):
    """create simple AWS ResourceType"""

    CURRENT_MODEL.objects.create(
        name="us-east-1",
        display_name="US East 1",
        provider=mce_app_provider_aws
    )

def test_error_duplicate(mce_app_provider_aws):
    """check error if object duplicate"""

    CURRENT_MODEL.objects.create(
        name="us-east-1",
        display_name="US East 1",
        provider=mce_app_provider_aws
    )

    with pytest.raises(ValidationError) as excinfo:
        CURRENT_MODEL.objects.create(
            name="us-east-1",
            display_name="US East 1",
            provider=mce_app_provider_aws
        )

    assert excinfo.value.message_dict == {
        '__all__': ['Region with this Provider and Name already exists.']
    }

def test_error_max_length(mce_app_provider_aws):
    """Test max_length constraints"""

    with pytest.raises(ValidationError) as excinfo:
        CURRENT_MODEL.objects.create(
            name="x" * 256,
            display_name="y" * 256,
            provider=mce_app_provider_aws
        )
    assert excinfo.value.message_dict == {
        'name': ['Ensure this value has at most 255 characters (it has 256).'],
        'display_name': ['Ensure this value has at most 255 characters (it has 256).'],
    }

def test_error_null_and_blank_value(mce_app_provider_aws):
    """test null and blank value constraints"""

    with pytest.raises(ValidationError) as excinfo:
        CURRENT_MODEL.objects.create(
            name="",
            display_name="",
            provider=mce_app_provider_aws
        )
    assert excinfo.value.message_dict == {
        'name': ['This field cannot be blank.'],
        'display_name': ['This field cannot be blank.'],
    }

    with pytest.raises(ValidationError) as excinfo:
        CURRENT_MODEL.objects.create(
            name=None,
            display_name=None,
            provider=None,
        )
    assert excinfo.value.message_dict == {
        'name': ['This field cannot be null.'],
        'display_name': ['This field cannot be null.'],
        'provider': ['This field cannot be null.'],
    }