from datetime import datetime, timedelta
from unittest import TestCase

from bitcallback.commands import CallbackData, SubscriptionData
from bitcallback.models import Callback, Subscription, SubscriptionState

from bitcallback.bitmon_task import SubscriptionManager
from bitcallback.database import make_session_scope

from .database import create_memory_db

class MockTransactionMonitor(object):

    def __init__(self):
        self.monitored = set()
        self.added = []
        self.deleted = []

    def get_confirmed(self):
        return []

    def add_addr(self, addr):
        self.monitored.add(addr)

    def del_addr(self, addr):
        self.monitored.remove(addr)

    def __len__(self):
        return len(self.monitored)

class TestSubscriptionManager(TestCase):

    def setUp(self):
        self.db_session = create_memory_db()

        with make_session_scope(self.db_session) as session:
            self.subs1 = Subscription(
                address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7',
                callback_url='http://localhost:9779')
            self.subs2 = Subscription(
                address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7',
                callback_url='http://localhost:9779',
                expiration=datetime.utcnow()-timedelta(days=300))
            self.subs3 = Subscription(
                address='n2SjFgAhHAv8PcTuq5x2e9sugcXDpMTzX7',
                callback_url='http://localhost:9779',
                state=SubscriptionState.canceled)
            self.subs4 = Subscription(
                address='n2SjFgAhHAv8PcTuq5x2e9suffXspmtztt',
                callback_url='http://localhost:9779')
            session.add(self.subs1)
            session.add(self.subs2)
            session.add(self.subs3)
            session.add(self.subs4)

        # Create equivalent SubscriptionData commands
        self.com1 = self.subs1.to_subscription_data()
        self.com2 = self.subs2.to_subscription_data()
        self.com3 = self.subs3.to_subscription_data()
        self.com4 = self.subs4.to_subscription_data()

    def tearDown(self):
        #self.subscription_manager.close()
        pass

    def test_load_active_subscription(self):
        """Test active Subscriptions are loaded during initialization"""

        # Try Database with no subscriptions
        monitor = MockTransactionMonitor()
        subscription_manager = SubscriptionManager(monitor, self.db_session)

        self.assertEqual(len(subscription_manager), 3)
        self.assertEqual(len(monitor), 2)

    def test_expired_are_discarded(self):
        """Test expired subscription are discarded and their state updated in DB"""
        monitor = MockTransactionMonitor()
        subscription_manager = SubscriptionManager(monitor, self.db_session)

        self.assertEqual(len(subscription_manager), 3)
        self.assertEqual(len(monitor), 2)

        # Calling poll_bitcoin should remove expired subscriptions
        subscription_manager.poll_bitcoin()

        with make_session_scope(self.db_session) as session:
            expired_subs = session.query(Subscription).filter_by(
                                    id=self.subs2.id).first()

        self.assertEqual(expired_subs.state, SubscriptionState.expired)
        self.assertEqual(len(subscription_manager), 2)
        self.assertEqual(len(monitor), 2)

    def test_cancel_subscription(self):
        """Test subscription cancelation"""
        monitor = MockTransactionMonitor()
        subscription_manager = SubscriptionManager(monitor, self.db_session)

        # Initial subscriptions loaded from DB during initialization
        self.assertEqual(len(subscription_manager), 3)
        self.assertEqual(len(monitor), 2)

        # Cancel
        subscription_manager.cancel_subscription(self.subs1.id)
        self.assertEqual(len(subscription_manager), 2)
        self.assertEqual(len(monitor), 2) # 2 because one address is duplicated

        subscription_manager.cancel_subscription(self.subs2.id)
        self.assertEqual(len(subscription_manager), 1)
        self.assertEqual(len(monitor), 1)

        subscription_manager.cancel_subscription(self.subs4.id)
        self.assertEqual(len(subscription_manager), 0)
        self.assertEqual(len(monitor), 0)

    def test_add_subscription(self):
        """Test monitoring new subscriptions"""
        monitor = MockTransactionMonitor()
        subscription_manager = SubscriptionManager(monitor,
                                            self.db_session,
                                            db_reload=False)

        subscription_manager.add_subscription(self.com1)
        self.assertEqual(len(subscription_manager), 1)
        self.assertEqual(len(monitor), 1)

        subscription_manager.add_subscription(self.com2)
        self.assertEqual(len(subscription_manager), 2)
        self.assertEqual(len(monitor), 1)

        subscription_manager.add_subscription(self.com4)
        self.assertEqual(len(subscription_manager), 3)
        self.assertEqual(len(monitor), 2)

    def test_db_reload_subscriptions(self):
        """Test RELOAD_SUBSCRIPTIONS configuration option"""
        # Nothing loaded with reload_subscriptions disable
        monitor = MockTransactionMonitor()
        subs_manager = SubscriptionManager(monitor, self.db_session, False)
        self.assertEqual(len(subs_manager), 0)
        self.assertEqual(len(monitor), 0)

        # ALL active subscriptions loaded with reload_subscriptions enabled
        monitor = MockTransactionMonitor()
        subs_manager = SubscriptionManager(monitor, self.db_session, True)
        self.assertEqual(len(subs_manager), 3)
        self.assertEqual(len(monitor), 2)

    def test_set_transaction_monitor(self):
        """Test subscriptions are updated when transaction monitor is changed"""
        monitor = MockTransactionMonitor()
        subscription_manager = SubscriptionManager(monitor, self.db_session)


import os
import io

from setuptools import setup, find_packages

def read(fname):
    with io.open(os.path.join(os.path.dirname(__file__), fname), encoding="utf-8") as f:
        return f.read()

setup(
    name="ml",
    version='1.0',
    packages=find_packages("."),
    package_dir={"": "."},
    python_requires=">=3.6",
    install_requires=[
        "torch>=1.9",
        "mlflow",
    ],
    include_package_data=True,
    license="MIT",
)


import argparse
import random
from functools import partial
from pprint import pprint

import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision.models import resnet18, resnet34, resnet50, resnet101, resnet152
import wandb

from datasets import cifar10, cifar100, imagenet, fake_imagenet
from pytorch_resnet_cifar10.resnet import resnet20, resnet32, resnet44, resnet56, resnet110, resnet1202
from simulator import Simulator
from smega2 import SMEGA2
from trainer import Trainer
from worker_selection import GammaRandomWorkerSelection

torch.backends.cudnn.benchmark = True

DATASETS = {
    'cifar10': cifar10,
    'cifar100': cifar100,
    'imagenet': imagenet,
    'fake-imagenet': fake_imagenet,
}

CIFAR_MODELS = {
    'resnet20': resnet20,
    'resnet32': resnet32,
    'resnet44': resnet44,
    'resnet56': resnet56(),
    'resnet110': resnet110,
    'resnet1202': resnet1202,
}

IMAGENET_MODELS = {
    'resnet18': resnet18,
    'resnet34': resnet34,
    'resnet50': resnet50,
    'resnet101': resnet101,
    'resnet152': resnet152,
}

def main():
    # Training settings
    parser = argparse.ArgumentParser(description="PyTorch Environment")

    train_parser = parser.add_argument_group("Train Parameters")
    train_parser.add_argument("--dataset", default="cifar10", metavar="DS",
                              help="train dataset: %(choices)s (default: cifar10)",
                              choices=DATASETS.keys())
    train_parser.add_argument("--model", default="resnet20", metavar="MDL",
                              help="model to train: %(choices)s (default: resnet20)",
                              choices=CIFAR_MODELS.keys() | IMAGENET_MODELS.keys())
    train_parser.add_argument("--epochs", type=int, default=160, metavar="E",
                              help="number of epochs to train (default: 10)")
    train_parser.add_argument("--batch-size", type=int, default=128, metavar="B",
                              help="input batch size for training (default: 128)")
    train_parser.add_argument("--test-batch-size", type=int, default=128, metavar="BT",
                              help="input batch size for testing (default: 128)")
    train_parser.add_argument("--lr_decay", type=float, default=0.1, metavar="LD",
                              help="learning rate decay rate (default: 0.1)")
    train_parser.add_argument("--schedule", type=int, nargs="*", default=[80, 120],
                              help="learning rate is decayed at these epochs (default: [80, 120])")
    train_parser.add_argument("--warmup-epochs", type=int, default=5, metavar="WE",
                              help="number of warmup epochs (default: 5)")
    train_parser.add_argument("--no-cuda", action="store_true", default=False,
                              help="disables CUDA training (default: False)")
    train_parser.add_argument("--seed", type=int, default=None, metavar="S",
                              help="random seed (default: None)")

    simulator_parser = parser.add_argument_group("Simulator Parameters")
    simulator_parser.add_argument("--sim-size", type=int, default=8, metavar="N",
                                  help="number of workers to simulate (default: 8)")
    simulator_parser.add_argument("--sim-gamma-shape", type=float, default=100, metavar="GSH",
                                  help="gamma shape parameter (default: 100)")
    simulator_parser.add_argument("--sim-gamma-scale", type=float, default=None, metavar="GSC",
                                  help="gamma scale parameter (default: B / GSH)")

    optimizer_parser = parser.add_argument_group("Optimizer Parameters")
    optimizer_parser.add_argument("--lr", type=float, default=0.1, metavar="LR", help="learning rate (default: 0.1)")
    optimizer_parser.add_argument("--momentum", type=float, default=0.9, metavar="M",
                                  help="SGD momentum (default: 0.9)")
    optimizer_parser.add_argument("--weight-decay", type=float, default=1e-4, metavar="WD",
                                  help="SGD weight decay (default: 1e-4)")
    optimizer_parser.add_argument("--naive", default=False, action="store_true",
                                  help="disables SMEGA2 advanced estimation")

    args = parser.parse_args()

    args.cuda = not args.no_cuda and torch.cuda.is_available()

    if args.seed:
        torch.manual_seed(args.seed)
        if args.cuda:
            torch.cuda.manual_seed(args.seed)
    random.seed(torch.initial_seed())

    if not args.sim_gamma_scale:
        args.sim_gamma_scale = args.batch_size / args.sim_gamma_shape

    pprint(vars(args))

    train_set, test_set = DATASETS[args.dataset]()
    data_loader = partial(DataLoader, num_workers=2, pin_memory=True)
    train_loader = data_loader(train_set, shuffle=True, batch_size=args.batch_size)
    test_loader = data_loader(test_set, shuffle=False, batch_size=args.test_batch_size)

    available_models = CIFAR_MODELS if 'cifar' in args.dataset else IMAGENET_MODELS
    try:
        model = available_models[args.model]()
    except KeyError as e:
        raise ValueError(f"{args.model} is not supported for {args.dataset}."
                         f" Please use one of {list(available_models.keys())}") from e
    if args.cuda:
        model = model.cuda()

    loss_fn = nn.CrossEntropyLoss()

    optimizer = SMEGA2(model.parameters(), lr=args.lr, momentum_cycle=args.sim_size, advanced=not args.naive,
                       momentum=args.momentum, weight_decay=args.weight_decay)
    assert len(optimizer.param_groups) == 1

    worker_selector = GammaRandomWorkerSelection(args.sim_size, args.sim_gamma_shape, args.sim_gamma_scale)

    simulator = Simulator(args.sim_size, worker_selector, model)

    trainer = Trainer(
        model,
        simulator,
        train_loader,
        test_loader,
        optimizer,
        loss_fn,
        args.epochs,
        args.warmup_epochs,
        args.schedule,
        args.lr_decay,
        args.cuda,
    )

    wandb.init(config=args, anonymous="allow")
    trainer.run()
    wandb.finish()

if __name__ == "__main__":
    main()


"""
After embedding with a fine-tuned embedder (scripts/run_embedding.fine_tuned.py), scale and PCA the anndatas.
Already divided into train-smaller, validation, test.

This reads from config.paths.anndatas_dir and writes to config.paths.scaled_anndatas_dir.

Usage examples:
    > python scripts/scale_anndatas_created_with_finetuned_embedding.py --help;
    > python scripts/scale_anndatas_created_with_finetuned_embedding.py;               # all folds
    > python scripts/scale_anndatas_created_with_finetuned_embedding.py --fold_id 0;   # single fold
    > python scripts/scale_anndatas_created_with_finetuned_embedding.py --fold_id 0 --fold_id 1;
"""

from typing import List
import scanpy as sc
import gc
import joblib

import choosegpu

import malid.external.genetools_scanpy_helpers
from malid import config, cli_utils
from malid.datamodels import GeneLocus
from pathlib import Path
import anndata

choosegpu.configure_gpu(enable=False)
import logging

logger = logging.getLogger(__name__)
import click

n_components = 50  # Number of principal components to compute

def load_original_anndata(fold_id: int, fold_label: str, input_dir: Path):
    adata = sc.read(input_dir / f"fold.{fold_id}.{fold_label}.h5ad")
    # Sanity check:
    if adata.obs_names.duplicated().any():
        raise ValueError("Obs names duplicated")
    return adata

def write_part_to_disk(
    fold_id: int, adata_part: anndata.AnnData, part_name: str, output_dir: Path
):
    fname_out = output_dir / f"fold.{fold_id}.{part_name}.h5ad"
    adata_part.write(fname_out)
    print(fname_out, adata_part.shape)

def run_on_single_locus(gene_locus: GeneLocus, fold_ids: List[int]):
    GeneLocus.validate_single_value(gene_locus)
    input_dir = config.paths.anndatas_dir / gene_locus.name
    output_dir = config.paths.scaled_anndatas_dir / gene_locus.name
    output_dir.mkdir(exist_ok=True, parents=True)

    for fold_id in fold_ids:
        # Fit scaling and PCA on train-smaller anndata and apply to validation and test anndata.
        # At this point these are all unscaled.

        # Run scale and PCA operations on CPU because matrices are too big for GPU.
        # Do this on one at a time to conserve RAM.

        logger.info(f"Processing: fold {fold_id}, train_smaller")
        adata_train_smaller = load_original_anndata(
            fold_id=fold_id, fold_label="train_smaller", input_dir=input_dir
        )
        # Sanity checks:
        if adata_train_smaller.uns["embedded"] != "unirep_fine_tuned":
            raise ValueError(
                f"Expected anndata to be embedded with 'unirep_fine_tuned', got {adata_train_smaller.uns['embedded']}"
            )
        if adata_train_smaller.uns["embedded_fine_tuned_on_fold_id"] != fold_id:
            raise ValueError(
                f"Expected anndata to be embedded with fold_id {fold_id}, got {adata_train_smaller.uns['embedded_fine_tuned_on_fold_id']}"
            )

        # Scale inplace and set raw
        (
            adata_train_smaller,
            scale_transformer,
        ) = malid.external.genetools_scanpy_helpers.scale_anndata(
            adata_train_smaller, scale_transformer=None, inplace=True, set_raw=True
        )
        # PCA inplace
        (
            adata_train_smaller,
            pca_transformer,
        ) = malid.external.genetools_scanpy_helpers.pca_anndata(
            adata_train_smaller,
            pca_transformer=None,
            n_components=n_components,
            inplace=True,
        )

        write_part_to_disk(
            fold_id=fold_id,
            adata_part=adata_train_smaller,
            part_name="train_smaller",
            output_dir=output_dir,
        )
        del adata_train_smaller
        gc.collect()

        # Save the transformations so we can apply to future new test sets
        joblib.dump(
            {"scale": scale_transformer, "pca": pca_transformer},
            output_dir / f"fold.{fold_id}.train_smaller.transformations.joblib",
        )

        # Apply those same transformations to validation and test sets:
        for fold_label in ["validation", "test"]:
            if fold_id == -1 and fold_label == "test":
                # skip: global fold does not have a test set
                continue
            logger.info(f"Processing: fold {fold_id}, {fold_label}")
            adata_dependent = load_original_anndata(
                fold_id=fold_id, fold_label=fold_label, input_dir=input_dir
            )

            # Scale inplace - using existing transformer - and set raw
            adata_dependent, _ = malid.external.genetools_scanpy_helpers.scale_anndata(
                adata_dependent,
                scale_transformer=scale_transformer,
                inplace=True,
                set_raw=True,
            )
            # PCA inplace - no more copying - using existing transformer
            adata_dependent, _ = malid.external.genetools_scanpy_helpers.pca_anndata(
                adata_dependent,
                pca_transformer=pca_transformer,
                inplace=True,
            )

            write_part_to_disk(
                fold_id=fold_id,
                adata_part=adata_dependent,
                part_name=fold_label,
                output_dir=output_dir,
            )
            del adata_dependent
            gc.collect()

@click.command()
@cli_utils.accepts_gene_loci
@cli_utils.accepts_fold_ids
def run(
    gene_locus: List[GeneLocus],
    fold_ids: List[int],
):
    gene_locus: GeneLocus = GeneLocus.combine_flags_list_into_single_multiflag_value(
        gene_locus
    )
    click.echo(f"Selected gene_locus: {gene_locus}")
    click.echo(f"Selected fold_ids: {fold_ids}")
    GeneLocus.validate(gene_locus)  # packed (by OR) into single item here
    for single_gene_locus in gene_locus:
        click.echo(f"Running on {single_gene_locus}...")
        run_on_single_locus(gene_locus=single_gene_locus, fold_ids=fold_ids)

if __name__ == "__main__":
    run()


from kivy.animation import Animation
from kivy.clock import Clock
from kivy.core.window import Window
from kivy.properties import StringProperty
from kivy.uix.bubble import Bubble

class InfoBubble(Bubble):
    '''Bubble to be used to display short Help Information'''

    message = StringProperty('')
    '''Message to be displayed
       :data:`message` is a :class:`~kivy.properties.StringProperty`
    '''

    def show(self, pos, duration, width=None):
        '''Animate the bubble into position'''
        if width:
            self.width = width
        # wait for the bubble to adjust it's size according to text then animate
        Clock.schedule_once(lambda dt: self._show(pos, duration))

    def _show(self, pos, duration):
        '''To show Infobubble at pos with Animation of duration.
        '''
        def on_stop(*l):
            if duration:
                Clock.schedule_once(self.hide, duration + .5)

        self.opacity = 0
        arrow_pos = self.arrow_pos
        if arrow_pos[0] in ('l', 'r'):
            pos = pos[0], pos[1] - (self.height / 2)
        else:
            pos = pos[0] - (self.width / 2), pos[1]

        self.limit_to = Window
        self.pos = pos
        Window.add_widget(self)

        anim = Animation(opacity=1, d=0.75)
        anim.bind(on_complete=on_stop)
        anim.cancel_all(self)
        anim.start(self)

    def hide(self, *dt):
        ''' Auto fade out the Bubble
        '''
        def on_stop(*l):
            Window.remove_widget(self)
        anim = Animation(opacity=0, d=0.75)
        anim.bind(on_complete=on_stop)
        anim.cancel_all(self)
        anim.start(self)


from base import BaseDataLoader
import numpy as np
import os
import torch
from torch.utils import data

class MnistDataLoader(BaseDataLoader):
    """
    MNIST data loading demo using BaseDataLoader
    """
    def __init__(self, data_dir, batch_size, shuffle=True, validation_split=0.0, num_workers=1, training=True):
        trsfm = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,))
        ])
        self.data_dir = data_dir
        self.dataset = datasets.MNIST(self.data_dir, train=training, download=True, transform=trsfm)
        super().__init__(self.dataset, batch_size, shuffle, validation_split, num_workers)

class CatsVsDogsDataLoader(BaseDataLoader):

    def __init__(self, data_dir, batch_size, shuffle=True, validation_split=0.0, num_workers=1, training=True):
        self.data_dir = data_dir
        raw_data = np.load(os.path.join(data_dir, "processed_img_data.npy"), allow_pickle=True)
        X_data = torch.tensor([i[0] for i in raw_data]).view(-1, 1, 50, 50)
        X_data = X_data/255.0
        y_data = torch.tensor([np.argmax(i[1]) for i in raw_data], dtype=torch.long)

        self.dataset = data.TensorDataset(X_data, y_data)

        super().__init__(self.dataset, batch_size, shuffle, validation_split, num_workers)


import face_recognition
import cv2
import os
import pickle
print("cv2 version: ", cv2.__version__)
print("face_recognition version: ", face_recognition.__version__)

Encodings=[]
Names=[]

with open('train.pkl', 'rb') as f:
    Names = pickle.load(f)
    Encodings = pickle.load(f)

# cam = cv2.VideoCapture(0)
cam = cv2.VideoCapture('/home/fred/Videos/Familie_openCV_VGA.mp4')

# Check if camera opened successfully
if (cam.isOpened()== False):
    print("Error opening video stream or file!")

font=cv2.FONT_HERSHEY_SIMPLEX

while (cam.isOpened()):
    ret, frame=cam.read()
    # frameSmall = cv2.resize(frame, (0,0), fx=.33, fy=.33)
    # frameRGB = cv2.cvtColor(frameSmall, cv2.COLOR_BGR2RGB)
    frameRGB = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    facePositions = face_recognition.face_locations(frameRGB,model='hog')
    # facePositions = face_recognition.face_locations(frameRGB,model='cnn')
    allEncodings = face_recognition.face_encodings(frameRGB, facePositions)
    for (top, right, bottom, left),face_encoding in zip(facePositions, allEncodings):
        name = 'Unknown Person'
        matches = face_recognition.compare_faces(Encodings, face_encoding)
        if True in matches:
            first_match_index = matches.index(True)
            name = Names[first_match_index]

        #top = top*3
        #right = right*3
        #bottom = bottom*3
        #left=left*3

        cv2.rectangle(frame, (left, top),(right, bottom), (0,0,255), 2)
        cv2.putText(frame, name, (left, top-6), font, 1, (0,0,255))

    cv2.imshow('Picture', frame)
    cv2.moveWindow('Picture', 10, 10)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
cam.release()
cv2.destroyAllWindows


import jieba
from wordcloud import WordCloud
from collections import Counter
import csv

# path 文件路径
# 统计词频
def count_word_frequency(path):
    fn = open(path,"r",encoding='utf-8')
    string_data = fn.read()
    fn.close()
    # 使用jieba分词将文本切分成词语
    words = jieba.cut(string_data)
    # 使用Counter计算词频
    word_count = Counter(words)

    return word_count

# 保存词频统计结果为csv文件
def save_word_frequency_to_csv(word_count, filename):
    # 将词频统计结果保存为CSV文件
    with open(filename, 'w', newline='', encoding='utf-8-sig') as file:
        writer = csv.writer(file)
        writer.writerow(['Word', 'Frequency'])  # 写入表头
        writer.writerows(word_count.items())  # 写入词频统计结果

def generate_wordcloud(text):
    new_txt = " ".join(text)
    wc = WordCloud(
        background_color='white',
        font_path='C:/Windows/Fonts/FZSTK.TTF',
        max_words=200,
        max_font_size=100,
        scale=32,
        # stopwords 停用词 生成图云时会过滤这些词
        stopwords={'的', '，', '和', '是', '随着', '对于', '对', '等', '能', '都', '。',
                    ' ', ',', '、', ',', '中', '在', '了', '通常', '如果', '我们', '需要'}
    )
    wc.generate(new_txt)
    wc.to_file("2022231550黄哲宇a.png")

# 统计词频
word_count = count_word_frequency("a.txt")
# 保存为CSV文件
filename = '2022231550黄哲宇a.csv'
save_word_frequency_to_csv(word_count, filename)
# 生成词云图
generate_wordcloud(word_count)


import configparser
import io
import pickle
from pathlib import Path
from typing import Tuple

import pandas as pd
from sqlalchemy.orm import Session

from reia.datamodel.asset import Asset
from reia.db.crud import (read_asset_collection, read_fragility_model,
                          read_taxonomymap, read_vulnerability_model)
from reia.io import (ASSETS_COLS_MAPPING, LOSSCATEGORY_FRAGILITY_MAPPING,
                     LOSSCATEGORY_VULNERABILITY_MAPPING)
from reia.utils import create_file_pointer

def create_fragility_input(
    fragility_model_oid: int,
    session: Session,
    template_name: Path = Path('reia/templates/fragility.xml')) \
        -> io.StringIO:
    """
    Create an in memory fragility xml file for OpenQuake.

    :param fragility_model_oid: oid of the VulnerabilityModel to be used.
    :param session: SQLAlchemy database session.
    :param template_name: Template to be used for the fragility file.
    :returns: Filepointer for exposure xml and one for csv list of assets.
    """

    fragility_model = read_fragility_model(
        fragility_model_oid, session)

    data = fragility_model._asdict()
    data['_type'] = next((k for k, v in
                          LOSSCATEGORY_FRAGILITY_MAPPING.items(
                          ) if k == data['_type'].value))
    data['fragilityfunctions'] = []

    for vf in fragility_model.fragilityfunctions:
        vf_dict = vf._asdict()
        vf_dict['limitstates'] = [lr._asdict() for lr in vf.limitstates]
        data['fragilityfunctions'].append(vf_dict)

    return create_file_pointer(template_name, data=data)

def create_taxonomymap_input(
    oid: int,
    session: Session,
    name: str = 'taxonomy_mapping.csv') \
        -> io.StringIO:
    """
    Create an in memory vulnerability xml file for OpenQuake.

    :param vulnerability_model_oid: oid of the VulnerabilityModel to be used.
    :param session: SQLAlchemy database session.
    :param template_name: Template to be used for the vulnerability file.
    :returns: Filepointer for exposure xml and one for csv list of assets.
    """

    taxonomy_map = read_taxonomymap(oid, session)

    mappings = taxonomy_map.mappings
    mappings = pd.DataFrame([vars(s) for s in mappings], columns=[
        'taxonomy', 'conversion', 'weight'])

    taxonomy_map_csv = io.StringIO()
    mappings.to_csv(taxonomy_map_csv, index=False)
    taxonomy_map_csv.seek(0)
    taxonomy_map_csv.name = name

    return taxonomy_map_csv

def create_vulnerability_input(
    vulnerability_model_oid: int,
    session: Session,
    template_name: Path = Path('reia/templates/vulnerability.xml')) \
        -> io.StringIO:
    """
    Create an in memory vulnerability xml file for OpenQuake.

    :param vulnerability_model_oid: oid of the VulnerabilityModel to be used.
    :param session: SQLAlchemy database session.
    :param template_name: Template to be used for the vulnerability file.
    :returns: Filepointer for exposure xml and one for csv list of assets.
    """

    vulnerability_model = read_vulnerability_model(
        vulnerability_model_oid, session)

    data = vulnerability_model._asdict()
    data['_type'] = next((k for k, v in
                          LOSSCATEGORY_VULNERABILITY_MAPPING.items(
                          ) if k == data['_type'].value))
    data['vulnerabilityfunctions'] = []

    for vf in vulnerability_model.vulnerabilityfunctions:
        vf_dict = vf._asdict()
        vf_dict['lossratios'] = [lr._asdict() for lr in vf.lossratios]
        data['vulnerabilityfunctions'].append(vf_dict)

    return create_file_pointer(template_name, data=data)

def create_exposure_input(
    asset_collection_oid: int,
    session: Session,
    template_name: Path = Path('reia/templates/exposure.xml'),
    assets_csv_name: Path = Path('exposure_assets.csv')) \
        -> Tuple[io.StringIO, io.StringIO]:
    """
    Creates in-memory exposure input files for OpenQuake.

    :param asset_collection_oid: oid of the ExposureModel to be used.
    :param session: SQLAlchemy database session.
    :param template_name: Template to be used for the exposure file.
    :returns: Filepointer for exposure xml and one for csv list of assets.
    """

    asset_collection = read_asset_collection(asset_collection_oid, session)
    data = asset_collection._asdict()

    data['assets_csv_name'] = assets_csv_name.name
    data['costtypes'] = [c._asdict() for c in asset_collection.costtypes]
    # first asset's tag types must be the same as all other's
    data['tagnames'] = [agg.type for agg in
                        asset_collection.assets[0].aggregationtags]

    exposure_xml = create_file_pointer(template_name, data=data)

    exposure_df = assets_to_dataframe(asset_collection.assets)

    exposure_csv = io.StringIO()
    exposure_df.to_csv(exposure_csv)
    exposure_csv.seek(0)
    exposure_csv.name = assets_csv_name.name

    return (exposure_xml, exposure_csv)

def assets_to_dataframe(assets: list[Asset]) -> pd.DataFrame:
    """
    Parses a list of Asset objects to a DataFrame.
    """

    assets_df = pd.DataFrame([x._asdict() for x in assets]).set_index('_oid')

    sites_df = pd.DataFrame([x.site._asdict() for x in assets])[
        ['longitude', 'latitude']]

    aggregationtags_df = pd.DataFrame(map(
        lambda asset: {tag.type: tag.name for tag in asset.aggregationtags},
        assets))

    result_df = pd.concat([assets_df,
                           sites_df.set_index(assets_df.index),
                           aggregationtags_df.set_index(assets_df.index)],
                          axis=1)

    selector = {**{'longitude': 'lon', 'latitude': 'lat'},
                **{v: k for k, v in ASSETS_COLS_MAPPING.items()},
                **{k: k for k in aggregationtags_df.columns}}

    result_df = result_df.rename(columns=selector)[[*selector.values()]] \
        .dropna(axis=1, how='all') \
        .fillna(0)
    result_df.index.name = 'id'

    return result_df

def assemble_calculation_input(job: configparser.ConfigParser,
                               session: Session) -> list[io.StringIO]:
    # create deep copy of configparser
    tmp = pickle.dumps(job)
    working_job = pickle.loads(tmp)

    calculation_files = []

    exposure_xml, exposure_csv = create_exposure_input(
        working_job['exposure']['exposure_file'], session)
    exposure_xml.name = 'exposure.xml'
    working_job['exposure']['exposure_file'] = exposure_xml.name

    calculation_files.extend([exposure_xml, exposure_csv])

    if 'vulnerability' in working_job.keys():
        for k, v in working_job['vulnerability'].items():
            if k == 'taxonomy_mapping_csv':
                file = create_taxonomymap_input(v, session)
                file.name = "{}.csv".format(k.replace('_file', ''))
            else:
                file = create_vulnerability_input(v, session)
                file.name = "{}.xml".format(k.replace('_file', ''))
            working_job['vulnerability'][k] = file.name
            calculation_files.append(file)

    elif 'fragility' in working_job.keys():
        for k, v in working_job['fragility'].items():
            if k == 'taxonomy_mapping_csv':
                file = create_taxonomymap_input(v, session)
            else:
                file = create_fragility_input(v, session)
                file.name = "{}.xml".format(k.replace('_file', ''))
            working_job['fragility'][k] = file.name
            calculation_files.append(file)

    for k, v in working_job['hazard'].items():
        with open(v, 'r') as f:
            file = io.StringIO(f.read())
        file.name = Path(v).name
        working_job['hazard'][k] = file.name
        calculation_files.append(file)

    job_file = create_job_file(working_job)
    job_file.name = 'job.ini'
    calculation_files.append(job_file)

    return calculation_files

def create_job_file(settings: configparser.ConfigParser) -> io.StringIO:
    job_ini = io.StringIO()
    settings.write(job_ini)
    job_ini.seek(0)

    return job_ini


#!/usr/bin/python3
# -*- coding: UTF-8 -*-

import struct
import sys
import time
import logging
from SerialCommunication import *  # module SerialCommunication.py
import platform
import copy
import threading
import os

FORMAT = '%(asctime)-15s %(name)s - %(levelname)s - %(message)s'
'''
Level: The level determines the minimum priority level of messages to log.
Messages will be logged in order of increasing severity:
DEBUG is the least threatening,
INFO is also not very threatening,
WARNING needs attention,
ERROR needs immediate attention,
and CRITICAL means “drop everything and find out what’s wrong.”
The default starting point is INFO,
which means that the logging module will automatically filter out any DEBUG messages.
'''
# logging.basicConfig(level=logging.DEBUG, format=FORMAT)
logging.basicConfig(level=logging.INFO, format=FORMAT)
logger = logging.getLogger(__name__)

def printH(head, value):
    print(head, end=' ')
    print(value)

def encode(in_str, encoding='utf-8'):
    if isinstance(in_str, bytes):
        return in_str
    else:
        return in_str.encode(encoding)

def serialWriteNumToByte(port, token, var=None):  # Only to be used for c m u b I K L o within Python
    # print("Num Token "); print(token);print(" var ");print(var);print("\n\n");
    logger.debug(f'serialWriteNumToByte, token={token}, var={var}')
    in_str = ""
    if var is None:
        var = []
    if token == 'K':
        period = var[0]
        #        print(encode(in_str))
        if period > 0:
            skillHeader = 4
        else:
            skillHeader = 7

        if period > 1:
            frameSize = 8  # gait
        elif period == 1:
            frameSize = 16  # posture
        else:
            frameSize = 20  # behavior
    # divide large angles by 2
        angleRatio = 1
        for row in range(abs(period)):
            for angle in var[skillHeader + row * frameSize:skillHeader + row * frameSize + min(16,frameSize)]:
                if angle > 125 or angle<-125:
                    angleRatio = 2
                    break
            if angleRatio ==2:
                break

        if angleRatio == 2:
            var[3] = 2
            for row in range(abs(period)):
                for i in range(skillHeader + row * frameSize,skillHeader + row * frameSize + min(16,frameSize)):
                    var[i] //=2
            printH('rescaled:\n',var)

        var = list(map(int, var))

        in_str = token.encode() + struct.pack('b' * skillHeader, *var[0:skillHeader])  # +'~'.encode()
        port.Send_data(in_str)
        time.sleep(0.001)

        for f in range(abs(period)):
            in_str = struct.pack('b' * (frameSize),
                                 *var[skillHeader + f * frameSize:skillHeader + (f + 1) * frameSize])  # + '~'.encode()
            if f == abs(period) - 1:
                in_str = in_str + '~'.encode()
            port.Send_data(in_str)
            time.sleep(0.001)
    else:
        if token == 'L' or token == 'I' or token == 'B' or token == 'C':
#            if token == 'C':
#                packType = 'B'
#            else:
#                packType = 'b'
            var = list(map(int, var))

            in_str = token.encode() + struct.pack('b' * len(var), *var) + '~'.encode()

        elif token == 'c' or token == 'm' or token == 'i' or token == 'b' or token == 'u' or token == 't':
            in_str = token + " "
            for element in var:
                in_str = in_str + str(element) + " "
        logger.debug(f"!!!! {in_str}")
        #        print(encode(in_str))
        port.Send_data(encode(in_str))

def serialWriteByte(port, var=None):
    logger.debug(f'serial_write_byte, var={var}')
    if var is None:
        var = []
    token = var[0][0]
    # print var
    if (token == 'c' or token == 'm' or token == 'i' or token == 'b' or token == 'u' or token == 't') and len(var) >= 2:
        in_str = ""
        for element in var:
            in_str = in_str + element + " "
    elif token == 'L' or token == 'I':
        if len(var[0]) > 1:
            var.insert(1, var[0][1:])
        var[1:] = list(map(int, var[1:]))
        in_str = token.encode() + struct.pack('b' * (len(var) - 1), *var[1:]) + '~'.encode()
    elif token == 'w' or token == 'k':
        in_str = var[0] + '\n'
    else:
        in_str = token
    logger.debug(f"!!!!!!! {in_str}")
    port.Send_data(encode(in_str))

def printSerialMessage(port, token, timeout=0):
    if token == 'k' or token == 'K':
        threshold = 4
    else:
        threshold = 1
    #    if token == 'K':
    #        timeout = 1
    startTime = time.time()
    allPrints = ''
    while True:
        time.sleep(0.001)
        #            return 'err'
        if port:
            response = port.main_engine.readline().decode('ISO-8859-1')
            if response != '':
                #                startTime = time.time()
                response = response[:-2]  # delete '\r\n'
                if response.lower() == token.lower():
                    return [response, allPrints]
                else:
#                    print(response, flush=True)
                    allPrints += response + '\n'
        now = time.time()
        if (now - startTime) > threshold:
            print('Elapsed time: ', end='')
            print(threshold, end=' seconds\n', flush=True)
            threshold += 2
        if 0 < timeout < now - startTime:
            return -1

def sendTask(goodPorts, port, task, timeout=0):  # task Structure is [token, var=[], time]
    logger.debug(f"{task}")
    global returnValue
    #    global sync
    #    print(task)
    if port:
        try:
            previousBuffer = port.main_engine.read_all().decode('ISO-8859-1')
            if previousBuffer:
                printH('Previous buffer:', previousBuffer)
            if len(task) == 2:
                #        print('a')
                #        print(task[0])
                serialWriteByte(port, [task[0]])
            elif isinstance(task[1][0], int):
                #        print('b')
                serialWriteNumToByte(port, task[0], task[1])
            else:
                #        print('c') #which case
                serialWriteByte(port, task[1])
            token = task[0][0]
            lastMessage = printSerialMessage(port, token, timeout)
            time.sleep(task[-1])
        #    with lock:
        #        sync += 1
        #        printH('sync',sync)
        #    if initialized:
        #        printH('thread',portDictionary[port])
        except Exception as e:
            #        printH('Fail to send to port',goodPorts[port])
            if port in goodPorts:
                goodPorts.pop(port)
            lastMessage = -1
    else:
        lastMessage = -1
    returnValue = lastMessage
    return lastMessage

def sendTaskParallel(ports, task, timeout=0):
    global returnValue
    #    global sync
    #    sync = 0
    threads = list()
    for p in ports:
        t = threading.Thread(target=sendTask, args=(goodPorts, p, task, timeout))
        threads.append(t)
        t.start()
    for t in threads:
        if t.is_alive():
            t.join()
    return returnValue

def splitTaskForLargeAngles(task):
    token = task[0]
    queue = list()
    if token == 'L' or token == 'I':
        var = task[1]
        indexedList = list()
        if token == 'L':
            for i in range(4):
                for j in range(4):
                    angle = var[4 * j + i]
                    if angle < -125 or angle > 125:
                        indexedList += [4 * j + i, angle]
                        var[4 * j + i] = max(min(angle, 125), -125)
            if len(var):
                queue.append(['L', var, task[-1]])
            if len(indexedList):
                queue[0][-1] = 0
                queue.append(['i', indexedList, task[-1]])
        #                print(queue)
        elif token == 'I':
            if min(var) < -125 or max(var) > 125:
                task[0] = 'i'
            queue.append(task)
    else:
        queue.append(task)
    return queue

def send(port, task, timeout=0):
    if isinstance(port, dict):
        p = list(port.keys())
    elif isinstance(port, list):
        p = port
    queue = splitTaskForLargeAngles(task)
    for task in queue:
        if len(port) > 1:
            returnResult = sendTaskParallel(p, task, timeout)
        elif len(port) == 1:
            returnResult = sendTask(goodPorts, p[0], task, timeout)
        else:
            #            print('no ports')
            return -1
    return returnResult

def keepReadingInput(ports):
    while True and len(ports):
        time.sleep(0.001)
        x = input()  # blocked waiting for the user's input
        if x != "":
            if x == "q" or x == "quit":
                break
            else:
                token = x[0]
                task = x[1:].split()  # white space
                if len(task) <= 1:
                    send(ports, [x, 1])
                else:
                    send(ports, [token, list(map(int, task)), 1])

def closeSerialBehavior(port):
    try:
        port.Close_Engine()
    except Exception as e:
        port.Close_Engine()
        raise e
    logger.info("close the serial port.")

def closeAllSerial(ports):
    send(ports, ['d', 0], 1)
    for p in ports:
        t = threading.Thread(target=closeSerialBehavior, args=(p,))
        t.start()
        t.join()
    ports.clear()

balance = [
    1, 0, 0, 1,
    0, 0, 0, 0, 0, 0, 0, 0, 30, 30, 30, 30, 30, 30, 30, 30]
buttUp = [
    1, 0, 15, 1,
    20, 40, 0, 0, 5, 5, 3, 3, 90, 90, 45, 45, -60, -60, 5, 5]
calib = [
    1, 0, 0, 1,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
dropped = [
    1, 0, -75, 1,
    0, 30, 0, 0, -5, -5, 15, 15, -75, -75, 45, 45, 60, 60, -30, -30]
lifted = [
    1, 0, 75, 1,
    0, -20, 0, 0, 0, 0, 0, 0, 60, 60, 75, 75, 45, 45, 75, 75]
rest = [
    1, 0, 0, 1,
    -30, -80, -45, 0, -3, -3, 3, 3, 70, 70, 70, 70, -55, -55, -55, -55]
sit = [
    1, 0, -30, 1,
    0, 0, -45, 0, -5, -5, 20, 20, 45, 45, 105, 105, 45, 45, -45, -45]
stretch = [
    1, 0, 20, 1,
    0, 30, 0, 0, -5, -5, 0, 0, -75, -75, 30, 30, 60, 60, 0, 0]
zero = [
    1, 0, 0, 1,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

balanceNybble = [
    1, 0, 0, 1,
    0, 0, 0, 0, 0, 0, 0, 0, 30, 30, -30, -30, 30, 30, -30, -30, ]
buttUpNybble = [
    1, 0, 15, 1,
    20, 40, 0, 0, 5, 5, 3, 3, 90, 90, -45, -45, -60, -60, -5, -5, ]
calibNybble = [
    1, 0, 0, 1,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ]
droppedNybble = [
    1, 0, 75, 1,
    0, 30, 0, 0, -5, -5, 15, 15, -75, -75, -60, -60, 60, 60, 30, 30, ]
liftedNybble = [
    1, 0, -75, 1,
    0, -70, 0, 0, 0, 0, 0, 0, 55, 55, 20, 20, 45, 45, 0, 0, ]
luNybble = [
    1, -30, 15, 1,
    -45, 60, -60, 0, 5, 5, 3, 3, -60, 70, -45, -35, 15, -60, 10, -65, ]
restNybble = [
    1, 0, 0, 1,
    -30, -80, -45, 0, -3, -3, 3, 3, 60, 60, -60, -60, -45, -45, 45, 45, ]
sitNybble = [
    1, 0, -20, 1,
    10, -20, -60, 0, -5, -5, 20, 20, 30, 30, -90, -90, 60, 60, 45, 45, ]
sleepNybble = [
    1, 0, 0, 1,
    -10, -100, 0, 0, -5, -5, 3, 3, 80, 80, -80, -80, -55, -55, 55, 55, ]
strNybble = [
    1, 0, 15, 1,
    10, 70, -30, 0, -5, -5, 0, 0, -75, -75, -45, -45, 60, 60, -45, -45, ]
zeroNybble = [
    1, 0, 0, 1,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ]

postureTableBittle = {
    "balance": balance,
    "buttUp": buttUp,
    "calib": calib,
    "dropped": dropped,
    "lifted": lifted,
    "rest": rest,
    "sit": sit,
    "str": stretch,
    "zero": zero
}

postureTableNybble = {
    "balance": balanceNybble,
    "buttUp": buttUpNybble,
    "calib": calibNybble,
    "dropped": droppedNybble,
    "lifted": liftedNybble,
    "lu": luNybble,
    "rest": restNybble,
    "sit": sitNybble,
    "sleep": sleepNybble,
    "str": strNybble,
    "zero": zeroNybble
}
postureTableDoF16 = {
    "balance": balance,
    "buttUp": buttUp,
    "calib": calib,
    "dropped": dropped,
    "lifted": lifted,
    "rest": rest,
    "sit": sit,
    "str": stretch,
    "zero": zero
}

postureDict = {
    'Nybble': postureTableNybble,
    'Bittle': postureTableBittle,
    'DoF16': postureTableDoF16
}
model = 'Bittle'
postureTable = postureDict[model]

def schedulerToSkill(ports, testSchedule):
    compactSkillData = []
    newSkill = []
    outputStr = ""

    for task in testSchedule:  # execute the tasks in the testSchedule
        print(task)
        token = task[0][0]
        if token == 'k' and task[0][1:] in postureTable:
            currentRow = postureTable[task[0][1:]][-16:]
            skillRow = copy.deepcopy(currentRow)
            compactSkillData.append(skillRow + [8, int(task[1] * 1000 / 500), 0, 0])
            newSkill = newSkill + skillRow + [8, int(task[1] * 1000 / 500), 0, 0]

        elif token == 'i' or token == 'I':
            currentRow = copy.deepcopy(skillRow)
            for e in range(0, len(task[1]), 2):
                #                    print(task[1][e],task[1][e+1])
                currentRow[task[1][e]] = task[1][e + 1]
            skillRow = copy.deepcopy(currentRow)
            compactSkillData.append(skillRow + [8, int(task[2] * 1000 / 500), 0, 0])
            newSkill = newSkill + skillRow + [8, int(task[2] * 1000 / 500), 0, 0]
        elif token == 'L':
            skillRow = copy.deepcopy(task[1][:16])
            compactSkillData.append(skillRow + [8, int(task[2] * 1000 / 500), 0, 0])
            newSkill = newSkill + skillRow + [8, int(task[2] * 1000 / 500), 0, 0]

        elif token == 'm':
            currentRow = copy.deepcopy(skillRow)
            for e in range(0, len(task[1]), 2):
                currentRow[task[1][e]] = task[1][e + 1]
                skillRow = copy.deepcopy(currentRow)
                compactSkillData.append(skillRow + [8, int(task[2] * 1000 / 500), 0, 0])
                newSkill = newSkill + skillRow + [8, int(task[2] * 1000 / 500), 0, 0]
    if len(compactSkillData) > 0:
        print('{')
        print('{:>4},{:>4},{:>4},{:>4},'.format(*[-len(compactSkillData), 0, 0, 1]))
        print('{:>4},{:>4},{:>4},'.format(*[0, 0, 0]))
    else:
        return
    angleRatio = 1
    for row in compactSkillData:
        if min(row) < -125 or max(row) > 125:
            angleRatio = 2
        print(('{:>4},' * 20).format(*row))
    print('};')
    newSkill = list(map(lambda x: x // angleRatio, newSkill))
    newSkill = [-len(compactSkillData), 0, 0, angleRatio, 0, 0, 0] + newSkill
    print(newSkill)
    #    sendTaskParallel(['K', newSkill, 1])
    send(ports, ['K', newSkill, 1])

def testPort(goodPorts, serialObject, p):
    global goodPortCount
    #    global sync
    try:
        time.sleep(5)
        result = serialObject.main_engine.read_all().decode('ISO-8859-1')
        if result != '':
            print('Waiting for the robot to boot up')
            time.sleep(2)
            waitTime = 3
        else:
            waitTime = 2
        result = sendTask(goodPorts, serialObject, ['b', [20, 50], 0], waitTime)
        print(result)
        if result != -1:# or "Petoi" in p:
            printH('Adding', p)
            goodPorts.update({serialObject: p})
            goodPortCount += 1
        else:
            serialObject.Close_Engine()
            print('* Port ' + p + ' is not connected to a Petoi device!')
    #    sync +=1
    except Exception:
        print('* Port ' + p + ' cannot be opened!')

def checkPortList(goodPorts, allPorts):
    threads = list()
    for p in reversed(allPorts):  # assuming the last one is the most possible port
        # if p == '/dev/ttyAMA0':
        #     continue
        serialObject = Communication(p, 115200, 1)
        t = threading.Thread(target=testPort,
                             args=(goodPorts, serialObject, p.split('/')[-1]))  # remove '/dev/' in the port name
        threads.append(t)
        t.start()
    for t in threads:
        if t.is_alive():
            t.join(8)

def keepCheckingPort(portList, check=True):
    allPorts = Communication.Print_Used_Com()
    while len(portList) > 0:
        currentPorts = Communication.Print_Used_Com()
        time.sleep(0.01)
        if set(currentPorts) - set(allPorts):
            newPort = list(set(currentPorts) - set(allPorts))
            if check:
                time.sleep(0.5)
                checkPortList(portList, newPort)

        elif set(allPorts) - set(currentPorts):
            closedPort = list(set(allPorts) - set(currentPorts))
            inv_dict = {v: k for k, v in portList.items()}
            for p in closedPort:
                if inv_dict.get(p.split('/')[-1], -1) != -1:
                    printH('Removing', p)
                    portList.pop(inv_dict[p.split('/')[-1]])

        allPorts = copy.deepcopy(currentPorts)

def connectPort(PortList):
    global initialized
    allPorts = Communication.Print_Used_Com()
    if platform.uname()[1] == 'raspberrypi':
        allPorts.append('/dev/ttyS0')

    for index in range(len(allPorts)):
        logger.info(f"port[{index}] is {allPorts[index]} ")

    if len(allPorts) > 0:
        goodPortCount = 0
        checkPortList(PortList, allPorts)
    initialized = True
    if len(PortList) == 0:
        print('No port found!')
    else:
        logger.info(f"Connect to serial port:")
        for p in PortList:
            logger.info(f"{PortList[p]}")

goodPorts = {}
initialized = False
goodPortCount = 0
sync = 0
lock = threading.Lock()
returnValue = ''

if __name__ == '__main__':
    try:
        connectPort(goodPorts)
        t = threading.Thread(target=keepCheckingPort, args=(goodPorts,))
        t.start()
        if len(sys.argv) >= 2:
            if len(sys.argv) == 2:
                cmd = sys.argv[1]
                token = cmd[0][0]
            else:
                token = sys.argv[1][0]
            #                sendTaskParallel([sys.argv[1][0], sys.argv[1:], 1])
            send(goodPorts, [sys.argv[1][0], sys.argv[1:], 1])

        print("You can type 'quit' or 'q' to exit.")

        keepReadingInput(goodPorts)

        closeAllSerial(goodPorts)
        logger.info("finish!")
        os._exit(0)

    except Exception as e:
        logger.info("Exception")
        closeAllSerial(goodPorts)
        raise e


import re
from typing import List

PRIORITY_DICT = {
    "#"  : 0,
    "("  : 1,
    "if" : 2,
    "iff": 2,
    "or" : 3,
    "and": 4,
    "not": 5,
    ")"  : 6
}

class LogicalExpression(object):
    def __init__(self, content: str) -> None:
        super(LogicalExpression, self).__init__()
        if content in PRIORITY_DICT:
            self.connective = content
            self.symbol = None
        else:
            self.symbol = content
            self.connective = None
        self.children: List[LogicalExpression] = []
        self.parent: LogicalExpression = None

    def appendChild(self, child) -> None:
        child: LogicalExpression = child
        self.children.append(child)
        child.parent = self

def readLinesFromTxt(file: str) -> list:
    in_file = open(file)
    lines = in_file.readlines()
    new_lines = []
    for line in lines:
        if line != "\n":
            if line[-1] == '\n':
                line = line[:-1]
            new_lines.append(line)
    in_file.close()
    return new_lines

def line2Tokens(line: str) -> list:
    # "(" and ")" are independent tokens
    tokens = re.split('([ ()])', line)
    tokens = [token for token in tokens if token != '' and token != ' ']
    return tokens

def toRPN(tokens: list) -> list:
    op = ["#"]  # S1
    var = []    # S2
    for token in tokens:
        if token in PRIORITY_DICT:  # it is an operator
            if token == "(":
                # push
                op.append(token)
            elif token == ")":
                # pop till (
                while op[-1] != "(":
                    var.append(op.pop(-1))
                op.pop(-1)  # pop "("
            elif PRIORITY_DICT[token] > PRIORITY_DICT[op[-1]]:
                # push
                op.append(token)
            else:
                # pop till smaller than token
                while PRIORITY_DICT[op[-1]] >= PRIORITY_DICT[token]:
                    var.append(op.pop(-1))
                op.append(token)
        else:
            var.append(token)
    while len(op) > 1:
        var.append(op.pop(-1))
    assert op == ["#"]
    return var

def RPN2Tree(var: list) -> LogicalExpression:
    def theOperation(op_node: LogicalExpression, children: List[LogicalExpression]) -> LogicalExpression:
        assert op_node.symbol is None
        assert op_node.connective is not None
        for child in children:
            op_node.appendChild(child)
        return op_node
    var = [LogicalExpression(node) for node in var]
    result: list[LogicalExpression] = []
    for node in var:
        if node.symbol is not None:  # a leaf node
            # push
            result.append(node)
        else:                        # an operator
            # do "the operation"
            if node.connective == "not":
                result.append(theOperation(node, [result.pop(-1)]))
            else:
                result.append(theOperation(node, [result.pop(-1), result.pop(-1)]))
    return result[0]

def buildSentences(file: str) -> LogicalExpression:
    h_node = LogicalExpression("and")
    lines = readLinesFromTxt(file)
    for line in lines:
        tokens = line2Tokens(line)
        var = toRPN(tokens)
        result = RPN2Tree(var)
        h_node.appendChild(result)
    return h_node


import flet as ft

"""
NavRail é tipo o NavBar, mas é utilizado na lateral da aplicação (esquerda ou direita)
"""

def main(page: ft.Page):
    page.title = "NavRail"

    rail = ft.NavigationRail(
        selected_index=0,                           # Define o índice inicial da destinação selecionada. (tipo o autofocus)
        label_type=ft.NavigationRailLabelType.ALL,
        #extended=True,
        min_width=100,                              # Define a largura mínima da barra de navegação.
        min_extended_width=400,                     # Define a largura mínima quando a barra de navegação está estendida.
        leading=ft.FloatingActionButton(icon=ft.icons.CREATE, text="Add"),  # Adiciona um botão flutuante no início
        group_alignment=-0.9,                       # Define a distância do botão flutuante para os itens de destino da barra (valores = -1.0 até 1.0)

        destinations=[                              # Define os itens/destinos que terá na NavRail

            ft.NavigationRailDestination(icon=ft.icons.FAVORITE_BORDER,
                                         selected_icon=ft.icons.FAVORITE,
                                         label="Favoritar"),
            # a diferença de Icon_content para Icon, é que Icon_content permite transformar o icon em um controle e ter mais estilização do mesmo.
            ft.NavigationRailDestination(icon_content=ft.Icon(ft.icons.BOOKMARK_BORDER),
                                         selected_icon_content=ft.Icon(ft.icons.BOOKMARK),
                                         label="Salvos"),
        ],

        on_change=lambda e: print("Selected destination:", # quando tiver clique, ele informara para qual destino foi (baseado em index, de 0 em diante)
                                 e.control.selected_index),# o "e.control.selected_index" que informa o index selecionado
    )

    # Adiciona uma estrutura de layout à página.
    page.add(
        ft.Row(
            [
                rail,                            # adiciona a NavRail na linha
                ft.VerticalDivider(width=1),     # adiciona uma divisão vertical (para a NavRail)
                ft.Column([ ft.Text("Body!")],
                          # https://flet.dev/docs/controls/row/#alignment <- link para saber mais do alinhamento
                          alignment=ft.MainAxisAlignment.START, # O MainAxisAlignment.Start alinha os filhos no início do eixo principal (á esquerda)
                          expand=True),
            ],
            expand=True,
        )
    )

ft.app(target=main)


import power

class InvalidInfoException(Exception):
    pass

power_types = [
    "AC",
    "BATTERY",
    "UPS"
]

power_warnings = [
    "NONE",
    "EARLY",
    "FINAL"
]

def main(info):
    if info in ('type', 'time', 'timeh', 'warn'):
        battery = power.PowerManagement()
        if info == 'type':
            print(get_power_type(battery))
        elif info == 'time':
            print(get_time_remaining(battery, 'm'))
        elif info == 'timeh':
            print(get_time_remaining(battery, 'h'))
        else:
            print(get_warning_level(battery))

    else:
        raise InvalidInfoException()

def get_power_type(battery):
    return power_types[battery.get_providing_power_source_type()]

def get_time_remaining(battery, mode='m'):
    time = battery.get_time_remaining_estimate()
    if time > 0:
        if mode == 'm':
            return time
        elif mode == 'h':
            h, m = divmod(time, 60)
            return "{}:{}".format(int(h), int(m))
    else:
        return time

def get_warning_level(battery):
    return power_warnings[battery.get_low_battery_warning_level()-1]

if __name__ == '__main__':

    import argparse

    parser = argparse.ArgumentParser()

    parser.add_argument('info', help='Set which info you want')
    args = parser.parse_args()

    try:
        main(args.info)
    except InvalidInfoException as ime:
        print("Invalid info name")
        parser.print_help()


import matplotlib.pyplot as plt
import numpy as np

# a)
v0 = 0
t0 = 0
y0 = 800
tf = 4
dt = 0.001
g = -9.8
n = int((tf-t0)/dt)

t = np.linspace(0, 1000, n)
v = np.zeros(n) #or np.empty(n)
y = np.zeros(n)

v[0] = v0
y[0] = y0
for i in range(n-1):
    v[i+1] = v[i] + g*dt
    y[i+1] = y[i] + v[i]*dt

plt.plot(t, y, label = "v(t)")
plt.xlabel("t /(s)")
plt.ylabel("y /(m)")
plt.legend()


import tkinter as tk
from tkinter import ttk

import customtkinter
from PIL import ImageTk, Image

from PopUps.ErrorPopUp import ErrorPopUp
from PopUps.ConfirmationPopUp import ConfirmationPopUp

from Services.OfficeServices import OfficeService
from logger import logging

logger = logging.getLogger('Office Application')

class UserOfficeGUI:
    _rootWindow: tk.Tk
    _parentFrame: tk.Frame
    _OfficeService: OfficeService

    def __init__(self, rootWindow, Frame, userData):
        self._rootWindow = rootWindow
        self._parentFrame = Frame
        self.confirm = ImageTk.PhotoImage(Image.open("assets/confirm.webp").resize((50, 50), Image.ANTIALIAS))
        self.back = ImageTk.PhotoImage(Image.open("assets/back.png").resize((50, 50), Image.ANTIALIAS))
        self._OfficeService = OfficeService(userData)

    def addOfficeOption(self):
        try:
            frame = customtkinter.CTkFrame(self._rootWindow, width=1000, height=800)
            frame.grid(row=0, column=0, sticky='news')
            logger.info("Office Adding Menu Opened.")
            customtkinter.CTkLabel(text_font=("Malgun Gothic", 10), master=frame, fg_color="white",
                                   text_color="black", text="What type of office would you like to add??", width=100,
                                   corner_radius=50, height=30, compound="top").place(relx=0.5, rely=0.2,
                                                                                anchor='center')
            international = ImageTk.PhotoImage(Image.open("assets/local.png").resize((50, 50), Image.ANTIALIAS))
            local = ImageTk.PhotoImage(Image.open("assets/international.webp").resize((50, 50), Image.ANTIALIAS))
            customtkinter.CTkButton(text_font=("Malgun Gothic", 10), image=international, master=frame, fg_color="white",
                                    text_color="black", text="International", width=80, corner_radius=50, height=70,
                                    compound="top",
                                    command=lambda: self.renderMenuForOffice(frame, True)).place(
                                    relx=0.4, rely=0.3, anchor='center')
            customtkinter.CTkButton(text_font=("Malgun Gothic", 10), image=local, master=frame, fg_color="white",
                                    text_color="black", text="Local", width=80, corner_radius=50, height=70,
                                    compound="top",
                                    command=lambda: self.renderMenuForOffice(frame, False)).place(
                relx=0.6, rely=0.3, anchor='center')
        except Exception as E:
            logger.error(f"Problem creating office. {E}")

    def renderMenuForOffice(self, frame, choice):
        try:
            name = tk.StringVar()
            location = tk.StringVar()
            employeeCount = tk.IntVar()
            customtkinter.CTkLabel(text_font=("Malgun Gothic", 10), master=frame, fg_color="white",
                                   text_color="black", text="Enter the name of the office:", width=100,
                                   corner_radius=50, height=30, compound="top").place(relx=0.2, rely=0.4,
                                                                                anchor='center')
            customtkinter.CTkEntry(frame, textvariable=name, border_color="white", corner_radius=5,
                                   text_color="white", width=100, height=2, text_font=("B old", 8)).place(relx=0.5,
                                                                                rely=0.4,
                                                                                anchor='center')
            customtkinter.CTkLabel(text_font=("Malgun Gothic", 10), master=frame, fg_color="white",
                                   text_color="black", text="Enter the location of the office:", width=100,
                                   corner_radius=50, height=30, compound="top").place(relx=0.2, rely=0.5,
                                                                                anchor='center')
            customtkinter.CTkEntry(frame, textvariable=location, border_color="white", corner_radius=5,
                                   text_color="white", width=100, height=2, text_font=("B old", 8)).place(relx=0.5,
                                                                                rely=0.5,
                                                                                anchor='center')
            customtkinter.CTkLabel(text_font=("Malgun Gothic", 10), master=frame, fg_color="white",
                                   text_color="black", text="Enter the number of employees in the office:", width=100,
                                   corner_radius=50, height=30, compound="top").place(relx=0.2, rely=0.65,
                                                                                anchor='center')
            customtkinter.CTkEntry(frame, textvariable=employeeCount, border_color="white", corner_radius=5,
                                   text_color="white", width=100, height=2, text_font=("B old", 8)).place(relx=0.5,
                                                                                rely=0.65,
                                                                                anchor='center')
            officeCreationPopUp = ConfirmationPopUp()
            customtkinter.CTkButton(text_font=("Malgun Gothic", 10), image=self.confirm, master=frame, fg_color="white",
                                    text_color="black", text="Confirm", width=30, corner_radius=50, height=30,
                                    compound="top",
                                    command=lambda: [officeCreationPopUp.createConfirmationPopUp("Office Created."),self._OfficeService.addOffice(name.get(), location.get(), True,
                                                                     employeeCount.get()) if choice else
                                       self._OfficeService.addOffice(name.get(), location.get(), False, employeeCount.get())
                          , frame.destroy()]).place(relx=0.2, rely=0.75, anchor='center')
            customtkinter.CTkButton(text_font=("Malgun Gothic", 10), image=self.back, master=frame, fg_color="white",
                                        text_color="black", text="Back", width=30, corner_radius=50, height=30,
                                        compound="top",
                                        command=frame.destroy).place(relx=0.4, rely=0.75, anchor='center')
        except Exception as ex:
            logger.error(f"Problem while creating office. {ex}")

    def listOffices(self):
        try:
            frame = customtkinter.CTkFrame(self._rootWindow, width=1000, height=800)
            frame.grid(row=0, column=0, sticky='news')
            style = ttk.Style()
            logger.info("Listing Offices.")
            style.theme_use("default")

            style.configure("Treeview",
                            background="#2a2d2e",
                            foreground="white",
                            rowheight=25,
                            fieldbackground="#343638",
                            bordercolor="#343638",
                            borderwidth=0)
            style.map('Treeview', background=[('selected', '#22559b')])

            style.configure("Treeview.Heading",
                            background="#565b5e",
                            foreground="white",
                            relief="flat")
            style.map("Treeview.Heading",
                      background=[('active', '#3484F0')])
            table = tk.ttk.Treeview(frame)
            table.place(relx=0.5, rely=0.25, anchor='center')
            table['columns'] = ('Name', 'Location', 'Type', "Employee Count", "Expenses", "Limit")
            table.column("#0", width=0, stretch="no")
            table.column("Name", anchor="center", width=200)
            table.column("Location", anchor="center", width=200)
            table.column("Type", anchor="center", width=200)
            table.column("Employee Count", anchor="center", width=200)
            table.column("Expenses", anchor="center", width=120)
            table.column("Limit", anchor="center", width=200)

            table.heading("#0", text="", anchor="center")
            table.heading("Name", text="Name", anchor="center")
            table.heading("Location", text="Location", anchor="center")
            table.heading("Type", text="Type", anchor="center")
            table.heading("Employee Count", text="Employee Count", anchor="center")
            table.heading("Expenses", text="Expenses", anchor="center")
            table.heading("Limit", text="Limit", anchor="center")

            customtkinter.CTkButton(text_font=("Malgun Gothic", 10), image=self.back, master=frame, fg_color="white",
                                    text_color="black", text="Back", width=30, corner_radius=50, height=30,
                                    compound="top",
                                    command=frame.destroy).place(relx=0.5, rely=0.5, anchor='center')
            userOfficeInfo = self._OfficeService.listOffices()
            for i in range(len(userOfficeInfo)):
                table.insert(parent='', index='end', iid=i, text='',
                             values=userOfficeInfo[i])
        except Exception as E:
            logger.error(f"Error while listing offices. {E}")

    def addExpenseOption(self):
        try:
            frame = customtkinter.CTkFrame(self._rootWindow, width=1000, height=800)
            frame.grid(row=0, column=0, sticky='news')
            newExpense = tk.IntVar(frame)
            officeName = tk.StringVar(frame)
            officeNameList=self._OfficeService.returnOfficeNames()
            officeName.set("Choose Office")
            customtkinter.CTkLabel(text_font=("Malgun Gothic", 10), master=frame, fg_color="white",
                                   text_color="black", text="Pick Office:", width=100,
                                   corner_radius=50, height=30, compound="top").place(relx=0.2, rely=0.2,
                                                                                anchor='center')

            officeMenu = customtkinter.CTkOptionMenu(frame, variable=officeName, values=officeNameList,
                                                       fg_color="white", text_color="black", button_color="white")
            officeMenu.place(relx=0.5, rely=0.2, anchor='center')
            choose = ImageTk.PhotoImage(Image.open("assets/choose.png").resize((50, 50), Image.ANTIALIAS))
            customtkinter.CTkButton(text_font=("Malgun Gothic", 10), image=choose, master=frame, fg_color="white",
                                    text_color="black", text="Choose", width=30, corner_radius=50, height=30,
                                    compound="top",
                                    command=lambda: addExpense()).place(relx=0.7, rely=0.2, anchor='center')

            customtkinter.CTkButton(text_font=("Malgun Gothic", 10), image=self.back, master=frame, fg_color="white",
                                    text_color="black", text="Back", width=30, corner_radius=50, height=30,
                                    compound="top",
                                    command=frame.destroy).place(relx=0.65, rely=0.6, anchor='center')
        except Exception as error:
            logger.error(f"Error while setting expenses. {error}")

        def addExpense():
            try:
                customtkinter.CTkLabel(text_font=("Malgun Gothic", 10), master=frame, fg_color="white",
                                       text_color="black", text="Enter Expense:", width=100,
                                       corner_radius=50, height=30, compound="top").place(relx=0.2, rely=0.4,
                                                                                anchor='center')
                customtkinter.CTkEntry(frame, textvariable=newExpense, border_color="white", corner_radius=5,
                                       text_color="white", width=100, height=2, text_font=("B old", 8)).place(relx=0.5,
                                                                                rely=0.4,
                                                                                anchor='center')
                expenseSetPopUp=ConfirmationPopUp()
                customtkinter.CTkButton(text_font=("Malgun Gothic", 10), image=self.confirm, master=frame, fg_color="white",
                                    text_color="black", text="Confirm", width=30, corner_radius=50, height=30,
                                    compound="top",
                                    command=lambda: [expenseSetPopUp.createConfirmationPopUp("Expense Set."),
                                        self._OfficeService.setExpense(officeName.get(), newExpense.get()), frame.destroy()]).place(relx=0.5,
                                                                                rely=0.6,
                                                                                anchor='center')
            except Exception as ex:
                logger.error(f"Error while adding expenses. {ex}")


import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import  train_test_split
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
cancer = load_breast_cancer()

fig, axes = plt.subplots(15,2,figsize=(10,20))
malignant = cancer.data[cancer.target == 0]
benign = cancer.data[cancer.target == 1]
ax = axes.ravel()

for i in range(30):
    _, bins = np.histogram(cancer.data[:,i], bins =50)
    ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5)
    ax[i].hist(benign[:,i], bins = bins, alpha = 0.5)
    ax[i].set_title(cancer.feature_names[i])
    ax[i].set_yticks(())
ax[0].set_xlabel("Feature magnitude")
ax[0].set_ylabel("Frequency")
ax[0].legend(["Malignant", "Benign"], loc ="best")
fig.tight_layout()
plt.show()

X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
print(X_train.shape)
print(X_test.shape)

logreg = LogisticRegression()
logreg.fit(X_train, y_train)
print("Test set accuracy from Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))

from sklearn.preprocessing import MinMaxScaler, StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)

logreg.fit(X_train_scaled, y_train)
print("Test set accuracy scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))


from datetime import date, timedelta, datetime
import standards
import traceback
import sys

def number(string):
    if type(string) == float or type(string) == int:
        return string
    string = string.strip()
    if not string:
        return 0
    string = string.split()[0]
    number = string.replace(",", "").replace("+", "").replace(".", "").replace("*", "")
    try:
        return int(number)
    except:
        return 0

def date_t(s):
    return datetime.strptime(s, "%Y-%m-%dT%H:%M:%S").date()

json_methods = {
    "::unixtime": lambda x: datetime.utcfromtimestamp(x//1000).strftime("%Y-%m-%d"),
    "::number": lambda x: number(x),
    "::text": lambda x: x.text,
    "::strip": lambda x: x.strip(),
    "::cap": lambda x: x.capitalize(),
    "::dmy": lambda x: datetime.strptime(x, "%d%m%Y").date(),
    "::ymd": lambda x: datetime.strptime(x, "%Y%m%d").date(),
    "::date_t": date_t,
    "::us_state_code": standards.state_codes['United States'].get,
    "::str": lambda x: str(x),
    "::dividethousands": lambda x: x/1000,
    "::china_province_eng": standards.state_codes['China'].get
}

def find_json(head, selectors):
    try:
        if type(selectors) != list:
            return selectors
        for selector in selectors:
            if selector.startswith("::"):
                head = json_methods[selector](head)
            else:
                head = head[selector]
        return head
    except KeyError as e:
        sys.stderr.write("KeyError warning on {} for selectors {}".format(e, selectors))
        traceback.print_tb(e.__traceback__)

def extract_json_row(row, labels):
    result = {}
    for label in labels:
        j = find_json(row, labels[label])
        if j is not None:
            result[label] = j
    return result

def column_total(rows, column, start=0, converter=int):
    for row in rows:
        start += converter(row[column])
    return start


# Import Libraries
import pandas as pd
import requests
import json
pd.set_option('display.max_columns', None)

### Parameters ###
org_id = hcl.variable["v_org_id"]                                 # Launchpad org id
org_region = hcl.variable["v_region"]                             # Launchpad org region
results_id = hcl.variable["v_results_id"]                         # Result container to push data to
hb_token = hcl.secret['v_hb_token'].unmask()                      # Request library requires token to be in plain text

### Pull Users for this org  ###

# URL API
api_env = "https://apis-" + org_region + ".highbond.com/v1"      # Adjust your environment to PROD or S1

# URL Endpoint
request_url = api_env +"/orgs/"+ org_id +"/users"

# Headers
request_headers = {
  'Authorization': 'Bearer '+hb_token,
  'Content-Type' : 'application/vnd.api+json',
  'Accept-encoding' : ''
}

# Submit the request and grab the response, and convert it to JSON
request_response = requests.get(request_url, headers=request_headers)
request_json = request_response.json()

# Confirm that the request was made successfully
# print(request_response)

# Create users Dataframe
df_users = pd.json_normalize(request_json['data'])
df_users

### Pull all Asset Types for this org  ###

# URL Endpoint as defined from https://docs-apis.highbond.com/public.html#operation/getRecords
request_url = api_env +"/orgs/"+ org_id +"/asset_types"

# Headers set for authentication; also need to add Accept-encoding as blank for the API
request_headers = {
  'Authorization': 'Bearer '+hb_token,
  'Content-Type' : 'application/vnd.api+json',
  'Accept-encoding':''
}

# Submit the request and grab the response, and convert it to JSON
request_response = requests.get(request_url, headers=request_headers)
request_json = request_response.json()

# Asset Types Dataframe
df_asset_types = pd.json_normalize(request_json['data'])
df_asset_types

### Pull Asset Info for a particular asset type
### Paginate if necessary and join all info into one dataframe

def get_asset_dataframe(asset_type_id, asset_type_name):
    ### # A Function to get assets for a specific asset type ###

    # URI based on org and asset type
    request_url = api_env +"/orgs/"+ org_id +"/asset_types/"+ asset_type_id +"/assets"

    # Headers set for authentication; also need to add Accept-encoding as blank for the API
    request_headers = {
      'Authorization' : 'Bearer '+hb_token,
      'Content-Type' : 'application/vnd.api+json',
      'Accept-encoding' : ''
    }

    # Make the Request and extract the JSON payload
    response = requests.get(request_url, headers=request_headers)
    raw = response.json()

    if (len(raw['data']) == 0):
        return None

    # Initialize the dataframes with the information of the first page
    # 1. Asset info (that needs to be pivoted)
    # 2. Asset type info
    # 3. Workflow info
    df_all = pd.json_normalize(raw['data'], record_path=['attributes', 'asset_attributes'], meta='id')
    df_all = df_all.pivot(index = 'id', columns = 'field_name', values = 'value')
    df_all = df_all.reset_index()

    df_all_type = pd.json_normalize(raw['data'])

    df_workflow_status_id = pd.json_normalize(raw['data'], record_path=['attributes', 'asset_attributes'], meta='id')
    df_workflow_status_id = df_workflow_status_id.pivot(index = 'id', columns = 'field_name', values = 'value.id')
    df_workflow_status_id = df_workflow_status_id.reset_index()

    df_workflow_status_name = pd.json_normalize(raw['data'], record_path=['attributes', 'asset_attributes'], meta='id')
    df_workflow_status_name = df_workflow_status_name.pivot(index = 'id', columns = 'field_name', values = 'value.name')
    df_workflow_status_name = df_workflow_status_name.reset_index()

    df_owner = pd.json_normalize(raw['data'], record_path=['attributes', 'asset_attributes'], meta='id')
    df_owner = df_owner.pivot(index = 'id', columns = 'field_name', values = 'value.user_ids')
    df_owner = df_owner.reset_index()

    # Retrieve remaining pages and append to combined dataframes
    while raw['links']['next'] != None and raw['links']['next'] != raw['links']['last'] :
        response = requests.get(api_env +"/orgs/"+org_id +raw['links']['next'], headers=request_headers)
        raw = response.json()

        df_next_data = pd.json_normalize(raw['data'], record_path=['attributes', 'asset_attributes'], meta='id')
        df_next_data = df_next_data.pivot(index = 'id', columns = 'field_name', values = 'value')
        df_next_data = df_next_data.reset_index()

        df_next_type = pd.json_normalize(raw['data'])

        df_next_workflow_status_id = pd.json_normalize(raw['data'], record_path=['attributes', 'asset_attributes'], meta='id')
        df_next_workflow_status_id = df_next_workflow_status.pivot(index = 'id', columns = 'field_name', values = 'value.id')
        df_next_workflow_status_id = df_next_workflow_status.reset_index()

        df_next_workflow_status_name = pd.json_normalize(raw['data'], record_path=['attributes', 'asset_attributes'], meta='id')
        df_next_workflow_status_name = df_next_workflow_status.pivot(index = 'id', columns = 'field_name', values = 'value.name')
        df_next_workflow_status_name = df_next_workflow_status.reset_index()

        df_next_owner = pd.json_normalize(raw['data'], record_path=['attributes', 'asset_attributes'], meta='id')
        df_next_owner = df_next_owner.pivot(index = 'id', columns = 'field_name', values = 'value.user_ids')
        df_next_owner = df_next_owner.reset_index()

        df_all = pd.concat([df_all, df_next_data])
        df_all_type = pd.concat([df_all_type, df_next_type])
        df_workflow_status_id = pd.concat([df_workflow_status_id, df_next_workflow_status_id])
        df_workflow_status_name = pd.concat([df_workflow_status_name, df_next_workflow_status_name])
        df_owner = pd.concat([df_owner, df_next_owner])

    ### Join all dataframe information

    df_combined = df_all

    # Get Asset Type ID
    df_combined['Asset Type Id'] = asset_type_id

    # Get Asset Type Name
    df_combined['Asset Type'] = asset_type_name

    df_combined.rename(columns={'id':'Asset Id'}, inplace=True)

    # Get Workflow Status  - todo: use df_combined
    df_combined = df_combined.merge(df_workflow_status_id[['id', 'metadata.workflow_status']], left_on='Asset Id', right_on='id')
    df_combined

    df_combined = df_combined.merge(df_workflow_status_name[['id', 'metadata.workflow_status']], left_on='Asset Id', right_on='id')
    df_combined

    # Get Owners
    df_combined = df_combined.merge(df_owner[['id', 'business_owner_(required)', 'technical_owner', 'metadata.created_by', 'metadata.updated_by']], left_on='Asset Id', right_on='id')
    df_combined

    print(df_combined)

    return df_combined

### Pull all Assets of interest

# Determine filters for asset types of interest
# and pull information
filter_hardware = df_asset_types['attributes.name'] == "IT Asset - Hardware"
filter_software = df_asset_types['attributes.name'] == "IT Asset - Software"
filter_systems = df_asset_types['attributes.name'] == "IT Asset - Information System"
filter_cloud = df_asset_types['attributes.name'] == "IT Asset - Cloud"

asset_type_id = df_asset_types[filter_hardware]['id'].values[0]
df_hardware =  get_asset_dataframe(asset_type_id, "IT Asset - Hardware")
if (df_hardware is None):
    print('IT Asset - Hardware assets have not been installed.')

asset_type_id = df_asset_types[filter_software]['id'].values[0]
df_software =  get_asset_dataframe(asset_type_id, "IT Asset - Software")
if (df_software is None):
    print('IT Asset - Software assets have not been installed.')

asset_type_id = df_asset_types[filter_systems]['id'].values[0]
df_systems =  get_asset_dataframe(asset_type_id, "IT Asset - Information System")
if (df_systems is None):
    print('IT Asset - Information System assets have not been installed.')

asset_type_id = df_asset_types[filter_cloud]['id'].values[0]
df_cloud =  get_asset_dataframe(asset_type_id, "IT Asset - Cloud")
if (df_cloud is None):
    print('IT Asset - Cloud assets have not been installed.')

### New Cell
df_all = pd.concat([df_hardware, df_software, df_systems, df_cloud])
df_all = df_all.reset_index()
result = df_all
result

### New Cell
list(result.columns)

### New Cell
### Remove column that cause the upload to fail  'metadata.updated_at',

### Remove columns for reporting
result.drop('id_x', axis=1, inplace=True)    #duplicate Asset Type ID from secondary table
result.drop('id_y', axis=1, inplace=True)    #duplicate Asset Type ID from secondary table
result.drop('Asset Id', axis=1, inplace=True)    #duplicate Asset ID
result.drop('index', axis=1, inplace=True)   #no use, df specific
result.drop('business_owner_(required)_x', axis=1, inplace=True)   #no use, df specific
result.drop('technical_owner_x', axis=1, inplace=True)   #no use, df specific
result.drop('metadata.created_by_x', axis=1, inplace=True)  #duplicate from original record but empty
result.drop('metadata.updated_by_x', axis=1, inplace=True)  #duplicate from original record but empty
result.drop('metadata.workflow_status_x', axis = 1, inplace = True)  #duplicate from original record but empty

### Rename columns for reporting
result.rename(columns={'relationships.asset_type.data.id':'Asset Type Id'}, inplace=True)
result.rename(columns={'relationships.workflow_status.data.id':'Workflow State Id'}, inplace=True)
result.rename(columns={'attributes.name_x':'Asset Type'}, inplace=True)
result.rename(columns={'attributes.name_y':'Workflow State'}, inplace=True)

result.rename(columns={'business_owner_(required)_y':'business_owner'}, inplace=True)
result.rename(columns={'technical_owner_y':'technical_owner'}, inplace=True)
result.rename(columns={'reference_id_(required)':'reference_id'}, inplace=True)

result.rename(columns={'metadata.id':'Asset Id'}, inplace=True)
result.rename(columns={'metadata.created_at':'Asset Created At'}, inplace=True)
result.rename(columns={'metadata.created_by_y':'Asset Created By'}, inplace=True)
result.rename(columns={'metadata.updated_at':'Asset Updated At'}, inplace=True)
result.rename(columns={'metadata.updated_by_y':'Asset Updated By'}, inplace=True)
result.rename(columns={'metadata.workflow_status_y': 'Workflow State Id'}, inplace=True)
result.rename(columns={'metadata.workflow_status': 'Workflow State'}, inplace=True)

list(result.columns)

### New Cell
def remove_brackets(df, columns):
	for column in columns:
	     if column in df:
	        df[column] = df[column].str[0]

fields = ['active_date',
 'availability_impact_level',
 'business_unit',
 'confidentiality_impact_level',
 'criticality_level',
 'description_(required)',
 'end_of_support_date',
 'hardware_device_type',
 'hostname_/_domain_name_/_url',
 'integrity_impact_level',
 'ip_address',
 'location',
 'manufacturer',
 'Asset Created At',
 'Asset Created By',
 'Asset Id',
 'Asset Updated At',
 'Asset Updated By',
 'model',
 'name',
 'reference_id',
 'serial_number',
 'total_asset_value',
 'business_owner',
 'technical_owner',
 'expiration_date',
 'license_id',
 'software_type',
 'version',
 'cloud_environment',
 'resource_state',
 'resource_type',
 'unique_cloud_id'
 ]

remove_brackets(result, fields)
result

### New Cell
# Match the names to the user ID
result = result.merge(df_users[['uid', 'name']], left_on='Asset Created By', right_on='uid', how='left')
result.drop('uid', axis=1, inplace=True)
result.drop('Asset Created By', axis=1, inplace=True)
result.rename(columns={'name_x':'name'}, inplace=True)
result.rename(columns={'name_y':'Asset Created By'}, inplace=True)

result = result.merge(df_users[['uid', 'name']], left_on='Asset Updated By', right_on='uid', how='left')
result.drop('uid', axis=1, inplace=True)
result.drop('Asset Updated By', axis=1, inplace=True)
result.rename(columns={'name_x':'name'}, inplace=True)
result.rename(columns={'name_y':'Asset Updated By'}, inplace=True)

result = result.merge(df_users[['uid', 'name']], left_on='technical_owner', right_on='uid', how='left')
result.drop('uid', axis=1, inplace=True)
result.drop('technical_owner', axis=1, inplace=True)
result.rename(columns={'name_x':'name'}, inplace=True)
result.rename(columns={'name_y':'technical_owner'}, inplace=True)

result = result.merge(df_users[['uid', 'name']], left_on='business_owner', right_on='uid', how='left')
result.drop('uid', axis=1, inplace=True)
result.drop('business_owner', axis=1, inplace=True)
result.rename(columns={'name_x':'name'}, inplace=True)
result.rename(columns={'name_y':'business_owner'}, inplace=True)
result

### New Cell
hcl_result = hcl.from_pandas(result)
hcl_result

### New Cell
hcl_result.to_hb_results(results_id, overwrite=True)


from django.urls import path
from . import views

urlpatterns = [
    path("", views.home, name='i_sell_homework'),
    path("python-basic", views.python_basic, name='python-basic'),
    path("django", views.django_framework, name='django-framework'),
    path("os-linux", views.os_linux, name='os-linux'),
    path("sql", views.sql, name='sql'),
    path("web-layout", views.web_layout, name='web-layout')

]


from setuptools import setup, find_packages

with open("requirements.txt") as f:
	install_requires = f.read().strip().split("\n")

# get version from __version__ variable in victory/__init__.py
from victory import __version__ as version

setup(
	name="victory",
	version=version,
	description="Custom Reporting and Custom Doctype",
	author="Atul Sah",
	author_email="atul.sah7@gmail.com",
	packages=find_packages(),
	zip_safe=False,
	include_package_data=True,
	install_requires=install_requires
)


import os

import keras
from keras.preprocessing.image import ImageDataGenerator, DirectoryIterator
from keras import backend as K
from keras.optimizers import SGD, RMSprop, Adam

import matplotlib.pyplot as plt

import pandas
import numpy as np

#DATA_FOLDER = "../data/dl/"
#DATA_FOLDER = ""
DATA_FOLDER = "bla/"

img_width = 224
img_height = 224
batch_size = 5

class MyDirectoryIterator(DirectoryIterator):
    pass

train_datagen = ImageDataGenerator(
#        rescale=1./255,
        #shear_range=0.2,
        #zoom_range=0.2,
        #horizontal_flip=True
        )

test_datagen = ImageDataGenerator(rescale=1./255)

# train_generator = train_datagen.flow_from_directory(
#         DATA_FOLDER + 'train',
#         target_size=(img_width, img_height),
#         batch_size=batch_size,
#         class_mode='binary')
#
# validation_generator = test_datagen.flow_from_directory(
#         DATA_FOLDER + 'val',
#         target_size=(img_width, img_height),
#         batch_size=batch_size,
#         class_mode='binary')

import numpy as np

# Transformation matrix for Deuteranope (a form of red/green color deficit)
lms2lmsd = np.array([[1,0,0],[0.494207,0,1.24827],[0,0,1]])
# Transformation matrix for Protanope (another form of red/green color deficit)
lms2lmsp = np.array([[0,2.02344,-2.52581],[0,1,0],[0,0,1]])
# Transformation matrix for Tritanope (a blue/yellow deficit - very rare)
lms2lmst = np.array([[1,0,0],[0,1,0],[-0.395913,0.801109,0]])
# Colorspace transformation matrices
rgb2lms = np.array([[17.8824,43.5161,4.11935],[3.45565,27.1554,3.86714],[0.0299566,0.184309,1.46709]])
lms2rgb = np.linalg.inv(rgb2lms)
# Daltonize image correction matrix
err2mod = np.array([[0,0,0],[0.7,1,0],[0.7,0,1]])

def _invoke(f, img):
    new = np.zeros_like(img)
    for i in range(new.shape[0]):
        for j in range(new.shape[1]):
            new[i, j, :3] = np.dot(f, img[i, j, :3])
    return new

def daltonize(img, mode='protanope'):
    lms = _invoke(rgb2lms, img)
    if mode == 'protanope':
        error_function = lms2lmsd
    errored = _invoke(lms2lmsd, lms)
    return _invoke(lms2rgb, errored)

from vgg16 import VGG16
model = VGG16(include_top=True, weights='imagenet')

from keras.applications.imagenet_utils import decode_predictions
from keras.applications.imagenet_utils import preprocess_input
from keras.preprocessing import image

def show_samples():
    train_generator = train_datagen.flow_from_directory(
            DATA_FOLDER + 'train',
            target_size=(img_width, img_height),
            batch_size=batch_size,
            class_mode='categorical')
    imgs, labels = train_generator.next()
    for i in range(len(imgs)):
        img = imgs[i]
        label = labels[i]
        print(label)
        print(np.argmax(label))
        # findex = samples[1][i]
        # print(findex)
        # print(img.shape)
        # fname = train_generator.filenames[int(findex)]
        # fpath = os.path.join(DATA_FOLDER, 'train', fname)
        # print(fpath)

        #img = image.load_img(fpath, target_size=(224, 224))
        imgOrig = img
        img = img.copy() * 255.0
        img = image.img_to_array(img)
        img = np.expand_dims(img, axis=0)
        img = preprocess_input(img)

        #imgOrig = img
        #img = image.img_to_array(img)
        #img = img * 255.0
        #img = np.expand_dims(img, axis=0)
        #img = preprocess_input(img)

        preds = model.predict(img)
        print('Predicted[Original]:', decode_predictions(preds))

        fig = plt.figure()
        a = fig.add_subplot(1, 2, 1)
        plt.imshow(imgOrig)
        a.set_title('Original: ')

        a = fig.add_subplot(1, 2, 2)
        daltonized = daltonize(image.img_to_array(imgOrig) / 255.0) * 255.0
        daltonized = image.array_to_img(daltonized)
        plt.imshow(daltonized)

        daltonized = daltonized.copy()
        daltonized = image.img_to_array(daltonized)
        daltonized = np.expand_dims(daltonized, axis=0)
        daltonized = preprocess_input(daltonized)
        preds = model.predict(daltonized)
        print('Predicted[Daltonized]:', decode_predictions(preds))
        #print(preds[:3])
        a.set_title('Color blindness: ')

        plt.show()
        img, label = train_generator.next()

def displayImages(train_generator):
    imgs, labels = train_generator.next()
    for i in range(len(imgs)):
        img = imgs[i]
        label = labels[i]

        imgOrig = img
        img = img.copy()
        img = image.img_to_array(img)
        img = np.expand_dims(img, axis=0) * 255.0
        img = preprocess_input(img)

        preds = model.predict(img)
        print
        print('Predicted[Original]:', decode_predictions(preds)[0][0])
        print('Correct[Original]:', decode_predictions(np.expand_dims(label, axis=0))[0][0])
        print
        plt.imshow(imgOrig)
        plt.show()

def plots(ims, figsize=(12,6), rows=1, interp=False, titles=None):
    if type(ims[0]) is np.ndarray:
        ims = np.array(ims).astype(np.uint8)
        if (ims.shape[-1] != 3):
            ims = ims.transpose((0,2,3,1))
    f = plt.figure(figsize=figsize)
    for i in range(len(ims)):
        sp = f.add_subplot(rows, len(ims)//rows, i+1)
        sp.axis('Off')
        if titles is not None:
            sp.set_title(titles[i], fontsize=16)
        plt.imshow(ims[i], interpolation=None if interp else 'none')
    plt.show()

def evaluate_network():
    train_generator = train_datagen.flow_from_directory(
            DATA_FOLDER + 'train',
            target_size=(img_width, img_height),
            batch_size=batch_size,
            class_mode='categorical')

    # validation_generator = test_datagen.flow_from_directory(
    #         DATA_FOLDER + 'val',
    #         target_size=(img_width, img_height),
    #         batch_size=batch_size,
    #         class_mode='categorical')

    #print(train_generator.num_class)
    #print(train_generator.class_indices)

    train_generator.num_class = 1000
    import json
    classIndex = json.load(open("imagenet_class_index.json"))
    #print(classIndex)
    totalFound = 0
    oldNewMapping = {}
    for index, node in classIndex.iteritems():
        folder = node[0]
        if folder in train_generator.class_indices:
            oldNewMapping[train_generator.class_indices[folder]] = int(index)
            totalFound += 1

    print(train_generator.classes)
    print(oldNewMapping)
    for i in range(len(train_generator.classes)):
        train_generator.classes[i] = oldNewMapping[train_generator.classes[i]]
    print(train_generator.classes, train_generator.classes.shape)

    print("Total found: {}".format(totalFound))
    #train_generator.class_indices = newClassIndices

    #displayImages(train_generator)

    model.compile(optimizer=Adam(lr=0.01),
                loss='categorical_crossentropy', metrics=['accuracy'])

    #results = model.evaluate_generator(train_generator, steps=5)
    #print(results)
    #print(model.metrics_names)

    #model.predict_generator(train_generator, steps=100)
    for i in range(10):
        X_train, Y_train = train_generator.next()
        X_orig = X_train.copy()
        X_train = preprocess_input(X_train)
        #X_train = X_train * 255.0
        #results = model.evaluate(X_train, Y_train, batch_size=batch_size)
        results = model.predict(X_train, batch_size=batch_size)

        correct = decode_predictions(results)
        correct = [p[0][1] for p in correct]
        plots(X_orig, titles=correct)

        #print(results)

def fit_network():
    from keras.models import Sequential
    from keras.layers import Conv2D, MaxPooling2D
    from keras.layers import Activation, Dropout, Flatten, Dense

    print(train_generator.next())
    if K.image_data_format() == 'channels_first':
        input_shape = (3, img_width, img_height)
    else:
        input_shape = (img_width, img_height, 3)

    model = Sequential()
    model.add(Conv2D(32, (3, 3), input_shape=input_shape))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(32, (3, 3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(64, (3, 3)))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Flatten())  # this converts our 3D feature maps to 1D feature vectors
    model.add(Dense(64))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(train_generator.num_class))
    model.add(Activation('sigmoid'))

    model.compile(loss='binary_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])
    model.fit_generator(
            train_generator,
            steps_per_epoch=2000 // batch_size,
            epochs=50,
            validation_data=validation_generator,
            validation_steps=800 // batch_size)
    model.save_weights('first_try.h5')

#show_samples()
evaluate_network()


from typing import List

class Solution:
    def divideString(self, s: str, k: int, fill: str) -> List[str]:
        s_list = list(s)
        cnt = 0
        res_str = ''
        res = []
        while s_list:
            res_str += s_list.pop(0)
            cnt += 1
            if cnt % k == 0:
                res.append(res_str)
                res_str = ''
        if len(res_str) < k:
            res_str += fill*(k-len(res_str))
            res.append(res_str)
        return res

if __name__ == '__main__':
    Solution().divideString(s = "abcdefghij", k = 3, fill = "x")