import os
import cv2
import timm
import torch
import numpy
import warnings
import pandas as pd
from tqdm import tqdm
import torch.nn as nn
import seaborn as sns
from scipy.stats import zscore
import matplotlib.pyplot as plt
import torch.nn.functional as F
from siren_pytorch import SirenNet
from sklearn.compose import ColumnTransformer
from torch.utils.data import Dataset, DataLoader
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import minmax_scale, StandardScaler

warnings.filterwarnings("ignore")
device = 'cuda'
class CustomDataset(Dataset):
    def __init__(self, x, y):
        self.x = x
        self.y = y

    def __len__(self):
        return len(self.x)

    def __getitem__(self, idx):
        x = self.x[idx, [0, 1, 2]]
        y = self.y[idx]
        return x, y

dataframe = pd.read_csv('/mnt/ramdisk/dados_combinados_R1.11.csv')
dataframe.drop(dataframe.columns.difference(['latitude', 'longitude', 'last_scraped [norm. PI]', 'price']), inplace=True, axis=1)
X = dataframe.drop('price', axis=1)
y = dataframe['price']
del dataframe

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=2)
del X, y

ct = ColumnTransformer(
    transformers=[
        ("scale", StandardScaler(), ['latitude', 'longitude']),
        ("pass", "passthrough", ['last_scraped [norm. PI]'])
    ])

X_train = ct.fit_transform(X_train)
X_val = ct.transform(X_val)

scaler = StandardScaler()
scaler.fit(y_train.values.reshape(-1, 1))
y_train, y_val = scaler.transform(y_train.values.reshape(-1, 1)), scaler.transform(y_val.values.reshape(-1, 1))
y_train = nn.Sigmoid()(torch.from_numpy(y_train.squeeze(1)))

X_train, X_val = torch.tensor(X_train, dtype=torch.float32), torch.tensor(X_val, dtype=torch.float32)

train_dataset = CustomDataset(X_train, y_train)
val_dataset = CustomDataset(X_val, y_val)

train_dataloader = DataLoader(train_dataset, batch_size=1024, shuffle=True, drop_last=True)
val_dataloader = DataLoader(val_dataset, batch_size=512, shuffle=True, drop_last=True)

model = SirenNet(
    dim_in = 3,                        # input dimension, ex. 2d coor
    dim_hidden = 256,                  # hidden dimension
    dim_out = 1,                       # output dimension, ex. rgb value
    num_layers = 16,                    # number of layers
    final_activation = nn.Identity(),   # activation of final layer (nn.Identity() for direct output)
    w0_initial = 30.,                   # different signals may require different omega_0 in the first layer - this is a hyperparameter
    dropout = 0.05
).to(device)
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)

fig, ax = plt.subplots()

loss_list = []
for ep in range(50) :
    inner_pbar = tqdm(train_dataloader)
    for X, y in inner_pbar:
        X = X.to(device)
        y = y.to(device)

        optimizer.zero_grad()

        y_pred = model(X)
        loss = torch.nn.L1Loss()(nn.Sigmoid()(y_pred), y)
        loss.backward()
        optimizer.step()

        inner_pbar.set_postfix({"L1": "{:.6f}".format(loss.item())})
        loss_list.append(loss.item())

ax.plot(range(len(loss_list)), loss_list, color='red', label='MAE')
ax.set_title('Erro de treinamento do modelo')
ax.set_xlabel('Número de Interações')
ax.set_ylabel('MAE')
ax.legend()

plt.savefig('/mnt/ramdisk/treinamento_siren.png')

ax.clear()

predictions = []
true_y = []

inner_pbar = tqdm(val_dataloader)
for X, y in inner_pbar:
    X = X.to(device)
    y = y.to(device)

    y_pred = model(X)
    predictions.append(y_pred)
    true_y.append(y)

    inner_pbar.set_postfix({"L1": "{:.6f}".format(loss.item())})

predictions = numpy .concatenate([p.cpu().detach().numpy() for p in predictions], axis=0)
true_y = numpy.concatenate([y.cpu().detach().numpy() for y in true_y], axis=0)

predictions = scaler.inverse_transform(predictions)
true_y = scaler.inverse_transform(true_y)

predictions = predictions.mean()
true_y = true_y.mean()

L1 = numpy.abs(predictions - true_y)
print(f"L1 = {L1:.4f}")


import logging as log
from common.config import DEFAULT_TABLE
from indexer.index import milvus_client, create_table, insert_vectors, create_index, has_table
from indexer.tools import connect_mysql, create_table_mysql, search_by_image_id, load_data_to_mysql
import datetime
import time
from indexer.logs import write_log
import uuid
import os

def get_img_ids(conn, cursor, ids_image, img, table_name):
    img_list = []
    ids_img = []
    info = []

    for i in range(len(ids_image)):
        has_id = search_by_image_id(conn, cursor, ids_image[i], table_name)
        if has_id:
            print("The id of image has exists:", ids_image[i])
            info.append(ids_image[i])
            continue
        else:
            img_list.append(img[i])
            ids_img.append(ids_image[i])

    return img_list, ids_img, info

def get_ids_file(ids_milvus, ids_image, file_name):
    with open(file_name,'w') as f:
        for i in range(len(ids_image)):
            line = str(ids_milvus[i]) + "," + ids_image[i] + '\n'
            f.write(line)

def init_table(index_client, conn, cursor, table_name):
    status, ok = has_table(index_client, table_name)
    print("has_table:", status, ok)
    if not ok:
        print("create table.")
        create_table(index_client, table_name)
        create_index(index_client, table_name)
        create_table_mysql(conn, cursor, table_name)

def insert_img(index_client, conn, cursor, img_to_vec, insert_img_list, insert_ids_img, table_name):
    vectors_img = img_to_vec(insert_img_list)
    # print(len(insert_img_list),len(insert_ids_img))
    status, ids_milvus = insert_vectors(index_client, table_name, vectors_img)

    file_name = str(uuid.uuid1()) + ".csv"
    get_ids_file(ids_milvus, insert_ids_img, file_name)
    print("load data to mysql:", file_name)
    load_data_to_mysql(conn, cursor, table_name, file_name)
    if os.path.exists(file_name):
        os.remove(file_name)
    return status

def do_insert(index_client, conn, cursor, img_to_vec, ids_image, img, size, table_name):
    if not table_name:
        table_name = DEFAULT_TABLE
    if not size:
        size = 200
    print("table_name:", table_name, ", num of orgin ids:", len(ids_image),", num of orgin img:", len(img))

    if len(ids_image)!= len(img):
        return "The number of pictures is not consistent with the ID number, please check!", None

    init_table(index_client, conn, cursor, table_name)
    img_list, ids_img, info = get_img_ids(conn, cursor, ids_image, img, table_name)
    print("num of the insert images:", len(img_list))
    if not img_list:
        return None, "All the image id exists!"
    try:
        i = 0
        while i+size<len(ids_img):
            insert_img_list = img_list[i:i+size]
            insert_ids_img = ids_img[i:i+size]
            i = i+size
            print("doing insert, size:", size, "the num of insert vectors:", len(insert_img_list))
            status = insert_img(index_client, conn, cursor, img_to_vec, insert_img_list, insert_ids_img, table_name)
        else:
            insert_img_list = img_list[i:len(ids_image)]
            insert_ids_img = ids_img[i:len(ids_image)]
            print("doing insert, size:", size, ",the num of insert vectors:", len(insert_img_list))
            status = insert_img(index_client, conn, cursor, img_to_vec, insert_img_list, insert_ids_img, table_name)

        return status, info
    except Exception as e:
        # log.error(e)
        write_log(e, 1)
        return None, "Error with {}".format(e)


from torch.utils.data import Dataset
import numpy as np
# import main as env
import math
import pandas as pd
from sklearn.model_selection import train_test_split
import torch

from imblearn.over_sampling import SMOTE
from imblearn.combine import SMOTETomek

class SmartHomeDataset(Dataset):
    def __init__(self, filename, random_state = 10, rawdata = None, classes = None, permu_task_order=False):
        if rawdata is None:
            self.pddata = pd.read_csv(filename).dropna()
        else:
            self.pddata = rawdata

        self.classes = classes
        if permu_task_order:
            lst = np.random.permutation(classes)
            self.classes = lst[0:10]

        # print("Task order: ", self.classes)

        self.pddata = self.pddata[self.pddata['ActivityName'].isin(self.classes)]
        self.labels = self.pddata["ActivityName"].values
        self.arr = self.pddata.drop('ActivityName', 1).values

        self.random_state = random_state

        self.config = {
            'feature': len(self.pddata.columns) - 1,
            'classes': len(self.classes)
        }

        self.target_tranform = None

    def train_test_split(self):
        traindata, testdata = train_test_split(self.pddata, test_size=0.2, random_state=self.random_state)

        traindata = SmartHomeDataset("", rawdata=traindata, classes=self.classes)
        testdata = SmartHomeDataset("", rawdata=testdata, classes=self.classes)

        return (traindata, testdata)

    def set_target_tranform(self, transform):
        self.target_tranform = transform

    def __len__(self):
        return len(self.pddata)

    def __getitem__(self, idx):
        if self.target_tranform is not None:
            return torch.FloatTensor(list(self.arr[idx])), self.target_tranform(self.labels[idx])

        raise Exception("No key `"+self.labels[idx]+"` in classmap")
        # return torch.FloatTensor(list(self.arr[idx])), self.labels[idx]

    def __str__(self):
        return "Smart Home dataset"

    def resampling(self):

        # SMOTE
        # It will synthesize a new data for the minority class, based on those that already exist.

        # Tomek links
        # It pairs an instance with another instance but of opposite classes
        # and then remove the instances of the bigger class of each pair.

        smote = SMOTE(random_state=self.random_state)

        classes = self.classes
        mapping = dict(zip(classes, range(len(classes))))

        pddata = self.pddata.replace({'ActivityName': mapping})
        labels = pddata["ActivityName"]

        pddata = pddata.drop('ActivityName', 1)
        columns = pddata.columns

        X_sm, y_sm = smote.fit_sample(pddata, labels)

        # convert the output back to pandas
        X_sm = pd.DataFrame(data=X_sm, columns=columns)
        rmapping = dict(zip(range(len(classes)), classes))
        X_sm["ActivityName"] = y_sm
        X_sm = X_sm.replace({'ActivityName': rmapping})

        return SmartHomeDataset("", rawdata=X_sm, classes=self.classes)

    def split(self, tasks=5):

        classes_per_task = int(np.floor(len(self.classes) / tasks))
        datasets = []

        labels_per_task = [
            list(np.array(range(classes_per_task)) + classes_per_task * task_id) for task_id in range(tasks)
        ]

        # split them up into sub-tasks
        datasets = []
        for labels in labels_per_task:
            datasets.append(self.filter(labels))

        return datasets, self.config, classes_per_task

    def filter(self, labels):
        classes = [self.classes[l] for l in labels]
        filtered = self.pddata[self.pddata["ActivityName"].isin(classes)]

        return SmartHomeDataset("", rawdata=filtered, classes=classes)

    def merge(self, dataset):
        dataset.pddata.columns = self.pddata.columns
        mpddata = pd.concat([self.pddata, dataset.pddata], axis=0, ignore_index=True)

        mpddata = mpddata.sample(frac=1).reset_index(drop=True)
        classes = self.classes + dataset.classes
        return SmartHomeDataset("", rawdata=mpddata, classes=classes)

    def detail(classes, train_datasets, test_datasets):
        print ("===== Smart Home dataset ======")
        d = {'Activity Name': [], '#Train': [], "#Test": [], "Total":[]}

        cid = {}
        i = 0
        for c in classes:
            d["Activity Name"].append(c)
            d["#Train"].append(0)
            d["#Test"].append(0)
            d["Total"].append(0)

            cid[c] = i
            i += 1

        for dataset in train_datasets:
            data = dataset.pddata
            for c in classes:
                tr = data[data["ActivityName"] == c]
                d["#Train"][cid[c]] += len(tr)

                d["Total"][cid[c]] += len(tr)

        for dataset in test_datasets:
            data = dataset.pddata
            for c in classes:
                tr = data[data["ActivityName"] == c]
                d["#Test"][cid[c]] += len(tr)
                d["Total"][cid[c]] += len(tr)

        df = pd.DataFrame(data=d)
        print (df)


import collections
import datetime
import getpass
import grp
import io
import itertools
import logging
import operator
import os
import os.path
import pathlib
import shutil
import stat
import sys
import typing
from functools import partial
from typing import Optional, Union, Iterable, Iterator, TextIO, Tuple

import jinja2
import netaddr
from jinja2.exceptions import FilterArgumentError
try:
    from jinja2.filters import pass_environment
except ImportError:
    # Jinja2 <3
    # noinspection PyUnresolvedReferences
    from jinja2.filters import environmentfilter as pass_environment

from hades import constants

logger = logging.getLogger(__name__)

template_filters: dict[str, typing.Callable] = {}

def template_filter(name):
    def decorator(f):
        template_filters[name] = f
        return f

    return decorator

@template_filter('unique')
def do_unique(a):
    return set(a)

@template_filter('intersection')
def do_intersection(a, b):
    return set(a).intersection(set(b))

@template_filter('difference')
def do_difference(a, b):
    return set(a).difference(set(b))

@template_filter('symmetric_difference')
def do_symmetric_difference(a, b):
    return set(a).symmetric_difference(set(b))

@template_filter('union')
def do_union(a, b):
    return set(a).union(set(b))

@template_filter('min')
def do_min(a):
    return min(a)

@template_filter('max')
def do_max(a):
    return max(a)

@template_filter('sorted')
@pass_environment
def do_sorted(env, iterable, *, attribute=None, item=None, reverse=False):
    if attribute is None and item is None:
        key = None
    elif attribute is not None and item is not None:
        raise FilterArgumentError("Only one of attribute and item may be"
                                  "specified")
    elif attribute is not None:
        key = partial(env.getattr, attribute=attribute)
    elif item is not None:
        key = partial(env.getitem, argument=item)
    return sorted(iterable, key=key, reverse=reverse)

@template_filter('zip')
def do_zip(*iterables):
    return zip(*iterables)

@template_filter('zip_longest')
def do_zip_longest(*iterables, fillvalue=None):
    return itertools.zip_longest(*iterables, fillvalue=fillvalue)

@template_filter('dirname')
def do_dirname(a):
    return os.path.dirname(a)

PathArg = Union[str, pathlib.PurePath]

def yield_all_sources(
    *search_path: pathlib.Path,
) -> Iterator[Tuple[pathlib.Path, pathlib.Path]]:
    """Find all files in the directories on ``search_path``.

    The algorithm is similar to systemd's unit search or the shell's ``PATH``
    search algorithm.

    The directories on the search path are recursively walked in parallel.
    Directories at the beginning of ``search_path`` take precedence over later
    directories, thus for files with the same name, the version in the
    leftmost directory that contains the file is selected. The content of
    directories is merged.

    Symbolic links are never followed and the path of symbolic link is returned
    instead of the path of its target.

    Symbolic links to ``/dev/null`` are handled specially and denote a
    *deletion marker*, that indicates that a file or directory should not be
    included. For directories this means that no directories and their contents
    to the right of the respective directory in the search path should be
    included. Version of the directory and their contents that are on the search
    path to the left of the deletion marker are still included.

    The kind of paths that are returned depend on the kind of paths, that are
    passed as ``search_path``. Relative search path components will yield
    relative paths and absolute components absolute paths.

    :param search_path: A list of paths to directories
    :return: An iterator that yields pairs, where the first element is the
     component of ``search_path``, where the file was found and the second
     element is the path of the file.
    """
    if logger.isEnabledFor(logging.DEBUG):
        logger.debug("Searching for all sources in %s",
                     ', '.join(map(str, search_path)))
    # Perform a bread-first search via a queue.
    # The queue contains a sequence of (search-path, file) pairs for each
    # version of a file on the search path
    queue: typing.Deque[
        typing.Sequence[typing.Tuple[
            pathlib.Path,  # base
            pathlib.Path,  # children
        ]]
    ] = collections.deque()
    queue.appendleft(tuple(zip(search_path, search_path)))
    while queue:
        sources = queue.pop()
        if not sources:
            raise ValueError("No search paths")
        else:
            relative_source = sources[0][1].relative_to(sources[0][0])

        # If the current file is a directory, we collect all versions of it in
        # a list, for all other elements we break the loop and continue with the
        # next element. If an error occurs we continue with the next version of
        # the file instead.
        directories: typing.List[
            typing.Tuple[
                pathlib.Path,  # base
                pathlib.Path,  # source
                typing.Dict[
                    str,  # child.name
                    pathlib.Path  # child
                ]
            ]
        ] = []
        for base, source in sources:
            if source.is_symlink():
                target = os.readlink(str(source))
                # Check for deletion marker
                if target != os.path.devnull:
                    if directories:
                        logger.error("%s is a symlink, while %s is a directory",
                                     source, directories[-1][1])
                        continue
                    else:
                        logger.debug("Found symlink %s to %s in %s",
                                     relative_source, target, base)
                        yield base, source
                else:
                    logging.debug("Skipping %s due to deletion marker in %s",
                                  relative_source, base)
                break
            elif source.is_dir():
                logger.debug("Found directory %s in %s", relative_source, base)
                directories.append(
                    (base, source, {
                        child.name: child
                        for child in source.iterdir()
                    })
                )
            elif source.is_file():
                logger.debug("Found file %s in %s", relative_source, base)
                if directories:
                    logger.error("%s is not a directory, while %s is",
                                 source, directories[-1])
                    continue
                yield base, source
                break
            else:
                logger.error("%s must be a regular file, directory, or symlink",
                             source)
                continue

        if directories:
            if logger.isEnabledFor(logging.DEBUG):
                dir_names = ', '.join(
                    str(base) for base, directory, children in directories)
                logger.debug("Descending into %s in %s",
                             relative_source, dir_names)
            # Merge all names of children
            names = set(itertools.chain(*(
                children.keys()
                for base, directory, children in directories
            )))
            for name in names:
                queue.appendleft(tuple(
                    (base, children[name])
                    for base, directory, children in directories
                    if name in children
                ))
            yield directories[0][:2]

def yield_all_versions(
    name: pathlib.PurePath,
    *search_path: pathlib.Path,
) -> Iterator[Tuple[pathlib.Path, pathlib.Path]]:
    """Find all versions of `name` on `search_path`.

    The principle behind the search algorithm are the same as
    :func:`yield_all_sources`. This function however will search for a single
    file or directory named ``name`` and return all versions of ``name`` that
    can be found on the ``search_path``. The version with the highest precedence
    (i.e. from the leftmost component of ``search_path`` that contains a version
    of ``name``) will be returned first.

    :param name: The name of the file or directory
    :param search_path: A list of of paths to directories
    :return: An iterator that yields pairs, where the first element is the
     component of ``search_path``, where the file was found and the second
     element is the path of the file.
    """
    if logger.isEnabledFor(logging.DEBUG):
        logger.debug("Searching for versions of %s in %s", name,
                     ', '.join(map(str, search_path)))
    if not search_path:
        raise ValueError("Empty search path")

    for directory in search_path:
        filename = directory
        # Check for invalid components and /dev/null deletion markers in all
        # components of the path
        for idx, part in enumerate(name.parts):
            if filename.is_symlink():
                raise ValueError("Intermediate components can't be symbolic "
                                 "links")
            if part == os.path.pardir or part == os.path.sep:
                raise ValueError("Illegal component {!r} at index {} of path {}"
                                 .format(part, idx, name))
            filename = filename / part
            # Abort search if deletion marker found
            if (filename.is_symlink() and
                    os.readlink(str(filename)) == os.path.devnull):
                logger.debug("Skipping %s due to deletion marker at %s",
                             directory, filename)
                return
            if not filename.exists() and not filename.is_symlink():
                break
        # Yield if no break occurred
        else:
            logger.debug("Found version of %s in %s at %s",
                         name, directory, filename)
            yield directory, filename

class OverridableFileSystemLoader(jinja2.BaseLoader):
    """A Jinja2 loader that loads templates with the :func:`yield_all_sources`
    algorithm, if the template name is a relative.

    The loader accepts :class:`pathlib.PurePath` objects in addition to strings
    to identify templates.
    """

    def __init__(self, search_paths, encoding='utf-8', template_suffix='.j2'):
        self.search_paths = search_paths
        self.encoding = encoding
        self.template_suffix = template_suffix

    def get_source(self, environment: jinja2.Environment,
                   template: str):
        path_template = pathlib.Path(template)
        if not path_template.is_absolute():
            try:
                base, path_template = next(yield_all_versions(path_template,
                                                         *self.search_paths))
            except StopIteration:
                raise jinja2.TemplateNotFound(path_template.name) from None
        try:
            with path_template.open('r', encoding=self.encoding) as f:
                contents = f.read()
        except (FileNotFoundError, IsADirectoryError):
            raise jinja2.TemplateNotFound(path_template.name)

        mtime = path_template.stat().st_mtime

        def has_changed():
            """Checks whether the template loaded via
            :class:`OverridableFileSystemLoader` has changed.

            This function won't detect new versions of the template in other
            search paths with higher precedence or new deletion markers in other
            search paths. We would additionally need a list of all parent
            directories in the other search paths and check all of them."""
            try:
                return path_template.stat().st_mtime == mtime
            except OSError:
                return False

        return contents, str(path_template), has_changed

    def list_templates(self):
        return set(str(path.relative_to(base))
                   for base, path in yield_all_sources(*self.search_paths)
                   if not path.is_symlink() and path.is_file()
                   and path.suffix == self.template_suffix)

class GeneratorError(Exception):
    pass

class ConfigGenerator:
    DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S%z"
    """Format string for producing RFC3339-compatible datetime strings"""
    TEMPLATE_SUFFIX = ".j2"

    def __init__(self, template_dirs: Union[PathArg, Iterable[PathArg]],
                 config, mode: int = 0o0750,
                 group: Optional[grp.struct_group] = None):
        self._current_user = getpass.getuser()
        self.group = group
        mode = stat.S_IMODE(mode)
        # The mode could be masked with much less characters.
        # The verbosity of these statements is for clarity and documentation.
        # Allow only read, write, sticky, and setgid for directories
        self.dir_mode = mode & (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
                                stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH |
                                stat.S_ISVTX | stat.S_ISGID)
        # Set execution bits if corresponding read bits are set
        self.dir_mode |= stat.S_IXUSR if stat.S_IRUSR else 0
        self.dir_mode |= stat.S_IXGRP if stat.S_IRGRP else 0
        self.dir_mode |= stat.S_IXOTH if stat.S_IROTH else 0
        # Allow only read, write, and sticky for files
        self.file_mode = mode & (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
                                 stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH |
                                 stat.S_ISVTX)
        self.config = config
        if isinstance(template_dirs, (pathlib.PurePath, str)):
            template_dirs = (template_dirs, )
        self.template_dirs = tuple(pathlib.Path(template_dir).resolve()
                                   for template_dir in template_dirs)
        self.env = jinja2.Environment(
            loader=OverridableFileSystemLoader(
                self.template_dirs,
                template_suffix=self.TEMPLATE_SUFFIX
            ),
            auto_reload=False, autoescape=False, keep_trailing_newline=True,
            undefined=jinja2.StrictUndefined,
            extensions=['jinja2.ext.do', 'jinja2.ext.loopcontrols',
                        'jinja2.ext.with_'],
            trim_blocks=True, lstrip_blocks=True,
        )
        self.env.globals.update({
            'collections': collections,
            'itertools': itertools,
            'netaddr': netaddr,
            'dirname': os.path.dirname,
            'constants': constants,
        })
        self.env.filters.update(template_filters)

    def _format_search_path(self):
        return ':'.join(map(str, self.template_dirs))

    def _setgroup(self, destination: pathlib.Path):
        if self.group is not None:
            os.chown(str(destination), -1, self.group.gr_gid,
                     follow_symlinks=False)

    def _setstat(self, destination: pathlib.Path, mode: int):
        destination.chmod(mode)
        self._setgroup(destination)

    def generate(self, source: PathArg, destination: Optional[PathArg]):
        source = pathlib.PurePath(source)
        sources = tuple(yield_all_versions(source, *self.template_dirs))
        if not sources:
            raise GeneratorError(
                "No file or directory named {} found (search path: {})".format(
                    str(source), self._format_search_path()
                )
            ) from None
        base, source = sources[0]
        if source.is_dir():
            if destination is None:
                raise GeneratorError(
                    "Destination can't be stdout, if source is a directory"
                )
            destination = pathlib.Path(destination)
            search_path = tuple(map(operator.itemgetter(1), sources))
            self._do_generate_directory(search_path, destination)
        else:
            self._do_generate_file(base, source, destination)

    def generate_directory(self, source: PathArg, destination: PathArg):
        source = pathlib.PurePath(source)
        if source.is_absolute():
            raise GeneratorError("Source {} must be a relative path"
                                 .format(source))
        sources = tuple(map(operator.itemgetter(1), yield_all_versions(source)))
        if not sources:
            raise GeneratorError("No directory named {} found (search path {})"
                                 .format(source, self._format_search_path()))
        destination = pathlib.Path(destination)
        self._do_generate_directory(sources, destination)

    def _do_generate_directory(self, sources: Tuple[pathlib.Path, ...],
                               destination: pathlib.Path):
        destination_base = destination
        logger.info("Generating %s from %s", destination,
                    ', '.join(map(str, sources)))
        for base, source in yield_all_sources(*sources):
            destination = destination_base / source.relative_to(base)
            if source.is_symlink():
                self._create_symlink(source, destination)
            elif source.is_dir():
                if destination.exists():
                    logger.debug("Clearing directory %s", destination)
                    # Clear destination directory contents
                    if destination.is_symlink() or not destination.is_dir():
                        raise GeneratorError("Destination {} is not a directory"
                                             .format(destination))
                    for path in destination.iterdir():
                        if path.is_dir():
                            shutil.rmtree(str(path))
                        else:
                            path.unlink()
                else:
                    logger.debug("Creating directory %s", destination)
                    destination.mkdir(self.dir_mode, exist_ok=True)
                shutil.copystat(str(source), str(destination),
                                follow_symlinks=False)
                self._setstat(destination, self.dir_mode)
            else:
                if source.suffix == self.TEMPLATE_SUFFIX:
                    destination = destination.with_name(destination.stem)
                    self._generate_template_to_file(base, source, destination)
                else:
                    self._copy_to_file(source, destination)

    def generate_file(self, source: PathArg, destination: Optional[PathArg]):
        source = pathlib.PurePath(source)
        try:
            base, source = next(yield_all_versions(source, *self.template_dirs))
        except StopIteration:
            raise GeneratorError("File {} not found (search path {})"
                                 .format(source, self._format_search_path())
                                 ) from None
        if not source.is_file() and not source.is_symlink():
            raise GeneratorError("{} is not a file".format(source))
        destination = pathlib.Path(destination)
        self._do_generate_file(base, source, destination)

    def _do_generate_file(self, base: pathlib.Path, source: pathlib.Path,
                          destination: Optional[PathArg]):
        if destination is None:
            if source.suffix == self.TEMPLATE_SUFFIX:
                self._generate_template_to_stdout(base, source)
            else:
                self._copy_to_stdout(source)
        else:
            destination = pathlib.Path(destination)
            if source.is_symlink():
                self._create_symlink(source, destination)
            elif source.suffix == self.TEMPLATE_SUFFIX:
                if destination.is_dir():
                    destination = destination / source.stem
                self._generate_template_to_file(base, source, destination)
            else:
                if destination.is_dir():
                    destination = destination / source.name
                self._copy_to_file(source, destination)

    def _create_symlink(self, source: pathlib.Path, destination: pathlib.Path):
        target = pathlib.Path(os.readlink(str(source)))
        logger.debug("Creating symlink %s to %s", destination, target)
        destination.symlink_to(target)
        self._setgroup(destination)

    def _copy_to_file(self, source: pathlib.Path, destination: pathlib.Path):
        logger.debug("Copying %s to %s", source, destination)
        shutil.copy2(str(source), str(destination), follow_symlinks=False)
        self._setstat(destination, self.file_mode)

    def _copy_to_stdout(self, source: pathlib.Path):
        logger.debug("Copying %s to stdout", source)
        self._copy_to_fd(source, sys.stdout.fileno())

    def _copy_to_fd(self, source: pathlib.Path, fd: int):
        with source.open('r') as file:
            end = file.seek(0, io.SEEK_END)
            os.sendfile(fd, file.fileno(), 0, end)

    def _create_file(self, name: pathlib.Path) -> int:
        flags = os.O_WRONLY | os.O_CLOEXEC | os.O_EXCL | os.O_CREAT
        return os.open(str(name), flags, mode=self.file_mode)

    def _generate_template_to_file(self, base: pathlib.Path,
                                   source: pathlib.Path,
                                   destination: pathlib.Path):
        destination = pathlib.Path(destination)
        # Safely create file:
        # 1. Create with O_CREAT | O_EXCL
        # 2. Unlink if fails
        # 3. Try again
        logger.info("Creating %s from template %s", destination, source)
        try:
            fd = self._create_file(destination)
        except FileExistsError:
            destination.unlink()
            fd = self._create_file(destination)
        self._setgroup(destination)
        with os.fdopen(fd, mode='w', encoding='utf-8') as writer:
            self._generate_template_to_writer(
                base, source, writer,
                destination_dir=destination.parent,
                destination=destination)

    def _generate_template_to_stdout(self, base: pathlib.Path,
                                     source: pathlib.Path):
        logger.info("Instantiating template %s", source)
        self._generate_template_to_writer(base, source, sys.stdout)

    def _generate_template_to_writer(self, base: pathlib.Path,
                                     source: pathlib.Path,
                                     writer: TextIO, **extra_variables):
        try:
            template = self.env.get_template(str(source))
        except jinja2.TemplateNotFound as e:
            raise GeneratorError("Template {} not found (search path {})"
                                 .format(source,
                                         self._format_search_path())
                                 ) from e
        now = datetime.datetime.now(datetime.timezone.utc).astimezone()
        comment = ("Generated on {} from {} by {}"
                   .format(now.strftime(self.DATETIME_FORMAT), source,
                           self._current_user))
        relative_source = source.relative_to(base)
        stream = template.stream(**self.config, **extra_variables,
                                 comment=comment,
                                 source_base=base,
                                 source=relative_source,
                                 source_dir=relative_source.parent)
        writer.writelines(stream)


import sqlite3
import re
import jwt
import base64

from flask import request, make_response
from utils.util import f, secret_key

from flask_restful import Resource, reqparse
from flask_jwt_extended import jwt_required, get_jwt_identity

from models.user import UserModel
from models.card import CardModel

class Card(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument("card_type",
                        type=str,
                        required=True,
                        help='card_number is missing')
    parser.add_argument("card_no",
                        type=str,
                        required=True,
                        help='card_number is missing')
    parser.add_argument("cvv",
                        type=str,
                        required=True,
                        help='cvv is missing')
    parser.add_argument("account_holder",
                        type=str,
                        required=True,
                        help='id is missing')
    parser.add_argument("phone_number",
                        type=str,
                        required=True,
                        help='id is missing')
    parser.add_argument("expiry_date",
                        type=str,
                        required=True,
                        help='expiry date is missing')

    @classmethod
    @jwt_required()
    def get(cls, username):
        try:
            if not check(username):
                return {'message': 'invalid username or password'}, 401
            username = username.lower()

            identity = get_jwt_identity()['identity']
            user = UserModel.find_by_username(username)
            user_identity = UserModel.find_by_userid(identity)

            if user:
                if user.id != user_identity.id or user.username != user_identity.username:
                    return {'message': 'invalid token'}, 401

                # bad query
                # query = "SELECT * FROM cards WHERE id in (SELECT id FROM users WHERE username='" + username + "')"

                query = "SELECT * FROM cards WHERE id in (SELECT id FROM users WHERE username=?)"

                connection = sqlite3.connect('data.db')
                cursor = connection.cursor()

                cards = []
                for _id, card_type, card_number, cvv, account_holder, phone_number, expiry_date in cursor.execute(query,
                                                                                (
                                                                                username,)):
                    cards.append(
                        {
                            "id": _id,
                            "card_type": f.decrypt(card_type).decode(),
                            "card_no": f.decrypt(card_number).decode(),
                            "cvv": f.decrypt(cvv).decode(),
                            "account_holder": f.decrypt(account_holder).decode(),
                            "phone_number": f.decrypt(phone_number).decode(),
                            "expiry_date": f.decrypt(expiry_date).decode()
                        }
                    )
                return {"username": username, "cards": cards}, 200
            else:
                return {'message': 'no user found'}, 404

        except ErrorException:
            return {'message': 'some error has occurred'}

    @classmethod
    @jwt_required()
    def post(cls, username):

        try:
            identity = get_jwt_identity()['identity']
            verify_csrf_token = request.headers['X-CSRFToken']

            token = request.cookies.get('token')

            if token_check(token) == -1:
                return {'message': 'invalid token in token check'}, 401

            data = jwt.decode(token, secret_key, algorithms=['HS256'])

            sub = data.get('sub')

            if sub is None:
                return {'message': 'invalid token'}, 401

            csrf_token = sub.get('csrf_token')

            if csrf_token is None:
                return {'message': 'token missing'}, 401

            if csrf_token != verify_csrf_token:
                return {'message': 'login required'}, 401

            if not check(username):
                return {'message': 'invalid username or password'}, 401

            username = username.lower()
            user = UserModel.find_by_username(username)
            user_identity = UserModel.find_by_userid(identity)

            if user:
                if user.id != user_identity.id or user.username != user_identity.username:
                    return {'message': 'invalid token'}, 401

                data = Card.parser.parse_args()

                _id = user.id
                card_type = f.encrypt(data['card_type'].encode())
                card_no = f.encrypt(data['card_no'].encode())
                cvv = f.encrypt(data['cvv'].encode())
                account_holder = f.encrypt(data['account_holder'].encode())
                phone_number = f.encrypt(data['phone_number'].encode())
                expiry_date = f.encrypt(data['expiry_date'].encode())

                # check if card with the provided number exists or not in the database
                if CardModel.find_by_card_number(card_no):
                    return {'message': 'Card already exists'}

                query = "INSERT INTO cards VALUES (?, ?, ?, ?, ?, ?, ?)"
                connection = sqlite3.connect('data.db')
                cursor = connection.cursor()
                cursor.execute(query, (_id, card_type, card_no, cvv, account_holder, phone_number, expiry_date))

                connection.commit()
                connection.close()
                response = make_response(
                    {'message': 'card added'},
                    201
                )
                response.headers.add('Access-Control-Allow-Credentials', True)
                return response
            else:
                return {'message': 'no user exist'}, 404
        except TokenInvalidException:
            return {'message': 'token invalid or not provided'}, 400

class TokenInvalidException(Exception):
    pass

class ErrorException(Exception):
    pass

# checking for any special character
def check(string):
    regex = re.compile("['@_!#$%^&*()<>?/|}{~:]")
    if regex.search(string) is None:
        return True

    else:
        return False

def token_check(token):
    try:
        # 1. verify that the token has three components
        components = token.split('.')
        signature = components[2]
        if len(components) != 3:
            return -1

        # 2. check algorithm
        header = jwt.get_unverified_header(token)
        algorithm = header.get('alg')
        if algorithm is None or algorithm == 'RS256':
            return -1

        # verifying the signature
        payload = jwt.decode(token, secret_key, algorithms=['HS256'])

        encoded_jwt = jwt.encode(
            payload,
            secret_key,
            algorithm='HS256',
            headers=header
        )

        if encoded_jwt != token:
            return -1
        return 1
    except Exception as e:
        print(e.args)
        return -1

def is_base64(s):
    try:
        return base64.b64encode(base64.b64decode(s)) == s
    except Exception:
        return False


# USAGE
# python gradient_descent.py

# import the necessary packages
import matplotlib.pyplot as plt
from sklearn.datasets.samples_generator import make_blobs
import numpy as np
import argparse

def sigmoid_activation(x):
	return 1.0 / (1 + np.exp(-x))

def next_training_batch(X, y, batchSize):
    for i in np.arange(0, X.shape[0], batchSize):
        yield (X[i:i + batchSize], y[i:i + batchSize])

ap = argparse.ArgumentParser()
ap.add_argument("-e", "--epochs", type=float, default=100,
	help="# of epochs")
ap.add_argument("-a", "--alpha", type=float, default=0.01,
	help="learning rate")
ap.add_argument("-b", "--batch-size", type=int, default=32,
    help="size of SGD mini-batches")
args = vars(ap.parse_args())

(X, y) = make_blobs(n_samples=250, n_features=2, centers=2,
    cluster_std=1.05, random_state=20)
X = np.c_[np.ones((X.shape[0])), X]

print("[INFO] starting training...")
W = np.random.uniform(size=(X.shape[1],))

lossHistory = []

for epoch in np.arange(0, args["epochs"]):
    epochLoss = []

    for (batchX, batchY) in next_training_batch(X, y, args["batch_size"]):

        preds = sigmoid_activation(batchX.dot(W))

        error = preds - batchY

        loss = np.sum(error ** 2)
        epochLoss.append(loss)

        gradient = batchX.T.dot(error) / batchX.shape[0]

        W += -args["alpha"] * gradient

    lossHistory.append(np.average(epochLoss))
    print("[INFO] epoch #{}, loss={:.7f}".format(epoch + 1, loss))

for i in np.random.choice(250, 10):
	activation = sigmoid_activation(X[i].dot(W))

	label = 0 if activation < 0.5 else 1

	# show our output classification
	print("activation={:.4f}; predicted_label={}, true_label={}".format(
		activation, label, y[i]))

Y = (-W[0] - (W[1] * X)) / W[2]
plt.figure()
plt.scatter(X[:, 1], X[:, 2], marker="o", c=y)
plt.plot(X, Y, "r-")

fig = plt.figure()
plt.plot(np.arange(0, args["epochs"]), lossHistory)
fig.suptitle("Training Loss")
plt.xlabel("Epoch #")
plt.ylabel("Loss")
plt.show()


import xml.etree.ElementTree as ET
import gpxpy
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from datetime import datetime,timedelta
import osmnx as ox
import networkx as nx
import xml.sax
import fiona
import csv
from rtree import index
import math
from math import exp, sqrt
from datetime import datetime
import itertools
from collections import Counter, defaultdict
from pyproj import Proj, transform
from sklearn.neighbors import NearestNeighbors
import scipy
from scipy import spatial
import xml.sax
import statistics
import ast
from os import listdir,path
import time
from numpy import ma
from pykalman import KalmanFilter
import pywt
from sklearn.metrics import confusion_matrix
from scipy.stats import entropy

from sklearn.cluster import KMeans
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.neural_network import MLPClassifier
from sklearn import svm, metrics
import hdbscan

plt.rcParams.update({'font.size': 20})

FMT = '%H:%M:%S'
def DTtoFloat(dstring):
    x = datetime.strptime(dstring, FMT)
    return x.hour * 3600 + x.minute * 60 + x.second

def lowpassfilter(signal, thresh, wavelet="db3"):
    thresh = thresh*np.nanmax(signal)
    coeff = pywt.wavedec(signal, wavelet, mode="per" )
    coeff[1:] = (pywt.threshold(i, value=thresh, mode="soft" ) for i in coeff[1:])
    reconstructed_signal = pywt.waverec(coeff, wavelet, mode="per" )
    return reconstructed_signal

def calculate_entropy(list_values):
    counter_values = Counter(list_values).most_common()
    probabilities = [elem[1]/len(list_values) for elem in counter_values]
    entropy=scipy.stats.entropy(probabilities)
    return entropy

def Kmeans(data):
    kmeans = KMeans(n_clusters=2, random_state=0).fit(data)
    eL = kmeans.labels_

    return eL

def RF(X,y):

    X, y = make_classification(n_samples=1000, n_features=4,
                           n_informative=2, n_redundant=0, random_state=0, shuffle=False)
    clf = RandomForestClassifier(max_depth=2, random_state=0)
    clf.fit(X, y)

    #RandomForestClassifier(...)
    #print(clf.predict([[0, 0, 0, 0]]))

def SWF(testdf,predDWT,delTs): ## SLIDING WINDOW FILTERING
    ## sequential sliding window filtering > Check for consecutive 0 and 1

    predictDWT = []
    lastGroupValue = -1 # create a flag to keep track of the window:
    #Grouped_item = [list(group) for idx, (key, group) in enumerate(itertools.groupby(predDWT))]
    end_index = 0
    for idx, (key, group) in enumerate(itertools.groupby(predDWT)):
        GrpList = list(group)
        #print(key, GrpList)
        start_index = end_index
        end_index = start_index + len(GrpList) # end index is the number of elements in the grp
        tempTimeOfStay = sum(delTs[start_index:end_index])
        #print(tempTimeOfStay)
        if tempTimeOfStay < 60:
            if lastGroupValue != -1:
                newTempList = [lastGroupValue] * len(GrpList)
                predictDWT = predictDWT + newTempList
            else:
                 predictDWT = predictDWT + GrpList
        else:
            predictDWT = predictDWT + GrpList
        lastGroupValue = predictDWT[-1]

    return predictDWT

def DWT_noise_filter(signal):
    ## DWT noise filtering of probabilities

    signalf = signal[:,1]
##    signal = predSVMProb[:,1]
##
    rec = lowpassfilter(signalf, 0.6)
    lenDiff = abs(len(rec) - len(testY))

    recDWT = rec[:(len(rec) - lenDiff)]
    mean = 0.5 #recDWT.mean()

    predDWT = np.array([val > mean for val in recDWT]) * 1 ## enable/disable DWT based filtering
    return predDWT

def PlotTime(x,y,xlabel,title,ylabel,lg1,lg2,xticks):

    poix = list(range(1,len(x)+1))
    fig, ax = plt.subplots()
    ax.plot(poix,x, c='red',marker='*', linestyle='-',label=lg1)
    #for an in x1:
        #ax.annotate(an,poi[an])
    ax.plot(poix,y,c='blue',marker='o', linestyle='-',label=lg2)
    plt.xlabel(xlabel)
    plt.title(title)
    plt.ylabel(ylabel)
    plt.xticks(np.arange(1, len(x)+1))
    ax.legend(loc='lower right', frameon=False)
    plt.xticks(poix, xticks, rotation=90)
    plt.grid(True)
    #plt.ylim(0,1)
    plt.show()

def CM(CMSVM):
    sen1 = CMSVM[0][0]/(CMSVM[0][0] + CMSVM[0][1])
    sen2 = CMSVM[1][1]/(CMSVM[1][1] + CMSVM[1][0])
    pr1 = CMSVM[0][0]/(CMSVM[0][0] + CMSVM[1][0])
    pr2 = CMSVM[1][1]/(CMSVM[1][1] + CMSVM[0][1])
    return np.round_([sen1,sen2,pr1,pr2],2)

dfTrainTest = pd.read_csv("TrainTestFile_filtered_CWSelected.csv",index_col=0) # + str(file)
#dfTrainTest.drop(['Unnamed: 0.1', 'Unnamed: 0.1.1'], axis=1,inplace=True)
dfTrainTest.replace([np.inf, -np.inf], 0,inplace=True)
dfTrainTest.fillna(0,inplace=True)

dfTrainTest.replace([np.inf, -np.inf], 0)
dfTrainTest.fillna(0)
df = dfTrainTest.reset_index(drop=True)

Gdf = df.groupby('filename').head(n=1)
start_list = Gdf.index.values.tolist()

TimeofStay = df.TimeOfStay.values
minTime = min(TimeofStay)

acSVM = []
acSVMf = []
acSVMdwt = []

acRF = []
acRFf = []
acRFdwt = []

acGB = []
acGBf = []
acGBdwt = []

acKM = []
acKMf = []

acHD = []
acHDf = []
acHDdwt = []

fn = []

trainMean = []
trainStd = []
dci = []
DS = []
te = []
qMeanStdTest = []
Entropy = []

CMSVMf = np.zeros((2,2))
CMSVM = np.zeros((2,2))

CMRFf = np.zeros((2,2))
CMRF = np.zeros((2,2))

CMGBf = np.zeros((2,2))
CMGB = np.zeros((2,2))

clsSVM = svm.SVC(probability=True,kernel='linear')#'rbf',gamma='auto')
clsSVMf = svm.SVC(probability=True,kernel='linear')#'rbf',gamma='auto')
clsRF = RandomForestClassifier(random_state=0)
clsRFf = RandomForestClassifier(random_state=0)
clsGB = GradientBoostingClassifier(random_state=0)
clsGBf = GradientBoostingClassifier(random_state=0)

TRfiles = list(dfTrainTest.filename.unique())
for i_start,cfile in enumerate(TRfiles): #enumerate(start_list):#

    testdf = df[df.filename == cfile]

    TIMELIST = [DTtoFloat(i) for i in testdf.time.values]
    delTs = np.ediff1d(TIMELIST).tolist()
    #minTime = min(delTs)
    delTs.insert(0,0)

    testY = testdf.observed.values #
    testX = testdf[['PointVelocityPrj','PointAccPrj','PointAnglePrj']].values

    testX =  testX/testX.max(axis=0)
    testXf = testdf[['velocityFiltered','accelFiltered','AngleFiltered']].values

    testXf =  testXf/testXf.max(axis=0)

    ### define train dataset
    traindf = df.drop(testdf.index.values, axis=0)
    traindf = traindf[traindf.PointVelocityPrj != 0]

    trainY = traindf.observed.values

    trainX = traindf[['PointVelocityPrj','PointAccPrj','PointAnglePrj']].values
    trainX =  trainX/trainX.max(axis=0)

    trainXf = traindf[['velocityFiltered','accelFiltered','AngleFiltered']].values
    trainXf =  trainXf/trainXf.max(axis=0)

       ####
    clsSVM.fit(trainX, trainY)
    clsSVMf.fit(trainXf, trainY)

    clsRF.fit(trainX, trainY)
    clsRFf.fit(trainXf, trainY)

    clsGB.fit(trainX, trainY)
    clsGBf.fit(trainXf, trainY)
    ##################
     ### prediction using SVM

    predSVM = clsSVM.predict(testX)
    #predSVMProb = clsSVM.predict_proba(testX)

    predSVMf = clsSVMf.predict(testXf)
    predSVMfProb = clsSVMf.predict_proba(testXf)

    ### prediction using RF

    predRF = clsRF.predict(testX)
    #predRFProb = clsRF.predict_proba(testX)

    predRFf = clsRFf.predict(testXf)
    predRFfProb = clsRFf.predict_proba(testXf)

    ### prediction using GB
    predGB = clsGB.predict(testX)
    #predGBProb = clsGB.predict_proba(testX)
    predGBf = clsGBf.predict(testXf)
    predGBfProb = clsGBf.predict_proba(testXf)

    ### append scores SVM
    acSVM.append(metrics.accuracy_score(testY,y_pred=predSVM))
    y_pred_SVMf = SWF(testdf,predSVMf,delTs)
    acSVMf.append(metrics.accuracy_score(testY,y_pred=y_pred_SVMf))

    SVMdwt = DWT_noise_filter(predSVMfProb)
    y_pred_SVMdwt = SWF(testdf,SVMdwt,delTs)
    acSVMdwt.append(metrics.accuracy_score(testY,y_pred=y_pred_SVMdwt))

    ### append scores RF
    acRF.append(metrics.accuracy_score(testY,y_pred=predRF))
    y_pred_RFf = SWF(testdf,predRFf,delTs)
    acRFf.append(metrics.accuracy_score(testY,y_pred=y_pred_RFf))

    RFdwt = DWT_noise_filter(predRFfProb)
    y_pred_RFdwt = SWF(testdf,RFdwt,delTs)
    acRFdwt.append(metrics.accuracy_score(testY,y_pred=y_pred_RFdwt))

    ### append scores GB
    acGB.append(metrics.accuracy_score(testY,y_pred=predGB))
    y_pred_GBf = SWF(testdf,predGBf,delTs)
    acGBf.append(metrics.accuracy_score(testY,y_pred=y_pred_GBf))

    GBdwt = DWT_noise_filter(predGBfProb)
    y_pred_GBdwt = SWF(testdf,GBdwt,delTs)
    acGBdwt.append(metrics.accuracy_score(testY,y_pred=y_pred_GBdwt))
    ###############

    te.append(len(testdf)/sum(delTs))
    ModeCounts = int(scipy.stats.mode(delTs)[1])
    dci.append(ModeCounts / len(testdf))
    DS.append(len(testdf))
    qMeanStdTest.append([np.mean(delTs),np.std(delTs)])
    Entropy.append(calculate_entropy(delTs))
    print(acSVM[-1])
    print(acSVMf[-1])
    print(acRF[-1])
    print(acRFf[-1])
    print(cfile)

    CMSVMf = np.add(CMSVMf,confusion_matrix(testY,y_pred_SVMf))
    CMSVM = np.add(CMSVM,confusion_matrix(testY,predSVM))

    CMRFf = np.add(CMRFf,confusion_matrix(testY,y_pred_RFf))
    CMRF = np.add(CMRF,confusion_matrix(testY,predRF))

    CMGBf = np.add(CMGBf,confusion_matrix(testY,y_pred_GBf))
    CMGB = np.add(CMGB,confusion_matrix(testY,predGB))

errordf = pd.DataFrame({'Dataset':TRfiles,'acRF':acRF,'acRFf':acRFf,'acRFdwt':acRFdwt,'acSVM':acSVM,'acSVMf':acSVMf,'acSVMdwt':acSVMdwt,'acGB':acGB,
                        'acGBf':acGBf,'acGBdwt':acGBdwt,
                      'Entropy':Entropy,'dci':dci,'MeanStd':qMeanStdTest,'TE':te})
errordf.to_csv('errordf_Subset_only.csv')

if 1 == 1:
    print(CMSVM)
    print(CMSVMf)
    print(CMRF)
    print(CMRFf)
    print(CMGB)
    print(CMGBf)

    print(CM(CMSVM))
    print(CM(CMSVMf))
    print(CM(CMRF))
    print(CM(CMRFf))
    print(CM(CMGB))
    print(CM(CMGBf))

    print(errordf.acRFf.mean())
    print(errordf.acRF.mean())
    print(errordf.acSVMf.mean())
    print(errordf.acSVM.mean())
    print(errordf.acGBf.mean())
    print(errordf.acGB.mean())


import os
import time

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import torch
from torch.autograd import Variable

def save_results_in_csv(y_pred):
    '''
    Saves the predictions of test lung images in the provided csv file
    :param y_pred: numpy array containing the boolean predictions
    '''
    fname = os.path.join(os.path.abspath(__file__), '..', 'data', 'CPSC340_Q2_SUBMISSION.csv')
    y_pred = y_pred.astype(bool)

    df = pd.DataFrame(columns=['Id', 'Predicted'])
    df['Id'] = np.arange(0, y_pred.shape[0])
    df['Predicted'] = y_pred
    df.to_csv(fname, index=False)

# Reference: Mitchell, L., Subramanian, V., & K., S. Y. (2019). Deep learning with PyTorch 1.x: implement deep
# learning techniques and neural network architecture variants using Python. page 173. Birmingham: Packt Publishing.
def train(model, dataloaders, dataset_sizes, criterion, optimizer, scheduler, num_epochs=20):
    since = time.time()
    if torch.cuda.is_available():
        model.cuda()

    best_model_wts = model.state_dict()
    best_acc = 0.0

    for epoch in range(num_epochs):
        print(scheduler.get_lr())
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)
        # Each epoch has a training and validation phase
        for phase in ['train', 'valid']:
            if phase == 'train':
                scheduler.step()
                model.train(True)  # Set model to training mode
            else:
                model.train(False)  # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for data in dataloaders[phase]:
                # get the inputs
                inputs, labels = data

                # wrap them in Variable
                if torch.cuda.is_available():
                    inputs = Variable(inputs.cuda())
                    labels = Variable(labels.cuda())
                else:
                    inputs, labels = Variable(inputs), Variable(labels)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                outputs = model(inputs)
                _, preds = torch.max(outputs.data, 1)
                loss = criterion(outputs, labels)

                # backward + optimize only if in training phase
                if phase == 'train':
                    loss.backward()
                    optimizer.step()

                # statistics
                # print(loss)
                running_loss += loss.item()
                print(preds, labels.data)
                print(preds.item() == labels.data.item())
                running_corrects += np.sum(int(preds.item()) == int(labels.data.item()))
                print("running_corrects", running_corrects)
            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects / dataset_sizes[phase]
            print("epoch_acc running_corrects ", running_corrects)
            print("epoch_acc ", dataset_sizes[phase])

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))

            # deep copy the model
            if phase == 'valid' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = model.state_dict()

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model

def save_images(data_loaders, train_labels):
    """
    Load and save the images in the following folders
    :param data_loaders: contains the tensors of the image pixels
    :param train_labels: contains 1/0 labels indicating whether the lung image is of covid patient or not
    :return:
    """
    training_images_rgb_folder = os.path.join(os.path.abspath(__file__), '..', 'training_images_rgb')
    training_images_grayscale_folder = os.path.join(os.path.abspath(__file__), '..', 'training_images_grayscale')
    test_images_rgb_folder = os.path.join(os.path.abspath(__file__), '..', 'test_images_rgb')
    test_images_grayscale_folder = os.path.join(os.path.abspath(__file__), '..', 'test_images_grayscale')

    if not os.listdir(training_images_rgb_folder):
        index = 0
        for sample in data_loaders["train"].dataset.imgs:
            image_name = f"Image_{index}_covid{train_labels[index].numpy()}.png"
            plt.imsave(os.path.join(training_images_rgb_folder, image_name), sample[0])
            plt.imsave(os.path.join(training_images_grayscale_folder, image_name), sample[0], cmap='gray')
            print(f"Saved {image_name}")
            index += 1

    if not os.listdir(test_images_rgb_folder):
        index = 0
        for sample in data_loaders["test"].dataset.imgs:
            image_name = f"Image_{index}.png"
            plt.imsave(os.path.join(test_images_rgb_folder, image_name), sample[0])
            plt.imsave(os.path.join(test_images_grayscale_folder, image_name), sample[0], cmap='gray')
            print(f"Saved {image_name}")
            index += 1


import numpy as np
from eigen_rootfinding.Macaulay import build_macaulay, find_degree, \
                                       create_matrix
from eigen_rootfinding.Multiplication import indexarray,indexarray_cheb,\
                                             msroots,get_rand_combos_matrix
from eigen_rootfinding.polynomial import is_power
from eigen_rootfinding.utils import row_swap_matrix, slice_top, mon_combos
from scipy import linalg as la
from scipy.special import comb
from itertools import combinations

def get_nullity_macaulay(dim,deg,polydegs):
    """Computes the nullity of a degree 'deg' Macaulay matrix analytically.

    Parameters:
    -----------
    dim : int
        Dimension of the system
    deg : int
        Macaulay degree
    polydegs : list of ints
        Degrees of the polynomials in the system

    Returns:
    --------
    nullity : int
        Nullity of the degree-'deg' Macaulay matrix
    """
    nullity = comb(dim+deg,dim,exact=True)
    for j in range(1,dim+1):
        innersum = 0
        for polydegsubset in combinations(polydegs,j):
            innersum += comb(dim+deg-sum(polydegsubset),dim,exact=True)
        nullity += (-1)**j*innersum
    return nullity

def svd_nullspace(a,nullity=None, rank=None):
    """Computes the nullspace of a matrix via the singular value decomposition.

    Parameters:
    -----------
    a : 2d ndarray
        Matrix to compute the nullspace of
    nullity : int
        Nullity of matrix. Results are more reliable if the nullity of a
        is known from the start.

    Returns:
    --------
    N : 2d ndarray
        Matrix whose columns form a basis for the nullspace of a
    """
    U,S,Vh = np.linalg.svd(a)
    if rank is None:
        if nullity is None:
            #mimics np.linalg.matrix_rank
            tol = S.max()*max(a.shape)*np.finfo(S.dtype).eps
            rank = np.count_nonzero(S>tol)
        else:
            rank = a.shape[1] - nullity
    return Vh[rank:].T.conj()

def nullspace_solve(polys, return_all_roots=True, method='svd', nullmethod='svd',
                    randcombos=False, normal=False):
    '''
    Finds the roots of the given list of multidimensional polynomials using
    the nullspace of the Macaulay matrix to create Moller-Stetter matrices.

    Parameters
    ----------
    polys : list of polynomial objects
        Polynomials to find the common roots of.
    return_all_roots : bool
        If True returns all the roots, otherwise just the ones in the unit box.
    method : str
        Which method to use to compute the Moller Stetter matrices from the nullspace.
        Options are 'qrp','lq','svd'.
    nullmethod : str
        Which method to use to compute the nullspace of the Macaulay matrix.
        Options are 'svd', 'fast'.
    randcombos : bool
        Whether or not to first take random linear combinations of the Macaulay matrix.
        Not allowed for fast nullspace computations (nullmethod='fast').
    normal : bool
        If randcombos is True, whether or not to use a matrix with entries
        drawn from the standard normal dsitribution when taking random
        linear combinations of the Macaulay matrix.

    returns
    -------
    roots : numpy array
        The common roots of the polynomials. Each row is a root.
    '''
    #setup
    degs = [poly.degree for poly in polys]
    dim = len(polys)
    bezout_bound = np.prod(degs)
    #compute nullspace
    if nullmethod=='svd':
        #build macaulay marix
        M,matrix_terms,cut = build_macaulay(polys)
        if randcombos:
            C = get_rand_combos_matrix(M.shape[1]-bezout_bound,M.shape[0], normal=normal)
            M = C@M
        nullspace = svd_nullspace(M,bezout_bound).T
    elif nullmethod=='fast':
        #todo change fast_null to make it
        #  return things in the order we want later
        nullspace,matrix_terms,cut = fast_null(polys,degs)
        srt = np.argsort(matrix_terms.sum(axis=1))[::-1]
        nullspace = nullspace[srt]
        matrix_terms = matrix_terms[srt]
        nullspace = nullspace.T
    #create MS matrices
    if method=='svd': MSfunc=get_MS_svd_nullspace
    elif method=='lq': MSfunc=get_MS_lq_nullspace
    elif method=='qrp': MSfunc=get_MS_qrp_nullspace
    MS = MSfunc(nullspace,matrix_terms,cut,bezout_bound,dim,power=is_power(polys))
    #return roots
    roots = msroots(MS)
    if return_all_roots:
        return roots
    else:
        # only return roots in the unit complex hyperbox
        return roots[[np.all(np.abs(root) <= 1) for root in roots]]

def get_MS_qrp_nullspace(nullspace,matrix_terms,cut,bezout_bound,dim,power=True):
    '''
    Constructs the Moller-Stetter matrices from the nullspace of the macaulay matrix
    via a QRP of the low degree terms in the nullspace.

    Parameters
    ----------
    nullspace : 2d ndarray
        Matrix whose rows (transposed) span the nullspace of the Macaulay matrix.
    matrix_terms : 2d integer ndarray
        Array containing the ordered column labels, where the ith row contains the
        exponent/degree of the ith monomial.
    cut : int
        Number of highest degree columns in the nullspace
    bezout_bound : int
        Number of roots of the system
    dim : int
        Dimension of the system.
    power : bool
        Whether the polynomails are expressed in the Chebyshev or Power basis.

    returns
    -------
    MS : (n, n, dim) ndarray
        Array containing the nxn Möller-Stetter matrices, where the matrix
        corresponding to multiplication by x_i is MS[..., i]
    '''
    #QRP on low degree columns
    Q,R,P = la.qr(nullspace[:,cut:],pivoting=True)
    R1 = R[:,:bezout_bound]
    #reorder columns
    matrix_terms[cut:] = matrix_terms[cut:][P]
    nullspace[:,cut:] = nullspace[:,cut:][:,P]
    #reduce columns we need for MS matrices
    cut2 = cut+bezout_bound
    basiscols = slice(cut,cut2)
    if power:
        idx_arrs = [indexarray(matrix_terms,basiscols,i) for i in range(dim)]
    else:
        idx_arrs = [indexarray_cheb(matrix_terms,basiscols,i) for i in range(dim)]
    needcols = np.unique(idx_arrs) #all the cols we need to reduce
    needhighdeg = needcols[needcols<cut] #high deg cols to reduce
    #left multiply by Q.H to get [Q.HN R1 R2]
    nullspace[:,needhighdeg] = Q.T.conj()@nullspace[:,needhighdeg]
    nullspace[:,cut2:] = R[:,bezout_bound:]
    #left multiply by R1inv to get [R1invQ.HN I R1invR2]
    nullspace[:,basiscols] = np.eye(bezout_bound)
    neednonbasis = np.concatenate((needhighdeg,needcols[needcols>=cut2]))
    nullspace[:,neednonbasis] = la.solve_triangular(R1,nullspace[:,neednonbasis])
    #construct MS matrices
    MS = np.empty((bezout_bound, bezout_bound, dim),dtype=nullspace.dtype)
    for i,arr in enumerate(idx_arrs):
        if power:
            MS[...,i] = nullspace[:,arr]
        else:
            MS[...,i] = (nullspace[:,arr[0]] + nullspace[:,arr[1]])/2
    return MS

def get_MS_lq_nullspace(nullspace,matrix_terms,cut,bezout_bound,dim,power=True):
    '''
    Constructs the Moller-Stetter matrices from the nullspace of the macaulay matrix
    via a LQ of the low degree terms in the nullspace.

    Parameters
    ----------
    nullspace : 2d ndarray
        Matrix whose rows (transposed) span the nullspace of the Macaulay matrix.
    matrix_terms : 2d integer ndarray
        Array containing the ordered column labels, where the ith row contains the
        exponent/degree of the ith monomial.
    cut : int
        Number of highest degree columns in the nullspace
    bezout_bound : int
        Number of roots of the system
    dim : int
        Dimension of the system.
    power : bool
        Whether the polynomails are expressed in the Chebyshev or Power basis.

    returns
    -------
    MS : (n, n, dim) ndarray
        Array containing the nxn Möller-Stetter matrices, where the matrix
        corresponding to multiplication by x_i is MS[..., i]
    '''
    #QRP on low degree columns transposed
    Q,R,P = la.qr(nullspace[:,cut:].conj().T,mode='economic',pivoting=True)
    L = R.conj().T#R[:bezout_bound].conj().T
    # Q = #Q[:,:bezout_bound]
    #rearrange rows in nullspace
    nullspace = nullspace[P]
    #reduce columns we need for MS matrices
    nullspace[:,:cut] = la.solve_triangular(L,nullspace[:,:cut],lower=True)
    nullspace[:,cut:] = Q.conj().T
    #construct MS matrices
    MS = np.empty((bezout_bound, bezout_bound, dim),dtype=nullspace.dtype)
    lowdeg = slice(cut,None)
    for i in range(dim):
        if power:
            arr = indexarray(matrix_terms,lowdeg,i)
            MS[...,i] = (nullspace[:,arr]@Q).T
        else:
            arr = indexarray_cheb(matrix_terms,lowdeg,i)
            MS[...,i] = (nullspace[:,arr[0]] + nullspace[:,arr[1]])@Q/2
    return MS

def get_MS_svd_nullspace(nullspace,matrix_terms,cut,bezout_bound,dim,power=True):
    '''
    Constructs the Moller-Stetter matrices from the nullspace of the macaulay matrix
    via a SVD of the low degree terms in the nullspace.

    Parameters
    ----------
    nullspace : 2d ndarray
        Matrix whose rows (transposed) span the nullspace of the Macaulay matrix.
    matrix_terms : 2d integer ndarray
        Array containing the ordered column labels, where the ith row contains the
        exponent/degree of the ith monomial.
    cut : int
        Number of highest degree columns in the nullspace
    bezout_bound : int
        Number of roots of the system
    dim : int
        Dimension of the system.
    power : bool
        Whether the polynomails are expressed in the Chebyshev or Power basis.

    returns
    -------
    MS : (n, n, dim) ndarray
        Array containing the nxn Möller-Stetter matrices, where the matrix
        corresponding to multiplication by x_i is MS[..., i]
    '''
    #SVD on low degree columns
    U,S,Vh = la.svd(nullspace[:,cut:],full_matrices=False)
    V = Vh.conj().T#[:bezout_bound].conj().T
    # S = S[:bezout_bound]
    #reduce columns we need for MS matrices
    #left multiply by U
    nullspace[:,:cut] = U.conj().T@nullspace[:,:cut]
    nullspace[:,:cut] = (nullspace[:,:cut].T/S).T #transposed for broadcasting
    nullspace[:,cut:] = V.conj().T
    #construct MS matrices
    MS = np.empty((bezout_bound, bezout_bound, dim),dtype=nullspace.dtype)
    lowdeg = slice(cut,None)
    for i in range(dim):
        if power:
            arr = indexarray(matrix_terms,lowdeg,i)
            MS[...,i] = nullspace[:,arr]@V
        else:
            arr = indexarray_cheb(matrix_terms,lowdeg,i)
            MS[...,i] = (nullspace[:,arr[0]] + nullspace[:,arr[1]])@V/2
    return MS

def all_shifts(polys, matrix_degree):
    """ Creates a dictionary of all the necessary shifts in the Macaulay
    matrix up to degree matrix_degree (from the smallest degree of the
    polys).

    Parameters
    ----------
        polys : list of polynomial objects
            The polynomial system of which the common roots are desired.
        all_shifts

    Returns
    -------
        shifts : dict of ints to lists of tuples
            The dictionary of all the necessary shifts. The keys are the
            different possible degrees from the minimum degree to the
            specified matrix degree, and the values are lists of tuples,
            corresponding to the polynomial and the monomial by which it
            is multiplied.
    """
    shifts = dict()
    min_degree = np.min([poly.degree for poly in polys])
    for i in range(min_degree,matrix_degree+1):
        shifts[i] = list()
    for poly in polys:
        degree = matrix_degree - poly.degree
        dim = poly.dim
        mons = mon_combos([0]*dim,degree)
        for mon in mons:
            shifts[np.sum(mon)+poly.degree].append(tuple([poly,mon]))
    return shifts

def new_terms(coeffs, old_term_set):
    """ Gets the new terms for the next iteration of nullspace construction.

    Parameters
    ----------
        coeffs : list of ndarrays
            The coefficients of the polynomials with all the "shifts" or
            monomial multiplications performed on them.
        old_term_set : set of ndarrays
            The set of the terms already accounted for in the nullspace
            construction (found in the previous step).

    Returns
    -------
        ndarray
            The new terms (monomials expressed as rows where
            sum(row) <= deg) in the Macaulay matrix.
            If there are no new terms, then returns None.
    """
    new_term_set = set()
    for coeff in coeffs:
        for term in zip(*np.where(coeff!=0)):
            if term not in old_term_set:
                new_term_set.add(tuple(term))
    if len(new_term_set)==0:
        return None
    else:
        return np.vstack(tuple(new_term_set))

def null_reduce(N,shifts,old_matrix_terms,bigShape,dim,deg,polydegs):
    """Create the next iteration of the nullspace construction
    algorithm.

    Parameters
    ----------
        N : ndarray
            The nullspace computed in the previous iteration.
        shifts : dict of ints to lists of tuples
            A dictionary of the shifts where the keys are the
            different possible degrees from the minimum degree to the
            specified matrix degree, and the values are lists of tuples,
            corresponding to the polynomial and the monomial by which it
            is multiplied.
        old_matrix_terms : ndarray
            An array where the ith row corresponds to the monomial
            of the ith column of the nullspace (the passed in value of
            N) of the Macaulay matrix.
        bigShape : list of ints
            The shape that the next (degree higher) Macaulay matrix must
            be to ensure that every term is accounted for.
        dim : int
            Dimension of the polynomial system.
        deg : int
            The degree of the Macaulay matrix to find the nullspace of.
        polydegs : list of ints
            List of the degrees of the polynomials in the system.

    Returns
    -------
        N : ndarray
            The next nullspace computed in the iterative algorithm.
        matrix_terms : ndarray
            An array where the ith row corresponds to the monomial
            of the ith column of the nullspace N of the Macaulay matrix.
    """

    old_term_set = set()
    for term in old_matrix_terms:
        old_term_set.add(tuple(term))
    coeffs = list()
    for poly,shift in shifts:
        coeffs.append(poly.mon_mult(shift,returnType = 'Matrix'))

    new_matrix_terms = new_terms(coeffs, old_term_set)

    matrix_terms = np.vstack(tuple([old_matrix_terms, new_matrix_terms]))

    new_matrix_term_indexes = list()
    old_matrix_term_indexes = list()
    for row in new_matrix_terms.T:
        new_matrix_term_indexes.append(row)
    for row in old_matrix_terms.T:
        old_matrix_term_indexes.append(row)

    #Adds the poly_coeffs to flat_polys, using added_zeros to make sure every term is in there.
    added_zeros = np.zeros(bigShape,dtype=N.dtype)
    new_flat_polys = list()
    old_flat_polys = list()
    for coeff in coeffs:
        slices = slice_top(coeff.shape)
        added_zeros[tuple(slices)] = coeff
        new_flat_polys.append(added_zeros[tuple(new_matrix_term_indexes)])
        old_flat_polys.append(added_zeros[tuple(old_matrix_term_indexes)])
        added_zeros[tuple(slices)] = np.zeros_like(coeff,dtype=coeff.dtype)

    R1 = np.reshape(old_flat_polys, (len(old_flat_polys),len(old_matrix_terms)))
    R2 = np.reshape(new_flat_polys, (len(new_flat_polys),len(new_matrix_terms)))

    X = np.hstack([R1@N,R2])
    nullity = get_nullity_macaulay(dim,deg,polydegs)
    K = svd_nullspace(X,nullity=nullity)

    cut = N.shape[1]
    K1 = K[:cut]
    K2 = K[cut:]
    N = np.vstack([N@K1,K2])

    return N, matrix_terms

def fast_null(polys, polydegs):
    """ Construct the nullspace of the Macaulay matrix created using the
    given system of polynomials using the Degree By Degree Method
    proposed by Telen in TODO: Cite paper.

    Parameters
    ----------
        polys : list of Polynomial objects
            The polynomial system of which the common roots are desired.
        polydegs : list of ints
            List of the degrees of the polynomials in the system.

    Returns
    -------
        N : ndarray
            The nullspace of the Macaulay matrix created by the system
            of polynomials.
        matrix_terms : ndarray
            An array where the ith row corresponds to the monomial
            of the ith column of the nullspace N of the Macaulay matrix.
        matrix_degree : int
            The degree of the Macaulay matrix created by the system of
            polynomials.
    """
    matrix_degree = find_degree(polys)
    dim = polys[0].dim
    bigShape = [matrix_degree+1]*dim

    shifts = all_shifts(polys, matrix_degree)
    degs = list(shifts.keys())

    initial_shifts = shifts[degs[0]]
    initial_coeffs = list()
    for poly,shift in initial_shifts:
        initial_coeffs.append(poly.mon_mult(shift,returnType = 'Matrix'))

    matrix, matrix_terms, cut = create_matrix(initial_coeffs, degs[0], dim)

    nullity = get_nullity_macaulay(dim,degs[0],polydegs)
    N = svd_nullspace(matrix,nullity=nullity)

    spot = 1
    while spot < len(degs):
        deg = degs[spot]
        new_shifts = shifts[deg]
        N,matrix_terms = null_reduce(N,new_shifts,matrix_terms,bigShape,dim,deg,polydegs)
        spot += 1
    return N, matrix_terms, comb(dim+matrix_degree-1,matrix_degree,exact=True)


import tkinter as tk
from tkinter import ttk
import os
from twisted.internet import tksupport, reactor
from autobahn.twisted.component import Component
from twisted.internet.defer import inlineCallbacks
import pickle
from pathlib import Path
from googleapiclient.discovery import build
import pandas as pd
from itertools import zip_longest
from functools import partial
from datetime import datetime

# TODO Server GUI rows are too long. Only extends to 'Add ' comment on vertical monitor
# TODO If booth closes, clear server's rat selection

class Server(tk.Frame):
    def __init__(self, parent, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.computer = os.environ["COMPUTERNAME"]
        self.parent = parent
        self.parent.geometry("400x300")
        self.parent.columnconfigure(0, weight=1)
        self.parent.rowconfigure(0, weight=1)
        self.frame = ttk.Frame(self.parent, padding=[3, 3, 3, 3])
        self.frame.grid(column=0, row=0, sticky=(tk.N, tk.W, tk.S))
        self.frame.columnconfigure(0, weight=1)
        self.frame.rowconfigure(0, weight=1)
        self.parent.title(f"Server {self.computer}")

        path = Path(__file__).parent / "../../resources/credentials/token.pickle"
        with path.open("rb") as token:
            sheets_creds = pickle.load(token)
        sheets_service = build('sheets', 'v4', credentials=sheets_creds)
        self._sheets = sheets_service.spreadsheets()

        server_info = pd.read_csv(Path(__file__).parent / "../../resources/credentials/server_info.csv")
        host, port, realm = server_info.iloc[0].values
        self.router_session = None
        self.component = Component(transports=f"ws://{host}:{port}", realm=realm)
        self.component.on_join(self.__joined)
        self.component.on_leave(self.__left)
        self.component.start(reactor)

        df = pd.read_csv(Path(__file__).parent / "../../resources/credentials/sheet_ids.csv")
        self.sheets_ids = {val.Name: val.ID for idx, val in df.iterrows()}

        # Get session running orders
        sess_info = self._sheets.values().get(spreadsheetId=self.sheets_ids["ATAT Behavior"],
                                              range="Sessions!A:Z", majorDimension="COLUMNS").execute()['values']
        session_dicts = []
        for session in range(1, len(sess_info)):
            rat_list = [rat if rat else None for rat in sess_info[session][1:]]
            session_dicts.extend([{"Booth": int(booth), "Session": int(session), "Rat": rat} for booth, rat in
                                  zip_longest(sess_info[0][1:], rat_list)])
        self.sessions = pd.DataFrame(session_dicts)

        # Get parameters for active rats
        parameters_info = self._sheets.values().get(spreadsheetId=self.sheets_ids["ATAT Behavior"],
                                                    range="Parameters!A:Z").execute()['values']
        self.parameters_info = pd.DataFrame(parameters_info[1:], columns=parameters_info[0])
        self.rat_list_values = self.parameters_info["Rat"].values.tolist()

        # Super control widgets
        self.control_frame = ttk.LabelFrame(self.frame, text="Controls", padding=[1, 1, 1, 1])
        self.booth_control_frame = ttk.LabelFrame(self.frame, text="Booths", padding=[1, 1, 1, 1])
        self.control_frame.pack()
        self.booth_control_frame.pack()
        self.refresh_parameters_button = ttk.Button(self.control_frame, text="Refresh parameters",
                                                    command=self.refresh_parameters)
        self.start_all_button = ttk.Button(self.control_frame, text="Start All", command=self.start_all_booths)
        self.stop_all_button = ttk.Button(self.control_frame, text="Stop and Save All", command=self.stop_all_booths)
        self.session_label = ttk.Label(self.control_frame, text="Select Session: ")
        self.session_selection = ttk.Combobox(self.control_frame, state="readonly",
                                              values=self.sessions["Session"].unique().tolist())
        self.session_selection.bind("<<ComboboxSelected>>", self.select_session)
        self.refresh_parameters_button.pack(side="left")
        self.start_all_button.pack(side="left")
        self.stop_all_button.pack(side="left")
        self.session_label.pack(side="left")
        self.session_selection.pack(side="left")

        # Iterate booth widgets
        self.booth_frames = {}
        self.booth_labels = {}
        self.booth_refresh_buttons = {}
        self.booth_open_buttons = {}
        self.booth_close_buttons = {}
        self.booth_rat_selecions = {}
        self.booth_status_labels = {}
        self.booth_time_labels = {}
        self.booth_trial_labels = {}
        self.booth_pellet_labels = {}
        self.booth_sound_labels = {}
        self.booth_percent_labels = {}
        self.booth_daily_labels = {}
        self.booth_attempt_labels = {}
        self.booth_start_buttons = {}
        self.booth_stop_buttons = {}
        self.booth_pause_buttons = {}
        self.booth_comment_buttons = {}
        self.booth_last_activity_labels = {}
        self.booth_test_func_buttons = {}
        self.booth_weight_labels = {}  # TODO
        for idx, booth in enumerate(self.sessions["Booth"].unique()):
            booth = int(booth)
            self.booth_frames[booth] = ttk.Frame(self.booth_control_frame, padding=[1, 1, 1, 1])
            self.booth_frames[booth].pack(fill="y", expand=True)
            self.booth_labels[booth] = ttk.Label(self.booth_frames[booth], text=f"Booth {booth}")
            self.booth_refresh_buttons[booth] = ttk.Button(self.booth_frames[booth], text="Refresh",
                                                           command=partial(self.refresh_booth, booth))
            self.booth_open_buttons[booth] = ttk.Button(self.booth_frames[booth], text="Open",
                                                        command=partial(self.open_booth, booth))
            self.booth_close_buttons[booth] = ttk.Button(self.booth_frames[booth], text="Close",
                                                         command=partial(self.close_booth, booth))
            self.booth_rat_selecions[booth] = ttk.Combobox(self.booth_frames[booth], justify="right", state="readonly",
                                                           values=self.rat_list_values + [""])
            self.booth_rat_selecions[booth].bind("<<ComboboxSelected>>", partial(self.select_rat, booth))
            self.booth_status_labels[booth] = ttk.Label(self.booth_frames[booth], text="Disconnected")
            self.booth_time_labels[booth] = ttk.Label(self.booth_frames[booth], text="--:--")
            self.booth_trial_labels[booth] = ttk.Label(self.booth_frames[booth], text="Trial: -")
            self.booth_pellet_labels[booth] = ttk.Label(self.booth_frames[booth], text="Pellets: -")
            self.booth_sound_labels[booth] = ttk.Label(self.booth_frames[booth], text="Sound:")
            self.booth_percent_labels[booth] = ttk.Label(self.booth_frames[booth], text="-% correct")
            self.booth_daily_labels[booth] = ttk.Label(self.booth_frames[booth], text="Session: -")
            self.booth_attempt_labels[booth] = ttk.Label(self.booth_frames[booth], text="Attempt: -")
            self.booth_start_buttons[booth] = ttk.Button(self.booth_frames[booth], text="Start",
                                                         command=partial(self.start_booth, booth))
            self.booth_stop_buttons[booth] = ttk.Button(self.booth_frames[booth], text="Stop/Save", state="disabled",
                                                        command=partial(self.stop_booth, booth))
            self.booth_pause_buttons[booth] = tk.Button(self.booth_frames[booth], text="Pause", state="disabled",
                                                        command=partial(self.pause_booth, booth))
            self.booth_comment_buttons[booth] = ttk.Button(self.booth_frames[booth], text="Add comment",
                                                           state="disabled", command=partial(self.add_comment, booth))
            self.booth_last_activity_labels[booth] = ttk.Label(self.booth_frames[booth], text="Last update: --:--")
            self.booth_test_func_buttons[booth] = ttk.Button(self.booth_frames[booth], text="Test func",
                                                             command=partial(self.test_func, booth))
            self.booth_labels[booth].pack(side="left")
            self.booth_refresh_buttons[booth].pack(side="left")
            self.booth_open_buttons[booth].pack(side="left")
            self.booth_close_buttons[booth].pack(side="left")
            self.booth_rat_selecions[booth].pack(side="left")
            self.booth_status_labels[booth].pack(side="left")
            self.booth_time_labels[booth].pack(side="left")
            self.booth_trial_labels[booth].pack(side="left")
            self.booth_pellet_labels[booth].pack(side="left")
            self.booth_sound_labels[booth].pack(side="left")
            self.booth_percent_labels[booth].pack(side="left")
            self.booth_daily_labels[booth].pack(side="left")
            self.booth_attempt_labels[booth].pack(side="left")
            self.booth_start_buttons[booth].pack(side="left")
            self.booth_stop_buttons[booth].pack(side="left")
            self.booth_pause_buttons[booth].pack(side="left")
            self.booth_comment_buttons[booth].pack(side="left")
            self.booth_last_activity_labels[booth].pack(side="left")
            self.booth_test_func_buttons[booth].pack(side="left")

    @inlineCallbacks
    def __joined(self, session, _details):
        print("Server joined session!")
        self.router_session = session
        yield self.router_session.subscribe(self.session_status, "server.session_status")
        yield self.router_session.subscribe(self.update_pause, "server.pause_booth")
        yield self.router_session.subscribe(self.update_rat, "server.select_rat")
        yield self.router_session.subscribe(self.update_booth, "server.refresh_booth")
        yield self.router_session.subscribe(self.update_running, "server.running_status")

    @inlineCallbacks
    def __left(self, _details, _was_clean):
        print("Server left session!")

    def test_func(self, booth_num):
        self.router_session.publish("test.test_func", booth_num)

    def refresh_parameters(self):
        # Update session running orders
        sess_info = self._sheets.values().get(spreadsheetId=self.sheets_ids["ATAT Behavior"],
                                              range="Sessions!A:Z", majorDimension="COLUMNS").execute()['values']
        session_dicts = []
        for session in range(1, len(sess_info)):
            rat_list = [rat if rat else None for rat in sess_info[session][1:]]
            session_dicts.extend([{"Booth": int(booth), "Session": int(session), "Rat": rat} for booth, rat in
                                  zip_longest(sess_info[0][1:], rat_list)])
        self.sessions = pd.DataFrame(session_dicts)
        self.session_selection["values"] = self.sessions["Session"].unique().tolist()

        # Update rat parameters
        parameters_info = self._sheets.values().get(spreadsheetId=self.sheets_ids["ATAT Behavior"],
                                                    range="Parameters!A:Z").execute()['values']
        self.parameters_info = pd.DataFrame(parameters_info[1:], columns=parameters_info[0])
        self.rat_list_values = self.parameters_info["Rat"].values.tolist()
        for booth in self.booth_rat_selecions.values():
            booth["values"] = self.rat_list_values + [""]

    def select_session(self, event):
        session_num = int(event.widget.get())
        session = self.sessions[self.sessions["Session"] == session_num]
        for idx, row in session.iterrows():
            if row["Rat"]:
                self.booth_rat_selecions[row["Booth"]].set(row["Rat"])
                self.router_session.publish("client.select_rat", row["Booth"], row["Rat"])
            else:
                self.booth_rat_selecions[row["Booth"]].set("")
                self.router_session.publish("client.select_rat", row["Booth"], "")

    def open_booth(self, booth_num):
        self.router_session.publish("client.open_booth", booth_num)

    def refresh_booth(self, booth_num):
        self.router_session.publish("client.refresh_booth", booth_num)

    def close_booth(self, booth_num):
        self.router_session.publish("client.close_booth", booth_num)

    def start_booth(self, booth_num):
        self.router_session.publish("client.start_booth", booth_num)

    def start_all_booths(self):
        for booth_num, booth_obj in self.booth_rat_selecions.items():
            if booth_obj["state"] == "normal" and booth_obj.get():
                self.start_booth(booth_num)

    def stop_booth(self, booth_num):
        self.router_session.publish("client.stop_booth", booth_num)

    def stop_all_booths(self):
        for booth_num, booth_obj in self.booth_stop_buttons.items():
            if booth_obj["state"] == "normal":
                self.stop_booth(booth_num)

    def pause_booth(self, booth_num):
        if self.booth_status_labels[booth_num]["text"] == "Paused":
            self.router_session.publish("client.pause_booth", booth_num, False)
        else:
            self.router_session.publish("client.pause_booth", booth_num, True)

    def update_pause(self, booth_num, pause):
        if pause:
            self.booth_pause_buttons[booth_num].configure(relief="sunken")
            self.booth_status_labels[booth_num].configure(text="Paused")
        else:
            self.booth_pause_buttons[booth_num].configure(relief="raised")
            self.booth_status_labels[booth_num].configure(text="Other")  # TODO fix this

    def update_rat(self, booth_num, rat):
        self.booth_start_buttons[booth_num]["state"] = "normal"
        self.booth_rat_selecions[booth_num].set(rat)

    def update_running(self, booth_num, running):
        if running:
            self.booth_pause_buttons[booth_num]["state"] = "normal"
            self.booth_stop_buttons[booth_num]["state"] = "normal"
            self.booth_comment_buttons[booth_num]["state"] = "normal"
            self.booth_start_buttons[booth_num]["state"] = "disabled"
            self.booth_close_buttons[booth_num]["state"] = "disabled"
            self.booth_rat_selecions[booth_num]["state"] = "disabled"
        else:
            self.booth_pause_buttons[booth_num]["state"] = "disabled"
            self.booth_stop_buttons[booth_num]["state"] = "disabled"
            self.booth_comment_buttons[booth_num]["state"] = "disabled"
            self.booth_close_buttons[booth_num]["state"] = "normal"
            self.booth_rat_selecions[booth_num]["state"] = "readonly"

    def update_booth(self, booth_info):
        # TODO
        booth_num = booth_info["Number"]

    def add_comment(self, booth_num):
        self.router_session.publish("client.add_comment", booth_num)

    def select_rat(self, booth_num, event):
        self.booth_start_buttons[booth_num]["state"] = "normal"
        self.router_session.publish("client.select_rat", booth_num, event.widget.get())

    def session_status(self, status_dict, booth_num):
        self.booth_status_labels[booth_num].configure(text=status_dict["Status"])
        self.booth_time_labels[booth_num].configure(text=f"{status_dict['Time'][0]}:{status_dict['Time'][1]}")
        self.booth_trial_labels[booth_num].configure(text=f"Trial: {status_dict['Trial']}")
        self.booth_pellet_labels[booth_num].configure(text=f"Pellet: {status_dict['Pellet']}")
        self.booth_sound_labels[booth_num].configure(text=f"Sound: {status_dict['Sound']}")
        self.booth_percent_labels[booth_num].configure(text=f"{status_dict['Percent']:.1f}% correct")
        self.booth_attempt_labels[booth_num].configure(text=f"Attempt: {status_dict['Attempt']}")
        self.booth_daily_labels[booth_num].configure(text=f"Session: {status_dict['Session']}")
        self.update_last_active(booth_num)

    def update_last_active(self, booth_num):
        last_active = datetime.now().strftime("%H:%M")
        self.booth_last_activity_labels[booth_num].configure(text=f"Last active: {last_active}")

    def quit(self):
        reactor.stop()

def create_gui():
    root = tk.Tk()
    tksupport.install(root)
    server = Server(root)
    root.protocol("WM_DELETE_WINDOW", server.quit)
    return server

if __name__ == "__main__":
    server_gui = create_gui()
    reactor.run()


import environ,os, boto3
from pathlib import Path
import base64
import hashlib
import datetime

BASE_DIR = Path(__file__).resolve().parent.parent

environ.Env.read_env(os.path.join(BASE_DIR, '.env'))

env = environ.Env(
    DEBUG=(bool, True)
)

def upload(user,image,*args):
    s3 = boto3.client('s3',
        aws_access_key_id=env('AWSAccessKeyId'),
        aws_secret_access_key=env('AWSSecretKey'),
        region_name='ap-northeast-2',
    )

    Bucket = "pracs3"

    if args:
        key = f'{user}/{args}/profile.jpg'
    else:
        key = f'{user}/profile.jpg'

    s3.put_object(
        ACL="public-read",
        Bucket=Bucket,
        Body=image,
        Key=key,
        ContentType=image.content_type,
        )

    url = f'https://{Bucket}.s3.ap-northeast-2.amazonaws.com/{key}'

    return url


from graphviz import Digraph as DotGraph
from wordcloud import WordCloud
import matplotlib.pyplot as plt

def showGraph(dot, show=True, file_name='textgraph.gv'):
  dot.render(file_name, view=show)

'''
def gshow0(g, file_name='textgraph.gv', show=True):
  dot = DotGraph()
  for e in g.edges():
    f, t = e
    # w = g[f][t]['weight']
    w = ''
    dot.edge(str(f), str(t), label=str(w))
  dot.render(file_name, view=show)
'''

def show_ranks(rank_dict,file_name="cloud.pdf",show=True) :
  if not show : return
  cloud=WordCloud(width=800,height=400)
  cloud.fit_words(rank_dict)
  f=plt.figure()
  plt.imshow(cloud, interpolation='bilinear')
  plt.axis("off")
  # plt.show()
  f.savefig(file_name,bbox_inches='tight')
  plt.close('all')

if __name__=="__main__":
  d = {'a': 0.1, 'b': 0.2, 'c': 0.33, 'd': 0.2}
  show_ranks(d)