# @Author : bamtercelboo
# @Datetime : 2018/1/30 15:58
# @File : DataConll2003_Loader.py
# @Last Modify Time : 2018/1/30 15:58
# @Contact : bamtercelboo@{gmail.com, 163.com}

"""
    FILE :
    FUNCTION :
"""
import sys
import os
import re
import random
import torch
import json
from Dataloader.Instance import Instance

from DataUtils.Common import *
torch.manual_seed(seed_num)
random.seed(seed_num)

class DataLoaderHelp(object):
    """
    DataLoaderHelp
    """

    @staticmethod
    def _clean_str(string):
        """
        Tokenization/string cleaning for all datasets except for SST.
        Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
        """
        string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
        string = re.sub(r"\'s", " \'s", string)
        string = re.sub(r"\'ve", " \'ve", string)
        string = re.sub(r"n\'t", " n\'t", string)
        string = re.sub(r"\'re", " \'re", string)
        string = re.sub(r"\'d", " \'d", string)
        string = re.sub(r"\'ll", " \'ll", string)
        string = re.sub(r",", " , ", string)
        string = re.sub(r"!", " ! ", string)
        string = re.sub(r"\(", " \( ", string)
        string = re.sub(r"\)", " \) ", string)
        string = re.sub(r"\?", " \? ", string)
        string = re.sub(r"\s{2,}", " ", string)
        return string.strip().lower()

    @staticmethod
    def _normalize_word(word):
        """
        :param word:
        :return:
        """
        new_word = ""
        for char in word:
            if char.isdigit():
                new_word += '0'
            else:
                new_word += char
        return new_word

    @staticmethod
    def _sort(insts):
        """
        :param insts:
        :return:
        """
        sorted_insts = []
        sorted_dict = {}
        for id_inst, inst in enumerate(insts):
            sorted_dict[id_inst] = inst.words_size
        dict = sorted(sorted_dict.items(), key=lambda d: d[1], reverse=True)
        for key, value in dict:
            sorted_insts.append(insts[key])
        print("Sort Finished.")
        return sorted_insts

class DataLoader(DataLoaderHelp):
    """
    DataLoader
    """
    def __init__(self, path, shuffle, config):
        """
        :param path: data path list
        :param shuffle:  shuffle bool
        :param config:  config
        """
        #
        print("Loading Data......")
        self.data_list = []
        self.max_count = config.max_count
        self.path = path
        self.shuffle = shuffle

        # BERT
        self.bert_path = [config.bert_train_file,
                          config.bert_dev_file,
                          config.bert_test_file]

        self.use_bert = config.use_bert

    def dataLoader(self):
        """
        :return:
        """
        path = self.path
        shuffle = self.shuffle
        assert isinstance(path, list), "Path Must Be In List"
        print("Data Path {}".format(path))
        for id_data in range(len(path)):
            print("Loading Data Form {}".format(path[id_data]))
            insts = self._Load_Each_Data(path=path[id_data], path_id=id_data)
            print("shuffle train data......")
            random.shuffle(insts)
            self.data_list.append(insts)
        # return train/dev/test data
        if len(self.data_list) == 3:
            return self.data_list[0], self.data_list[1], self.data_list[2]
        elif len(self.data_list) == 2:
            return self.data_list[0], self.data_list[1]

    def _Load_Each_Data(self, path=None, path_id=None):
        """
        :param path:
        :param shuffle:
        :return:
        """
        assert path is not None, "The Data Path Is Not Allow Empty."
        insts = []
        now_lines = 0
        with open(path, encoding="UTF-8") as f:
            inst = Instance()
            for line in f.readlines():
                line = line.strip()
                now_lines += 1
                if now_lines % 200 == 0:
                    sys.stdout.write("\rreading the {} line\t".format(now_lines))
                if line == "\n":
                    print("empty line")

                inst = Instance()
                line = line.split()
                label = line[0]
                word = " ".join(line[1:])
                if label not in ["0", "1"]:
                    print("Error line: ", " ".join(line))
                    continue
                inst.words = self._clean_str(word).split()
                inst.labels.append(label)
                inst.words_size = len(inst.words)
                insts.append(inst)

                if len(insts) == self.max_count:
                    break
            # print("\n")
        if self.use_bert:
            insts = self._read_bert_file(insts, path=self.bert_path[path_id])
        return insts

    def _read_bert_file(self, insts, path):
        """
        :param insts:
        :param path:
        :return:
        """
        print("\nRead BERT Features File From {}".format(path))
        now_lines = 0
        with open(path, encoding="utf-8") as f:
            for inst, bert_line in zip(insts, f.readlines()):
                now_lines += 1
                if now_lines % 2000 == 0:
                    sys.stdout.write("\rreading the {} line\t".format(now_lines))
                bert_fea = json.loads(bert_line)
                inst.bert_tokens = bert_fea["features"]["tokens"]
                inst.bert_feature = bert_fea["features"]["values"]
                # print(inst.bert_feature)
            sys.stdout.write("\rReading the {} line\t".format(now_lines))
        return insts


import discord
from discord.ext import commands

from bot import BaseBot
from core import error
from core.utils import read_config

config = read_config()

class Log(commands.Cog):
    def __init__(self, bot: BaseBot):
        self.bot = bot

    @commands.Cog.listener()
    async def on_command_error(self, ctx, err):
        err = getattr(err, "original", err)
        err = error.BotException(err, self.bot)
        return await err.propagate()

    @commands.Cog.listener()
    async def on_message_edit(
        self, message_before: discord.Message, message_after: discord.Message
    ):
        if (
            not message_after.author.bot
            and message_before.guild.id == config["server"]["guild_id"]
        ):
            message_log_channel_id = self.bot.config["server"]["message_log_channel"]
            message_log_channel = await self.bot.fetch_channel(message_log_channel_id)
            e = discord.Embed(
                title="Message Edit",
                color=0xF6F600,
                description=f"**Original**: {message_before.content}\n**New**: {message_after.content}\n[jump url]({message_after.jump_url})",
            )
            e.set_author(
                name=message_after.author.name,
                icon_url=message_after.author.display_avatar,
            )
            await message_log_channel.send(embed=e)

    @commands.Cog.listener()
    async def on_message_delete(self, message: discord.Message):
        if not message.author.bot and message.guild.id == config["server"]["guild_id"]:
            message_log_channel_id = self.bot.config["server"]["message_log_channel"]
            message_log_channel = await self.bot.fetch_channel(message_log_channel_id)
            e = discord.Embed(
                title="Message delete",
                color=0xF64800,
                description=f"{message.content}",
            )
            e.set_author(
                name=message.author.name, icon_url=message.author.display_avatar
            )
            await message_log_channel.send(embed=e)

async def setup(bot):
    await bot.add_cog(Log(bot))


from __future__ import print_function, division, absolute_import
import os
import cv2
import glob
import random

import mxnet as mx
import numpy as np

from threading import Thread
from collections import defaultdict

try:
    from queue import Queue
except ImportError:
    from Queue import Queue

from mxnet.io import DataBatch, DataIter

from utils.random_erasing import random_erasing

def pop(x, size):
    return [x.pop(0) for _ in range(size)]

class GroupIterator(DataIter):
    def __init__(self, data_dir, p_size, k_size, image_size, rand_mirror=False, rand_crop=False, random_erasing=False,
                 force_resize=None, resize_shorter=None, num_worker=4, random_seed=None):

        if random_seed is None:
            random_seed = random.randint(0, 2 ** 32 - 1)
        np.random.RandomState(random_seed)
        np.random.seed(random_seed)
        random.seed(random_seed)

        self.random_seed = random_seed

        self.p_size = p_size
        self.k_size = k_size
        self.batch_size = p_size * k_size
        self.image_size = image_size
        self.rand_mirror = rand_mirror
        self.rand_crop = rand_crop
        self.random_erasing = random_erasing
        self.resize_shorter = resize_shorter
        self.num_worker = num_worker
        self.force_resize = tuple(force_resize) if force_resize is not None else force_resize
        self.img_list = None
        self.cursor = 0
        self.size = 0

        resize_set = {self.resize_shorter, self.force_resize}
        assert len(resize_set) == 2 and None in resize_set, "resize_shorter and force_resize are mutually exclusive!"

        if self.force_resize is not None:
            if self.force_resize[0] < self.image_size[0] or self.force_resize[1] < self.image_size[1]:
                raise ValueError("each edge of force_resize must be larger than or equal to target image size")

        if self.resize_shorter is not None:
            if self.resize_shorter < min(self.image_size):
                raise ValueError("resize_shorter must be larger than or equal to minimum of target image side size")

        print("Data loading..")
        self.id2imgs = self._preprocess(data_dir)
        print("Data loaded!")

        self.num_id = len(self.id2imgs)
        print(self.num_id)

        # multi-thread primitive
        self.result_queue = Queue(maxsize=8 * num_worker)
        self.index_queue = Queue()
        self.workers = None

        self._thread_start(num_worker)

        self.reset()

    def _preprocess(self, data_dir):
        img_list = glob.glob(os.path.join(data_dir, "*.jpg")) + glob.glob(os.path.join(data_dir, "*.png"))
        img_list.sort()
        self.img_list = img_list

        assert len(img_list) != 0

        id2imgs = defaultdict(list)
        for img_name in img_list:
            idx = int(os.path.basename(img_name).split("_")[0])
            id2imgs[idx].append(img_name)

        id2imgs.pop(0, None)
        id2imgs.pop(-1, None)

        id2imgs_organized = {}
        for i, v in enumerate(id2imgs.values()):
            id2imgs_organized[i] = v

        return id2imgs_organized

    def _insert_queue(self):
        data = []

        # pids = list(self.id2imgs.keys()).copy()
        pids = list(self.id2imgs.keys())[:]
        random.shuffle(pids)

        sample_range = list(range(0, len(pids), self.p_size))[:-1]
        self.size = len(sample_range)

        for i in sample_range:
            start = i
            stop = start + self.p_size

            for pid in pids[start:stop]:
                if len(self.id2imgs[pid]) >= self.k_size:
                    imgs = np.random.choice(self.id2imgs[pid], replace=False, size=self.k_size)
                else:
                    imgs = np.random.choice(self.id2imgs[pid], replace=True, size=self.k_size)

                data.extend(imgs)

            label = np.array(pids[start:stop]).repeat(self.k_size)

            # self.index_queue.put([data.copy(), label])
            self.index_queue.put([data[:], label])

            assert len(data) == self.batch_size
            assert len(label) == self.batch_size

            del data[:]

    def _thread_start(self, num_worker):
        self.workers = [Thread(target=self._worker, args=(self.random_seed + i,)) for i in range(num_worker)]

        for worker in self.workers:
            worker.daemon = True
            worker.start()

    def _worker(self, seed):
        np.random.RandomState(seed)
        np.random.seed(seed)
        random.seed(seed)

        while True:
            indices = self.index_queue.get()
            result = self._get_batch(indices=indices)

            if result is None:
                return

            self.result_queue.put(result)

    def _get_batch(self, indices):
        img_paths = indices[0]
        label = indices[1]

        # Loading
        data = []
        for img_path in img_paths:
            img = cv2.imread(img_path)
            cv2.cvtColor(img, code=cv2.COLOR_BGR2RGB, dst=img)
            img = img.astype(np.float32)

            ori_h, ori_w = img.shape[:2]

            if self.force_resize is not None:
                img = cv2.resize(img, self.force_resize[::-1], interpolation=cv2.INTER_LINEAR)
            else:
                if ori_h < ori_w:
                    h, w = self.resize_shorter, int(round(self.resize_shorter / ori_h * ori_w))
                else:
                    h, w = int(round(self.resize_shorter / ori_w * ori_h)), self.resize_shorter

                if h != ori_h or w != ori_w:
                    img = cv2.resize(img, (w, h), interpolation=cv2.INTER_LINEAR)

            h, w = img.shape[:2]
            x, y = int(round((w - self.image_size[1]) / 2)), int(round((h - self.image_size[0]) / 2))

            # random crop
            if self.rand_crop:
                x = random.randint(0, w - self.image_size[1])
                y = random.randint(0, h - self.image_size[0])

            if self.image_size != (h, w):
                img = img[y:y + self.image_size[0], x:x + self.image_size[1], :]

            # random mirror
            if self.rand_mirror and random.randint(0, 1) == 1:
                img = cv2.flip(img, flipCode=1)

            if self.random_erasing:
                img = random_erasing(img)

            data.append(img)

        data = np.stack(data, axis=0).transpose([0, 3, 1, 2])
        data = mx.nd.array(data)
        label = mx.nd.array(label)

        assert data.shape[0] == label.shape[0] == self.batch_size

        return [data], [label]

    @property
    def provide_data(self):
        return [('data', (self.batch_size, 3, self.image_size[0], self.image_size[1]))]

    @property
    def provide_label(self):
        return [('softmax_label', (self.batch_size,))]

    def reset(self):
        self.cursor = 0
        self.index_queue.queue.clear()

        self._insert_queue()

    def next(self):
        if self.cursor >= self.size:
            raise StopIteration

        data, label = self.result_queue.get()

        self.cursor += 1

        return DataBatch(data=data, label=label, provide_data=self.provide_data, provide_label=self.provide_label)

if __name__ == '__main__':
    import time

    data_dir = "/home/chencp/dataset/Market-1501-v15.09.15"
    train = GroupIterator(data_dir=os.path.join(data_dir, "bounding_box_train"),
                          p_size=64, k_size=4, image_size=(128, 64),
                          force_resize=(128, 64), rand_mirror=True, rand_crop=False, random_erasing=True, num_worker=8)

    tic = time.time()
    tmp = []
    for i in range(100):
        print(i)
        batch = train.next()
        tmp.append(batch.label[0].asnumpy().tolist())
        print(batch.data[0].shape)
        print(batch.label[0].shape)
        print(batch.label[0])
        imgs = batch.data[0].transpose([0, 2, 3, 1]).asnumpy()

        print(imgs.dtype)
        for j in range(imgs.shape[0]):
            img = imgs[j]
            cv2.imwrite("%d-%d.jpg" % (i, j), img[:, :, [2, 1, 0]].astype(np.uint8))

    print((time.time() - tic) / 100)


import logging
import os
import shutil
from pathlib import Path
import tempfile

import pandas as pd

from autumn.core import db, plots
from autumn.settings import REMOTE_BASE_DIR
from autumn.infrastructure.tasks.utils import get_project_from_run_id
from autumn.core.utils.s3 import download_from_run_s3, list_s3, upload_to_run_s3, get_s3_client
from autumn.core.utils.timer import Timer
from .storage import StorageMode, MockStorage, S3Storage, LocalStorage
from autumn.core.runs import ManagedRun

logger = logging.getLogger(__name__)

def powerbi_task(run_id: str, urunid: str, quiet: bool, store="s3"):

    REMOTE_BASE_DIR = Path(tempfile.mkdtemp())

    POWERBI_PLOT_DIR = os.path.join(REMOTE_BASE_DIR, "plots", "uncertainty")
    POWERBI_DATA_DIR = os.path.join(REMOTE_BASE_DIR, "data", "powerbi")
    POWERBI_DIRS = [POWERBI_DATA_DIR, POWERBI_PLOT_DIR]
    POWERBI_PRUNED_DIR = os.path.join(POWERBI_DATA_DIR, "pruned")
    POWERBI_COLLATED_PATH = os.path.join(POWERBI_DATA_DIR, "collated")
    POWERBI_COLLATED_PRUNED_PATH = os.path.join(POWERBI_DATA_DIR, "collated-pruned")

    s3_client = get_s3_client()
    project = get_project_from_run_id(run_id)

    # Set up directories for plots and output data.
    with Timer(f"Creating PowerBI directories"):
        for dirpath in POWERBI_DIRS:
            if os.path.exists(dirpath):
                shutil.rmtree(dirpath)

            os.makedirs(dirpath)

    mr = ManagedRun(run_id, s3_client=s3_client)

    if store == StorageMode.MOCK:
        storage = MockStorage()
    elif store == StorageMode.S3:
        s3_client = get_s3_client()
        storage = S3Storage(s3_client, run_id, REMOTE_BASE_DIR, not quiet)
    elif store == StorageMode.LOCAL:
        storage = LocalStorage(run_id, REMOTE_BASE_DIR)

    # Find the full model run databases in AWS S3.
    # key_prefix = os.path.join(run_id, os.path.relpath(FULL_RUN_DATA_DIR, REMOTE_BASE_DIR))
    # chain_db_keys = []
    # for filename_base in ["mcmc_run", "mcmc_params", "derived_outputs"]:
    #    chain_db_keys += list_s3(s3_client, key_prefix, key_suffix=f"{filename_base}.feather")

    # Download the full model run databases.
    # with Timer(f"Downloading full model run data"):
    #    for src_key in chain_db_keys:
    #        download_from_run_s3(s3_client, run_id, src_key, quiet)

    # No urunid supplied; get a single candidate dataframe (ie the MLE run)
    if urunid == "mle":
        all_mcmc_df = mr.full_run.get_mcmc_runs()
        candidates_df = db.process.select_pruning_candidates(all_mcmc_df, 1)
    else:
        c, r = (int(x) for x in urunid.split("_"))
        candidates_df = pd.DataFrame(columns=["chain", "run"])
        candidates_df.loc[0] = dict(chain=c, run=r)

    # Remove unnecessary data from each full model run database.
    full_db_paths = db.load.find_db_paths(mr.full_run.local_path)
    with Timer(f"Pruning chain databases"):
        get_dest_path = lambda p: os.path.join(POWERBI_PRUNED_DIR, os.path.basename(p))
        for full_db_path in full_db_paths:
            chain_id = int(full_db_path.split("-")[-1])
            chain_candidates = candidates_df[candidates_df["chain"] == chain_id]
            db.process.prune_chain(full_db_path, get_dest_path(full_db_path), chain_candidates)

    # Collate data from each pruned full model run database into a single database.
    pruned_db_paths = db.load.find_db_paths(POWERBI_PRUNED_DIR)
    with Timer(f"Collating pruned databases"):
        db.process.collate_databases(pruned_db_paths, POWERBI_COLLATED_PATH)

    # Calculate uncertainty for model outputs.
    with Timer(f"Calculating uncertainty quartiles"):
        db.uncertainty.add_uncertainty_quantiles(POWERBI_COLLATED_PATH, project.plots)

    # Remove unnecessary data from the database.
    with Timer(f"Pruning final database"):
        db.process.prune_final(POWERBI_COLLATED_PATH, POWERBI_COLLATED_PRUNED_PATH, candidates_df)

    # Unpivot database tables so that they're easier to process in PowerBI.
    run_slug = run_id.replace("/", "-")
    dest_db_path = os.path.join(POWERBI_DATA_DIR, f"powerbi-{run_slug}.db")
    with Timer(f"Applying PowerBI specific post-processing final database"):
        db.process.powerbi_postprocess(POWERBI_COLLATED_PRUNED_PATH, dest_db_path, run_id)

    # Upload final database to AWS S3
    with Timer(f"Uploading PowerBI data to AWS S3"):
        # upload_to_run_s3(s3_client, run_id, dest_db_path, quiet)
        storage.store(dest_db_path)

    # Create uncertainty plots
    with Timer(f"Creating uncertainty plots"):
        plots.uncertainty.plot_uncertainty(project.plots, dest_db_path, POWERBI_PLOT_DIR)

    # Upload the plots to AWS S3.
    with Timer(f"Uploading plots to AWS S3"):
        # upload_to_run_s3(s3_client, run_id, POWERBI_PLOT_DIR, quiet)
        storage.store(POWERBI_PLOT_DIR)

    shutil.rmtree(REMOTE_BASE_DIR)


############################################################
# -*- coding: utf-8 -*-
#
#       #   #  #   #   #    #
#      ##  ##  #  ##  #    #
#     # # # #  # # # #    #  #
#    #  ##  #  ##  ##    ######
#   #   #   #  #   #       #
#
# Python-based Tool for interaction with the 10micron mounts
# GUI with PyQT5 for python
#
# written in python3, (c) 2019-2023 by mworion
# Licence APL2.0
#
###########################################################
from base import packageConfig

# standard libraries
from datetime import datetime
import time

# external packages
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QTransform
from skyfield.almanac import dark_twilight_day, TWILIGHTS

# local import
if packageConfig.isAvailable:
    from gui.extWindows.simulatorW import SimulatorWindow

from gui.utilities.toolsQtWidget import sleepAndEvents
from gui.extWindows.keypadW import KeypadWindow
from gui.utilities.stylesQtCss import Styles
from gui.utilities.toolsQtWidget import MWidget
from gui.extWindows.messageW import MessageWindow
from gui.extWindows.hemisphereW import HemisphereWindow
from gui.extWindows.measureW import MeasureWindow
from gui.extWindows.imageW import ImageWindow
from gui.extWindows.satelliteW import SatelliteWindow
from gui.extWindows.analyseW import AnalyseWindow
from gui.extWindows.videoW1 import VideoWindow1
from gui.extWindows.videoW2 import VideoWindow2
from gui.extWindows.videoW3 import VideoWindow3
from gui.extWindows.videoW4 import VideoWindow4
from gui.extWindows.bigPopupW import BigPopup
from gui.widgets.main_ui import Ui_MainWindow
from gui.mainWmixin.tabMount import Mount
from gui.mainWmixin.tabEnviron import Environ
from gui.mainWmixin.tabAlmanac import Almanac
from gui.mainWmixin.tabModel import Model
from gui.mainWmixin.runBasic import BasicRun
from gui.mainWmixin.tabBuildPoints import BuildPoints
from gui.mainWmixin.tabManageModel import ManageModel
from gui.mainWmixin.tabSat_Search import SatSearch
from gui.mainWmixin.tabSat_Track import SatTrack
from gui.mainWmixin.tabMinorPlanet import MinorPlanet
from gui.mainWmixin.tabRelay import Relay
from gui.mainWmixin.tabTools_Rename import Rename
from gui.mainWmixin.tabTools_IERSTime import IERSTime
from gui.mainWmixin.tabAnalysis import Analysis
from gui.mainWmixin.tabPower import Power
from gui.mainWmixin.tabSett_Device import SettDevice
from gui.mainWmixin.tabSett_Mount import SettMount
from gui.mainWmixin.tabSett_Dome import SettDome
from gui.mainWmixin.tabSett_ParkPos import SettParkPos
from gui.mainWmixin.tabSett_Relay import SettRelay
from gui.mainWmixin.tabSett_Misc import SettMisc
from gui.mainWmixin.tabImage_Manage import ImageManage
from gui.mainWmixin.tabImage_Stats import ImagsStats
from logic.profiles.profile import loadProfile, saveProfile, blendProfile

class MainWindow(
    MWidget,
    SettMisc,
    Mount,
    Environ,
    Almanac,
    Model,
    BasicRun,
    BuildPoints,
    ManageModel,
    SatSearch,
    SatTrack,
    MinorPlanet,
    Relay,
    Power,
    Rename,
    IERSTime,
    Analysis,
    SettDevice,
    SettMount,
    ImageManage,
    ImagsStats,
    SettDome,
    SettParkPos,
    SettRelay,
):
    """
    the main window class handles the main menu as well as the show and no show
    part of any other window. all necessary processing for functions of that gui
    will be linked to this class. therefore, window classes will have a threadpool
    for managing async processing if needed.
    """

    __all__ = ['MainWindow']

    def __init__(self, app):
        # has to be put before super to adjust the color before the stylesheet
        # on the parent classes is drawn first
        colSet = app.config.get('colorSet', 0)
        Styles.colorSet = colSet
        super().__init__()
        self.app = app
        self.msg = app.msg
        self.threadPool = app.threadPool
        self.deviceStat = app.deviceStat
        self.uiWindows = app.uiWindows
        self.setAttribute(Qt.WA_DeleteOnClose)
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        self.satStatus = False
        self.gameControllerRunning = False
        self.setWindowTitle(f'MountWizzard4 - v{self.app.__version__}')

        self.uiWindows['showMessageW'] = {
            'button': self.ui.openMessageW,
            'classObj': None,
            'name': 'MessageDialog',
            'class': MessageWindow,
        }
        self.uiWindows['showHemisphereW'] = {
            'button': self.ui.openHemisphereW,
            'classObj': None,
            'name': 'HemisphereDialog',
            'class': HemisphereWindow,
        }
        self.uiWindows['showImageW'] = {
            'button': self.ui.openImageW,
            'classObj': None,
            'name': 'ImageDialog',
            'class': ImageWindow,
        }
        self.uiWindows['showMeasureW'] = {
            'button': self.ui.openMeasureW,
            'classObj': None,
            'name': 'MeasureDialog',
            'class': MeasureWindow,
        }
        self.uiWindows['showSatelliteW'] = {
            'button': self.ui.openSatelliteW,
            'classObj': None,
            'name': 'SatelliteDialog',
            'class': SatelliteWindow,
        }
        self.uiWindows['showAnalyseW'] = {
            'button': self.ui.openAnalyseW,
            'classObj': None,
            'name': 'AnalyseDialog',
            'class': AnalyseWindow,
        }
        self.uiWindows['showVideoW1'] = {
            'button': self.ui.openV1,
            'classObj': None,
            'name': 'Video1',
            'class': VideoWindow1,
        }
        self.uiWindows['showVideoW2'] = {
            'button': self.ui.openV2,
            'classObj': None,
            'name': 'Video2',
            'class': VideoWindow2,
        }
        self.uiWindows['showVideoW3'] = {
            'button': self.ui.openV3,
            'classObj': None,
            'name': 'Video3',
            'class': VideoWindow3,
        }
        self.uiWindows['showVideoW4'] = {
            'button': self.ui.openV4,
            'classObj': None,
            'name': 'Video4',
            'class': VideoWindow4,
        }
        if packageConfig.isAvailable:
            self.uiWindows['showSimulatorW'] = {
                'button': self.ui.mountConnected,
                'classObj': None,
                'name': 'SimulatorDialog',
                'class': SimulatorWindow,
            }
        self.uiWindows['showKeypadW'] = {
            'button': self.ui.openKeypadW,
            'classObj': None,
            'name': 'KeypadDialog',
            'class': KeypadWindow,
        }
        self.uiWindows['showBigPopupW'] = {
            'button': self.ui.big,
            'classObj': None,
            'name': 'BigPopup',
            'class': BigPopup,
        }
        self.deviceStatGui = {
            'dome': self.ui.domeConnected,
            'camera': self.ui.cameraConnected,
            'refraction': self.ui.refractionConnected,
            'plateSolve': self.ui.plateSolveConnected,
            'mount': self.ui.mountConnected,
        }
        self.mwSuper('__init__')
        self.app.mount.signals.pointDone.connect(self.updateStatusGUI)
        self.app.mount.signals.mountUp.connect(self.updateMountConnStat)
        self.app.mount.signals.settingDone.connect(self.updateMountWeatherStat)
        self.app.remoteCommand.connect(self.remoteCommand)
        self.app.plateSolve.signals.message.connect(self.updatePlateSolveStatus)
        self.app.dome.signals.message.connect(self.updateDomeStatus)
        self.app.camera.signals.message.connect(self.updateCameraStatus)
        self.ui.saveConfigQuit.clicked.connect(self.quitSave)
        self.ui.loadFrom.clicked.connect(self.loadProfileGUI)
        self.ui.addFrom.clicked.connect(self.addProfileGUI)
        self.ui.saveConfigAs.clicked.connect(self.saveConfigAs)
        self.ui.saveConfig.clicked.connect(self.saveConfig)
        self.app.seeingWeather.b = self.ui.label_b.property('a')

        for window in self.uiWindows:
            self.uiWindows[window]['button'].clicked.connect(self.toggleWindow)

        self.initConfig()
        self.ui.colorSet.currentIndexChanged.connect(self.refreshColorSet)
        self.ui.collectWindows.clicked.connect(self.collectWindows)
        self.showExtendedWindows()
        self.activateWindow()

        self.ui.tabsMovable.clicked.connect(self.enableTabsMovable)
        self.app.update1s.connect(self.updateTime)
        self.app.update1s.connect(self.updateControllerStatus)
        self.app.update1s.connect(self.updateThreadAndOnlineStatus)
        self.app.update1s.connect(self.updateWindowsStats)
        self.app.update1s.connect(self.smartFunctionGui)
        self.app.update1s.connect(self.smartTabGui)
        self.app.update1s.connect(self.updateWindowsStats)
        self.app.update1s.connect(self.setEnvironDeviceStats)
        self.app.update1s.connect(self.updateDeviceStats)

    def initConfig(self):
        """
        :return: True for test purpose
        """
        config = self.app.config
        self.ui.profile.setText(config.get('profileName'))
        colSet = config.get('colorSet', 0)
        Styles.colorSet = colSet
        self.ui.colorSet.setCurrentIndex(colSet)
        self.setStyleSheet(self.mw4Style)

        if 'mainW' not in config:
            config['mainW'] = {}
        config = config['mainW']

        self.positionWindow(config)
        self.mwSuper('initConfig')
        self.setTabAndIndex(self.ui.mainTabWidget, config, 'orderMain')
        self.setTabAndIndex(self.ui.mountTabWidget, config, 'orderMount')
        self.setTabAndIndex(self.ui.imagingTabWidget, config, 'orderImaging')
        self.setTabAndIndex(self.ui.modelingTabWidget, config, 'orderModeling')
        self.setTabAndIndex(self.ui.manageTabWidget, config, 'orderManage')
        self.setTabAndIndex(self.ui.settingsTabWidget, config, 'orderSettings')
        self.setTabAndIndex(self.ui.toolsTabWidget, config, 'orderTools')
        self.setTabAndIndex(self.ui.satTabWidget, config, 'orderSatellite')

        self.changeStyleDynamic(self.ui.mountConnected, 'color', 'gray')
        self.smartTabGui()
        self.enableTabsMovable()
        self.setupIcons()
        self.show()
        return True

    def storeConfigExtendedWindows(self):
        """
        :return: True for test purpose
        """
        config = self.app.config
        for window in self.uiWindows:
            config[window] = bool(self.uiWindows[window]['classObj'])
            if config[window]:
                self.uiWindows[window]['classObj'].storeConfig()

        return True

    def storeConfig(self):
        """
        :return: True for test purpose
        """
        config = self.app.config
        config['colorSet'] = self.ui.colorSet.currentIndex()
        config['profileName'] = self.ui.profile.text()
        if 'mainW' not in config:
            config['mainW'] = {}
        else:
            config['mainW'].clear()
        config = config['mainW']

        config['winPosX'] = self.pos().x()
        config['winPosY'] = self.pos().y()
        self.getTabAndIndex(self.ui.mainTabWidget, config, 'orderMain')
        self.getTabAndIndex(self.ui.mountTabWidget, config, 'orderMount')
        self.getTabAndIndex(self.ui.imagingTabWidget, config, 'orderImaging')
        self.getTabAndIndex(self.ui.modelingTabWidget, config, 'orderModeling')
        self.getTabAndIndex(self.ui.manageTabWidget, config, 'orderManage')
        self.getTabAndIndex(self.ui.settingsTabWidget, config, 'orderSettings')
        self.getTabAndIndex(self.ui.toolsTabWidget, config, 'orderTools')
        self.getTabAndIndex(self.ui.satTabWidget, config, 'orderSatellite')
        self.mwSuper('storeConfig')
        self.storeConfigExtendedWindows()
        return True

    def enableTabsMovable(self):
        """
        :return: True for test purpose
        """
        isMovable = self.ui.tabsMovable.isChecked()
        self.ui.mainTabWidget.setMovable(isMovable)
        self.ui.mountTabWidget.setMovable(isMovable)
        self.ui.imagingTabWidget.setMovable(isMovable)
        self.ui.modelingTabWidget.setMovable(isMovable)
        self.ui.manageTabWidget.setMovable(isMovable)
        self.ui.settingsTabWidget.setMovable(isMovable)
        self.ui.toolsTabWidget.setMovable(isMovable)
        self.ui.satTabWidget.setMovable(isMovable)
        self.app.tabsMovable.emit(isMovable)
        return True

    def closeEvent(self, closeEvent):
        """
        :return:    nothing
        """
        self.gameControllerRunning = False
        self.app.timer0_1s.stop()
        self.changeStyleDynamic(self.ui.pauseModel, 'pause', False)
        self.closeExtendedWindows()
        self.stopDrivers()
        self.threadPool.waitForDone(5000)
        super().closeEvent(closeEvent)
        self.app.quit()

    def quitSave(self):
        """
        :return:    true for test purpose
        """
        self.app.storeConfig()
        self.saveConfig()
        self.close()
        return True

    def setupIcons(self):
        """
        :return:    True if success for test
        """
        # main window
        self.wIcon(self.ui.saveConfigAs, 'save')
        self.wIcon(self.ui.loadFrom, 'load')
        self.wIcon(self.ui.addFrom, 'load')
        self.wIcon(self.ui.saveConfig, 'save')
        self.wIcon(self.ui.saveConfigQuit, 'save')
        self.wIcon(self.ui.mountOn, 'power-on')
        self.wIcon(self.ui.mountOff, 'power-off')
        self.wIcon(self.ui.stop, 'hand')
        self.wIcon(self.ui.tracking, 'target')
        self.wIcon(self.ui.followSat, 'satellite')
        self.wIcon(self.ui.flipMount, 'flip')
        self.wIcon(self.ui.setSiderealTracking, 'sidereal')
        self.wIcon(self.ui.setLunarTracking, 'lunar')
        self.wIcon(self.ui.setSolarTracking, 'solar')
        self.wIcon(self.ui.park, 'park')

        # model points
        self.wIcon(self.ui.loadBuildPoints, 'load')
        self.wIcon(self.ui.saveBuildPoints, 'save')
        self.wIcon(self.ui.saveBuildPointsAs, 'save')
        self.wIcon(self.ui.clearBuildP, 'trash')

        # model
        self.wIcon(self.ui.plateSolveSync, 'start')
        pixmap = self.img2pixmap(':/pics/azimuth.png').scaled(101, 101)
        self.ui.picAZ.setPixmap(pixmap)
        pixmap = self.img2pixmap(':/pics/altitude.png').scaled(101, 101)
        self.ui.picALT.setPixmap(pixmap)

        self.wIcon(self.ui.cancelModel, 'cross-circle')
        self.wIcon(self.ui.runModel, 'start')
        self.wIcon(self.ui.pauseModel, 'pause')
        self.wIcon(self.ui.endModel, 'stop_m')
        self.wIcon(self.ui.dataModel, 'choose')

        # manage model
        self.wIcon(self.ui.runOptimize, 'start')
        self.wIcon(self.ui.cancelOptimize, 'cross-circle')
        self.wIcon(self.ui.deleteWorstPoint, 'circle-minus')
        self.wIcon(self.ui.clearModel, 'trash')
        self.wIcon(self.ui.openAnalyseW, 'bar-chart')
        self.wIcon(self.ui.showActualModelAnalyse, 'copy')
        self.wIcon(self.ui.showOriginalModelAnalyse, 'copy')

        self.wIcon(self.ui.loadName, 'load')
        self.wIcon(self.ui.saveName, 'save')
        self.wIcon(self.ui.deleteName, 'trash')
        self.wIcon(self.ui.refreshName, 'reload')
        self.wIcon(self.ui.refreshModel, 'reload')

        # minor planets
        self.wIcon(self.ui.progMinorPlanetsFull, 'run')
        self.wIcon(self.ui.progMinorPlanetsFiltered, 'run')
        self.wIcon(self.ui.progMinorPlanetsSelected, 'run')
        self.wIcon(self.ui.progEarthRotationData, 'run')
        self.wIcon(self.ui.downloadIERS, 'run')

        # satellite
        self.wIcon(self.ui.stopSatelliteTracking, 'cross-circle')
        self.wIcon(self.ui.startSatelliteTracking, 'start')
        self.wIcon(self.ui.progSatellitesFull, 'run')
        self.wIcon(self.ui.progSatellitesFiltered, 'run')
        self.wIcon(self.ui.progTrajectory, 'run')

        # analyse
        self.wIcon(self.ui.runFlexure, 'start')
        self.wIcon(self.ui.runHysteresis, 'start')
        self.wIcon(self.ui.cancelAnalysis, 'cross-circle')

        # tools
        self.wIcon(self.ui.renameStart, 'start')
        self.wIcon(self.ui.renameInputSelect, 'folder')
        self.wIcon(self.ui.posButton0, 'target')
        self.wIcon(self.ui.posButton1, 'target')
        self.wIcon(self.ui.posButton2, 'target')
        self.wIcon(self.ui.posButton3, 'target')
        self.wIcon(self.ui.posButton4, 'target')
        self.wIcon(self.ui.posButton5, 'target')
        self.wIcon(self.ui.posButton6, 'target')
        self.wIcon(self.ui.posButton7, 'target')
        self.wIcon(self.ui.posButton8, 'target')
        self.wIcon(self.ui.posButton9, 'target')

        self.wIcon(self.ui.moveNorth, 'north')
        self.wIcon(self.ui.moveEast, 'east')
        self.wIcon(self.ui.moveSouth, 'south')
        self.wIcon(self.ui.moveWest, 'west')
        self.wIcon(self.ui.moveNorthEast, 'northEast')
        self.wIcon(self.ui.moveNorthWest, 'northWest')
        self.wIcon(self.ui.moveSouthEast, 'southEast')
        self.wIcon(self.ui.moveSouthWest, 'southWest')
        self.wIcon(self.ui.moveNorthAltAz, 'north')
        self.wIcon(self.ui.moveEastAltAz, 'east')
        self.wIcon(self.ui.moveSouthAltAz, 'south')
        self.wIcon(self.ui.moveWestAltAz, 'west')
        self.wIcon(self.ui.moveNorthEastAltAz, 'northEast')
        self.wIcon(self.ui.moveNorthWestAltAz, 'northWest')
        self.wIcon(self.ui.moveSouthEastAltAz, 'southEast')
        self.wIcon(self.ui.moveSouthWestAltAz, 'southWest')
        self.wIcon(self.ui.stopMoveAll, 'stop_m')
        self.wIcon(self.ui.moveAltAzAbsolute, 'target')
        self.wIcon(self.ui.moveRaDecAbsolute, 'target')

        # driver setting
        for driver in self.drivers:
            if self.drivers[driver]['uiSetup'] is not None:
                ui = self.drivers[driver]['uiSetup']
                self.wIcon(ui, 'cogs')

        self.wIcon(self.ui.ascomConnect, 'link')
        self.wIcon(self.ui.ascomDisconnect, 'unlink')

        # imaging
        self.wIcon(self.ui.copyFromTelescopeDriver, 'copy')
        self.wIcon(self.ui.haltFocuser, 'bolt-alt')
        self.wIcon(self.ui.moveFocuserIn, 'exit-down')
        self.wIcon(self.ui.moveFocuserOut, 'exit-up')
        self.wIcon(self.ui.coverPark, 'exit-down')
        self.wIcon(self.ui.coverUnpark, 'exit-up')

        # dome setting
        pixmap = self.img2pixmap(':/dome/radius.png')
        self.ui.picDome1.setPixmap(pixmap)
        pixmap = self.img2pixmap(':/dome/north.png')
        self.ui.picDome2.setPixmap(pixmap)
        pixmap = self.img2pixmap(':/dome/east.png')
        self.ui.picDome3.setPixmap(pixmap)
        pixmap = self.img2pixmap(':/dome/z_gem.png')
        self.ui.picDome4.setPixmap(pixmap)
        pixmap = self.img2pixmap(':/dome/z_10micron.png')
        self.ui.picDome5.setPixmap(pixmap)
        pixmap = self.img2pixmap(':/dome/gem.png')
        self.ui.picDome6.setPixmap(pixmap)
        pixmap = self.img2pixmap(':/dome/lat.png')
        self.ui.picDome7.setPixmap(pixmap)
        pixmap = self.img2pixmap(':/dome/shutter.png')
        self.ui.picDome8.setPixmap(pixmap)
        pixmap = self.img2pixmap(':/dome/hysteresis.png')
        self.ui.picDome9.setPixmap(pixmap)
        pixmap = self.img2pixmap(':/dome/zenith.png')
        self.ui.picDome10.setPixmap(pixmap)
        self.wIcon(self.ui.copyFromDomeDriver, 'copy')
        self.wIcon(self.ui.domeCloseShutter, 'exit-down')
        self.wIcon(self.ui.domeOpenShutter, 'exit-up')
        self.wIcon(self.ui.domeAbortSlew, 'bolt-alt')

        # park positions
        self.wIcon(self.ui.posSave0, 'download')
        self.wIcon(self.ui.posSave1, 'download')
        self.wIcon(self.ui.posSave2, 'download')
        self.wIcon(self.ui.posSave3, 'download')
        self.wIcon(self.ui.posSave4, 'download')
        self.wIcon(self.ui.posSave5, 'download')
        self.wIcon(self.ui.posSave6, 'download')
        self.wIcon(self.ui.posSave7, 'download')
        self.wIcon(self.ui.posSave8, 'download')
        self.wIcon(self.ui.posSave9, 'download')

        # misc setting
        self.wIcon(self.ui.installVersion, 'world')
        pixmap = self.svg2pixmap(':/icon/controller.svg', self.M_BLUE)
        self.ui.controller1.setPixmap(pixmap.scaled(16, 16))
        self.ui.controller2.setPixmap(pixmap.scaled(16, 16))
        self.ui.controller3.setPixmap(pixmap.scaled(16, 16))
        self.ui.controller4.setPixmap(pixmap.scaled(16, 16))
        self.ui.controller5.setPixmap(pixmap.scaled(16, 16))
        pixmap = self.svg2pixmap(':/icon/controllerNew.svg', self.M_BLUE)
        self.ui.controllerOverview.setPixmap(pixmap)
        self.ui.controller1.setEnabled(False)
        self.ui.controller2.setEnabled(False)
        self.ui.controller3.setEnabled(False)
        self.ui.controller4.setEnabled(False)
        self.ui.controller5.setEnabled(False)

        # environment
        pixmap = self.svg2pixmap(':/icon/meteoblue.svg', '#124673')
        pixmap = pixmap.transformed(QTransform().rotate(-90))
        pixmap = pixmap.scaled(37, 128, 1)
        self.ui.meteoblueIcon.setPixmap(pixmap)
        self.ui.meteoblueIcon.setVisible(False)
        self.ui.meteoblueSeeing.setVisible(False)
        return True

    def updateMountConnStat(self, status):
        """
        updateMountConnStat show the connection status of the mount. if status
        is None, which means there is no valid host entry for connection,
        the status is grey

        :param status:
        :return: true for test purpose
        """
        hasSim = packageConfig.isAvailable
        self.deviceStat['mount'] = status

        if status and hasSim:
            self.ui.mountConnected.setEnabled(status)
            self.ui.mountConnected.setText('Mount 3D')
        elif status:
            self.ui.mountConnected.setText('Mount')
        elif not status and hasSim:
            self.ui.mountConnected.setText('Mount')
            if self.uiWindows['showSimulatorW']['classObj']:
                self.uiWindows['showSimulatorW']['classObj'].close()
        return True

    def updateMountWeatherStat(self, setting):
        """
        updateMountWeatherStat show the connection status of the mount weather
        station connected. if the data values are None there is no station
        attached to the GPS port.

        :return: true for test purpose
        """
        if setting.weatherTemperature is None and setting.weatherPressure is None:
            self.deviceStat['directWeather'] = None
        else:
            if setting.weatherStatus is None:
                self.deviceStat['directWeather'] = False
            else:
                self.deviceStat['directWeather'] = True
        return True

    def smartFunctionGui(self):
        """
        smartFunctionGui enables and disables gui actions depending on the
        actual state of the different devices. this should be the core of
        avoiding user misused during running operations. smartGui is run every
        1 second synchronously, because it can't be simpler done with dynamic
        approach. all different situations in a running environment is done
        locally.

        :return: true for test purpose
        """
        isMountReady = bool(self.deviceStat.get('mount'))
        isDomeReady = bool(self.deviceStat.get('dome'))
        isModelingReady = all(
            bool(self.deviceStat.get(x)) for x in ['mount', 'camera', 'plateSolve'])
        isPause = self.ui.pauseModel.property('pause')

        if isModelingReady and self.app.data.buildP and not isPause:
            self.ui.runModel.setEnabled(True)
        else:
            self.ui.runModel.setEnabled(False)

        if isModelingReady:
            self.ui.runModel.setEnabled(True)
            self.ui.plateSolveSync.setEnabled(True)
            self.ui.dataModel.setEnabled(True)
            self.ui.runFlexure.setEnabled(True)
            self.ui.runHysteresis.setEnabled(True)
        else:
            self.ui.runModel.setEnabled(False)
            self.ui.plateSolveSync.setEnabled(False)
            self.ui.dataModel.setEnabled(False)
            self.ui.runFlexure.setEnabled(False)
            self.ui.runHysteresis.setEnabled(False)

        if isMountReady:
            self.ui.refractionGroup.setEnabled(True)
            self.ui.dsoGroup.setEnabled(True)
            self.ui.mountCommandTable.setEnabled(True)
        else:
            self.ui.dsoGroup.setEnabled(False)
            self.ui.refractionGroup.setEnabled(False)
            self.ui.mountCommandTable.setEnabled(False)

        if isDomeReady and isMountReady:
            self.ui.useDomeAz.setEnabled(True)
        else:
            self.ui.useDomeAz.setEnabled(False)

        return True

    def smartTabGui(self):
        """
        smartTabGui enables and disables tab visibility depending on the actual
        state of the different devices.
        :return: true for test purpose
        """
        smartTabs = {
            'Power': {
                'statID': 'power',
                'tab': self.ui.toolsTabWidget,
            },
            'Relay': {
                'statID': 'relay',
                'tab': self.ui.toolsTabWidget,
            },
            'RelaySett': {
                'statID': 'relay',
                'tab': self.ui.settingsTabWidget,
            },
        }
        tabChanged = False

        for key, tab in smartTabs.items():
            tabIndex = self.getTabIndex(smartTabs[key]['tab'], key)
            tabStatus = smartTabs[key]['tab'].isTabVisible(tabIndex)

            stat = bool(self.deviceStat.get(smartTabs[key]['statID']))
            smartTabs[key]['tab'].setTabVisible(tabIndex, stat)
            actChanged = tabStatus != stat
            tabChanged = tabChanged or actChanged

        tabIndex = self.getTabIndex(self.ui.imagingTabWidget, 'reference')
        self.ui.imagingTabWidget.setTabVisible(tabIndex, packageConfig.isReference)
        tabIndex = self.getTabIndex(self.ui.imagingTabWidget, 'AnalyseFlexure')
        self.ui.toolsTabWidget.setTabVisible(tabIndex, packageConfig.isAnalyse)

        # redraw tabs only when a change occurred. this is necessary, because
        # enable and disable does not remove tabs
        if tabChanged:
            ui = self.ui.mainTabWidget
            ui.setStyleSheet(ui.styleSheet())
            ui = self.ui.toolsTabWidget
            ui.setStyleSheet(ui.styleSheet())
        return True

    def updateWindowsStats(self):
        """
        :return: True for test purpose
        """
        for win in self.uiWindows:
            winObj = self.uiWindows[win]

            if winObj['classObj']:
                self.changeStyleDynamic(winObj['button'], 'running', True)
            else:
                self.changeStyleDynamic(winObj['button'], 'running', False)
        return True

    def setEnvironDeviceStats(self):
        """
        :return:
        """
        refracOn = self.app.mount.setting.statusRefraction == 1
        isManual = self.ui.refracManual.isChecked()
        isTabEnabled = self.ui.showTabEnviron.isChecked()
        if not refracOn or not isTabEnabled:
            self.deviceStat['refraction'] = None
            self.ui.refractionConnected.setText('Refraction')
        elif isManual:
            self.ui.refractionConnected.setText('Refrac Manu')
            self.deviceStat['refraction'] = True
        else:
            self.ui.refractionConnected.setText('Refrac Auto')
            isSource = self.deviceStat.get(self.refractionSource, False)
            self.deviceStat['refraction'] = isSource
        return True

    def updateDeviceStats(self):
        """
        :return: True for test purpose
        """
        for device, ui in self.deviceStatGui.items():
            if self.deviceStat.get(device) is None:
                self.changeStyleDynamic(ui, 'color', 'gray')
            elif self.deviceStat[device]:
                self.changeStyleDynamic(ui, 'color', 'green')
            else:
                self.changeStyleDynamic(ui, 'color', 'red')

        isMount = self.deviceStat.get('mount', False)
        self.changeStyleDynamic(self.ui.mountOn, 'running', isMount)
        self.changeStyleDynamic(self.ui.mountOff, 'running', not isMount)
        return True

    def updatePlateSolveStatus(self, text):
        """
        :param text:
        :return: true for test purpose
        """
        self.ui.plateSolveText.setText(text)
        return True

    def updateDomeStatus(self, text):
        """
        :param text:
        :return: true for test purpose
        """
        self.ui.domeText.setText(text)
        return True

    def updateCameraStatus(self, text):
        """
        :param text:
        :return: true for test purpose
        """
        self.ui.cameraText.setText(text)
        return True

    def updateControllerStatus(self):
        """
        :return: True for test purpose
        """
        gcStatus = self.gameControllerRunning
        self.ui.controller1.setEnabled(gcStatus)
        self.ui.controller2.setEnabled(gcStatus)
        self.ui.controller3.setEnabled(gcStatus)
        self.ui.controller4.setEnabled(gcStatus)
        self.ui.controller5.setEnabled(gcStatus)
        return True

    def updateThreadAndOnlineStatus(self):
        """
        :return: True for test purpose
        """
        if self.ui.isOnline.isChecked():
            mode = 'Online'
        else:
            mode = 'Offline'

        moon = self.ui.moonPhaseIllumination.text()

        f = dark_twilight_day(self.app.ephemeris, self.app.mount.obsSite.location)
        twilight = TWILIGHTS[int(f(self.app.mount.obsSite.ts.now()))]

        activeCount = self.threadPool.activeThreadCount()
        t = f'{mode} - {twilight} - Moon: {moon}% - Threads:{activeCount:2d} / 30'
        self.ui.statusOnline.setTitle(t)
        return True

    def updateTime(self):
        """
        :return: True for test purpose
        """
        self.ui.timeComputer.setText(datetime.now().strftime('%H:%M:%S'))
        tzT = time.tzname[1] if time.daylight else time.tzname[0]
        t = f'TZ: {tzT}'
        self.ui.statusTime.setTitle(t)
        return True

    def updateStatusGUI(self, obs):
        """
        :return:    True if ok for testing
        """
        if obs.statusText() is not None:
            self.ui.statusText.setText(obs.statusText())
        else:
            self.ui.statusText.setText('-')

        if self.app.mount.obsSite.status == 0:
            self.changeStyleDynamic(self.ui.tracking, 'running', True)
        else:
            self.changeStyleDynamic(self.ui.tracking, 'running', False)

        if self.app.mount.obsSite.status == 5:
            self.changeStyleDynamic(self.ui.park, 'running', True)
        else:
            self.changeStyleDynamic(self.ui.park, 'running', False)

        if self.app.mount.obsSite.status == 1:
            self.changeStyleDynamic(self.ui.stop, 'running', True)
        else:
            self.changeStyleDynamic(self.ui.stop, 'running', False)

        if self.app.mount.obsSite.status == 10 and not self.satStatus:
            self.app.playSound.emit('SatStartTracking')
            self.satStatus = True
        elif self.app.mount.obsSite.status != 10:
            self.satStatus = False

        return True

    def deleteWindowResource(self, widget=None):
        """
        :return: success
        """
        if not widget:
            return False

        for window in self.uiWindows:
            if self.uiWindows[window]['name'] != widget.objectName():
                continue
            self.uiWindows[window]['classObj'] = None

        return True

    def setColorSet(self):
        """
        :return:
        """
        Styles.colorSet = self.ui.colorSet.currentIndex()
        return True

    def refreshColorSet(self):
        """
        :return:
        """
        self.setColorSet()
        self.setStyleSheet(self.mw4Style)
        self.setupIcons()
        self.app.colorChange.emit()
        return True

    def buildWindow(self, window):
        """
        buildWindow makes new object instance from window class. both are
        stored in the uiWindows dict for usage.

        :return: true for test purpose
        """
        self.uiWindows[window]['classObj'] = self.uiWindows[window]['class'](self.app)
        self.uiWindows[window]['classObj'].destroyed.connect(self.deleteWindowResource)
        self.uiWindows[window]['classObj'].initConfig()
        self.uiWindows[window]['classObj'].showWindow()
        return True

    def toggleWindow(self):
        """
        :return: true for test purpose
        """
        for window in self.uiWindows:
            if self.uiWindows[window]['button'] != self.sender():
                continue

            if not self.uiWindows[window]['classObj']:
                self.buildWindow(window)
            else:
                self.uiWindows[window]['classObj'].close()
        return True

    def showExtendedWindows(self):
        """
        showExtendedWindows opens all extended windows depending on their
        opening status stored in the configuration dict.

        :return: true for test purpose
        """
        for window in self.uiWindows:
            if window == 'showSimulatorW':
                continue
            if not self.app.config.get(window, False):
                continue

            self.buildWindow(window)
        return True

    def waitClosedExtendedWindows(self):
        """
        waits until the window class is deleted
        :return:
        """
        waitDeleted = True
        while waitDeleted:
            for window in self.uiWindows:
                if self.uiWindows[window]['classObj']:
                    continue

                waitDeleted = False
            sleepAndEvents(100)
        return True

    def closeExtendedWindows(self):
        """
        closeExtendedWindows closes all open extended windows by calling
        close.

        :return: true for test purpose
        """
        for window in self.uiWindows:
            if not self.uiWindows[window]['classObj']:
                continue

            self.uiWindows[window]['classObj'].close()
        self.waitClosedExtendedWindows()
        return True

    @staticmethod
    def checkExtension(filePath, ext):
        """
        :param filePath:
        :param ext:
        :return:
        """
        if not filePath.endswith(ext):
            filePath += ext
        return filePath

    def switchProfile(self, config):
        """
        :param config:
        :return:
        """
        self.closeExtendedWindows()
        # self.stopDrivers()
        self.app.config = config
        topo = self.app.initConfig()
        self.app.mount.obsSite.location = topo
        self.initConfig()
        self.showExtendedWindows()
        return True

    def loadProfileGUI(self):
        """
        :return:
        """
        folder = self.app.mwGlob['configDir']
        loadFilePath, name, ext = self.openFile(
            self, 'Open config file', folder, 'Config files (*.cfg)',
            enableDir=False)
        if not name:
            return False

        config = loadProfile(configDir=self.app.mwGlob['configDir'], name=name)
        if config:
            self.ui.profile.setText(name)
            self.msg.emit(1, 'System', 'Profile', f'loaded {name}')
        else:
            self.msg.emit(2, 'System', 'Profile error',
                          f'{name}] cannot no be loaded')
            return False

        self.switchProfile(config)
        return True

    def addProfileGUI(self):
        """
        :return:
        """
        config = self.app.config
        folder = self.app.mwGlob['configDir']
        loadFilePath, name, ext = self.openFile(
            self, 'Open add-on config file', folder, 'Config files (*.cfg)',
            enableDir=False)
        if not name:
            self.ui.profileAdd.setText('-')
            return False

        self.storeConfig()
        self.app.storeConfig()
        configAdd = loadProfile(configDir=self.app.mwGlob['configDir'], name=name)
        if configAdd:
            self.ui.profileAdd.setText(name)
            profile = self.ui.profile.text()
            self.msg.emit(1, 'System', 'Profile', f'Base: {profile}')
            self.msg.emit(1, 'System', 'Profile', f'Add : {name}')
        else:
            self.ui.profileAdd.setText('-')
            self.msg.emit(2, 'System', 'Profile error',
                          f'{name}] cannot no be loaded')
            return False

        config = blendProfile(config, configAdd)
        self.switchProfile(config)
        return True

    def saveConfigAs(self):
        """
        :return:
        """
        folder = self.app.mwGlob['configDir']
        saveFilePath, name, ext = self.saveFile(
            self, 'Save config file', folder, 'Config files (*.cfg)',
            enableDir=False)
        if not name:
            return False

        self.storeConfig()
        self.app.storeConfig()
        suc = saveProfile(configDir=self.app.mwGlob['configDir'],
                          name=name,
                          config=self.app.config)
        if suc:
            self.ui.profile.setText(name)
            self.msg.emit(1, 'System', 'Profile', f'saved {name}')
            self.ui.profileAdd.setText('-')
        else:
            self.msg.emit(2, 'System', 'Profile error',
                          f'{name}] cannot no be saved')
        return True

    def saveConfig(self):
        """
        :return: nothing
        """
        self.storeConfig()
        self.app.storeConfig()
        suc = saveProfile(configDir=self.app.mwGlob['configDir'],
                          name=self.ui.profile.text(),
                          config=self.app.config)
        if suc:
            self.ui.profileAdd.setText('-')
            self.msg.emit(1, 'System', 'Profile', 'Actual profile saved')
        else:
            self.msg.emit(2, 'System', 'Profile',
                          'Actual profile cannot not be saved')
        return suc

    def remoteCommand(self, command):
        """
        remoteCommand received signals from remote class and executes them.
        :param command:
        :return: True for test purpose
        """
        if command == 'shutdown':
            self.quitSave()
            self.msg.emit(2, 'System', 'Remote', 'Shutdown MW4 remotely')
        elif command == 'shutdown mount':
            self.mountShutdown()
            self.msg.emit(2, 'System', 'Remote', 'Shutdown MW4 remotely')
        elif command == 'boot mount':
            self.mountBoot()
            self.msg.emit(2, 'System', 'Remote', 'Boot Mount remotely')
        return True

    def collectWindows(self):
        """
        :return:
        """
        for i, window in enumerate(self.uiWindows):
            if self.uiWindows[window]['classObj']:
                self.uiWindows[window]['classObj'].resize(800, 600)
                self.uiWindows[window]['classObj'].move(i * 50 + 10, i * 50 + 10)
                self.uiWindows[window]['classObj'].activateWindow()
        self.move(i * 50 + 10, i * 50 + 10)
        self.activateWindow()
        return True


from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
import torch
"""
使用该脚本，将lora的权重合并大base model中
"""

def merge_lora_to_base_model():
    model_name_or_path = 'baichuan-inc/baichuan-7B'
    adapter_name_or_path = 'YeungNLP/firefly-baichuan-7b-qlora-sft'
    save_path = 'checkpoint/firefly-baichuan-7b-qlora-sft-merge'

    config = AutoConfig.from_pretrained(model_name_or_path)
    tokenizer = AutoTokenizer.from_pretrained(
        model_name_or_path,
        trust_remote_code=True,
        # llama不支持fast
        use_fast=False if config.model_type == 'llama' else True
    )
    model = AutoModelForCausalLM.from_pretrained(
        model_name_or_path,
        trust_remote_code=True,
        low_cpu_mem_usage=True,
        torch_dtype=torch.float16,
        # device_map='auto',
        device_map={'': 'cpu'}
    )
    model = PeftModel.from_pretrained(model, adapter_name_or_path, device_map={'': 'cpu'})
    model = model.merge_and_unload()

    tokenizer.save_pretrained(save_path)
    model.save_pretrained(save_path)

if __name__ == '__main__':
    merge_lora_to_base_model()


#!/usr/bin/env python
"""
Converts an image to fit a style given by a texture.

Usage:
  convert.py convert <texture> <source> <destination>
  convert.py build <source> <destination>
"""

from __future__ import print_function

import numpy as np
import scipy.ndimage as nd
import sys
import skimage.io
from PIL import Image
import os
import sys
import glob
import warnings
warnings.filterwarnings("ignore", category=UserWarning)

# insert pycaffe into pythonpath
caffe_path = os.path.abspath(os.getenv('CAFFE'))
pycaffe_path = caffe_path + '/python'
if pycaffe_path not in sys.path:
  sys.path.insert(0, pycaffe_path)
# suppress wall of text logging
os.environ['GLOG_minloglevel'] = '2'

import caffe
from caffe.proto import caffe_pb2

DEPLOY_PATH = './models/vgg/deploy.prototxt'
CAFFEMODEL_PATH = './snapshots/VGG_ILSVRC_19_layers.caffemodel'
DEFAULT_CONTENT_LAYERS = ['conv4_2']
DEFAULT_STYLE_LAYERS = ['conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv5_1']

def build_net(deploy_path, caffemodel_path, mode=caffe.TRAIN):
  caffe.set_mode_cpu()

  return caffe.Classifier(deploy_path, caffemodel_path, mode,
      mean=np.array([103.939, 116.779, 123.680]), channel_swap=(2, 1, 0))

def preprocess(net, im):
  return np.float32(np.rollaxis(im, 2)[::-1]) - net.transformer.mean['data']

def deprocess(net, im):
  return np.dstack((im + net.transformer.mean['data'])[::-1])

def layer_texture(net, layer):
  # activations already exist in the net
  activations = net.blobs[layer].data[0]
  n_filters = activations.shape[0]
  activations = activations.reshape((n_filters, -1))
  return activations.dot(activations.T) #/ activations.shape[1]

def build_texture(net, im_files, layers=None):
  """
  Texture is G, where G[l][i, j] is the dot product of the activations of filters i and j of layer l.
  Each dot product is normalized by the dimensions of the activations.

  If multiple images are given, the textures for each image are averaged together.
  """
  layers = layers or DEFAULT_STYLE_LAYERS
  G = {}

  for layer in layers:
    n_filters = net.blobs[layer].data.shape[1]
    G[layer] = np.zeros((len(im_files), n_filters, n_filters))

  for n, im_file in enumerate(im_files):
    # feed forward
    im = np.float32(Image.open(im_file))
    im = preprocess(net, im)

    if im.shape[1] * im.shape[2] > 720 * 442:
      # XXX temporary resize
      print("Warning: image may be too big and cause caffe to segfault, cropping")
      im = im[:, :720, :442]
    blob = net.blobs['data']
    # resize net input
    blob.reshape(1, *im.shape)
    blob.data[0] = im
    net.forward(end=layers[-1])

    # compute dot products
    for layer in layers:
      activations = net.blobs[layer].data[0]
      n_filters = activations.shape[0]
      activations = activations.reshape((n_filters, -1))
      G[layer][n, :, :] = activations.dot(activations.T) #/ activations.shape[1]

  # average imagewise textures
  for layer in G:
    G[layer] = np.average(G[layer], axis=0)

  return G

def adadelta(data, diff, param):
  p = param.get('p', 0.95) # decay constant
  e = param.get('e', 1e-2) # epsilon

  if not 'g_t' in param:
    # initialize accumulation variables
    param['g_t'] = np.zeros_like(data)
    param['dx_t'] = np.zeros_like(data)

  g_t = param['g_t']
  dx_t = param['dx_t']

  # accumulate gradient
  g_t[:] = p * g_t + (1.0 - p) * diff * diff

  # compute update
  rms_g = np.sqrt(g_t + e)
  rms_dx = np.sqrt(dx_t + e)
  dx = -rms_dx / rms_g * diff

  # accumulate update
  dx_t[:] = p * dx_t + (1.0 - p) * dx * dx

  # apply update
  data += dx
  return dx

def irpropm(data, diff, param):
  n_p = param.get('n_p', 1.2) # increase factor
  n_m = param.get('n_m', 0.5) # decrease factor
  step_max = param.get('step_max', 5.0)
  step_min = param.get('step_min', 1e-6)

  if not 'g_t' in param:
    param['g_t'] = np.zeros_like(data)
    param['dx_t'] = np.ones_like(data) / 2.0

  g_t = param['g_t']   # prev gradient
  dx_t = param['dx_t'] # prev step

  d_p = (diff * g_t) >= 0
  d_m = (diff * g_t) < 0

  dx_t[:] = np.maximum(dx_t, step_min)
  d = (d_p * np.minimum(dx_t * n_p, step_max)) + (d_m * np.maximum(dx_t * n_m, step_min))
  diff[d_m] = 0.0

  step = -np.sign(diff) * d
  data += step

  g_t[:] = diff
  dx_t[:] = d
  return step

def train_content_layer(net, layer, reference_data, alpha, param):
  net.forward(end=layer)

  activation = net.blobs[layer].data[0]
  ref_activation = reference_data[layer]
  net.blobs[layer].diff[0] = alpha * (activation - ref_activation) * (activation > 0)

  net.backward(start=layer)

  blob = net.blobs['data']
  irpropm(blob.data[0], blob.diff[0], param)

def train_style_layer(net, layer, texture, beta, layer_weight, param):
  net.forward(end=layer)

  G_l = layer_texture(net, layer)

  activation = net.blobs[layer].data[0]
  n_filters = activation.shape[0]
  n_size = activation.shape[1] * activation.shape[2]
  net.blobs[layer].diff[0] = beta * layer_weight / ((n_filters * n_size) ** 2) * \
    (G_l - texture[layer]).T.dot(activation.transpose(1, 0, 2)) * (activation > 0)

  net.backward(start=layer)

  blob = net.blobs['data']
  irpropm(blob.data[0], blob.diff[0], param)

def convert(net, im, texture, n_iter=250, content_layers=None, style_layers=None, alpha=1.0, beta=1000.0):
  content_layers = content_layers or DEFAULT_CONTENT_LAYERS
  style_layers = style_layers or DEFAULT_STYLE_LAYERS
  layer_weight = 1.0 / len(style_layers)
  im = preprocess(net, im)

  # compute reference activations
  reference_data = {}
  blob = net.blobs['data']
  blob.reshape(1, *im.shape)
  blob.data[0] = im
  for layer in content_layers:
    net.forward(end=layer)
    reference_data[layer] = np.copy(net.blobs[layer].data[0])

  param = {}

  for i in range(n_iter):
    # backprop onto reference image according to loss from http://arxiv.org/abs/1508.06576
    for layer in content_layers:
      train_content_layer(net, layer, reference_data, alpha, param)
    for layer in style_layers:
      train_style_layer(net, layer, texture, beta, layer_weight, param)

    print(i, end=" ")
    sys.stdout.flush()

  im = blob.data[0]
  return deprocess(net, im)

def convert_cmd(target, source, destination):
  texture = np.load(target)
  im = np.float32(Image.open(source))
  net = build_net(DEPLOY_PATH, CAFFEMODEL_PATH)
  im_new = convert(net, im, texture)

  im_new = np.uint8(np.clip(im_new, 0, 255))
  skimage.io.imsave(destination, im_new)

def build_cmd(source, destination):
  if os.path.isfile(source):
    im_files = [source]
  elif os.path.isdir(source):
    im_files = glob.iglob("{}/*.*".format(source))
  else:
    raise ValueError("Source does not exist.")

  net = build_net(DEPLOY_PATH, CAFFEMODEL_PATH)
  G = build_texture(net, im_files)
  with open(destination, 'wb') as f:
    np.savez(f, **G)

if __name__ == '__main__':
  if len(sys.argv) == 5 and sys.argv[1] == 'convert':
    convert_cmd(sys.argv[2], sys.argv[3], sys.argv[4])

  elif len(sys.argv) == 4 and sys.argv[1] == 'build':
    build_cmd(sys.argv[2], sys.argv[3])

  else:
    print(__doc__)


import os
import sys
import inspect
# insert env dir into sys
# env must contain env folder:
currentdir = os.path.dirname(os.path
                             .abspath(inspect.getfile(inspect.currentframe())))
env = currentdir.find("env")
env_dir = currentdir[:env]
# print(env_dir)
if env_dir not in sys.path:
    sys.path.insert(0, env_dir)

from env.equation.data.terms.output.cpp.additions.someFuncs import NewtonBinomCoefficient

import logging

# create logger
log_level = logging.INFO  # logging.DEBUG
logging.basicConfig(level=log_level)
logger = logging.getLogger('gen_pure_common.py')
logger.setLevel(level=log_level)

class GenPureCommon():
    '''
    DESCRIPTION:

    Find d^{n}(u)/d(x)^{n}
     where n is
      n > 1 for central functions
            (used method common_diff)

    INPUT:

    See ``self.__init__``

    '''
    def __init__(self, blockNumber, unknownVarIndex,
                 indepVar, indepVarOrders):

        '''
        Inputs:

        - ``blockNumber`` -- used for strides and cellsize
        variables names

        - ``unknownVarIndex`` -- shift index for variable
        (like (U, V)-> (source[+0], source[+1]))
        (Ex: dict([('U', 0), ('V', 1)])['U'])

        - ``indepVar`` -- like 'x' i.e. for which diff maked

        - ``indepVarOrders`` -- dict([(var, float(order))])

        '''

        # PARAMS FOR ALL
        self.blockNumber = blockNumber

        # shift index for variable like
        # like (U,V)-> (source[+0], source[+1])
        self.unknownVarIndex = unknownVarIndex

        # like ['x'] i.e. for which diff maked
        # see in begining of callDerivGenerator
        # TODO use 'x' instead:
        self.indepVar = indepVar
        x = self.indepVar

        logger.debug("indepVarOrders:")
        logger.debug(indepVarOrders)

        self.derivOrder = int(indepVarOrders[x])

        # like ['x', 'y', 'z']
        self.userIndepVariables = ['x', 'y', 'z']

        # for method choosing for special
        var = self.indepVar
        self.indepVarIndexList = [self.userIndepVariables.index(var)]

        # for all
        self.make_general_data()
        # END FOR ALL

    def make_general_data(self):
        '''
        DESCRIPTION:
        DXM1 - means dx
        DXM2 - means dx^2
        '''
        self.increment = ('D' + self.indepVar.upper()
                          + 'M' + str(self.derivOrder))
        self.specialIncrement = 'D' + self.indepVar.upper()
        self.stride = ('Block' + str(self.blockNumber)
                       + 'Stride' + self.indepVar.upper())

    def createIndicesList(self):
        '''
        DESCRIPTION:

        Т.к. для CentralFunction умеем генерировать аппроксимации
        производных любого порядка, то эти аппроксимации содержат
        много слагаемых, каждое из которых имеет свой индекс.

        EXAMPLES:

        For derivOrder = 1
        [' + 0', '-1']

        For derivOrder = 2
        [' + 1', ' + 0', '-1']

        For derivOrder = 3
        [' + 1', ' + 0', '-1', '-2']

        USED FUNCTIONS::

            self.derivOrder
        '''
        leftIndex = self.derivOrder // 2
        rightIndex = -(self.derivOrder - leftIndex)
        reverseList = [i for i in range(rightIndex, leftIndex + 1)]
        comfortableList = reverseList[::-1]
        indicesListAsString = []
        for index in comfortableList:
            if index >= 0:
                indicesListAsString.extend([' + ' + str(int(index))])
            else:
                indicesListAsString.extend([str(int(index))])
        return indicesListAsString

    def createCoefficientList(self):
        '''
        DESCRIPTION:

        Т.к. для CentralFunction умеем генерировать аппроксимации
        производных любого порядка, то эти аппроксимации содержат
        много слагаемых, перед каждым из которых имеется свой
        коэффициент.

        Create Newton binom coefficients

        self.derivOrder
        '''
        numberList = [NewtonBinomCoefficient(self.derivOrder, k)
                      for k in range(0, self.derivOrder + 1)]
        stringList = []
        for number in numberList:
            stringList.extend([str(number)])
        return stringList

    def common_diff(self):
        '''
        DESCRIPTION:

        generate cpp derivative for central function
        or for border in case of non border variable
        (for example: for y at border x=0)

        for ``derivOrder = 1``:

        .. math::  du/dx = (u_{i+1}-u_{i-1})/(2 dx)

        for ``derivOrder = 2``:

        .. math:: ddu/ddx = (u_{i+1}-2*u_{i}+u_{i-1})/(dx^2)

        Template:

        source[idx  # point in that derive will find
        +  stride  * Block0CELLSIZE  # +1 to some of {x,y,z} direction
                                     # (defined by stride)
                                     # ('x': +1,
                                     #  'y': + Block0StrideY*Block0CELLSIZE,
                                     #  'z': + Block0StrideZ*Block0CELLSIZE
                                     #         = Block0SizeY*Block0CountY*Block0CELLSIZE)
        +  $ unknownVarIndex[0] $ ]  # shift, differ for each variable ('U': +0, 'V': +1)

        USED FUNCTIONS:

        str self.blockNumber
        self.delay
        str self.unknownVarIndex

        self.createIndicesList
        self.createCoefficientList

        '''
        try:
            increment = self.increment
            stride = self.stride
        except:
            raise(SyntaxError("increment or stride are not defined"
                              + "\n use make_general_data first"))

        if self.derivOrder == 1:
            toLeft = ('source['+'delay'+'][idx - '
                      + stride + ' * ' + 'Block'
                      + str(self.blockNumber) + 'CELLSIZE + '
                      + str(self.unknownVarIndex) + ']')
            toRight = ('source['+'delay'+'][idx + '
                       + stride + ' * ' + 'Block'
                       + str(self.blockNumber) + 'CELLSIZE + '
                       + str(self.unknownVarIndex) + ']')
            return('0.5 * ' + increment + ' * '
                   + '(' + toRight + ' - ' + toLeft + ')')
        else:
            indicesList = self.createIndicesList()
            coefficientList = self.createCoefficientList()
            finiteDifference = ''
            for i, index in enumerate(indicesList):
                m1 = i % 2
                m2 = (i + 1) % 2
                m3 = i > 0
                m4 = coefficientList[i] != '1.0'
                startOfLine = (finiteDifference
                               + m1 * ' - ' + m2 * m3 * ' + '
                               + m4 * (str(coefficientList[i])
                                       + ' * '))
                restOfLine = ('source['+'delay'+'][idx' + str(index)
                              + ' * ' + stride + ' * ' + 'Block'
                              + str(self.blockNumber) + 'CELLSIZE + '
                              + str(self.unknownVarIndex) + ']')
                finiteDifference = startOfLine + restOfLine
            return '(' + increment + ' * ' + '(' + finiteDifference + ')' + ')'


#!/usr/bin/env python3
""" Base class for Face Aligner plugins
    Plugins should inherit from this class

    See the override methods for which methods are
    required.

    The plugin will receive a dict containing:
    {"filename": <filename of source frame>,
     "image": <source image>,
     "detected_faces": <list of DlibRectangles>}

    For each source item, the plugin must pass a dict to finalize containing:
    {"filename": <filename of source frame>,
     "image": <source image>,
     "detected_faces": <list of dlibRectangles>,
     "landmarks": <list of landmarks>}
    """

import logging
import os
import traceback

from io import StringIO

from lib.aligner import Extract
from lib.gpu_stats import GPUStats
from lib.faces_detect import DetectedFace

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

class Aligner():
    """ Landmarks Aligner Object """
    def __init__(self, loglevel):
        logger.debug("Initializing %s", self.__class__.__name__)
        self.loglevel = loglevel
        self.cachepath = os.path.join(os.path.dirname(__file__), ".cache")
        self.extract = Extract()
        self.init = None

        # The input and output queues for the plugin.
        # See lib.queue_manager.QueueManager for getting queues
        self.queues = {"in": None, "out": None}

        #  Path to model if required
        self.model_path = self.set_model_path()

        # Approximate VRAM required for aligner. Used to calculate
        # how many parallel processes / batches can be run.
        # Be conservative to avoid OOM.
        self.vram = None
        logger.debug("Initialized %s", self.__class__.__name__)

    # <<< OVERRIDE METHODS >>> #
    # These methods must be overriden when creating a plugin
    @staticmethod
    def set_model_path():
        """ path to data file/models
            override for specific detector """
        raise NotImplementedError()

    def initialize(self, *args, **kwargs):
        """ Inititalize the aligner
            Tasks to be run before any alignments are performed.
            Override for specific detector """
        logger_init = kwargs["log_init"]
        log_queue = kwargs["log_queue"]
        logger_init(self.loglevel, log_queue)
        logger.debug("_base initialize %s: (PID: %s, args: %s, kwargs: %s)",
                     self.__class__.__name__, os.getpid(), args, kwargs)
        self.init = kwargs["event"]
        self.queues["in"] = kwargs["in_queue"]
        self.queues["out"] = kwargs["out_queue"]

    def align(self, *args, **kwargs):
        """ Process landmarks
            Override for specific detector
            Must return a list of dlib rects"""
        if not self.init:
            self.initialize(*args, **kwargs)
        logger.debug("Launching Align: (args: %s kwargs: %s)", args, kwargs)

    # <<< DETECTION WRAPPER >>> #
    def run(self, *args, **kwargs):
        """ Parent align process.
            This should always be called as the entry point so exceptions
            are passed back to parent.
            Do not override """
        try:
            self.align(*args, **kwargs)
        except Exception:  # pylint: disable=broad-except
            logger.error("Caught exception in child process: %s", os.getpid())
            # Display traceback if in initialization stage
            if not self.init.is_set():
                logger.exception("Traceback:")
            tb_buffer = StringIO()
            traceback.print_exc(file=tb_buffer)
            exception = {"exception": (os.getpid(), tb_buffer)}
            self.queues["out"].put(exception)
            exit(1)

    # <<< FINALIZE METHODS>>> #
    def finalize(self, output):
        """ This should be called as the final task of each plugin
            aligns faces and puts to the out queue """
        if output == "EOF":
            logger.trace("Item out: %s", output)
            self.queues["out"].put("EOF")
            return
        logger.trace("Item out: %s", {key: val
                                      for key, val in output.items()
                                      if key != "image"})
        self.queues["out"].put((output))

    # <<< MISC METHODS >>> #
    @staticmethod
    def get_vram_free():
        """ Return free and total VRAM on card with most VRAM free"""
        stats = GPUStats()
        vram = stats.get_card_most_free()
        logger.verbose("Using device %s with %sMB free of %sMB",
                       vram["device"],
                       int(vram["free"]),
                       int(vram["total"]))
        return int(vram["card_id"]), int(vram["free"]), int(vram["total"])

    def get_item(self):
        """ Yield one item from the queue """
        while True:
            item = self.queues["in"].get()
            if isinstance(item, dict):
                logger.trace("Item in: %s", {key: val
                                             for key, val in item.items()
                                             if key != "image"})
                # Pass Detector failures straight out and quit
                if item.get("exception", None):
                    self.queues["out"].put(item)
                    exit(1)
            else:
                logger.trace("Item in: %s", item)
            yield item
            if item == "EOF":
                break


import sys
import json

sys.path.insert(0, 'src/data')
sys.path.insert(0, 'src/utils')
sys.path.insert(0, 'src/models')

from clean import remove_data
from scrape import scrape_data
from fbpreprocessing import fb_preprocessing
from model_preprocessing import get_data
from run_models import run_models
from youtube import get_youtube

def gather_data(data_params):
    with open('config/chromedriver.json') as fh:
        chromedriver_path = json.load(fh)['chromedriver_path']

    print("Scraping data...")
    scrape_data(chromedriver_path,
                data_params['all_links_pickle_path'],
                data_params['fbworkouts_path'],
                data_params['comments_path'])
    print("Scraping done.")

    print("Querying Youtube API...")
    get_youtube(data_params['fbworkouts_path'],
                data_params['youtube_csv_path'])
    print("Querying done")

def preprocess(data_params, d):
    print("Preprocessing...")
    fb_preprocessing(
        fbworkouts_path = data_params['fbworkouts_path'],
        fbworkouts_clean_path = data_params['fbworkouts_clean_path'],
        comments_path = data_params['comments_path'],
        fbcommenters_path = data_params['fbcommenters'],
        user_item_interactions_path = data_params['user_item_interactions_path'],
        fbworkouts_meta_path = data_params['fbworkouts_meta_path'],
        all_links_pickle_path = data_params['all_links_pickle_path'],
        youtube_csv_path = data_params['youtube_csv_path'],
        d=d
        )
    print("Data preprocessing done.")

def run_model(data_params, k):
    data = get_data(data_params['user_item_interactions_path'])
    run_models(data, k=k)

def main(targets):
    if 'clean' in targets:
        remove_data()
        print("Data cleaned.")
        return

    if 'test' in targets:
        with open('config/test-params.json') as fh:
            data_params = json.load(fh)

        preprocess(data_params, d=0)
        run_model(data_params, k=None)
        return

    with open('config/data-params.json') as fh:
        data_params = json.load(fh)

    if 'all' in targets:
        gather_data(data_params)
        preprocess(data_params, d=5)
        run_model(data_params, k=20)
        return

    if 'data' in targets:
        gather_data(data_params)
        preprocess(data_params, d=5)

    if 'model' in targets:
        run_model(data_params, k=20)

    return

if __name__ == '__main__':
    targets = sys.argv[1:]
    main(targets)


from datetime import datetime
from core.validators import FieldValidator
from .constants import CREATE, UPDATE

class Field:
    """
    Base class of any field.
    """
    field_type = None
    _name = None
    _value = None
    is_sub_field = False
    _reserved_attributes = {
        'null': bool,
        'blank': bool,
        'min_length': int,
        'max_length': int,
        'unique': bool,
        'choices': (tuple, list),
        'on_delete': int
    }

    def __setattr__(self, key, value):
        if key in self._reserved_attributes:
            required_type = self._reserved_attributes.get(key)

            if value is not None and not isinstance(value, required_type):
                raise TypeError(
                    'Reserved attr `{attr_name}` has wrong type! '
                    'Expected `{attr_type}`'.format(
                        attr_name=key,
                        attr_type=required_type.__name__
                    )
                )

        super().__setattr__(key, value)

    def set_field_name(self, name):
        self._name = name

    def get_field_name(self):
        return self._name

    def set_field_value(self, value):
        self._value = value

    def get_field_value(self):
        return self._value

    def get_choice_key(self, value):
        choices = getattr(self, 'choices', None)

        if choices:
            choices = {key: value for key, value in choices}

            if value in choices.keys():
                value = choices.get(value)

            elif value not in choices.values():
                raise ValueError(
                    'The value \'{field_value}\' is not specified in the \'choices\' attribute.'.format(
                        field_value=value
                    )
                )

        return value

    def get_choice_value(self, key):
        choices = getattr(self, 'choices', None)
        value = key

        if choices:
            choices = {value: key for key, value in choices}

            if key not in choices.keys():
                raise ValueError(
                    'The value \'{field_value}\' is not specified in the \'choices\' attribute.'.format(
                        field_value=key
                    )
                )

            value = choices.get(key)

        return value

    def get_value(self, name, value, action):
        default = getattr(self, 'default', None)
        choices = getattr(self, 'choices', None)

        # Process the 'default' attribute
        if default is not None and not value:
            value = default() if callable(default) else default

        # Process the 'choices' attribute
        if value and choices:
            value = self.get_choice_key(value)

        return value

    def validate(self, name, value):
        field_validator = FieldValidator(self, name, value)
        field_validator.validate()

        return value

    def to_internal_value(self, value):
        """
        Define this method if you need to save specific data format.
        Here you must to serialize your data.
        """
        return value

    def to_external_value(self, value):
        """
        Define this method if you specified `to_internal_value`,
        to represent data from the storage.
        Here you must to deserialize your data.
        """
        return value

class BoolField(Field):
    field_type = bool

    def __init__(self, null=False, default=None, choices=None):
        self.null = null
        self.default = default
        self.choices = choices

class StringField(Field):
    field_type = str

    def __init__(self, null=False, blank=True, min_length=None, max_length=None,
                 unique=False, index=None, default=None, choices=None):
        self.null = null
        self.blank = blank
        self.min_length = min_length
        self.max_length = max_length
        self.unique = unique
        self.index = index
        self.default = default
        self.choices = choices

class IntegerField(Field):
    field_type = int

    def __init__(self, null=False, unique=False, default=None, choices=None):
        self.null = null
        self.unique = unique
        self.default = default
        self.choices = choices

class FloatField(Field):
    field_type = float

    def __init__(self, null=False, unique=False, default=None, choices=None):
        self.null = null
        self.unique = unique
        self.default = default
        self.choices = choices

class ListField(Field):
    field_type = list

    def __init__(self, child=None, null=False, length=None, unique=False, default=None):
        self.child = child
        self.null = null
        self.length = length
        self.unique = unique
        self.default = default

    # For IDE tips
    def __iter__(self):
        return self

    def validate(self, name, value):
        value = super().validate(name, value)

        # TODO: use multiprocessing pool of workers
        if value and self.child is not None:
            for list_item in value:
                self.child.is_sub_field = True
                self.child.validate(name, list_item)

        return value

class DictField(Field):
    field_type = dict

    def __init__(self, null=False, unique=False, min_length=None, max_length=None, default=None):
        self.null = null
        self.unique = unique
        self.min_length = min_length
        self.max_length = max_length
        self.default = default

    # For IDE tips
    def __iter__(self):
        return self

class DateTimeField(Field):
    field_type = datetime

    def __init__(self, null=False, auto_now_create=False, auto_now_update=False):
        self.null = null
        self.auto_now_create = auto_now_create
        self.auto_now_update = auto_now_update

    def get_value(self, name, value, action):
        if (action is CREATE and self.auto_now_create) or (action is UPDATE and self.auto_now_update) and not value:
            # MonogoDB rounds microseconds, and ODM does not request the created document,
            # for data consistency I reset them
            value = datetime.now().replace(microsecond=0)

        return value

class BaseRelationField(Field):
    backward_class = None
    _query = None

    def __init__(self, relation, related_name=None, null=False, on_delete=None):
        self.relation = relation
        self.related_name = related_name
        self.null = null
        self.on_delete = on_delete

    def __aiter__(self):
        return self

    # TODO: Test it!
    async def __anext__(self):
        async for item in self._get_query():
            return item
        raise StopAsyncIteration()

    def __await__(self):
        return self._get_query().__await__()

    def _get_query(self):
        raise NotImplementedError

class BaseBackwardRelationField(Field):
    _query = None

    def get_query(self):
        raise NotImplementedError

    def __init__(self, relation):
        self.relation = relation

    def __aiter__(self):
        return self

    async def __anext__(self):
        async for item in self.get_query():
            return item
        raise StopAsyncIteration()

    def __await__(self):
        return self.get_query().__await__()

class ForeignKeyBackward(BaseBackwardRelationField):
    def get_query(self):
        if not self._query:
            filter_kwargs = {self._name: self._value}
            self._query = self.relation.objects.filter(**filter_kwargs)

        return self._query

class OneToOneBackward(BaseBackwardRelationField):
    def get_query(self):
        if not self._query:
            filter_kwargs = {self._name: self._value}
            self._query = self.relation.objects.get(**filter_kwargs)

        return self._query

class ForeignKey(BaseRelationField):
    backward_class = ForeignKeyBackward

    def _get_query(self):
        if not self._query:
            get_kwargs = {'_id': self._value.id}
            self._query = self.relation.objects.get(**get_kwargs)

        return self._query

class OneToOne(BaseRelationField):
    backward_class = OneToOneBackward

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.unique = True

    def _get_query(self):
        get_kwargs = {'_id': self._value.id}
        return self.relation.objects.get(**get_kwargs)


"""
Code borrowed and adapted from
https://github.com/shubham-goel/ucmr/blob/master/src/nnutils/geom_utils.py
"""
from __future__ import absolute_import, division, print_function

import torch
import numpy as np

def get_viewpoint(azimouth):
    ''' Returns the viewpoint given azimouth angle '''
    return 'side' if (azimouth>=45 and azimouth<135) or (azimouth>=-135 and azimouth<-45) else 'front-back'

def quat_inverse(quat):
    """
    quat: B x 4: [quaternions]
    returns inverted quaternions
    """
    flip = torch.tensor([1,-1,-1,-1],dtype=quat.dtype,device=quat.device)
    quat_inv = quat * flip.view((1,)*(quat.dim()-1)+(4,))
    return quat_inv

def hamilton_product(qa, qb):
    """Multiply qa by qb.
    Args:
        qa: B X N X 4 quaternions
        qb: B X N X 4 quaternions
    Returns:
        q_mult: B X N X 4
    """
    qa_0 = qa[..., 0]
    qa_1 = qa[..., 1]
    qa_2 = qa[..., 2]
    qa_3 = qa[..., 3]

    qb_0 = qb[..., 0]
    qb_1 = qb[..., 1]
    qb_2 = qb[..., 2]
    qb_3 = qb[..., 3]

    # See https://en.wikipedia.org/wiki/Quaternion#Hamilton_product
    q_mult_0 = qa_0*qb_0 - qa_1*qb_1 - qa_2*qb_2 - qa_3*qb_3
    q_mult_1 = qa_0*qb_1 + qa_1*qb_0 + qa_2*qb_3 - qa_3*qb_2
    q_mult_2 = qa_0*qb_2 - qa_1*qb_3 + qa_2*qb_0 + qa_3*qb_1
    q_mult_3 = qa_0*qb_3 + qa_1*qb_2 - qa_2*qb_1 + qa_3*qb_0

    return torch.stack([q_mult_0, q_mult_1, q_mult_2, q_mult_3], dim=-1)

def quat_rotate(X, quat):
    """Rotate points by quaternions.

    Args:
        X: B X N X 3 points
        quat: B X 4 quaternions

    Returns:
        X_rot: B X N X 3 (rotated points)
    """
    # repeat q along 2nd dim
    # import ipdb; ipdb.set_trace()
    # ones_x = X[[0], :, :][:, :, [0]]*0 + 1
    # q = torch.unsqueeze(q, 1)*ones_x
    quat      = quat[:,None,:].expand(-1,X.shape[1],-1)
    quat_conj = torch.cat([ quat[:, :, 0:1] , -1*quat[:, :, 1:4] ], dim=-1)
    X         = torch.cat([ X[:, :, 0:1]*0, X ], dim=-1)
    X_rot     = hamilton_product(quat, hamilton_product(X, quat_conj))
    return X_rot[:, :, 1:4]

# ------------------------------

def axisangle2quat(axis, angle):
    """
    axis : Bx3: [axis]
    angle: B  : [angle]
    returns
        quaternion: Bx4
    """
    axis  = torch.nn.functional.normalize(axis,dim=-1)
    angle = angle.unsqueeze(-1)/2
    quat  = torch.cat([angle.cos(), angle.sin()*axis], dim=-1)
    return quat

def quat2axisangle(quat):
    """
    quat: B x 4: [quaternions]
    returns quaternion axis, angle
    """
    cos   = quat[..., 0]
    sin   = quat[..., 1:].norm(dim=-1)
    axis  = quat[..., 1:]/sin[..., None]
    angle = 2*cos.clamp(-1+1e-6,1-1e-6).acos()
    return axis, angle

def get_base_quaternions(num_pose_az=8, num_pose_el=5, initial_quat_bias_deg=90., elevation_bias=0, azimuth_bias=0):
    _axis = torch.eye(3).float()

    # Quaternion base bias
    xxx_base = [1.,0.,0.]
    aaa_base = initial_quat_bias_deg
    axis_base = torch.tensor(xxx_base).float()
    angle_base = torch.tensor(aaa_base).float() / 180. * np.pi
    qq_base = axisangle2quat(axis_base, angle_base) # 4

    # Quaternion multipose bias
    azz = torch.as_tensor(np.linspace(0,2*np.pi,num=num_pose_az,endpoint=False)).float() + azimuth_bias * np.pi/180
    ell = torch.as_tensor(np.linspace(-np.pi/2,np.pi/2,num=(num_pose_el+1),endpoint=False)[1:]).float() + elevation_bias * np.pi/180
    quat_azz = axisangle2quat(_axis[1], azz) # num_pose_az,4
    quat_ell = axisangle2quat(_axis[0], ell) # num_pose_el,4
    quat_el_az = hamilton_product(quat_ell[None,:,:], quat_azz[:,None,:]) # num_pose_az,num_pose_el,4
    quat_el_az = quat_el_az.view(-1,4)                  # num_pose_az*num_pose_el,4
    _quat = hamilton_product(quat_el_az, qq_base[None,...]).float()

    return _quat

def camera_quat_to_position_az_el(quat, initial_quat_bias_deg=0.):
    """Quat: N,4"""
    assert(quat.dim()==2)
    assert(quat.shape[1]==4)
    quat    = quat_to_camera_position(quat, initial_quat_bias_deg=initial_quat_bias_deg)
    quat_uv = convert_3d_to_uv_coordinates(quat)
    return quat_uv

def quat_to_camera_position(quat, initial_quat_bias_deg):
    """Quat: N,4"""
    X = torch.zeros((quat.shape[0],1,3),dtype=torch.float32,device=quat.device)
    X[:,:,2] = -1
    new_quat = quat_inverse(quat)

    xxx_base   = [1.,0.,0.]
    aaa_base   = initial_quat_bias_deg
    axis_base  = torch.tensor(xxx_base)
    angle_base = torch.tensor(aaa_base) / 180. * np.pi
    qq_base    = axisangle2quat(axis_base, angle_base) # 4
    new_quat   = hamilton_product(qq_base[None,:], new_quat)

    new_quat = hamilton_product(axisangle2quat( torch.eye(3, dtype=torch.float32)[0], torch.tensor(np.pi/2))[None,:],  new_quat ) # rotate 90deg about X
    new_quat = hamilton_product(axisangle2quat( torch.eye(3, dtype=torch.float32)[2], torch.tensor(-np.pi/2))[None,:], new_quat ) # rotate -90deg about Z
    rotX     = quat_rotate(X, new_quat).squeeze(1)   # ...,3
    return rotX

def convert_3d_to_uv_coordinates(X):
    """
    X : N,3
    Returns UV: N,2 normalized to [-1, 1]
    U: Azimuth: Angle with +X [-pi,pi]
    V: Inclination: Angle with +Z [0,pi]
    """
    if type(X) == torch.Tensor:
        eps=1e-4
        rad = torch.norm(X, dim=-1).clamp(min=eps)
        theta = torch.acos( (X[..., 2] / rad).clamp(min=-1+eps,max=1-eps) )  # Inclination: Angle with +Z [0,pi]
        phi = torch.atan2(X[..., 1], X[..., 0])                              # Azimuth    : Angle with +X [-pi,pi]
        vv = (theta / np.pi) * 2 - 1
        uu = ((phi + np.pi) / (2*np.pi)) * 2 - 1
        uv = torch.stack([uu, vv],dim=-1)
    else:
        rad = np.linalg.norm(X, axis=-1)
        rad = np.clip(rad, 1e-12, None)
        theta = np.arccos(X[..., 2] / rad)                                   # Inclination: Angle with +Z [0,pi]
        phi = np.arctan2(X[..., 1], X[..., 0])                               # Azimuth    : Angle with +X [-pi,pi]
        vv = (theta / np.pi) * 2 - 1
        uu = ((phi + np.pi) / (2*np.pi)) * 2 - 1
        uv = np.stack([uu, vv],-1)
    return uv