import numpy
from openquake.baselib import sap, datastore
from openquake.hazardlib.stats import mean_curve, compute_pmap_stats
from openquake.calculators import getters

def make_figure(indices, n, imtls, spec_curves, curves=(), label=''):
    """
    :param indices: the indices of the sites under analysis
    :param n: the total number of sites
    :param imtls: ordered dictionary with the IMTs and levels
    :param spec_curves: a dictionary of curves IMT -> array(n_sites, n_levels)
    :param curves: a dictionary of dictionaries IMT -> array
    :param label: the label associated to `spec_curves`
    """
    # NB: matplotlib is imported inside since it is a costly import
    import matplotlib.pyplot as plt

    fig = plt.figure()
    n_imts = len(imtls)
    n_sites = len(indices)
    spec_curves = spec_curves.convert(imtls, n)
    all_curves = [c.convert(imtls, n) for c in curves]
    for i, site in enumerate(indices):
        for j, imt in enumerate(imtls):
            ax = fig.add_subplot(n_sites, n_imts, i * n_imts + j + 1)
            ax.grid(True)
            ax.set_xlabel('site %d, %s' % (site, imt))
            ax.set_ylim([0, 1])
            if j == 0:  # set Y label only on the leftmost graph
                ax.set_ylabel('PoE')
            if spec_curves is not None:
                ax.plot(imtls[imt], spec_curves[imt][site], '--', label=label)
            for r, curves in enumerate(all_curves):
                ax.plot(imtls[imt], curves[imt][site], label=str(r))
            ax.legend()
    return plt

def get_pmaps(dstore, indices):
    rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
    getter = getters.PmapGetter(dstore, rlzs_assoc)
    getter.init()
    pmaps = getter.get_pmaps(indices)
    weights = dstore['csm_info'].rlzs['weight']
    mean = compute_pmap_stats(pmaps, [mean_curve], weights)
    return mean, pmaps

@sap.Script
def plot(calc_id, other_id=None, sites='0'):
    """
    Hazard curves plotter.
    """
    # read the hazard data
    haz = datastore.read(calc_id)
    other = datastore.read(other_id) if other_id else None
    oq = haz['oqparam']
    indices = numpy.array(list(map(int, sites.split(','))))
    n_sites = len(haz['sitecol'])
    if not set(indices) <= set(range(n_sites)):
        invalid = sorted(set(indices) - set(range(n_sites)))
        print('The indices %s are invalid: no graph for them' % invalid)
    valid = sorted(set(range(n_sites)) & set(indices))
    print('Found %d site(s); plotting %d of them' % (n_sites, len(valid)))
    if other is None:
        mean_curves, pmaps = get_pmaps(haz, indices)
        single_curve = len(pmaps) == 1
        plt = make_figure(valid, n_sites, oq.imtls, mean_curves,
                          [] if single_curve else pmaps, 'mean')
    else:
        mean1, _ = get_pmaps(haz, indices)
        mean2, _ = get_pmaps(other, indices)
        plt = make_figure(valid, n_sites, oq.imtls, mean1,
                          [mean2], 'reference')
    plt.show()

plot.arg('calc_id', 'a computation id', type=int)
plot.arg('other_id', 'optional id of another computation', type=int)
plot.opt('sites', 'comma-separated string with the site indices')


# -*- coding=UTF-8 -*-
# pyright: strict
"""template matching.  """
from __future__ import annotations

import logging
import os
import pathlib
from typing import Dict, Iterator, Optional, Set, Text, Tuple, Union

import cv2
import numpy as np
from PIL.Image import Image
from PIL.Image import open as open_image

from . import imagetools, mathtools, app

TARGET_WIDTH = 540

class g:
    last_screenshot_save_path: str = ""

    @property
    def _legacy_screenshot_width(self):
        import warnings

        warnings.warn("use app.device.width() instead", DeprecationWarning)
        return app.device.width()

    @_legacy_screenshot_width.setter
    def _legacy_screenshot_width(self, v: int):
        import warnings

        warnings.warn("use app.device.width() instead", DeprecationWarning)

_LOADED_TEMPLATES: Dict[Text, Image] = {}

def _cv_image(img: Image):
    return cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)

def load(name: Text) -> Image:
    if name not in _LOADED_TEMPLATES:
        app.log.text("load: %s" % name, level=app.DEBUG)
        img = open_image(pathlib.Path(__file__).parent / "templates" / name)
        _LOADED_TEMPLATES[name] = img
    return _LOADED_TEMPLATES[name]

_NOT_EXISTED_NAMES: Set[Text] = set()

def try_load(name: Text) -> Optional[Image]:
    if name in _NOT_EXISTED_NAMES:
        return None
    try:
        return load(name)
    except Exception as ex:
        app.log.text("can not load: %s: %s" % (name, ex), level=app.DEBUG)
        _NOT_EXISTED_NAMES.add(name)
        return None

def add_middle_ext(name: Text, value: Text) -> Text:
    parts = name.split(".")
    parts.insert(max(len(parts) - 1, 1), value)
    return ".".join(parts)

class Specification:
    @classmethod
    def from_input(cls, input: Input) -> Specification:
        if isinstance(input, Specification):
            return input
        return Specification(input)

    def __init__(
        self,
        name: Text,
        pos: Optional[Text] = None,
        *,
        threshold: float = 0.9,
        lightness_sensitive: bool = True,
    ):
        self.name = name
        self.pos = pos
        self.threshold = threshold
        self.lightness_sensitive = lightness_sensitive

    def load_pos(self) -> Optional[Image]:
        return try_load(self.pos or add_middle_ext(self.name, "pos"))

    def match(self, img: Image, pos: Tuple[int, int]) -> bool:
        x, y = pos
        if self.lightness_sensitive:
            tmpl_img = load(self.name)
            match_img = img.crop((x, y, x + tmpl_img.width, y + tmpl_img.height))

            cv_tmpl_img = np.asarray(tmpl_img.convert("L"))
            cv_match_img = np.asarray(match_img.convert("L"))
            match_min, match_max, _, _ = cv2.minMaxLoc(cv_match_img)
            tmpl_min, tmpl_max, _, _ = cv2.minMaxLoc(cv_tmpl_img)

            max_diff = (match_max - tmpl_max) / 255.0
            min_diff = (match_min - tmpl_min) / 255.0
            if max_diff < 0:
                max_diff *= -1
                min_diff *= -1

            lightness_similarity = 1 - (abs(max_diff + min_diff) / 2)
            app.log.text(
                "lightness match: tmpl=%s, similarity=%.3f"
                % (self, lightness_similarity),
                level=app.DEBUG,
            )
            if lightness_similarity < self.threshold:
                return False
        return True

    def __repr__(self):
        return self.__str__()

    def __str__(self):
        return f"tmpl<{self.name}+{self.pos}>" if self.pos else f"tmpl<{self.name}>"

Input = Union[Text, Specification]
_DEBUG_TMPL = os.getenv("DEBUG_TMPL") or "debug.png"

def _match_one(
    img: Image, tmpl: Input
) -> Iterator[Tuple[Specification, Tuple[int, int]]]:
    rp = mathtools.ResizeProxy(TARGET_WIDTH)
    cv_img = _cv_image(
        imagetools.resize(
            img,
            width=rp.vector(
                img.width,
                app.device.width(),
            ),
        )
    )
    tmpl = Specification.from_input(tmpl)

    pos = tmpl.load_pos()
    pil_tmpl = load(tmpl.name)
    cv_tmpl = _cv_image(pil_tmpl)
    tmpl_h, tmpl_w = cv_tmpl.shape[:2]
    if pos:
        cv_pos = np.array(pos.convert("L"))
    else:
        cv_pos = np.full(cv_img.shape[:2], 255.0, dtype=np.uint8)
    res = cv2.matchTemplate(cv_img, cv_tmpl, cv2.TM_CCOEFF_NORMED)
    if tmpl.name == _DEBUG_TMPL:
        app.log.image(
            "match template",
            cv_img,
            layers={"tmpl": cv_tmpl, "match": res.astype(np.uint8)},
            level=app.DEBUG,
        )
    reverse_rp = mathtools.ResizeProxy(app.device.width())
    while True:
        mask = cv_pos[0 : res.shape[0], 0 : res.shape[1]]
        _, max_val, _, max_loc = cv2.minMaxLoc(res, mask=mask)
        x, y = max_loc
        client_pos = reverse_rp.vector2((x, y), TARGET_WIDTH)
        if max_val < tmpl.threshold or not tmpl.match(img, client_pos):
            app.log.text(
                "not match: tmpl=%s, pos=%s, similarity=%.3f"
                % (tmpl, max_loc, max_val),
                level=app.DEBUG,
            )
            break
        app.log.text(
            "match: tmpl=%s, pos=%s, similarity=%.2f" % (tmpl, max_loc, max_val)
        )
        yield (tmpl, client_pos)

        # mark position unavailable to avoid overlap
        cv_pos[max(0, y - tmpl_h) : y + tmpl_h, max(0, x - tmpl_w) : x + tmpl_w] = 0

def match(
    img: Image, *tmpl: Union[Text, Specification]
) -> Iterator[Tuple[Specification, Tuple[int, int]]]:
    match_count = 0
    for i in tmpl:
        for j in _match_one(img, i):
            match_count += 1
            yield j
    if match_count == 0:
        app.log.text(f"no match: tmpl={tmpl}")

# DEPRECATED
# spell-checker: disable
def _legacy_screenshot(*, max_age: float = 1) -> Image:
    import warnings

    warnings.warn("use `app.device.screenshot` instead", DeprecationWarning)
    return app.device.screenshot(max_age=max_age)

def _legacy_invalidate_screenshot():
    import warnings

    warnings.warn(
        "screenshot invalidation is handled by device service", DeprecationWarning
    )

globals()["LOGGER"] = logging.getLogger(__name__)
globals()["invalidate_screeshot"] = _legacy_invalidate_screenshot
globals()["invalidate_screenshot"] = _legacy_invalidate_screenshot
globals()["screenshot"] = _legacy_screenshot
g.screenshot_width = g._legacy_screenshot_width  # type: ignore


import json
from json.decoder import JSONDecodeError

from aiogram import Router, types
from aiogram.dispatcher.filters import Text, CommandObject
from aiogram.dispatcher.fsm.context import FSMContext
from aiogram.exceptions import TelegramBadRequest

from config import config, bot, cookies_path
from models import (
    User, BlackWord, ALL_BANNED, Parser, ParserFirstBanned, Alert
)
from data import texts
from data.states import NewBlackWord
from data.keyboards import get_log_markup, settings_markup, \
    get_sellers_black_markup, get_words_black_markup, tools_markup, confirm_markup, menu_markup
from log import encode_log_config, decode_log_config
from parsers.sahibinden import AlertMessage
from routers.filters.admin import MainAdminFilter

router = Router()

@router.message(Text(text_contains="настройки", text_ignore_case=True))
async def settings_handler(message: types.Message):
    await message.answer(
        text=texts.submenu,
        reply_markup=settings_markup
    )

@router.message(Text(text_contains="конфигуратор лога", text_ignore_case=True))
async def parser_logging_config(message: types.Message):
    user = User.get(user_id=message.from_user.id)
    await message.answer(
        text=texts.parse_config,
        reply_markup=get_log_markup(
            decode_log_config(user.log_config),
            config.bot_username
        )
    )

@router.callback_query(lambda cb: cb.data.split(":")[0] == "cfgtoggle")
async def config_toggle(query: types.CallbackQuery):
    user = User.get(user_id=query.from_user.id)
    field = query.data.split(":")[1]
    log = decode_log_config(user.log_config)
    setattr(log, field, not getattr(log, field))
    user.log_config = encode_log_config(log)
    user.save()
    await query.message.edit_text(
        text=texts.parse_config,
        reply_markup=get_log_markup(log, "config_bot")
    )

@router.message(Text(text_contains="черный список продавцов", text_ignore_case=True))
async def sellers_black_list(message: types.Message):
    user = User.get(user_id=message.from_user.id)
    count = 0
    for banned_model in ALL_BANNED:
        count += banned_model.select().where(banned_model.owner == user).count()

    await message.answer(
        text=texts.seller_black_list.format(
            sellers_black=count
        ),
        reply_markup=get_sellers_black_markup(user.other_banned)
    )

@router.callback_query(text="toggle_sellers_black")
async def toggle_sellers_black(query: types.CallbackQuery):
    user = User.get(user_id=query.from_user.id)
    user.other_banned = not user.other_banned
    user.save()
    await query.message.edit_text(
        text=query.message.text,
        reply_markup=get_sellers_black_markup(user.other_banned)
    )

@router.callback_query(text="delete_sellers_black")
async def delete_sellers_black(query: types.CallbackQuery):
    user = User.get(user_id=query.from_user.id)
    for banned_model in ALL_BANNED:
        banned_model.delete().where(banned_model.owner == user).execute()

    await query.answer("Успешно удалил")
    try:
        await query.message.edit_text(
            text=texts.seller_black_list.format(
                sellers_black=0
            ),
            reply_markup=get_sellers_black_markup(user.other_banned)
        )
    except TelegramBadRequest:
        pass

@router.message(Text(text_contains="черный список слов", text_ignore_case=True))
async def words_black_list(message: types.Message):
    user = User.get(user_id=message.from_user.id)
    user_words = BlackWord.select().where(BlackWord.owner == user)
    words = "\n".join(w.name for w in user_words)
    if not words:
        words = "У вас нет запрещенных слов"
    await message.answer(
        text=texts.word_black_list.format(
            words=words
        ),
        reply_markup=get_words_black_markup(user.black_words)
    )

@router.callback_query(text="toggle_words_black")
async def toggle_words_black(query: types.CallbackQuery):
    user = User.get(user_id=query.from_user.id)
    user.black_words = not user.black_words
    user.save()
    await query.message.edit_text(
        text=query.message.text,
        reply_markup=get_words_black_markup(user.black_words)
    )

@router.callback_query(text="delete_words_black")
async def delete_words_black(query: types.CallbackQuery):
    user = User.get(user_id=query.from_user.id)
    BlackWord.delete().where(BlackWord.owner == user).execute()
    words = "У вас нет запрещенных слов"
    await query.answer("Успешно удалил")
    try:
        await query.message.edit_text(
            text=texts.word_black_list.format(
                words=words
            ),
            reply_markup=get_words_black_markup(user.black_words)
        )
    except TelegramBadRequest:
        pass

@router.callback_query(text="add_black_word")
async def add_black_word_handler(query: types.CallbackQuery, state: FSMContext):
    await query.message.edit_text(
        text=texts.new_black_word
    )
    await state.set_state(NewBlackWord.main)

@router.message(NewBlackWord.main)
async def new_black_words(message: types.Message, state: FSMContext):
    user = User.get(user_id=message.from_user.id)
    for word in message.text.split():
        BlackWord.create(
            owner=user,
            name=word
        )
    user_words = BlackWord.select().where(BlackWord.owner == user)
    words = "\n".join(w.name for w in user_words)
    if not words:
        words = "У вас нет запрещенных слов"
    await message.answer(
        text=texts.word_black_list.format(
            words=words
        ),
        reply_markup=get_words_black_markup(user.black_words)
    )
    await state.clear()

@router.message(Text(text_contains="инструменты", text_ignore_case=True))
async def tools_handler(message: types.Message):
    await message.answer(
        text=texts.submenu,
        reply_markup=tools_markup
    )

@router.message(Text(text_contains="задать куки", text_ignore_case=True))
async def all_urls_handler(message: types.Message, state: FSMContext):
    user = User.get(user_id=message.from_user.id)
    await message.answer(
        'Отправь файл с куки'
    )
    await state.set_state('set_cookies')

@router.message(MainAdminFilter(), content_types=types.ContentType.DOCUMENT)
async def set_cookies_handler(message: types.Message, state: FSMContext):
    destination = r"./file.txt"
    await bot.download(
        message.document,
        destination=destination
    )
    with open(destination) as f:
        text = f.read()
    cookies = text.strip()
    try:
        cookies = json.loads(cookies)
    except JSONDecodeError:
        await message.answer('Неверный формат куки')
        return
    if (not isinstance(cookies, dict)) and (not isinstance(cookies, list)):
        print(type(cookies))
        await message.answer('Неверный формат куки')
        return
    with open(cookies_path, 'w', encoding='utf-8') as f:
        json.dump(cookies, f, ensure_ascii=False, indent=4)
    Alert.delete().where(Alert.message == AlertMessage.need_cookies).execute()
    await message.answer('Куки сохранены')

@router.message(commands={'del'})
async def delete_urls_handler(message: types.Message, command: CommandObject, state: FSMContext):
    args = command.args
    user = User.get(user_id=message.from_user.id)
    print(args)
    if not args:
        await message.answer(
            text=texts.invalid_pars_url_del,
            disable_web_page_preview=True
        )
        return
    parser = Parser.get_or_none(url=args, owner=user)
    if not parser:
        await message.answer(
            text='Такой ссылки не найдено'
        )
        return
    await message.answer(
        f'Удалить ссылку {parser.url}?',
        reply_markup=confirm_markup,
        disable_web_page_preview=True,
    )
    await state.update_data(del_url=parser.url)
    await state.set_state('confirm_del')

@router.callback_query(state='confirm_del', text='no')
async def no_confirm_del(query: types.CallbackQuery, state: FSMContext):
    await query.message.delete()
    await query.message.answer(
        f'Ссылка не будет удалена',
        reply_markup=tools_markup
    )
    await state.clear()

@router.callback_query(state='confirm_del', text='yes')
async def confirm_del(query: types.CallbackQuery, state: FSMContext):
    data = await state.get_data()
    del_url = data['del_url']
    user = User.get(user_id=query.from_user.id)
    Parser.delete().where(Parser.url == del_url, Parser.owner == user).execute()
    ParserFirstBanned.delete().where(ParserFirstBanned.search_url == del_url, ParserFirstBanned.owner == user).execute()
    await query.answer()
    await query.message.delete()
    await query.message.answer(
        f'Ссылка успешно удалена',
        reply_markup=tools_markup
    )
    await state.clear()

@router.message(Text(text_contains="удалить все ссылки", text_ignore_case=True))
async def delete_all_urls_handler(message: types.Message, state: FSMContext):
    await message.answer(
        'Удалить все ссылки из парсера?',
        reply_markup=confirm_markup
    )
    await state.set_state('confirm_del_all')

@router.callback_query(state='confirm_del_all', text='no')
async def no_confirm_del_all(query: types.CallbackQuery, state: FSMContext):
    await query.message.delete()
    await query.message.answer(
        f'Ссылки не будут удалены',
        reply_markup=tools_markup
    )
    await state.clear()

@router.callback_query(state='confirm_del_all', text='yes')
async def confirm_del_all(query: types.CallbackQuery, state: FSMContext):
    user = User.get(user_id=query.from_user.id)
    Parser.delete().where(Parser.owner == user).execute()
    ParserFirstBanned.delete().where(ParserFirstBanned.owner == user).execute()
    await query.answer()
    await query.message.delete()
    await query.message.answer(
        f'Все ссылки успешно удалены',
        # reply_markup=tools_markup
    )
    await state.clear()


## 作業目標

# * 請利用 API: https://www.dcard.tw/_api/forums/pet/posts?popular=true 回答下列問題：

# 1. 這個 API 一次會回傳幾筆資料？每一筆資料包含哪些欄位？

import requests
import json
r = requests.get('https://www.dcard.tw/_api/forums/pet/posts?popular=true')
response = json.loads(r.text)
len(response)
print(len(response))
# 30個

# 2. 取出每一筆資料的「標題」、「貼文時間」、「留言人數」、「按讚人數」
count_num = 0
likeCount_num = 0
for response_content in response:
    title = response_content['title']
    time = response_content['createdAt']
    count = response_content['commentCount']
    count_num+=count
    likeCount = response_content['likeCount']
    likeCount_num+=likeCount

# 3. 計算熱門/非熱門文章的「平均留言人數」與「平均按讚人數」
notPopular = requests.get('https://www.dcard.tw/_api/forums/pet/posts?popular=false')
notPopular_response = json.loads(notPopular.text)
notPopular_count_num = 0
notPopular_likeCount_num = 0
for notPopular_content in notPopular_response:
    count = notPopular_content['commentCount']
    notPopular_count_num += count
    likeCount = notPopular_content['likeCount']
    notPopular_likeCount_num += likeCount

not_popular_count_average = notPopular_count_num/len(notPopular_response)
print(not_popular_count_average)
not_popular_likeCount_average = notPopular_likeCount_num/len(notPopular_response)
print(not_popular_likeCount_average)
popular_count_average = count_num/len(response)
print(popular_count_average)
popular_likeCount_average = likeCount_num/len(response)
print(popular_likeCount_average)


"""empty message

Revision ID: e91416f1802f
Revises: f4ff2af7f44b
Create Date: 2016-10-22 22:34:58.777715

"""

# revision identifiers, used by Alembic.
revision = 'e91416f1802f'
down_revision = 'f4ff2af7f44b'

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('auth_roles',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('created_at', sa.DateTime(), nullable=True),
    sa.Column('modified_at', sa.DateTime(), nullable=True),
    sa.Column('name', sa.String(length=80), nullable=False),
    sa.Column('description', sa.String(length=255), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('name')
    )
    op.create_table('roles_users',
    sa.Column('user_id', sa.Integer(), nullable=True),
    sa.Column('role_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['role_id'], ['auth_roles.id'], ),
    sa.ForeignKeyConstraint(['user_id'], ['users.id'], )
    )
    op.add_column('users', sa.Column('active', sa.Boolean(), nullable=True))
    op.add_column('users', sa.Column('current_login_at', sa.DateTime(), nullable=True))
    op.add_column('users', sa.Column('email', sa.String(length=255), nullable=False))
    op.add_column('users', sa.Column('last_login_at', sa.DateTime(), nullable=True))
    op.add_column('users', sa.Column('login_count', sa.Integer(), nullable=True))
    op.alter_column('users', 'password',
               existing_type=sa.VARCHAR(),
               nullable=False)
    op.alter_column('users', 'username',
               existing_type=sa.VARCHAR(),
               nullable=False)
    op.create_unique_constraint(None, 'users', ['email'])
    op.create_unique_constraint(None, 'users', ['username'])
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'users', type_='unique')
    op.drop_constraint(None, 'users', type_='unique')
    op.alter_column('users', 'username',
               existing_type=sa.VARCHAR(),
               nullable=True)
    op.alter_column('users', 'password',
               existing_type=sa.VARCHAR(),
               nullable=True)
    op.drop_column('users', 'login_count')
    op.drop_column('users', 'last_login_at')
    op.drop_column('users', 'email')
    op.drop_column('users', 'current_login_at')
    op.drop_column('users', 'active')
    op.drop_table('roles_users')
    op.drop_table('auth_roles')
    ### end Alembic commands ###


import math
import numpy as np
import pandas as pd
from sklearn import metrics
from IPython.display import display
from pandas.api.types import is_string_dtype
from pandas.api.types import is_numeric_dtype
from sklearn.ensemble import RandomForestRegressor

def train_cats(df):
    for n, c in df.items():
        if is_string_dtype(c):
            df[n] = c.astype("category").cat.as_ordered()

def apply_cats(df, train):
    for n, c in df.items():
        if train[n].dtype == "category":
            df[n] = pd.Categorical(
                c, categories=tran[n].categories, ordered=True)

def numericalize(df, col, name):
    if not is_numeric_dtype(col):
        df[name] = col.cat.codes+1

def numericalize(df, col, name):
    if not is_numeric_dtype(col):
        df[name] = col.cat.codes+1

def fix_missing(df, col, name, nan_dict, is_train):
    if is_numeric_dtype(col):
        if pd.isnull(col).sum():
            df[name+"_na"] = pd.isnull(col)
            nan_dict[name] = col.median()
            df[name] = col.fillna(nan_dict[name])
    else:
        if is_numeric_dtype(col):
            if name in nan_dict:
                df[name+"_na"] = pd.isnull(col)
                df[name] = col.fillna(nan_dict[name])
            else:
                df[name] = col.fillna(df[name].median())

def proc_df(df, y_fld, nan_dict=None, is_train=True):
    df = df.copy()
    y = df[y_fld].values

    df.drop(y_fld, axis=1, inplace=True)

    if nan_dict is None:
        nan_dict = {}
    for n, c in df.items():
        fix_missing(df, c, n, nan_dict, is_train)
        numericalize(df, c, n)
    if is_train:
        return df, y, nan_dict
    return df, y

def split_train_val(df, n):
    return df[:n].copy(), df[n:].copy()

def rmse(x, y):
    return math.sqrt(((x-y)**2).mean())

def print_score(m):
    print(f"RMSE of valid set {rmse(m.predict(x_valid),y_valid)}")
    print(f"R^2 of valid set {m.score(x_valid,y_valid)}")
    print(f"RMSE of train set {rmse(m.predict(x_train),y_train)}")
    print(f"R^2 of train set {m.score(x_train,y_train)}")

if __name__ == '__main__':
    df = pd.read_csv("Train.csv", low_memory=False)
    print(df)
    print(df.isnull().sum())
    train_cats(df)
    print(df["image_id"])
    print(df["image_id"].cat.codes)
    numericalize(df, df["image_id"], "image_id")
    print(df["image_id"])
    df = df.sort_values(by="image_id")
    print(df)
    n_valid = 100
    n_train = len(df)-n_valid
    raw_train, raw_valid = split_train_val(df, n_train)
    x_train, y_train, nas = proc_df(raw_train, 'healthy')
    x_valid, y_valid = proc_df(
        raw_valid, 'healthy', nan_dict=nas, is_train=False)
    m = RandomForestRegressor(n_estimators=1000, n_jobs=-1)
    m.fit(x_train, y_train)
    m.score(x_train, y_train)
    print_score(m)
    print(x_train)
    print(y_train)
    print(x_valid)
    print(y_valid)
    print(df)
    print(df["healthy"].mean())
    print(df["healthy"]-df["healthy"].mean())
    print(math.sqrt(((df.healthy-df.healthy.mean()).pow(2).sum())/len(df)))
    filt = (df.rust > 0)
    df2 = df[filt]
    df3 = df[~filt]
    print(df2)
    print(math.sqrt(((df2.healthy-df2.healthy.mean()).pow(2).sum() +
          (df3.healthy-df3.healthy.mean()).pow(2).sum())/len(df)))
    filt2 = (df3.scab > 0)
    df4 = df3[filt2]
    df5 = df3[~filt2]
    print(df4)
    print(df5)
    print(math.sqrt(((df2.healthy-df2.healthy.mean()).pow(2).sum()+(df4.healthy -
          df4.healthy.mean()).pow(2).sum()+(df5.healthy-df5.healthy.mean()).pow(2).sum())/len(df)))


import math
import argparse

def get_nominal_interest(rate, periods):
    return rate / (periods * 100)

def get_number_payments(loan_principal, monthly_payment, loan_interest):
    nominal_interest = get_nominal_interest(loan_interest, 12)
    base = monthly_payment / (monthly_payment - nominal_interest * loan_principal)
    number = math.log(base, 1 + nominal_interest)
    # years = math.floor(math.ceil(number) / 12)
    # remaining_months = math.ceil(number) % 12
    months_left = math.ceil(number)
    return months_left

def print_period(months_left):
    years = math.floor(math.ceil(months_left) / 12)
    remaining_months = math.ceil(months_left) % 12
    if remaining_months == 0 and years > 1:
        return f'{years} years'
    elif remaining_months == 0 and years == 1:
        return f'{years} year'
    elif years > 1:
        if remaining_months > 1:
            return f'{years} years and {remaining_months} months'
        elif remaining_months == 1:
            return f'{years} years and {remaining_months} month'
    elif years == 1:
        if remaining_months > 1:
            return f'{years} year and {remaining_months} months'
        elif remaining_months == 1:
            return f'{years} year and {remaining_months} month'

def get_monthly_payment(loan_principal, number_of_periods, loan_interest):
    nominal_interest = get_nominal_interest(loan_interest, 12)
    # ((i*p)*(1+i)^n) / ((1+i)^n - 1)
    result = ((nominal_interest * loan_principal) * (1 + nominal_interest)**number_of_periods) / \
             ((1 + nominal_interest)**number_of_periods - 1)
    return math.ceil(result)

def get_loan_principal(annuity_amount, number_of_periods, loan_interest):
    nominal_interest = get_nominal_interest(loan_interest, 12)
    # P = A / ((i * (1+i)**n) / ((1+i)**n) - 1))
    result = annuity_amount / ((nominal_interest * (1 + nominal_interest)**number_of_periods) /
                               ((1 + nominal_interest)**number_of_periods - 1))
    return math.floor(result)

def get_overpayment(loan_principal, number_of_periods, payment):
    return abs(int(loan_principal - (number_of_periods * payment)))

def get_differentiated_payments(loan_principal, number_of_periods, loan_interest):
    nominal_interest = get_nominal_interest(loan_interest, 12)
    total_payment = 0
    for current_period in range(1, number_of_periods+1):
        current_payment = math.ceil((loan_principal / number_of_periods) + nominal_interest *
                                    (loan_principal - ((loan_principal * (current_period - 1)) / number_of_periods)))
        total_payment += current_payment
        print(f'Month {current_period}: payment is {current_payment}')
    overpayment = int(abs(loan_principal - total_payment))
    print(f'\nOverpayment = {overpayment}')

def loan_calculator():
    invalid_params_msg = 'Incorrect parameters'
    parser = argparse.ArgumentParser(description="This program calculates interest based on "
                                                 "type and presence of provided arguments.")
    parser.add_argument("-t", "--type", choices=['diff', 'annuity'],
                        help="Type must be one specified from list")
    parser.add_argument("--principal", default=None,
                        help="Used for calculations of both types of payment. You can get its value if you know"
                             " the interest, annuity payment, and number of months.")
    parser.add_argument("--periods", default=None,
                        help="Denotes the number of months needed to repay the loan. It's calculated based on the"
                             " interest, annuity payment, and principal.")
    parser.add_argument("--interest", default=None,
                        help="Specified without a percent sign. Note that it can accept a floating-point value. "
                             "The loan calculator can't calculate the interest, so it must always be provided.")
    parser.add_argument("--payment", default=None,
                        help="The monthly payment amount.")
    args = parser.parse_args()

    if args.type == 'annuity':
        if not args.interest:
            print(invalid_params_msg)
        if args.principal and args.payment and args.interest:
            loan_principal = float(args.principal)
            monthly_payment = int(args.payment)
            loan_interest = float(args.interest)
            if loan_principal < 0 or monthly_payment < 0 or loan_interest < 0:
                print(invalid_params_msg)
            else:
                number_of_periods = get_number_payments(loan_principal, monthly_payment, loan_interest)
                print(f'It will take {print_period(number_of_periods)} to repay this loan!')
                print(f'Overpayment = {get_overpayment(loan_principal, number_of_periods, monthly_payment)}')
        if args.principal and args.periods and args.interest:
            loan_principal = float(args.principal)
            number_of_periods = int(args.periods)
            loan_interest = float(args.interest)
            if loan_principal < 0 or number_of_periods < 0 or loan_interest < 0:
                print(invalid_params_msg)
            else:
                monthly_payment = get_monthly_payment(loan_principal, number_of_periods, loan_interest)
                print(f'Your annuity payment = {monthly_payment}!')
                print(f'Overpayment = {get_overpayment(loan_principal, number_of_periods, monthly_payment)}')
        if args.payment and args.periods and args.interest:
            annuity_amount = float(args.payment)
            number_of_periods = int(args.periods)
            loan_interest = float(args.interest)
            if annuity_amount < 0 or number_of_periods < 0 or loan_interest < 0:
                print(invalid_params_msg)
            else:
                loan_principal = get_loan_principal(annuity_amount, number_of_periods, loan_interest)
                print(f'Your loan principal = {loan_principal}!')
                print(f'Overpayment = {get_overpayment(loan_principal, number_of_periods, annuity_amount)}')
    elif args.type == 'diff':
        if args.payment:
            print(invalid_params_msg)
        elif args.principal and args.periods and args.interest:
            loan_principal = float(args.principal)
            number_of_periods = int(args.periods)
            loan_interest = float(args.interest)
            if loan_principal < 0 or number_of_periods < 0 or loan_interest < 0:
                print(invalid_params_msg)
            else:
                get_differentiated_payments(loan_principal, number_of_periods, loan_interest)
        else:
            print(invalid_params_msg)
    else:
        print(invalid_params_msg)

if __name__ == '__main__':
    loan_calculator()


from django.contrib import admin
from django.conf import settings
from django.conf.urls.static import static
from django.urls import path, include, re_path
from graphene_file_upload.django import FileUploadGraphQLView
from rest_framework.routers import SimpleRouter
# from drf_yasg.views import get_schema_view
# from drf_yasg import openapi

from drf_spectacular.views import SpectacularJSONAPIView, SpectacularAPIView, \
    SpectacularSwaggerView, SpectacularRedocView
from rest_framework_simplejwt.views import TokenObtainPairView, TokenRefreshView, TokenVerifyView

from accounts.views import SocialComplete, SocialDisconnect, ShareDocView, GetSharedDoc
from common.views import CalendarView, GetOpenGraphTagsView, StandardIconViewSet
from contacts.views import ContactViewSet, ContactSearchViewSet
from integration.urls import all_integrations_endpoints, google_auth_endpoints
from integration.views.googlesheet import GoogleSheetIntegrationViewSet
from projects.views import ProjectViewSet
from trash.views import TrashViewSet
from django.views.decorators.csrf import csrf_exempt
from graphene_django.views import GraphQLView
from poll.urls import poll_endpoints, endpoints_v2

router = SimpleRouter()
router.register(r'contacts', ContactViewSet)
router.register(r'projects', ProjectViewSet)
router.register(r'trash', TrashViewSet)
router.register(r'icons', StandardIconViewSet)
router.register(r'googlesheetintegration', GoogleSheetIntegrationViewSet, 'googlesheetintegration')

urlpatterns = [
    path('api/v1/', include([
        path('schema/', SpectacularAPIView.as_view(api_version='v1'), name='schema'),
        path('schema_json/', SpectacularJSONAPIView.as_view(api_version='v1'), name='schema_json'),
        path('swagger/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'),
        path('redoc/', SpectacularRedocView.as_view(url_name='schema'), name='redoc'),
        path('admin/', admin.site.urls),
        path('auth/', include('accounts.urls', namespace='accounts')),
        path('oauth/social-auth/', SocialComplete.as_view()),
        path('oauth/social-disconnect/', SocialDisconnect.as_view()),
        path('oauth/', include('social_django.urls', namespace='social')),
        path('calendar/<str:from>/<str:to>/', CalendarView.as_view()),
        path('poll/', include(poll_endpoints)),
        path('integrations/', include(all_integrations_endpoints)),
        path('googleauth/', include(google_auth_endpoints)),
        path('url_opengraph/', GetOpenGraphTagsView.as_view()),
        path('', include(router.urls)),
        path('', include('projects.urls')),
        path('', include('contacts.urls')),
        # path('', include('storyboards.urls')),
        # path('pay/', include('tinkoff_pay.urls')),
        path('', include('subscription.urls')),
        path('', include('document.urls')),
        path('', include('catalog_templates.urls')),
        path('', include('ai.urls')),
        path('', include('folders.urls')),
        path('document/share/', include([
            path('', ShareDocView.as_view({
                'post': 'create'}),
                 name='share_document'),
            path('<uuid:doc_uuid>/', ShareDocView.as_view({
                'delete': 'delete'}),
                 name='delete_share_document'),
            path('token/<uuid:doc_uuid>/',
                 GetSharedDoc.as_view({'get': 'retrieve'}),
                 name='get_shared_doc_token')])),
        path('contact/search/',
             ContactSearchViewSet.as_view({'post': 'search'}),
             name='search_for_contact'),
        path('graphql/', csrf_exempt(FileUploadGraphQLView.as_view(graphiql=True))),
    ])),
    path('api/v2/', include([
        path('schema/', SpectacularAPIView.as_view(api_version='v2'), name='schema', ),
        path('schema_json/', SpectacularJSONAPIView.as_view(), name='schema_json'),
        path('swagger/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'),
        path('redoc/', SpectacularRedocView.as_view(url_name='schema'), name='redoc'),
        path('token/', TokenObtainPairView.as_view(), name='token_obtain_pair'),
        path('token/refresh/', TokenRefreshView.as_view(), name='token_refresh'),
        path('token/verify/', TokenVerifyView.as_view(), name='token_verify'),
    ])),
]
# urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
# static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

# schema_view = get_schema_view(
#     openapi.Info(
#         title="Kinomanager API",
#         default_version='v1',
#         # description="Test description",
#         # terms_of_service="https://www.google.com/policies/terms/",
#         contact=openapi.Contact(email="dmz55@ya.ru"),
#         license=openapi.License(name="BSD License"),
#     ),
#     public=True,
#     permission_classes=[permissions.AllowAny],
# )

# urlpatterns += [
#     re_path(r'^api/v1/swagger(?P<format>\.json|\.yaml)$', schema_view.without_ui(cache_timeout=0),
#             name='schema-json'),
#     re_path(r'^api/v1/swagger/$', schema_view.with_ui('swagger', cache_timeout=0),
#             name='schema-swagger-ui'),
#     re_path(r'^api/v1/redoc/$', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'),
# ]


from pymongo import MongoClient
import time
from datetime import datetime

# connect
def connect():
    client = MongoClient('172.16.2.70', 27017)
    return client

def load_dkline(client):
    table = client.kline.history_klines_day_0
    start = time.time()

    now = datetime.now()
    etime = now.strftime("%Y%m%d")
    print("dkline end time:", etime)

    # read stocks
    with open('/data/quant/kline/stocks', 'r') as fd:
        stock = fd.readline()
        while stock:
            stock = stock[0:9]
            # creat data file
            df = '/data/quant/kline/.cache/dkline/' + stock
            with open(df, 'w') as dfd:
                print(df, etime, stock)
                ret = table.find({'date': {'$lte': etime}, 'stock_code': stock})
                cnt = ret.count()
                print(stock, ":", cnt)
                for i in range(cnt):
                    r = ret[i]
                    try:
                        d = str(r['open_px']) + ' ' + str(r['high_px']) + ' '+ str(r['low_px']) + ' ' + str(r['close_px']) + ' ' \
                                + str(r['preclose_px']) + ' ' + str(r['date']) + ' ' + str(r['business_amount']) + ' ' + str(r['business_balance']) + '\n'

                    except KeyError as e:
                        d = str(r['data'][0][1]) + ' ' + str(r['data'][0][2]) + ' ' + str(r['data'][0][3]) + ' ' + str(r['data'][0][4]) + ' ' \
                                + str(r['preclose_px']) + ' ' + str(r['data'][0][0]) + ' ' + str(r['data'][0][5]) + ' ' + str(r['data'][0][6]) + '\n'
                    except Exception as e:
                        print(r)
                        print("error:", e)
                        continue
                    dfd.write(d)
                #dfd.flush()
                dfd.close()

            stock = fd.readline()

    end = time.time()
    print("cost:", end-start)

if __name__ == '__main__':
    client = connect()
    load_dkline(client)


#!/usr/bin/python

import sys
import os
import json

from Xlib import X
from Xlib.display import Display

from wmctrl_window import WmctrlWindow

def main(argv):
  display = Display()
  root = display.screen().root
  root.change_attributes(event_mask=X.PropertyChangeMask | X.SubstructureNotifyMask)
  skip_windows = ['tk.TkN/Awindowswitcher', 'xfce4-appfinder.Xfce4-appfinderxavierApplicationFinder']

  NET_ACTIVE_WINDOW = display.intern_atom('_NET_ACTIVE_WINDOW')

  try:
    while True:
      event = display.next_event()
      if event.type != X.PropertyNotify:
        continue

      response = root.get_full_property(NET_ACTIVE_WINDOW, X.AnyPropertyType)
      win_id = hex(response.value[0]).rstrip('L').lstrip('0x')

      if len(win_id) == 0:
        continue

      wmctrl_out = os.popen('sleep 0.1; wmctrl -dliGux | grep -i {win_id}'.format(win_id=win_id)).read()
      wmctrl_out = wmctrl_out.split('\n')
      wmctrl_out.pop()

      if len(wmctrl_out) != 1:
        continue

      window = WmctrlWindow(wmctrl_out[0])
      # ignore window switcher
      if window.name in skip_windows:
        continue

      str_json = open("/home/danilo/scripts/flip360_wids.json", "r").read()
      jjson = json.loads(str_json)

      key_json = 'm{m}{w}'.format(m=window.monitor, w=window.workspace)
      if jjson.get(key_json, None) == None:
        continue

      window_previous = WmctrlWindow(jjson[key_json])
      if window.id == window_previous.id:
        continue

      jjson[key_json] = window.str_win
      flip360_wids = open("/home/danilo/scripts/flip360_wids.json", "w")
      flip360_wids.write(json.dumps(jjson))
      flip360_wids.close()

  finally:
    display.close()

if __name__ == '__main__':
  sys.exit(main(sys.argv))


import os

from PyQt5.QtWidgets import QMessageBox

from orangewidget import gui, widget
from orangewidget.settings import Setting
from oasys.widgets import gui as oasysgui, congruence

from syned.beamline.beamline import Beamline

class FileWriter(widget.OWWidget):
    name = "Syned File Writer"
    description = "Utility: Syned File Writer"
    icon = "icons/file_writer.png"
    maintainer = "Manuel Sanchez del Rio"
    maintainer_email = "srio(@at@)esrf.eu"
    priority = 2
    category = "Utility"
    keywords = ["data", "file", "load", "read"]

    want_main_area = 0

    syned_file_name = Setting("")
    is_automatic_run= Setting(1)

    inputs = [("SynedBeamline" , Beamline, "setBeamline" )]

    outputs = [{"name":"SynedBeamline",
                "type":Beamline,
                "doc":"Syned Beamline",
                "id":"data"}]

    beamline = None

    def __init__(self):
        super().__init__()

        self.runaction = widget.OWAction("Write Syned File", self)
        self.runaction.triggered.connect(self.write_file)
        self.addAction(self.runaction)

        self.setFixedWidth(590)
        self.setFixedHeight(180)

        left_box_1 = oasysgui.widgetBox(self.controlArea, "Syned File Selection", addSpace=True, orientation="vertical",
                                         width=570, height=100)

        gui.checkBox(left_box_1, self, 'is_automatic_run', 'Automatic Execution')

        gui.separator(left_box_1, height=10)

        figure_box = oasysgui.widgetBox(left_box_1, "", addSpace=True, orientation="horizontal", width=550, height=50)

        self.le_syned_file_name = oasysgui.lineEdit(figure_box, self, "syned_file_name", "Syned File Name",
                                                    labelWidth=120, valueType=str, orientation="horizontal")
        self.le_syned_file_name.setFixedWidth(330)

        gui.button(figure_box, self, "...", callback=self.selectFile)

        gui.separator(left_box_1, height=10)

        button = gui.button(self.controlArea, self, "Write Syned File", callback=self.write_file)
        button.setFixedHeight(45)

        gui.rubber(self.controlArea)

    def selectFile(self):
        self.le_syned_file_name.setText(oasysgui.selectFileFromDialog(self, self.syned_file_name, "Open Syned File"))

    def setBeamline(self, data):
        if not data is None:
            self.beamline = data

            if self.is_automatic_run:
                self.write_file()

    def write_file(self):
        self.setStatusMessage("")

        try:
            if not self.beamline is None:
                congruence.checkDir(self.syned_file_name)

                self.beamline.to_json(self.syned_file_name)

                path, file_name = os.path.split(self.syned_file_name)

                self.setStatusMessage("File Out: " + file_name)

                self.send("SynedBeamline", self.beamline)
            else:
                QMessageBox.critical(self, "Error",
                                     "Syned Data not present",
                                     QMessageBox.Ok)
        except Exception as exception:
            QMessageBox.critical(self, "Error", str(exception), QMessageBox.Ok)

if __name__ == "__main__":
    from PyQt5.QtWidgets import QApplication
    import sys

    a = QApplication(sys.argv)
    ow = FileWriter()

    ow.show()
    a.exec_()


import sys
import os
import configparser
import xml.etree.ElementTree as ET
from PyQt6 import QtWidgets, QtCore, QtGui
from PyQt6.QtWidgets import QMainWindow, QWidget, QLabel, QLineEdit, QTextEdit, QPushButton, QVBoxLayout, QHBoxLayout, QTreeWidget, QTreeWidgetItem, QSizePolicy, QDialog, QScrollArea, QCheckBox, QMessageBox, QFrame, QStatusBar, QApplication
from PyQt6.QtGui import QGuiApplication, QClipboard
from message_parser import MessageParser
from typing import Tuple

basedir = os.path.dirname(__file__)

try:
	# Only applicable for Windows application
    from ctypes import windll
    myappid = 'rkcoding.fixv.1'
    windll.shell32.SetCurrentProcessExplicitAppUserModelID(myappid)
except ImportError:
    pass

def get_clipboard(clipboard: QClipboard) -> str:
	mime_data = clipboard.mimeData()
	if mime_data.hasText():
		return mime_data.text()
	return ''

def set_clipboard(clipboard: QClipboard, source: str):
	clipboard.setText(source)

class MessageEditor(QDialog):
	apply_signal = QtCore.pyqtSignal()

	def __init__(self, parent = None) -> None:
		super().__init__(parent)
		self.init_ui()

	def init_ui(self):
		self.setWindowTitle('Message Editor')
		self.setModal(True)

		central_vbox = QVBoxLayout()
		toolbar_hbox = QHBoxLayout()
		toolbar_hbox.setSpacing(5)
		copy_button = QPushButton('Copy')
		paste_button = QPushButton('Paste')
		clear_button = QPushButton('Clear')
		msg_text_edit = QTextEdit()
		msg_text_edit.setWordWrapMode(QtGui.QTextOption.WrapMode.WrapAnywhere)

		apply_button = QPushButton('Apply')
		apply_button.setDefault(True)
		cancel_button = QPushButton('Cancel')
		button_box = QHBoxLayout()

		self.setLayout(central_vbox)
		central_vbox.addLayout(toolbar_hbox)
		toolbar_hbox.addWidget(copy_button)
		toolbar_hbox.addWidget(paste_button)
		toolbar_hbox.addWidget(clear_button)
		central_vbox.addWidget(msg_text_edit)
		central_vbox.addLayout(button_box)
		button_box.addWidget(apply_button)
		button_box.addStretch()
		button_box.addWidget(cancel_button)

		self.msg_text_edit = msg_text_edit
		self.msg_text_edit.installEventFilter(self)
		self.clipboard = QGuiApplication.clipboard()

		copy_button.clicked.connect(lambda: set_clipboard(self.clipboard, msg_text_edit.toPlainText()))
		paste_button.clicked.connect(lambda: msg_text_edit.setText(get_clipboard(self.clipboard)))
		clear_button.clicked.connect(msg_text_edit.clear)
		cancel_button.clicked.connect(lambda: self.close())
		apply_button.clicked.connect(self.apply_signal)

	def eventFilter(self, widget, event):
		if event.type() == QtCore.QEvent.Type.KeyPress and widget is self.msg_text_edit:
			if event.key() == QtCore.Qt.Key.Key_Return and self.msg_text_edit.hasFocus():
				self.apply_signal.emit()
		return super().eventFilter(widget, event)

class AppWindow(QMainWindow):
	def __init__(self, app: QtWidgets.QApplication) -> None:
		super().__init__()

		# Setting up app config and initialize message parser
		app_config_file_name = 'app_config.ini'
		self.init_config(os.path.join(basedir, app_config_file_name))
		self.msg_parser = self.build_msg_parser()

		self.app = app
		self.message_editor = MessageEditor()
		self.init_ui()
		self.init_logic()
		self.show()

	def init_config(self, config_path: str):
		app_config = configparser.ConfigParser()
		if len(app_config.read(os.path.join(basedir, config_path))) == 0:
			QMessageBox.warning(self, "Error",
				f"""
Config file not found. Please ensure that config file exists.

Application will now exit.
""",
				QMessageBox.StandardButton.Ok)
			sys.exit()
		if 'CONFIG' not in app_config.sections():
			QMessageBox.warning(self, "Error",
				f"""
Error reading in config file. Please ensure that config file is in correct format:

[CONFIG]
Option1: ...
Option2: ...

Application will now exit.
""",
				QMessageBox.StandardButton.Ok)
			sys.exit()
		config = app_config['CONFIG']

		transport_data_dictionary = config['TransportDataDictionary'] if 'TransportDataDictionary' in config else ''
		app_data_dictionary = config['AppDataDictionary'] if 'AppDataDictionary' in config else ''
		if transport_data_dictionary and app_data_dictionary:
			self.data_dict_path = os.path.join(basedir, transport_data_dictionary)
			self.app_dict_path = os.path.join(basedir, app_data_dictionary)
		else:
			data_dictionary = config['DataDictionary'] if 'DataDictionary' in config else ''
			self.data_dict_path = os.path.join(basedir, data_dictionary) if data_dictionary else ''
			self.app_dict_path = None
		# Setting config with default values
		self.always_on_top = False
		self.expand_tree_on_launch = False
		self.error_on_statusbar = False
		self.is_right_align = False
		self.start_pos = (0, 0)
		self.start_size = (500, 600)
		try:
			if 'AlwaysOnTop' in config:
				self.always_on_top = config.getboolean('AlwaysOnTop')
			if 'ExpandOnLaunch' in config:
				self.expand_tree_on_launch = config.getboolean('ExpandOnLaunch')
			if 'ErrorOnStatusbar' in config:
				self.error_on_statusbar = config.getboolean('ErrorOnStatusbar')
			if 'IsRightAlign' in config:
				self.is_right_align = config.getboolean('IsRightAlign')
			start_pos_x = config.getint('StartPosX') if 'StartPosX' in config else self.start_pos[0]
			start_pos_y = config.getint('StartPosY') if 'StartPosY' in config else self.start_pos[1]
			self.start_pos = (start_pos_x, start_pos_y)
			start_width = config.getint('StartWidth') if 'StartWidth' in config else self.start_size[0]
			start_height = config.getint('StartHeight') if 'StartHeight' in config else self.start_size[1]
			self.start_size = (start_width, start_height)
		except ValueError:
			print("Encountered value error in setting config")
		# always_on_top = config['AlwaysOnTop'] if 'AlwaysOnTop' in config else None
		# self.always_on_top = True if always_on_top and always_on_top == 'Yes' else False
		# expand_on_launch = config['ExpandOnLaunch'] if 'ExpandOnLaunch' in config else None
		# self.expand_on_launch = True if expand_on_launch and expand_on_launch == 'Yes' else False
		# error_on_statusbar = config['ErrorOnStatusbar'] if 'ErrorOnStatusbar' in config else None
		# self.error_on_statusbar = True if error_on_statusbar and error_on_statusbar == 'Yes' else False

		# Set other initial configs for the app
		self.is_compact = False
		self.is_autocompact = False

	def build_msg_parser(self):
		try:
			if self.data_dict_path or self.app_dict_path:
				return MessageParser(self.data_dict_path, self.app_dict_path)
		except Exception as e:
			print(e)
			QMessageBox.warning(self, "Error",
				f"""
Could not parse quickfix dictionary file at specified path. Please check the dictionary path in config file.

Application will now exit.
""",
				QMessageBox.StandardButton.Ok)
			sys.exit()

		QMessageBox.warning(self, "Error",
			f"""
Quickfix dictionary path incorrectly set. Please check the config file.

Application will now exit.
""",
			QMessageBox.StandardButton.Ok)
		sys.exit()

	def init_ui(self):
		self.setObjectName('MainWindow')
		self.setWindowTitle('FIXV')
		self.resize(self.start_size[0], self.start_size[1])
		windowPosX = QApplication.primaryScreen().size().width() - self.start_size[0] - self.start_pos[0] if self.is_right_align else self.start_pos[0]
		self.move(windowPosX, self.start_pos[1])
		self.setup_actions()
		self.setup_menu()

		central_widget = QWidget(self)
		self.setCentralWidget(central_widget)

		central_vbox = QVBoxLayout()

		compact_container = QWidget()
		compact_vbox = QVBoxLayout()
		compact_vbox.setContentsMargins(5, 5, 5, 5)
		compact_vbox.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
		expand_arrow = QLabel()
		expand_arrow.setPixmap(QtGui.QPixmap(os.path.join(basedir, "assets", "DownArrow.png")))

		viewer_container = QWidget()
		viewer_vbox = QVBoxLayout()
		viewer_vbox.setContentsMargins(0, 0, 0, 0)
		viewer_vbox.setSpacing(10)

		toolbar_hbox = QHBoxLayout()
		toolbar_hbox.setContentsMargins(0, 0, 0, 0)
		toolbar_hbox.setSpacing(20)
		compact_button = QPushButton('Compact')

		msg_entry_vbox = QVBoxLayout()
		msg_entry_vbox.setSpacing(0)
		title_hbox = QHBoxLayout()
		title_hbox.setSpacing(5)
		title_hbox.setContentsMargins(0, 0, 0, 10)
		msg_title_label = QLabel('Fix Message')
		msg_title_label.setContentsMargins(5, 0, 5, 0)
		msg_title_label.setSizePolicy(QSizePolicy.Policy.Preferred, QSizePolicy.Policy.Fixed)
		autopaste_checkbox = QCheckBox('AutoPaste  ')
		seperator_line = QFrame()
		seperator_line.setFrameShape(QFrame.Shape.VLine)
		copy_button = QPushButton('Copy')
		paste_button = QPushButton('Paste')
		clear_button = QPushButton('Clear')
		msg_line_hbox = QHBoxLayout()
		msg_line = QTextEdit()
		msg_line.setReadOnly(True)
		text_height = msg_line.document().documentLayout().documentSize().height()
		msg_line.setFixedHeight(int(text_height) + 20)
		msg_line.setLineWrapMode(QTextEdit.LineWrapMode.NoWrap)
		msg_delim_hbox = QHBoxLayout()
		msg_delim_hbox.setContentsMargins(15, 0, 0, 0)
		msg_delim_hbox.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop)
		msg_delim_label = QLabel('Delimiter:')
		msg_delim_label.setSizePolicy(QSizePolicy.Policy.Maximum, QSizePolicy.Policy.Preferred)
		msg_delim_edit = QLineEdit('|')
		msg_delim_edit.setTextMargins(5, 5, 5, 5)
		msg_delim_edit.setMaxLength(1)
		msg_delim_edit.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
		msg_delim_edit.setFixedWidth(25)

		actions_hbox = QHBoxLayout()
		parse_button = QPushButton('Re-Parse')

		output_tree = QTreeWidget()
		output_tree.setColumnCount(3)
		output_tree.setHeaderLabels(['Tag', 'Name', 'Value'])

		CENTRAL_VBOX = True
		COMPACT_VBOX = True
		VIEWER_VBOX = True
		TOOLBAR_HBOX = True
		MSG_ENTRY_VBOX = True
		TITLE_HBOX = True
		MSG_LINE_HBOX = True
		MSG_DELIM_HBOX = True
		ACTIONS_HBOX = True

		central_widget.setLayout(central_vbox)
		if CENTRAL_VBOX:
			central_vbox.addWidget(compact_container)
			compact_container.setLayout(compact_vbox)

			if COMPACT_VBOX:
				compact_vbox.addWidget(expand_arrow)

			central_vbox.addWidget(viewer_container)
			viewer_container.setLayout(viewer_vbox)

			if VIEWER_VBOX:
				viewer_vbox.addLayout(toolbar_hbox)
				viewer_vbox.addLayout(msg_entry_vbox)
				viewer_vbox.addLayout(actions_hbox)

				if TOOLBAR_HBOX:
					toolbar_hbox.addWidget(compact_button)

				if MSG_ENTRY_VBOX:
					msg_entry_vbox.addLayout(title_hbox)
					msg_entry_vbox.addLayout(msg_line_hbox)

					if TITLE_HBOX:
						title_hbox.addWidget(msg_title_label)
						title_hbox.addStretch()
						title_hbox.addWidget(autopaste_checkbox)
						title_hbox.addWidget(seperator_line)
						title_hbox.addWidget(copy_button)
						title_hbox.addWidget(paste_button)
						title_hbox.addWidget(clear_button)

					if MSG_LINE_HBOX:
						msg_line_hbox.addWidget(msg_line)
						msg_line_hbox.addLayout(msg_delim_hbox)

						if MSG_DELIM_HBOX:
							msg_delim_hbox.addWidget(msg_delim_label)
							msg_delim_hbox.addWidget(msg_delim_edit)

				if ACTIONS_HBOX:
					actions_hbox.addWidget(parse_button)
					actions_hbox.addStretch()

				viewer_vbox.addWidget(output_tree)

		## Export variable to self
		self.central_vbox = central_vbox
		self.central_widget = central_widget
		self.compact_container = compact_container
		self.viewer_container = viewer_container
		self.compact_button = compact_button
		self.autopaste_checkbox = autopaste_checkbox
		self.msg_title_label = msg_title_label
		self.copy_button = copy_button
		self.paste_button = paste_button
		self.clear_button = clear_button
		self.parse_button = parse_button
		self.msg_line = msg_line
		self.msg_delim_edit = msg_delim_edit
		self.output_tree = output_tree
		self.clipboard = QGuiApplication.clipboard()
		self.statusBar = QStatusBar()

	def setup_actions(self):
		self.auto_compact_act = QtGui.QAction('&Auto Compact')
		self.auto_compact_act.setCheckable(True)
		self.auto_compact_act.setShortcut('Ctrl+O')
		self.auto_compact_act.toggled.connect(self.toggle_autocompact)
		self.edit_message_act = QtGui.QAction('&Edit Message')
		self.edit_message_act.setShortcut('Ctrl+E')
		self.edit_message_act.triggered.connect(self.show_message_editor)
		self.always_top_act = QtGui.QAction('&Always On Top')
		self.always_top_act.setShortcut('Ctrl+T')
		self.always_top_act.setCheckable(True)
		self.always_top_act.toggled.connect(self.toggle_stays_on_top)
		if self.always_on_top:
			self.always_top_act.toggle()
		self.show_err_msg_act = QtGui.QAction('&Show Message')
		self.show_err_msg_act.setCheckable(True)
		self.show_err_status_act = QtGui.QAction('&Show in Status Bar')
		self.show_err_status_act.setCheckable(True)
		self.err_act_grp = QtGui.QActionGroup(self)
		self.err_act_grp.addAction(self.show_err_msg_act)
		self.err_act_grp.addAction(self.show_err_status_act)
		self.err_act_grp.triggered.connect(self.change_error_notification)
		if self.error_on_statusbar:
			self.show_err_status_act.setChecked(True)
			self.change_error_notification(self.show_err_status_act)
		else:
			self.show_err_msg_act.setChecked(True)
			self.change_error_notification(self.show_err_msg_act)

	def setup_menu(self):
		edit_menu = self.menuBar().addMenu('Edit')
		edit_menu.addAction(self.edit_message_act)
		view_menu = self.menuBar().addMenu('View')
		view_menu.addAction(self.auto_compact_act)
		view_menu.addAction(self.always_top_act)
		error_menu = view_menu.addMenu('Error')
		error_menu.addAction(self.show_err_msg_act)
		error_menu.addAction(self.show_err_status_act)

	def init_logic(self):
		self.message_editor.apply_signal.connect(self.apply_message_editor)
		self.compact_container.mousePressEvent = lambda _: self.toggle_compact(False)
		self.compact_button.clicked.connect(lambda: self.toggle_compact(True))
		self.autopaste_checkbox.toggled.connect(self.toggle_autopaste)
		self.copy_button.clicked.connect(lambda: set_clipboard(self.clipboard, self.msg_line.toPlainText()))
		self.paste_button.clicked.connect(self.paste_and_decode)
		self.clear_button.clicked.connect(self.msg_line.clear)
		self.parse_button.clicked.connect(self.decode_and_show_msg)
		self.msg_line.mousePressEvent = self.show_message_editor
		self.compact_container.setVisible(False)
		self.prev_size = self.size()
		self.prev_compact_size = None
		self.toggle_compact_sc = QtGui.QShortcut(QtGui.QKeySequence('Ctrl+M'), self)
		self.toggle_compact_sc.activated.connect(lambda: self.toggle_compact(not self.is_compact))
		self.toggle_autopaste_sc = QtGui.QShortcut(QtGui.QKeySequence('Ctrl+Shift+P'), self)
		self.toggle_autopaste_sc.activated.connect(lambda: self.autopaste_checkbox.toggle())
		self.paste_decode_sc = QtGui.QShortcut(QtGui.QKeySequence('Ctrl+P'), self)
		self.paste_decode_sc.activated.connect(self.paste_and_decode)
		self.reparse_sc = QtGui.QShortcut(QtGui.QKeySequence('Ctrl+R'), self)
		self.reparse_sc.activated.connect(self.decode_and_show_msg)

		self.setStatusBar(self.statusBar)

	def toggle_compact(self, is_compact: bool):
		if self.message_editor.isVisible():
			return
		right_edge_x = self.pos().x() + self.size().width()
		if is_compact:
			self.prev_size = self.size()
			self.viewer_container.setVisible(False)
			self.compact_container.setVisible(True)
			self.menuBar().hide()
			self.statusBar.hide()
			self.central_vbox.setContentsMargins(0, 0, 0, 0)
		else:
			self.prev_compact_size = self.size()
			self.viewer_container.setVisible(True)
			self.compact_container.setVisible(False)
			self.menuBar().show()
			self.statusBar.show()
			self.central_vbox.setContentsMargins(10, 10, 10, 10)
		self.adjustSize()
		self.resize(self.prev_size if not is_compact else (self.prev_compact_size if self.prev_compact_size else QtCore.QSize(self.size().width(), 12)))
		self.is_compact = is_compact
		if self.is_right_align:
			self.move(right_edge_x - self.size().width(), self.pos().y())

	def toggle_stays_on_top(self, checked):
		self.setWindowFlag(QtCore.Qt.WindowType.WindowStaysOnTopHint, checked)
		self.message_editor.setWindowFlag(QtCore.Qt.WindowType.WindowStaysOnTopHint, checked)
		self.show()

	def change_error_notification(self, qaction: QtGui.QAction):
		self.is_show_err_msg = False
		self.is_show_err_status = False
		if qaction == self.show_err_msg_act:
			self.is_show_err_msg = True
		elif qaction == self.show_err_status_act:
			self.is_show_err_status = True

	def autopaste_decode(self):
		if self.message_editor.isVisible():
			return
		self.paste_and_decode()

	def toggle_autopaste(self, checked: bool):
		if checked:
			self.clipboard.changed.connect(self.autopaste_decode)
		else:
			self.clipboard.changed.disconnect(self.autopaste_decode)

	def changeEvent(self, e: QtCore.QEvent) -> None:
		if e.type() == QtCore.QEvent.Type.ActivationChange:
			if self.isActiveWindow():
				if self.is_autocompact and self.is_compact:
					self.toggle_compact(False)
			else:
				if self.is_autocompact and not self.is_compact:
					self.toggle_compact(True)
		return super().changeEvent(e)

	def toggle_autocompact(self, checked: bool):
		self.is_autocompact = checked

	def show_message_editor(self, *arg):
		if self.is_compact:
			return
		self.message_editor.msg_text_edit.setText(self.msg_line.toPlainText())
		self.message_editor.move(self.pos())
		self.message_editor.show()

	def apply_message_editor(self):
		self.msg_line.setText(self.message_editor.msg_text_edit.toPlainText())
		self.message_editor.close()
		self.decode_and_show_msg()

	def show_status_bar_msg(self, msg: str):
		self.statusBar.clearMessage()
		self.statusBar.showMessage(msg)
		self.statusBar.setToolTip(msg)

	def paste_and_decode(self):
		self.msg_line.setText(get_clipboard(self.clipboard).replace('\n', '').replace('\r', ''))
		self.decode_and_show_msg()

	def decode_and_show_msg(self):
		if not self.msg_line.toPlainText():
			self.show_status_bar_msg('Nothing to parse')
			return
		try:
			self.output_tree.clear()
			msg_tree = self.msg_parser.parse_msg(self.msg_line.toPlainText(), self.msg_delim_edit.text() if self.msg_delim_edit.text() else '|')
			self.build_output_tree(msg_tree)
			self.show_status_bar_msg('Successfully parsed')
		except Exception as error:
			error_msg = f'[{type(error).__name__}Error] {error}'
			if self.is_show_err_msg:
				QMessageBox.warning(self, "Error", f"""
Message Parsing Error

{error_msg}
""", QMessageBox.StandardButton.Ok)
			elif self.is_show_err_status:
				self.show_status_bar_msg(error_msg)

	def build_output_tree(self, msg_tree: ET.Element):
		for section in msg_tree.iterfind('./'):
			section_item = QTreeWidgetItem([section.tag])
			self.output_tree.addTopLevelItem(section_item)
			section_item.setExpanded(self.expand_tree_on_launch)
			self.build_tree_item(section, section_item)
		# Only resize the first and second column
		for i in range(2):
			self.output_tree.resizeColumnToContents(i)

	def build_tree_item(self, msg_elem: ET.Element, tree_parent: QTreeWidgetItem):
		for elem in msg_elem.iterfind('./'):
			field_tag = elem.get('number') if 'number' in elem.attrib else elem.tag
			if elem.tag == 'field':
				field_name = elem.get('name') if 'name' in elem.attrib else 'UNDEFINED'
			else:
				field_name = elem.get('name') if 'name' in elem.attrib else None
			field_raw = elem.get('raw') if 'raw' in elem.attrib else None
			field_enum = elem.get('enum') if 'enum' in elem.attrib else None
			field_value = None
			if field_raw:
				if field_enum:
					field_value = f'{field_raw} ({field_enum})'
				else:
					field_value = field_raw
			item_entries = [field_tag]
			if field_name:
				item_entries.append(field_name)
			if field_value:
				if len(item_entries) < 2:
					item_entries.append('')
				item_entries.append(field_value)
			if len(item_entries) > 3:
				print(f'[WARNING] Row for {field_tag} contains more than 3 entries')
			tree_item = QTreeWidgetItem(item_entries)
			if len(item_entries) == 3:
				tree_item.setToolTip(2, field_value)
			tree_parent.addChild(tree_item)
			self.build_tree_item(elem, tree_item)
			tree_item.setExpanded(self.expand_tree_on_launch)

if __name__ == "__main__":

	app = QtWidgets.QApplication(sys.argv)
	app.setApplicationName('FIX Viewer')
	app.setWindowIcon(QtGui.QIcon(os.path.join(basedir, "assets", "appicon.ico")))
	app_window = AppWindow(app)

	# msg_string = '8=FIX.4.4|9=224|35=D|34=1080|49=TESTBUY1|52=20180920-18:14:19.508|56=TESTSELL1|11=636730640278898634|15=USD|21=2|38=7000|40=1|54=1|55=MSFT|60=20180920-18:14:19.492|453=2|448=111|447=6|802=1|523=test1|448=222|447=8|802=2|523=test2|523=test3|528=10|94959=1234|10=225|'
	# app_window.msg_line.setText(msg_string)

	sys.exit(app.exec())


from __future__ import annotations

import pytest

from marimo._ast.app import App
from marimo._ast.cell import cell_factory, parse_cell

class TestParseCell:
    @staticmethod
    def test_parse_simple() -> None:
        code = "x = 0\nz = y"
        cell = parse_cell(code)
        assert cell.key == hash(code)
        assert cell.code == code
        assert cell.defs == set(["x", "z"])
        assert cell.refs == set(["y"])

    @staticmethod
    def test_local_variables() -> None:
        code = "_, y = f(x)\ndef _foo():\n  _bar = 0\nimport _secret_module"
        cell = parse_cell(code)
        assert cell.defs == {"y"}
        assert cell.refs == {"f", "x"}

    @staticmethod
    def test_dunder_dunder_excluded() -> None:
        code = "__ = 10; __name__ = 20"
        cell = parse_cell(code)
        assert cell.defs == {"__", "__name__"}
        assert cell.refs == set()

    @staticmethod
    def test_local_class() -> None:
        code = "class _A: pass"
        cell = parse_cell(code)
        assert cell.defs == set()
        assert cell.refs == set()

    @staticmethod
    def test_alias_underscored_name() -> None:
        code = "import _m as m"
        cell = parse_cell(code)
        assert cell.defs == {"m"}
        assert cell.refs == set()

    @staticmethod
    def test_ref_local_var() -> None:
        code = "x = _y"
        cell = parse_cell(code)
        assert cell.defs == {"x"}
        assert cell.refs == set()

        code = "def f(x= _y): pass"
        cell = parse_cell(code)
        assert cell.defs == {"f"}
        assert cell.refs == set()

class TestCellFactory:
    @staticmethod
    def test_missing_return() -> None:
        def f() -> None:
            x = 10  # noqa: F841

        with pytest.raises(ValueError) as e:
            cell_factory(f)

        assert "missing a return statement" in str(e.value)

    @staticmethod
    def test_not_tuple() -> None:
        def f() -> int:
            x = 10  # noqa: F841
            return x

        with pytest.raises(ValueError) as e:
            cell_factory(f)  # type: ignore[type-var]

        assert "must return a tuple" in str(e.value)

    @staticmethod
    def test_missing_some_defs() -> None:
        def f() -> tuple[int]:
            x = 10
            y = x  # noqa: F841
            return (x,)

        with pytest.raises(ValueError) as e:
            cell_factory(f)

        assert "must return a tuple of all its defs" in str(e.value)

    @staticmethod
    def test_missing_some_refs() -> None:
        z = 0

        app = App()

        @app.cell
        def f(y: int) -> tuple[int]:
            x = y + z
            return (x,)

        with pytest.raises(ValueError) as e:
            app._validate_args()

        assert "must take all its refs as args" in str(e.value)

    @staticmethod
    def test_extra_returns() -> None:
        def f() -> tuple[int]:
            return (1,)

        with pytest.raises(ValueError) as e:
            cell_factory(f)

        assert "shouldn't return anything" in str(e.value)

    @staticmethod
    def return_local_variable() -> None:
        def f() -> tuple[int]:
            _x = 0
            return (_x,)

        with pytest.raises(ValueError) as e:
            cell_factory(f)

        assert "Names starting with underscores should not be returned" in str(
            e.value
        )

    @staticmethod
    def local_variable_as_arg() -> None:
        def f(_x: int) -> None:
            return

        with pytest.raises(ValueError) as e:
            cell_factory(f)

        assert (
            "Names starting with underscores should not be taken as "
            "parameters" in str(e.value)
        )


import re
import ast
from collections import defaultdict
from textwrap import TextWrapper
from html import escape
from intervaltree import IntervalTree as Intervals
from ipymarkup.show import show_html
from ipymarkup.record import Record
from ipymarkup.palette import Color, MaterialRgb, material, Palette
from ipymarkup.span import order_spans, span_text_sections, format_span_box_markup, format_span_line_markup, wrap_multilines, prepare_spans, get_multilines
from ipymarkup.dep import format_dep_markup
import matplotlib.pyplot as plt

from xml.etree import ElementTree

from dash import html, dcc
import dash_mantine_components as dmc

AMBER = Color(
    'amber',
    background=material('Amber', '50'),
    border=material('Amber', '300'),
    text=material('Amber', '400'),
    line=material('Amber', '400')
)

BLUE = Color(
    'blue',
    background=material('Blue', '50'),
    border=material('Blue', '300'),
    text=material('Blue', '400'),
    line=material('Blue', '400')
)
GREEN = Color(
    'green',
    background=material('Green', '50'),
    border=material('Green', '300'),
    text=material('Green', '400'),
    line=material('Green', '400')
)
RED = Color(
    'red',
    background=material('Red', '50'),
    border=material('Red', '300'),
    text=material('Red', '400'),
    line=material('Red', '400')
)

PINK = Color(
    'pink',
    background=material('Pink', '50'),
    border=material('Pink', '300'),
    text=material('Pink', '400'),
    line=material('Pink', '400')
)
ORANGE = Color(
    'orange',
    background=material('Orange', '50'),
    border=material('Orange', '300'),
    text=material('Orange', '400'),
    line=material('Orange', '400')
)
PURPLE = Color(
    'purple',
    background=material('Deep Purple', '50'),
    border=material('Deep Purple', '300'),
    text=material('Deep Purple', '400'),
    line=material('Deep Purple', '400')
)
BROWN = Color(
    'brown',
    background=material('Brown', '50'),
    border=material('Brown', '300'),
    text=material('Brown', '400'),
    line=material('Brown', '400')
)
GREY = Color(
    'grey',
    text=material('Grey', '500'),
    line=material('Grey', '400')
)

CYAN = Color(
    'cyan',
    background=material('Cyan', '50'),
    border=material('Cyan', '300'),
    text=material('Cyan', '400'),
    line=material('Cyan', '400')
)

LIME = Color(
    'lime',
    background=material('Lime', '50'),
    border=material('Lime', '300'),
    text=material('Lime', '400'),
    line=material('Lime', '400')
)

TEAL = Color(
    'teal',
    background=material('Teal', '50'),
    border=material('Teal', '300'),
    text=material('Teal', '400'),
    line=material('Teal', '400')
)

Indigo = Color(
    'Indigo',
    background=material('Indigo', '50'),
    border=material('Indigo', '300'),
    text=material('Indigo', '400'),
    line=material('Indigo', '400')
)

colors_ent = [ORANGE, RED, TEAL, Indigo ,BROWN, GREEN,  LIME]

color_fact = [GREEN]

color_temporal = [BLUE]

color_risk_att = [GREEN, RED, GREY]

ents = ['LabValues', 'DiagLab', 'Measure', 'Condition', 'Medication', 'Process', ',HealthState', 'TimeInfo', 'Other']
temporal = ['past', 'past_present', 'present', 'future']
factuality = ['negative', 'speculated', 'minor', 'unlikely', 'possible_future']
risk_att = ['increase_risk_factor', 'increase_symptom', 'decrease_risk_factor', 'decrease_symptom']
rels = ['has_TimeInfo', 'has_Measure', 'has_State']

def retrieve_annos(file):
    annotate_lines = [l.strip().split('\t') for l in open(file)]
    spans_ent = {l[0]: l[1].split() for l in annotate_lines if l[0][0] == 'T' }
    spans_att = {l[0]: l[1].split()  for l in annotate_lines if l[0][0] == 'A'}
    spans_rl = {l[0]:l[1].split() for l in annotate_lines  if l[0][0] == 'R'}
    return spans_ent, spans_att, spans_rl

def retrieve_ents(spans_ent, spans_att, to_show=[], att='Factuality'):
    # box + line show
    # att is either 'Factuality' or 'Risk'
    if len(to_show) == 0:
        to_show = ['LabValues', 'DiagLab', 'Measure', 'Condition', 'Medication', 'Process', ',HealthState', 'TimeInfo', 'Other']

    spans_to_show = [(int(v[1]), int(v[2]), k) for k,v in spans_ent.items() if v[0] in to_show]
    ent_dict = {k:v[0] for k, v in spans_ent.items()}
    att_dict = {v[1]:v[2] for k, v in spans_att.items() if v[0] == att}
    return spans_to_show, ent_dict, att_dict

def retrieve_conl(spans_ent):
    # line show
    conclusions = [(int(v[1]), int(v[2]), 'Conclusion') for k,v in spans_ent.items() if v[0] == 'Conclusion' ]
    return conclusions

import numpy as np
def find_most_closest(idx, indices):
    min_d = np.array([abs(d-idx) for d in indices])
    min_i = indices[np.argmin(min_d)]
    return min_i

def resolve_rel_idx(spans_rl, spans_ent, idx_w):
    rl_tuples = []
    for k, v in spans_rl.items():
        left = v[1].split(':')[1]
        s_left = int(spans_ent[left][1])
        if s_left not in idx_w:
            s_left = find_most_closest(s_left, list(idx_w.keys()))
        w_left = idx_w[s_left][0]
        right = v[2].split(':')[1]
        s_right = int(spans_ent[right][1])
        if s_right not in idx_w:
            s_right = find_most_closest(s_right, list(idx_w.keys()))
        w_right = idx_w[s_right][0]
        if w_left == w_right:
            if left > right:
                w_right += 1
            else:
                w_right -=1
        rl_tuples.append((w_left, w_right, v[0]))
    return rl_tuples

def retrieve_temp(spans_ent, spans_att, to_show=[], att='Temporal_Element'):
    # box + line show
    spans_to_show = [(int(v[1]), int(v[2]), k) for k,v in spans_ent.items() if v[0] == 'Temporal']
    ent_dict = {k:v[0] for k, v in spans_ent.items()}
    att_dict = {v[1]:v[2] for k, v in spans_att.items() if v[0] == att}
    return spans_to_show, ent_dict, att_dict

import numpy as np
def find_most_closest(idx, indices):
    min_d = np.array([abs(d-idx) for d in indices])
    min_i = indices[np.argmin(min_d)]
    return min_i

def resolve_rel_idx(spans_rl, spans_ent, idx_w):
    rl_tuples = []
    for k, v in spans_rl.items():
        left = v[1].split(':')[1]
        s_left = int(spans_ent[left][1])
        if s_left not in idx_w:
            s_left = find_most_closest(s_left, list(idx_w.keys()))
        w_left = idx_w[s_left][0]
        right = v[2].split(':')[1]
        s_right = int(spans_ent[right][1])
        if s_right not in idx_w:
            s_right = find_most_closest(s_right, list(idx_w.keys()))
        w_right = idx_w[s_right][0]
        if w_left == w_right:
            if left > right:
                w_right += 1
            else:
                w_right -=1
        rl_tuples.append((w_left, w_right, v[0]))
    return rl_tuples

def format_span_box_line_markup(text, spans, entities=None, attributes=None, palette_ent=None, palette_att = None,
                            width=200, line_gap=16, line_width=3,
                            label_size=14, background='white'):
    spans = order_spans(prepare_spans(spans))
    multilines = list(get_multilines(spans))

    level_width = line_gap + line_width
    yield (
        '
'
    )
    for offset, line, multilines in wrap_multilines(text, multilines, width):
        yield '
'  # line block
        for text, multi in span_text_sections(line, multilines):
            text = escape(text)
            if not multi:
                yield (
                    ''
                )
                yield text
                yield ''
                continue

            level = max(_.level for _ in multi.lines)
            margin = (level + 1) * level_width
            yield (
                ''.format(
                    margin=margin
                )
            )

            for line in multi.lines:
                padding = line_gap + line.level * level_width
                color_ent = palette_ent.get(entities[line.type])
                if attributes is not None and line.type in attributes:
                    color_att = palette_att.get(attributes[line.type])

                    yield (
                        ''.format(
                            background=color_ent.background.value,
                            border=color_ent.border.value,
                            line_width=line_width,
                            padding=padding,
                            color=color_att.line.value

                        )
                    )
                else:
                    #level = max(_.level for _ in multi.lines)
                    margin = (level + 1) * level_width
                    yield (
                        ''.format(
                            margin=margin,
                            background=color_ent.background.value,
                            border=color_ent.border.value

                        )
                    )
                yield text
                yield ''

            for line in multi.lines:
                if not line.type or offset + multi.start != line.start:
                    continue

                bottom = -line.level * level_width - line_gap
                color_ent = palette_ent.get(entities[line.type])

                yield (
                ''.format(
                    color=color_ent.text.value
                    )
                )
                yield entities[line.type]
                yield ''
                if line.type in attributes:
                    yield (
                        ''.format(
                            label_size=label_size,
                            background=background,
                            bottom=bottom
                        )
                    )
                    yield attributes[line.type]
                else:
                    margin = (level + 1) * level_width
                    yield (
                        ''.format(
                            margin=margin

                        )
                    )

                yield ''

            yield ''  # close relative
        yield '
'  # close line
    yield '
'

#box_line_lines = format_span_box_line_markup(text=text_inj, spans=spans, entities=ent_dict, attributes=att_dict, palette_ent=PALETTE_ent, palette_att=PALETTE_att)
def get_w_idx(text_split):
    idx_w = {0:0}
    #text_list = text_inj[0].split()
    past = 0
    for i, w in enumerate(text_split):

        if i> 0:
            past += len(text_split[i-1])+1
            idx_w[past] = (i, w)
        else:
            idx_w[0] = (0, w)
    return idx_w

def convert_html_to_dash(html_code,  dash_modules = [html, dcc]):
    """Convert standard html (as string) to Dash components.

    Looks into the list of dash_modules to find the right component (default to [html, dcc, dbc])."""

    def find_component(name):
        for module in dash_modules:
            try:
                return getattr(module, name)
            except AttributeError:
                pass
        raise AttributeError(f"Could not find a dash widget for '{name}'")

    def parse_css(css):
        """Convert a style in ccs format to dictionary accepted by Dash"""
        return {k: v for style in css.strip(";").split(";") for k, v in [style.split(":")]}

    def parse_value(v):
        try:
            return ast.literal_eval(v)
        except (SyntaxError, ValueError):
            return v

    parsers = {"style": parse_css, "id": lambda x: x}

    def _convert(elem):
        comp = find_component(elem.tag.capitalize())
        children = [_convert(child) for child in elem]
        if not children:
            children = elem.text
        attribs = elem.attrib.copy()
        if "class" in attribs:
            attribs["className"] = attribs.pop("class")
        attribs = {k: parsers.get(k, parse_value)(v) for k, v in attribs.items()}

        return comp(children=children, **attribs)

    et = ElementTree.fromstring(html_code)

    return _convert(et)

# Word relevance heatmap
def rescale_score_by_abs(score, max_score, min_score):
 """
 Normalize the relevance value (=score), accordingly to the extremal relevance values (max_score and min_score),
 for visualization with a diverging colormap.
 i.e. rescale positive relevance to the range [0.5, 1.0], and negative relevance to the range [0.0, 0.5],
 using the highest absolute relevance for linear interpolation.
 """

 # CASE 1: positive AND negative scores occur --------------------
 if max_score > 0 and min_score < 0:

  if max_score >= abs(min_score):  # deepest color is positive
   if score >= 0:
    return 0.5 + 0.5 * (score / max_score)
   else:
    return 0.5 - 0.5 * (abs(score) / max_score)

  else:  # deepest color is negative
   if score >= 0:
    return 0.5 + 0.5 * (score / abs(min_score))
   else:
    return 0.5 - 0.5 * (score / min_score)

    # CASE 2: ONLY positive scores occur -----------------------------
 elif max_score > 0 and min_score >= 0:
  if max_score == min_score:
   return 1.0
  else:
   return 0.5 + 0.5 * (score / max_score)

 # CASE 3: ONLY negative scores occur -----------------------------
 elif max_score <= 0 and min_score < 0:
  if max_score == min_score:
   return 0.0
  else:
   return 0.5 - 0.5 * (score / min_score)

def getRGB(c_tuple):
 return "#%02x%02x%02x" % (int(c_tuple[0] * 255), int(c_tuple[1] * 255), int(c_tuple[2] * 255))

def dash_span_word(word, score, colormap):
    if score == None:
        #score = 0.5
        score = 0.0
    word = word.strip('\n')
    style = {'margin-left': '1px', 'margin-right': '2px', 'background-color': getRGB(colormap(score))}
    span = html.Span([word], style = style)
    return span

def html_span_word(word, score, colormap):
    if score == None:
        #score = 0.5
        score = 0.0
    word = word.strip('\n')
    style = {'margin-left': '1px', 'margin-right': '2px', 'background-color': getRGB(colormap(score))}
    span = html.Span([word], style = style)
    return span

def html_span_word_check(word, score, colormap):
    if score == None:
        #score = 0.5
        score = 0.0
    word = word.strip('\n')
    style = {'margin-left': '1px', 'margin-right': '2px', 'background-color': getRGB(colormap(score))}
    span = html.Span([word], style = style)
    return span

def dash_html_prediction_score(words, scores,cmap_name="bwr"):
 """
 Return word-level heatmap in HTML format,
 with words being the list of words (as string),
 scores the corresponding list of word-level relevance values,
 and cmap_name the name of the matplotlib diverging colormap.
 Refer to https://matplotlib.org/stable/tutorials/colors/colormaps.html for cmap_name
 'Greens' for prediction, 'bwr' for relevance
 """

 colormap = plt.get_cmap(cmap_name)

 #try:
 # assert len(words) < len(scores)
 #except:
 # print(len(words), len(scores))
 try:
  max_s = max(scores)
 except:
  print("No max score")
  max_s = 0.5
  scores = [0.5] * len(words)

 try:
  min_s = min(scores)
 except:
  print("No min score")
  min_s = 0.5
  scores = [0.5] * len(words)

 output_text_div_children = []
 scaled_scores = []
 for idx, w in enumerate(words):
      #w = w.replace('<','(').replace('>',')')
      #if max_s > 0.5:
        #score = rescale_score_by_abs(scores[idx], max_s, min_s)
      #elif max_s < 0.1:
      #    score = 0.5+scores[idx]

      if scores[idx] >= 0.5:
          # range between 0.6 and 0.9
          score = 0.5 + scores[idx]/2
      elif  0.1 <= scores[idx] < 0.5:
          score = 0.5 + scores[idx]/3
      else:
          score = 0.5 + scores[idx]
      #scaled_scores.append(score)
      #output_text = output_text + table_row(span_word(w, score, colormap))
      w_span = dash_span_word(w, score, colormap)
      output_text_div_children.append(w_span)

 return output_text_div_children

def dash_html_prediction_score_annotate(words, scores,cmap_name="bwr"):
 """
 Return word-level heatmap in HTML format,
 with words being the list of words (as string),
 scores the corresponding list of word-level relevance values,
 and cmap_name the name of the matplotlib diverging colormap.
 Refer to https://matplotlib.org/stable/tutorials/colors/colormaps.html for cmap_name
 'Greens' for prediction, 'bwr' for relevance
 """

 colormap = plt.get_cmap(cmap_name)

 #try:
 # assert len(words) < len(scores)
 #except:
 # print(len(words), len(scores))
 try:
  max_s = max(scores)
 except:
  print("No max score")
  max_s = 0.5
  scores = [0.5] * len(words)

 try:
  min_s = min(scores)
 except:
  print("No min score")
  min_s = 0.5
  scores = [0.5] * len(words)

 output_text_div_children = []
 scaled_scores = []
 for idx, w in enumerate(words):
      #w = w.replace('<','(').replace('>',')')
      #if max_s > 0.5:
        #score = rescale_score_by_abs(scores[idx], max_s, min_s)
      #elif max_s < 0.1:
      #    score = 0.5+scores[idx]

      if scores[idx] >= 0.5:
          # range between 0.6 and 0.9
          score = 0.5 + scores[idx]/2
      elif  0.1 <= scores[idx] < 0.5:
          score = 0.5 + scores[idx]/3
      else:
          score = 0.5 + scores[idx]
      #scaled_scores.append(score)
      #output_text = output_text + table_row(span_word(w, score, colormap))
      w_span = dash_span_word(w, score, colormap)
      output_text_div_children.append(w_span)

 return output_text_div_children


import os
import numpy as np
import astropy.io.ascii as ascii
import matplotlib.pylab as plt
from numpy import random as random

import code

def sigma_correct(veldisp,width,index):
    """
        Correct Lick index measurements for observed
        velocity dispersion if larger than Lick resolution
    """

    sigcorr_slope_file = '/import/opus1/nscott/Stellar_Populations/misc/sigcorr_slope.dat'
    sigcorr_icpt_file = '/import/opus1/nscott/Stellar_Populations/misc/sigcorr_icpt.dat'

    tab1 = ascii.read(sigcorr_slope_file)
    tab2 = ascii.read(sigcorr_icpt_file)

    slope_arr = tab1[index]
    icpt_arr = tab2[index]

    sigma = np.arange(17)*25.
    if (np.ceil(veldisp) % 25) == 0:
        width = width * slope_arr[np.int(np.ceil(veldisp)//25)] + icpt_arr[np.int(np.ceil(veldisp)//25)]
    else:
        try:
            high = width * slope_arr[np.int(np.ceil(veldisp)//25+1)] + icpt_arr[np.int(np.ceil(veldisp)//25+1)]
            low = width * slope_arr[np.int(np.ceil(veldisp)//25)] + icpt_arr[np.int(np.ceil(veldisp)//25)]
            width = ((25. - (veldisp - sigma[np.int(np.ceil(veldisp)//25)]))*low +
                 (25. - (sigma[np.int(np.ceil(veldisp)//25+1)] - veldisp))*high)/25.
        except:
            code.interact(local=dict(globals(),**locals()))
    return width

def measure_ind(rest_wavelength,spectrum,index,plot=False,obswav=''):

    # Measure the requested Lick index

    index_definition_file = os.path.expanduser('/import/opus1/nscott/Stellar_Populations/misc/pseudocontinuum_absorption_line_indices.dat')
    #names = ['Index','Index_Lower','Index_Upper','BC_Lower','BC_Upper','RC_Lower','RC_Upper','Source']
    index_definitions = ascii.read(index_definition_file)#,names=names)

    #Check requested index exists in the table

    index_names = index_definitions['Index']
    aa = np.where(index == index_names)[0]
    if len(aa) == 0:
        raise Exception('Invalid index entered')

    #Need to check for D4000 or R94 defined indices

    ps_blue_inds = np.where((rest_wavelength > float(index_definitions[aa]['BC_lower'].data[0])) &
                             (rest_wavelength < float(index_definitions[aa]['BC_upper'].data[0])))[0]
    ps_red_inds = np.where((rest_wavelength > float(index_definitions[aa]['RC_lower'].data[0])) &
                           (rest_wavelength < float(index_definitions[aa]['RC_upper'].data[0])))[0]
    ps_blue_centre = np.mean(rest_wavelength[ps_blue_inds])
    ps_red_centre = np.mean(rest_wavelength[ps_red_inds])
    ps_blue = np.median(spectrum[ps_blue_inds])
    ps_red = np.median(spectrum[ps_red_inds])

    ps_gradient = (ps_red - ps_blue)/(ps_red_centre - ps_blue_centre)
    ps_zeropoint = ps_red - ps_gradient*ps_red_centre

    index_inds = np.where((rest_wavelength > float(index_definitions[aa]['I_lower'].data[0])) &
                          (rest_wavelength < float(index_definitions[aa]['I_upper'].data[0])))[0]
    index_centre = np.mean(rest_wavelength[index_inds])

    ps_flux = 0.0
    obs_flux = 0.0
    for ind in index_inds:
        ps_flux += ps_zeropoint + ps_gradient*rest_wavelength[ind]
        obs_flux += spectrum[ind]

    missing_flux = ps_flux - obs_flux
    ps_ind_flux = ps_zeropoint + ps_gradient*index_centre
    equivalent_width = missing_flux/ps_ind_flux

    # Handle bad columns - just return NaN where present

    if plot:
        plt.figure(4)
        plt.clf()
        plt.plot(rest_wavelength,spectrum)
        plt.xlabel('Wavelength (Angstroms)')
        plt.ylabel('Flux')
        plt.title(index)
        plt.plot([ps_blue_centre,ps_red_centre],[ps_blue,ps_red],'ks-',markersize=10,linewidth=2.5)
        plt.plot([rest_wavelength[ps_blue_inds[0]],rest_wavelength[ps_blue_inds[-1]]],[ps_blue*1.05,ps_blue*1.05],'k--',linewidth=2.0)
        plt.text(ps_blue_centre,ps_blue*1.1,'Blue Pseudo-\ncontinuum Band',ha='center')
        plt.plot([rest_wavelength[ps_red_inds[0]],rest_wavelength[ps_red_inds[-1]]],[ps_red*1.05,ps_red*1.05],'k--',linewidth=2.0)
        plt.text(ps_red_centre,ps_red*1.1,'Red Pseudo-\ncontinuum Band',ha='center')
        plt.plot([rest_wavelength[index_inds[0]],rest_wavelength[index_inds[-1]]],[ps_ind_flux*1.1,ps_ind_flux*1.1],'k--',linewidth=2.0)
        plt.text(index_centre,ps_ind_flux*1.15,'Index Band',ha='center')
        plt.plot([rest_wavelength[index_inds[0]],rest_wavelength[index_inds[0]]],[0.0,spectrum[index_inds[0]]],'k-')
        plt.plot([rest_wavelength[index_inds[-1]],rest_wavelength[index_inds[-1]]],[0.0,spectrum[index_inds[-1]]],'k-')
        ps_line = ps_zeropoint + ps_gradient*rest_wavelength[index_inds]
        plt.fill_between(rest_wavelength[index_inds],ps_line,spectrum[index_inds],facecolor='green',alpha=0.5)
        plt.axis([min(rest_wavelength[ps_blue_inds])-10,max(rest_wavelength[ps_red_inds])+10,
          min(spectrum[index_inds])*0.9,max([ps_blue,ps_red])*1.1])
        plt.draw()
        plt.show(block=False)

    if np.abs(np.mean(obswav[index_inds]) - 5577.0) < 15.:
        equivalent_width = np.nan

    if index_definitions['Molecular'][aa] == 1:
        equivalent_width = -2.5*np.log10(1 - equivalent_width/(index_definitions[aa]['I_upper'].data-index_definitions[aa]['I_lower'].data))
        equivalent_width = equivalent_width[0]

    return equivalent_width

def mc_ind_errors(rest_wavelength,spectrum,variance,index,n=100,obswav=''):

    # Determine uncertainty on a Lick index measurement by randomly drawing
    # noise from the variance spectrum and re-measuring the index.

    ind_vals = np.zeros((n))

    spectrum[np.isfinite(spectrum) == False] = np.nanmedian(spectrum)
    variance[np.isfinite(variance) == False] = np.nanmedian(variance)

    for i in range(n):
        spec_new = np.copy(spectrum) + random.randn(len(spectrum))*np.sqrt(variance)
        ind_vals[i] = measure_ind(rest_wavelength,spec_new,index,obswav=obswav)

    mean_ind = np.mean(ind_vals)
    err_ind = np.std(ind_vals)

    return mean_ind,err_ind


# -*- coding: utf-8 -*-

import re
from nltk.corpus import stopwords
from langdetect import detect, DetectorFactory
from wordsegment import load, segment
load()

def pre_process(text, spell_correction_model, word_segment):
    text = clean(text, spell_correction_model, word_segment)
    return text

def word_segmentation(word):
    return segment(word)

def clean(text, spell_correction_model, word_segment):
    lang = ''
    try:
        DetectorFactory.seed = 0
        lang = str(detect(text))
    except:
        pass
    if lang != 'en':
        print(lang)
        return None
    text = re.sub(r'''(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s
                    ()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’]))''', " ", text)

    # text = re.sub(r'^https?:\/\/.*[\r\n]*', '', text, flags=re.MULTILINE)
    letter_only = re.sub('[^a-zA-Z]', ' ', text)

    print('Before Spell Correction', letter_only)
    spell_corrected_text = ''
    for word in letter_only.split():
        word = spell_correction_model.correction(word)
        # corrected_text_array = word_segment.segment(word)
        corrected_text_array = word_segmentation(word)
        arr = []
        if len(corrected_text_array)>1:
            for each in corrected_text_array:
                arr.append(spell_correction_model.correction(each))
        else:
            arr = corrected_text_array

        spell_corrected_text = spell_corrected_text + ' ' + ' '.join(arr)

    print('After Spell Correction', spell_corrected_text)
    words = spell_corrected_text.lower().split()
    stopwords_eng = set(stopwords.words("english"))
    useful_words = [x for x in words if x not in stopwords_eng]

    # Combine words into a paragraph again
    useful_words_string = ' '.join(useful_words)
    return useful_words_string