import os
from aiogram import types
from aiogram.dispatcher import FSMContext
from docgen.creation.new_image import docMaking
from docgen.validate import validate
from states import MyStatesGroup
from keyboards import *

# докген начало
async def imgStart(msg: types.Message, state: FSMContext):
    example = '`Иван Иванов 24-08-1998 М UA +`'
    await msg.answer(text=f'Пришлите данные в формате: Имя Фамилия дд\-мм\-гггг пол\(м/ж\) страна\(UA/KZ/USA\) делать ли метаданные\(\+/\-\)\nПример: {example}',
                    parse_mode='MarkdownV2', reply_markup=setImgStartKb()
                     )
    await MyStatesGroup.imgReady.set()

# докген генерация и выдача
async def imgReady(msg: types.Message, state: FSMContext):
    imgData = await state.get_data()

    # проверяем есть ли уже данные, отличаются ли
    # новые данные от старых и была ли ошибка
    try:
        if msg.text == imgData['imgData']:
            pass
        else:
            if not (msg.text == 'Еще'):
                await state.update_data(imgData=msg.text)
    except:
        await state.update_data(imgData=msg.text)
    imgData = await state.get_data()

    # генерация изображения c ответом и обработка ошибок
    validatedData = validate(imgData['imgData'].split())
    if validatedData:
        waiting = await msg.answer(text='\u231B_Изображение генерируется\.\.\._', parse_mode='MarkdownV2', reply_markup=setImgReadyKb())
        doc = docMaking(validatedData)
        photo = types.InputFile(doc)
        await msg.answer_document(photo)
        await waiting.delete()
        await msg.answer(text='\u2705Готово!\nДля изображения с этими же данными, нажмите кнопку «Еще», или пришлите другие данные.',
                         reply_markup=setImgReadyKb())
        os.remove(doc)
        await MyStatesGroup.imgReady.set()
    else:
        await state.reset_state()
        await MyStatesGroup.imgReady.set()
        important = ('*соблюдайте пробелы и формат данных*')
        example = '`Иван Иванов 24-08-1998 М UA +`'
        await msg.answer(text=f'\u274C Данные введены некорректно, {important} как в примере:\n{example}',
                         parse_mode='MarkdownV2', reply_markup=setImgReadyKb())


import numpy as np
import torch
from matplotlib import pyplot as plt
from torchvision.io.image import read_image

import my_transforms
from Metrics import Metrics
from evaluation import Evaluation
from modelcodes.models.detection import fasterrcnn_resnet50_fpn_v2, FasterRCNN_ResNet50_FPN_V2_Weights
from torchvision.utils import draw_bounding_boxes
from torchvision.transforms.functional import to_pil_image
from tqdm import tqdm
from my_dataset import COCOdataset
import json

from visualization import visualization

device = "cuda"
coco_root = './data/coco2017'
batchsize = 1

def Inference():
    def transxywh(bbx):
        bbx[2] = bbx[2] - bbx[0]
        bbx[3] = bbx[3] - bbx[1]
        return bbx

    weights = FasterRCNN_ResNet50_FPN_V2_Weights.DEFAULT
    model = fasterrcnn_resnet50_fpn_v2(weights=weights, box_score_thresh=0.7)

    data_transform = weights.transforms()

    val_dataset = COCOdataset(coco_root, "val", data_transform)  #

    val_dataset_loader = torch.utils.data.DataLoader(val_dataset,
                                                     batch_size=batchsize,
                                                     shuffle=False,
                                                     pin_memory=True,
                                                     num_workers=0,
                                                     collate_fn=val_dataset.collate_fn)
    model.to(device)
    model.eval()
    results = []
    with torch.no_grad():
        for image, targets in tqdm(val_dataset_loader, desc="validation..."):
            image = list(img.to(device) for img in image)

            outputs = model(image)

            for i, prediction in enumerate(outputs):
                cat_ids = prediction["labels"].cpu()
                bboxs = prediction["boxes"].cpu().numpy().tolist()
                scores = prediction['scores'].cpu()
                dis_scores = prediction['dis_scores'].cpu().numpy().tolist()
                full_score = prediction['full_scores'].cpu().numpy().tolist()
                for j in range(prediction["labels"].shape[0]):
                    content_dic = {
                        "image_id": int(targets[i]["image_id"].numpy()[0]),
                        "category_id": int(cat_ids[j]),
                        "bbox": transxywh(bboxs[j]),
                        "score": float(scores[j]),
                        "full_score": full_score[j],
                        "dis_score": dis_scores[j]
                    }
                    results.append(content_dic)

        json_str = json.dumps(results, indent=4)
        with open('./data/FRCNN_COCO070.json', 'w') as json_file:
            json_file.write(json_str)

def DeepView(datapath):
    with open(datapath) as f1:
        pre_list = json.load(f1)
    eva = Evaluation(datatype='COCO')
    eva2 = Evaluation(datatype='COCO')
    fault_list, fault_type_num = eva.get_faulttype(pre_list)
    _, _, tp_list = eva2.get_score_iou(pre_list)
    result = {}
    metrics = Metrics(pre_list=pre_list, dif_list=None, imgpre_list=None)
    result['DeepView'] = metrics.deepview('COCO')
    metric_names = [_[0] for _ in result.items()]
    vis = visualization()
    colorlist = ['xkcd:red', 'xkcd:peach', 'xkcd:green', 'xkcd:light purple', 'xkcd:black', 'xkcd:grey']

    deepview_instance = None
    X = []
    Tro_diversity = []
    for i in range(len(result)):
        metric = result[metric_names[i]]
        zipped = zip(metric, fault_list, tp_list, pre_list)
        sort_zipped = sorted(zipped, key=lambda x: (x[0], x[1]))
        srt_ = zip(*sort_zipped)
        srtd_metric_list, srtd_fault_list, srtd_tp_list, srtd_pre_list = [list(x) for x in srt_]
        deepview_instance = srtd_pre_list[::-1]
        # rq3 = RQ3engine(slice=[0.1])
        # rq3.map2image(srtd_pre_list[::-1], metric_names[i], modeltype)
        X, Y, Tro_diversity = vis.plt_fault_type_rauc(srtd_fault_list[::-1], fault_type_num)
        plt.plot(X, Y, label=metric_names[i], color=colorlist[i])
        x_list, Tro_effective, y_list, rauc_1, rauc_2, rauc_3, rauc_5, rauc_all = eva.RAUC_cls(
            metric_tplist=srtd_tp_list[::-1])
        print(metric_names[i] + ' diversity: RAUC =' + ' ' + str(round(sum(Y) / sum(Tro_diversity) * 100, 2)))
        # plt.plot(x_list, y_list, label=metric_names[i])
        print(metric_names[i] + ' effectiveness: RAUC-n = ' + str(
            [round(rauc_1 * 100, 2), round(rauc_2 * 100, 2), round(rauc_3 * 100, 2), round(rauc_5 * 100, 2),
             round(rauc_all * 100, 2)]))

    plt.plot(X, Tro_diversity, label='Theoretical', color='tab:blue')

    plt.xlabel('$Number\ of\ prioritized\ test\ instances$')
    plt.ylabel('$Number\ of\ error\ type\ detected$')

    plt.legend()

    plt.show()

    return deepview_instance

if __name__ == "__main__":
    Inference()  # Model Inference

    deepview_result = DeepView(datapath='./data/FRCNN_COCO070.json')

    json_str = json.dumps(deepview_result, indent=4)
    with open('./data/deepview_result', 'w') as json_file:
        json_file.write(json_str)


import gym
from PIL import Image
import numpy as np
from gym.envs.box2d.car_dynamics import Car
from gym.envs.box2d import CarRacing

dst = '/media/ray/SSD/workspace/python/dataset/save_here'

def generate_action():
    action = [0, 0, 0]
    action[0] = np.random.normal(0, np.random.random())
    acc_chance = 0.9
    if np.random.random() < acc_chance:
        action[1] = np.random.random()
        action[2] = 0
    else:
        action[1] = 0
        action[2] = np.random.random()
    return action

MAX_GAME_TIME = 1000
MAX_RUNS = 20
REST_NUM = 250
# actions = []
def multiple_runs(on):
    env = CarRacing()
    frame_and_action = []
    for run in range(MAX_RUNS):
        env.reset()
        # done = False
        counter = 0
        for game_time in range(MAX_GAME_TIME):
            # env.render()
            action = generate_action()
            state, r, done, _ = env.step(action)
            frame_and_action.append({'state': state,
                           'action': action})
            # print(r)
            counter += 1
            if counter > REST_NUM:
                print(
                    'RUN:{},GT:{},DATA:{}'.format(
                        run, game_time, len(frame_and_action)
                    )
                )
                position = np.random.randint(len(env.track))
                env.car = Car(env.world, *env.track[position][1:4])
                counter = 0
    save_name = 'rollout_{}.npy'.format(on)
    np.save(dst + '/' + save_name, frame_and_action)

for i in range(1, 2):
    multiple_runs(i)

# print(frame)
# print(state)
# print(state.shape)
# print(state[10:20, 10:20, 2])
# frames = np.array(frames)
# print(len(frames))
# print(frames.shape)

# for _ in range(10):
#     print(generate_action())


from typing import Callable, Any, Optional
import functools
import time
file_name: str = input("Введите адрес сайта (имя файла): ")
saved_text: Optional[str] = None

def dec_with_time(_func=None, *, stop_time: int = 1) -> Callable:
    def stopper_decorator(func: Callable) -> Callable:
        """
        Функция декоратор, делает задержку перед выполнением обернутой функции
        :param func: обернутвая функция
        :return: обернутая функция
        """
        @functools.wraps(wrapped=func)
        def wrapper(**kwargs: Any) -> Any:
            # print("Ждем {} секунд".format(stop_time))
            time.sleep(stop_time)
            check = func(kwargs["name"])
            if check is None:
                print("-"*90)
                print("Текст сайта записан.\nЗаписаный текст: \n{}".format(saved_text))
            elif check:
                print("-"*90)
                print("Текст на сайте изменился.\nНовый текст: \n{}".format(saved_text))
            elif not check:
                pass
        return wrapper
    if _func is None:
        return stopper_decorator
    return stopper_decorator(_func)

@dec_with_time(stop_time=5)
def my_func(name: str) -> Any:
    """
    Функция првоерки изменений в файле (сайт)
    :param name: название файла с кодом сайта
    """
    global saved_text
    with open(name, "r", encoding="utf-8") as file:
        site_text = file.read()

    if saved_text:
        if not saved_text == site_text:
            saved_text = site_text
            return True
        else:
            return False
    else:
        saved_text = site_text
        return None

while True:
    my_func(name=file_name)


from unittest import mock

from elasticsearch.exceptions import TransportError

from elasticsearch_django.management.commands import (
    BaseSearchCommand,
    create_search_index,
    delete_search_index,
    prune_search_index,
    rebuild_search_index,
    update_search_index,
)

class BaseSearchCommandTests:
    """Tests for the elasticsearch_django management commands base command."""

    @mock.patch("elasticsearch_django.management.commands.logger")
    @mock.patch.object(BaseSearchCommand, "do_index_command")
    def test_handle(self, mock_do, mock_log):
        """Test the main handle method calls do_index_command."""
        obj = BaseSearchCommand()
        obj.handle(indexes=["foo", "bar"])
        # this should have called the do_index_command twice
        mock_do.assert_has_calls([mock.call("foo"), mock.call("bar")])
        mock_do.reset_mock()
        mock_do.side_effect = TransportError(message="oops", errors=(Exception(),))
        obj.handle(indexes=["baz"])
        mock_do.assert_called_once_with("baz")
        mock_log.exception.assert_called_once()

class NamedCommandTests:
    """Test each named command."""

    @mock.patch(
        "elasticsearch_django.management.commands.create_search_index.create_index"
    )
    def test_create_search_index(self, mock_create):
        """Test the create_search_index command."""
        cmd = create_search_index.Command()
        cmd.do_index_command("foo")
        mock_create.assert_called_once_with("foo")

    @mock.patch(
        "elasticsearch_django.management.commands.delete_search_index.delete_index"
    )
    def test_delete_search_index(self, mock_delete):
        """Test the delete_search_index command."""
        cmd = delete_search_index.Command()
        retval = cmd.do_index_command(
            "foo", interactive=False
        )  # True would hang the tests
        assert retval == mock_delete.return_value.body
        mock_delete.assert_called_once_with("foo")
        mock_delete.reset_mock()

        # mock out thw raw_input so the test doesn't hang
        with mock.patch.object(
            delete_search_index.Command, "_confirm_action"
        ) as mock_confirm:
            mock_confirm.return_value = False
            retval = cmd.do_index_command("foo", interactive=True)
            mock_delete.assert_not_called()
            assert retval is None

    @mock.patch(
        "elasticsearch_django.management.commands.prune_search_index.prune_index"
    )
    def test_prune_search_index(self, mock_prune):
        """Test the prune_search_index command."""
        cmd = prune_search_index.Command()
        cmd.do_index_command("foo")
        mock_prune.assert_called_once_with("foo")

    @mock.patch(
        "elasticsearch_django.management.commands.update_search_index.update_index"
    )
    def test_update_search_index(self, mock_update):
        """Test the update_search_index command."""
        cmd = update_search_index.Command()
        cmd.do_index_command("foo")
        mock_update.assert_called_once_with("foo")

    @mock.patch(
        "elasticsearch_django.management.commands.rebuild_search_index.delete_index"
    )
    @mock.patch(
        "elasticsearch_django.management.commands.rebuild_search_index.create_index"
    )
    @mock.patch(
        "elasticsearch_django.management.commands.rebuild_search_index.update_index"
    )
    def test_rebuild_search_index(self, mock_update, mock_create, mock_delete):
        """Test the rebuild_search_index command."""
        cmd = rebuild_search_index.Command()
        result = cmd.do_index_command(
            "foo", interactive=False
        )  # True would hang the tests
        mock_delete.assert_called_once_with("foo")
        mock_create.assert_called_once_with("foo")
        mock_update.assert_called_once_with("foo")
        assert result["delete"] == mock_delete.return_value.body
        assert result["create"] == mock_create.return_value.body
        assert result["update"] == mock_update.return_value
        # check that the delete is handled if the index does not exist
        mock_delete.side_effect = TransportError("Index not found")
        result = cmd.do_index_command(
            "foo", interactive=False
        )  # True would hang the tests
        assert result["delete"] == {}


from rest_framework import serializers

from users.models import User, Follow
from recipes.models import Recipe

class UserSerializer(serializers.ModelSerializer):
    """Сериализатор для модели User."""
    is_subscribed = serializers.SerializerMethodField()

    class Meta:
        model = User
        fields = (
            'username',
            'email',
            'first_name',
            'last_name',
            'is_subscribed',
            'id'
        )

    def get_is_subscribed(self, obj):

        request = self.context.get('request')
        if request.user.is_anonymous:
            return False
        if Follow.objects.filter(
            user=request.user,
            following__id=obj.id
        ).exists():
            return True
        return False

class FollowListSerializer(serializers.ModelSerializer):
    """Сериализатор для модели Follow"""

    recipes = serializers.SerializerMethodField()
    recipes_count = serializers.SerializerMethodField()
    is_subscribed = serializers.SerializerMethodField()

    class Meta:
        model = User
        fields = (
            'email',
            'id',
            'username',
            'first_name',
            'last_name',
            'recipes',
            'recipes_count',
            'is_subscribed'
        )
        read_only_fields = (
            'email',
            'id',
            'username',
            'first_name',
            'last_name'
        )

    def get_is_subscribed(self, obj):

        request = self.context.get('request')
        if request.user.is_anonymous:
            return False
        if Follow.objects.filter(
            user=request.user, following__id=obj.id
        ).exists():
            return True
        return False

    def get_recipes(self, obj):
        request = self.context.get('request')
        if request.method == 'GET':
            queryset = Recipe.objects.filter(author__id=obj.id).order_by('id')
        else:
            queryset = Recipe.objects.filter(author__id=obj.id).order_by('id')
        return RecipeMinifieldSerializer(queryset, many=True).data

    def get_recipes_count(self, obj):
        request = self.context.get('request')
        if request.method == 'GET':
            queryset = Recipe.objects.filter(author__id=obj.id).order_by('id')
        return queryset.count()

class RecipeMinifieldSerializer(serializers.ModelSerializer):
    """Сериализатор для показа небольшого количества информации о рецепте"""
    class Meta:
        model = Recipe
        fields = (
            'id',
            'name',
            'cooking_time',
            'image'
        )


'''
Created on Feb 10, 2019

@author: bperlman1
'''
import time
import dash
from dash.dependencies import Input, Output, State

import dash_core_components as dcc
import dash_html_components as html
import pandas as pd
import datetime
import plotly.graph_objs as go

from dashrisk import var_models as varm
from dashrisk import option_models as opmod
import base64
import pytz
import io
from dashrisk import dash_grid as dg

# Step 3: Define some often used variables
TIME_COLUMN='Date'
DEFAULT_PAGE_SIZE = 100 # number of rows to show
DEFAULT_TIMEZONE = 'US/Eastern'

# Step 4.2 define some helpfule css
button_style={
    'line-height': '40px',
    'borderWidth': '1px',
    'borderStyle': 'dashed',
    'borderRadius': '1px',
    'textAlign': 'center',
    'background-color':'#fffff0',
    'vertical-align':'middle',
}

select_file_style={
    'line-height': '40px',
    'borderWidth': '1px',
    'borderStyle': 'dashed',
    'borderRadius': '1px',
    'textAlign': 'center',
#     'background-color':'#42e2f4',
    'background-color':'#b0e2f4',
    'vertical-align':'middle',
}

buttons_grid_style = {'display': 'grid',
  'grid-template-columns': '49.5% 49.5%',
  'grid-gap': '1px'
}

grid_style = {'display': 'grid',
  'grid-template-columns': '98.5%',
  'grid-gap': '10px'
}

app = dash.Dash()

#********
# DEFAULT_PORTFOLIO_NAME=  'hi_volume_stocks.csv'
DEFAULT_PORTFOLIO_NAME=  'test_portfolio2.csv'
DF_NO_POSITION = pd.read_csv(DEFAULT_PORTFOLIO_NAME)

dt = dg.GridTable('dt','Risk Profile').html
dt_pos = dg.GridTable('dt_pos','Original Position').html
dt_greeks_full = dg.GridTable('dt_greeks_full','Greeks Full').html
dt_greeks_by_underlying = dg.GridTable('dt_greeks_by_underlying','Greeks By Underlying').html

# my_graph = dg.GridGraph('my-graph','Var By Underlying',['no_position'],[0],'Underlying','Value at Risk').html

app.layout = html.Div([
        html.Div([html.Div([],style={'display': 'none'})],id='spinner'),
        html.Div([html.H1("LiveRisk Analysis")],
                 style={'background-color':'#2a3f5f','border':'1px solid #C8D4E3','border-radius': '3px'}
        ),
         html.Div([
               html.Span(
                    dcc.Upload(
                        id='upload-data',
                        children=html.Div(['CLICK to select a portfolio csv'],id='select_div'),
                        # Allow multiple files to be uploaded
                        multiple=False,
                    ),
                    style=select_file_style
                ),
               html.Span(
#                     html.Div([DEFAULT_PORTFOLIO_NAME],id='portfolio_name'),
                    html.Div([],id='portfolio_name'),
                   style=select_file_style
                ),
               html.Span(
                    html.Div(['Portfolio 1/99 VaR = 0'],id='pvar_div'),
                    style=button_style
                ),
               html.Span(
                    html.Div(['S&P500 Equivilant Position = 0'],id='sp_eq_div'),
                    style=button_style
                ),
            ],
        style=buttons_grid_style),
        html.Div(
            [html.H2("Var Profile")],
            style={'background-color':'#eefaf7','border':'1px solid #C8D4E3','border-radius': '3px'}
        ),
        dcc.Graph(id='my-graph',style={'background-color':'#f5f5f0'}),
        # datatable grid
        html.Div(
            [html.H2("Risk Tables")],
            style={'background-color':'#eefaf7','border':'1px solid #C8D4E3','border-radius': '3px'}
        ),
        html.Div(
            html.Div([
                dt, dt_pos, dt_greeks_full,dt_greeks_by_underlying,#my_graph
                ],
                className='item1',style=grid_style
            ),
            id='risk_tables'
        ),
        html.Div(
            [html.H2("Sector Spdr Best Hedge")],
            style={'background-color':'#eefaf7','border':'1px solid #C8D4E3','border-radius': '3px'}
        ),
        dcc.Store(id='var_dict'),
        # Hidden div inside the app that stores the intermediate value
        html.Div(id='intermediate_value', style={'display': 'none'}),
    ]
)

@app.callback(
    Output('portfolio_name', 'children'),
    [
        Input('upload-data', 'filename'),
    ],
)
def  update_filename(filename):
    print('entering update_filename')
    if filename is None or len(filename)<1:
        r =  f'portfolio loaded: {DEFAULT_PORTFOLIO_NAME}'
    else:
        r =  f'portfolio loaded: {filename}'
    return [r]

@app.callback(
    Output('spinner','children'),
    [
        Input('risk_tables', 'children'),
        Input('portfolio_name', 'children')
    ],
    [
        State('var_dict', 'data'),
    ]
)
def hide_show_spinner(children1,children2,data):
# def hide_show_spinner(children1,data):
    print('entering spinner')
    if data is None:
        return html.Div(className='loader')
    # get yyyymmddhhmyyyymmddhhmmssmmmmmmmssmmm
    yyyymmddhhmmssmmmmmm = data['yyyymmddhhmmssmmmmmm']
    print(f'spinner callback: {yyyymmddhhmmssmmmmmm}')
    t_now = datetime.datetime.now()
    y = int(str(yyyymmddhhmmssmmmmmm)[0:4])
    mon = int(str(yyyymmddhhmmssmmmmmm)[4:6])
    day = int(str(yyyymmddhhmmssmmmmmm)[6:8])
    hour = int(str(yyyymmddhhmmssmmmmmm)[8:10])
    minute = int(str(yyyymmddhhmmssmmmmmm)[10:12])
    second = int(str(yyyymmddhhmmssmmmmmm)[12:14])
    microseconds = int(str(yyyymmddhhmmssmmmmmm)[14:20])
    t_then = datetime.datetime(y,mon,day,hour,minute,second,microseconds)
    total_secs = (t_now - t_then).total_seconds()
    print(f'exiting spinner - total seconds = {str(total_secs)}')
    if total_secs < .8:
        return html.Div(style={'display':'none'})
    return html.Div(className='loader')

@app.callback(
    Output('var_dict', 'data'),
    [
        Input('portfolio_name', 'children'),
    ],
    [
        State('upload-data', 'contents'),
    ]

)
def  update_memory(style,contents):
    print('entering update_memory')
    if contents is None:
        print('contents is None')
        df = DF_NO_POSITION.copy()
    else:
        df = parse_contents(contents)
    print(f'Start computing VaR {datetime.datetime.now()}')
    vm = varm.VarModel(df)
    var_dict = vm.compute_var()
    port_var = var_dict['port_var']
#     df_var_all = var_dict['df_underlying_positions']
    sp_dollar_equiv = var_dict['sp_dollar_equiv']

    # do greeks
    df_positions_all = var_dict['df_positions_all']
    df_atm_price = var_dict['df_atm_price']
    df_atm_price = df_atm_price.rename(columns={'close':'price'})
    df_std = var_dict['df_std']
    model_per_underlying_dict = {u:opmod.BsModel for u in df_atm_price.underlying}
    greeks_dict = opmod.get_df_greeks(df_positions_all, df_atm_price, model_per_underlying_dict)
    df_greeks = greeks_dict['df_greeks']
    # merge var and greeks
    df_risk_all = df_greeks.merge(df_positions_all[['symbol','position_var']],how='inner',on='symbol')
    df_risk_all = df_risk_all.drop(['option_price'],axis=1)
    n = datetime.datetime.now()
    print(f'End computing VaR {n}')
    yyyymmddhhmmssmmmmmm = '%04d%02d%02d%02d%02d%02d%06d' %(n.year,n.month,n.day,n.hour,n.minute,n.second,n.microsecond)
    return {'yyyymmddhhmmssmmmmmm':str(yyyymmddhhmmssmmmmmm),'df_std':df_std.to_dict('rows'),'df_risk_all':df_risk_all.to_dict('rows'),'port_var':port_var,'sp_dollar_equiv':sp_dollar_equiv}

@app.callback(
    Output('pvar_div', 'children'),
    [
        Input(component_id='var_dict', component_property='data'),
    ]
)
def  update_pvar_div(data):
    print('entering update_pvar_div')
    if data is None:
        port_var = 0.0
    else:
        port_var = round(float(data['port_var']),3)
    return f"Portfolio VaR = {port_var}"

@app.callback(
    Output('sp_eq_div', 'children'),
    [
        Input('pvar_div', 'children'),
    ],
    [
        State(component_id='var_dict', component_property='data'),
    ]
)
def  update_sp_eq_div(children,data):
    print('entering update_sp_eq_div')
    if data is None:
        sp_dollar_equiv = 0.0
    else:
        sp_dollar_equiv = round(float(data['sp_dollar_equiv']),3)
    return f"S&P500 Equivilant Position = {sp_dollar_equiv}"

@app.callback(
    Output('my-graph', 'figure'),
    [
        Input('sp_eq_div', 'children'),
    ],
    [
        State(component_id='var_dict', component_property='data'),
    ]
)
def update_graph(children,data):
    print('entering update_graph')
    df = pd.DataFrame(data['df_risk_all'])
    x_vals=df.underlying.as_matrix().reshape(-1)
    y_vals=df.position_var.as_matrix().reshape(-1)
    fig = go.Figure(data = [go.Bar(
                x=x_vals,
                y=y_vals
        )],
        layout= go.Layout(plot_bgcolor='#f5f5f0')
    )

    return fig

@app.callback(
    Output('risk_tables', 'children'),
    [
        Input('sp_eq_div', 'children'),
    ],
    [
        State(component_id='var_dict', component_property='data'),
    ]
)
def update_risk_tables(fig,data):
    print('entering update_risk_tables')
    # all positions
    df_risk_by_symbol = pd.DataFrame(data['df_risk_all'])
    risk_agg_cols = [c for c in df_risk_by_symbol.columns.values if c not in ['symbol','position']]
    df_risk_by_symbol = format_df(df_risk_by_symbol,['symbol','underlying','position'])
    # all by underlying
    df_risk_by_underlying = df_risk_by_symbol[risk_agg_cols].groupby('underlying',as_index=False).sum()
    df_risk_by_underlying = format_df(df_risk_by_underlying,['underlying'])
    # get and format DataFrames

    # create GridTable's
    dt_risk_by_symbol = dg.GridTable('dt_risk_by_symbol','Value at Risk and Greeks by Symbol',df_risk_by_symbol).html
    dt_risk_by_underlying = dg.GridTable('dt_risk_by_underlying','Value at Risk and Greeks by Underlying',df_risk_by_underlying).html

    # create return Div
    ret = html.Div([
        dt_risk_by_symbol,dt_risk_by_underlying
        ],
        className='item1',style=grid_style
    )
    return ret

def parse_contents(contents):
    '''
    app.layout contains a dash_core_component object (dcc.Store(id='df_memory')),
      that holds the last DataFrame that has been displayed.
      This method turns the contents of that dash_core_component.Store object into
      a DataFrame.

    :param contents: the contents of dash_core_component.Store with id = 'df_memory'
    :returns pandas DataFrame of those contents
    '''
    c = contents.split(",")[1]
    c_decoded = base64.b64decode(c)
    c_sio = io.StringIO(c_decoded.decode('utf-8'))
    df = pd.read_csv(c_sio)
    # create a date column if there is not one, and there is a timestamp column instead
    cols = df.columns.values
    cols_lower = [c.lower() for c in cols]
    if 'date' not in cols_lower and 'timestamp' in cols_lower:
        date_col_index = cols_lower.index('timestamp')
        # make date column
        def _extract_dt(t):
            y = int(t[0:4])
            mon = int(t[5:7])
            day = int(t[8:10])
            hour = int(t[11:13])
            minute = int(t[14:16])
            return datetime.datetime(y,mon,day,hour,minute,tzinfo=pytz.timezone(DEFAULT_TIMEZONE))
        # create date
        df['date'] = df.iloc[:,date_col_index].apply(_extract_dt)
    return df

def format_df(df_in,non_value_cols):
    df = df_in.copy()
#     non_value_cols = ['symbol','position','underlying']
    value_columns = [c for c in df.columns.values if c not in non_value_cols]
    for c in value_columns:
        try:
            df[c] = df[c].round(3)
        except:
            pass
    all_cols = non_value_cols + value_columns
    df = df[all_cols]
    return df

if __name__ == '__main__':
    app.run_server(port=8400)


# noinspection PyUnresolvedReferences
from PIL import Image, ImageEnhance

import numpy as np
from scipy import signal

from Utility.constants import *
from Utility.math_utility import *

def convert_pixel_to(pixel, nr_channels=3):
    new_pixel = [0, 0, 0, 0]

    for j, pixel_value in enumerate(pixel):
        new_pixel[j] = pixel_value

    if nr_channels == 3:
        new_pixel = (new_pixel[0], new_pixel[1], new_pixel[2])
    else:
        new_pixel = (new_pixel[0], new_pixel[1], new_pixel[2], new_pixel[3])
    return new_pixel

def get_common_part(image_name1, image_name2):
    im_handler1 = ImageHandler()
    im_handler1.read_image(image_name1)

    im_handler2 = ImageHandler()
    im_handler2.read_image(image_name2)

    minWidth = min(im_handler1.width, im_handler2.width)
    minHeight = min(im_handler1.height, im_handler2.height)

    pixels_img1 = im_handler1.get_pixels_copy(w=minWidth, h=minHeight)
    pixels_img2 = im_handler2.get_pixels_copy(w=minWidth, h=minHeight)

    pixels_img1 = np.asarray(pixels_img1)
    pixels_img2 = np.asarray(pixels_img2)

    return pixels_img1, pixels_img2

def calculate_psnr(image_name1, image_name2):
    pixels_img1, pixels_img2 = get_common_part(image_name1, image_name2)

    mse = np.mean((pixels_img1 - pixels_img2)**2)
    if abs(mse) < EPS:
        return float('inf')

    return 20*np.log10(255.0 / np.sqrt(mse))

def calculate_ssim(image_name1, image_name2, K1=0.01, K2=0.03, L=255):
    img1, img2 = get_common_part(image_name1, image_name2)

    C1 = (K1 * L) ** 2
    C2 = (K2 * L) ** 2

    avg1 = np.average(img1)
    avg2 = np.average(img2)

    cov = covariance(img1, img2)
    var1 = covariance(img1, img1)
    var2 = covariance(img2, img2)

    return (2*avg1*avg2 + C1)*(2*cov + C2) / ((avg1**2 + avg2**2 + C1)*(var1 + var2 + C2))

class ImageHandler:
    def __init__(self):
        self.image = None
        self.pixels = None
        self.height = None
        self.width = None

    def read_image(self, image_name):
        self.image = Image.open(image_name)
        self.pixels = self.image.load()
        self.width, self.height = self.image.size
        return self.image

    def get_pixels_pointer(self):
        return self.pixels

    def get_pixels_copy(self, w=None, h=None):
        if w is None:
            w = self.width

        if h is None:
            h = self.height

        width, height = w, h
        pixels = []
        for x in range(width):
            row = []
            for y in range(height):
                row.append(self.pixels[x, y])

            pixels.append(row)
        return pixels

    def get_size(self):
        return self.image.size

    def save_image(self, image_name):
        self.image.save(image_name)

    def rescale_image_keep_ratio_width(self, new_width, new_image_name):
        width_percent = new_width / self.width
        new_height = int(self.height * width_percent)
        self.rescale_image(new_width, new_height, new_image_name)

    def rescale_image_keep_ratio_height(self, new_height, new_image_name):
        height_percent = new_height / self.height
        new_width = int(self.width * height_percent)
        self.rescale_image(new_width, new_height, new_image_name)

    def rescale_image(self, new_width, new_height, new_image_name):
        new_width = int(new_width)
        new_height = int(new_height)
        new_image = self.image.resize((new_width, new_height), Image.ANTIALIAS)
        new_image.save(new_image_name)

    def rescale_image_by_factor(self, scale_factor, new_image_name):
        new_width = int(self.width*scale_factor)
        new_height = int(self.height*scale_factor)
        new_image = self.image.resize((new_width, new_height), Image.ANTIALIAS)
        new_image.save(new_image_name)

    def apply_contrast(self, factor, save_name):
        enhancer = ImageEnhance.Contrast(self.image)
        new_image = enhancer.enhance(factor)
        new_image.save(save_name)

    def apply_sharpness(self, factor, save_name):
        enhancer = ImageEnhance.Sharpness(self.image)
        new_image = enhancer.enhance(factor)
        new_image.save(save_name)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import yfinance as yf
from keras.models import load_model
import streamlit as st

def show_stock_price_prediction():

    plt.rcParams['font.sans-serif'] = ['Taipei Sans TC Beta']

    plt.style.use("dark_background")

    st.title('股價預測')

    user_input = st.text_input('輸入股票代號', '2330.TW')
    selected_year = st.slider('選擇期間（年）', 1, 10, 3)

    if st.button('預測股價'):

        df = yf.download(user_input, period=f'{selected_year}y')

        st.subheader(f'至今{selected_year}年期間的股票資料')
        st. write(df.describe())

        st.subheader('收盤價走勢')
        fig = plt.figure(figsize = (12,6))
        plt.plot(df.Close, 'b', label="收盤價")
        plt.legend(loc="upper left")
        st.pyplot(fig)

        # Splitting Data into Training and Testing

        data_training = pd.DataFrame(df['Close'][0:int(len(df)*0.70)])
        data_testing = pd.DataFrame(df['Close'][int(len(df)*0.70): int(len(df))])

        from sklearn.preprocessing import MinMaxScaler
        scaler = MinMaxScaler(feature_range=(0,1))

        data_training_array = scaler.fit_transform(data_training)

        #Load my model
        model = load_model('lstm_model_2330.h5', compile = False)

        #Testing Part

        past_100_days = data_training.tail(100)
        # final_df = past_100_days.append(data_testing, ignore_index=True)
        final_df = pd.concat([past_100_days, data_testing], ignore_index=True)
        input_data = scaler.fit_transform(final_df)

        x_test = [] # 產生一個名叫x_test的list
        y_test = [] # 產生一個名叫y_test的list

        for i in range(100, data_training_array.shape[0]):
            x_test.append(data_training_array[i-100: i])
            y_test.append(data_training_array[i, 0])

        x_test, y_test = np.array(x_test), np.array(y_test)
        y_predicted = model.predict(x_test)
        scaler = scaler.scale_

        scaler_factor = 1/scaler[0]
        y_predicted = y_predicted * scaler_factor
        y_test = y_test * scaler_factor

        #Final Graph

        st.subheader('使用LSTM模型預測收盤價走勢')
        fig2 = plt.figure(figsize=(12,6))
        plt.plot(y_test, 'b', label = '原收盤價')
        plt.plot(y_predicted, 'r', label = '預測收盤價')
        plt.xlabel('時間')
        plt.ylabel('收盤價')
        plt.legend()
        st.pyplot(fig2)


# -*- coding: utf-8 -*-

import json
import os

import torch
from dataclasses import dataclass
from tensorboardX import SummaryWriter

from .logging import log_info, log_warning, open_file

@dataclass
class ManagerOptions:
    base_path: str
    checkpoint_freq: int
    log_freq: int

def summary_model(model):
    num_params = 0
    for name, params in model.named_parameters():
        if params.requires_grad:
            num_params += params.numel()
    log_info('params count: %.2fM', num_params / 1024 / 1024)

def optimizer_to_device(op, device):
    for state in op.state.values():
        for k, v in state.items():
            if torch.is_tensor(v):
                state[k] = v.to(device)

def list_checkpoints(checkpoint_dir, include_best=False):
    names = []
    if not os.path.exists(checkpoint_dir):
        return names
    for filename in os.listdir(checkpoint_dir):
        parts = filename.split('.')
        if len(parts) != 3 or parts[0] != 'ckpt' or parts[2] != 'pt':
            continue
        try:
            step = int(parts[1])
        except Exception:
            continue
        names.append((step, os.path.join(checkpoint_dir, filename)))
    checkpoints = [filename for _, filename in sorted(names, reverse=True)]

    extra_paths = [(-1, 'pretrained.pt')]

    if include_best:
        extra_paths.append((0, 'best.pt'))

    for index, path in extra_paths:
        path = os.path.join(checkpoint_dir, path)
        if os.path.exists(path):
            if index == -1:
                index = len(checkpoints)
            checkpoints.insert(index, path)

    return checkpoints

class SessionManager:
    def __init__(self, options: ManagerOptions, training=True):
        self.use_cuda = torch.cuda.is_available()

        self.writer = None
        self.options = options

        self.base_path = options.base_path
        self.checkpoint_freq = options.checkpoint_freq
        self.log_freq = options.log_freq
        self.training = training

        self.device = torch.device('cuda:0' if self.use_cuda else 'cpu')

        self.checkpoint_dir = self.get_path('checkpoints')
        self.summaries_dir = self.get_path('summaries')

        self.metrics_name = None
        self.ckpt_count = 5

    def _restore(self):
        checkpoints = list_checkpoints(self.checkpoint_dir, True)
        assert checkpoints, 'can not find checkpoint files'

        log_info('Session restores from %s', checkpoints[0])

        saved_state = torch.load(checkpoints[0], map_location=self.device)

        self.global_step = saved_state['global_step']
        self.writer = SummaryWriter(log_dir=self.summaries_dir, purge_step=self.global_step)
        self.model = self.build_model()
        self.optimizer = self.build_optimizer()

        self.load_state_dict(saved_state)

        self.model.to(self.device)
        optimizer_to_device(self.optimizer, self.device)

        metrics_name = self.metrics_name
        if metrics_name is not None:
            metrics_value = saved_state.get(metrics_name, 0)
            setattr(self, metrics_name, metrics_value)
            log_info('Start from step: %d (%s %.3f)',
                     self.global_step, metrics_name, metrics_value)
        else:
            log_info('Start from step: %d', self.global_step)

    def _init(self):
        assert not list_checkpoints(self.checkpoint_dir), 'checkpoint file(s) already exists !!!'

        os.makedirs(self.checkpoint_dir, exist_ok=True)
        if self.metrics_name is not None:
            setattr(self, self.metrics_name, 0)

        self.global_step = 0
        self.writer = SummaryWriter(log_dir=self.summaries_dir)
        self.model = self.build_model()
        self.optimizer = self.build_optimizer()

        self.model.to(self.device)
        optimizer_to_device(self.optimizer, self.device)

        with open_file(os.path.join(self.base_path, 'config'), 'w') as fp:
            json.dump(self.options.__dict__, fp, indent=2)

    @classmethod
    def restore_or_init(cls, path, options_fn, restore=False, **kwargs):
        config_path = os.path.join(path, 'config') if restore else path
        options = options_fn(config_path)
        session_manager = cls(options, **kwargs)

        if restore:
            session_manager._restore()
        else:
            session_manager._init()

        summary_model(session_manager.model)
        return session_manager

    def build_model(self):
        raise NotImplementedError

    def build_optimizer(self):
        raise NotImplementedError

    def state_dict(self, saved_state):
        raise NotImplementedError

    def load_state_dict(self, saved_state):
        raise NotImplementedError

    def close(self):
        self.writer.close()

    def on_update(self, stats, prefix='main'):
        if not self.training:
            return

        self.global_step += 1
        step = self.global_step
        if step % self.log_freq == 0:
            self.log_stats(f'{step}', stats, prefix=prefix)
            self.try_save()

    def log_stats(self, prompt, stats, prefix):
        string = []
        for name, value in stats.items():
            string.append(f'{name}={value:.3f}')
        string = ' ; '.join(string)

        log_info('%s: %s', prompt, string)
        if self.training:
            self.add_summaries(stats.keys(), stats.values(), prefix=prefix)

    def try_save(self, metrics_value=None):
        step = self.global_step
        if step % self.checkpoint_freq != 0:
            if metrics_value is None:
                return
            model_paths = []
        else:
            model_paths = [os.path.join(self.checkpoint_dir, f'ckpt.{self.global_step}.pt')]

        state = {'global_step': step}
        metrics_name = self.metrics_name
        if metrics_name is not None and metrics_value is not None:
            if getattr(self, metrics_name) < metrics_value:
                setattr(self, metrics_name, metrics_value)
                model_paths.append(os.path.join(self.checkpoint_dir, 'best.pt'))
            state[metrics_name] = getattr(self, metrics_name)

        for path in model_paths:
            with open_file(path, 'wb') as fp:
                torch.save(self.state_dict(state), fp)

        for old_checkpoint in list_checkpoints(self.checkpoint_dir)[self.ckpt_count:]:
            try:
                os.remove(old_checkpoint)
            except Exception as err:
                log_warning('%s: can not remove old checkpoint: %s', err, old_checkpoint)

    def add_summaries(self, tags, values, prefix='main', global_step=None):
        if global_step is None:
            global_step = self.global_step
        tags = [tag.replace(' ', '_') for tag in tags]
        self.writer.add_scalars(prefix, dict(zip(tags, values)), global_step)

    def add_summary(self, tag, value, prefix='', global_step=None):
        if not self.training:
            return
        tag = tag.replace(' ', '_')
        if global_step is None:
            global_step = self.global_step
        self.writer.add_scalar(f'{prefix}/{tag}', value, global_step)

    def get_path(self, *name):
        return os.path.join(self.base_path, *name)


import abc
import asyncio
import copy
import logging
from collections import deque
from datetime import datetime as dt
from typing import Awaitable, Callable, List, Optional
from uuid import UUID, uuid4

from controllers.wq.progress import WQItemProgress
from controllers.wq.status import (
    WorkQueueState,
    WorkQueueStatus,
    WorkQueueStatusEntry,
    WorkQueueStatusItem,
)
from controllers.wq.types import (
    WQItemConfigType,
    WQItemKind,
    WQItemProgressType,
)

class WQItem(abc.ABC):

    _uuid: UUID
    _time_start: Optional[dt]
    _time_end: Optional[dt]
    _is_running: bool
    _is_done: bool
    logger: logging.Logger

    def __init__(self, logger: logging.Logger) -> None:
        self._uuid = uuid4()
        self._time_start = None
        self._time_end = None
        self._is_running = False
        self._is_done = False
        self.logger = logger

    @property
    def uuid(self) -> UUID:
        return self._uuid

    @property
    def time_start(self) -> Optional[dt]:
        return self._time_start

    @property
    def time_end(self) -> Optional[dt]:
        return self._time_end

    @property
    def duration(self) -> int:
        if self._time_start is None:
            return 0
        end = self._time_end if self._time_end is not None else dt.now()
        return (end - self._time_start).seconds

    @property
    def progress(self) -> WQItemProgress:
        return WQItemProgress(
            uuid=self.uuid,
            is_running=self._is_running,
            is_done=self._is_done,
            time_start=self._time_start,
            time_end=self._time_end,
            duration=self.duration,
            progress=self._progress,
        )

    def is_running(self) -> bool:
        return self._is_running

    def is_done(self) -> bool:
        return self._is_done

    async def run(self) -> None:
        assert not self._is_done
        assert self._time_end is None
        self._is_running = True
        self._time_start = dt.now()

        self.logger.debug(f"running work item uuid {self.uuid}")
        await self._run()
        self.logger.debug(f"done running work item uuid {self.uuid}")

        self._is_done = True
        self._is_running = False
        self._time_end = dt.now()

    async def stop(self) -> None:
        if not self._is_running:
            return

        await self._stop()
        self._is_running = False
        self._time_end = dt.now()

    @abc.abstractmethod
    async def _run(self) -> None:
        pass

    @abc.abstractmethod
    async def _stop(self) -> None:
        pass

    @property
    @abc.abstractmethod
    def _progress(self) -> Optional[WQItemProgressType]:
        pass

    @property
    @abc.abstractmethod
    def config(self) -> WQItemConfigType:
        pass

WQItemFinishCB = Callable[[WQItem], Awaitable[None]]
WQItemStartCB = Callable[[WQItem], Awaitable[None]]

class WQItemCB:
    start: WQItemStartCB
    finish: WQItemFinishCB

    def __init__(self, start: WQItemStartCB, finish: WQItemFinishCB) -> None:
        self.start = start
        self.finish = finish

class WQEntry:
    item: WQItem
    kind: WQItemKind
    cb: WQItemCB

    def __init__(self, item: WQItem, kind: WQItemKind, cb: WQItemCB) -> None:
        self.item = item
        self.kind = kind
        self.cb = cb

class WorkQueue:

    _lock: asyncio.Lock
    _task: Optional[asyncio.Task[None]]
    _is_shutting_down: bool
    _is_running: bool

    _waiting: deque[WQEntry]
    _running: Optional[WQEntry]
    _finished: List[WQEntry]

    _running_task: Optional[asyncio.Task[None]]

    logger: logging.Logger

    def __init__(self, logger: logging.Logger = logging.getLogger()) -> None:
        self._lock = asyncio.Lock()
        self._task = None
        self._is_shutting_down = False
        self._is_running = False
        self._waiting = deque()
        self._running = None
        self._finished = []
        self._running_task = None
        self.logger = logger

    async def start(self) -> None:
        async with self._lock:
            if self._is_shutting_down:
                self.logger.info("already shutting down.")
                return
            self.logger.info("starting workqueue")
            self._task = asyncio.create_task(self._tick())
            self._is_running = True

    async def stop(self) -> None:
        async with self._lock:
            if self._is_shutting_down:
                return
            self.logger.info("shutting down.")
            self._is_shutting_down = True

    async def _tick(self) -> None:
        while True:
            async with self._lock:
                if self._is_shutting_down:
                    await self._shutdown_queue()
                    break
                await self._handle_queue()

            await asyncio.sleep(1.0)

        self.logger.info("stopping main task, shutting down.")
        self._is_running = False

    async def _handle_queue(self) -> None:
        if self._running is None:
            await self._maybe_promote()
            return

        if self._running.item.is_done():
            item = self._running
            assert self._running_task is not None
            await self._running_task
            self._running = None
            self._running_task = None
            self._finished.append(item)
            await item.cb.finish(item.item)

    async def _maybe_promote(self) -> None:
        assert self._running is None
        if len(self._waiting) == 0:
            return

        next = self._waiting.popleft()
        self.logger.debug(f"promoting entry to running, kind: {next.kind}")
        await self._run_entry(next)

    async def _run_entry(self, entry: WQEntry) -> None:
        self._running = entry
        self._running_task = asyncio.create_task(entry.item.run())
        await self._running.cb.start(self._running.item)

    async def _shutdown_queue(self) -> None:
        self._waiting.clear()
        if self._running is None:
            return
        await self._running.item.stop()
        self._running = None

    async def put(self, item: WQItem, kind: WQItemKind, cb: WQItemCB) -> None:
        async with self._lock:
            if self._is_shutting_down:
                return
            self.logger.debug(f"append work item, kind: {kind}")
            self._waiting.append(WQEntry(item, kind, cb))

    async def waiting(self) -> List[WQEntry]:
        async with self._lock:
            lst = self._waiting.copy()
        return list(lst)

    async def finished(self) -> List[WQEntry]:
        async with self._lock:
            lst = self._finished.copy()
        return lst

    async def running(self) -> Optional[WQEntry]:
        ret: Optional[WQEntry] = None
        async with self._lock:
            if self._running is not None:
                ret = copy.copy(self._running)
        return ret

    def _convert_to_status_item(self, entry: WQEntry) -> WorkQueueStatusItem:
        return WorkQueueStatusItem(
            uuid=entry.item.uuid,
            kind=entry.kind,
            is_running=entry.item.is_running(),
            is_done=entry.item.is_done(),
            time_start=entry.item.time_start,
            time_end=entry.item.time_end,
            duration=entry.item.duration,
        )

    async def state(self) -> WorkQueueState:
        waiting: List[WorkQueueStatusItem] = []
        finished: List[WorkQueueStatusItem] = []
        current: Optional[WorkQueueStatusItem] = None

        async with self._lock:
            for entry in self._waiting:
                waiting.append(self._convert_to_status_item(entry))

            for entry in self._finished:
                finished.append(self._convert_to_status_item(entry))

            if self._running is not None:
                current = self._convert_to_status_item(self._running)

        return WorkQueueState(
            waiting=waiting, finished=finished, current=current
        )

    async def status(self) -> WorkQueueStatus:
        current: Optional[WQEntry] = await self.running()

        if current is None:
            return WorkQueueStatus(is_running=False, current=None)

        return WorkQueueStatus(
            is_running=True,
            current=WorkQueueStatusEntry(
                item=self._convert_to_status_item(current),
                progress=current.item.progress,
                config=current.item.config,
            ),
        )


import telebot
import openpyxl

bot = telebot.TeleBot('1745813569:AAFqiQHg5cszsEa0D5CU9YnEwxpSz70OlrY')

@bot.message_handler(commands=['start', 'help'])
def send_welcome(message):
    bot.reply_to(message, f'Я бот. Приятно познакомиться, {message.from_user.first_name}')

file = openpyxl.open("goods4bot.xlsx", read_only=True)

sheet = file.active

#print(sheet[1][0].value)
print(sheet["B2"].value)

cells = sheet["A1":"C10"]
for type, coll, roll in cells:
    print(type.value, coll.value, roll.value)

#for row in range(1, sheet.max_row + 1):
#    type = sheet[row][0].value
#    collection = sheet[row][1].value
#    roll = sheet[row][2].value
#    print(row, type, collection, roll)


import torch
from tqdm import tqdm
from transformers import RobertaTokenizer, RobertaForMaskedLM
from dataset import SemEvalDataSet, load_data
from sound_similarity.sound_similarity import find_similar_sounding_words

MODEL = 'roberta-large'

tokenizer = RobertaTokenizer.from_pretrained(MODEL)
model = RobertaForMaskedLM.from_pretrained(MODEL)

def load_dataset_for_inference():
  data = load_data(
    path_input='subtask3-heterographic-test.xml',
    path_label='subtask3-heterographic-test.gold'
  )
  train_data = SemEvalDataSet(data)
  return train_data
train_data = []

def get_target_word_from_pun(masked_sent, cand_list):
  inputs = tokenizer(masked_sent, return_tensors="pt")
  loss_arr = []

  for cand in tqdm(cand_list):
    sent_option = masked_sent.replace("<mask>", cand)
    labels = tokenizer(sent_option, return_tensors="pt")["input_ids"][0][:inputs.input_ids.shape[1]]
    outputs = model(**inputs, labels=labels)
    loss_arr.append(outputs.loss.item())

  min_idx = torch.argmin(torch.tensor(loss_arr)).item()
  pred_word = cand_list[min_idx]
  pred_sent = masked_sent.replace('<mask>', pred_word)

  return pred_word, pred_sent

src_target_list = []

def run_inference_on_dataset():
  for train in train_data:
    sent = train['sentence']
    src = train['src']
    tgt_root = train['tgt_root']
    masked_sent = sent.replace(src, '<mask>')
    cand_list = find_similar_sounding_words(src)

    if cand_list != []:
      target_pred_pun, target_sent = get_target_word_from_pun(masked_sent, cand_list)
      print("Sentence: ", sent)
      print("Source word: ", src)
      print("Target root: ", tgt_root)
      print("Predicted target: ", target_pred_pun)
      print("Predicted sentence: ", target_sent)
      print("===========================================================")
      src_target_list.append({ "src": src, "target": tgt_root, "pred": target_pred_pun, "sent": sent })

  return src_target_list

def run_inference_pipeline(sentence, source):
  masked_sent = sentence.replace(source, '<mask>')
  cand_list = find_similar_sounding_words(source)
  if cand_list != []:
    target_pred_pun, target_sent = get_target_word_from_pun(masked_sent, cand_list)
    print("=" * 50)
    print("Sentence: ", sentence)
    print("Source word: ", source)
    print("Predicted target: ", target_pred_pun)
    print("Predicted sentence: ", target_sent)
    print("=" * 50)
    return [{ "source": source, "predicted": target_pred_pun, "sentence": sentence }]


#!/usr/bin/env python3
"""Module for Deletion-resilient hypermedia pagination
"""

import csv
import math
from typing import List, Dict

class Server:
    """Server class to paginate a database of popular baby names.
    """
    DATA_FILE = "Popular_Baby_Names.csv"

    def __init__(self):
        self.__dataset = None
        self.__indexed_dataset = None

    def dataset(self) -> List[List]:
        """Cached dataset
        """
        if self.__dataset is None:
            with open(self.DATA_FILE) as f:
                reader = csv.reader(f)
                dataset = [row for row in reader]
            self.__dataset = dataset[1:]

        return self.__dataset

    def indexed_dataset(self) -> Dict[int, List]:
        """Dataset indexed by sorting position, starting at 0
        """
        if self.__indexed_dataset is None:
            dataset = self.dataset()
            self.__indexed_dataset = {
                i: dataset[i] for i in range(len(dataset))
            }
        return self.__indexed_dataset

    def get_hyper_index(self, index: int = None, page_size: int = 10) -> Dict:
        """
        Return a dictionary with hypermedia
        pagination information based on the index and page size.
        """
        indexed_dataset = self.indexed_dataset()
        dataset_size = len(indexed_dataset)

        if index is None:
            index = 0
        else:
            assert 0 <= index < dataset_size, "Index is out of range."

        data = []
        next_index = index

        while len(data) < page_size and next_index < dataset_size:
            if next_index in indexed_dataset:
                data.append(indexed_dataset[next_index])
            next_index += 1

        return {
            "index": index,
            "data": data,
            "page_size": len(data),
            "next_index": next_index if next_index < dataset_size else None
        }

    def delete_item(self, index: int):
        """
        Delete an item from the indexed dataset.
        """
        if index in self.__indexed_dataset:
            del self.__indexed_dataset[index]
            self.__indexed_dataset = {
                i: self.__indexed_dataset[i] for i in range(
                                             len(self.__indexed_dataset))
            }


# coding=utf-8
"""
Created on Oct 14, 2010
@author: Peter Harrington
"""
import matplotlib.pyplot as plt

plt.rcParams['font.sans-serif']=['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus']=False    # 用来正常显示负号

decisionNode = dict(boxstyle="sawtooth", fc="0.8")
leafNode = dict(boxstyle="round4", fc="0.8")
arrow_args = dict(arrowstyle="<-")

def get_num_leafs(_my_tree):
    num_leafs = 0
    first_str = list(_my_tree.keys())[0]
    second_dict = _my_tree[first_str]
    for key in second_dict.keys():
        # test to see if the nodes are dictionaries, if not they are leaf nodes
        if type(second_dict[key]).__name__ == 'dict':
            num_leafs += get_num_leafs(second_dict[key])
        else:
            num_leafs += 1
    return num_leafs

def get_tree_depth(_my_tree):
    max_depth = 0
    first_str = list(_my_tree.keys())[0]
    second_dict = _my_tree[first_str]
    for key in second_dict.keys():
        # test to see if the nodes are dictionaries, if not they are leaf nodes
        if type(second_dict[key]).__name__ == 'dict':
            this_depth = 1 + get_tree_depth(second_dict[key])
        else:
            this_depth = 1
        if this_depth > max_depth:
            max_depth = this_depth
    return max_depth

def plot_node(node_text, center_pt, parent_pt, node_type):
    """
    参考：https://matplotlib.org/3.1.0/api/_as_gen/matplotlib.pyplot.annotate.html
    :param node_text:
    :param center_pt:
    :param parent_pt:
    :param node_type:
    :return:
    """
    create_plot.ax1.annotate(
        node_text, xy=parent_pt, xycoords='axes fraction',
        xytext=center_pt, textcoords='axes fraction',
        va="center", ha="center", bbox=node_type, arrowprops=arrow_args
    )

def plot_mid_text(cntr_pt, parent_pt, txt_string):
    x_mid = (parent_pt[0] - cntr_pt[0]) / 2.0 + cntr_pt[0]
    y_mid = (parent_pt[1] - cntr_pt[1]) / 2.0 + cntr_pt[1]
    create_plot.ax1.text(x_mid, y_mid, txt_string, va="center", ha="center", rotation=30)

def plot_tree(_my_tree, parent_pt, _node_text):    # if the first key tells you what feat was split on
    num_leafs = get_num_leafs(_my_tree)      # this determines the x width of this tree
    depth = get_tree_depth(_my_tree)
    first_str = list(_my_tree.keys())[0]         # the text label for this node should be this
    cntr_pt = (plot_tree.xOff + (1.0 + float(num_leafs)) / 2.0 / plot_tree.totalW, plot_tree.yOff)
    plot_mid_text(cntr_pt, parent_pt, _node_text)
    plot_node(first_str, cntr_pt, parent_pt, decisionNode)
    second_dict = _my_tree[first_str]
    plot_tree.yOff = plot_tree.yOff - 1.0 / plot_tree.totalD
    for key in second_dict.keys():
        if type(second_dict[key]).__name__ == 'dict':
            # test to see if the nodes are dictionaries, if not they are leaf nodes
            plot_tree(second_dict[key], cntr_pt, str(key))        # recursion
        else:
            # it's a leaf node print the leaf node
            plot_tree.xOff = plot_tree.xOff + 1.0 / plot_tree.totalW
            plot_node(second_dict[key], (plot_tree.xOff, plot_tree.yOff), cntr_pt, leafNode)
            plot_mid_text((plot_tree.xOff, plot_tree.yOff), cntr_pt, str(key))
    plot_tree.yOff = plot_tree.yOff + 1.0 / plot_tree.totalD

# def create_plot():
#     fig = plt.figure(1, facecolor='white')
#     fig.clf()
#     create_plot.ax1 = plt.subplot(111, frameon=False)
#     plot_node(U'决策节点', (0.5, 0.1), (0.1, 0.5), decisionNode)
#     plot_node(U'叶子节点', (0.8, 0.1), (0.3, 0.8), leafNode)
#     plt.show()

def create_plot(in_tree):
    fig = plt.figure(1, facecolor='white')
    fig.clf()
    ax_props = dict(xticks=[], yticks=[])
    create_plot.ax1 = plt.subplot(111, frameon=False, **ax_props)    # no ticks
    # createPlot.ax1 = plt.subplot(111, frameon=False) # ticks for demo puropses
    plot_tree.totalW = float(get_num_leafs(in_tree))
    plot_tree.totalD = float(get_tree_depth(in_tree))
    plot_tree.xOff = -0.5 / plot_tree.totalW
    plot_tree.yOff = 1.0
    plot_tree(in_tree, (0.5, 1.0), '')
    plt.show()

def retrieve_tree(i):
    list_of_trees = [
        {'no surfacing': {0: 'no', 1: {'flippers': {0: 'no', 1: 'yes'}}}},
        {'no surfacing': {0: 'no', 1: {'flippers': {0: {'head': {0: 'no', 1: 'yes'}}, 1: 'no'}}}}
    ]
    return list_of_trees[i]

if __name__ == '__main__':
    my_tree = retrieve_tree(0)
    # my_tree['no surfacing'][3] = "maybe"
    # print("my_tree")
    # print(my_tree)
    # print(get_num_leafs(my_tree))
    # print(get_tree_depth(my_tree))
    create_plot(my_tree)
    print("Run Decision Tree plot finish")


from sklearn.naive_bayes import MultinomialNB

import logging

logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', \
                    level=logging.INFO, \
                    datefmt='%Y-%m-%d %H:%M:%S')

from pdb import set_trace as bp

class NaiveBayesMultinomial:

    def fit(self, X, y):
        logging.info('naive_bayes_multinomial.fit')
        self.classifier = MultinomialNB()
        self.classifier.fit(X, y)

    def predict(self, X):
        logging.info('naive_bayes_multinomial.predict')
        prediction = self.classifier.predict(X)
        return prediction


import argparse
import os
import time
from collections import Counter
import logging

from omnidata import load_mnist
from omnimodel import Statistician
from omniplot import save_test_grid
from torch import optim
from torch.autograd import Variable
from torch.nn import functional as F
from torch.utils import data
from tqdm import tqdm

import gzip
import numpy as np
import os
import pickle
import torch

from skimage.transform import rotate
from torch.utils import data

from omnidata import OmniglotSetsDataset

try:
    from utils import (kl_diagnormal_diagnormal, kl_diagnormal_stdnormal,
                       gaussian_log_likelihood)
except ModuleNotFoundError:
    # put parent directory in path for utils
    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
    from utils import (kl_diagnormal_diagnormal, kl_diagnormal_stdnormal,
                       gaussian_log_likelihood)

np.random.seed(0)
n_features = 256 * 4 * 4  # output shape of convolutional encoder
# create model
from torch import optim
from torch.autograd import Variable
from torch.nn import functional as F
from torch.utils import data
from tqdm import tqdm
model_kwargs = {
    'batch_size': 32,
    'sample_size': 5,
    'n_features': n_features,
    'c_dim': 512,
    'n_hidden_statistic': 3,
    'hidden_dim_statistic': 256,
    'n_stochastic':1,
    'z_dim': 16,
    'n_hidden': 3,
    'hidden_dim': 256,
    'nonlinearity': F.elu,
    'print_vars': False
}

model = Statistician(**model_kwargs)
model.cuda()
optimizer = optim.Adam(model.parameters(), lr=1e-3)

#filename = "../outputdilate4/checkpoints/15-02-2019-03:43:01-400.m"
filename = "../outputs/outputnewmore/checkpoints/07-03-2019-21:47:26-300.m"
checkpoint = torch.load(filename)
model.load_state_dict(checkpoint['model_state'])
optimizer.load_state_dict(checkpoint['optimizer_state'])
model.eval()

def classify_datapoint(x, D_means, D_vars, K, single_sample=True):
    dataset = torch.from_numpy(x)
    with torch.no_grad():
        inputs = Variable(dataset.cuda())
    h1 = model.shared_convolutional_encoder(inputs)
    c_mean_, c_logvar_ = model.statistic_network(h1, summarize=False, single_sample=single_sample)
    kl_divergences = []
    for i in range(K):
        kl = kl_diagnormal_diagnormal(D_means[i], D_vars[i], c_mean_, c_logvar_)
        kl_divergences.append(kl.data.item())
    best_index = kl_divergences.index(min(kl_divergences))
    return best_index

def get_omniglot_kshot(images, lb, K=5, support=1, n_test_samples=100):
    test_classes = list(Counter(lb).keys())
    chosen_K = np.random.choice(test_classes, K)
    n_test_samples = 20 - support
    D = []
    x_test = []
    x_labels = []
    for i in range(K):
        idx = chosen_K[i]
        targets_idx = np.where(lb == idx)[0]
        actual_inputs = images[targets_idx[:support]]
        D.append(actual_inputs)
        samples = images[targets_idx[support : support + n_test_samples]]
        x_test.append(samples)
        x_labels += ([i] * samples.shape[0])
    x_labels = np.array(x_labels)
    x_test = np.vstack(x_test)
    D = np.array(D)
    return D, x_test, x_labels

def get_mnist_kshot(images, lb, K=10, support =1, n_test_samples=100):
    W = support
    D = []
    x_test = []
    x_labels = []
    for i in range(K):
        idx = np.where(lb == i)[0]
        np.random.shuffle(idx)
        D.append(images[idx[:W]])
        samples = images[idx[W : W + n_test_samples]]
        x_test.append(samples)
        x_labels += ([i] * samples.shape[0])
    x_labels = np.array(x_labels)
    x_test = np.vstack(x_test)
    D = np.array(D)
    return D, x_test, x_labels

def do_kshot(dataset="mnist", K=10, support=1, n_trials=100, n_test_samples=1):
    W = support
    if dataset == "mnist":
        data_dir = "../mnist-data"
        images, one_hot_labels = load_mnist(data_dir=data_dir)
        lb = np.argmax(one_hot_labels, 1)
        get_kshot_data_fc = get_mnist_kshot
    else:
        data_dir = "../omniglot-data"
        kshot =  OmniglotSetsDataset(data_dir=data_dir, split='kshot')
        lb = kshot.data['targets']
        images = kshot.data['inputs']
        get_kshot_data_fc = get_omniglot_kshot
        n_test_samples = 20 - support
    accs = []
    for trial in tqdm(range(n_trials)):
        D, x_test, x_labels = get_kshot_data_fc(images, lb, K=K, support=support, n_test_samples=n_test_samples)
        D_means = []
        D_vars = []
        for i in range(K):
            dataset = torch.from_numpy(D[i])
            with torch.no_grad():
                inputs = Variable(dataset.cuda())
            model.eval()
            h = model.shared_convolutional_encoder(inputs)
            c_mean_full, c_logvar_full = model.statistic_network(h, summarize=True)
            D_means.append(c_mean_full)
            D_vars.append(c_logvar_full)
        test_loader = data.DataLoader(dataset=x_test, batch_size=n_test_samples,
                                  shuffle=False, num_workers=0, drop_last=False)
        preds = []
        for batch in test_loader:
            with torch.no_grad():
                inputs = Variable(batch.cuda())
            h1 = model.shared_convolutional_encoder(inputs)
            c_mean_, c_logvar_ = model.statistic_network(h1, single_sample=True)
            for bi, x in enumerate(batch):
                kl_divergences = []
                for i in range(K):
                    kl = kl_diagnormal_diagnormal(D_means[i], D_vars[i], c_mean_[bi], c_logvar_[bi])
                    kl_divergences.append(kl.data.item())
                best_index = kl_divergences.index(min(kl_divergences))
                preds.append(best_index)
        acc = np.mean(np.array(preds) == x_labels)
        logging.info(acc)
        accs.append(acc)
    return accs

# accs_1 = mnist_one_shot(support=1, n_test_samples=10, n_trials=100)
# print("1-shot: {}".format(np.mean(accs_1)))
dataset = "omniglot"
for support in [1]:
    for K in [20]:
        logging.info("************************")
        logging.info("OMNIGLOT {}-Shot {}-Way".format(support, K))
        accs = do_kshot(dataset=dataset, K=K, support=support, n_test_samples=20, n_trials=100)
        logging.info("OMNIGLOT {}-Shot {}-Way".format(support, K))
        logging.info("k-shot: {}".format(np.mean(accs)))
        print(np.mean(accs))


'''
    This file is part of PM4Py (More Info: https://pm4py.fit.fraunhofer.de).

    PM4Py is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    PM4Py is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with PM4Py.  If not, see <https://www.gnu.org/licenses/>.
'''

from pm4py.objects.log.obj import EventLog, EventStream
import pandas as pd
from typing import Union, Optional, Dict, Any
from pm4py.objects.conversion.log import converter as log_converter
from enum import Enum
from pm4py.util import exec_utils, constants, xes_constants, pandas_utils

class Parameters(Enum):
    ARRIVAL_RATE = "arrival_rate"
    FINISH_RATE = "finish_rate"
    CASE_ID_COLUMN = constants.PARAMETER_CONSTANT_CASEID_KEY
    START_TIMESTAMP_COLUMN = constants.PARAMETER_CONSTANT_START_TIMESTAMP_KEY
    TIMESTAMP_COLUMN = constants.PARAMETER_CONSTANT_TIMESTAMP_KEY
    RESOURCE_COLUMN = constants.PARAMETER_CONSTANT_RESOURCE_KEY
    ACTIVITY_COLUMN = constants.PARAMETER_CONSTANT_ACTIVITY_KEY
    GROUPER_FREQ = "grouper_freq"
    SERVICE_TIME = "service_time"
    WAITING_TIME = "waiting_time"
    SOJOURN_TIME = "sojourn_time"
    DIFF_START_END = "diff_start_end"

def apply(log: Union[EventLog, EventStream, pd.DataFrame], parameters: Optional[Dict[Any, Any]] = None) -> pd.DataFrame:
    """
    Extracts temporal features with the provided granularity from the Pandas dataframe.

    Implements the approach described in the paper:
    Pourbafrani, Mahsa, Sebastiaan J. van Zelst, and Wil MP van der Aalst. "Supporting automatic system dynamics model generation for simulation in the context of process mining." International Conference on Business Information Systems. Springer, Cham, 2020.

    Parameters
    ---------------
    log
        Event log / Event stream / Pandas dataframe
    parameters
        Parameters of the algorithm, including:
        - Parameters.GROUPER_FREQ => the time interval to be used for the grouping
        - Parameters.ARRIVAL_RATE => column of the dataframe which is going to host the arrival rate
        - Parameters.FINISH_RATE => column of the dataframe which is going to host the finishing rate
        - Parameters.SERVICE_TIME => column of the dataframe which is going to host the service time
        - Parameters.WAITING_TIME => column of the dataframe which is going to host the waiting time
        - Parameters.SOJOURN_TIME => column of the dataframe which is going to host the sojourn time
        - Parameters.CASE_ID_COLUMN => case ID column in the dataframe (default: case:concept:name)
        - Parameters.ACTIVITY_COLUMN => activity column in the dataframe (default: concept:name)
        - Parameters.TIMESTAMP_COLUMN => timestamp column in the dataframe (default: time:timestamp)
        - Parameters.RESOURCE_COLUMN => resource column in the dataframe (default: org:resource)
        - Parameters.START_TIMESTAMP_COLUMN => start timestamp column in the dataframe (if not provided, the timestamp column is used)

    Returns
    ----------------
    features_df
        Dataframe with temporal features
    """
    if parameters is None:
        parameters = {}

    grouper_freq = exec_utils.get_param_value(Parameters.GROUPER_FREQ, parameters, "W")
    timestamp_column = exec_utils.get_param_value(Parameters.TIMESTAMP_COLUMN, parameters, xes_constants.DEFAULT_TIMESTAMP_KEY)
    start_timestamp_column = exec_utils.get_param_value(Parameters.START_TIMESTAMP_COLUMN, parameters, None)
    if start_timestamp_column is None:
        start_timestamp_column = timestamp_column
    case_id_column = exec_utils.get_param_value(Parameters.CASE_ID_COLUMN, parameters, constants.CASE_CONCEPT_NAME)
    diff_start_end = exec_utils.get_param_value(Parameters.DIFF_START_END, parameters, "@@diff_start_end")
    arrival_rate = exec_utils.get_param_value(Parameters.ARRIVAL_RATE, parameters, "@@arrival_rate")
    finish_rate = exec_utils.get_param_value(Parameters.FINISH_RATE, parameters, "@@finish_rate")
    service_time = exec_utils.get_param_value(Parameters.SERVICE_TIME, parameters, "@@service_time")
    waiting_time = exec_utils.get_param_value(Parameters.WAITING_TIME, parameters, "@@waiting_time")
    sojourn_time = exec_utils.get_param_value(Parameters.SOJOURN_TIME, parameters, "@@sojourn_time")
    resource_column = exec_utils.get_param_value(Parameters.RESOURCE_COLUMN, parameters, xes_constants.DEFAULT_RESOURCE_KEY)
    activity_column = exec_utils.get_param_value(Parameters.ACTIVITY_COLUMN, parameters, xes_constants.DEFAULT_NAME_KEY)

    log = log_converter.apply(log, variant=log_converter.Variants.TO_DATA_FRAME, parameters=parameters)
    log = pandas_utils.insert_case_arrival_finish_rate(log, case_id_column=case_id_column, timestamp_column=timestamp_column, arrival_rate_column=arrival_rate, finish_rate_column=finish_rate)
    log = pandas_utils.insert_case_service_waiting_time(log, case_id_column=case_id_column, timestamp_column=timestamp_column, diff_start_end_column=diff_start_end, service_time_column=service_time, sojourn_time_column=sojourn_time, waiting_time_column=waiting_time)

    grouped_log = log.groupby(pd.Grouper(key=start_timestamp_column, freq=grouper_freq))

    final_values = []

    for gkey, gval in grouped_log:
        dct = {}
        dct["timestamp"] = gkey

        gval_first = gval.groupby(case_id_column).first()

        dct["unique_resources"] = gval[resource_column].nunique()
        dct["unique_cases"] = gval[case_id_column].nunique()
        dct["unique_activities"] = gval[activity_column].nunique()
        dct["num_events"] = len(gval)

        dct["average_arrival_rate"] = gval_first[arrival_rate].mean()
        dct["average_finish_rate"] = gval_first[finish_rate].mean()

        dct["average_waiting_time"] = gval_first[waiting_time].mean()
        dct["average_sojourn_time"] = gval_first[sojourn_time].mean()
        dct["average_service_time"] = gval_first[service_time].mean()

        final_values.append(dct)

    dataframe = pd.DataFrame(final_values)
    dataframe.fillna(0, inplace=True)
    return dataframe


from langchain.document_loaders import TextLoader
from langchain.llms import OpenAI
import json
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import Chroma
from langchain.prompts import PromptTemplate
from langchain.chains import RetrievalQA
import openai
import os
import singlestoredb as s2
import glob
from db import add_vector, search
import json
from dotenv import load_dotenv

load_dotenv()

os.environ["OPENAI_API_KEY"] = os.getenv('OPENAI_API_KEY')
openai.api_key = os.environ["OPENAI_API_KEY"]

#identify the key points in a transcript
#We describe a key point as a major speech point that the speaker makes in an effort
#to emphasize a certain concept or emotion. We use ChatGPT which does a good job of
#pattern matching and classification to essentially to find keypoints in a speech. Given
#the large amount of media, it's processed--it should be adept with speeches.
def identify_keypoints(filepath):
    loader = TextLoader(filepath)
    docs = loader.load()

    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
    texts = text_splitter.split_documents(docs)
    print("@identify_keypoints")
    print("\n\n What are the texts? \n\n")
    print(texts)
    print("\n\n")
    embeddings = OpenAIEmbeddings()
    docsearch = Chroma.from_documents(texts, embeddings)

    prompt_template = """Use the following pieces of context to answer the question at the end.
    {context}

    Question: {question}
    """
    PROMPT = PromptTemplate(
        template=prompt_template, input_variables=["context", "question"]
    )
    chain_type_kwargs = {"prompt": PROMPT}

    qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
    #Add a confidence level field alongside each field you extract representing how confident you are in your result.
    query = '''
    You are given a speech's transcript here. Your goal here is to identify the key points in this speech. For the purposes of this task,
    you can define a key point as something that makes a new significant argument in the speech.

    Follow these guidelines for a keypoint.
    1. It is a direct quote from the speech word for word.
    2. The maximum size of the keypoint you can choose is one sentence. Do not choose a keypoint consisting of multiple sentences.
    3. All keypoints are meant to be unique and shouldn't overlap with other keypoints.
    4. It should be relevant and hold high significance to the argument.

    You MUST express your list where the elements are the keypoints with the opening and
    ending brace as such: ["keypoint1", "keypoint2", ...]

    Choose at most 5 keypoints.
    You MUST give an answer--even if it is an empty list. Do not say "I don't know", provide your
    best guess.
    '''

    result = qa.run(query)
    return result

def keypoints(filepath):
    result = identify_keypoints(filepath)
    if result == "I don't know.":
        return []
    #convert the text output into a list
    #print("Result of chat is: ", result)

    begin = result.index("[")
    end = result.index("]")
    list_string = result[begin: end + 1]
    print(list_string)
    list = json.loads(list_string)
    return list

#now that we have our keypoints, we address the entire labeling problem--primary system
#Let's run this method over all the media that we're given to develop a holistic view
#aggregating the necessary keypoints for us to run similarity search and determine labels
#returns a list of dictionary elements that contain an id with a list of keypoints for the file of filename of id
def aggregate_keypoints(fileselector, speaker_id):
    ungrouped_kps = []
    for fpath in glob.glob(fileselector):
        kp_associated_data = {
            'speaker_id': speaker_id,
            'transcript_id': fpath,
            'keypoints': keypoints(fpath)
        }
        ungrouped_kps.append(kp_associated_data)
    for ungrouped_kp in ungrouped_kps:
        print(ungrouped_kp['keypoints'])
        print('\n\n...\n\n')

    return ungrouped_kps

#speakers = aggregate_keypoints('./transcripts/*.txt')

def get_embedding(text):
    return openai.Embedding.create(input = text, model="text-embedding-ada-002")['data'][0]['embedding']

def group_keypoints(spkr_id, trscpt_id, speakers):
    for speaker in speakers:
        for kp in speaker['keypoints']:
            embd = get_embedding(kp)
            add_vector(spkr_id, trscpt_id, kp, embd)

#Difference function to evaluate the differences in speech and emotional cues of
#two specific keypoint analyses from Hume.AIg
def difference(model_speech_emos, human_speech_emos):
    return

def score(transcript, speaker_id):
    #identif
    #return json()
    #keypoints from your transcript
    #{kp:[
    #  your_keypoints,
    #  kp1: {
    #     expected_emotion_data
    #  }
    #
    # }
    speakers = aggregate_keypoints('./data/model/*.txt')
    #update the database by extracting embedding for each keypoint for speech for each speaker
    group_keypoints(speaker_id, transcript + uuid(), speakers)
    #access hume functions
    customer_kps = aggregate_keypoints('./data/customer/*.txt')
    differences = [{}]
    for customer_kp in customer_kps:
        for customer_keypoint in customer_kp['keypoints']:
            customer_embedding = get_embedding(customer_keypoint)
            most_similar_model_keypoint, transcript_id = search(customer_embedding)
            customer_emotion_at_keypoint = get_human_emotion(customer_keypoint)
            differences.append({
                'source_customer_keypoint': customer_keypoint,
                'nearest_model_keypoint': most_similar_model_keypoint,
                'customer_emotion_data': customer_emotion_at_keypoint,
                'most_similar_model_data': hume_emotion_from_clip(clip_path(get_timestep_for_keypoint(transcript_id, most_similar_model_keypoint))),
                'emotional_difference': difference(most_similar_model_keypoint, customer_emotion_at_keypoint)
            })

def get_difference_summary(summary):
    str_summary = json.dumps(summary)
    response = openai.ChatCompletion.create(
        model="gpt-3.5-turbo",
        messages=[
            {"role": "user", "content": f'''I'm going to give you a json representing the emotions relevant to major key times in a speech.
                I want you to act like a speech coach that analyzes the given data to concisely summarize the insights for
                the speaker. Each object in the json comes with the main emotion, the difference between how much of this emotion
                we want you to show and how much it was shown, and the mode of communication (voice/face).
                A positive number means the speaker should portray this emotion more through that mode whereas
                a negative means you should tone it down.'''},
            {"role": "user", "content": f"Here's the JSON -- ${str_summary}"},
            {"role": "user", "content": "Can you give me a few sentences of feedback with your interpretations?"}
        ]
    )

    return response["choices"][0]["message"]["content"]

jsony = '''"summary": {
        "In these tough times, we need to gather together and really, really like help each other.": [
            {
                "difference": 0.27074927455270126,
                "emotion": "Sadness",
                "type": "face"
            },
            {
                "difference": 0.3909689262509346,
                "emotion": "Calmness",
                "type": "voice"
            },
            {
                "difference": 0.41626258194446564,
                "emotion": "Concentration",
                "type": "voice"
            },
            {
                "difference": 0.3203507959842682,
                "emotion": "Determination",
                "type": "voice"
            },
            {
                "difference": -0.2709709294140339,
                "emotion": "Joy",
                "type": "voice"
            }
        ]
    }'''
print(get_difference_summary(jsony))
#print(openai.Embedding.create(input = ['work for them'], model="text-embedding-ada-002")['data'][0]['embedding'])
'''
print('\n\nParent\n\n')
print(get_embedding('parent'))
print('\n\nChildren\n\n')
print(get_embedding('children'))
print('\n\nFuture of our children is doomed.\n\n')
print(get_embedding('Future of our children is doomed.'))
print('\n\n.')
print('\n\nHow we treat our kids determines our future.\n\n')
print(get_embedding('How we treat our kids determines our future.'))
print('\n\n.')
print('\n\nWe must think as parents to best determine the future.\n\n')
print(get_embedding('We must think as parents to best determine the future.'))'''


from flask import Markup
import psycopg2
import modules.config as c
from modules.categorization import *
from sqlalchemy import Column, Integer, String, create_engine, or_, and_, any_

from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker

engine = create_engine(c.ENGINE, echo=False)
Base = declarative_base()
session = sessionmaker(bind=engine)
s = session()

class Jobs(Base):
	__tablename__ = c.TABLE
	id = Column(Integer, primary_key=True)
	title = Column(String, nullable=False)
	url = Column(String)
	company = Column(String)
	exp = Column(Integer)
	salary = Column(Integer)
	description = Column(String)
	img = Column(String)
	date = Column(String)
	links = Column(String)

def search(search_terms, exp_in, tag_list, salary_in, nullzp, sources, sort_by):
	jobs = s.query(Jobs.url, Jobs.title, Jobs.company, Jobs.salary, Jobs.description, Jobs.img, Jobs.date, Jobs.id, Jobs.exp, Jobs.links).all()
	job_list = []
	if nullzp == 1:
		q = s.query(Jobs).filter(or_(Jobs.salary >= salary_in, Jobs.salary == 0)).filter(Jobs.exp <= exp_in).filter(Jobs.url.like(any_(sources))).order_by('date').all()
	else:
		q = s.query(Jobs).filter(and_(Jobs.salary >= salary_in, Jobs.salary != 0)).filter(Jobs.exp <= exp_in).filter(Jobs.url.like(any_(sources))).order_by('date').all()

	for row in q:
		match = detect_category(search_terms, row.title)
		if match > 0:
			if tag_list != "anything":
				if any(tag in (row.title.lower() + row.description.lower()) for tag in tag_list.split(' ')):
					job_list.append([row.url, row.title, row.company, int(row.salary), Markup(row.description), row.date, match, row.img, row.id, row.exp, row.links])
			else:
				job_list.append([row.url, row.title, row.company, int(row.salary), Markup(row.description), row.date, match, row.img, row.id, row.exp, row.links])

	sorted_job_list = sorted(job_list, key=lambda x: x[5], reverse=True)
	return(sorted_job_list)


### Data Model for api data is defined here. ###
import datetime
import pytz

class Mini_Pattern():
    """
        Represents a search result object.
        Values are subset of information found in Pattern.
    """
    def __init__(self, pattern_values):

        self.pattern_id = pattern_values['pattern_id']
        self.name = pattern_values['name']
        self.is_free = pattern_values['is_free']
        self.img_fullsize_url = pattern_values['img_fullsize_url']
        self.img_small_url = pattern_values['img_small_url']

    def __repr__(self):
        """
            Provides helpful representation when object is printed.
        """
        repr_s = "<Pattern pattern_id='{}' name='{}' is_free='{}' img_fullsize_url='{}' img_small_url='{}'>".format(
            self.pattern_id, self.name, self.is_free, self.img_fullsize_url, self.img_small_url)

        return repr_s

    @classmethod
    def as_dict(self, pattern):
        """
            Returns object in dictionary format.
        """
        pattern_dict = {"pattern_id" : pattern.pattern_id, "name" : pattern.name, "is_free": pattern.is_free, "img_fullsize_url": pattern.img_fullsize_url, "img_small_url": pattern.img_small_url}

        return pattern_dict

class Pattern(Mini_Pattern):
    """
        Represents a pattern with detailed information.
    """
    def __init__(self, pattern_values):
        self.pattern_id = pattern_values['pattern_id']
        self.name = pattern_values['name']
        self.author = pattern_values['author']
        self.is_free = pattern_values['is_free']
        self.img_fullsize_url = pattern_values['img_fullsize_url']
        self.img_small_url = pattern_values['img_small_url']
        self.is_clothing = pattern_values['is_clothing']
        self.pattern_type = pattern_values['pattern_type']
        self.is_downloadable = pattern_values['is_downloadable']
        self.url = pattern_values['url']
        self.yardage = pattern_values['yardage_description']
        self.yarn_weight = pattern_values['yarn_weight_description']
        self.gauge = pattern_values['gauge_description']
        self.sizes = pattern_values['sizes_available']
        self.description = pattern_values['notes']
        self.created_at = self.convert_date_time(pattern_values['created_at'])
        self.suggested_yarn = Yarn.get_yarn_list(pattern_values['suggested_yarn_list'])
        self.suggested_needles = Needles.get_needles_list(pattern_values['suggested_needles_list'])

    def convert_date_time(self, date_time_str):
        """
            Converts date_time string to date_time object with UTC timezone.
        """
        date_time_list = date_time_str.split(" -")
        print(date_time_list)
        date_time_str = date_time_list[0]
        print(date_time_str)

        date_time_obj = datetime.datetime.strptime(date_time_str, '%Y/%m/%d %H:%M:%S')
        utc_date_time = date_time_obj.astimezone(pytz.utc)

        return utc_date_time

    def __repr__(self):
        """
            Provides helpful representation when object is printed.
        """
        repr_s = "<Pattern pattern_id='{}' name='{}' author='{}' is_free='{}' img_fullsize_url='{}' img_small_url='{}' is_clothing='{}' pattern_type='{}' is_downloadable='{}' url='{}' yardage='{}' yarn_weight='{}' gauge='{}' sizes='{}' description='{}' created_at='{}'".format(
            self.pattern_id, self.name, self.author, self.is_free, self.img_fullsize_url,
            self.img_small_url, self.is_clothing, self.pattern_type, self.is_downloadable,
            self.url, self.yardage, self.yarn_weight, self.gauge, self.sizes, self.description, self.created_at)

        return repr_s

    @classmethod
    def as_dict(self, pattern):
        """
            Returns object in dictionary format.
        """
        suggested_yarn_dict = Yarn.get_yarn_dicts(pattern.suggested_yarn)

        needles_dict = Needles.get_needles_dicts(pattern.suggested_needles)

        pattern_dict = {
            "pattern_id" : pattern.pattern_id,
            "name" : pattern.name,
            "author" : pattern.author,
            "is_free": pattern.is_free,
            "img_fullsize_url": pattern.img_fullsize_url,
            "img_small_url": pattern.img_small_url,
            "is_clothing" : pattern.is_clothing,
            "pattern_type" : pattern.pattern_type,
            "is_downloadable" : pattern.is_downloadable,
            "url" : pattern.url,
            "yardage" : pattern.yardage,
            "yarn_weight" : pattern.yarn_weight,
            "gauge" : pattern.gauge,
            "sizes" : pattern.sizes,
            "description" : pattern.description,
            "created_at" : pattern.created_at,
            "suggested_yarn" : suggested_yarn_dict,
            "needles" : needles_dict
            }

        return pattern_dict

class Yarn():
    """
        Represents a yarn object.
    """
    def __init__(self, name, weight):
        self.name = name
        self.weight = weight

    def __repr__(self):
        """
            Provides helpful representation when object is printed.
        """
        repr_s = "<Yarn name='{}' weight='{}'>".format(self.name, self.weight)
        return repr_s

    @classmethod
    def get_yarn_list(self, suggested_yarn_list):
        """
            Returns list with yarn objects given dictionaries with yarn information.
        """
        yarn_list = []

        for yarn in suggested_yarn_list:
            yarn_name = yarn['yarn_name']
            yarn_weight = yarn['yarn_weight']

            yarn = Yarn(yarn_name, yarn_weight)

            yarn_list.append(yarn)

        return yarn_list

    @classmethod
    def get_yarn_dicts(self, yarn_list):
        """
            Returns dictionaries with yarn information given list with yarn objects.
        """
        yarn_dict_list = []

        for yarn in yarn_list:
            yarn_dict = Yarn.as_dict(yarn)

            yarn_dict_list.append(yarn_dict)

        return yarn_dict_list

    @classmethod
    def as_dict(self, yarn):
        """
            Returns object in dictionary format.
        """
        yarn_dict ={
            "name" : yarn.name,
            "weight" : yarn.weight
        }

        return yarn_dict

class Needles():
    """
        Represents a needles object.
    """
    def __init__(self, name):
        self.name = name

    def __repr__(self):
        """
            Provides helpful representation when object is printed.
        """
        repr_s = "<Needles name='{}'>".format(self.name)

        return repr_s

    @classmethod
    def get_needles_list(self, suggested_needles_list):
        """
            Returns list with needles objects given dictionaries with needles information.
        """
        needles_list = []

        for needles in suggested_needles_list:
            needles_name = needles['name']

            needles = Needles(needles_name)

            needles_list.append(needles)

        return needles_list

    @classmethod
    def get_needles_dicts(self, needles_list):
        """
            Returns dictionaries with needles information given list with yarn objects.
        """
        needles_dict_list = []

        for needles in needles_list:
            needles_dict = Needles.as_dict(needles)

            needles_dict_list.append(needles_dict)

        return needles_dict_list

    @classmethod
    def as_dict(self, needles):
        """
            Returns object in dictionary format.
        """
        needles_dict ={
            "name" : needles.name
        }

        return needles_dict


#%% Change working directory from the workspace root to the ipynb file location. Turn this addition off with the DataScience.changeDirOnImportExport setting
# ms-python.python added
import os
try:
	os.chdir(os.path.join(os.getcwd(), 'Main'))
	print(os.getcwd())
except:
	pass
#%% [markdown]
# ## Challenge
# Oh dear, so this did seem not to work very well. In fact it is remarkably poor. Now there are many things that we could do here.
#
# Firstly the overfit is a problem, even though it was poor in the first place. We could go back and clean up our feature set. There might be some gains to be made by getting rid of the noise.
#
# We could also see how removing the nulls but including dietary information performs. Though its a slight change to the question we could still possibly get some improvements there.
#
# Lastly, we could take our regression problem and turn it into a classifier. With this number of features and a discontinuous outcome, we might have better luck thinking of this as a classification problem. We could make it simpler still by instead of classifying on each possible value, group reviews to some decided high and low values.
#
# __And that is your challenge.__
#
# Transform this regression problem into a binary classifier and clean up the feature set. You can choose whether or not to include nutritional information, but try to cut your feature set down to the 30 most valuable features.
#
# Good luck!

#%%
import warnings
from warnings import simplefilter
simplefilter(action='ignore', category=FutureWarning)
import numpy as np
import pandas as pd
import scipy
import matplotlib.pyplot as plt
import seaborn as sns
get_ipython().run_line_magic('matplotlib', 'inline')

#%%
raw_data = pd.read_csv('https://tf-assets-prod.s3.amazonaws.com/tf-curric/data-science/epi_r.csv')

#%%
#list(raw_data.columns)

#%%
raw_data.sample(5)

#%%
raw_data.rating.describe()

#%%
raw_data.rating.hist(bins=20)
plt.title('Histogram of Recipe Ratings')
plt.show()

#%%
# Count nulls
null_count = raw_data.isnull().sum()
null_count[null_count>0]

#%%
from sklearn.svm import SVR
svr = SVR()
X = raw_data.drop(['rating', 'title', 'calories', 'protein', 'fat', 'sodium'], 1).sample(frac=0.3, replace=True, random_state=1)
Y = raw_data.rating.sample(frac=0.3, replace=True, random_state=1)

#%%
from sklearn.model_selection import cross_val_score
cross_val_score(svr, X, Y, cv=5)

#%%
# Transform rating into categorical variable
raw_data['rating'] = np.where(np.logical_and(raw_data['rating'] >= 0.0, raw_data['rating'] < 1.0 ), 0, raw_data['rating'])
raw_data['rating'] = np.where(np.logical_and(raw_data['rating'] >= 1.0, raw_data['rating'] < 2.0 ), 1, raw_data['rating'])
raw_data['rating'] = np.where(np.logical_and(raw_data['rating'] >= 2.0, raw_data['rating'] < 3.0 ), 2, raw_data['rating'])
raw_data['rating'] = np.where(np.logical_and(raw_data['rating'] >= 3.0, raw_data['rating'] < 4.0 ), 3, raw_data['rating'])
raw_data['rating'] = np.where(np.logical_and(raw_data['rating'] >= 4.0, raw_data['rating'] < 5.0 ), 4, raw_data['rating'])
raw_data['rating'] = np.where(np.logical_and(raw_data['rating'] >= 5.0, raw_data['rating'] < 6.0 ), 5, raw_data['rating'])
raw_data['rating'].head(5)

#%%
from sklearn.svm import SVC
svc = SVC()
X = raw_data.drop(['rating', 'title', 'calories', 'protein', 'fat', 'sodium'], 1)
Y = raw_data.rating

#%%
cross_val_score(svc, X, Y, cv=5)

#%% [markdown]
# When you've finished that, also take a moment to think about bias. Is there anything in this dataset that makes you think it could be biased, perhaps extremely so?
#
# There is. Several things in fact, but most glaringly is that we don't actually have a random sample. It could be, and probably is, that the people more likely to choose some kinds of recipes are more likely to give high reviews.
#
# After all, people who eat chocolate _might_ just be happier people.