#!/usr/bin/env python
# -*- coding: utf-8; py-indent-offset:4 -*-
"""
featurelib
==========
Date time features.

"""

from pathlib import Path

import numpy as np
import pandas as pd
import re

from pandas import DataFrame
from dataclasses import dataclass

from .core import FtAbstract

@dataclass()
class DateTimeFt(FtAbstract):
    """Date time manipulation"""

    def add_datepart_(self, df: DataFrame, fldname: str, drop=True, time=False):
        """Helper function that adds columns relevant to a date.

        Arguments:
            df {DataFrame} -- Pandas data frame.
            fldname {string} -- Field name.

        Keyword Arguments:
            drop {bool} -- Drop the date field (default: {True})
            time {bool} -- Time part (default: {False})
        """
        fld = df[fldname]
        fld_dtype = fld.dtype
        if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):
            fld_dtype = np.datetime64

        if not np.issubdtype(fld_dtype, np.datetime64):
            df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)
        targ_pre = re.sub('[Dd]ate$', '', fldname)
        attr = ['Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear',
                'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']
        if time:
            attr = attr + ['Hour', 'Minute', 'Second']
        for n in attr:
            df[targ_pre + n] = getattr(fld.dt, n.lower())
        df[targ_pre + 'Elapsed'] = fld.astype(np.int64) // 10 ** 9
        if drop:
            df.drop(fldname, axis=1, inplace=True)

    def get_elapsed_(self, df: DataFrame, fld, pre):
        """Get elapsed between 2 fields.
        """
        day1 = np.timedelta64(1, 'D')
        last_date = np.datetime64()
        last_store = 0
        res = []

        for s, v, d in zip(df.Store.values, df[fld].values, df.Date.values):
            if s != last_store:
                last_date = np.datetime64()
                last_store = s
            if v:
                last_date = d
            res.append(((d-last_date).astype('timedelta64[D]') / day1))
        df[pre+fld] = res


'''
    The RESTful style api server
'''
from flask import render_template
from flask import Flask, request, redirect, jsonify, send_from_directory, make_response
from flask_cors import *
from server import app
from functools import wraps
import pandas as pd
import hashlib
import random
import time
import datetime
import os
import json
import requests
import subprocess
from math import sqrt
from server.dataprocessing.sentenceCls import SentenceEmbedder as se
from server.dataprocessing import featureExtractor as fe
import torch
import transformers as ppb
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import MultinomialNB, GaussianNB
from sklearn.neighbors import KNeighborsClassifier
import time
import re
import numpy as np
sentenceEmbedder = se()

import sklearn
print("Sklearn verion is {}".format(sklearn.__version__))

import joblib

def load_models():
    dir_path = os.path.dirname(os.path.realpath(__file__))
    # argument model
    with open(os.path.join(dir_path, "../models/argument_model.joblib"), 'rb') as f:
        arg_model = joblib.load(f)

    with open(os.path.join(dir_path, "../models/ml_argument.joblib"), 'rb') as f:
        ml_arg = joblib.load(f)

    # claim type model
    with open(os.path.join(dir_path, "../models/claim_model.joblib"), 'rb') as f:
        claim_model = joblib.load(f)

    with open(os.path.join(dir_path, "../models/ml_claims.joblib"), 'rb') as f:
        ml_claim = joblib.load(f)

    # premise type model
    with open(os.path.join(dir_path, "../models/premise_model.joblib"), 'rb') as f:
        premise_model = joblib.load(f)

    with open(os.path.join(dir_path, "../models/ml_premises.joblib"), 'rb') as f:
        ml_premise = joblib.load(f)

    # relationship model
    with open(os.path.join(dir_path, "../models/relations_mnodel.joblib"), 'rb') as f:
        relation_model = joblib.load(f)

    with open(os.path.join(dir_path, "../models/relation_len.joblib"), 'rb') as f:
        relation_len = joblib.load(f)

    return [[arg_model, ml_arg], [claim_model, ml_claim], [premise_model, ml_premise], [relation_model, relation_len]]
all_models = load_models()

def run_models(sentence):
    ## calculate eloquence
    errors = fe.find_sentence_errors(sentence)
    try:
        # print("query labels:", all_models)
        # argumentation
        ## non-argument: 0, claim: 1, premise: 2
        a_model = all_models[0][0]
        ml_arg = all_models[0][1]
        a_f = sentenceEmbedder.encode_one(sentence, ml_arg)
        a_label = a_model.predict(a_f)[0]
        print("prediction results:", a_model.predict(a_f))

        if a_label == 0:
            return {
                "content": sentence,
                "is_claim": 0,
                "claim_type": "0",
                "logos": 0,
                "pathos": 0,
                "evidence": 0,
                "relevance": 0,
                "ethos": 0,
                "concreteness": -1,
                "eloquence": errors[0],
                "elo_info": errors,
            }
        elif a_label == 1:
            # claim_type model
            c_model = all_models[1][0]
            ml_claim = all_models[1][1]
            c_f = sentenceEmbedder.encode_one(sentence, ml_claim)
            c_label = c_model.predict(c_f)[0]
            print("prediction results claim type:", c_model.predict(c_f))
            # interpretation: 1, evaluation: 2, disagreement/agreement: 0
            # convert to claim_type
            claim_type = "0"
            if c_label == 1:
                claim_type = "interpretation"
            elif c_label == 2:
                claim_type = "evaluation"
            else:
                claim_type = "disagreement"
            return {
                "content": sentence,
                "is_claim": 1,
                "claim_type": claim_type,
                "logos": 0,
                "pathos": 0,
                "evidence": 0,
                "relevance": 0,
                "ethos": 0,
                "concreteness": -1,
                "eloquence": errors[0],
                "elo_info": errors,
            }
        elif a_label == 2:
            p_model = all_models[2][0]
            ml_premise = all_models[2][1]
            p_f = sentenceEmbedder.encode_one(sentence, ml_premise)
            p_label = p_model.predict(p_f)[0]
            print("prediction results premises:", p_model.predict(p_f))
            # p_label: [logos, pathos, evidence, relevance, ethos]
            return {
                "content": sentence, #+ " .",
                "is_claim": 0,
                "claim_type": "0",
                "logos": int(p_label[0]),
                "pathos": int(p_label[1]),
                "evidence": int(p_label[2]),
                "relevance": int(p_label[3]),
                "ethos": int(p_label[4]),
                "concreteness": -1,
                "eloquence": int(errors[0]),
                "elo_info": errors,
            }

    except Exception as e:
        print("---model issues---")
        print(e)
        return {
            "content": sentence, # + " .",
            "is_claim": 0,
            "claim_type": "0",
            "logos": 0,
            "pathos": 0,
            "evidence": 0,
            "relevance": 0,
            "ethos": 0,
            "concreteness": -1,
            "eloquence": int(errors[0]),
            "elo_info": errors,
        }

def run_relationship(sentences):
    # sentence relationship only computed when there are more than two sentences
    if len(sentences) >= 2:
        claims = []
        claim_id = []
        supports = []
        support_id = []
        for senid, sentence in enumerate(sentences):
            is_claim = sentence["is_claim"]
            # add claims
            if is_claim == 1:
                claims.append(sentence["content"])
                claim_id.append(senid)
            # add premises
            if sentence["logos"] + sentence["pathos"] + sentence["evidence"] + sentence["relevance"] + sentence["ethos"] >0:
                supports.append(sentence["content"])
                support_id.append(senid)
        # claim-support-pairs
        cspairs = []
        final_pairs = []
        ### HARCODE: if no claim, then 1st sentence will be the claim
        if len(claim_id) == 0:
            claim_id = [0]
        ###
        if len(claim_id) > 0 and len(support_id) > 0:
            for cid in claim_id:
                for sid in support_id:
                    if cid != sid: # claim != premise/supports
                        cspairs.append([cid, sid])
            #
            # print(cspairs)
            # if not empty, run relationship
            rmodel = all_models[3][0]
            rmodel_len = all_models[3][1]
            senEm = []
            for spair in cspairs:
                sen0 = sentences[spair[0]]["content"]
                sen1 = sentences[spair[1]]["content"]
                em0 = sentenceEmbedder.encode_one(sen0, rmodel_len)
                em1 = sentenceEmbedder.encode_one(sen1, rmodel_len)
                em = em0.tolist()[0] + em1.tolist()[0]
                senEm.append(em)
            # run pretrained model
            relation_scores = rmodel.predict(senEm)
            reidxs = np.where(relation_scores == 0)
            final_pairs = [cspairs[reidx] for reidx in reidxs[0]]

            # print("relationship score:",relation_scores, np.where(relation_scores == 0))

        # print("final pairs:", final_pairs)
        print("relationship pairs", final_pairs)
        return final_pairs
    else:
        return []

# run_models("hello")
# exit()

# app = Flask(__name__)
# CORS(app, supports_credentials=True)

@app.route('/')
def index():
    return render_template('index.html')

@app.route("/test", methods=['POST','GET'])
def register():
    print('hello world!')
    return 'get your request!'

@app.route('/uploadInput', methods=['POST','GET'])
def uploadInput():
    # userid and texts
    # # usid = ""
    # # temp = ""
    # reg = re.compile('<[^>]*')
    # content = reg.sub('', request)

    response = json.loads(request.data.decode("utf-8"))
    print("response in the backend",response)
    txt = response["input"]["content"]
    userid = response["input"]["userid"]

    ### TODO: add more splitting operations
    sen_idxs = [m.start() for m in re.finditer(r'[\.\?\!]+', txt)]
    ini_idx = 0
    all_results = []

    for idx in sen_idxs:
        sen_txt = txt[ini_idx: idx+1]
        ini_idx = idx + 1
        if len(sen_txt.strip()) > 0:
            results = run_models(sen_txt)
            all_results.append(results)

    # --- OLD version
    # sentence_list = [s.strip() for s in re.split(r'[\.\?\!]+', txt) if len(s.strip()) > 0]
    # print(sentence_list)
    # all_results = []

    # for senid, sentence in enumerate(sentence_list):
    #     print("sentence:", sentence)
    #     results = run_models(sentence)
    #     all_results.append(results)

    ### OLD one: have BUG
    # pattern = r'\.|/|\'|`|\[|\]|<|>|\?|:|\{|\}|\~|!||\(|\)|-|=|\_|、|；|‘|’|【|】|·|…'
    # pattern = r'\.|/|\'|`|\[|\]|<|>|\?|:|\{|\}|\~|!||\(|\)|-|=|\_|、|；|‘|’|【|】|·|…'
    # sentence_list = re.split(pattern, txt)
    # for index_sentence in range(0, len(sentence_list)-1): # split re without the last one
    #     results = run_models(sentence_list[index_sentence])
    #     all_results.append(results)

    # run relation models
    relationship_pairs = run_relationship(all_results)
    print("all_results:", all_results)
    print("relationship_pairs:", relationship_pairs)

    # save results
    dir_path = os.path.dirname(os.path.realpath(__file__))
    user_results_name = userid + str(time.time()) + ".json"
    with open(os.path.join(dir_path, "../dataset/user_inputs/"+user_results_name), "w") as f:
        json.dump({
        "results": all_results,
        "relationships": relationship_pairs}, f)

    return json.dumps({
        "results": all_results,
        "relationships": relationship_pairs
    })


from appium import webdriver
import pytest

@pytest.fixture(scope="session")
def setup_driver():

    desired_caps = {
        'platformName': 'Android',
        'deviceName': 'Android Emulator',
        'app': 'app-debug.apk',
        "appPackage": "ru.netology.testing.uiautomator"
    }

    driver = webdriver.Remote("http://127.0.0.1:4723/wd/hub", desired_caps)
    yield driver

    driver.quit()


# tests/test_expense.py

import json
from typer.testing import CliRunner
import pytest

from tracker import (
        DB_READ_ERROR,
        SUCCESS,
        __app_name__,
        __version__,
        cli,
        tracker,
)

runner = CliRunner()

def test_version():
    result = runner.invoke(cli.app, ["--version"])
    assert result.exit_code == 0
    assert "%s v %s" % (__app_name__,__version__) in result.stdout

@pytest.fixture
def mock_json_file(tmp_path):
    todo = [{
        "Description": "some milk.",
        "amount": 2.0,
        "Done": False
        }]
    db_file = tmp_path / "todo.json"
    with db_file.open("w") as db:
        json.dump(todo, db, indent=4)
    return db_file

test_data1 = {
        "description": ["Clean", "the", "house"],
        "amount": 1.0,
        "todo": {
            "Description": "Clean the house.",
            "amount": 1.0,
            "Done": False,
            },
}

test_data2 = {
        "description" : ["Wash the car"],
        "amount": 2.0,
        "todo": {
            "Description": "Wash the car.",
            "amount": 2.0,
            "Done": False,
        },
}

@pytest.mark.parametrize(
        "description, amount, expected",
        [
            pytest.param(
                test_data1["description"],
                test_data1["amount"],
                (test_data1["expense"],SUCCESS),
                ),
            pytest.param(
                test_data2["description"],
                test_data2["amount"],
                (test_data2["todo"], SUCCESS),
                ),
        ],
)

def test_add(mock_json_file, description, amount, expected):
    expenser = tracker.Tracker(mock_json_file)
    assert expenser.add(description, amount) == expected
    read = expenser._db_handler.read_expense()
    assert len(read.todo_list) == 2


import pprint
from pydantic import BaseModel, Field
from github import GithubException, BadCredentialsException, UnknownObjectException

class InputSchema(BaseModel):
    organization_name: str = Field(
        description='Name of Github Organization. Eg: "unskript"',
        title='Organization Name',
    )
    username: str = Field(
        description='Organization member\'s username. Eg: "jane-mitch-unskript"',
        title='Member\'s Username',
    )

def github_remove_member_from_org_printer(output):
    if output is None:
        return
    pprint.pprint(output)

def github_remove_member_from_org(handle, organization_name:str, username:str)-> str:
    """github_remove_member_from_org returns the status to remove a member

        :type organization_name: string
        :param organization_name: Name of Github Organization. Eg: "unskript"

        :type username: string
        :param username: Organization member's username. Eg: "jane-mitch-unskript"

        :rtype: List of return status of removing a member from Org
    """
    organization = handle.get_organization(organization_name)
    try:
        user = handle.get_user(username)
        mem_exist = organization.has_in_members(user)
        if mem_exist:
            result = organization.remove_from_members(user)
    except GithubException as e:
        if e.status == 403:
            raise BadCredentialsException("You need admin access") from e
        if e.status == 404:
            raise UnknownObjectException("No organization or user found") from e
        raise e.data
    except Exception as e:
        raise e
    if result is None:
        return f"Successfully removed user {username}"
    return None


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @Time    : 2020/11/9 16:47
# @Author  : wyb
# @Site    :
# @File    : cls_data.py 某联社主页网站sign
# @Software: PyCharm
import sys
import os

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(BASE_DIR)

import requests
import time

from 新手入门.encrypt_utils import md5, hmac_sha1

# 目标网站 aHR0cHM6Ly93d3cuY2xzLmNuL3RlbGVncmFwaA==

def params_sort(content):
    """字符串排序"""
    encrypt_list = [each for each in content.split('&') if "sign" not in each]
    temp_sort = sorted(encrypt_list)
    result = '&'.join(temp_sort)
    return result

def get_sign(source_url):
    encrypt_str = source_url[source_url.index('?') + 1:]
    return md5(hmac_sha1(encrypt_str))

def get_roll_list():
    url = "https://www.cls.cn/v1/roll/get_roll_list?app=CailianpressWeb&category=jpush&last_time={}&os=web&refresh_type=1&rn=20&sv=7.2.2".format(
        int(time.time()))
    sign = get_sign(url)
    print("获取sign: ", sign)
    target_url = url + "&sign=" + sign
    payload = {}
    headers = {
        'Connection': 'keep-alive',
        'Accept': 'application/json, text/plain, */*',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36',
        'Content-Type': 'application/json;charset=utf-8',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Dest': 'empty',
        'Referer': 'https://www.cls.cn/telegraph',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cookie': 'HWWAFSESID=4d99da9ede9500e5a9; HWWAFSESTIME=1604910723785; hasTelegraphNotification=on; hasTelegraphRemind=on; hasTelegraphSound=on; vipNotificationState=on; Hm_lvt_fa5455bb5e9f0f260c32a1d45603ba3e=1603092968,1604555395,1604910728; Hm_lpvt_fa5455bb5e9f0f260c32a1d45603ba3e=1604913124'
    }

    response = requests.get(target_url, headers=headers, data=payload)
    print(response.text)

if __name__ == '__main__':
    for i in range(2):
        get_roll_list()


from abc import ABC

import discord
import riotwatcher

import APICommands.Command
from APICommands.Command import intTryParse, championIdToName, getChampionsJSON, getSummonerNameFromMessage

class HighestMastery(APICommands.Command.Command, ABC):
    commandName = "Highest mastery"
    keywords = ["highestmastery", "highestMastery", "HM", "hm", "Hm", "HighestMastery"]

    def __init__(self, pref, api: riotwatcher.LolWatcher, region: str, additionalKeywords: list):
        if additionalKeywords is None:
            additionalKeywords = []
        super().__init__(pref, api, region, additionalKeywords)

    async def execute(self, message: discord.Message):
        sumname = ""
        firstIsInt = intTryParse(message.content.split(" ")[1])[1]

        try:
            sumname = getSummonerNameFromMessage(message, 2 if firstIsInt else 1)
        except:
            await self.usage(message)
        if sumname != "":
            if not await self.checkSumname(sumname, message):
                return
            listlen = int(message.content.split(" ")[1]) if firstIsInt else 10
            try:
                output = self.getChampionMasteryList(sumname, listlen)
                for i in output:
                    await message.channel.send(i)
            except:
                await self.usage(message)

    async def info(self, message: discord.Message):
        pass

    async def usage(self, message: discord.Message):
        await message.channel.send("Wrong usage of" + self.commandName + "!\nUsage: " + self.pref + "hm [count] ["
                                                                                "Summonername]")

    def getChampionMasteryList(self, sumname, listlen):
        output = ["Der Spieler " + sumname + " hat den höchsten Mastery auf diesen " + str(
            listlen) + " Champions\n"]
        count = 0
        response = self.api.champion_mastery.by_summoner(self.region, self.getEncryptedSummonerID(sumname))[
                   :listlen]
        championsJSON = getChampionsJSON()
        for i in response:
            champname = championIdToName(i["championId"], championsJSON)
            mpoints = i["championPoints"]
            mastery = i["championLevel"]
            out = "**" + champname + "**" + " Points: " + str(mpoints) + " Level: " + str(
                mastery) + "\n"
            if len(output[count]) + len(out) >= 2000:
                output.append("")
                count += 1
            output[count] += out
        return output


import aiohttp.server
import asyncio
import time
from concurrent.futures import ProcessPoolExecutor

def slow_func():
  time.sleep(10)
  return "test"

class HttpProtocol(aiohttp.server.ServerHttpProtocol):

    def __init__(self, loop, ppexecutor):
        super(HttpProtocol, self).__init__(loop=loop, debug=True)
        self.loop = loop
        self.ppexecutor = ppexecutor
        #print(ppexecutor)

    async def handle_request(self, message, payload):
# ppexecutor is a ProcessPoolExecutor instance
        print('Handle request', message)
        data = await self.loop.run_in_executor(None, slow_func)
        response = aiohttp.Response(self.writer, 200, http_version=message.version, close=True)

        response.add_headers(
          ('Content-Type', 'text'),
          ('Content-Length', str(len(data)))
        )
        response.send_headers()
        response.write(data.encode())
        response.write_eof()
        self.keep_alive(False)

def main():
    pp = ProcessPoolExecutor(3)
    loop = asyncio.get_event_loop()
    myserver =  HttpProtocol(loop=loop, ppexecutor=pp)
    f = loop.create_server(
            myserver, '127.0.0.1', 8080)
    svr = loop.run_until_complete(f)
    socks = svr.sockets
    print('serving on', socks[0].getsockname())
    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2016 Massachusetts Institute of Technology

"""Extract images from a rosbag.
"""
import sys
import os
import argparse

import cv2

import rosbag
from sensor_msgs.msg import Image
from cv_bridge import CvBridge

def main():

    """Extract tipics: from a rosbag.
    """
    bag_file = sys.argv[1]
    bag = rosbag.Bag(bag_file, "r")
    bridge = CvBridge()
    count = 0
    types = []
    topic_lis = []
    topics = bag.get_type_and_topic_info()[1].keys()
    print( "extracting topics")
    for i in range(0,len(bag.get_type_and_topic_info()[1].values())):
        types.append(bag.get_type_and_topic_info()[1].values()[i][0])
    for i in types:
            if i == "sensor_msgs/Image":
               topic_lis.append( topics[types.index(i)]  )
    print(topic_lis)
    f = open("topiclist.txt", "a")
    f.truncate(0)
    f.write('\n'.join(topic_lis))
    f.close()
    print("extraction done")
    return

if __name__ == '__main__':
    main()


import xml.etree.ElementTree as ET
import xmltodict
import os
cwd = os.getcwd()
print("cwd {0}".format(cwd))
fName = "export1.xml"
filePath = os.path.join(cwd, fName)
print("filePath {0}".format(filePath))

#read as string
with open(filePath) as xmlFile:
	dataGPS = xmlFile.read().replace("\n", "")
#print("dataGPS {0}".format(dataGPS))
xmlDict = xmltodict.parse(dataGPS)
print("xmlDict {0}".format(xmlDict))

tree = ET.parse(filePath)
root = tree.getroot()
print("dir root {0}".format(dir(root)))
#print("root.items {0}".format(root.items()))
print("root.getchildren {0}".format(root.getchildren()))
print("root.iterfind {0}".format(root.iterfind('trk')))
for item in root.iterfind('trk'):
	print("iterfind tag {0}, iterfind attrib {1}".format(item.tag, item.attrib))
print("root.tag {0}, root.attrib {1}".format(root.tag, root.attrib))

for i, child in enumerate(root.iter()):
	print("child.findall {0}".format(child.findall('trkpt]')))
	print("root child {0:3>0} - {1}:{2}".format(i, child.tag, child.attrib))
	name = child.findall("name")
	trkseg = child.findall("trkseg")
	print("name {0}".format(name))
	print("trkseg {0}".format(trkseg))
	for j, chChild in enumerate(child.iter()):
		print("{0}-child child {0:3>0} - {1}:{2}".format(j, chChild, chChild))
		print("chChild.findall {0}".format(chChild.findall('trk')))
		for k, chChChild in enumerate(chChild):
			#name = chChild.find("name").text
			lat = chChChild.get("lat")
			lon = chChChild.get("lon")
			print("{0:0>3}\nlat {1}\nlon {2}".format(k,lat, lon))
			#print("{0}-child child child {0:3>0} - {1}:{2}".format(k, chChChild, chChChild))

name = root.findall('trk')
#print("iter trk {0}".format(trk.attrib))
print("name:{0}".format(root.findall("name")))

#for trkpt in root.iter('trkpt'):
#	lat = trkpt.get("lat")
#	lon = trkpt.get("lon")
#	print("trkpt.attrib {0}".format(trkpt.attrib))
#	#print("{0:3>0 - lat={1}:lon={2}".format(i, lat, lon))

for name in root.findall('name'):
	print("name.text {0}".format(name.text))
	#print("{0:3>0 - lat={1}:lon={2}".format(i, lat, lon))

fName = "test.xml"
filePath = os.path.join(cwd, fName)
print("filePath {0}".format(filePath))
tree = ET.parse(filePath)
root2 = tree.getroot()
print("root2.tag {0}, root2.attrib {1}".format(root2.tag, root2.attrib))
for rank in root2.iter('rank'):
	print("rank {0}".format(rank.text))
trkpts = root.findall('trkpt')
print("trkpts {0} - len {1}".format(trkpts, len(trkpts)))
for item in root.iter('trkpt'):
	print("iterfind tag {0}, iterfind attrib {1}".format(item.tag, item.attrib))

def getElementToAppend(inCountry, **kwargs):
	rank = kwargs['rank'] if 'rank' in kwargs else None
	year = kwargs['year'] if 'year' in kwargs else None
	neighbors = kwargs['neighbors'] if 'neighbors' in kwargs else None
	gdppc = kwargs['gdppc'] if 'gdppc' in kwargs else None
	newEl = ET.Element("country", {"name":inCountry})
	ET.SubElement(newEl, "rank").text="{0}".format(rank) if rank else None
	ET.SubElement(newEl, "year").text = "{0}".format(year) if year else None
	ET.SubElement(newEl, "gdppc").text="{0}".format(gdppc) if gdppc else None
	if neighbors:
		for neighbor in neighbors:
			ET.SubElement(newEl, "neighbor", {"name":neighbor["name"], "direction":neighbor["direction"]})
	return newEl

neighborsHu = [{"name":"Slovakia", "direction":"N"}, {"name":"Austria", "direction":"W"}]
newEl = getElementToAppend("Hungary", rank=20, year=2000, gdppc=15654, neighbors=neighborsHu)
root2.append(newEl)
for country in root2.findall('country'):
	print("appended {0}".format(country.get("name")))
tree.write("output.xml")

fName = "ÚV Želivka 3.stavba.xml"
filePath = os.path.join(cwd, fName)
print("filePath {0}".format(filePath))
tree = ET.parse(filePath)
root3 = tree.getroot()
print("root3.tag {0}, root3.attrib {1}".format(root3.tag, root3.attrib))
for check in root3.findall("./Heading/Section/Check"):
	print("{0}".format(check.get('CheckName')))

##read as string
#with open(filePath) as xmlFile:
#	data = xmlFile.read().replace("\n", "")

#print("data {0}".format(data))
#xmlDict = xmltodict.parse(data)
#print("xmlDict {0}".format(xmlDict))


import logging
import threading
from BotThreader import BotThreader

from Dataclasses import ModuleStatus

class TwitchChat:
    _bot = None

    def __init__(
        self,
        access_token,
        client_id,
        client_secret,
        *args,
        **kwargs,
    ):
        self.access_token = access_token
        self.client_id = client_id
        self.client_secret = client_secret
        self._status = ModuleStatus.IDLE
        self.module_name = "twitch_chat"

    def set_status(self, status: ModuleStatus):
        self._status = status

    def get_status(self):
        return self._status

    async def start(self, *args, **kwargs):
        self.set_status(ModuleStatus.RUNNING)
        self.threader = BotThreader()  # Store the BotThreader instance
        self.set_bot(self.threader.create_bot())

        bot_run_thread = threading.Thread(
            target=self.threader.run_bot, args=(self._bot,)
        )
        bot_run_thread.start()
        self.bot_thread = bot_run_thread

    async def stop(self, *args, **kwargs):
        logging.debug("TwitchChat stop initiated")
        self.set_status(ModuleStatus.STOPPING)

        self.threader.stop_bot(
            self._bot
        )  # Use the stored BotThreader instance

        self.bot_thread.join()

        self.set_status(ModuleStatus.IDLE)

    @classmethod
    def get_bot(cls):
        return cls._bot

    @classmethod
    def set_bot(cls, bot):
        cls._bot = bot

    @classmethod
    async def join_channel(cls, channel):
        await cls._bot.join_channels([channel])

    @classmethod
    async def leave_channel(cls, channel):
        await cls._bot.send_message_to_channel(channel, "Bye, world!")
        await cls._bot.part_channels([channel])
        cls._bot.active_channels.remove(channel)

    @classmethod
    def list_channels(cls):
        logging.info(f"{cls._bot.active_channels}")
        return cls._bot.active_channels


import pytest

@pytest.fixture()
def ca_words():
    return ["cat", "car", "cartoon", "carate"]

@pytest.fixture()
def ta_words():
    return ["tabla", "table", "taboo"]

@pytest.fixture()
def words(ta_words, ca_words):
    return ta_words + ca_words

def test_find(words, trie):
    for word in words:
        assert word in trie

    assert "rofl" not in trie

def test_prefix_not_in_trie(trie):
    assert "ca" not in trie

def test_len(words, trie):
    assert len(trie) == len(words)

def test_iter(trie, words):
    assert sorted(trie) == sorted(words)

def test_complete(ca_words, ta_words, trie):
    assert list(trie.complete("cat")) == ["cat"]
    assert sorted(trie.complete("car")) == sorted(["car", "cartoon", "carate"])
    assert sorted(trie.complete("ca")) == sorted(ca_words)
    assert sorted(trie.complete("ta")) == sorted(ta_words)

@pytest.mark.parametrize(
    ("words", "expected"),
    [
        (["cat"] * 3 + ["car"] * 1 + ["cartoon"] * 2, ["cat", "cartoon", "car"]),
        (["cat"] * 2 + ["car"] * 1 + ["cartoon"] * 3, ["cartoon", "cat", "car"]),
        (["cat"] * 2 + ["car"] * 3 + ["cartoon"] * 1, ["car", "cat", "cartoon"]),
    ],
)
def test_sorted_by_use(trie, expected):
    assert list(trie) == expected
    assert list(trie.complete("ca")) == expected

@pytest.mark.parametrize("words", [["cat"] * 3 + ["car"] * 1 + ["cartoon"] * 2])
def test_query_count(trie):
    assert trie.count("cat") == 3
    assert trie.count("car") == 1
    assert trie.count("cartoon") == 2
    assert trie.count("carate") == 0
    assert trie.count("ca") == 0

@pytest.mark.parametrize(
    ("input_groups", "expected"),
    [
        (["cv", "as", "rt"], ["car", "cartoon", "carate", "cat"]),
        (["atc", "atc", "atc"], ["cat"]),
        (["c", "a", "r"], ["car", "cartoon", "carate"]),
        (["ct", "a", "tb"], ["cat", "tabla", "table", "taboo"]),
    ],
)
def test_ambiguous_completion(trie, input_groups, expected):
    assert sorted(trie.complete(input_groups)) == sorted(expected)

@pytest.mark.parametrize(
    ("words", "input_groups", "expected"),
    [(["cat"] * 3 + ["car"], ["cv", "as", "rt"], ["cat", "car"])],
)
def test_ambiguous_completion_sorted_by_use(trie, input_groups, expected):
    assert list(trie.complete(input_groups)) == expected


#!/usr/bin/python

import sys
import os
import curses

class PresenTerm:

	def __init__(self, screen, dirname='.'):
		self.screen = screen
		self.height, self.width = screen.getmaxyx()
		self.offsety, self.offsetx = -self.height / 2, -self.width / 2
		self.cur = 0
		self.slides = []
		self.dirname = dirname

	def readSlides(self):
		for filename in os.listdir(self.dirname):
			filename = os.path.join(self.dirname, filename)
			if os.path.isfile(filename) and filename.endswith('.slide'):
				with open(filename) as f:
					self.slides.append(f.read())

	def showCounter(self):
		if self.cur < 1:
			self.cur = 1
		if self.cur > len(self.slides):
			self.cur = len(self.slides)
		self.screen.addstr(0, 0, 'presenterm: %(dir)s: %(cur)s of %(count)s' %
			{ 'dir': self.dirname, 'cur': self.cur, 'count': len(self.slides) },
			curses.A_REVERSE)

	def showSlide(self):
		self.screen.addstr(3, 0,
			'%(slide)s' % { 'slide': self.slides[self.cur - 1] },
			curses.A_NORMAL)

	def moveSlide(self, moveTo):
		self.cur = self.cur + moveTo
		self.refresh()

	def refresh(self):
		self.screen.clear()
		self.showCounter()
		self.showSlide()

	def main(self):
		self.readSlides()
		self.refresh()
		while 1:
			key = self.screen.getch()
			if key == ord('q'):
				break
			elif key == curses.KEY_LEFT or key == ord('a'):
				self.moveSlide(-1)
			elif key == curses.KEY_RIGHT or key == ord('d'):
				self.moveSlide(1)

def main(stdscr):
	try:
		slides_dir = os.path.abspath(sys.argv[1])
	except IndexError:
		slides_dir = os.getcwd()
	screen = PresenTerm(stdscr, slides_dir)
	screen.main()

if __name__ == "__main__":
	curses.wrapper(main)


# wx_my_button.py       13Jun2023  crs, From my_button.py
import wx

def ok_cmd(e):
    print("ok_cmd")

def not_ok_cmd(e):
    print("not_ok_cmd")

def quit_cmd(e):
    print("quit_cmd")
    wx.Exit()

app = wx.App()
frame = wx.Frame(None)
frame.Show()
panel = wx.Panel(frame)
btn_height = 50
btn_width = int(1.5 * btn_height)

ok_height = btn_height
ok_width = btn_width

not_ok_height = int(1.5 * btn_height)
not_ok_width = btn_width

quit_height = int(2 * btn_height)
quit_width = btn_width
pos_1 = wx.Point(int(btn_width/2),btn_height)
btn_inc = wx.Size(btn_width,0)
ok_button = wx.Button(panel, label="OK", pos=pos_1,
                      size=(ok_width, ok_height))
ok_button.Bind(wx.EVT_BUTTON, ok_cmd)
ok_button.SetForegroundColour(wx.GREEN)
ok_button.SetBackgroundColour(wx.LIGHT_GREY)

not_ok_pos = pos_1 + btn_inc
not_ok_font = wx.Font(14, wx.FONTFAMILY_DEFAULT, 0,90)
not_ok_button = wx.Button(panel, label="Not OK", pos=not_ok_pos,
                          size=(not_ok_width, not_ok_height))
not_ok_button.SetFont(not_ok_font)
not_ok_button.Bind(wx.EVT_BUTTON, not_ok_cmd)
not_ok_button.SetForegroundColour(wx.BLUE)
not_ok_button.SetBackgroundColour(wx.LIGHT_GREY)

quit_pos = not_ok_pos + btn_inc
quit_font = wx.Font(18, wx.FONTFAMILY_DEFAULT, 0,90)
quit_button = wx.Button(panel, label="QUIT", pos=quit_pos,
                        style=wx.BORDER_RAISED,
                        size=(quit_width, quit_height))
quit_button.SetFont(quit_font)
quit_button.Bind(wx.EVT_BUTTON, quit_cmd)
quit_button.SetForegroundColour(wx.RED)
quit_button.SetBackgroundColour(wx.LIGHT_GREY)

frame.SetSize((350, 250))
frame.SetTitle('wx_my_button')
frame.Centre()

app.MainLoop()
print("After mainloop")


import time
import copy
import datetime

from django.contrib import admin
from django import forms
from django.forms.widgets import HiddenInput
from django.contrib import messages
from django.contrib.admin.util import unquote
from django.template.loader import render_to_string

from admin_locking.exceptions import ObjectChangedException, LockExistsException
from admin_locking import (assert_no_lock_exists,
                           assert_object_not_changed_since)
from admin_locking.api import lock, unlock, log_change

LOCK_ERROR_TEMPLATE_NAME = "admin_locking/lock_error.html"
PAGE_LOAD_TIME_ERROR_TEMPLATE_NAME = "admin_locking/page_load_time_error.html"

def render_error(template_name, request, obj):
    return render_to_string(template_name, dict(
        obj = obj,
        until = (datetime.datetime.fromtimestamp(obj.until)
                 if hasattr(obj, 'until') else None),
        time = (datetime.datetime.fromtimestamp(obj.time)
                 if hasattr(obj, 'time') else None),
        request = request,
    ))

def try_lock(obj, request, show_message=False):
    try:
        lock(obj, request.user)
    except LockExistsException as e:
        if not show_message:
            raise forms.ValidationError(render_error(LOCK_ERROR_TEMPLATE_NAME,
                                               request, e.obj))
        else:
            if request.method == 'GET':
                messages.add_message(request, messages.ERROR,
                    render_error(LOCK_ERROR_TEMPLATE_NAME, request, e.obj))

class LockingAdmin(admin.ModelAdmin):
    class Media():
        js = ('admin_locking/js/admin_locking.js', )

    def get_form(self, request, obj=None, **kwargs):
        form = super(LockingAdmin, self).get_form(request, obj=None, **kwargs)

        if obj is None:
            return form

        def try_load_time(on_exception_do):
            try:
                load_time = request.POST['admin_locking_page_load_time']
                assert_object_not_changed_since(obj, load_time)
            except ObjectChangedException as e:
                on_exception_do(e.obj)

        class ExtendedForm(form):
            admin_locking_page_load_time = forms.FloatField(
                initial=time.time(), widget=HiddenInput())

            def clean(self, *args, **kwargs):
                def page_load_time_error(obj):
                    raise forms.ValidationError(render_error(
                        PAGE_LOAD_TIME_ERROR_TEMPLATE_NAME, request, obj))
                try_lock(obj, request)
                try_load_time(page_load_time_error)
                return super(ExtendedForm, self).clean(*args, **kwargs)

        return ExtendedForm

    def get_fieldsets(self, request, obj=None, *args, **kwargs):
        fieldsets = super(LockingAdmin, self).get_fieldsets(request, obj=obj,
                                                            *args, **kwargs)
        if obj is not None:
            fieldsets = copy.deepcopy(fieldsets)
            fields = fieldsets[0][1]['fields']
            if 'admin_locking_page_load_time' not in fields:
                fields.extend(['admin_locking_page_load_time'])

        return fieldsets

    def save_model(self, request, obj, form, change):
        def assertions(obj, request):
            assert_no_lock_exists(obj, request.user)
            load_time = request.POST['admin_locking_page_load_time']
            assert_object_not_changed_since(obj, load_time)

        if obj.pk is not None:
            assertions(obj, request)
            log_change(obj, request.user)

        return super(LockingAdmin, self).save_model(request, obj, form, change)

    def change_view(self, request, object_id, *args, **kwargs):
        obj = self.get_object(request, unquote(object_id))
        try_lock(obj, request, True)
        return super(LockingAdmin, self).change_view(request, object_id,
                                                     *args, **kwargs)

    def response_change(self, request, obj, *args, **kwargs):
        response = super(LockingAdmin, self).response_change(request, obj,
                                                             *args, **kwargs)
        continuations = ("_continue", "_saveasnew", "_addanother")
        if any(x in request.POST for x in continuations):
            return response

        if obj is not None:
            unlock(obj)

        return response


import os
import cv2
import glob

carpeta = "/home/alvaro.nieva/Documents/nieva_TFM/data/TUM_KITCHEN/cam3"
fps = 25
image_list = os.listdir(carpeta)
image_list.sort()

img_array = []
for filename in image_list:
    name = carpeta + '/' + filename
    img = cv2.imread(name)
    height, width, layers = img.shape
    size = (width,height)
    img_array.append(img)

out = cv2.VideoWriter('./data/tmktch_cam3.avi',cv2.VideoWriter_fourcc(*'DIVX'), fps, size)

for i in range(len(img_array)):
    out.write(img_array[i])
out.release()
print("Done")


import json
import requests
import decimal
import numpy as np
import pandas as pd
from tabulate import tabulate
from decimal import Decimal

import os, sys

import os, sys
from os.path import dirname, realpath
proj_path = dirname(dirname(dirname(realpath(__file__))))

# This is so Django knows where to find stuff.
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
sys.path.append(proj_path)

# This is so my local_settings.py gets loaded.
os.chdir(proj_path)

import django
django.setup()

import matplotlib
matplotlib.use('TkAgg')   # use for OSX

import matplotlib.pyplot as plt
from matplotlib.collections import LineCollection
from matplotlib.colors import ListedColormap, BoundaryNorm
import statsmodels.api as sm
from datetime import datetime, timedelta
import matplotlib.dates as mdates
from ticker.strategies.base import BaseStrategy
from trader.btc_trader import BTCTrader

def get_trend2(df, dt_start, dt_end, ax=None):
    """
    returns coef. A from equation y = A*x + B
    If A > 0 then trend goes UP
    If A < 0 then trend goes DOWN
    """
    x = df['jDate'][dt_start:dt_end]
    X1 = np.column_stack((x, x))
    X1 = sm.add_constant(X1)
    y1 = df['close'][dt_start:dt_end]
    fitted = sm.OLS(y1, X1).fit()
    if ax:
        ax.plot(x, fitted.fittedvalues, color='red')
    return fitted.params.x1 * 1000

def convert_ts(ts):
    return datetime.fromtimestamp(int(ts))

def get_trend(arr):
    n = len(arr)
    x = np.arange(1.0, n+1, 1.0)
    y = arr
    A = np.vstack([x, np.ones(len(x))]).T
    m, c = np.linalg.lstsq(A, y)[0]
    return m

def get_color(arr):
    if min(arr) == 1:
        return 1
    if max(arr) == -1:
        return -1
    return 0

class SimpleStrategy(BaseStrategy):

    def get_precolor(self, row):
        return -1 if row.close < row.ema_hour else 1

    def get_final_color(self, row):
        if row.color_0 == 0:
            if row.trend_1h > 0:
                return 1
            else:
                return -1
        return row.color_0

    def get_color(self, row):
        return 1 if row.ema_hour_trend > 0 else -1

    def calculate_signal(self, is_print=False):
        """
        Calculating all helper data columns
        Mostly based on exponential weighted mean and other rolling functions
        EMA and difference
        """
        self.market_data['jDate'] = self.market_data.index.to_julian_date()
        self.market_data['ema_hour'] = self.market_data['close'].ewm(span=12, adjust=False).mean()
        self.market_data['ema_hour_trend'] = pd.rolling_apply(self.market_data['ema_hour'], 3, lambda x: get_trend(x))
        self.market_data['ema_12hours'] = self.market_data['close'].ewm(span=72, adjust=False).mean()
        self.market_data['ema_12hours_trend'] = pd.rolling_apply(self.market_data['ema_12hours'], 3,
                                                              lambda x: get_trend(x))
        self.market_data['ema_day'] = self.market_data['close'].ewm(span=self.prices_in_a_day,
                                                                    adjust=False).mean()
        self.market_data['ema_day_trend'] = pd.rolling_apply(self.market_data['ema_day'], 3,
                                                              lambda x: get_trend(x))

        self.market_data['ema_week'] = self.market_data['close'].ewm(span=self.prices_in_a_day * 7,
                                                                     adjust=False).mean()
        self.market_data['ema_month'] = self.market_data['close'].ewm(span=self.prices_in_a_day * 30,
                                                                      adjust=False).mean()
        #self.market_data['trend_day'] = pd.rolling_apply(self.market_data['close'], 288, lambda x: get_trend(x))
        self.market_data['trend_4h'] = pd.rolling_apply(self.market_data['close'], 48, lambda x: get_trend(x))
        self.market_data['trend_1h'] = pd.rolling_apply(self.market_data['close'], 12, lambda x: get_trend(x))
        self.market_data['trend_30min'] = pd.rolling_apply(self.market_data['close'], 6, lambda x: get_trend(x))
        self.market_data['trend_15min'] = pd.rolling_apply(self.market_data['close'], 3, lambda x: get_trend(x))

        self.market_data['max_15min'] = self.market_data['close'].rolling(window=3).max()
        self.market_data['max_30min'] = self.market_data['close'].rolling(window=6).max()

        #self.market_data['pre_color'] = self.market_data.apply(self.get_precolor, axis=1)
        #self.market_data['color_0'] = pd.rolling_apply(self.market_data['pre_color'], 5, lambda x: get_color(x))
        #self.market_data['color'] = self.market_data.apply(self.get_final_color, axis=1)
        self.market_data['color'] = self.market_data.apply(self.get_color, axis=1)

        self.market_data['ema5'] = self.market_data['close'].ewm(5).mean()
        self.market_data['buy_line'] = self.market_data['ema5'].mean()

        return super(SimpleStrategy, self).calculate_signal(is_print)

    def get_signal_buy(self, row):
        """ Handle BUY conditions """
        if row.ema_12hours_trend > 0 and row.ema_day_trend > -0.000007:
            if row.close < row.ema_month * 0.99:
                if row.close < row.ema_week * 0.99:
                    if row.close < row.ema_day * 0.99:
                        if row.trend_15min > 0:
                            #print(row)
                            return self.execute_buy(row)

        # if row.close < row.ema_day:
        #     if row.trend_15min > 0 and row.trend_30min > 0:
        #         return self.execute_buy(row)

    def get_signal_sell(self, row):
        """ Handle SELL conditions """
        if row.close > row.ema_day * 1.00:
            if row.close > row.ema_week * 1.00:
                if row.close > row.ema_month * 1.00:
                    if row.trend_15min < 0:
                        return self.execute_sell(row)

        last_trade = self.trader.get_last_trade()
        if last_trade:
            buy_price = last_trade.price
            if row.close < buy_price * self.bad_buy_limit:
                return self.execute_sell(row)       # High Loss - wrong buy

    def draw_line(self):
        plt.plot(self.market_data.index, self.market_data['ema_hour'], color='red')
        plt.plot(self.market_data.index, self.market_data['ema_day'], color='yellow')
        plt.plot(self.market_data.index, self.market_data['ema_12hours'], color='orange')
        plt.plot(self.market_data.index, self.market_data['ema_month'], color='yellow')

    def draw_color_line(self, ax, data_col_name, factor_col_name):
        # convert dates to numbers first
        inxval = mdates.date2num(self.market_data.index.to_pydatetime())
        points = np.array([inxval, self.market_data[data_col_name]]).T.reshape(-1, 1, 2)
        segments = np.concatenate([points[:-1], points[1:]], axis=1)

        # colorize line
        # factor is from -1 -> 1
        cmap = ListedColormap(['r', 'b', 'g'])
        norm = BoundaryNorm([-200, -0.000002, 0.000002, 200], cmap.N)
        lc = LineCollection(segments, cmap=cmap, norm=norm, linewidth=1)
        lc.set_array(self.market_data[factor_col_name])
        ax.add_collection(lc)
        ax.plot(self.market_data.index, self.market_data[data_col_name], alpha=0.0)

    def draw_graph(self, from_symbol='BTC', to_symbol='BCH'):
        fig, ax = plt.subplots()
        fig.set_size_inches(10, 8)
        #ax.plot(self.market_data.index, self.market_data.close, color='gray')

        self.draw_line()
        self.draw_color_line(ax, 'close', 'color')

        #ax.plot(df.index, df['long_trend'], alpha=0.0)
        #ax.plot(df.index, df['long_trend'], color='orange')

        # SELL Signal
        below_threshold = self.market_data[self.market_data.signal == -1]
        plt.scatter(below_threshold.index, below_threshold['close'], color='red')

        # BUY Signal
        below_threshold = self.market_data[self.market_data.signal == 1]
        plt.scatter(below_threshold.index, below_threshold['close'], color='darkgreen')

        # helper lines
        # plt.plot(market_data.index, market_data['average'], color='gray')
        # plt.plot(df.index, df['ema100'], color='pink')

        ax.xaxis.set_major_locator(mdates.MonthLocator())
        ax.xaxis.set_minor_locator(mdates.DayLocator())
        monthFmt = mdates.DateFormatter("%b")
        ax.xaxis.set_major_formatter(monthFmt)

        plt.title('market: {}'.format(self.market))
        plt.grid(True)

        xfmt = mdates.DateFormatter('%m-%d %H')
        ax.xaxis.set_major_formatter(xfmt)
        ax.xaxis.set_major_locator(mdates.DayLocator(interval=7))

        h = self.market_data.close.max() - self.market_data.close.min()
        y_min = self.market_data.close.min() - h / 2
        y_max = self.market_data.close.max() + h / 2
        plt.axis([self.market_data.index.min() + timedelta(hours=12),
                  self.market_data.index.max(), y_min, y_max])
        plt.setp(plt.gca().xaxis.get_majorticklabels(), 'rotation', 90)
        plt.show()

    def start_from(self, dt_from, days=30, from_symbol='BTC', plot=False):
        self.read_data(dt_from, days)
        print('read_data.. done ')
        self.bad_buy_limit = Decimal('0.97')
        self.calculate_signal(True)
        print(self.trader.balance)
        self.draw_graph()

if __name__ == "__main__":
    # dt_from = datetime.strptime('2017-08-30 9:40:00', '%Y-%m-%d %H:%M:%S')
    dt_from = datetime.utcnow()
    s = SimpleStrategy('BTC_ETH')
    s.start_from(dt_from, days=65)

    # dt_from = datetime.utcnow()
    # s = SimpleStrategy()
    # df = s.read_data(dt_from, days=10)
    # df['jDate'] = df.index.to_julian_date()
    #
    # fig, ax = plt.subplots()
    # fig.set_size_inches(10, 8)
    # ax.plot(df.index, df['close'], color='orange')
    #
    # ts_end = dt_from.timestamp()
    # start = datetime.fromtimestamp(ts_end - (86400 * 7))
    # end = datetime.fromtimestamp(ts_end - (86400 * 6))
    #
    # fig, ax = plt.subplots(figsize=(8, 6))
    # ax.plot(df.jDate, df.close)
    # a = get_trend(df, start, end, ax)
    # print(a)
    # plt.show()


from django.urls import path
from . import views

app_name = "polls"

urlpatterns = [
    path('list/', views.polls_list, name='list'),
    path('list/user/', views.list_by_user, name='list_by_user'),
    path('add/', views.polls_add, name='add'),
    path('edit/<int:poll_id>/', views.polls_edit, name='edit'),
    path('delete/<int:poll_id>/', views.polls_delete, name='delete_poll'),
    path('end/<int:poll_id>/', views.endpoll, name='end_poll'),
    path('edit/<int:poll_id>/choice/add/', views.add_choice, name='add_choice'),
    path('edit/choice/<int:choice_id>/', views.choice_edit, name='choice_edit'),
    path('delete/choice/<int:choice_id>/',
         views.choice_delete, name='choice_delete'),
    path('<int:poll_id>/', views.poll_detail, name='detail'),
    path('<int:poll_id>/vote/', views.poll_vote, name='vote'),
]


import json
import os
import re
import gzip
import random
import sys
import unidecode
from collections import Counter, OrderedDict
from tqdm import tqdm as tqdm
from arango import DocumentInsertError, IndexCreateError
from arango.database import StandardDatabase

from dataloader_constants import (
    DEFAULT_LANGS,
    LANG_PALI,
    LANG_CHINESE,
    LANG_TIBETAN,
    LANG_SANSKRIT,
    COLLECTION_PARALLELS,
    COLLECTION_PARALLELS_SORTED_BY_FILE,
    COLLECTION_SEGMENTS,
    COLLECTION_FILES,
    COLLECTION_FILES_PARALLEL_COUNT,
    COLLECTION_REGEX,
    COLLECTION_CATEGORIES_PARALLEL_COUNT
)

from dataloader_models import Parallel, Segment, MenuItem
from dataloader_utils import (
    get_database,
    should_download_file,
    execute_in_parallel,
    get_segments_and_parallels_from_gzipped_remote_file,
    get_collection_list_for_language,
    get_categories_for_language_collection,
    natural_keys
)

# allow importing from api directory
PACKAGE_PARENT = ".."
SCRIPT_DIR = os.path.dirname(
    os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__)))
)
sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT)))

from api.queries import menu_queries, main_queries
from api.utils import get_language_from_filename

def load_segment_data_from_menu_files(root_url: str, threads: int, langs: list):
    for language in langs:
        with open(f"../data/{language}-files.json") as f:
            print(f"\nLoading segment data from menu files in {language}:...")
            files_data = json.load(f)
            filtered_file_data = (
                [
                    file
                    for file in files_data
                    if should_download_file(language, file["filename"])
                ]
                if os.environ["TESTING_LIMIT"]
                else files_data
            )

            execute_in_parallel(
                lambda item: load_segments_and_parallels_data_from_menu_file(
                    item, lang=language, root_url=root_url
                ),
                filtered_file_data,
                threads,
            )

def get_folios_from_segment_keys(segment_keys, lang):
    folios = []
    if lang == LANG_CHINESE:
        last_num = ''
        for segment_key in segment_keys:
            num = segment_key.split("_")[1].split(":")[0]
            if num != last_num:
                folios.append({"num": num, "segment_nr": segment_key})
                last_num = num
    elif lang == LANG_TIBETAN:
        last_num = ''
        for segment_key in segment_keys:
            num = segment_key.split(":")[1].split("-")[0]
            if num != last_num:
                folios.append({"num": num, "segment_nr": segment_key})
                last_num = num
    elif lang == LANG_PALI:
        last_num = ''
        for segment_key in segment_keys:
            num = segment_key.split(":")[1].split(".")[0].split("_")[0]
            if re.search(r"^(anya|tika|atk)", segment_key):
                if num.endswith('0') and num != last_num:
                    folios.append({"num": num, "segment_nr": segment_key})
                    last_num = num
            else:
                if num != last_num:
                    folios.append({"num": num, "segment_nr": segment_key})
                    last_num = num
    elif lang == LANG_SANSKRIT:
        last_num = ''
        for segment_key in segment_keys:
            if re.search(r"^(XXdhppat|S10udanav)", segment_key):
                num = segment_key.split(":")[1].split("_")[1]
                if num != last_num:
                    folios.append({"num": num, "segment_nr": segment_key})
                    last_num = num
            if re.search(r"^(OT)", segment_key):
                num = segment_key.split(":")[1].split("_")[0].strip('x')
                if num != last_num:
                    folios.append({"num": num, "segment_nr": segment_key})
                    last_num = num
            else:
                num = segment_key.split(":")[1].split(".")[0].split("_")[0]
                if num.endswith('0') and num != last_num:
                    folios.append({"num": num, "segment_nr": segment_key})
                    last_num = num
    return folios

def load_segments_and_parallels_data_from_menu_file(
    menu_file_json, lang: str, root_url: str
) -> None:
    file_url = f"{root_url}{lang}/{menu_file_json['filename']}.json.gz"
    db = get_database()

    if not file_url.endswith("gz"):
        print(f"{file_url} is not a gzip file. Ignoring.")
        return

    [segments, parallels] = get_segments_and_parallels_from_gzipped_remote_file(
        file_url
    )

    if segments:
        segment_keys, totallengthcount, totalfilelengthcount = load_segments(
            segments, parallels, db
        )
        load_files_collection(menu_file_json, segment_keys, lang, db)
        load_file_parallel_counts(
            menu_file_json, totallengthcount, totalfilelengthcount, db
        )

    if parallels:
        load_parallels(parallels, db)
        load_parallels_sorted(parallels, db,menu_file_json['filename'])

def load_segments(segments: list, all_parallels: list, db: StandardDatabase) -> list:
    """ Returns list of segment numbers. """
    segment_keys = []
    segmentnr_parallel_ids_dic = {}
    segmentnr_parallel_ids_dic_limited = {}
    parallel_total_list = []
    parallel_file_total_list = []

    for parallel in all_parallels:
        if isinstance(
            parallel, dict
        ):  # this relates to a strange bug in the generated data, I hope I can fix it in the future. TODO 4/20: check if this problem still exists, maybe we can get rid of it
            if parallel["root_segnr"]:
                for segment_key in parallel["root_segnr"]:
                    if segment_key not in segmentnr_parallel_ids_dic.keys():
                        segmentnr_parallel_ids_dic[segment_key] = [parallel["id"]]
                    else:
                        segmentnr_parallel_ids_dic[segment_key].append(parallel["id"])
                    # the limited-dic collects only the first 10 parallels, because we don't need more for the text-view.
                    if parallel['co-occ'] <= 20:
                        if segment_key not in segmentnr_parallel_ids_dic_limited.keys():
                            segmentnr_parallel_ids_dic_limited[segment_key] = [parallel["id"]]
                        else:
                            segmentnr_parallel_ids_dic_limited[segment_key].append(parallel["id"])

            if parallel["par_segnr"]:
                collection_key = re.search(COLLECTION_REGEX, parallel["par_segnr"][0])
                if collection_key:
                    parallel_total_list.append(
                        {collection_key.group(): parallel["root_length"]}
                    )

                parallel_file_key = parallel["par_segnr"][0].split(":")[0]
                parallel_file_total_list.append(
                    {parallel_file_key: parallel["root_length"]}
                )

    segment_count = 0
    for segment in segments:
        parallel_ids = []
        parallel_ids_limited = []
        if isinstance(segment, dict):
            if segment["segnr"] in segmentnr_parallel_ids_dic.keys():
                parallel_ids = segmentnr_parallel_ids_dic[segment["segnr"]]
            if segment["segnr"] in segmentnr_parallel_ids_dic_limited.keys():
                parallel_ids_limited = segmentnr_parallel_ids_dic_limited[segment["segnr"]]

            # Loads segment into database
            # todo: remove parallel_ids table
            segment_key = load_segment(segment, segment_count, parallel_ids,parallel_ids_limited, db)

            segment_keys.append(segment_key)
            segment_count += 1
    totallengthcount = Counter()
    for totalcount in parallel_total_list:
        totallengthcount += Counter(totalcount)

    totalfilelengthcount = Counter()
    for totalparallelcount in parallel_file_total_list:
        totalfilelengthcount += Counter(totalparallelcount)

    return segment_keys, totallengthcount, totalfilelengthcount

def load_segment(
        json_segment: Segment, count: int, parallel_ids: list, parallel_ids_limited: list, db: StandardDatabase
) -> str:
    """
    Given a single segment object, load it into the `segments` collection.

    :param json_segment: Segment JSON data
    :param count: Incrementing number of segments
    :param parallel_ids: Array of IDs of parallels of which segment is root
    :param db: ArangoDB database object
    :return: Segment nr
    """
    collection = db.collection(COLLECTION_SEGMENTS)
    try:
        doc = {
            "_key": json_segment["segnr"],
            "_id": f'segments/{json_segment["segnr"]}',
            "segnr": json_segment["segnr"],
            "segtext": json_segment["segtext"],
            "lang": json_segment["lang"],
            "position": json_segment["position"],
            "count": count,
            "parallel_ids": parallel_ids,
            "parallel_ids_limited": parallel_ids_limited
        }
        collection.insert(doc)
    except (KeyError, AttributeError) as e:
        print("Could not load segment. Error: ", e)
    except DocumentInsertError as e:
        print(f"Could not save segment {json_segment['segnr']}. Error: ", e)

    return json_segment["segnr"]

def load_file_parallel_counts(
    file: MenuItem,
    total_length_count: list,  # TODO: this is not typed correctly
    total_file_length_count: list,  # TODO: same as above
    db: StandardDatabase,
):
    db_collection = db.collection(COLLECTION_FILES_PARALLEL_COUNT)
    sorted_total_file_length_count = sorted(
        total_file_length_count.items(), key=lambda kv: kv[1], reverse=True
    )

    doc = {
        "_key": file["filename"],
        "category": file["category"],
        "language": get_language_from_filename(file["filename"]),
        "filenr": file["filenr"],
        "totallengthcount": total_length_count,
        "totalfilelengthcount": OrderedDict(sorted_total_file_length_count),
        "totallength": sum(total_length_count.values()),
    }
    try:
        db_collection.add_hash_index(["category"], unique=False)
        if db_collection.get(file['filename']):
            db_collection.delete(file['filename'])
        db_collection.insert(doc)
    except (DocumentInsertError, IndexCreateError) as e:
        print("Could not load file. Error: ", e)

def load_files_collection(file: MenuItem, segment_keys: list,lang: str, db: StandardDatabase):
    files_db_collection = db.collection(COLLECTION_FILES)
    file_key = file["filename"]
    folios = get_folios_from_segment_keys(segment_keys,lang)
    search_field = file['textname'] + " " + file['displayName'] + " " + unidecode.unidecode(file['displayName'])
    doc = {"_key": file_key, "segment_keys": segment_keys, "folios" : folios, "language":lang, "search_field" : search_field}
    doc.update(file)
    try:
        files_db_collection.insert(doc)
    except DocumentInsertError as e:
        print("Could not load file. Error: ", e)

def load_parallels(json_parallels: [Parallel], db: StandardDatabase) -> None:
    """
    Given an array of parallel objects, load them all into the `parallels` collection

    :param json_parallels: Array of parallel objects to be loaded as-they-are.
    :param db: ArangoDB connection object
    """
    db_collection = db.collection(COLLECTION_PARALLELS)
    db_collection_sorted = db.collection(COLLECTION_PARALLELS_SORTED_BY_FILE)
    parallels_to_be_inserted = []
    root_file_parallel_edges_to_be_inserted = []

    for parallel in json_parallels:
        if isinstance(parallel, dict):
            root_filename = parallel["root_segnr"][0].split(":")[0]
            root_filename = re.sub("_[0-9][0-9][0-9]","",root_filename)
            parallel_id = f"parallels/{parallel['id']}"
            parallel["_key"] = parallel["id"]
            parallel["_id"] = parallel_id
            # here we delete some things that we don't need in the DB:
            del parallel["par_pos_end"]
            del parallel["root_pos_end"]
            del parallel["par_segtext"]
            del parallel["root_segtext"]
            del parallel["par_string"]
            del parallel["root_string"]
            # todo: delete the root_filename key after it's not needed anymore
            parallel["root_filename"] = root_filename
            parallels_to_be_inserted.append(parallel)

    chunksize = 10000

    for i in range(0, len(parallels_to_be_inserted), chunksize):
        try:
            db_collection.insert_many(parallels_to_be_inserted[i : i + chunksize])
        except (DocumentInsertError, IndexCreateError) as e:
            print(f"Could not save parallel {parallel}. Error: ", e)

def load_parallels_sorted(json_parallels: [Parallel], db: StandardDatabase,filename: str) -> None:
    """
    Given an array of parallel objects, load them all into the `sorted parallels` collection presorted.

    :param json_parallels: Array of parallel objects to be loaded as-they-are.
    :param db: ArangoDB connection object

    """

    db_collection_sorted = db.collection(COLLECTION_PARALLELS_SORTED_BY_FILE)
    # I wonder if this can be done more efficiently, a lot of spaghetti code...
    parallels_sorted_by_src_position = sorted(json_parallels, key=lambda k: k['root_pos_beg'])
    ids_sorted_by_src_position = list(map(lambda parallel: parallel['id'], parallels_sorted_by_src_position))

    parallels_sorted_by_tgt_position = sorted(json_parallels, key=lambda k: natural_keys(k['par_segnr'][0]))
    ids_sorted_by_tgt_position = list(map(lambda parallel: parallel['id'], parallels_sorted_by_tgt_position))

    parallels_sorted_by_length_par = sorted(json_parallels, key=lambda k: k['par_length'])
    ids_sorted_by_length_par = list(map(lambda parallel: parallel['id'], parallels_sorted_by_length_par))

    parallels_sorted_by_length_root = sorted(json_parallels, key=lambda k: k['root_length'])
    ids_sorted_by_length_root = list(map(lambda parallel: parallel['id'], parallels_sorted_by_length_root))

    ids_sorted_by_length_par.reverse()
    ids_sorted_by_length_root.reverse()

    ids_sorted_random = ids_sorted_by_length_root
    random.shuffle(ids_sorted_random)

    src_lang = json_parallels[0]['src_lang']
    try:
        db_collection_sorted.insert({'_key':filename,
                                     'filename':filename,
                                     'lang':src_lang,
                                     'parallels_sorted_by_src_pos': ids_sorted_by_src_position,
                                     'parallels_sorted_by_tgt_pos': ids_sorted_by_tgt_position,
                                     'parallels_sorted_by_length_src': ids_sorted_by_length_root,
                                     'parallels_randomized': ids_sorted_random,
                                     'parallels_sorted_by_length_tgt': ids_sorted_by_length_par})
    except (DocumentInsertError, IndexCreateError) as e:
        print(f"Could not save sorted parallel for {filename}. Error: ", e)

def create_indices(db: StandardDatabase):
    db_collection = db.collection(COLLECTION_PARALLELS)
    db_collection.add_hash_index(["root_filename"], unique=False)
    db_collection.add_hash_index(["src_lang"], unique=False)
    db_collection = db.collection(COLLECTION_FILES)
    db_collection.add_hash_index(["language"], unique=False)
    db_collection.add_hash_index(["category"], unique=False)

def load_sources(db: StandardDatabase,root_url):
    source_json_path = root_url + "sources.json"
    db_file_collection = db.collection(COLLECTION_FILES)
    with open(source_json_path,"rb") as f:
        source_list = json.load(f)
    for entry in source_list:
        filename = entry['filename']
        current_file = db_file_collection.get(filename)
        if current_file:
            current_file['source_string'] = entry['source']
            print(current_file)
            db_file_collection.update(current_file)

# TODO: Refactor this function. Split into smaller chunks.
def calculate_parallel_totals():
    # This function goes over all the data and groups it into totals for the visual view
    # This takes some time to run on the full dataset.
    db = get_database()
    collection_query_cursor = db.aql.execute(
        menu_queries.QUERY_CATEGORIES_PER_COLLECTION
    )
    collections = [doc for doc in collection_query_cursor]

    # for each collection, the totals to each other collection of that same language are calculated
    for col in collections:
        language = col["language"]
        source_collection = col["collection"]
        source_col_dict = {}
        for source_cat in col["categories"]:
            source_col_dict.update(source_cat)

        language_collection_list = get_collection_list_for_language(
            language, collections
        )

        for target_collection in language_collection_list:
            selected_category_dict = get_categories_for_language_collection(
                target_collection, collections
            )

            counted_parallels = []
            for category, cat_name in source_col_dict.items():
                all_files_cursor = db.aql.execute(
                    menu_queries.QUERY_FILES_PER_CATEGORY,
                    batch_size=100000,
                    bind_vars={"category": category, "language": language},
                )
                all_files = [doc for doc in all_files_cursor]
                add_category_totals_to_db(
                    all_files,
                    category,
                    target_collection,
                    selected_category_dict,
                    language,
                )

                total_par_list = {}
                for filename in all_files:
                    parallel_count = filename["totallengthcount"]
                    for categoryname in selected_category_dict:
                        if categoryname not in total_par_list.keys():
                            if categoryname not in parallel_count.keys():
                                total_par_list[categoryname] = 0
                            else:
                                total_par_list[categoryname] = parallel_count[
                                    categoryname
                                ]
                        elif categoryname in parallel_count.keys():
                            total_par_list[categoryname] += parallel_count[categoryname]

                for key, value in total_par_list.items():
                    counted_parallels.append(
                        [
                            cat_name + " (" + category + ")",
                            selected_category_dict[key].rstrip() + "_(" + key + ")",
                            value,
                        ]
                    )

            load_parallel_counts(
                source_collection, target_collection, counted_parallels
            )

def add_category_totals_to_db(
    all_files, category, target_collection, selected_category_dict, language
):
    # for each collection, the totals of each category in that collection
    # to each other collection of that same language are calculated
    counted_parallels = []
    for filename in all_files:
        file_counted_parallels = []
        file_counted_parallels_no_zeros = []
        parallel_count = filename["totallengthcount"]
        for categoryname in selected_category_dict:
            weight_value = 0
            if categoryname in parallel_count:
                weight_value = parallel_count[categoryname]

            displayFileName = filename["filename"]
            if language == 'skt' or language == 'pli':
                displayFileName = filename["displayName"] + ' ('+displayFileName+')'

            new_paralllel_entry = [
                displayFileName,
                selected_category_dict[categoryname].rstrip()
                + "_("
                + categoryname
                + ")",
                weight_value,
            ]
            file_counted_parallels.append(new_paralllel_entry)
            if weight_value > 0:
                file_counted_parallels_no_zeros.append(new_paralllel_entry)
        load_parallel_counts(
            language + "_" + filename["filename"],
            target_collection,
            file_counted_parallels_no_zeros,
        )
        counted_parallels += file_counted_parallels

    load_parallel_counts(
        language + "_" + category, target_collection, counted_parallels
    )

def load_parallel_counts(source_name: str, target_name: str, total_length_count: list):
    if total_length_count:
        db = get_database()
        collection = db.collection(COLLECTION_CATEGORIES_PARALLEL_COUNT)
        doc = {
            "_key": source_name + "_" + target_name,
            "sourcecollection": source_name,
            "targetcollection": target_name,
            "totallengthcount": total_length_count,
        }
        try:
            collection.add_hash_index(["sourcecollection"], unique=False)
            collection.insert(doc)
        except (DocumentInsertError, IndexCreateError) as e:
            print("Could not load file. Error: ", e)


import os
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

from ad import plot, utils

class PlotCallback(tf.keras.callbacks.Callback):
    """Custom callback that plots reconstructions"""

    def __init__(self, path: str, freq=1, amount=1, data: np.ndarray = None, normalize_fn=None,
                 predict_fn=None, post_fn=None, **kwargs):
        self.freq = int(freq)
        self.amount = int(amount)
        self.kwargs = kwargs

        # load samples from path
        if isinstance(path, str):
            self.data = utils.read_npz(path, dtype=np.float32, verbose=False, keys='images')
        else:
            assert isinstance(data, np.ndarray)
            self.data = data

        if callable(normalize_fn):
            self.data = normalize_fn(self.data)

        if callable(predict_fn):
            self.predict_fn = lambda model, x: predict_fn(model, x)
        else:
            self.predict_fn = lambda model, x: self.model(x)

        if callable(post_fn):
            self.post_process = lambda x: post_fn(x)
        else:
            self.post_process = lambda x: x

    def plot(self):
        x = self.data[np.random.choice(len(self.data), size=self.amount, replace=False)]
        y = self.predict_fn(self.model, x)
        x = self.post_process(x)

        if self.amount > 1:
            x = np.mean(x, axis=0)
            y = np.mean(y, axis=0)

        plot.compare(x, y, **self.kwargs)

    def on_epoch_end(self, epoch, logs=None):
        if epoch % self.freq == 0:
            self.plot()

class PlotLatents(PlotCallback):
    def __init__(self, *args, predict_fn, bins=100, **kwargs):
        super().__init__(*args, **kwargs)
        assert callable(predict_fn)

        self.bins = int(bins)
        self.buffer = []
        self.predict = predict_fn

    def plot(self, z1, z2):
        ax1, ax2 = utils.get_plot_axes(rows=1, cols=2)

        # histogram
        ax1.hist(z1, bins=self.bins, density=True, histtype='step', label=r'z$_0$')
        ax1.hist(z2, bins=self.bins, density=True, histtype='step', label=r'$z_1$')
        ax1.legend(loc='best')

        # scatter plot
        ax2.scatter(z1, z2, marker='o', s=10, label=r'$z$')
        ax2.set_xlabel(r'$z_0$')
        ax2.set_ylabel(r'$z_1$')
        ax2.legend(loc='best')

        plt.show()

    def on_epoch_end(self, epoch, logs=None):
        x = self.data[np.random.choice(len(self.data), size=self.amount, replace=False)]
        z = self.predict(self.model, x)

        z1 = z[:, 0]
        z2 = z[:, 1]

        self.buffer.append((z1, z2))

        if epoch % self.freq == 0:
            self.plot(z1, z2)

    def save_latents(self, path: str, name: str, clear=True):
        for i, (z1, z2) in enumerate(self.buffer):
            plt.figure(figsize=(12, 10))

            plt.scatter(z1, z2, marker='o', s=10, label=r'$z$')
            plt.xlabel(r'$z_0$')
            plt.ylabel(r'$z_1$')
            plt.legend(loc='best')

            # save figure
            path = utils.makedir(path)
            plt.savefig(os.path.join(path, f'{name}_{i}.png'), bbox_inches='tight')

            plt.show()

        # empty buffer
        if clear:
            self.buffer.clear()


import settings
import threading

from common.funcs import S3, QueryHandler
from common.custom_error import InvalidBucketRequest, KeyAlreadyExists
from models.s3_object import S3Object

class ConsoleS3Object(S3):

    def __init__(self, object_type, image_name, content, acl='private'):
        self.bucket_name = self.get_bucket_name(object_type)
        super(ConsoleS3Object, self).__init__(bucket_name = self.bucket_name, name = image_name, content = content, acl = acl)

    def get_bucket_name(self, object_type):
        bucket_name = settings.articles_BUCKETS.get(object_type, None)
        if bucket_name:
            return bucket_name
        else:
            raise InvalidBucketRequest

    def handle_upload(self):
        if self.check_exists():
            raise KeyAlreadyExists
        else:
            threading.Thread(group = None, target = self.upload, name = None, args = ()).start()

    def upload(self):
        self.client.put_object(Bucket = self.bucket_name, Key = self.name, Body=self.content, ACL = self.acl)

def get_group_users(discussion_id):
    query = "SELECT users.name, users.username FROM users, discussions_users WHERE (users.username=discussions_users.username AND discussion_id=%s)";
    variables = (discussion_id, )
    group_users = QueryHandler.get_results(query, variables)
    return group_users