#!/usr/bin/env python3

"""
trivial strategy
"""

import archon.config as config
import traceback
from datetime import datetime

#import archon.broker as broker
from archon.brokersrv.brokerservice import BrokerService
import archon.exchange.exchanges as exc
import archon.exchange.bitmex.bitmex as mex
import archon.exchange.bitmex.book_util as book_util
import archon.exchange.bitmex.timeseries as timeseries
import archon.broker.facade as facade
import archon.model.models as models
from archon.custom_logger import setup_logger, remove_loggers
import archon.orders as orders
from archon.util import *

import pandas as pd
import numpy

import time
import json
import atexit
import logging
import threading

from abc import ABC, abstractmethod

from strat import Strategy

class TrivialStrategy(Strategy):

    def __init__(self, abroker, mex_sym, order_qty, zoff):
        setup_logger(logger_name=__name__, log_file=__name__ + '.log')
        self.log = logging.getLogger(__name__)
        self.log.info('marb strategy')

        remove_loggers()

        # init data
        self.book_mex = None
        self.pos_qty = 0

        self.zoff = zoff # the offset in $
        self.diff_cancel = 1.0 # cancel if diff is larger than this
        self.order_qty = order_qty

        super().__init__(abroker, mex_sym)

    def calc_mid(self, book):
        print (book)
        bids = list(filter(lambda x: x['side']=='Buy', book))
        asks = list(filter(lambda x: x['side']=='Sell', book))
        print (">>>> " ,bids[0])
        #, asks)
        mb = bids[0]
        ma = asks[0]
        mid = (mb['price'] + ma['price'])/2
        return mid

    def round_tick(self, price):
        """ round to bitmex tick size of 0.50$ """
        ip = round(price,0)
        if ip > price: ip=ip-1
        rest = price - ip
        r = -1
        if rest > 0.5 and rest <= 0.75:
            r = ip + 0.5
        elif rest > 0.75:
            r = ip
        elif rest <= 0.5:
            r = ip
        return r

    def calc_quotes(self):
        """ calculate one buy and sell quote
        trivial offset from bid and ask"""

        bids = list(filter(lambda x: x['side']=='Buy', self.book))
        asks = list(filter(lambda x: x['side']=='Sell', self.book))
        da = asks[0]['price']
        db = bids[0]['price']

        FV = self.calc_mid(self.book)

        buyorder = None
        sellorder = None

        oq = self.zoff #offset parameter in $

        qty = self.order_qty
        target_price = self.round_tick(FV - oq) # offset from mid
        order = [orders.ORDER_SIDE_BUY, qty, target_price]
        buyorder = order

        qty = self.order_qty
        target_price = self.round_tick(FV + oq) # offset from mid
        order = [ orders.ORDER_SIDE_SELL, qty, target_price]
        sellorder = order

        return [buyorder, sellorder]

    def replace_order(self, old_order, new_order):
        oid = old_order['orderID']
        result = self.abroker.cancel_order(oid, exc.BITMEX)
        return result
        #self.submit_order(new_order)

    def order_str(self, o):
        s = "%s %5.3f %5.3f"%(o['side'],o['orderQty'],o['price'])
        return s

    def handle_no_position(self):
        self.log.info("handle_no_position")
        target_orders = self.calc_quotes()
        self.log.info("target orders %s"%target_orders)
        #TODO adjust orders instead
        oo = abroker.openorders(exc.BITMEX)
        self.log.info("open orders %s"%str(oo))

        buys = list(filter(lambda x: x['side']=='Buy', oo))
        sells = list(filter(lambda x: x['side']=='Sell', oo))
        [target_buyorder, target_sellorder] = target_orders

        # if existing orders adjust orders by checking against targets

        if len(buys) > 0:
            for o in buys:
                self.log.info("check order %s"%str(o))
                dif = target_buyorder[2] - o['price']
                self.log.info("order dif %s"%str(dif))
                if dif >= 1:
                    self.log.info("price too low. cancel")
                    self.replace_order(o, target_buyorder)

                elif dif <= -1:
                    self.log.info("price too high. cancel")
                    self.replace_order(o, target_buyorder)

                else:
                    self.log.info("price of existing order ok")

        if len(sells) > 0:
            for o in sells:
                self.log.info("check order %s"%self.order_str(o))
                dif = target_sellorder[2] - o['price']
                self.log.info("order dif %s"%str(dif))
                if dif >= self.diff_cancel:
                    self.log.info("price too low. cancel")
                    self.replace_order(o, target_sellorder)

                elif dif <= -self.diff_cancel:
                    self.log.info("price too high. cancel")
                    self.replace_order(o, target_sellorder)

                else:
                    self.log.info("price of existing order ok")

        if len(buys) == 0:
            for order in [target_buyorder]:
                #TODO cleanup order array to dict
                ttype = order[0]
                market = self.mex_sym
                qty = order[1]
                order_price = order[2]
                order = [market,ttype,order_price,qty]
                result = self.abroker.submit_order_post(order, exc.BITMEX)
                self.log.info("order result %s"%str(result))
                time.sleep(1)

        if len(sells) == 0:
            for order in [target_sellorder] :
                #TODO cleanup order array to dict
                ttype = order[0]
                market = self.mex_sym
                qty = order[1]
                order_price = order[2]
                order = [market,ttype,order_price,qty]
                result = self.abroker.submit_order_post(order, exc.BITMEX)
                self.log.info("order result %s"%str(result))
                time.sleep(1)

    def handle_position(self):
        #TODO same with skew
        return

    def handle_books(self, mexbook):
        self.log.info("handle books")
        #db = self.abroker.get_db()
        #db.books.insert_one({"orderbook":mexbook})

        self.book = mexbook
        #self.update_orders()

        #book_util.display_book(self.book)
        mid = self.calc_mid(self.book)
        self.log.info("mid price %s"%str(mid))

        pos = abroker.position(exc.BITMEX)
        try:
            self.pos_qty = int(pos[0]['currentQty'])
        except:
            self.pos_qty = 0

        got_max_position = self.pos_qty > 50
        oo = abroker.openorders(exc.BITMEX)
        self.log.info("finished setting data")

        if len(oo)>4:
            self.log.info("too many orders")
            #self.cancel_all()
            self.alive_flag = False
            return

        self.log.info("got_max_position %s"%str(got_max_position))

        if got_max_position:
            #self.handle_position()
            self.log.info("todo...")
            self.alive_flag = False
            return

        else:
            self.handle_no_position()

if __name__=='__main__':
    #todo register exit handler
    #atexit.register(clean_exit)

    try:
        #assume service is running, this is just a proxy
        abroker = BrokerService(setAuto=True,initFeeder=False)
        abroker.set_keys_exchange_file(exchanges=[exc.BITMEX])
        mex_sym = mex.instrument_btc_perp
        zoff = 1.0
        order_qty = 1.0
        strategy = TrivialStrategy(abroker, mex_sym, order_qty, zoff)
        strategy.start()
        strategy.join()
    except Exception as e:
        traceback.print_exc()


from selenium import webdriver
from selenium.webdriver.common.alert import Alert
from random import choice, randint
import win32com.client as comclt
import win32gui
import win32con
import time

def handle_upload_file_dialog(file_path):
    sleep = 1
    windowsShell = comclt.Dispatch("WScript.Shell")
    time.sleep(sleep)
    windowsShell.SendKeys(file_path)
    time.sleep(sleep)
    windowsShell.SendKeys("{TAB}{TAB}")
    time.sleep(sleep)
    windowsShell.SendKeys("{ENTER}{ENTER}")

def login():
    driver.find_element_by_xpath('//div[text()="Login"]').click()
    time.sleep(1)
    driver.find_element_by_xpath("//input[@type='email']").send_keys("cheekit.chong98@gmail.com")
    time.sleep(1)
    driver.find_element_by_xpath("//input[@type='password']").send_keys("bts")
    time.sleep(1)
    driver.find_element_by_xpath('//div[text()="Login as Tenant"]').click()
    time.sleep(1)
    links = driver.find_elements_by_xpath('//div[text()="LOGIN"]')
    links[-1].click()
    time.sleep(5)

if __name__ == "__main__":
    driver = webdriver.Chrome()
    driver.get('https://esc-group-10.netlify.app/')
    time.sleep(5)
    login()
    driver.get('https://esc-group-10.netlify.app/staff/manage-tenants/create')
    time.sleep(3)
    links = driver.find_elements_by_xpath("//input[@type='text']")
    links[-1].send_keys("TEST")
    links[-2].send_keys("TEST")
    driver.find_element_by_xpath("//input[@type='email']").send_keys("Test@test.com")
    driver.find_element_by_xpath("//input[@type='password']").send_keys("testtest")
    driver.find_element_by_xpath('//div[text()="CREATE ACCOUNT"]').click()
    time.sleep(2)
    driver.get('https://esc-group-10.netlify.app/staff/manage-tenants/delete')
    time.sleep(2)
    driver.find_element_by_xpath('//div[text()="Sengkang General Hospital"]').click()
    time.sleep(1)
    driver.find_element_by_xpath('//div[text()="TEST"]').click()
    time.sleep(1)
    driver.find_element_by_xpath('//div[text()="DELETE TENANT"]').click()
    time.sleep(1)
    Alert(driver).accept()
    time.sleep(3)
    driver.close()


from __future__ import absolute_import
from pymongo import MongoClient
from pymongo.database import Database
from pymongo.collection import Collection
import six

class_to_class_name = {
    str:     'unicode',
    six.text_type: 'unicode',
    bool:    'bool',
    list:    'list',
    tuple:   'list',
    int:     'int',
    float:   'float',
}

class FedjiMongo(MongoClient):

    def __getattr__(self, db):
        return FedjiMongoDB(self, db)

class FedjiMongoDB(Database):
    # def __init__(self, fedji_client, db):
        # self.db = getattr(fedji_client.client, db)

    def __getattr__(self, collection):
        return FedjiMongoCollection(self, collection)

class FedjiMongoCollection(Collection):

    def __init__(self, fedji_db, collection):
        Collection.__init__(self, fedji_db, collection)
        attributes_collection = Database.__getattr__(
            self.database, '_attributes')
        fields = attributes_collection.find_one({})
        if fields is None:
            self.fields_id = attributes_collection.save({}, manipluate=True)
            self._fields = {}
        else:
            self.fields_id = fields.pop('_id')
            self._fields = fields

    @property
    def fields(self):
        return self._fields

    def new_field(self, field, cls):
        if field not in self.fields:
            atype = class_to_class_name[cls]
            self._fields[field] = atype
            doc = self.fields.copy()
            doc['_id'] = self.fields_id
            attributes_collection = Database.__getattr__(
                self.database, '_attributes')
            import pprint
            pprint.pprint(doc)
            attributes_collection.save(doc)

    def insert(self, doc_or_docs, **kwargs):
        if isinstance(doc_or_docs, dict):
            doc_or_docs = [doc_or_docs]
        for document in doc_or_docs:
            for k, v in six.iteritems(document):
                self.new_field(k, v.__class__)
        Collection.insert(self, doc_or_docs, **kwargs)

    # def __getattr__(self, attribute):
        # return getattr(self.collection, attribute)


""" Utilities module.
"""

import datetime as dt

import pkg_resources

import barbante

def local_import(name):
    """ Returns the module *name*.

        Attributes:
            name - the name of the module to be imported.

        Exception:
            TypeError - if *name* is not a string.
            ImportError - if there is no module *name* in your namespace.
    """
    if type(name) is not str:
        raise TypeError("barbante.api.recommend: name must be a string.")
    try:
        mod = __import__(name)
    except ImportError:
        raise ImportError(
            "barbante.api.recommend: there is no module name {0}".format(name))
    components = name.split('.')
    for comp in components[1:]:
        mod = getattr(mod, comp)
    return mod

def resource_stream(resource_path, package=barbante.__name__):
    """ Encapsulates pkg_resources.resource_stream.
    """
    return pkg_resources.resource_stream(package, resource_path)

def resource_filename(resource_path, package=barbante.__name__):
    """ Encapsulates pkg_resources.resource_filename.
    """
    return pkg_resources.resource_filename(package, resource_path)

def parse_date_series(series, date_format, queue=None):
    """ Implements pandas' apply function.

        **Parameters**
            date - date string in some format, e.g.: 31/12/2014.

            date_format - string format for *date*, e.g.: %d/%m/%Y %H:%M:%S.

            queue - multiprocessing.Queue for returning results.
    """
    new_series = series.apply(string_to_date, args=[date_format])
    if queue is not None:
        queue.put(new_series)

def parse_date_list(*args):
    """ Replaces pandas' *apply* function.

        :param args: (date_list,
                      start_index,
                      end_index,
                      date string in some format, e.g.: 31/12/2014,
                      string format for *date*, e.g.: %d/%m/%Y %H:%M:%S,
                      multiprocessing.Queue for returning results)
    """
    date_list = args[0]
    queue = args[1]
    date_format = args[2]
    new_series = []
    for date in date_list:
        new_series.append(string_to_date(date, date_format))

    queue.put(new_series)

def string_to_date(date, date_format):
    """ Returns the Unix date format from pandas.datetime.

        **Parameters**
        date - date string in some format, e.g.: 31/12/2014.

        date_format - string format for *date*, e.g.: %d/%m/%Y %H:%M:%S.
    """
    try:
        # return int(time.mktime(dt.datetime.strptime(date,
        # date_format).timetuple()))
        return dt.datetime.strptime(date, date_format)
    except TypeError:
        return

def flatten_dict(product_model_dict):
    """ Converts a dict with nested attributes into a flattened dict,
        where the path to each attribute in the original dict maps to
        a key in the new dict.

            > d = {'a': 1, 'b': {'c': 2, 'd': {'e': 3, 'f': 4}}}
            > ProductModel._flatten_dict(d)
            {'a': 1, 'b.c': 2, 'b.d.e': 3, 'b.d.f': 4}

        :param product_model_dict: the original dict.
        :returns: a flattened dict.
    """
    result = {}
    for key, value in product_model_dict.items():
        if isinstance(value, dict):
            subdict = flatten_dict(value)
            for sk, sv in subdict.items():
                result[key + '.' + sk] = sv
        else:
            result[key] = value
    return result


# -*- coding: utf-8 -*-
"""Model Examples."""

import numpy as np
import sympy
from sympy import symbols

from causing.model import Model
from causing.simulate import SimulationParams, simulate

data_path = __file__.split("causing")[0]

def example():
    """model example"""

    X1, X2, Y1, Y2, Y3 = symbols(["X1", "X2", "Y1", "Y2", "Y3"])
    equations = (  # equations in topological order (Y1, Y2, ...)
        X1,
        X2 + 2 * Y1 ** 2,
        Y1 + Y2,
    )
    m = Model(
        xvars=[X1, X2],  # exogenous variables in desired order
        yvars=[Y1, Y2, Y3],  # endogenous variables in topological order
        equations=equations,
        final_var=Y3,  # final variable of interest, for mediation analysis
    )

    ymvars = [Y3]  # manifest endogenous variables
    xdat, ymdat = simulate(
        m,
        SimulationParams(
            ymvars=ymvars,
            xmean_true=[3, 2],  # mean of exogeneous data
            sigx_theo=1,  # true scalar error variance of xvars
            sigym_theo=1,  # true scalar error variance of ymvars
            rho=0.2,  # true correlation within y and within x vars
            tau=200,  # nr. of simulated observations
        ),
    )

    estimate_input = dict(
        ymvars=ymvars,
        ymdat=ymdat,
        alpha=None,  # regularization parameter, is estimated if None
        dof=None,  # effective degrees of freedom, corresponding to alpha
    )

    return m, xdat, ymdat, estimate_input

def example2():
    """model example 2, no regularization required, no latent variables"""

    X1, Y1 = symbols(["X1", "Y1"])
    equations = (X1,)
    ymvars = [Y1]
    m = Model(
        equations=equations,
        xvars=[X1],
        yvars=[Y1],
        final_var=Y1,
    )

    xdat, ymdat = simulate(
        m,
        SimulationParams(
            ymvars=ymvars,
            xmean_true=[3],
            sigx_theo=1,
            sigym_theo=1,
            rho=0.2,
            tau=200,
        ),
    )

    estimate_input = dict(
        ymvars=ymvars,
        ymdat=ymdat,
        alpha=None,
        dof=None,
    )

    return m, xdat, ymdat, estimate_input

def example3():
    """model example 3

    difficult to estimate:
    if just Y3 is manifest, huge regularization is required and direct effects are strongly biased,
    (if all yvars are manifest, just slight regularization is required and some standard errors are huge)
    """

    X1, Y1, Y2, Y3 = symbols(["X1", "Y1", "Y2", "Y3"])
    equations = (
        2 * X1,
        -X1,
        Y1 + Y2,
    )
    ymvars = [Y3]
    m = Model(
        equations=equations,
        xvars=[X1],
        yvars=[Y1, Y2, Y3],
        final_var=Y3,
    )

    xdat, ymdat = simulate(
        m,
        SimulationParams(
            ymvars=ymvars,
            xmean_true=[3],
            sigx_theo=1,
            sigym_theo=1,
            rho=0.2,
            tau=200,
        ),
    )

    estimate_input = dict(
        ymvars=ymvars,
        ymdat=ymdat,
        alpha=None,
        dof=None,
    )

    return m, xdat, ymdat, estimate_input

def education():
    """Education

    The dataset contains following variables in this order, the variables 0.
    to 4. being time varying and variables 5. to 9. being time invariant:

    0. PERSONID = Person id (ranging from 1 to 2,178) # not used by us
    1. EDUC     = Education (years of schooling)
    2. LOGWAGE  = Log of hourly wage, at most recent job, in real 1993 dollars # we use wage instead of log wage
    3. POTEXPER = Potential experience (= AGE - EDUC - 5)
    4. TIMETRND = Time trend (starting at 1 in 1979 and incrementing by year) # not used by us
    5. ABILITY  = Ability (cognitive ability measured by test score)
    6. MOTHERED = Mother's education (highest grade completed, in years)
    7. FATHERED = Father's education (highest grade completed, in years)
    8. BRKNHOME = Dummy variable for residence in a broken home at age 14
    9. SIBLINGS = Number of siblings

    Model identified without regularization if wage instead of logwage and all observations. # yyyy

    ToDo: Automatic Hessian gives wrong results for this example: # yyyy
    Algebraic and numeric   Hessian allclose: True.
    Automatic and numeric   Hessian allclose: False.
    Automatic and algebraic Hessian allclose: False.
    No problem if ABILITY has zero effect
    """

    (
        FATHERED,
        MOTHERED,
        SIBLINGS,
        BRKNHOME,
        ABILITY,
        AGE,
        EDUC,
        POTEXPER,
        WAGE,
    ) = symbols(
        [
            "FATHERED",
            "MOTHERED",
            "SIBLINGS",
            "BRKNHOME",
            "ABILITY",
            "AGE",
            "EDUC",
            "POTEXPER",
            "WAGE",
        ]
    )

    equations = (
        # EDUC
        13
        + 0.1 * (FATHERED - 12)
        + 0.1 * (MOTHERED - 12)
        - 0.1 * SIBLINGS
        - 0.5 * BRKNHOME,
        # POTEXPER
        sympy.Max(AGE - EDUC - 5, 0),
        # WAGE
        7 + 1 * (EDUC - 12) + 0.5 * POTEXPER + 1 * ABILITY,
    )
    m = Model(
        equations=equations,
        xvars=[FATHERED, MOTHERED, SIBLINGS, BRKNHOME, ABILITY, AGE],
        yvars=[EDUC, POTEXPER, WAGE],
        final_var=WAGE,
    )

    # load and transform data
    from numpy import array, concatenate, exp, loadtxt

    xymdat = loadtxt("{}/data/education.csv".format(data_path), delimiter=",").reshape(
        -1, 10
    )
    xymdat = xymdat.T  # observations in columns
    # xymdat = xymdat[:, 0:200]      # just some of the 17,919 observations
    xdat = xymdat[[7, 6, 9, 8, 5]]  # without PERSONID, TIMETRND
    age = array(xymdat[3, :] + xymdat[1, :] + 5).reshape(
        1, -1
    )  # age = POTEXPER + EDUC + 5
    ymdat = xymdat[[1, 3, 2]]
    ymdat[2, :] = exp(ymdat[2, :])  # wage instead of log wage
    xdat = concatenate((xdat, age))

    estimate_input = dict(
        ymvars=[EDUC, POTEXPER, WAGE],
        ymdat=ymdat,
        alpha=2.637086,
        dof=0.068187,
    )

    return m, xdat, ymdat, estimate_input

def heaviside():
    """Minimal example exercise correct Heaviside(0) handling"""

    X1, Y1 = symbols(["X1", "Y1"])
    m = Model(
        xvars=[X1],
        yvars=[Y1],
        equations=(sympy.Max(X1, 0),),
        final_var=Y1,
    )

    xdat = np.array([[-1, -2, 3, 4, 5, 6]])
    ymdat = np.array([[-1.1, -2.1, 2.9, 4, 5.1, 5.9]])
    estimate_input = dict(
        ymvars=[Y1],
        ymdat=ymdat,
        alpha=None,
        dof=None,
    )
    return m, xdat, ymdat, estimate_input


import discord
import asyncio
import crawl_rss as cr
from tokens import Token

client = discord.Client()

token = Token.discord_token

async def my_background_task():
    await client.wait_until_ready()

    channel = client.get_channel(Token.discord_channel)
    await channel.send(cr.messageFormat())
    while not client.is_closed():
      print("I will die")
      await client.close()

        # await asyncio.sleep(60) # task runs every 60 seconds

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('------')

client.loop.create_task(my_background_task())
client.run(token)


import requests
from twilio.rest import Client
from decouple import config

STOCK_NAME = "TSLA"
COMPANY_NAME = "Tesla Inc"
FUNCTION = "TIME_SERIES_DAILY"

MY_API = config("MY_API", default="")
NEW_API = config("NEW_API", default="")
TWILIO_SID = config("TWILIO_SID", default="")
TWILIO_TOKEN = config("TWILIO_TOKEN", default="")

STOCK_ENDPOINT = "https://www.alphavantage.co/query"
NEWS_ENDPOINT = "https://newsapi.org/v2/everything"

parameters = {
    "function": FUNCTION,
    "symbol": STOCK_NAME,
    "apikey": MY_API
}

response = requests.get(url=STOCK_ENDPOINT, params=parameters)
response.raise_for_status()

data = response.json()["Time Series (Daily)"]
data_list = [value for (key, value) in data.items()]
yesterday_data = data_list[0]
yesterday_closing_data = yesterday_data["4. close"]
print(yesterday_closing_data)

day_before_yesterday_data = data_list[1]
day_before_yesterday_closing_price = day_before_yesterday_data["4. close"]
print(day_before_yesterday_closing_price)

difference = float(yesterday_closing_data) - float(day_before_yesterday_closing_price)
up_down = None
if difference > 0:
    up_down = "up"
else:
    up_down = "down"

diff_percent = (difference / float(yesterday_closing_data)) * 100
print(diff_percent)

if abs(diff_percent) > .1:
    news_params = {
        "apiKey": NEW_API,
        "qInTitle": COMPANY_NAME
    }

    news_response = requests.get(url=NEWS_ENDPOINT, params=news_params)
    articles = news_response.json()["articles"]
    print(articles)

    three_articles = articles[:3]
    print(three_articles)

    formatted_articles = [f"{STOCK_NAME}: {up_down}{diff_percent}%\nHeadline: {article['title']}. "
                          f"\nBrief {article['description']}" for article in three_articles]

    client = Client(TWILIO_SID, TWILIO_TOKEN)

    for article in formatted_articles:
        message = client.messages.create(
            body=article,
            from_="TWILIO PHONE NUMBER",
            to="RECEPIENT PHONE NUMBER",
        )


from django.db import models
from django.utils.html import mark_safe
from edc_base.model_mixins import BaseUuidModel
from edc_base.sites.site_model_mixin import SiteModelMixin
from edc_base.utils import get_utcnow
from edc_consent.field_mixins import VerificationFieldsMixin
from edc_identifier.model_mixins import NonUniqueSubjectIdentifierModelMixin

class ConsentCopies(VerificationFieldsMixin, NonUniqueSubjectIdentifierModelMixin,
                    SiteModelMixin, BaseUuidModel):
    version = models.CharField(
        verbose_name='Consent version',
        max_length=10,
        help_text='See \'Consent Type\' for consent versions by period.')

    @property
    def related_objects(self):
        return getattr(self, 'consent_images')

    class Meta:
        app_label = 'edc_odk'
        verbose_name = 'Consent Copies'
        verbose_name_plural = 'Consent Copies'
        unique_together = ('subject_identifier', 'version')

class ConsentImage(BaseUuidModel):
    consent_copies = models.ForeignKey(
        ConsentCopies,
        on_delete=models.PROTECT,
        related_name='consent_images', )

    image = models.FileField(upload_to='consent_copies/')

    user_uploaded = models.CharField(
        max_length=50,
        blank=True,
        verbose_name='user uploaded', )

    datetime_captured = models.DateTimeField(
        default=get_utcnow)

    def consent_image(self):
        return mark_safe(
            '<embed src="%(url)s" style="border:none" height="100" width="150"'
            'title="consent copy"></embed>' % {'url': self.image.url})

    consent_image.short_description = 'Consent Image'
    consent_image.allow_tags = True


from bokeh.plotting import figure
from bokeh.io import curdoc
from bokeh.models import GeoJSONDataSource, ColorBar
from bokeh.models import DatetimeTickFormatter, PrintfTickFormatter, NumeralTickFormatter, NumeralTickFormatter
from bokeh.models import HoverTool, WheelZoomTool
from bokeh.models import LogColorMapper, LinearColorMapper
from bokeh.models import BasicTicker
from bokeh.models import Label
from app import config as cfg
from app import constants as cts
from app import data

def plot_map(ds, palette=cts.MAP_PALLETE, breeze=cfg.MAP_BREEZE):
    """Plot cases map."""

    # Calculating map bounds
    minx, miny, maxx, maxy = ds.total_bounds
    x_range = maxx - minx
    y_range = maxy - miny
    ar = (maxx - minx) / (maxy - miny)
    x_range = (minx - breeze * x_range, maxx + breeze * x_range)
    y_range = (miny - breeze * y_range, maxy + breeze * y_range)

    # Preparing data
    ds = GeoJSONDataSource(geojson=ds.to_json())
    tools = "pan,reset,save"
    color_mapper = LogColorMapper(palette=tuple(reversed(palette)))

    color_bar = ColorBar(color_mapper=color_mapper, major_label_text_font_size="7px",
                     ticker=BasicTicker(desired_num_ticks=len(tuple(reversed(palette)))),
                     label_standoff=6, border_line_color=None, location=(0, 0))

    p = figure(tools=tools, x_range=x_range, y_range=y_range, aspect_ratio=ar,
               x_axis_location=None, y_axis_location=None, match_aspect=True)

    p.grid.grid_line_color = None
    p.toolbar.logo = None
    p.add_layout(color_bar, 'right')

    p.patches('xs', 'ys', fill_alpha=0.7,
              fill_color={'field': 'total_color', 'transform': color_mapper},
              line_color='gray', line_width=0.25, source=ds)

    p.add_tools(HoverTool(tooltips=cts.MAP_TOOLTIP,
                          point_policy="follow_mouse",
                          toggleable=False))
    # Adding zoom tool
    wheel_zoom = WheelZoomTool(zoom_on_axis=False)
    p.add_tools(wheel_zoom)
    p.toolbar.active_scroll = wheel_zoom

    # Applying theme
    doc = curdoc()
    doc.theme = cfg.THEME
    return p


import json
import re
from collections import Counter, defaultdict
from pathlib import Path
from typing import Union

import arabic_reshaper
import demoji
import seaborn as sns
from hazm import Normalizer, sent_tokenize, word_tokenize
from loguru import logger
from src.data import DATA_DIR
from wordcloud import WordCloud

class Chat_Statistics:
    """Generates chat statistics from a telegram chat json file
    """
    def __init__(self, chat_json: Union[str, Path]):
        """
        :param chat_json (Union[str, Path]): path to telegram export json file
        """

        # load chat data
        logger.info(f" loading chat data from {chat_json} ")
        with open(chat_json) as f:
            self.chat_data = json.load(f)

        self.normalizer=Normalizer()

        # load stop words
        logger.info(f" loading stop words from {DATA_DIR / 'stopwords.txt'} ")
        stop_words = open(DATA_DIR / 'stopwords.txt').readlines()
        stop_words = map(str.strip,stop_words)
        self.stop_words=set(map(self.normalizer.normalize,stop_words))

    # delete emoji
    def de_emojify(self, text):
        """Removes emojis and some special characters from the text.

        :param text: Text that contains emoji
        """
        regrex_pattern = re.compile(pattern = "[\u2069\u2066]+", flags = re.UNICODE)
        text = regrex_pattern.sub(r'', text)
        return demoji.replace(text, " ")

    @staticmethod
    def rebuild_msg(sub_messages):
        msg_text = ''
        for sub_msg in sub_messages:
            if isinstance(sub_msg,str):
                msg_text += sub_msg
        return msg_text

    def get_top_users(self,top_n: int = 10):
        """Generating top users from telegram chat
        """

        is_question = defaultdict(bool)
        for msg in self.chat_data['messages']:
            if not isinstance(msg['text'], str):
                msg['text'] = self.rebuild_msg(msg['text'])

            sentences = sent_tokenize(msg['text'])
            for sentence in sentences:
                if ('?' not in sentence) and ('؟' not in sentence):
                    continue

                is_question[msg['id']] = True
                break

        logger.info(" Genetaing top users...")

        users=[]
        for msg in self.chat_data['messages']:
            if not msg.get('reply_to_message_id'):
                continue

            if is_question[msg['reply_to_message_id']] is False:
                continue

            users.append(msg['from'])

        return dict(Counter(users).most_common(top_n))

    def generate_word_cloud(self, output_dir:Union[str, Path]):
        """ generates a word cloud from the chat data

        Args:
            output_dir (Union[str, Path]): path to output directory for word cloud image
        """
        logger.info(" loading text content... ")
        text_content=''

        for msg in self.chat_data['messages']:
            if type(msg['text']) is str:
                tokens = word_tokenize(msg['text'])
                tokens = list(filter(lambda item:item not in self.stop_words,tokens))
                text_content += f" {' '.join(tokens)}"

        #normalize, reshape for final word cloud

        text_content = arabic_reshaper.reshape(self.de_emojify(text_content))
        text_content=self.normalizer.normalize(text_content)

        #generate word cloud
        logger.info(" generating word cloud... ")
        wordcloud = WordCloud(
            width=1200, height=1200,
            font_path=str(DATA_DIR /'Vazirmatn-Light.ttf'),
            background_color='white',
            max_font_size=150
            ).generate(text_content)

        logger.info(f" saving wordcloud to {output_dir} ")
        wordcloud.to_file(str(Path(output_dir) / "wordcloud.png"))

if __name__ == '__main__':
    chat_stats=Chat_Statistics(chat_json = DATA_DIR / 'chat.json')
    chat_stats.generate_word_cloud(output_dir= DATA_DIR)
    top_users = chat_stats.get_top_users(top_n=10)

    print(top_users)
    print('Done !')


import sys
sys.path.insert(0, './src')
# import os
# os.getcwd()
# os.chdir('../')
from importlib import reload
import numpy as np
np.set_printoptions(suppress=True)
import pickle
from envs import merge
reload(merge)
from envs.merge import EnvMerge
import os
import time
import json
import matplotlib.pyplot as plt

with open('./src/envs/config.json', 'rb') as handle:
    config = json.load(handle)

env = EnvMerge(config)

def pickle_this(item, data_files_dir, item_name):
    data_files_dir += '/'+item_name+'.pickle'
    if not os.path.exists(data_files_dir):
        with open(data_files_dir, 'wb') as handle:
            pickle.dump(item, handle)
    else:
        print('This data id exists')

def prep_data(training_data):
    all_epis = np.unique(training_data[0][:, 0, 0])
    np.random.seed(2021)
    np.random.shuffle(all_epis)

    train_epis = all_epis[:int(len(all_epis)*0.7)]
    val_epis = np.setdiff1d(all_epis, train_epis)
    train_samples = np.where(training_data[0][:, 0:1, 0] == train_epis)[0]
    val_samples = np.where(training_data[0][:, 0:1, 0] == val_epis)[0]

    _, history_sca, future_sca, future_idm_s,\
                future_m_veh_c, future_e_veh_a = training_data
    train_input = [history_sca[train_samples, :, 2:],
                future_sca[train_samples, :, 2:],
                future_idm_s[train_samples, :, 2:],
                future_m_veh_c[train_samples, :, 2:],
                future_e_veh_a[train_samples, :, 2:]]

    test_input = [history_sca[val_samples, :, 2:],
                future_sca[val_samples, :, 2:],
                future_idm_s[val_samples, :, 2:],
                future_m_veh_c[val_samples, :, 2:],
                future_e_veh_a[val_samples, :, 2:]]

    return train_input, test_input
# %%
"""
Generate datan
"""
from data import merge_data_gen
reload(merge_data_gen)
from data.merge_data_gen import DataGenMerge
data_gen = DataGenMerge(env=env, episodes_n=500)
sim_data = data_gen.prep_data()
sim_data.shape

# %%
"""
Pickle generated data.
"""
data_id = '049'
dataset_name = 'sim_data_'+data_id
data_files_dir = './src/datasets/'+dataset_name
if not os.path.exists(data_files_dir):
    os.makedirs(data_files_dir)

if not os.path.exists(data_files_dir+'/sim_data.pickle'):
    with open(data_files_dir+'/sim_data.pickle', 'wb') as handle:
        pickle.dump(sim_data, handle)
else:
    print('This data id exists')

# %%
"""
Load generated data.
"""
data_id = '049'
dataset_name = 'sim_data_'+data_id
data_files_dir = './src/datasets/'+dataset_name
with open(data_files_dir+'/sim_data.pickle', 'rb') as handle:
    sim_data = pickle.load(handle)
sim_data.shape
sim_data[20, 1]
sim_data[:, 2]
sim_data[:, 2]
# %%
"""
count cars
"""
count = 0
np.unique(sim_data[sim_data[:, 0] == 1][:, 2]).shape[0]
for i in range(500):
    count += np.unique(sim_data[sim_data[:, 0] == i][:, 2]).shape[0]-2 #
count
2000 * 200 * 0.1/60**2
# %%

lens = []
for i in range(100):
    lens.append(np.unique(sim_data[sim_data[:, 0] == i][:, 1]).shape[0])
lens
plt.hist(lens, bins=50)
# %%
"""
Prepare data
"""
history_len = 30 # steps
rollout_len = 50
from data import data_prep
reload(data_prep)
from data.data_prep import DataPrep
data_prep = DataPrep()
sim_data, dummy_value_set = data_prep.fill_missing_values(sim_data)
sim_data_scaled, env_scaler, m_scaler = data_prep.scale_data(sim_data)
history_future_seqs = data_prep.sequence(sim_data, history_len, rollout_len)
history_future_seqs_scaled = data_prep.sequence(sim_data_scaled, history_len, rollout_len)
data_arrays = data_prep.split_data(history_future_seqs, history_future_seqs_scaled)

history_future_usc, history_sca, future_sca, future_idm_s, \
                future_m_veh_c, future_e_veh_a = data_arrays

history_future_usc.shape
history_future_usc[:, 2]
# %%
"""
count cars
"""
count = 0
np.unique(sim_data[sim_data[:, 0] == 1][:, 2]).shape[0]
history_future_usc = history_future_usc[:, 0, 0:3]
for i in range(500):
    count += np.unique(history_future_usc[history_future_usc[:, 0] == i][:, 2]).shape[0]
count
2000 * 50 * 0.1/(60**2)
np.unique(history_future_usc[history_future_usc[:, 0] == 9][:, 2]).shape[0]
# %%

# %%
train_input, test_input = prep_data(data_arrays)
train_input[-1].shape
# %%

"""
Make displacement a target
"""
def get_target_vals(arr):
    dxs = np.zeros([arr.shape[0], rollout_len+1, 1])

    for step in range(1, rollout_len+1):
        dxs[:, step, :] = dxs[:, step-1, :] + arr[:, step-1, 1:]*0.1 \
                                                    + 0.5*arr[:, step-1, 0:1]*0.1**2
    return dxs[:, 1:, :]
dxs = get_target_vals(train_input[-1])
train_input[-1] = np.append(train_input[-1], dxs, axis=-1)

dxs = get_target_vals(test_input[-1])
test_input[-1] = np.append(test_input[-1], dxs, axis=-1)

plt.plot(train_input[-1][0, :, -1])
plt.plot(train_input[-1][1000, :, -1])
# %%
# """
# BALANCE DATA
# """
# hf_usc_indexs = {}
# col_names = [
#          'episode_id', 'time_step',
#          'e_veh_id', 'f_veh_id', 'm_veh_id',
#          'm_veh_exists', 'e_veh_att',
#          'e_veh_speed', 'f_veh_speed', 'm_veh_speed',
#          'e_veh_action', 'f_veh_action', 'm_veh_action',
#          'aggressiveness',
#          'desired_v','desired_tgap', 'min_jamx', 'max_act', 'min_act',
#          'el_delta_v', 'el_delta_x', 'em_delta_v', 'em_delta_x',
#          'em_delta_y', 'delta_x_to_merge']
#
# for i, item_name in enumerate(col_names):
#     hf_usc_indexs[item_name] = i
# # %%
# history_future_usc, history_sca, future_sca, future_idm_s, future_m_veh_c, future_e_veh_a = data_arrays
# balance_value = np.count_nonzero((\
#                         history_future_usc[:, :, hf_usc_indexs['e_veh_att']] == 1).any(axis=1))/\
#                         history_future_usc.shape[0]
# print(balance_value)
# cond = (history_future_usc[:, :, hf_usc_indexs['e_veh_att']] == 1).any(axis=1)
# data_arrays = [np.append(sim_dataay, sim_dataay[cond], axis=0) for sim_dataay in data_arrays]
# %%
"""
Anywhere an array is fed to a model, the data is first scaled.
Scalers are pickled here.
"""
item_name = 'env_scaler'
pickle_this(env_scaler, data_files_dir, item_name)

item_name = 'm_scaler'
pickle_this(m_scaler, data_files_dir, item_name)

item_name = 'dummy_value_set'
pickle_this(dummy_value_set, data_files_dir, item_name)

item_name = 'train_input{history_len}_f{rollout_len}'.format(\
                                history_len=history_len, rollout_len=rollout_len)

pickle_this(train_input, data_files_dir, item_name)

item_name = 'test_input{history_len}_f{rollout_len}'.format(\
                                history_len=history_len, rollout_len=rollout_len)

pickle_this(test_input, data_files_dir, item_name)

item_name = 'data_arrays_h{history_len}_f{rollout_len}'.format(\
                                history_len=history_len, rollout_len=rollout_len)

pickle_this(data_arrays, data_files_dir, item_name)


from AIPUBuilder.Optimizer.utils import *
from AIPUBuilder.Optimizer.framework import *
from AIPUBuilder.Optimizer.ops.rnn import *
from AIPUBuilder.Optimizer.ops.conv import clear_lower_bits_for_bias
from AIPUBuilder.Optimizer.logger import *
import torch.nn as nn

split_weights_name = ['wx_gk', 'wh_gk', 'wx_ck', 'wh_ck']

def list_equal(left, right):
    if isinstance(left, list) and isinstance(right, list):
        return left == right
    return False

@op_register(OpType.GRUv3)
def gruv3(self, *args):
    inp0 = self.inputs[0]
    inp1 = self.inputs[1]

    current_batch_idx = self.current_batch_idx
    input_seq = inp0.betensor.float()
    initial_batch = self.inputs[1].betensor.shape[0]
    batch_size = inp0.betensor.shape[0]
    time_step = self.get_param('time_steps')
    input_size = self.get_param('input_size')
    cell_size = self.get_param('cell_size')
    direction = self.get_param('direction')
    threshold = self.get_param('threshold', optional=True, default_value=float('inf'))

    if direction == 'reverse':
        input_seq = torch.flip(input_seq, [1])

    initial_H = inp1.betensor.float()
    start_data_idx = self.current_batch_idx * batch_size

    start_initial_idx = start_data_idx % initial_batch
    in_state = initial_H[start_initial_idx: start_initial_idx + 1]
    for initial_idx in range(1, batch_size):
        current_initial_idx = (start_data_idx + initial_idx) % initial_batch
        in_state = torch.cat((in_state, initial_H[current_initial_idx: current_initial_idx + 1]), dim=0)

    w = self.constants["weights"].betensor.float()
    bias = self.constants['biases'].betensor.float()
    out_sequence = self.get_param('out_sequence')
    activations_list = self.get_param('activations') \
        if 'activations' in self.params else ['SIGMOID', 'TANH']

    state_batch = torch.zeros([batch_size, time_step, cell_size], device=inp0.betensor.device)
    state_last = torch.zeros([batch_size, cell_size], device=inp0.betensor.device)

    weights = w.permute(1, 0).float()
    gates_kernel = weights[:, :2 * cell_size]
    candidate_kernel = weights[:, 2 * cell_size:]

    if not self.quantized:
        # save weights to placeholder for caculating max_key_axis and min_key_axis through per-channel
        f_lut_in = torch.zeros([batch_size, time_step, 2*cell_size], device=inp0.betensor.device)
        f_lut_out = torch.zeros([batch_size, time_step, 2*cell_size], device=inp0.betensor.device)
        g_lut_in = torch.zeros([batch_size, time_step, cell_size], device=inp0.betensor.device)
        g_lut_out = torch.zeros([batch_size, time_step, cell_size], device=inp0.betensor.device)
        hidden_gate_out = torch.zeros([batch_size, time_step, cell_size], device=inp0.betensor.device)

        wx_gk = gates_kernel[0:input_size, :].permute(1, 0)
        wh_gk = gates_kernel[input_size:, :].permute(1, 0)
        wx_ck = candidate_kernel[0:input_size, :].permute(1, 0)
        wh_ck = candidate_kernel[input_size:, :].permute(1, 0)
        for idx, weights_name in enumerate(split_weights_name):
            if weights_name not in self.constants:
                split_weights = eval(weights_name)
                self.constants[weights_name] = PyTensor(
                    self.name+"/constants"+str(idx), split_weights.cpu().numpy().astype(dtype2nptype(Dtype.FP32)))
                self.constants[weights_name].betensor = split_weights
                self.constants[weights_name].ir_shape = TensorShape(list(split_weights.cpu().numpy().shape))
                self.constants[weights_name].ir_dtype = self.constants[weights_name].dtype

        gates_bias = bias[: 2 * cell_size]
        candidate_bias = bias[2 * cell_size: 3 * cell_size]
        for b in range(batch_size):
            state = torch.unsqueeze(in_state[b], dim=0)
            state_all = torch.zeros([1, cell_size], device=inp0.betensor.device, dtype=torch.float)
            for ts in range(time_step):
                in_ts = torch.reshape(input_seq[b, ts, :], (-1, input_size))
                if 'version' in self.params and self.params['version'] == "GRUV1":

                    gate_kernel_x_f = gates_kernel[:input_size, :]
                    gate_kernel_h_f = gates_kernel[input_size:, :]
                    candidate_kernel_x_f = candidate_kernel[:input_size, :]
                    candidate_kernel_h_f = candidate_kernel[input_size:, :]

                    hidden_bias = bias[3 * cell_size:]
                    h_by_wg_f = torch.matmul(state, gate_kernel_h_f)
                    x_by_wg_f = torch.add(torch.matmul(in_ts, gate_kernel_x_f),
                                          torch.unsqueeze(gates_bias, 0))

                    mat_sum_f = x_by_wg_f + h_by_wg_f
                    mat_sum_f = torch.clamp(mat_sum_f, -threshold, threshold)
                    f_lut_in[b, ts, :] = torch.squeeze(mat_sum_f, 0)
                    sig_lut_out_f = g_rnn_activation_func[activations_list[0]][1](mat_sum_f)
                    f_lut_out[b, ts, :] = torch.squeeze(sig_lut_out_f, 0)
                    r_f, u_f = torch.chunk(sig_lut_out_f, 2, dim=1)

                    h_by_wc_f = torch.matmul(state, candidate_kernel_h_f)
                    h_by_wc_f = torch.add(h_by_wc_f, torch.unsqueeze(hidden_bias, 0))
                    hidden_gate_out[b, ts, :] = torch.squeeze(h_by_wc_f)
                    r_hwc_f = torch.multiply(r_f, h_by_wc_f)
                    x_by_wc_f = torch.add(torch.matmul(in_ts, candidate_kernel_x_f), torch.unsqueeze(candidate_bias, 0))
                    mat_sum2_f = x_by_wc_f + r_hwc_f
                    mat_sum2_f = torch.clamp(mat_sum2_f, -threshold, threshold)
                    g_lut_in[b, ts, :] = torch.squeeze(mat_sum2_f, 0)
                    c_f = g_rnn_activation_func[activations_list[1]][1](mat_sum2_f)
                    g_lut_out[b, ts, :] = torch.squeeze(c_f, 0)
                    state = torch.multiply((1.0 - u_f), c_f) + torch.multiply(u_f, state)
                else:  # GRUV3
                    gate_input = torch.cat((in_ts, state), dim=1)
                    sum0 = torch.add(torch.matmul(gate_input, gates_kernel), torch.unsqueeze(gates_bias, 0))  # [1,1024]
                    sum0 = torch.clamp(sum0, -threshold, threshold)
                    f_lut_in[b, ts, :] = torch.squeeze(sum0, 0)
                    f_out = g_rnn_activation_func[activations_list[0]][1](sum0)
                    f_lut_out[b, ts, :] = torch.squeeze(f_out, 0)
                    r, u = torch.chunk(f_out, 2, dim=1)
                    state_r = torch.multiply(state, r)

                    candidate_input = torch.cat((in_ts, state_r), dim=1)
                    sum1 = torch.add(torch.matmul(candidate_input, candidate_kernel),
                                     torch.unsqueeze(candidate_bias, 0))
                    sum1 = torch.clamp(sum1, -threshold, threshold)
                    g_lut_in[b, ts, :] = torch.squeeze(sum1, 0)
                    c = g_rnn_activation_func[activations_list[1]][1](sum1)
                    g_lut_out[b, ts, :] = torch.squeeze(c, 0)
                    state = torch.multiply((1.0 - u), c) + torch.multiply(u, state)
                state_all = state if ts == 0 else torch.cat((state_all, state), dim=0)

            state_last = state if b == 0 else torch.cat((state_last, state), dim=0)
            state_all = torch.unsqueeze(state_all, dim=0)
            state_batch = state_all if b == 0 else torch.cat((state_batch, state_all), dim=0)
        placeholders_list = ['state_batch', 'f_lut_in', 'f_lut_out', 'g_lut_in', 'g_lut_out', 'hidden_gate_out']
        if len(self.placeholders) < len(placeholders_list):
            for placeholder_name in placeholders_list:
                ph = PyTensor(self.name + '/' + placeholder_name,
                              eval(placeholder_name).cpu().numpy().astype(dtype2nptype(Dtype.FP32)))
                self.placeholders.append(ph)
        for idx, placeholder_name in enumerate(placeholders_list):
            self.placeholders[idx].betensor = eval(placeholder_name)

    else:
        dtype = dtype2str(self.outputs[0].dtype)
        q_bits_activation = dtype2bits(self.outputs[0].dtype)
        if dtype not in ['int8', 'int16']:
            OPT_FATAL("Currently gruv3/gruv1 only  support quantization bits  of activations is 8 or 16")
        qmin, qmax = bits2range(q_bits_activation, True)

        lut_in_bits = self.inputs[0].qbits
        lut_out_bits = self.outputs[0].qbits

        biases = torch.squeeze(bias)
        gates_bias_q = biases[: 2 * cell_size]
        candidate_bias_q = biases[2 * cell_size:3 * cell_size]
        if 'version' in self.params and self.params['version'] == "GRUV1":
            hidden_bias_q = biases[3 * cell_size:]

        lut_names = ['lut_rt', 'lut_zt', 'lut_ht']
        rt_table = self.constants['lut_rt'].betensor
        zt_table = self.constants['lut_zt'].betensor
        ht_table = self.constants['lut_ht'].betensor

        wx_gk_q = gates_kernel[0:input_size, :]
        wh_gk_q = gates_kernel[input_size:, :]
        wx_ck_q = candidate_kernel[0:input_size, :]
        wh_ck_q = candidate_kernel[input_size:, :]
        if 'scale_value' in self.params:
            scale = torch.tensor(self.params["scale_value"], device=inp0.betensor.device)
            shift = torch.tensor(self.params["shift_value"], device=inp0.betensor.device)
        elif "scale" in self.constants:
            requant_scale = self.constants["scale"].betensor
            requant_shift = self.constants["shift"].betensor
            gk_step = wx_gk_q.shape[1]
            ck_step = wx_ck_q.shape[1]
            step_length = [gk_step, gk_step, ck_step, ck_step, 1, 1]
            start_idx = 0
            scale = []
            shift = []
            for idx, step in enumerate(step_length):
                scale.append(requant_scale[start_idx: start_idx + step].to(inp0.betensor.device))
                shift.append(requant_shift[start_idx: start_idx + step].to(inp0.betensor.device))
                start_idx += step
        if dtype == 'int8':
            act_qmax = 2 ** 31 - 1
            act_qmin = -2 ** 31
        elif dtype == 'int16':
            act_qmax = 2 ** 47 - 1
            act_qmin = -2 ** 47
            qmax = 2**15 - 1
            qmin = -qmax

        for b in range(batch_size):
            state = torch.unsqueeze(in_state[b], dim=0).float()
            state_all = torch.zeros([1, cell_size], device=inp0.betensor.device, dtype=torch.float)
            for ts in range(time_step):
                in_ts = torch.reshape(input_seq[b, ts, :], (-1, input_size)).float()
                x_by_wg = torch.matmul(in_ts, wx_gk_q)
                h_by_wg = torch.matmul(state, wh_gk_q)
                re_scaled_h_by_wg = linear_requantize(h_by_wg, scale[0], shift[0], 0, act_qmin, act_qmax)

                mat_sum = x_by_wg + re_scaled_h_by_wg + torch.unsqueeze(gates_bias_q, 0)
                rescaled_mat_sum = linear_requantize(mat_sum, scale[1], shift[1], 0, qmin, qmax)
                rt_zt_lut_out = lookup_lut_powerof2(rescaled_mat_sum, rt_table, lut_in_bits, True, lut_out_bits, True)
                r, u = torch.chunk(rt_zt_lut_out, 2, dim=1)

                x_by_wc = torch.matmul(in_ts, wx_ck_q)
                if 'version' in self.params and self.params['version'] == "GRUV1":
                    hidden_scale = self.params["hidden_scale_value"]
                    hidden_shift = self.params["hidden_shift_value"]
                    h_by_wc = torch.add(torch.matmul(state, wh_ck_q), torch.unsqueeze(hidden_bias_q, 0))
                    h_by_wc = linear_requantize(h_by_wc, hidden_scale, hidden_shift, 0, qmin, qmax)
                    h_by_wc = torch.multiply(r, h_by_wc)
                else:
                    factor = 256.0 if dtype == 'int8' else 65536.0
                    hprev_r = torch.round(torch.div(torch.multiply(r, state), factor))
                    h_by_wc = torch.matmul(hprev_r, wh_ck_q)
                re_scaled_h_by_wc = linear_requantize(h_by_wc, scale[2], shift[2], 0, act_qmin, act_qmax)
                mat_sum2 = x_by_wc + re_scaled_h_by_wc + torch.unsqueeze(candidate_bias_q, 0)

                rescaled_mat_sum2 = linear_requantize(mat_sum2, scale[3], shift[3], 0, qmin, qmax)
                c = lookup_lut_powerof2(rescaled_mat_sum2, ht_table, lut_in_bits, True, lut_out_bits, True)
                max_pre = qmax
                c_times_1_minus_u = (max_pre - u) * c
                rescaled_c_times_1_minus_u = linear_requantize(
                    c_times_1_minus_u, scale[4], shift[4], 0, act_qmin, act_qmax)

                state = rescaled_c_times_1_minus_u + u * state
                state = linear_requantize(state, scale[5], shift[5], 0, qmin, qmax)
                state_all = state if ts == 0 else torch.cat((state_all, state), dim=0)
            state_last = state if b == 0 else torch.cat((state_last, state), dim=0)
            state_all = torch.unsqueeze(state_all, dim=0)
            state_batch = state_all if b == 0 else torch.cat((state_batch, state_all), dim=0)

    if direction == 'reverse':
        state_batch = torch.flip(state_batch, [1])

    if list_equal(out_sequence, ['H', 'Hn']):
        self.outputs[0].betensor = state_batch
        self.outputs[1].betensor = state_last
        return (state_batch, state_last)
    elif list_equal(out_sequence, ['Hn']):
        self.outputs[0].betensor = state_last
        return state_last
    else:
        self.outputs[0].betensor = state_batch
        return state_batch

def adaptation_weights_for_unify_shifts_for_aiff(node, weights_name, multiplicator):
    w_ele_t = node.constants[weights_name]
    w_ele = w_ele_t.betensor
    do_scale, do_scale_type, do_shift, do_shift_type = unify_shifts_for_aiff_with_per_n_channel_quant(
        node, w_ele_t, node.attrs["q_mode_weight"], node.attrs["q_bits_weight"], True, lambda xt: multiplicator/xt.scale)
    scale_zp_shape = [w_ele.shape[0]] + [1 for i in range(len(w_ele.shape) - 1)]
    w_scale_expand = w_ele_t.scale.reshape(scale_zp_shape)
    w_zerop_expand = w_ele_t.zerop.reshape(scale_zp_shape)
    quantized_weight = linear_quantize_clip(w_ele, w_scale_expand, w_zerop_expand, w_ele_t.qmin, w_ele_t.qmax)
    return quantized_weight, w_ele_t.scale, w_ele_t.zerop

@quant_register(OpType.GRUv3)
def gruv3_quantize(self, *args):
    q_mode_weight = self.attrs["q_mode_weight"]
    q_mode_activation = self.attrs["q_mode_activation"]
    if QuantMode.is_per_channel(q_mode_activation) == True:
        OPT_FATAL("Currently not support per-channel quantization of activations")
    q_bits_weight = self.attrs["q_bits_weight"]
    q_bits_bias = self.attrs["q_bits_bias"]
    q_bits_activation = self.attrs["q_bits_activation"]

    dtype = dtype2str(bits2dtype(q_bits_activation, is_signed=True))
    if dtype not in ['int8', 'int16']:
        OPT_FATAL("Currently gruv3/gruv1 only  support quantization bits  of activations is 8 or 16")

    inp = self.inputs[0]
    time_step = self.get_param('time_steps')
    input_size = self.get_param('input_size')
    cell_size = self.get_param('cell_size')
    activations_list = self.get_param('activations', optional=True, default_value=['SIGMOID', 'TANH'])
    activation_alpha = self.get_param('activation_alpha', optional=True, default_value=[1, 1])
    activation_beta = self.get_param('activation_beta', optional=True, default_value=[0, 0])

    h_all = self.placeholders[0]
    h_all.scale, h_all.zerop, h_all.qmin, h_all.qmax, h_all.dtype = \
        get_linear_quant_params_from_tensor(h_all, QuantMode.to_symmetric(
            q_mode_activation), q_bits_activation, is_signed=True)
    h_all.qbits = q_bits_activation
    h_all.qinvariant = False
    h_scale = h_all.scale

    bak_inp_tensor_property = get_bak_tensor_property(self.inputs[0])
    bak_outp_tensor_property = get_bak_tensor_property(self.outputs[0])

    activations_in_scale = []
    activations_out_scale = []
    activation_idx_lut_name = {0: ('lut_rt', 'lut_zt'), 1: ('lut_ht',)}
    for activation_idx, activation in enumerate(activations_list):
        if activation in ["SIGMOID", "TANH"]:  # "SOFTPLUS", "SOFTSIGN", "ELU", "HARDSIGMOID", "SCALEDTANH", "AFFINE", "THRESHOLDEDRELU"
            activation_in_scale, activation_out_scale = \
                generate_lut_with_placeholders(self, activation_idx, activation, q_mode_activation,
                                               q_bits_activation, activation_idx_lut_name, with_lut=True, with_clamp=True, *args)
            if dtype == 'int8':
                activation_in_scale = 256 * activation_in_scale
            activations_in_scale.append(activation_in_scale)
            activations_out_scale.append(activation_out_scale)
        else:
            OPT_FATAL("Don't support activation[%s]." % (activation))

    _tensor_default_property = get_tensor_default_property()
    for p in _tensor_default_property:
        self.inputs[0].__setattr__(p, bak_inp_tensor_property[self.inputs[0].name][p])
        self.outputs[0].__setattr__(p, bak_outp_tensor_property[self.outputs[0].name][p])

    f_in_scale, g_in_scale = activations_in_scale
    f_out_scale, g_out_scale = activations_out_scale

    if self.attrs['unify_shifts_for_aiff'] and QuantMode.is_per_channel(q_mode_weight):
        # quantize wx_gk_t_scale
        quantized_wx_gk, wx_gk_t_scale, wx_gk_t_zerop = adaptation_weights_for_unify_shifts_for_aiff(
            self, 'wx_gk', f_in_scale / inp.scale)
        quantized_wh_gk, wh_gk_t_scale, wh_gk_t_zerop = adaptation_weights_for_unify_shifts_for_aiff(
            self, 'wh_gk', inp.scale * wx_gk_t_scale / h_scale)
        quantized_wx_ck, wx_ck_t_scale, wx_ck_t_zerop = adaptation_weights_for_unify_shifts_for_aiff(
            self, 'wx_ck', g_in_scale / inp.scale)
        quantized_wh_ck, wh_ck_t_scale, wh_ck_t_zerop = adaptation_weights_for_unify_shifts_for_aiff(
            self, 'wh_ck', inp.scale * wx_ck_t_scale / (h_scale * f_out_scale))
        w_scale_expand = [wx_gk_t_scale, wh_gk_t_scale, wx_ck_t_scale, wh_ck_t_scale]
        w_zerop_expand = [wx_gk_t_zerop, wh_gk_t_zerop, wx_ck_t_zerop, wh_ck_t_zerop]
        [self.constants.pop(weights_name) for weights_name in split_weights_name]
    else:
        w_scale_expand = []
        w_zerop_expand = []
        quantized_w = []
        for idx, weights_name in enumerate(split_weights_name):
            w_ele_t = self.constants[weights_name]
            w_ele = w_ele_t.betensor
            w_ele_t.scale, w_ele_t.zerop, w_ele_t.qmin, w_ele_t.qmax, w_ele_t.dtype = \
                get_linear_quant_params_from_tensor(w_ele_t, q_mode_weight, q_bits_weight, is_signed=True)
            if QuantMode.is_per_channel(q_mode_weight):
                w_scale_expand.append(w_ele_t.scale)
                w_zerop_expand.append(w_ele_t.zerop)
                scale_zp_shape = [w_ele.shape[0]] + [1 for i in range(len(w_ele.shape) - 1)]
                quantized_weight = linear_quantize_clip(w_ele, w_scale_expand[idx].reshape(scale_zp_shape),
                                                        w_zerop_expand[idx].reshape(scale_zp_shape), w_ele_t.qmin,
                                                        w_ele_t.qmax)
            else:
                w_scale_expand.append(torch.tensor(w_ele_t.scale, device=w_ele.device))
                w_zerop_expand.append(torch.tensor(w_ele_t.zerop, device=w_ele.device))
                quantized_weight = linear_quantize_clip(
                    w_ele, w_scale_expand[idx], w_zerop_expand[idx], w_ele_t.qmin, w_ele_t.qmax)
            quantized_w.append(quantized_weight)
            self.constants.pop(weights_name)
        wx_gk_t_scale, wh_gk_t_scale, wx_ck_t_scale, wh_ck_t_scale = w_scale_expand
        wx_gk_t_zerop, wh_gk_t_zerop, wx_ck_t_zerop, wh_ck_t_zerop = w_zerop_expand
        quantized_wx_gk, quantized_wh_gk, quantized_wx_ck, quantized_wh_ck = quantized_w

    gate_kernel_q = torch.cat((quantized_wx_gk, quantized_wh_gk), dim=1)
    cand_kernel_q = torch.cat((quantized_wx_ck, quantized_wh_ck), dim=1)
    quantized_weight = torch.cat((gate_kernel_q, cand_kernel_q), dim=0).contiguous()
    weights = self.constants["weights"]
    weights.scale = w_scale_expand
    weights.zerop = w_zerop_expand
    weights.betensor = quantized_weight
    weights.qbits = q_bits_weight
    weights.dtype = bits2dtype(weights.qbits, is_signed=True)
    weights.qinvariant = False

    # quantized bias
    b = self.constants["biases"]
    bias = b.betensor
    gates_bias = bias[: 2 * cell_size]
    candidate_bias = bias[2 * cell_size: 3 * cell_size]
    xwg_scale = inp.scale * wx_gk_t_scale
    hwg_scale = h_scale * wh_gk_t_scale
    # torch.min(xwg_scale, hwg_scale) if isinstance(xwg_scale, torch.Tensor) else min(xwg_scale, hwg_scale)
    gb_scale = xwg_scale
    zerop = 0
    qmin = -2**(q_bits_bias-1)
    qmax = 2**(q_bits_bias-1) - 1
    gates_bias_q = linear_quantize_clip(gates_bias, gb_scale, zerop, qmin, qmax)

    if 'version' in self.params and self.params['version'] == "GRUV1":
        factor = 1
        hidden_out_placeholders = self.placeholders[5]
        hidden_out_placeholders.scale, hidden_out_placeholders.zerop, \
            hidden_out_placeholders.qmin, hidden_out_placeholders.qmax, hidden_out_placeholders.dtype = \
            get_linear_quant_params_from_tensor(
                hidden_out_placeholders, "per_tensor_symmetric_restricted_range", q_bits_activation, is_signed=True)
        hidden_out_placeholders.qbits = q_bits_activation
        hidden_out_placeholders.qinvariant = False
        hidden_scale, hidden_scale_type, hidden_shift, hidden_shift_type = \
            get_scale_approximation_params(hidden_out_placeholders.scale / (h_scale * wh_ck_t_scale),
                                           q_bits_activation,
                                           force_shift_positive=self.force_shift_positive)
        if QuantMode.is_per_channel(q_mode_weight):
            self.constants["hidden_scale"] = PyTensor(
                self.name + "/hidden_scale", hidden_scale.cpu().numpy().astype(dtype2nptype(hidden_scale_type)))
            self.constants["hidden_scale"].dtype = hidden_scale_type
            self.constants["hidden_shift"] = PyTensor(
                self.name + "/hidden_shift", hidden_shift.cpu().numpy().astype(dtype2nptype(hidden_shift_type)))
            self.constants["hidden_shift"].dtype = hidden_shift_type
        else:
            self.params["hidden_shift_value"] = int(hidden_shift)
            self.params["hidden_shift_type"] = hidden_shift_type
            self.params["hidden_scale_value"] = int(hidden_scale)
            self.params["hidden_scale_type"] = hidden_scale_type

        hidden_bias = bias[3 * cell_size:]
        hidden_bias_scale = h_scale * wh_ck_t_scale
        hidden_bias_q = linear_quantize_clip(hidden_bias, hidden_bias_scale, zerop, qmin, qmax)
        hwc_scale = hidden_scale * f_out_scale
        quantized_bias = hidden_bias_q
    else:
        factor = 2 ** q_bits_activation
        hwc_scale = h_scale * f_out_scale * wh_ck_t_scale
        quantized_bias = torch.zeros([0], device=inp.betensor.device)

    xwc_scale = inp.scale * wx_ck_t_scale
    # torch.min(xwc_scale, hwc_scale) if isinstance(xwc_scale, torch.Tensor) else min(xwc_scale, hwc_scale)  # xwc_scale if xwc_scale <= hwc_scale else hwc_scale
    cb_scale = xwc_scale
    candidate_bias_q = linear_quantize_clip(candidate_bias, cb_scale, zerop, qmin, qmax)
    quantized_bias = torch.cat((gates_bias_q, candidate_bias_q, quantized_bias), dim=0)
    b.zerop = 0
    b.scale = [gb_scale, cb_scale]  # min(gb_scale, cb_scale)
    b.qmin = qmin
    b.qmax = qmax
    b.qbits = q_bits_bias
    b.betensor = quantized_bias
    b.dtype = bits2dtype(b.qbits, is_signed=True)
    b.qinvariant = False

    absorb_input_h_zp_to_bias(self, *args)
    clear_lower_bits_for_bias(self, *args)

    # get_scale_approximation_params = compute_requantization_param
    hwg_do_scale, hwg_do_scale_type, hwg_do_shift, hwg_do_shift_type = \
        get_scale_approximation_params(gb_scale / hwg_scale,
                                       q_bits_activation,
                                       force_shift_positive=self.force_shift_positive)
    sum_to_sigm_do_scale, sum_to_sigm_do_scale_type, sum_to_sigm_do_shift, sum_to_sigm_do_shift_type = \
        get_scale_approximation_params(f_in_scale / gb_scale,
                                       q_bits_activation,
                                       force_shift_positive=self.force_shift_positive)

    hwc_do_scale, hwc_do_scale_type, hwc_do_shift, hwc_do_shift_type = \
        get_scale_approximation_params(factor * cb_scale / hwc_scale,
                                       q_bits_activation,
                                       force_shift_positive=self.force_shift_positive)
    sum_to_tanh_do_scale, sum_to_tanh_do_scale_type, sum_to_tanh_do_shift, sum_to_tanh_do_shift_type = \
        get_scale_approximation_params(g_in_scale / cb_scale,
                                       q_bits_activation,
                                       force_shift_positive=self.force_shift_positive)
    if dtype == 'int8':
        sum_to_sigm_do_shift += 8
        sum_to_tanh_do_shift += 8

    c1mu_do_scale, c1mu_do_scale_type, c1mu_do_shift, c1mu_do_shift_type = \
        get_scale_approximation_params(h_scale / g_out_scale,
                                       q_bits_activation,
                                       force_shift_positive=self.force_shift_positive)

    hm1_do_scale, hm1_do_scale_type, hm1_do_shift, hm1_do_shift_type = \
        get_scale_approximation_params(1.0 / f_out_scale,
                                       q_bits_activation,
                                       force_shift_positive=self.force_shift_positive)

    if QuantMode.is_per_channel(q_mode_weight):
        do_scale = torch.cat((hwg_do_scale, sum_to_sigm_do_scale, hwc_do_scale, sum_to_tanh_do_scale,
                              torch.tensor([c1mu_do_scale], device=inp.betensor.device),
                              torch.tensor([hm1_do_scale], device=inp.betensor.device)))
        do_shift = torch.cat((hwg_do_shift, sum_to_sigm_do_shift, hwc_do_shift, sum_to_tanh_do_shift,
                              torch.tensor([c1mu_do_shift], device=inp.betensor.device),
                              torch.tensor([hm1_do_shift], device=inp.betensor.device)))

        _, do_scale_type = range2dtype(0, do_scale.max().item())  # bits2dtype(q_bits_activation, is_signed=False)
        _, do_shift_type = range2dtype(do_shift.min().item(), do_shift.max().item(), force_int=True)
        self.constants["scale"] = PyTensor(
            self.name+"/scale", do_scale.cpu().numpy().astype(dtype2nptype(do_scale_type)))
        self.constants["scale"].dtype = do_scale_type
        self.constants["shift"] = PyTensor(
            self.name+"/shift", do_shift.cpu().numpy().astype(dtype2nptype(do_shift_type)))
        self.constants["shift"].dtype = do_shift_type
    else:
        do_scale = torch.tensor([hwg_do_scale, sum_to_sigm_do_scale, hwc_do_scale, sum_to_tanh_do_scale,
                                 c1mu_do_scale, hm1_do_scale], device=inp.betensor.device)
        do_shift = torch.tensor([hwg_do_shift, sum_to_sigm_do_shift, hwc_do_shift, sum_to_tanh_do_shift,
                                 c1mu_do_shift, hm1_do_shift], device=inp.betensor.device)
        _, do_scale_type = range2dtype(0, do_scale.max().item())  # bits2dtype(q_bits_activation, is_signed=False)
        _, do_shift_type = range2dtype(do_shift.min().item(), do_shift.max().item(), force_int=True)
        scale_value = do_scale.cpu().numpy().astype(dtype2nptype(do_scale_type)).tolist()
        shift_value = do_shift.cpu().numpy().astype(dtype2nptype(do_shift_type)).tolist()
        self.params["shift_value"] = shift_value
        self.params["shift_type"] = [do_shift_type] * len(shift_value)
        self.params["scale_value"] = scale_value
        self.params["scale_type"] = [do_scale_type] * len(scale_value)

    for out in self.outputs:
        out.qbits = q_bits_activation
        out.scale, out.zerop, out.qmin, out.qmax, out.dtype = get_linear_quant_params_from_tensor(
            out, QuantMode.to_symmetric(q_mode_activation), out.qbits, is_signed=True)
        out.qinvariant = False


from django.shortcuts import render
from django.http import HttpResponse
from property.choice import price_choices, bedroom_choices, county_choices

from property.models import Property
from accounts.models import Owner

def index(request):
    propertys = Property.objects.order_by('-list_date')

    context = {
        'propertys': propertys,
        'county_choices': county_choices,
        'bedroom_choices': bedroom_choices,
        'price_choices': price_choices
    }

    return render(request, 'pages/index.html', context)

def about(request):
    # Get all realtors
   owners = Owner.objects.order_by('-date_joined')

    # Get MVP
   mvp_owners = Owner.objects.all().filter(is_mvp=True)

   context = {
        'owners': owners,
        'mvp_owners': mvp_owners
    }

   return render(request, 'pages/about.html', context)

# Create your views here.


from rest_framework.views import APIView
from dj_rest_auth.views import AllowAny
from django.http import FileResponse, HttpRequest, HttpResponse, JsonResponse
from rest_framework.decorators import api_view
from rest_framework.request import Request
from tempfile import NamedTemporaryFile
from docxtpl import DocxTemplate
from drfauth.models import Document

from django.contrib.staticfiles.storage import staticfiles_storage

def write_file(data, filename):
    # Convert binary data to proper format and write it on Hard Disk
    with open("staticfiles/"+filename, 'wb') as file:
        file.write(data)
class PdfView(APIView):
    permission_classes=[AllowAny]
    def get(self,request):
        one = Document.objects.get(id=1)
        write_file(one.Doc_Content,one.FileName+"."+one.Extension)
        file_path = staticfiles_storage.path(one.FileName+"."+one.Extension)
        print(file_path)
        with open(file_path, 'rb') as pdf:
            response = HttpResponse(pdf.read(), content_type='application/pdf')
            response['Content-Disposition'] = 'attachment; filename="Instrumento de Evaluacion.pdf"'
            return response

class ReporteView(APIView):
    permission_classes=[AllowAny]
    def post(self,request:Request):
        one = Document.objects.get(id=2)
        write_file(one.Doc_Content,one.FileName+"."+one.Extension)
        file_path = staticfiles_storage.path(one.FileName+"."+one.Extension)
        template = DocxTemplate(file_path)
        contextVars = request.data.copy()
        template.render(contextVars)
        template.save("staticfiles/informe"+"."+one.Extension)
        file_path = staticfiles_storage.path("informe" +"."+one.Extension)
        with open(file_path, 'rb') as pdf:
            response = HttpResponse(pdf.read(), content_type='application/vnd.openxmlformats-officedocument.wordprocessingml.document')
            response['Content-Disposition'] = 'attachment; filename="Infome.docx"'
            return response
        # return JsonResponse(request.data,safe=False)

class Reporte2View(APIView):
    permission_classes=[AllowAny]
    def post(self,request:Request):
        one = Document.objects.get(id=3)
        write_file(one.Doc_Content,one.FileName+"."+one.Extension)
        file_path = staticfiles_storage.path(one.FileName+"."+one.Extension)
        template = DocxTemplate(file_path)
        contextVars = request.data.copy()
        template.render(contextVars)
        template.save("staticfiles/reporte"+"."+one.Extension)
        file_path = staticfiles_storage.path("reporte" +"."+one.Extension)
        with open(file_path, 'rb') as pdf:
            response = HttpResponse(pdf.read(), content_type='application/vnd.openxmlformats-officedocument.wordprocessingml.document')
            response['Content-Disposition'] = 'attachment; filename="reporte.docx"'
            return response
        # return JsonResponse(request.data,safe=False)


import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
df = pd.read_csv (r'Seed Yield.csv')
print (df)
sns.set_theme(style="whitegrid")
# Draw a nested barplot by cultivar and irrigation
g = sns.catplot(
    data=df, kind="bar",
    x="Cultivar", y="Seed Yield", hue="Irrigation",
    ci="sd", palette="dark", alpha=.6, height=6
)
g.despine(left=False)
g.set_axis_labels("", "Seed Yield (kg)")
g.legend.set_title("Irrigation")
plt.show()


import json
from os import PathLike
from pathlib import Path

from typing import Any, Dict, List, MutableMapping, Union

import yaml

STRUCTURE = {
    "Dataset": [
        "dataset_name",
        "dataset_config",
        "dataset_revision",
        "labelcolumn",
        "textcolumn",
    ],
    "Training": [
        "optim",
        "learning_rate",
        "per_device_train_batch_size",
        "per_device_eval_batch_size",
        "gradient_accumulation_steps",
        "max_steps",
        "save_steps",
        "metric_for_best_model",
    ],
    "Best checkedpoint based on validation": [
        "best_metric",
        "best_model_checkpoint",
    ]
}

def rec_merge(d1, d2):
    '''
    https://stackoverflow.com/a/24088493/1150683
    Update two dicts of dicts recursively,
    if either mapping has leaves that are non-dicts,
    the second's leaf overwrites the first's.
    '''
    for k, v in d1.items():
        if k in d2:
            # this next check is the only difference!
            if all(isinstance(e, MutableMapping) for e in (v, d2[k])):
                d2[k] = rec_merge(v, d2[k])
            # we could further check types and merge as appropriate here.
    d3 = d1.copy()
    d3.update(d2)
    return d3

def format_dict(d: Dict[str, Any]):
    return "\n".join(f"- {k}: {v}" for k, v in d.items())

def get_modelcard(modelcard_base: Union[str, PathLike]):
    pmodelcard = Path(modelcard_base)
    modelcard = yaml.safe_load(pmodelcard.read_bytes())
    return modelcard

def generate_readme(din: Union[str, PathLike], modelcard_base: Union[str, PathLike], overwrite_readme: bool = False):
    pdin = Path(din).resolve()
    preadme = pdin / "README.md"
    mode = "w" if overwrite_readme else "a"

    test_results: Dict[str, float] = json.loads(pdin.joinpath("test_results.json").read_bytes())
    env: Dict[str, Any] = json.loads(pdin.joinpath("env.json").read_bytes())
    trainer_state: Dict[str, Any] = json.loads(pdin.joinpath("trainer_state.json").read_bytes())
    args: Dict[str, Any] = json.loads(pdin.joinpath("args.json").read_bytes())
    args = {**args, **trainer_state}

    with preadme.open(mode=mode, encoding="utf-8") as fhout:
        modelcard = get_modelcard(modelcard_base)
        modelcard["datasets"] = [args["dataset_name"]]
        modelcard["metrics"] = list(test_results.keys())
        model_index = {
            "name": f"{args['model_name_or_path'].split('/')[-1]}-{args['dataset_name'].split('/')[-1]}",
            "results": [{
                "task": {
                    "type": modelcard["model-index"][0]["results"][0]["task"]["type"],
                    "name": modelcard["model-index"][0]["results"][0]["task"]["name"],
                },
                "dataset": {
                    "type": args["dataset_name"],
                    "name": f"{args['dataset_name']} - {args['dataset_config']} - {args['dataset_revision']}",
                    "config": args['dataset_config'],
                    "split": args["testsplit_name"],
                    "revision": args['dataset_revision']
                },
                "metrics": [{
                    "type": m,
                    "value": v,
                    "name": f"{args['testsplit_name'].capitalize()} {m}"
                } for m, v in test_results.items()]
            }]
        }
        modelcard["model-index"] = [rec_merge(modelcard["model-index"][0], model_index)]

        fhout.write(f"---\n{yaml.dump(modelcard)}---\n\n")

        fhout.write(f"# {args['model_name_or_path'].split('/')[-1]}-{args['dataset_name'].split('/')[-1]}\n\n")
        for title, keys in STRUCTURE.items():
            temp_d = {k: v for k, v in args.items() if k in keys}
            temp_d = {k: v for k, v in sorted(temp_d.items(), key=lambda k: keys.index(k[0]))}
            fhout.write(f"# {title}\n{format_dict(temp_d)}\n\n")

        fhout.write(f"# Test results of best checkpoint\n{format_dict(test_results)}\n\n")
        fhout.write(f"## Confusion matric\n\n")
        fhout.write("![cfm](fig/test_confusion_matrix.png)\n\n")
        fhout.write(f"## Normalized confusion matrix\n\n")
        fhout.write("![norm cfm](fig/test_confusion_matrix_norm.png)\n\n")
        fhout.write(f"# Environment\n{format_dict(env)}\n\n")

    print("Done! Verify the README before uploading!!!")

def main():
    import argparse

    cparser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    cparser.add_argument("din", help="Input directory that contains the model and all output jsons."
                                     "Will also write README.md here.")
    cparser.add_argument("modelcard_base", help="Path to model card base YAML.")
    cparser.add_argument("-f", "--overwrite_readme", help="Overwrite a README if it already exists.",
                         action="store_true")

    cargs = cparser.parse_args()
    generate_readme(**vars(cargs))

if __name__ == "__main__":
    main()


import os
import random

import torch
import torchvision.transforms as T
from PIL import Image
from torch.utils import data

def rreplace(s, old, new, occurrence):
    li = s.rsplit(old, occurrence)
    return new.join(li)

class ImageAttr(data.Dataset):
    """Dataset class for the ImageAttr dataset."""
    def __init__(self, image_dir, attr_path, transform, mode,
                 binary=False, n_style=4,
                 char_num=52, unsuper_num=968, train_num=120, val_num=28):
        """Initialize and preprocess the ImageAttr dataset."""
        self.image_dir = image_dir
        self.attr_path = attr_path
        self.n_style = n_style

        self.transform = transform
        self.mode = mode
        self.binary = binary

        self.super_train_dataset = []
        self.super_test_dataset = []
        self.unsuper_train_dataset = []

        self.attr2idx = {}
        self.idx2attr = {}

        self.char_num = char_num
        self.unsupervised_font_num = unsuper_num
        self.train_font_num = train_num
        self.val_font_num = val_num

        self.test_super_unsuper = {}
        for super_font in range(self.train_font_num+self.val_font_num):
            self.test_super_unsuper[super_font] = random.randint(0, self.unsupervised_font_num - 1)

        self.char_idx_offset = 10

        self.chars = [c for c in range(self.char_idx_offset, self.char_idx_offset+self.char_num)]

        self.preprocess()

        if mode == 'train':
            self.num_images = len(self.super_train_dataset) + len(self.unsuper_train_dataset)
        else:
            self.num_images = len(self.super_test_dataset)

    def preprocess(self):
        """Preprocess the font attribute file."""
        lines = [line.rstrip() for line in open(self.attr_path, 'r')]
        all_attr_names = lines[0].split()
        for i, attr_name in enumerate(all_attr_names):
            self.attr2idx[attr_name] = i
            self.idx2attr[i] = attr_name

        lines = lines[1:]

        train_size = self.char_num * self.train_font_num
        val_size = self.char_num * self.val_font_num

        for i, line in enumerate(lines):
            split = line.split()
            filename = split[0]
            values = split[1:]
            target_char = filename.split('/')[1].split('.')[0]
            char_class = int(target_char) - self.char_idx_offset
            font_class = int(i / self.char_num)

            attr_value = []
            for val in values:
                if self.binary:
                    attr_value.append(val == '1')
                else:
                    attr_value.append(eval(val) / 100.0)

            # print(filename, char_class, font_class, attr_value)

            if i < train_size:
                self.super_train_dataset.append([filename, char_class, font_class, attr_value])
            elif i < train_size + val_size:
                self.super_test_dataset.append([filename, char_class, font_class, attr_value])
            else:
                self.unsuper_train_dataset.append([filename, char_class, font_class, attr_value])

        print('Finished preprocessing the Image Attribute (Explo) dataset...')

    def __getitem__(self, index):
        """Return one image and its corresponding attribute label."""
        # dataset = self.super_train_dataset if self.mode == 'train' else self.super_test_dataset

        if self.mode == 'train':
            if index < len(self.super_train_dataset):
                filename_A, charclass_A, fontclass_A, attr_A = self.super_train_dataset[index]
                label_A = 1.0
                font_embed_A = self.unsupervised_font_num  # dummy id 968
                # B is supervised or unsupervised
                sample_p = random.random()
                if sample_p < 0.5:
                    # Unsupervise
                    index_B = index % self.char_num + self.char_num * random.randint(0, self.unsupervised_font_num - 1)
                    filename_B, charclass_B, fontclass_B, attr_B = self.unsuper_train_dataset[index_B]
                    label_B = 0.0
                    font_embed_B = fontclass_B - self.train_font_num - self.val_font_num  # convert to [0, 967]
                else:
                    # Supervise
                    # get B from supervise train !!
                    index_B = index % self.char_num + self.char_num * random.randint(0, self.train_font_num - 1)
                    filename_B, charclass_B, fontclass_B, attr_B = self.super_train_dataset[index_B]
                    label_B = 1.0
                    font_embed_B = self.unsupervised_font_num  # dummy id 968

            else:
                # get A from unsupervise train !!
                index = index - len(self.super_train_dataset)
                filename_A, charclass_A, fontclass_A, attr_A = self.unsuper_train_dataset[index]
                label_A = 0.0
                font_embed_A = fontclass_A - self.train_font_num - self.val_font_num
                # B is supervised or unsupervised
                sample_p = random.random()
                if sample_p < 0.5:
                    # Unsupervise
                    index_B = index % self.char_num + self.char_num * random.randint(0, self.unsupervised_font_num - 1)  # noqa
                    filename_B, charclass_B, fontclass_B, attr_B = self.unsuper_train_dataset[index_B]
                    label_B = 0.0
                    font_embed_B = fontclass_B - self.train_font_num - self.val_font_num  # convert to [0, 967]
                else:
                    # Supervise
                    # get B from supervise train !!
                    index_B = index % self.char_num + self.char_num * random.randint(0, self.train_font_num - 1)
                    filename_B, charclass_B, fontclass_B, attr_B = self.super_train_dataset[index_B]
                    label_B = 1.0
                    font_embed_B = self.unsupervised_font_num  # dummy id 968

        else:
            # load the random one from unsupervise data as the reference aka A
            # unsuper to super
            font_index_super = index // self.char_num + self.train_font_num
            font_index_unsuper = self.test_super_unsuper[font_index_super]
            char_index_unsuper = index % self.char_num + self.char_num * font_index_unsuper
            filename_A, charclass_A, fontclass_A, attr_A = self.unsuper_train_dataset[char_index_unsuper]
            label_A = 0.0
            font_embed_A = fontclass_A - self.train_font_num - self.val_font_num  # convert to [0, 967]

            filename_B, charclass_B, fontclass_B, attr_B = self.super_test_dataset[index]
            label_B = 1.0
            font_embed_B = self.unsupervised_font_num  # dummy id 968

        # Get style samples
        random.shuffle(self.chars)
        style_chars = self.chars[:self.n_style]
        styles_A = []
        if self.n_style == 1:
            styles_A.append(filename_A)
        else:
            for char in style_chars:
                styles_A.append(rreplace(filename_A, str(charclass_A+10), str(char), 1))

        random.shuffle(self.chars)
        style_chars = self.chars[:self.n_style]
        styles_B = []
        if self.n_style == 1:
            styles_B.append(filename_B)
        else:
            for char in style_chars:
                styles_B.append(rreplace(filename_B, str(charclass_B+10), str(char), 1))

        image_A = Image.open(os.path.join(self.image_dir, filename_A)).convert('RGB')
        image_B = Image.open(os.path.join(self.image_dir, filename_B)).convert('RGB')
        # Open and transform style images
        style_imgs_A = []
        for style_A in styles_A:
            style_imgs_A.append(self.transform(Image.open(os.path.join(self.image_dir, style_A)).convert('RGB')))
        style_imgs_A = torch.cat(style_imgs_A)
        style_imgs_B = []
        for style_B in styles_B:
            style_imgs_B.append(self.transform(Image.open(os.path.join(self.image_dir, style_B)).convert('RGB')))
        style_imgs_B = torch.cat(style_imgs_B)

        return {"img_A": self.transform(image_A), "charclass_A": torch.LongTensor([charclass_A]),
                "fontclass_A": torch.LongTensor([fontclass_A]), "attr_A": torch.FloatTensor(attr_A),
                "styles_A": style_imgs_A,
                "fontembed_A": torch.LongTensor([font_embed_A]),
                "label_A": torch.FloatTensor([label_A]),
                "img_B": self.transform(image_B), "charclass_B": torch.LongTensor([charclass_B]),
                "fontclass_B": torch.LongTensor([fontclass_B]), "attr_B": torch.FloatTensor(attr_B),
                "styles_B": style_imgs_B,
                "fontembed_B": torch.LongTensor([font_embed_B]),
                "label_B": torch.FloatTensor([label_B])}

    def __len__(self):
        """Return the number of images."""
        return self.num_images

def get_loader(image_dir, attr_path, image_size=256,
               batch_size=16, dataset_name='explor_all', mode='train', num_workers=8,
               binary=False, n_style=4,
               char_num=52, unsuper_num=968, train_num=120, val_num=28):
    """Build and return a data loader."""
    transform = []
    transform.append(T.Resize(image_size))
    transform.append(T.ToTensor())
    transform.append(T.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)))
    transform = T.Compose(transform)

    if dataset_name == 'explor_all':
        dataset = ImageAttr(image_dir, attr_path, transform,
                            mode, binary, n_style,
                            char_num=52, unsuper_num=968,
                            train_num=120, val_num=28)
    data_loader = data.DataLoader(dataset=dataset,
                                  drop_last=True,
                                  batch_size=batch_size,
                                  shuffle=(mode == 'train'),
                                  num_workers=num_workers)

    return data_loader


from typing import Optional

from fastapi import FastAPI
from pydantic import BaseModel

app = FastAPI()
persons = [{"id": 1, "ad": "Seyma", "soyad": "Sarigil", "meslek": "Gelistirici", "memleket": "Hatay"},
           {"id": 2, "ad": "Alp", "soyad": "Kara", "meslek": "Müzisyen", "memleket": "İstanbul"}]

class Person(BaseModel):
    id: Optional[int] = None
    ad: str
    soyad: str
    meslek: str
    memleket: str

@app.get("/")
def get_all_persons():
    return persons

@app.post("/add_person/")
def add_person(person: Person): #ad, soyad, meslek, memleket
    person_dict = person.dict()
    person_dict["id"] = persons[-1]["id"] + 1
    persons.append(dict(person_dict))
    return persons

@app.get("/get_person/{person_id}/")
def get_specific_person(person_id: int):
    person = [i for i in persons if i["id"] == person_id]
    return person

@app.put("/update_person/{person_id}/")
def update_name_person(person_id: int, person: Person):
    p = [i for i in persons if person.id == person_id]
    p[0]["ad"] = person.ad
    return persons

@app.delete("/delete_person/{person_id}/")
def delete_person(person_id: int):
    person = [i for i in persons if i["id"] == person_id]
    persons.remove(person[0])
    return person


from rest_framework_simplejwt.serializers import TokenObtainPairSerializer
from rest_framework_simplejwt.views import TokenObtainPairView
from rest_framework import exceptions

class StaffTokenObtainPairSerializer(TokenObtainPairSerializer):
    def validate(self, attrs):
        data = super().validate(attrs)
        if self.user.is_employee == True:
            refresh = self.get_token(self.user)
            data['refresh'] = str(refresh)
            data['access'] = str(refresh.access_token)
            data['work_area'] = self.user.employees.work_area.name
            data['fullname'] = self.user.fullname
            return data
        else:
            raise exceptions.AuthenticationFailed(
                self.error_messages['no_active_account'],
                'no_active_account',
            )

class StaffTokenObtainPairView(TokenObtainPairView):
    serializer_class = StaffTokenObtainPairSerializer


## set up logging
import logging, os
logging.basicConfig(level=os.environ.get("LOGLEVEL","INFO"))
log = logging.getLogger("glam_command_line")

import argparse, glob, json, subprocess, sys
import glam_data_processing.legacy as glam
from datetime import datetime

# create temporary directory for shenanigans
TEMP_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),"temp/")
try:
	os.mkdir(TEMP_DIR)
except FileExistsError:
	pass

def statsOneFile(file_path,admin_level, mask_level,admin_specified=None,mask_specified=None) -> bool:
	base = os.path.basename(file_path)
	log.info(f"Processing {base}")
	try:
		img = glam.getImageType(file_path)(file_path) # create Image object
		img.uploadStats(admin_level=admin_level,crop_level=mask_level,admin_specified=admin_specified,crop_specified=mask_specified)
		return True
	except:
		log.exception(f"FAILED in processing {base}")
		return False

def download_and_statsOneFile(product,date,directory,admin_level,mask_level,admin_specified=None,mask_specified=None,save=False) -> bool:
	"""Wrapper for statsOneFile in case downloading is required"""
	paths = ()
	try:
		log.info(f"Downloading {product} {date}")
		downloader = glam.Downloader()
		paths = downloader.pullFromS3(product,date,directory)
		log.info(f"Processing {product} {date}")
		for p in paths:
			assert statsOneFile(file_path=p,admin_level=admin_level,mask_level=mask_level,admin_specified=admin_specified,mask_specified=mask_specified)
		downloader = None
		return True
	except:
		log.exception(f"{product} {date} FAILED")
		return False
	finally:
		if not save:
			for p in paths:
				os.remove(p)

def getAllTiffs(in_dir) -> list:
	"""Takes input directory path, returns list of all tiff full paths"""
	return glob.glob(os.path.join(in_dir,"*.tif"))

def getProductDateTuple(file_path) -> tuple:
	"""extracts the product and date from a file path; returns as a tuple"""
	try:
		base = os.path.basename(file_path)
		parts= base.split(".")
		product = parts[0]
		try:
			date = datetime.strptime(f"{parts[1]}.{parts[2]}","%Y.%j").strftime("%Y-%m-%d")
		except:
			date = datetime.strptime(parts[1],"%Y-%m-%d").strftime("%Y-%m-%d")
		return (product,date)
	except:
		log.exception(f"Failed to extract product-date tuple for {os.path.basename(file_path)}")

def getUniqueFilename(directory,file_name) -> str:
	"""given desired directory and file name, iterates through
	integers until it finds a unique file_name_{i}.ext"""
	base,ext=os.path.splitext(file_name)
	outPath = os.path.join(directory,file_name)
	i = 0
	while os.path.exists(outPath):
		i += 1
		outPath = os.path.join(directory,f"{base}_{i}{ext}")
	return outPath

def main():
	## parse arguments
	parser = argparse.ArgumentParser(description="Generate new statistics, in parallel")
	parser.add_argument("file_directory",
		help="Directory where imagery files are / will be stored")
	parser.add_argument("-al",
		"--admin_level",
		default="ALL",
		choices=["ALL","GAUL","BRAZIL"],
		help="Run statistics for only a subset of administrative divisions")
	parser.add_argument("-as",
		"--admin_specified",
		choices = glam.admins,
		help="Run statistics for a single administrative division")
	parser.add_argument("-ml",
		"--mask_level",
		default="ALL",
		choices=["ALL","BRAZIL","CROPMONITOR","NOMASK"],
		help="Run statistics for only a subset of crop masks")
	parser.add_argument("-ms",
		"--mask_specified",
		choices=glam.crops,
		help="Run statistics for a single crop mask")
	parser.add_argument("-c",
		"--cores",
		required=True,
		help="How many cores to use")
	parser.add_argument("-l",
		"--logfile",
		help="Where to write output log messages")
	parser.add_argument("-d",
		"--download_files",
		action='store_true',
		help="Download any files not currently on disk. Pull them from S3 before running stats. Requires product flag.")
	parser.add_argument('-p',
		"--product",
		help="Which product to pull. Required if --download_files is set.")
	parser.add_argument("-s",
		"--save_results",
		action='store_true',
		help="If set, does not delete downloaded files")
	parser.add_argument("-mo",
		"--missing_only",
		action='store_true',
		help="Generate statistics only for files NOT currently on disk in file_directory")
	parser.add_argument("-META",
		metavar='metastring',
		default = None,
		action='store',
		help="Do not set this flag. For internal use only.")

	args = parser.parse_args()

	## META stores file path or product/date as json
	if args.META:
		meta_parts = args.META.split(".")
		if args.download_files:
			product = meta_parts[0]
			date = meta_parts[1]
			#print(product)
			#print(date)
			download_and_statsOneFile(product,date,args.file_directory,args.admin_level,args.mask_level,args.admin_specified,args.mask_specified,args.save_results)
		else:
			in_file = args.META
			#print(in_file)
			statsOneFile(in_file, args.admin_level,args.mask_level,args.admin_specified,args.mask_specified)
		sys.exit()

	## confirm argument validity
	# download_files requires product
	try:
		if args.download_files:
			assert args.product
	except AssertionError:
		log.error("If download_files is set, a product must be specified.")
		sys.exit()
	# no overlap of BRAZIL crops and GAUL admins
	if args.mask_level == "BRAZIL" and args.admin_level == "GAUL":
		log.error("Can't generate Brazil crop stats for Gaul regions")
		sys.exit()
	# don't set both _level and _specified
	if args.admin_specified and (args.admin_level != "ALL"):
		log.error("Do not set both --admin_level and --admin_specified")
		sys.exit()
	if args.mask_specified and (args.mask_level != "ALL"):
		log.error("Do not set both --mask_level and --mask_specified")
		sys.exit()
	# warn the user if arguments are being ignored
	if not args.download_files:
		try:
			assert not args.save_results
		except AssertionError:
			log.warning("--download_files not set; --save_results flag ignored")
		try:
			assert not args.missing_only
		except AssertionError:
			log.warning("--download_files not set; --missing_only flag ignored")
	if not args.logfile:
		args.logfile = getUniqueFilename(TEMP_DIR,"gns_log.txt")

	lines = []
	## perform stats on existing files
	extant = getAllTiffs(args.file_directory)
	if not args.missing_only:
		for f in extant:
			line = f"python {os.path.abspath(__file__)} {args.file_directory} -al {args.admin_level} -ml {args.mask_level} -c {args.cores} -META '{f}'"
			if args.save_results:
				line += " -s"
			if args.admin_specified:
					line += f" -as {args.admin_specified}"
			if args.mask_specified:
				line += f" -ms {args.mask_specified}"
			line += "\n"
			lines += line

	## download and process new files if requested
	if args.download_files:
		downloader= glam.Downloader()
		onDisk = [getProductDateTuple(f) for f in extant]
		#available = downloader.getAllS3(product=args.product)
		#missing = [t for t in available if t not in onDisk]
		missing = downloader.listMissing(args.file_directory)
		if len(missing) == 0:
			log.info(f"No missing files; all available files on S3 are also in {args.file_directory}")
		else:
			for t in missing:
				product,date=t
				meta = f"{product}.{date}"
				line = f"python {os.path.abspath(__file__)} {args.file_directory} -al {args.admin_level} -ml {args.mask_level} -c {args.cores} -d -META {meta}"
				if args.save_results:
					line += " -s"
				if args.admin_specified:
					line += f" -as {args.admin_specified}"
				if args.mask_specified:
					line += f" -ms {args.mask_specified}"
				line += " \n"
				lines += line

			downloader=None
	command_file = getUniqueFilename(TEMP_DIR,"gns_commands.txt")

	with open(command_file,'w') as wf:
		wf.writelines(lines)
	shell_call = ["nohup","sh","-c", f'"cat {command_file} | parallel -j {args.cores}"', "&>",args.logfile,"&"]
	print(" ".join(shell_call))
	subprocess.call(shell_call)
	#os.remove(command_file)

if __name__ == "__main__":
	main()


from aiogram import types
from aiogram.dispatcher import FSMContext
from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton

from data.config import admins
from keyboards.default.menu_keyboards import menu
from keyboards.inline.callback_datas import report_data
from loader import dp, db, bot
from states.menu_states import Reports

@dp.callback_query_handler(text='cancel_filter_change', state=Reports.WaitReport)
async def cancel_report(call: types.CallbackQuery, state: FSMContext):
    """Отмена отправки репорта"""
    await call.message.answer("Отмена", reply_markup=menu)
    await call.message.edit_reply_markup()
    await state.finish()

@dp.message_handler(state=Reports.WaitReport)
async def get_cancel(message: types.Message, state: FSMContext):
    """Нажатие на кнопку Главное меню"""
    report = message.text
    await db.add_report(user_id=message.from_user.id, name=message.from_user.full_name, status_report=False,
                        report=report)
    report_id = await db.sellect_last_report_id()
    for user in admins:
        await bot.send_message(chat_id=user, text=f"Новое сообщение об ошибке!\n\nid репорта: {report_id}\n{report}",
                               reply_markup=InlineKeyboardMarkup(row_width=2,
                                                                 inline_keyboard=[
                                                                     [
                                                                         InlineKeyboardButton(
                                                                             text='Отметить как исправленный',
                                                                             callback_data=report_data.new(
                                                                                report_id=report_id,
                                                                                user_id=message.from_user.id)
                                                                         )
                                                                     ]
                                                                 ]))
    await message.answer('Спасибо за Ваше сообщение!\n'
                         f'Номер Вашего отчета: {report_id}\n'
                         'Мы исправим ошибку в ближайшее время',
                         reply_markup=menu)
    await state.finish()