import os

import feedparser
from django.http import HttpRequest
from django.shortcuts import render
from datetime import datetime

# Create your views here.

from lxml import etree
from tp1.settings import BASE_DIR

fname = 'mondial.xml'
pname = os.path.join(BASE_DIR, 'app/data/' + fname)
tree = etree.parse(pname)
root = tree.getroot()

def home(request):
    tparams = {
        'title': 'Home Page',
        'year': datetime.now().year,
    }
    return render(request, 'index.html', tparams)

def about(request):
    tparams = {
        'title': 'About',
        'message': 'Acerca do projecto',
        'year': datetime.now().year,
    }
    return render(request, 'about.html', tparams)

def list(request, selection):
    #print(selection)
    xsl = "simpleList.xsl"

    if selection == "continent" or selection == "country" or selection == "organization":
        xpath = f'//{selection}'

    elif ("cty" in selection) or ("city" in selection) or ("stadt" in selection):
        xpath = f'//*[@id="{selection}"]'
        xsl = 'city.xsl'

    elif ("europe" in selection) or ("asia" in selection) or ("africa" in selection) or ("america" in selection) or ("australia" in selection):
        xpath = f'//country/encompassed[@continent="{selection}"]/parent::*'

    elif "org" in selection:
        xpath = f'//organization[@id="{selection}"]'
        xsl = 'organization.xsl'

    else:
        xpath = f'//country[@id="{selection}"]'
        xsl = 'country.xsl'
    #print(xpath)

    xslt_tree = etree.parse(os.path.join(BASE_DIR, 'app/data/' + xsl))
    transform = etree.XSLT(xslt_tree)
    print(transform(root, selection=xpath))
    tparams = {
        'year': datetime.now().year,
        'current': selection,
        'page': transform(root, selection=xpath),
    }

    return render(request, 'xsltDisplay.html', tparams)

def rss(request):
    feeds = feedparser.parse('https://news.google.com/rss?hl=en-US&gl=US&ceid=US%3Aen&x=1571747254.2933')

    tparams = {
        'title': 'RSS',
        'message': 'Noticias da Imprensa Americana',
        'year': datetime.now().year,
        'feeds': feeds,
    }
    return render(request, 'rss.html', tparams)


import os
import string
import numpy as np

from geospatialtools import gdal_tools
# import tensorflow as tf
import demfuncs as dem
import pickle
import pandas as pd
import matplotlib.pyplot as plt

import matplotlib
matplotlib.use('Agg')
# matplotlib.use('Qt5Agg') # dyn show plots

dem.matplotlib_update_settings()

# ------------------------------------------------------------------------------
# Functions needed for analyzing and plotting the results
# of nate's GFDL preprocessing module
# ------------------------------------------------------------------------------

def dprint(dict):
    for ki in list(dict.keys()):
        v = dict[ki]
        print('{}:: mean = {:.4f}, stdv = {:.4f}, min = {:.4f}, max = {:.4f}'.format(
            ki, np.mean(v), np.std(v), np.min(v), np.max(v)))

def skew(sample):

    # sample1 = sample[~np.isnan(sample)]
    sigma = np.nanstd(sample)
    mu = np.nanmean(sample)
    xn = (sample - mu)/sigma
    myskew = np.nanmean( xn**3)
    return myskew

def kurt(sample):
    # sample1 = sample[~np.isnan(sample)]
    sigma = np.nanstd(sample)
    mu = np.nanmean(sample)
    xn = (sample - mu) / sigma
    mykurt = np.nanmean( xn ** 4)
    return mykurt

def comp_4_stats(sample, weights = None):
    res = {}
    if weights is None:
        res['mean'] = np.nanmean(sample)
        res['stdv'] = np.nanstd(sample)
        res['skew'] = skew(sample)
        res['kurt'] = kurt(sample)
    else:
        # do it only for vectors (=> list of tiles)
        assert np.size(weights) == np.size(sample)
        assert np.ndim(weights) == 1
        res['mean'] = weighted_stats(sample, weights, k=1)
        res['stdv'] = weighted_stats(sample, weights, k=2)
        res['skew'] = weighted_stats(sample, weights, k=3)
        res['kurt'] = weighted_stats(sample, weights, k=4)
    return res

def weighted_stats(x, f, k=1):
    # x -> samples, f -> weights, k -> order
    ### TEST::
    # A = np.array([-9999, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 4, 4, 4, 7, 7, 12, 12, 121])
    # X = np.unique(A)
    # F = np.array([np.size(A[A == x]) / np.size(A) for x in X])
    # res = til.comp_4_stats(X, weights=F)
    # print(res['mean'], np.mean(A))
    # print(res['stdv'], np.std(A))
    # print(res['skew'], til.skew(A))
    # print(res['kurt'], til.kurt(A))
    if k == 1:
        res = np.sum(x*f)
    elif k==2:
        mu = weighted_stats(x, f, k=1)
        res = np.sqrt( np.sum(f*(x-mu)**2) )
    elif k==3:
        mu = weighted_stats(x, f, k=1)
        sigma = weighted_stats(x, f, k=2)
        res = np.sum(f*(x-mu)**3)/sigma**3
    elif k==4:
        mu = weighted_stats(x, f, k=1)
        sigma = weighted_stats(x, f, k=2)
        res = np.sum(f*(x-mu)**4)/sigma**4
    else:
        print("k={}".format(k))
        # raise Exception("order k must be in [1,2,3,4]")
    return res

def maphrus(hrus, hruprops,
            prop='hillslope_aspect'):
    """ map a terrain properties based on the map of tiles
      and the avergae property for each tile"""
    ny, nx = hrus.shape
    print(ny, nx)
    map_prop = np.zeros((ny, nx) ) *np.nan
    for i in range(ny):
        for j in range(nx):
            my_hru_index = hrus[i, j]
            if not np.isnan(my_hru_index) and my_hru_index > -1:
                my_hru_index = hrus[i ,j] - 1
                my_value = hruprops[prop][my_hru_index]
                map_prop[i ,j] = my_value
    return map_prop

def read_tile_properties(datadir=None, do_averaging=True, aveblock=110,
                         cosz = None, phi = None, adir=None,
                         modeldir=None):
    """-------------------------------------------------------------------------
    Read the output of Nate's GFDL: preprocessing stored in folder datadir
    read the maps of terrain quantities of interest
    and the tile-by-tile average values of these quantities

    cosz, phi -> Zenith and azimuth angles needed to compute the
    tile-by-tile solar incidence angle

    modeldir -> folder with the models used to compute rad fluxes differences
    -------------------------------------------------------------------------"""
    landfolder = os.path.join(datadir, 'land', 'tile:1,is:1,js:1')

    # read tiles and their areal-average properties
    with open(os.path.join(landfolder, 'hrus.pck'), "rb") as hru_file:
        hrus = pickle.load(hru_file)
        # print("shape of hrus = {}".format(hrus.shape))
        # hrus = np.flipud(hrus)
    with open(os.path.join(landfolder, 'hru_properties.pck'), "rb") as hru_file2:
        hruprops = pickle.load(hru_file2)

    # hillsfile = os.path.join(landfolder, 'hillslopes_ea.tif')
    # hillsfile0 = os.path.join(landfolder, 'ti_ea.tif')
    hillsfile = os.path.join(landfolder, 'tiles.tif')
    # hills0 = gdal_tools.read_data(hillsfile0).data
    hills = gdal_tools.read_data(hillsfile).data

    # with open(os.path.join(landfolder, 'hillslope_properties.pck')) as hill_file:
    #     hillprops = pickle.load(hill_file)

    # matplotlib.use('Qt5Agg') # dyn show plots
    # plt.figure()
    # plt.imshow(hills0)
    # plt.show()
    # hills[hills<0] = -1
    # plt.figure()
    # plt.imshow(hills)
    # plt.colorbar()
    # plt.show()
    # print('hills plot')

    # read the DEM or some other dataset in the equal area projection
    elevfile_latlon = os.path.join(landfolder, 'dem_latlon.tif')
    elevfile = os.path.join(landfolder, 'demns_ea.tif')
    # cossfile = os.path.join(landfolder, 'coss_ea.tif')
    sinscosafile = os.path.join(landfolder, 'sinscosa_ea.tif')
    sinssinafile = os.path.join(landfolder, 'sinssina_ea.tif')
    tcffile = os.path.join(landfolder, 'tcf_ea.tif')
    svffile = os.path.join(landfolder, 'svf_ea.tif')
    radavelevfile = os.path.join(landfolder, 'radavelev_ea.tif')
    radstelevfile = os.path.join(landfolder, 'radstelev_ea.tif')

    # RD = gdal_tools.read_data(elevfile)

    elev_map_latlon = gdal_tools.read_data(elevfile_latlon)
    minlat = elev_map_latlon.miny
    maxlat = elev_map_latlon.maxy
    minlon = elev_map_latlon.minx
    maxlon = elev_map_latlon.maxx

    elev_map_raster = gdal_tools.read_data(elevfile)
    elev_map = elev_map_raster.data
    tcf_map = gdal_tools.read_data(tcffile).data
    svf_map = gdal_tools.read_data(svffile).data
    # coss_map = gdal_tools.read_data(cossfile).data
    sinssina_map = gdal_tools.read_data(sinssinafile).data
    sinscosa_map = gdal_tools.read_data(sinscosafile).data
    radstelev_map = gdal_tools.read_data(radstelevfile).data
    radavelev_map = gdal_tools.read_data(radavelevfile).data

    # nx, ny = elev_map.shape
    # print(elev_map.shape)
    # print(radavelev_map.shape)

    nx = elev_map_raster.nx
    ny = elev_map_raster.ny
    ylat_ea = np.flipud( np.linspace(minlat, maxlat, ny) )
    xlon_ea = np.linspace(minlon, maxlon, nx)

    nx_latlon = elev_map_latlon.nx
    ny_latlon = elev_map_latlon.ny
    ylat_latlon = np.flipud( np.linspace(minlat, maxlat, ny_latlon) )
    xlon_latlon = np.linspace(minlon, maxlon, nx_latlon)

    # if do_averaging:
    #
    #     avblock = aveblock
    #
    #     # block average terrain maps
    #     elev_map =      dem.crop_and_average(elev_map,         average=True, aveblock=avblock)
    #     sinscosa_map =  dem.crop_and_average(sinscosa_map,     average=True, aveblock=avblock)
    #     sinssina_map =  dem.crop_and_average(sinssina_map,     average=True, aveblock=avblock)
    #     tcf_map =       dem.crop_and_average(tcf_map,          average=True, aveblock=avblock)
    #     svf_map =       dem.crop_and_average(svf_map,          average=True, aveblock=avblock)
    #     radstelev_map = dem.crop_and_average(radstelev_map,    average=True, aveblock=avblock)
    #     radavelev_map = dem.crop_and_average(radavelev_map,    average=True, aveblock=avblock)

    # create coordinates for original elevation map (equal area projection)::
    elev = gdal_tools.read_data(elevfile)
    x_ea = np.linspace(elev.minx, elev.maxx, elev.nx)
    y_ea = np.linspace(elev.miny, elev.maxy, elev.ny)

    ntiles = len(hruprops['hillslope_aspect'])
    print('number of HRUs = {}'.format(ntiles))
    print(np.unique(hrus))
    print(np.shape(hrus))

    hrus_plot = hrus.copy().astype(float)
    hrus_plot[hrus_plot < 0] = np.nan

    sinz = (1 - cosz ** 2) ** (0.5)

    def compute_sia(cosz=None, sinz=sinz, phi=None, SS=None, SC=None):
        sia = cosz + sinz * ( np.sin(phi)*SS + np.cos(phi)*SC)
        return sia

    tile_sia = compute_sia(cosz=cosz, sinz=sinz, phi=phi,
                        SS=hruprops['hillslope_sinssina'],
                        SC=hruprops['hillslope_sinscosa'])

    sia_map = compute_sia(cosz=cosz, sinz=sinz, phi=phi,
                       SS=sinssina_map, SC=sinscosa_map)

    hruprops['hillslope_sia'] = tile_sia

    # return a dictionary with tile data
    res = {}
    res['ntiles'] = ntiles
    res['cosz'] = cosz
    res['phi'] = phi

    # coordinates and bounds

    res['y_ea'] = y_ea # coordinates in [km], equal area map
    res['x_ea'] = x_ea # coordinates in [km], equal area map
    res['ylat_ea'] = ylat_ea # coorinates in [lat, lon], equal area map
    res['xlon_ea'] = xlon_ea # coorinates in [lat, lon], equal area map
    res['ylat_latlon'] = ylat_latlon # latlon coordinates, latlon map
    res['xlon_latlon'] = xlon_latlon # latlon coordinates, latlon map
    res['minlat'] = minlat # domain boundaries, lat-lon
    res['maxlat'] = maxlat # domain boundaries, lat-lon
    res['minlon'] = minlon # domain boundaries, lat-lon
    res['maxlon'] = maxlon # domain boundaries, lat-lon

    # average value for each tile
    res['tile_ele'] = hruprops['hillslope_radavelev']
    res['tile_sde'] = hruprops['hillslope_radstelev']
    res['tile_svf'] = hruprops['hillslope_svf']
    res['tile_tcf'] = hruprops['hillslope_tcf']
    res['tile_sia'] = hruprops['hillslope_sia']

    # average value for each tile, mapped over domain
    res['mappedtile_ele'] = maphrus(hrus, hruprops, 'hillslope_radavelev')
    res['mappedtile_sde'] = maphrus(hrus, hruprops, 'hillslope_radstelev')
    res['mappedtile_svf'] = maphrus(hrus, hruprops, 'hillslope_svf')
    res['mappedtile_tcf'] = maphrus(hrus, hruprops, 'hillslope_tcf')
    res['mappedtile_sia'] = maphrus(hrus, hruprops, 'hillslope_sia')

    # if do_averaging:
    #     res['mappedtile_ele'] = dem.crop_and_average( res['mappedtile_ele'], average=True, aveblock=aveblock)
    #     res['mappedtile_sde'] = dem.crop_and_average( res['mappedtile_sde'], average=True, aveblock=aveblock)
    #     res['mappedtile_svf'] = dem.crop_and_average( res['mappedtile_svf'], average=True, aveblock=aveblock)
    #     res['mappedtile_tcf'] = dem.crop_and_average( res['mappedtile_tcf'], average=True, aveblock=aveblock)
    #     res['mappedtile_sia'] = dem.crop_and_average( res['mappedtile_sia'], average=True, aveblock=aveblock)

    res['mappedtile_hrus'] = hrus
    res['mappedtile_hills'] = hills

    # values of original high res (or ave) maps
    res['map_ele'] = radavelev_map
    res['map_sde'] = radstelev_map
    res['map_svf'] = svf_map
    res['map_tcf'] = tcf_map
    res['map_sia'] = sia_map

    res['map_sinssina'] = sinssina_map
    res['map_sinscosa'] = sinscosa_map

    # LOAD MLR MODEL AND USE IT FOR PREDICTIONS
    FLUX_TERMS = ['fdir', 'frdirn', 'fdif', 'frdifn', 'fcoupn']
    for flux_term in FLUX_TERMS:

        # APPLY THE MODEL TO MAKE PREDICTIONS TILE-BY-TILE:
        dfhru = pd.DataFrame({
            # 'ele':hruprops['hillslope_radavelev'],
            # 'sde':hruprops['hillslope_radstelev'],
            'tcfn': hruprops['hillslope_tcf'],
            'svfn': hruprops['hillslope_svf'],
            'sian': tile_sia
        })

        # MAKE "TRUE" PREDICTIONS BASED ON HIGH-RESOLUTION ORIGINAL TERRAIN DATA::
        # using the equal area projection map, averaged if needed
        dfhrt = pd.DataFrame({
            # 'ele': radavelev_map.flatten(),
            # 'sde': radstelev_map.flatten(),
            'tcfn': tcf_map.flatten(),
            'svfn': svf_map.flatten(),
            'sian': sia_map.flatten()
        })

        # EZDEV: keep SIAN, changed order or predictors wrt version 1
        # if flux_term in ['frdirn', 'frdifn'] or (cosz > 0.99 and flux_term == 'fdir'):
        #     _ = dfhrt.pop('sian')
        #     _ = dfhru.pop('sian')

        mlr_savedir = modeldir
        mlr_savename = os.path.join(mlr_savedir,
                   'mlr_model_{}_cosz_{}_adir_{}'.format(flux_term, cosz, adir))
        with open(os.path.join(mlr_savedir,
                   '{}.pickle'.format(mlr_savename)), 'rb') as pklf:
            mlr_model = pickle.load(pklf)

        print('model char::')
        print(mlr_model.n_features_in_)
        print(dfhru.shape)
        print(dfhrt.shape)
        print('end model diagnostics')

        # compute fluxes
        # hru_pred_tiles = mlr_model.predict(dfhru).flatten()
        tile_pred = mlr_model.predict(dfhru)
        map_pred_1d = mlr_model.predict(dfhrt)

        # hruprops[flux_term] = hru_pred_tiles

        # plt.figure()
        # plt.imshow(map_pred_hru)
        # plt.colorbar()
        # plt.show()

        # MAP PREDICTIONS:
        # map_pred_hru = maphrus(hrus, hruprops, prop=flux_term)

        # average tile-by-tile predictions at same scale used for terrain vars
        # map_pred_ave_hru = dem.crop_and_average(tile_pred,
        #                         average=True, aveblock=aveblock)

        map_pred = map_pred_1d.reshape( np.shape(elev_map) )
        # hru_pred = map_pred_ave_hru.flatten()

        # save results to output for current flux component
        res['tile_{}'.format(flux_term)] = tile_pred
        res['map_{}'.format(flux_term)] = map_pred

        hruprops['hillslope_{}'.format(flux_term)] = tile_pred

        tile_pred_mapped = maphrus(hrus, hruprops,
                                    'hillslope_{}'.format(flux_term))

        if do_averaging:
            tile_pred_mapped_averaged = dem.crop_and_average(tile_pred_mapped,
                                average=True, aveblock=aveblock)
        else:
            tile_pred_mapped_averaged = tile_pred_mapped

        # res['mappedtile_{}'.format(flux_term)] = tile_pred_mapped
        res['mappedtile_{}'.format(flux_term)] = tile_pred_mapped_averaged

    return res


from __future__ import annotations

import os
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn as nn
from torch import optim
from tqdm import tqdm

import gymnasium as gym

class A2C(nn.Module):
    """
    (Synchronous) Advantage Actor-Critic (A2C) agent class.
    """

    def __init__(self,
                 n_features: int,
                 n_actions: int,
                 device: torch.device,
                 critic_lr: float,
                 actor_lr: float,
                 n_envs: int) -> None:
        """Initialize the actor and critic networks and their optimizers.
        ### Parameters
        - `n_features`: int, number of features in the state
        - `n_actions`: int, number of actions
        - `device`: torch.device, device to use for computation
        - `critic_lr`: float, learning rate for the critic
        - `actor_lr`: float, learning rate for the actor
        - `n_envs`: int, number of parallel environments to use"""
        super().__init__()
        self.device = device
        self.n_envs = n_envs

        critic_layers = [
            nn.Linear(n_features, 32),
            nn.ReLU(),
            nn.Linear(32,32),
            nn.ReLU(),
            nn.Linear(32, 1), # estimate V(s)
        ]

        actor_layers = [
            nn.Linear(n_features, 32),
            nn.ReLU(),
            nn.Linear(32, 32),
            nn.ReLU(),
            nn.Linear(
                32, n_actions
            ),  # estimate action logits (will be fed into a softmax later)
        ]
        # define actor and critic networks
        self.critic:nn.Sequential = nn.Sequential(*critic_layers).to(self.device)
        self.actor:nn.Sequential = nn.Sequential(*actor_layers).to(self.device)

        # define optimizer for actor and critic
        self.critic_optim = optim.RMSprop(self.critic.parameters(), lr=critic_lr)
        self.actor_optim = optim.RMSprop(self.actor.parameters(), lr=actor_lr)

    def forward(self, x:np.ndarray) -> tuple[torch.Tensor, torch.Tensor]:
        """Forward pass of the actor and critic networks.

        ### Parameters
        - `x`: A batched vector of states, [n_env, state_dim]

        ### Return
        - `state_values`: torch.Tensor, estimated state values, with shape [n_env, ]
        - `action_logits_vec`: torch.Tensor, estimated action logits, with shape [n_env, n_actions]
        """
        x = torch.Tensor(x).to(self.device)
        state_values = self.critic(x)
        action_logits_vec = self.actor(x)
        return (state_values, action_logits_vec)

    def select_action(self, x: np.ndarray) -> tuple[torch.tensor, torch.tensor, torch.tensor, torch.tensor]:
        """
        Return a tuple the chosen actions and the log-probs of the actions.

        ### Return
         - a tuple: (actions, action_log_probs, state_values, entropy)
         Four torch.Tensor objects with shape [n_env, ]
        """
        state_values, action_logits_vec = self.forward(x)
        action_pd = torch.distributions.Categorical(logits=action_logits_vec)
        actions = action_pd.sample()
        action_log_probs = action_pd.log_prob(actions)
        entropy = action_pd.entropy()
        return (actions, action_log_probs, state_values, entropy)

    def get_losses(self,
                   rewards:torch.Tensor,
                   action_log_probs: torch.Tensor,
                   value_preds: torch.Tensor,
                   entropy: torch.Tensor,
                   masks: torch.Tensor,
                   gamma: float,
                   lam: float,
                   ent_coeft: float,
                   device: torch.device
                   )-> tuple[torch.Tensor, torch.Tensor]:
        """
        ### Parameters
        - `rewards`: torch.Tensor, with the rewards for each time step in the episode, with shape `[n_steps_per_update, n_env]`
        - `action_log_probs`: torch.Tensor, with the log-probabilities of the actions taken, with shape `[n_steps_per_update, n_env]`
        - `value_preds`: torch.Tensor, with the state values prediction for each time step in the episode, with shape `[n_steps_per_update, n_env]`
        - `masks`: torch.Tensor
        - `gamma`: float, discount factor
        - `lam`: float, GAE hyperparameter

        ### Return
        - A tuple, (critic_loss, actor_loss)
        - critic_loss: torch.Tensor, with shape [1, ]
        - actor_loss: torch.Tensor, with shape [1, ]
        """
        T = len(rewards) # the n_steps_per_update
        advantages = torch.zeros_like(T, self.n_envs, device=device)
        # []
        # compute the advanteges using GAE
        gae = 0.0
        for t in reversed(range(T-1)):
            td_error = (-value_preds[t] + rewards[t] + gamma * masks[t]* value_preds[t+1])
            gae = td_error + gamma * lam * masks[t] * gae
            advantages[t] = gae

        # calculate the loss of the minibatch for actor and critic
        critic_loss = advantages.pow(2).mean()

        # give a bonus for higher entropy, to encourage exploration
        actor_loss = -(advantages.detach() * action_log_probs).mean() - ent_coeft * entropy.mean()
        return (critic_loss, actor_loss)

    def update_parameters(self,
                          critic_loss: torch.Tensor,
                          actor_loss: torch.Tensor)->None:
        self.critic_optim.zero_grad()
        critic_loss.backward()
        self.critic_optim.step()

        self.actor_optim.zero_grad()
        actor_loss.backward()
        self.actor_optim.step()

n_envs = 3
n_updates = 1000
n_steps_per_update = 128
randomize_domain = False

# agent hyperparams
gamma = 0.999
lam = 0.95
ent_coeft = 0.01
actor_lr = 0.001
critic_lr = 0.005

if __name__ == "__main__":
    # envs = gym.vector.make("LunarLander-v2", num_envs=3, max_episode_steps=600)
   if randomize_domain:
    envs = gym.vector.AsyncVectorEnv(
        [
            lambda: gym.make(
                "LunarLander-v2",
                gravity=np.clip(
                    np.random.normal(loc=-10.0, scale=1.0), a_min=-11.99, a_max=-0.01
                ),
                enable_wind=np.random.choice([True, False]),
                wind_power=np.clip(
                    np.random.normal(loc=15.0, scale=1.0), a_min=0.01, a_max=19.99
                ),
                turbulence_power=np.clip(
                    np.random.normal(loc=1.5, scale=0.5), a_min=0.01, a_max=1.99
                ),
                max_episode_steps=600,
            )
            for i in range(n_envs)
        ]
        )
   else:
    envs = gym.vector.make("LunarLander-v2", num_envs=n_envs, max_episode_steps=600)

    obs_shape = envs.single_action_space.shape[0]
    action_shape = envs.single_action_space.n

    use_cuda = False
    if use_cuda:
       device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    else:
       device = torch.device("cpu")

    # init the agent
    agent = A2C(obs_shape, action_shape, device, critic_lr, actor_lr, n_envs)


import visa
from random import random

class LockIn:
    def __init__(self, inst, **kwargs):
        self.inst = inst

    def initialize(self):
        self.inst.write("*RST")
        self.inst.write("FILTer:SLOPe 6")
        self.inst.write("ROUTe2 IOSC")  # reference is internal oscillator

    @property
    def frequency(self):
        return float(self.inst.query("SOUR:FREQ?"))

    @frequency.setter
    def frequency(self, value):
        resp = self.inst.write("SOUR:FREQ {:7E}".format(value))
        if resp[1] != 0:
            raise Exception

    @property
    def TC(self):
        return float(self.inst.query("FILT:TCON?"))

    @TC.setter
    def TC(self, value):
        resp = self.inst.write("FILT:TCON {:7E}".format(value))
        if resp[1] != 0:
            raise Exception

    @property
    def output(self):
        return float(self.inst.query("SOUR:VOLT?"))

    @output.setter
    def output(self, value):
        assert 0 <= value <= 1
        resp = self.inst.write("SOUR:VOLT {}".format(value))
        if resp[1] != 0:
            raise Exception

    def fetch(self):
        return self.inst.query("FETC?").strip()

    def close(self):
        self.inst.close()

def fake_measure():
    """return fake measured value in order of amplitude, phase, x, y"""
    return "{},{}".format(random(), random())

list_of_commands = """inst.write("INP:COU AC") # or DC
inst.write("INPUT:FILTER:NOTCH1 ON") # or OF, 1, 2
inst.write("INPUT:FILTER:NOTCH2 ON") # or OF, 1, 2
inst.write("INPUT:IMPedance 1E6") # or 50
inst.write("INPut:LOW FLOat") # or GROund
inst.write("ROUTe:TERMinals  A") # {A|AB|C|I|HF}
inst.write("FILTer:SLOPe 24") #{6|12|18|24}

inst.write("FILTer:TCONstant 1") #<time constant in sec>
inst.query("FETCH?")
inst.write("SOUR:FREQ {:7E}".format(freq)) #1.234567E+05
inst.write("SOURce:VOLTage <amplitude> )
{numeric, range 0.00000 to 1.000, setting resolution 4 digits
(at output voltage range full scale), unit Vrms} Suffix M (10-3),
 unit  V, MAX, and  MIN can be used.  Example: 100M (= 0.1)"""

if __name__ == "__main__":
    rm = visa.ResourceManager()
    inst = LockIn(rm.open_resource('GPIB0::1::INSTR'))

    inst.frequency = 2000
    inst.inst.query("FREQ?")
    inst.inst.write("FREQ 3000")


import datetime
a = """Turing: DOJ: 22 Aug 2017 | Subjects: Math, English | Salary: 18 LPA in hand | Also takes Web lectures.
Dinho: DOJ: 1 Jan 2016 | Subjects: Sports, Health Science | Salary: 25 LPA in hand
Adele:  DOJ: 1 Mar 2015 | Subjects: English | Salary: 10 LPA in hand
Freddie: DOJ: 1 Aug 2017 | Subjects: Music, English | Salary: 20 LPA in hand | Also takes Web lectures.
Dalton: DOJ: 1 Mar 2017 | Subjects: Botany, Zoology | Salary: 9 LPA in hand | Also takes Web lectures.
Harish: DOJ: 1 Feb 2017 | Subjects: Math, Science | Salary: 18 LPA in hand
Trump: DOJ: 1 Aug 2017 | Subjects: Political Science, Business Administration, Foreign Affairs | Salary: 8 LPA in hand
Swaraj: DOJ: 1 Sept 2019 | Subjects: Foreign Affairs, Negotiations | Salary: 28 LPA in hand | Also takes Web lectures.
Socrates: DOJ: 1 June 2015 | Subjects: Philosophy, Moral Science | Salary: 11.5 LPA in hand | Also takes Web lectures."""
lines = a.split('\n')
months = {
    'Jan':1,
    'Feb':2,
    'Mar':3,
    'Apr':4,
    'May':5,
    'June':6,
    'July':7,
    'Aug':8,
    'Sept':9,
    'Oct':10,
    'Nov':11,
    'Dec':12
}
for line in lines:
    if not line:
        continue
    all_entities = line.split(' | ')
    nameAndDoj = all_entities[0].split(': ')
    name = nameAndDoj[0]
    Doj = nameAndDoj[2]
    dArr = Doj.split(' ')
    dArr[1] = months[dArr[1]]
    date = datetime.date(int(dArr[2]), dArr[1], int(dArr[0]))
    subjects = all_entities[1]
    subjects = subjects.split(': ')[1]
    salary = all_entities[2]
    salary = salary.split(': ')[1]
    if len(all_entities) > 3:
        web_lecture_capability = 'yes'
    else:
        web_lecture_capability = 'no'
    # print('{} {} {} {} {}'.format(name, date, subjects, salary, web_lecture_capability))
    print('{} {} {}'.format(subjects, name, salary))


from django.db.models import Sum

from quiz.utils import lti_utils as lti
from quiz.models import Quiz, LTIUser, QuizSettings, Question, Response, QuizManager, Prompt

def get_quiz(request):
    """
    Return the non-archived quiz if the resourceLinkId and contextId pair exists, if not, then check whether the resourceLinkId exists. If it
    does, then a new quiz is to be created which would be copy of the latest non archived version of the quiz with given resourceLinkId
    return new quiz.
    """
    live_quiz = Quiz.objects.filter(consumer_key = lti.get_oauth_consumer_key(request), resourceLinkId = lti.get_resource_link_id(request), contextId = lti.get_context_id(request))
    if live_quiz.exists():
        return live_quiz.latest()

    # The resourceLinkId and the contextId pair doesn't exists, check whether the resourceLinkId exists.
    if Quiz.objects.filter(consumer_key = lti.get_oauth_consumer_key(request), resourceLinkId = lti.get_resource_link_id(request)).exists():
        # The quiz link is being migrated to another Context (new course term), hence the non-archived
        # latest quiz with same resourceLinkId has to be copied over as a new Quiz.
        quiz = create_new_quiz(request)
        latest_quiz = Quiz.objects.filter(consumer_key = lti.get_oauth_consumer_key(request), resourceLinkId = lti.get_resource_link_id(request)).latest()
        # copy all the settings of the latest quiz associated with the RESOURCE LINK ID.
        quiz_settings = QuizSettings.objects.get(quiz = quiz)
        quiz_settings.duration = latest_quiz.duration
        quiz_settings.timeBetweenAttempt = latest_quiz.timeBetweenAttempt
        quiz_settings.maxAttempts = latest_quiz.maxAttempts
        quiz_settings.graded = latest_quiz.graded
        quiz_settings.save()
        #TODO: Copy all the questions , doing it manually to avoid any useless trouble: is it the right way?
        questions = get_questions_by_quiz(latest_quiz)
        for question in questions:
            new_question = Question()
            new_question.quiz = question.quiz      # change the quiz to the new quiz
            new_question.question_type = question.question_type
            new_question.serial_number = question.serial_number
            new_question.question_weight = question.question_weight
            new_question.statement = question.statement
            new_question.options_data = question.options_data
            new_question.expected_response = question.expected_response
            new_question.other_data = question.other_data
            new_question.published = question.published
            new_question.save()

        return quiz

    # The quiz doesn't exists for associated request! Return False
    return False

def create_new_quiz(request):
    """
    Creates a new quiz and its default settings for the given request, and returns the quiz.
    """
    quiz = Quiz() #New Quiz is created
    quiz.resourceLinkId = lti.get_resource_link_id(request)
    quiz.contextTitle = lti.get_context_title(request)
    quiz.consumer_key = lti.get_oauth_consumer_key(request)
    quiz.contextId = lti.get_context_id(request)
    quiz.quizName = lti.get_quiz_name(request)
    quiz.save() #Save the Quiz
    quizSettings = QuizSettings(quiz = quiz)
    quizSettings.save()
    return quiz

def get_user(request):
    """
    Return the user associated with give request if it exists, or create a new user and return it.
    It also stores/updates the Outcome service data in the database.
    """
    userId = lti.get_user_id(request)
    if not userId:
        return False
    ltiUserList = LTIUser.objects.filter(consumer_key = lti.get_oauth_consumer_key(request),  userId = userId)
    if ltiUserList.count() == 0:
        ltiUser = LTIUser()
        ltiUser.userId = lti.get_user_id(request)
        ltiUser.consumer_key = lti.get_oauth_consumer_key(request)
        ltiUser.name = lti.get_user_name(request)
        ltiUser.email = lti.get_user_email(request)
        ltiUser.role = lti.get_user_role(request)
        ltiUser.save() #Save the ltiUser
    else:
        ltiUser = ltiUserList[0]
    return ltiUser

def get_questions_by_quiz(quiz):
    return Question.objects.order_by('serial_number').filter(quiz = quiz)

def get_prompts_by_quiz(quiz):
    return Prompt.objects.order_by('id').filter(quiz = quiz)

def get_published_questions(quiz, random=None):
    """
    Returns the QuerySet of the published questions for the given quiz
    """
    questions = get_questions_by_quiz(quiz) #Questions are ordered by serial number
    if random:
        return questions.filter(published = True).order_by('?')
    return questions.filter(published = True)

def get_published_questions_id(quiz):
    """
    Returns the list of the ids of the published questions for the given quiz
    """
    questions = get_published_questions(quiz) #Questions are ordered by serial number
    question_ids = []
    for question in questions:
        question_ids.append(question.id)
    return question_ids

def get_previous_attempt(request):
    """
    Returns the non-submitted Response/Attempt for the given user/student as identified by the request.
    """
    responses = Response.objects.filter(quiz = get_quiz(request), user = get_user(request), submitted = False)
    if responses:
        attempt = responses.latest()
        return attempt
    return False

def get_new_attempt(request):
    """
    Returns a new Response/attempt.
    If the max attempt number is not set, then previous attempt is deleted.
    """
    quiz, user = get_quiz(request), get_user(request)
    quiz_settings = QuizSettings.objects.get(quiz = quiz)
    questions = get_published_questions(quiz, random=quiz_settings.randomizeQuestionOrder)  # returns QuerySet of the published questions
    question_id_list = [ question['id'] for question in questions.values('id') ] # Save the order of questions in db
    used_attempt_number = None
    if not quiz_settings.maxAttempts:
        # Delete the last attempt and return a new attempt object if maxAttempts isn't set
        last_response = Response.objects.filter(quiz = quiz, user=user)
        if last_response:
            last_response.delete() # Delete the response
    else:
        used_attempt_number = get_used_attempt_number(quiz, user)
        max_attempts  = quiz_settings.maxAttempts
        if max_attempts and used_attempt_number >= max_attempts:
            # New attempt cannot be allowed, return False
            return False
    # Create a new Response and return.
    attempt = Response(quiz = quiz, user = user, question_ids = question_id_list, attempt_number=used_attempt_number+1 if used_attempt_number is not None else None)
    attempt.set_end_time()
    attempt.ip_address = get_ip(request)
    attempt.save()

    # set isEverAttempted variable of Quiz to be True if not already
    if not quiz.isEverAttempted:
        quiz.isEverAttempted = True
        quiz.save()

    return attempt

def get_used_attempt_number(quiz, user):
    return Response.objects.filter(quiz=quiz, user = user).count()

def get_submitted_responses(request):
    responses = Response.objects.filter(quiz = get_quiz(request), user = get_user(request), submitted = True).order_by('-start_time')
    if responses:
        return responses
    return []

def get_students_responses(request):
    """
    Returns the last response of all students in the quiz
    """
    quiz = get_quiz(request)
    responses = Response.objects.filter(quiz=quiz, submitted=True).order_by('-start_time')
    return responses

def get_quiz_settings(quiz):
    return QuizSettings.objects.get(quiz=quiz)

def get_quiz_total_marks(quiz):
    """
    Returns the total marks of the quiz by adding weight of all questions in the quiz.
    :param quiz:
    :return: number : total marks
    """
    # TODO: Wouldn't it be better to store the total marks in a table like cache?
    questions = get_published_questions(quiz)
    total = questions.aggregate(Sum('question_weight'))['question_weight__sum']
    return total

def get_managers(quiz):
    """
    Returns the query set managers of the given quiz
    """
    managers = QuizManager.objects.filter(quiz=quiz)
    return managers

def get_ip(request):
    x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
    if x_forwarded_for:
        ip = x_forwarded_for.split(',')[0]
    else:
        ip = request.META.get('REMOTE_ADDR')
    return ip


import sys
import argparse
import cv2
import numpy as np
from PIL import Image, ImageDraw
import torch
import torch.nn.functional as F
from dataset import get_loader
from model import *
from model.vgg_cfg import cfg as vgg_config
from utils import AverageMeter, load_model, Hooks
from explain.rap.modules.vgg import vgg16, vgg16_bn, vgg16_bn_voc
from torch.autograd import Variable
import torch.nn as nn
#from explains.ebp.exc_bp import ExcitationBackpropExplainer
import os
from sklearn.metrics import roc_curve, auc

def main():
    parser = argparse.ArgumentParser(description='')

    parser.add_argument('--gpu', action='store_true')
    parser.add_argument('--data_path', default="./data/VOCdevkit/VOC2012", type=str,
                        help="Data path, here for segmentation ground truth")
    parser.add_argument('--path', default='', type=str,
                        help="Saved Model Path, ref load_model in utils.py")
    parser.add_argument('--arch', default='VGG16_VOC', type=str,
                        help="Predefined Architecture for knowledge distillation" +
                        "For Structure Pruning and Unstructure Pruning, Keep VGG16_VOC")
    parser.add_argument('--iter', default=1, type=int,
                        help="how many iteration for Structure Pruning"+
                            "We use only one-shot pruning in the paper")
    parser.add_argument('--prune_rate', default=0.7, type=float,
                        help="Pruning ratio for each iteration")
    parser.add_argument('--compression', default='', type=str,
                        help='Choose in [kd | unstructured | structured]')
    parser.add_argument('--method', default='', type=str,
                        help='Choose in [gcam | ebp | lrp | rap]')
    parser.add_argument('--metric', default='auc', type=str,
                        help='Choose in [miou | iou | auc]')
    parser.add_argument('--save', default='result.txt', type=str,
                        help="Results Recording file")

    args = parser.parse_args()
    compression = args.compression
    method = args.method
    metric = args.metric

    print("Attribution Map Method : {}".format(method))
    print("Compression Method : {}".format(compression))
    print("Evaluation Metric : {}".format(metric))

    if (method == 'gcam') or (method =='ebp'):
        if compression == 'structured':
            layerwise_prune_rate = [args.prune_rate, 0.0,
                            0.0, 0.0,
                            0.0, 0.0, 0.0,
                            args.prune_rate, args.prune_rate, args.prune_rate,
                            args.prune_rate, args.prune_rate, args.prune_rate]
            model_config = prune_vgg_cfg(vgg_config['VGG16_VOC'], layerwise_prune_rate, args.iter)
            model = VGG(None, input_dims=(3,227,227), specific_cfg=model_config)

        elif compression == 'kd':
            model = VGG(args.arch, input_dims=(3, 227, 227))

        elif compression == 'unstructured':
            model = VGG(args.arch, input_dims=(3,227,227))
        else:
            raise NotImplementedError

    elif (method == 'lrp') or (method == 'rap'):
        if compression == 'structured':
            layerwise_prune_rate = [args.prune_rate, 0.0,
                        0.0, 0.0,
                        0.0, 0.0, 0.0,
                        args.prune_rate, args.prune_rate, args.prune_rate,
                        args.prune_rate, args.prune_rate, args.prune_rate]
            model_config = prune_vgg_cfg(vgg_config['VGG16_VOC'], layerwise_prune_rate, args.iter)
            model = vgg16_bn_voc(config=model_config)
        elif compression == 'kd':
            if args.arch == 'VGG16_VOC':
                r = 1
            elif args.arch == 'VGG16_VOC_x2':
                r = 2
            elif args.arch == 'VGG16_VOC_x4':
                r = 4
            else:
                r = 8
            model = vgg16_bn_voc(config=None, reduce=r, pretrained=False)
        elif compression == 'unstructured':
            r = 1
            model = vgg16_bn_voc(config=None, reduce=r, pretrained=False)
        else:
            raise NotImplementedError

    print('Number of model parameters: {}'.format(sum([p.data.nelement() for p in model.parameters()])))

    if args.gpu:
        model = torch.nn.DataParallel(model).cuda()
    else:
        model = torch.nn.DataParallel(model)

    checkpoint = args.path

    _, best_mAP = load_model(checkpoint, model)

    ## Batch size should be 1
    vocsegment = get_loader(data_name='VOC-seg',
                data_path= args.data_path,
                split= 'val',
                batch_size=1)

    if method == 'gcam':
        features = Hooks(model, ['42'])
    elif method == 'ebp':
        layer = model.module.features._modules['42']
        explainer = ExcitationBackpropExplainer(model, layer)
    else:
        model = model.module

    model.eval()
    auc_list = []
    correct_point = 0
    incorrect_point = 0
    pred_true = 0
    pred_false = 0

    for idx, (images, targets, labels) in enumerate(vocsegment):
        if args.gpu:
            images = images.cuda()
            labels = labels.cuda()

        if method == 'gcam':
            features.reset_()
            output, _, _ = model(images)
            activation = features.activations[0]

        elif method == 'ebp':
            output, _, _ = model(images)

        elif (method == 'lrp') or (method=='rap'):
            output = model(images)

        ## 0 index : BackGround
        lbl = (labels[:,1:].squeeze(0)).nonzero(as_tuple=False).cpu().data.numpy() #
        predictions = (output.squeeze()>0).nonzero(as_tuple=False).cpu().data.numpy() #
        pointacc_labels = []

        ## Class-wise heatmap
        for i, lb in enumerate(lbl):
            ## Class-wise Segmentation Map
            target = targets.squeeze().data.numpy()
            seg_mask = np.zeros((target.shape[0], target.shape[1]), dtype=np.uint8)
            whe = np.where(target == (lb+1))
            seg_mask[whe] = 1
            for m in range(len(whe[0])):
                pointacc_labels.append([whe[0][m], whe[1][m]])

            grad_out = output.data.clone()
            grad_out.fill_(0.0)
            grad_out[0, lb] = 1

            if method == 'gcam':
                model.zero_grad()
                output.backward(grad_out, retain_graph=True)
                gradient = features.gradients[i]
                linearization = torch.sum(torch.sum(gradient, 3, keepdim=True), 2, keepdim=True)
                channels = linearization * activation
                grad_cam = torch.sum(channels, 1, keepdim=True)
                grad_cam = F.relu(grad_cam)
                heatmap = grad_cam.squeeze().cpu().data.numpy()
                heatmap = cv2.resize(heatmap, (227, 227))

            elif method == 'ebp':
                ebp_heatmap = explainer.explain(images, grad_out)
                heatmap = ebp_heatmap.squeeze().cpu().data.numpy()
                heatmap = cv2.resize(heatmap, (227, 227))

            elif method == 'lrp':
                Res = model.relprop(R=output * grad_out, alpha=1).sum(dim=1, keepdim=True)
                heatmap = Res.squeeze().cpu().data.numpy()
                if output[0, lab] < 0:
                    heatmap = -1 * heatmap

            elif method == 'rap':
                RAP = model.RAP_relprop(R=grad_out)
                Res = (RAP).sum(dim=1, keepdim=True)
                heatmap = Res.squeeze().cpu().data.numpy()
                if output[0, lab] < 0:
                    heatmap = -1 * heatmap
                mask = np.int32(heatmap > 0)
                heatmap = heatmap * mask

            if np.isnan(heatmap).any():
                continue
            ##For Point Acc, maxpoint
            max_index = np.where(heatmap == np.max(heatmap))
            max_point = [max_index[0][0], max_index[1][0]]

            ##Get AUC
            h_max = np.nanmax(heatmap)
            h_min = np.nanmin(heatmap)
            heatmap = (heatmap - h_min) / (h_max - h_min + 10e-6)
            auc_value = get_auc(heatmap, seg_mask)

            if lb not in predictions:
                pred_false += 1
                continue

            if max_point in pointacc_labels:
                correct_point += 1
            else:
                incorrect_point += 1
            pred_true += 1
            auc_list.append(auc_value)

        if idx %100 == 0:
            print("Processing Image(Current|Total) {}|{}".format(idx, len(vocsegment)))

    meanAUC = sum(auc_list) / len(auc_list)
    print("Correct Prediction Samples: {}".format(pred_true))
    print("Incorrect Prediction Samples: {}".format(pred_false))
    print("Pointing game correct samples in correct prediction: {}".format(correct_point))
    print("Pointing game [(Point Correct and Prediction Correct)/(Prediction Correct)] : {}".format(float(correct_point/pred_true)))
    print("Metric AUC : {}".format(meanAUC))

    save_file = args.save
    with open(save_file, 'w') as f:
        f.write(checkpoint + '\t' + 'Metric AUC : {} \n'.format(float(meanAUC)))
        f.write(checkpoint + '\t' + 'Point Game Acc : {}'.format(float(correct_point/pred_true)))

def get_auc(pred, gt):
    pred = pred.reshape(-1)
    gt = gt.reshape(-1)
    flasePositive, truePositive, _ = roc_curve(gt, pred)
    roc_auc = auc(flasePositive, truePositive)
    return roc_auc

def get_miou(pred, gt, n_classes=2):
    pred_tmp = pred
    gt_tmp = gt
    intersect = [0] * n_classes
    union = [0] * n_classes
    for j in range(n_classes):
        # match = (pred_tmp == j) + (gt_tmp == j)
        match = (pred_tmp == j).astype(np.float) + (gt_tmp == j).astype(np.float)
        it = np.sum(match == 2)
        un = np.sum(match > 0)
        intersect[j] += it
        union[j] += un
    iou = []
    for k in range(n_classes):
        if union[k] == 0:
            continue
        iou.append(intersect[k] / union[k])
    miou = (sum(iou) / len(iou))
    return miou

def get_iou(pred, gt):
    pred_tmp = pred
    gt_tmp = gt
    match = (pred_tmp == 1).astype(np.float) + (gt_tmp == 1).astype(np.float)
    intersect = np.sum(match == 2)
    union = np.sum(match > 0)
    iou = intersect/union
    return iou

def thresh(a, coef):
    return coef * (np.max(a))

def denormalize(image):
    image[0,:,:] = image[0,:,:] * 0.2239 + 0.4589
    image[1,:,:] = image[1,:,:] * 0.2186 + 0.4355
    image[2,:,:] = image[2,:,:] * 0.2206 + 0.4032
    return image

def visualize(map):
    r = map.copy()
    g = map.copy()
    b = map.copy()
    r[np.where(map == 1)] = 255
    g[np.where(map == 1)] = 255
    b[np.where(map == 1)] = 255
    rgb = np.zeros((map.shape[0], map.shape[1], 3))
    rgb[:, :, 0] = r
    rgb[:, :, 1] = g
    rgb[:, :, 2] = b
    return rgb

def compute_pred(output):
    pred = output.data.max(1, keepdim=True)[1]  # get the index of the max log-probability
    print('Pred cls : '+str(pred))
    T = pred.squeeze().cpu().numpy()
    T = np.expand_dims(T, 0)
    T = (T[:, np.newaxis] == np.arange(20)) * 1.0
    T = torch.from_numpy(T).type(torch.FloatTensor)
    Tt = Variable(T).cuda()
    return Tt

def prune_vgg_cfg(cfg, prune_rate, iter):
    for idx in range(iter):
        # print('iter: {0}'.format(idx))
        conv_id = 0
        new_features = []
        for layer in cfg['features']:
            if isinstance(layer, int):
                new_features.append(int(layer * (1 - prune_rate[conv_id])))
                conv_id += 1
            elif layer == 'M':
                new_features.append('M')
            else:
                assert False
        cfg = {'features': new_features, 'classifier': cfg['classifier']}
    return {'features': new_features, 'classifier': cfg['classifier']}

if __name__ == '__main__':
    main()


#! /usr/bin/env python3

from lark import Lark

grammar =r"""
    variable : /[^()\n\s\.0-9][^()\n\s\.]*/

    expression : variable
        | abstraction
        | application

    abstraction : "(" "\\" variable "." expression ")"

    application : "(" expression expression ")"

    %import common.WS
    %ignore WS

 """

parser = Lark(grammar,start='expression')

def parse_file(path):
    with open(path,"r") as source_file:
        source = source_file.read()
    return parser.parse(source)

def parse_string(source):
    return parser.parse(source)

def test_file(path):
    tree = parse_file(path)
    print(tree.pretty())

def test_string(source):
    tree = parse_string(source)
    print(tree.pretty())

def interpreter():
    while(True):
        try :
            line = input("Lambda >> ")
        except (EOFError,KeyboardInterrupt) :
            break
        print("-> \n",end="")
        test_string(line)

if __name__ == '__main__':
    import sys
    if (len(sys.argv)>1):
        for path in sys.argv[1:]:
            test_file(path)
    else :
        interpreter()


import cv2 as cv
import numpy as np
from matplotlib import pyplot as plt
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
import os

img_path = r"D:\Documents\cut"

def cut_center(image):
    print(f'裁剪之前的图片尺寸{image.shape}')
    # img.shape[0]：图像的垂直尺寸（高度）
    # img.shape[1]：图像的水平尺寸（宽度）
    cut_width_left = 300
    cut_width_right = 300
    origin_width = image.shape[1]
    origin_high = image.shape[0]
    # 裁剪坐标为[y0:y1, x0:x1]
    central_line_part = image[0:origin_high, origin_width//2-cut_width_left:origin_width//2+cut_width_right]
    return central_line_part

def get_images(img_path):
    images = []
    files = os.listdir(img_path)
    n = len(files)
    for _ in range(5):
        ind = np.random.randint(0, n)
        img_dir = os.path.join(img_path, files[ind])
        print(img_dir)
        img = cv.imread(img_dir)
        img = cut_center(img)
        images.append(img)
    return images
images = get_images(img_path)

plt.subplot(141), plt.imshow(images[0]), plt.title('img1')
plt.xticks([]), plt.yticks([])

plt.subplot(142), plt.imshow(images[1]), plt.title('img2')
plt.xticks([]), plt.yticks([])

plt.subplot(143), plt.imshow(images[2]), plt.title('img3')
plt.xticks([]), plt.yticks([])

plt.subplot(144), plt.imshow(images[3]), plt.title('img4')
plt.xticks([]), plt.yticks([])

plt.tight_layout()
plt.show()


"""
Product of two expressions
"""
from sympy import expand, sympify, pprint
from sympy.core.sympify import SympifyError

def product(expr1, expr2):
    prod = expand(expr1 * expr2)
    pprint(prod, order='rev-lex')

if __name__ == '__main__':
    expr1 = input('Enter the 1st expression: ')
    expr2 = input('Enter the 2nd expression: ')

    try:
        expr1 = sympify(expr1)
        expr2 = sympify(expr2)
    except SympifyError:
        print('Invalid input')
    else:
        product(expr1, expr2)
    finally:
        print('Program has finished')


#!/usr/bin/python3
# -*- coding: utf-8 -*-

from aiohttp import web
from ..authorization import authorization, authorize

###############################
# Single vacancy removal handler
# DELETE /vacancy/{id}
###############################

@authorization
async def removeitem(request):
    """Single vacancy removal handler
        request: aiohttp.web_request.Request object

        returns: aiohttp.web_response.Response object
    """

    # Acquire requested ID
    try:
        _id = int(request.match_info["id"])

    # 400 Bad Request
    except ValueError:
        return web.Response(status=400)

    cursor = await request.app['db'].cursor()
    SQL = request.app["sql"]

    # Delete row by ID
    await cursor.execute(SQL["DELETE"], (_id,))
    request.app['db'].commit()
    # Check if specified row was found or not
    rc = cursor.rowcount

    # 404 Not Found
    if not rc:
        return web.Response(status=404)

    # Success
    return web.Response(status=200)


"""pledge amount in cents

Revision ID: eb8714b4b6ca
Revises: 401fef781277
Create Date: 2023-03-20 10:04:57.390744

"""
from alembic import op
import sqlalchemy as sa

# Polar Custom Imports

# revision identifiers, used by Alembic.
revision = "eb8714b4b6ca"
down_revision = "401fef781277"
branch_labels: tuple[str] | None = None
depends_on: tuple[str] | None = None

def upgrade() -> None:
    op.alter_column(
        "pledges",
        "amount",
        existing_type=sa.NUMERIC(precision=25, scale=10),
        type_=sa.BigInteger(),
        existing_nullable=False,
    )
    op.execute("update pledges set amount = amount * 100")

def downgrade() -> None:
    op.alter_column(
        "pledges",
        "amount",
        existing_type=sa.BigInteger(),
        type_=sa.NUMERIC(precision=25, scale=10),
        existing_nullable=False,
    )
    op.execute("update pledges set amount = amount / 100")


from ete3 import Tree, PhyloTree
from api_tools.tk import convert_genome_ID_rev,convert_genome_ID

intree = './223163/223163.newick'
new_tree = intree.replace('.newick', '_renamed.newick')
t = Tree(intree)
for l in t.get_leaves():
    l.name = convert_genome_ID_rev(l.name) + ';' + l.name
text = t.write()
with open(new_tree, 'w') as f1:
    f1.write(text)

import pandas as pd
from tqdm import tqdm
from collections import defaultdict

t = Tree(intree)
# all_g = set([convert_genome_ID_rev(_.split('_')[0]) for _ in t.get_leaf_names()])
all_ids = t.get_leaf_names()

id_dict = defaultdict(dict)
for g1 in tqdm(t.get_leaves()):
    for g2 in t.get_leaves():
        id_dict[g1.name][g2.name] = t.get_distance(g1, g2)
dis = pd.DataFrame.from_dict(id_dict)

from sklearn.cluster import KMeans

kmeans = KMeans(n_clusters=2, random_state=0, precompute_distances=True, tol=1e-10).fit(dis.values)
kmeans.labels_

id2info = defaultdict(list)
for idx, id in enumerate(dis.index):
    new_name = convert_genome_ID_rev(id.split('_')[0]) + '_' + id
    id2info[new_name] = [str(kmeans.labels_[idx])]
from api_tools.itol_func import *

text = to_binary_shape(id2info, {'1': {}, '0': {}})
with open('../itol_txt/separate_tmp.txt', 'w') as f1:
    f1.write(text)

t = PhyloTree(intree)
# t.set_outgroup(t.get_midpoint_outgroup())
t.set_species_naming_function(lambda node: convert_genome_ID_rev(node.name.split('_')[0]))

print(t.get_ascii(attributes=["name", "species"], show_internal=False))
t2 = t.collapse_lineage_specific_expansions()

ntrees, ndups, sptrees = t2.get_speciation_trees()
sptrees = list(sptrees)
print("Found %d species trees and %d duplication nodes" % (ntrees, ndups))
for spt in sptrees:
    print(len(spt.get_leaf_names()))


import requests
import json
import os
from threading import Thread, Lock
from queue import Queue

class Spider():

    def __init__(self, mid, name):
        self.mid = mid
        self.name = name
        self.download_api = "http://211.152.148.26/amobile.music.tc.qq.com/"
        self.api = ["https://u.y.qq.com/cgi-bin/musicu.fcg?-=getUCGI38350760984660526&g_tk=1125629726&loginUin=1791670972&hostUin=0&format=json&inCharset=utf8&outCharset=utf-8&notice=0&platform=yqq.json&needNewCode=0&data=%7B%22comm%22%3A%7B%22ct%22%3A24%2C%22cv%22%3A0%7D%2C%22singer%22%3A%7B%22method%22%3A%22get_singer_detail_info%22%2C%22param%22%3A%7B%22sort%22%3A5%2C%22singermid%22%3A%22","","%22%2C%22sin%22%3A","偏移量","%2C%22num%22%3A60%7D%2C%22module%22%3A%22music.web_singer_info_svr%22%7D%7D"]
        self.headers = {
            "Origin": "https://y.qq.com",
            "Referer": "https://y.qq.com/n/yqq/singer/%s.html" % self.mid,
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36"
        }
        self.music_api = ["http://u.y.qq.com/cgi-bin/musicu.fcg?-=getplaysongvkey2546320457778981&g_tk=1312345255&loginUin=1791670972&hostUin=0&format=json&inCharset=utf8&outCharset=utf-8&notice=0&platform=yqq.json&needNewCode=0&data=%7B%22req%22%3A%7B%22module%22%3A%22CDN.SrfCdnDispatchServer%22%2C%22method%22%3A%22GetCdnDispatch%22%2C%22param%22%3A%7B%22guid%22%3A%226543954968%22%2C%22calltype%22%3A0%2C%22userip%22%3A%22%22%7D%7D%2C%22req_0%22%3A%7B%22module%22%3A%22vkey.GetVkeyServer%22%2C%22method%22%3A%22CgiGetVkey%22%2C%22param%22%3A%7B%22guid%22%3A%226543954968%22%2C%22songmid%22%3A%5B%22",'',"%22%5D%2C%22songtype%22%3A%5B0%5D%2C%22uin%22%3A%221791670972%22%2C%22loginflag%22%3A1%2C%22platform%22%3A%2220%22%7D%7D%2C%22comm%22%3A%7B%22uin%22%3A1791670972%2C%22format%22%3A%22json%22%2C%22ct%22%3A24%2C%22cv%22%3A0%7D%7D"]
        self.head = {
            'User-Agent':"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.4098.3 Safari/537.36"
            }

    def get_musics(self):
        self.api[1] = self.mid
        index = 0
        res = []
        while 1:
            self.api[3] = str(index * 60)
            index += 1
            print("已提取%d页" % index)
            url = "".join(self.api)
            response = requests.get(url, headers=self.headers)
            data = json.loads(response.text)
            # print(data)
            res.extend(data['singer']['data']['songlist'])
            if len(data['singer']['data']['songlist']) <= 55:
                break
        return res

    def del_with_data(self, data):
        res = []
        for i in data:
            singer = []
            for j in i['singer']:
                singer.append(j['name'])
            res.append({"name": i['name'], "singers": ",".join(singer), "mid": i['file']['media_mid']})
        return res

    def get_vkey(self, mid):
        self.music_api[1] = mid
        data_url = ''.join(self.music_api)
        try:
            data = json.loads(requests.get(data_url, timeout=10).text)
        except:
            # print("网络错误")
            # time.sleep(2)
            return 0
        vkey = data['req']['data']['vkey']
        return vkey

    def download(self, queue, num):
        while 1:
            mid, filename = queue.get()
            vkey = self.get_vkey(mid)
            # 判断vkey
            if not vkey:
                print("vkey获取失败", filename)
                queue.task_done()
                continue
            # 拼接URL
            mmid = "C400" + mid + ".m4a"
            # print(vkey)
            download_url = self.download_api + mmid + '?' + 'guid=6543954968&vkey=' + vkey + "&uin=1224&fromtag=6"
            # print(download_url)
            # 验证歌曲名称有效性
            try:
                data = requests.get(download_url, headers=self.head, timeout=10).content
            except:
                print("连接超时")
                queue.task_done()
                # time.sleep(1)
                continue

            size = len(data)
            if size < 1024:
                print(filename, "未知原因")
                num.put(1)
                queue.task_done()
                continue
            try:
                with open(os.path.join(self.name, "%s.m4a" % filename), 'wb+') as f:
                    f.write(data)
            except:
                print("歌曲名称含有非法字符：%s" % filename)
                # continue
            else:
                print("已下载：%s    %dk " % (filename, (size / 1024)))
                num.put(1)
            finally:
                # 释放队列元素
                queue.task_done()

    def run(self):
        print("提取歌曲数据")
        data = self.del_with_data(self.get_musics())
        total = len(data)
        print("数据提取完毕，共%d首歌曲" % total)
        total = len(data)
        print("开始下载")
        tasks = Queue()
        num = Queue()
        for i in range(32):
            t = Thread(target=self.download, args=(tasks, num))
            t.daemon = True
            t.start()
        if not os.path.exists(self.name):
            os.makedirs(self.name)
        # 创建任务
        for i in data:
            # print(i['mid'])
            tasks.put((i['mid'], i['name']))
        tasks.join()
        down_successfully = num.qsize()
        print("成功下载%d首" % down_successfully)
        print("%d首下载失败" % (total - down_successfully))

a = Spider("0025NhlN2yWrP4", "周杰伦")
a.run()


"""
        ##############################################################################
        #                        KITE | Release  1.1                                 #
        #                                                                            #
        #                        Kite home: quantum-kite.com                         #
        #                                                                            #
        #  Developed by: Simao M. Joao, Joao V. Lopes, Tatiana G. Rappoport,         #
        #  Misa Andelkovic, Lucian Covaci, Aires Ferreira, 2018-2022                 #
        #                                                                            #
        ##############################################################################
"""

import numpy as np
import matplotlib.pyplot as plt
import sys
from matplotlib import rc

def process_arpes(name):
    # open file and read from it
    f = open(name,"r")
    whole = f.readlines()

    # inside that file, find where is each field. There are three fields:
    # list of energies, list of k-vectors and the arpes matrix
    print("Processing file: finding location of each dataset. ", end="")
    cutarpes, cutenergies, cutvectors = -1, -1, -1
    count = 0
    found = 0
    for i in whole:
      if("ARPES:" in i):
        cutarpes = count + 1
        found += 1
      elif("Energies:" in i):
        cutenergies = count + 1
        found += 1
      elif("k-vectors:" in i):
        cutvectors = count + 1
        found += 1

      count +=1
      if(found == 3):
        break
    print(f"Done. Locations: {cutarpes} {cutenergies} {cutvectors}")

    # process the energies and arpes matrix into numpy arrays
    print("Fetching energies. ", end="")
    energies = np.array([float(i[:-1]) for i in whole[cutenergies:cutarpes - 1]])
    print(f"Done. Number of energies: {len(energies)}")

    # process the arpes matrix
    print("Fetching ARPES matrix. ", end="")
    arpes = np.array([[float(y) for y in i[:-1].split(" ") if y!=""] for i in whole[cutarpes:]])
    print(f"Done. Arpes matrix dimensions: {len(arpes)} {len(arpes[0])}")

    # find the number of k-vectors
    print("Fetching k vectors. ", end="")
    kvectors = whole[cutvectors:cutenergies - 1]
    num_vectors = len(kvectors)
    nvectors = np.linspace(1, num_vectors, num_vectors)
    print(f"Done. Number of k vectors: {num_vectors}")
    # print(nvectors)

    # plot the matrix
    print("Making meshgrid. ", end="")
    X,Y = np.meshgrid(nvectors, energies)
    print("Done.")

    print("Plotting 2D color map. ", end="")
    fig, axs = plt.subplots(1,1,figsize=(20,15))
    axs.pcolor(X,Y,arpes,cmap='hot', rasterized=True)
    axs.set_xlabel("k-points", fontsize=30)
    axs.set_ylabel("Energy", fontsize=30)
    print("Done.")

    print("Saving figure.", end="")
    plt.savefig("arpes.pdf", format="pdf", dpi=50)
    print("Done.")

if __name__ == "__main__":
    name = sys.argv[1]
    process_arpes(name)


from PIL import Image, ImageDraw, ImageFont
from datetime import datetime
from qrcode import QRCode

from config import (
    FONT_PATH,
    label_width_px,
    label_length_px,
    header_fnt_size,
    normal_fnt_size,
    serial_fnt_size,
    vertical_padding,
    middle, LABEL_PATH
)

# QR and text dynamic positions.
text_1 = int(vertical_padding - 5 + header_fnt_size)
draw_1 = int(text_1 + vertical_padding - 5)
text_6 = int(label_length_px - 10)
text_5 = int(text_6 - vertical_padding - normal_fnt_size)
text_2 = int(text_5 - vertical_padding - normal_fnt_size)
text_4 = int(text_2 - vertical_padding - normal_fnt_size)
text_3 = int(text_4 - vertical_padding - normal_fnt_size)
text_7 = int(text_3 - vertical_padding - normal_fnt_size)
draw_2 = int(text_7 - 10 - normal_fnt_size)

# Font types initialization.
header_fnt = ImageFont.truetype(FONT_PATH, size=header_fnt_size)
body_fnt = ImageFont.truetype(FONT_PATH, size=normal_fnt_size)
serial_fnt = ImageFont.truetype(FONT_PATH, size=serial_fnt_size)

# Utility functions to get date and time.
def get_date():
    return datetime.today().strftime('%b/%d/%Y')

def get_date_for_id():
    return datetime.today().strftime('%m/%d/%Y')

def get_day_name():
    return datetime.today().strftime('%A')

def get_time():
    return datetime.today().strftime('%I:%M:%S %p')

# Main function to generate a label as an image.

def generate_label(lot, count, weight_str, barcode, unique_uuid, printer_serial="", label_path=LABEL_PATH):
    day_name = get_day_name()
    day_date = get_date()
    day_time = get_time()
    day_date_for_id = get_date_for_id()

    uniq_id = day_date_for_id.replace('/', '') + '-' + str(lot) + '-' + str(count)

    img = Image.new('RGB', (label_width_px, label_length_px), color='white')
    draw = ImageDraw.Draw(img)

    qr = QRCode(version=4, box_size=5, border=1)
    if barcode:
        qr.add_data(unique_uuid)
    else:
        qr.add_data(weight_str)
    qr.make(fit=True)
    qr_bitmap = qr.make_image(fill_color='black', back_color='white')

    draw.text((middle * 0.6, text_1), str(weight_str) + ' lb', fill='black', anchor='ms', font=header_fnt)
    img.paste(qr_bitmap, (50, draw_1))
    # draw.rectangle([((middle * 1.5) - 30, draw_2), ((middle * 1.5) + 30, draw_2 - 60)], fill=None, outline='black', width= 2)
    draw.text((middle * 1.5, text_7), day_name, fill='black', anchor='ms', font=serial_fnt)
    # draw the X
    # draw.line((((middle * 1.5) - 15), draw_2, ((middle * 1.5) + 15), (draw_2 - 30)), fill='black', width=3)
    # draw.line((((middle * 1.5) + 15), draw_2, ((middle * 1.5) - 15), (draw_2 - 30)), fill='black', width=3)
    # draw the serial
    draw.text((middle * 1.5, draw_2), printer_serial, fill='black', anchor='ms', font=serial_fnt)
    draw.text((middle * 1.5, text_3), day_date, fill='black', anchor='ms', font=serial_fnt)
    draw.text((middle * 1.5, text_4), day_time, fill='black', anchor='ms', font=serial_fnt)
    draw.text((middle * 1.5, text_2), uniq_id, fill='black', anchor='ms', font=serial_fnt)
    draw.text((middle * 1.5, text_5), barcode[-12:] if barcode else "No Barcode", fill='black', anchor='ms',
              font=serial_fnt)
    draw.text((middle * 1.5, text_6), unique_uuid[-12:], fill='black', anchor='ms', font=serial_fnt)
    img.save(label_path)


import argparse
import json
from collections import Counter
import heapq
import random

class Basic_hill_climb:
    def __init__(self, initial, random_start, para, goal, error, neighbors, verbose, sideways, restarts):
        self.init_state = initial
        self.random_start = random_start
        self.para = para
        self.goal_function = goal
        self.error_function = error
        self.neighbors_function = neighbors
        self.verbose = verbose
        self.sideways = sideways
        self.restarts = restarts
    def run(self):
        state = self.init_state
        print("Start: " + str(state) + " = " + str(self.error_function(state, self.para)))
        sideways_tmp = self.sideways
        sideways_vl = [] # visited list
        while not self.goal_function(state, self.para):
            neighbors = []
            for s in self.neighbors_function(state, self.para):
                if self.verbose:
                    print(str(s) + " = " + str(self.error_function(s, self.para)))
                heapq.heappush(neighbors, (self.error_function(s, self.para), s))
            if neighbors[0][0] >= self.error_function(state, self.para):
                if neighbors[0][0] == self.error_function(state, self.para) and sideways_tmp > 0:
                    # go sideway
                    flag = 0
                    for _ in range(len(neighbors)):
                        temp_state = heapq.heappop(neighbors)
                        if temp_state[0] > self.error_function(state, self.para):
                            break
                        if temp_state[1] in sideways_vl:
                            continue
                        state = temp_state[1]
                        sideways_vl.append(state)
                        sideways_tmp -= 1
                        print("Choose(sideways): " + str(state) + " = " + str(self.error_function(state, self.para)))
                        flag=1
                        break
                    if flag==1:
                        continue
                if self.restarts > 0:
                    # restart
                    self.restarts -= 1
                    sideways_tmp = self.sideways
                    state = self.random_start(self.para)
                    print("Restarting with: " + str(state) + " = " + str(self.error_function(state, self.para)))
                    sideways_vl = [state]
                    continue
                # no found.
                print("Cannot restart. Search fail.")
                quit()
            state = neighbors[0][1]
            sideways_tmp = self.sideways
            print("Choose: " + str(state) + " = " + str(neighbors[0][0]))
            sideways_vl = [state]
        print("Goal: " + str(state) + " = " + str(self.error_function(state, self.para)))

def error_nqueen(state, _):
    error = 0
    same_row = dict(Counter(state))
    for _, value in same_row.items():
        if value > 1:
            error += value*(value-1)/2
    state_1 = [state[i]+i for i in range(len(state))]
    same_row = dict(Counter(state_1))
    for _, value in same_row.items():
        if value > 1:
            error += value*(value-1)/2
    state_2 = [state[i]-i for i in range(len(state))]
    same_row = dict(Counter(state_2))
    for _, value in same_row.items():
        if value > 1:
            error += value*(value-1)/2
    return error

def error_knapsack(state, data):
    value = 0
    weight = 0
    for item in state:
        value += data["Items"][item]["V"]
        weight += data["Items"][item]["W"]
    return max(weight - data["M"], 0) + max(data["T"] - value, 0)

def goal_nqueen(state, _):
    if error_nqueen(state, _) == 0:
        return True
    else:
        return False

def goal_knapsack(state, data):
    if error_knapsack(state, data) == 0:
        return True
    else:
        return False

def neighbors_nqueen(state, _):
    neighbors = []
    for i in range(len(state)):
        for j in range(len(state)):
            if j == state[i]:
                continue
            neighbor = state[:]
            neighbor[i] = j
            neighbors.append(neighbor)
    return neighbors

def neighbors_knapsack(state, data):
    neighbors = []
    # delete
    for i in range(len(state)):
        temp = state[:]
        del temp[i]
        neighbors.append(temp)
    # Add
    for item in data["Items"]:
        if not item in state:
            temp = state[:]
            temp.append(item)
            neighbors.append(temp)
    # replace
    for i in range(len(state)):
        for item in data["Items"]:
            temp = state[:]
            if not item in state:
                del temp[i]
                temp.append(item)
                neighbors.append(temp)
    return neighbors

def rondom_start_nqueen(num):
    state = [i for i in range(num)]
    random.shuffle(state)
    return state

def rondom_start_knapsack(data):
    state = []
    for item in data["Items"]:
        if random.randint(0,1):
            state.append(item)
    return state

def run_nqueen(num_queen, verbose, sideways, restarts):
    initial_state = [i for i in range(num_queen)]
    hill_climb = Basic_hill_climb(initial_state, rondom_start_nqueen, num_queen, goal_nqueen, error_nqueen, neighbors_nqueen, verbose, sideways, restarts)
    hill_climb.run()

def run_knapsack(data, verbose, sideways, restarts):
    initial_state = data["Start"]
    hill_climb = Basic_hill_climb(initial_state, rondom_start_knapsack, data, goal_knapsack, error_knapsack, neighbors_knapsack, verbose, sideways, restarts)
    hill_climb.run()

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("-N", type=int)
    group.add_argument("jsonfile", nargs='?')
    parser.add_argument("-verbose", help="verbose mode", action="store_true")
    parser.add_argument("-sideways", type=int, default=0)
    parser.add_argument("-restarts", type=int, default=0)
    args = parser.parse_args()

    if args.verbose:
        verbose = True
    else:
        verbose = False
    sideways = args.sideways
    restarts = args.restarts

    if args.N:
        num_queen = args.N
        run_nqueen(num_queen, verbose, sideways, restarts)
    else:
        try:
            with open(args.jsonfile, "r") as fp:
                json_data = json.load(fp)
            trans_data = {"T": json_data["T"], "M": json_data["M"]}
            if "Start" in json_data:
                trans_data["Start"] = json_data["Start"]
            else:
                trans_data["Start"] = []
            trans_data["Items"] = {}
            for item in json_data["Items"]:
                trans_data["Items"][item["name"]] = {"V":item["V"], "W":item["W"]}
        except:
            print("Bad file.")
            quit()

        run_knapsack(trans_data, verbose, sideways, restarts)


#!/usr/bin/python3
'''This is a module'''

if __name__ == '__main__':

    import MySQLdb
    import sys

    db = MySQLdb.connect('localhost', sys.argv[1],
                         sys.argv[2], sys.argv[3])
    cursor = db.cursor()
    cursor.execute('''SELECT * FROM states WHERE name
                   LIKE BINARY "N%" ORDER BY id''')
    for i in cursor.fetchall():
        print(i)


# utils.py

import numpy as np
#from astropy import units as u # May be necesary because of object?
from astropy.time import Time
#from astropy.coordinates import Angle, ICRS  # May be necssary because of object?
import de421
import jplephem
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker

# Object that stores info for recreating plots
class track_data(object):
    # Plot metadata
    objname=''
    dt=0.
    nmctrials=0.
    startdate=0.
    enddate=0.
    refID=0.
    nobs=0
    npriorobs=0
    npharoobs=0
    # Tracks and their 1+2 sigma limits
    datearr=0.
    sep=0.
    sep_1sig=0. # 2D array, second index indicates min/max
    sep_2sig=0.
    pa=0.
    pa_1sig=0.
    pa_2sig=0.
    # measured dates/positions (and where they would be if they were bg)
    obsdates=0.
    meas_sep=0.
    meas_sep_err=0.
    bg_sep=0.
    meas_pa=0.
    meas_pa_err=0.
    bg_pa=0.
    # pharo measured dates/positions (and where they would be if they were bg)
    pharo_obsdates=0.
    pharo_sep=0.
    pharo_sep_err=0.
    pharo_bg_sep=0.
    pharo_pa=0.
    pharo_pa_err=0.
    pharo_bg_pa=0.
    # prior dates/positions (and where they would be if they were bg)
    prior_obsdates=0.
    prior_sep=0.
    prior_sep_err=0.
    prior_bg_sep=0.
    prior_pa=0.
    prior_pa_err=0.
    prior_bg_pa=0.
    prior_cite=''

    # The class "constructor"/"initializer"
    def __init__(self,objname,dt,nmctrials,startdate,enddate,refID,datearr,sep,sep_1sig,sep_2sig,pa,pa_1sig,pa_2sig,obsdates,meas_sep,meas_sep_err,bg_sep,meas_pa,meas_pa_err,bg_pa,pharo_obsdates,pharo_sep,pharo_sep_err,pharo_bg_sep,pharo_pa,pharo_pa_err,pharo_bg_pa,prior_obsdates,prior_sep,prior_sep_err,prior_bg_sep,prior_pa,prior_pa_err,prior_bg_pa,prior_cite):
        self.objname=objname
        self.dt=dt
        self.nmctrials=nmctrials
        self.startdate=startdate
        self.enddate=enddate
        self.refID=refID
        self.nobs=obsdates.size
        self.npharoobs=pharo_obsdates.size
        self.npriorobs=prior_obsdates.size
        self.datearr=datearr
        self.sep=sep
        self.sep_1sig=sep_1sig
        self.sep_2sig=sep_2sig
        self.pa=pa
        self.pa_1sig=pa_1sig
        self.pa_2sig=pa_2sig
        self.obsdates=obsdates
        self.meas_sep=meas_sep
        self.meas_sep_err=meas_sep_err
        self.bg_sep=bg_sep
        self.meas_pa=meas_pa
        self.meas_pa_err=meas_pa_err
        self.bg_pa=bg_pa
        self.pharo_obsdates=pharo_obsdates
        self.pharo_sep=pharo_sep
        self.pharo_sep_err=pharo_sep_err
        self.pharo_bg_sep=pharo_bg_sep
        self.pharo_pa=pharo_pa
        self.pharo_pa_err=pharo_pa_err
        self.pharo_bg_pa=pharo_bg_pa
        self.prior_obsdates=prior_obsdates
        self.prior_sep=prior_sep
        self.prior_sep_err=prior_sep_err
        self.prior_bg_sep=prior_bg_sep
        self.prior_pa=prior_pa
        self.prior_pa_err=prior_pa_err
        self.prior_bg_pa=prior_bg_pa
        self.prior_cite=prior_cite

# gets the background position of star (equivalent to background_position.pro)
# But in millarcseconds!
def get_bg_pos(sep,pa,pm_ra,pm_dec,ra,dec,parallax,ref_epoch,final_epoch,PA_near_zero=False,parallax_offset=True):
    """
    Takes separation and pos ang of stars, the proper motion, reference epoch and final epoch
    and returns final separation [mas] and position angle [deg]
    INPUTS:
    sep             -- separation between stars, in milliarcseconds
    pa              -- postion angle in degrees E of N
    pm_ra           -- proper motion of star, in RA, in milliarcseconds
    pm_dec          -- proper motion of star, in DEC, in milliarcseconds
    ra              -- RA of star in degrees
    dec             -- DEC of star in degrees
    parallax        -- parallax of star (in milliarcseconds; 1000.0*1/dist_pc)
    ref_epoch       -- Epoch at which ALL sep, pa, ra, dec were measured [decimal date]
    final_epoch     -- Epoch at which to compute output sep/pa [decimal date]
    KEYWORD INPUTS:
    PA_near_zero    -- Output PAs in interval (-180,180) instead of (0,360)
    parallax_offset -- Consider effect of Earth's motion (parallax) [default TRUE]
    OUTPUTS:
    out_sep         -- separation at final_epoch, in milliarcseconds
    out_pa          -- position angle at final_epoch, in degrees
    """
    # Get initial x,y separations
    x0=sep*np.sin(np.radians(pa))
    y0=sep*np.cos(np.radians(pa))

    # Get adjustment from proper motion (pm values in milliarcseconds/yr)
    # (-1 since companion moves opposite direction to star)
    dx_pm = (final_epoch-ref_epoch)*pm_ra*-1.0
    dy_pm = (final_epoch-ref_epoch)*pm_dec*-1.0
    new_x = x0 + dx_pm
    new_y = y0 + dy_pm

    # Get adjustment from parallax (if keyword set)
    if parallax_offset:
        # Get offsets at ref and final epochs
        # (unlike parallax.pro, we use real parallax here so no need to adjust for it later)
        (ref_raoff,ref_decoff)=get_parallax_offset(ra,dec,parallax,ref_epoch)
        (out_raoff,out_decoff)=get_parallax_offset(ra,dec,parallax,final_epoch)
        # Apply offset: first subtract off reference epoch to get true position, then add parallax offset
        # (However, apply -1 factor due to background object moving in opposite direction)
        new_x = (new_x+ref_raoff)  - out_raoff
        new_y = (new_y+ref_decoff) - out_decoff

    # Compute new sep and pa
    out_sep=np.sqrt(new_x**2.0+new_y**2.0)
    out_pa=np.degrees(np.arctan2(new_x,new_y)) # in degrees now
    out_pa=np.mod(out_pa,360.0) # between 0 and 360deg
    if PA_near_zero:
        if out_pa >180:
            out_pa-=360.0 # now between -180 to +180
    return (out_sep,out_pa)

# gets the offset due to parallax of Earth (equivalent to parallax.pro)
def get_parallax_offset(ra,dec,parallax,epoch,ra_units='detector'):
    """
    Takes the star's ra, dec, parallax, and a given epoch.
    Returns the parallax shift in star's position, in arcseconds (sexagesimal seconds for raoff; decimal seconds for decoff)
    INPUTS:
    ra          -- RA position of star, in degrees
    dec         -- DEC position of star, in degrees
    parallax    -- parallax of star, in MILLIarcseconds (1000.0*1/distance_pc)
    epoch       -- epoch (decimal years) to compute parallax offset (scalar or monotonically increasing vector)
    ra_units    -- USE EITHER:
                   'time': raoff is in seconds of time in RA (need to multiply by 15 to get same units as decoff)
                   'detector': [default] 1 unit displacement same in both horizontal/vertical direction
                               (factor of cos(dec) applied to raoff; multiply by 15 to get same units)
    OUTPUTS:
    raoff       -- parallax shift in star's position, in milliarcseconds
    decoff      -- parallax shift in star's position, in milliarcseconds
    """
    # Convert from deg to rad
    ra_rad=np.radians(ra)
    dec_rad=np.radians(dec)

    # Use jplephem with de421 to determine position of Earth geocenter wrt SS Barycenter
    eph=jplephem.Ephemeris(de421)
    t=Time(epoch,format='jyear',scale='ut1')
    JD=t.jd
    barycenter = eph.position('earthmoon', JD)
    moonvector = eph.position('moon', JD)
    earthPos = (barycenter - moonvector * eph.earth_share) / 1.49597871e+8
    usex=np.float(earthPos[0])
    usey=np.float(earthPos[1])
    usez=np.float(earthPos[2])

    # Compute offsets (in milliarcseconds)
    delta_alpha = parallax * 1./np.cos(dec_rad) * (usex*np.sin(ra_rad) - usey*np.cos(ra_rad))
    delta_delta = parallax * (usex*np.cos(ra_rad)*np.sin(dec_rad) + usey*np.sin(ra_rad)* np.sin(dec_rad) - usez*np.cos(dec_rad))
    # Convert RA seconds as necessary
    if ra_units=='time':
        delta_alpha = delta_alpha / 15.0
    elif ra_units=='detector':
        delta_alpha = delta_alpha * np.cos(dec_rad)
    else:
        raise Exception('ra_units set to unhandled option: {}'.format(ra_units))

    # Set outputs
    raoff=delta_alpha
    decoff=delta_delta

    return (raoff,decoff)

# Returns the value corresponding to given confidence interval
# Assumes 1D array!
def get_ci(array,ci):
    array_sort=np.sort(array)
    ind_low=np.floor((0.5-ci/2.0)*array.size).astype(np.int)
    ind_high=np.ceil((0.5+ci/2.0)*array.size).astype(np.int)
    return (array_sort[ind_low],array_sort[ind_high])

# Function to create and save the plot (o=track_data object)
def make_plot(o,outfilename,titlestring,titleloc,no_prior=False,no_pharo=False,no_bg=False,for_poster=False):
    # "for_poster" makes plots slightly differently formatted and coloured
    if for_poster:
        myplotfonts={'size': 36}
        max_n_xtics=4
        max_n_ytics=4
        xlabel_fontsize=24
        ylabel_fontsize=24
        our_data_fmt='ko'
    else: # this is for paper
        myplotfonts={'size': 24}
        max_n_xtics=4
        max_n_ytics=6
        xlabel_fontsize=24
        ylabel_fontsize=24
        our_data_fmt='ko'
    matplotlib.rc('font',**myplotfonts)
    # Use thicker axes border
    matplotlib.rc('axes', linewidth=2.0)
    # Get a few important parameters
    nobs=o.obsdates.size # not used
    # Create figure
    fig = plt.figure()
    # sep subplot
    ax_sep = fig.add_subplot(2,1,1)
    # Make title label (on SEP SUBPLOT) -- only if not making plots for poster!
    if not for_poster:
        if titleloc=='left':
            ax_sep.text(0.03,0.95,titlestring,weight='bold',horizontalalignment='left',verticalalignment='top',transform=ax_sep.transAxes)
        elif titleloc=='center':
            ax_sep.text(0.50,0.95,titlestring,weight='bold',horizontalalignment='center',verticalalignment='top',transform=ax_sep.transAxes)
        elif titleloc=='right':
            ax_sep.text(0.97,0.95,titlestring,weight='bold',horizontalalignment='right',verticalalignment='top',transform=ax_sep.transAxes)
        elif titleloc=='none':
            pass # don't make a title!
        else:
            raise Exception('Invalid value for titleloc: {}'.format(titleloc))
    # Plot BG track and 68+95 confidence intervals (use zorder to place shaded stuff behind the rest)
    if not no_bg:
        ax_sep.plot(o.datearr,o.sep,'k-',linewidth=2)
        ax_sep.fill_between(o.datearr,o.sep_1sig[:,0],o.sep_1sig[:,1],color='0.5',zorder=-1)
        ax_sep.fill_between(o.datearr,o.sep_2sig[:,0],o.sep_2sig[:,1],color='0.8',zorder=-2)
    # Plot constant sep value (with errorbar)
    #ax_sep.axhline(y=np.mean(o.meas_sep),color='black',linestyle=':')
    ax_sep.axhline(y=o.meas_sep[o.refID],color='black',linestyle=':')
    ax_sep.axhline(y=o.meas_sep[o.refID]+o.meas_sep_err[o.refID],color='black',linestyle=':')
    ax_sep.axhline(y=o.meas_sep[o.refID]-o.meas_sep_err[o.refID],color='black',linestyle=':')
    # For each measurement, plot actual position (filled) and if they were bg (open)
    if not for_poster and not no_bg:
        ax_sep.plot(o.obsdates,o.bg_sep,'ko',ms=10,mew=2,mfc='None')
    ax_sep.errorbar(o.obsdates,o.meas_sep,yerr=o.meas_sep_err,fmt=our_data_fmt,ms=10)
    ax_sep.set_xlim([o.startdate,o.enddate])
    #ax_sep.set_ylim([2440,2530]) manual adjustment for a specific plot
    # Plot PHARO measurements, if they exist AND ``no_pharo'' is not set:
    if o.npharoobs > 0 and not no_pharo:
        if not no_bg:
            ax_sep.plot(o.pharo_obsdates,o.pharo_bg_sep,'k^',ms=10,mew=2,mfc='None')
        ax_sep.errorbar(o.pharo_obsdates,o.pharo_sep,yerr=o.pharo_sep_err,fmt='k^',ms=10)
    # Plot prior measurements, if they exist AND ``no_prior'' is not set:
    if o.npriorobs > 0 and not no_prior:
        if not no_bg:
            ax_sep.plot(o.prior_obsdates,o.prior_bg_sep,'ks',ms=10,mew=2,mfc='None')
        ax_sep.errorbar(o.prior_obsdates,o.prior_sep,yerr=o.prior_sep_err,fmt='ks',ms=10)
    # Format x axis tics (turn off labels, but keep tic marks)
    ax_sep.xaxis.set_major_locator(ticker.MaxNLocator(nbins=max_n_xtics,integer=True)) # sets xtics at nice locations
    ax_sep.xaxis.set_major_formatter(ticker.NullFormatter()) # do not show xtic labels
    ax_sep.xaxis.set_minor_locator(ticker.AutoMinorLocator(n=4))
    # Format y axis tics (prune edge for ganged plots)
    ax_sep.yaxis.set_major_locator(ticker.MaxNLocator(nbins=max_n_ytics,prune='both')) # set ytics at nice locations
    ax_sep.yaxis.set_major_formatter(ticker.ScalarFormatter(useOffset=False))
    ax_sep.yaxis.set_minor_locator(ticker.AutoMinorLocator(n=2))
    if not for_poster:
        ax_sep.set_ylabel('Sep [mas]',fontsize=ylabel_fontsize)
    # Increase length and thickness of tics
    ax_sep.tick_params('both',length=10,width=2.0,which='major')
    ax_sep.tick_params('both',length=5,width=2.0,which='minor')
    # PA subplot
    ax_pa=fig.add_subplot(2,1,2)
    # Plot BG track and 68+95 confidence intervals (use zorder to place shaded stuff behind the rest)
    if not no_bg:
        ax_pa.plot(o.datearr,o.pa,'k-',linewidth=2)
        ax_pa.fill_between(o.datearr,o.pa_1sig[:,0],o.pa_1sig[:,1],color='0.5',zorder=-1)
        ax_pa.fill_between(o.datearr,o.pa_2sig[:,0],o.pa_2sig[:,1],color='0.7',zorder=-2)
    # Plot constant PA value (with errorbar)
    #ax_pa.axhline(y=np.mean(o.meas_pa),color='black',linestyle=':')
    ax_pa.axhline(y=o.meas_pa[o.refID],color='black',linestyle=':')
    ax_pa.axhline(y=o.meas_pa[o.refID]+o.meas_pa_err[o.refID],color='black',linestyle=':')
    ax_pa.axhline(y=o.meas_pa[o.refID]-o.meas_pa_err[o.refID],color='black',linestyle=':')
    # For each measurement, plot actual position (filled) and if they were bg (open)
    if not for_poster and not no_bg:
        ax_pa.plot(o.obsdates,o.bg_pa,'ko',ms=10,mew=2,mfc='None')
    ax_pa.errorbar(o.obsdates,o.meas_pa,yerr=o.meas_pa_err,fmt=our_data_fmt,ms=10)
    ax_pa.set_xlim([o.startdate,o.enddate])
    # Plot PHARO measurements, if they exist AND ``no_pharo'' is not set:
    if o.npharoobs > 0 and not no_pharo:
        if not no_bg:
            ax_pa.plot(o.pharo_obsdates,o.pharo_bg_pa,'k^',ms=10,mew=2,mfc='None')
        ax_pa.errorbar(o.pharo_obsdates,o.pharo_pa,yerr=o.pharo_pa_err,fmt='k^',ms=10)
    # Plot prior measurements, if they exist AND ``no_prior'' is not set:
    if o.npriorobs > 0 and not no_prior:
        if not no_bg:
            ax_pa.plot(o.prior_obsdates,o.prior_bg_pa,'ks',ms=10,mew=2,mfc='None')
        ax_pa.errorbar(o.prior_obsdates,o.prior_pa,yerr=o.prior_pa_err,fmt='ks',ms=10)
    # Format x axis tics
    ax_pa.xaxis.set_major_locator(ticker.MaxNLocator(nbins=max_n_xtics,integer=True)) # sets xtics at nice locations
    ax_pa.xaxis.set_major_formatter(ticker.FormatStrFormatter('%4d')) # format for xtic labels
    ax_pa.xaxis.set_minor_locator(ticker.AutoMinorLocator(n=4))
    if not for_poster:
        ax_pa.set_xlabel('Date [yr]',fontsize=xlabel_fontsize)
    # Format y axis tics (prune edges for ganged plots)
    ax_pa.yaxis.set_major_locator(ticker.MaxNLocator(nbins=max_n_ytics,prune='both')) # set ytics at nice locations
    ax_pa.yaxis.set_major_formatter(ticker.ScalarFormatter(useOffset=False))
    ax_pa.yaxis.set_minor_locator(ticker.AutoMinorLocator(n=2))
    if not for_poster:
        ax_pa.set_ylabel('PA [deg]',fontsize=ylabel_fontsize)
    # Increase length and thickness of tics
    ax_pa.tick_params('both',length=10,width=2.0,which='major')
    ax_pa.tick_params('both',length=5,width=2.0,which='minor')
    # Adjust so that everything fits and plots are stacked
    fig.tight_layout()
    plt.subplots_adjust(hspace=0.001) # no horizontal space between figures
    fig.savefig(outfilename) # png for display
    fig.savefig(outfilename.replace('.png','.svg'),format='svg',dpi=1200) # for plotting

# Output the data used in making plot
def make_data_table(o,outfilename):
    f=open(outfilename,'w')
    # Write: Object name, obs-date, separation, error, PA, error, citation
    for ii in range(0,o.npriorobs):
        f.write('{} & {:7.2f} & ${:6.1f} \pm {:6.1f}$ & ${:5.1f} \pm {:5.1f}$ & {}\\\\\n'.format(o.objname,o.prior_obsdates[ii],o.prior_sep[ii],o.prior_sep_err[ii],o.prior_pa[ii],o.prior_pa_err[ii],o.prior_cite[ii]))
    for jj in range(0,o.nobs):
        if o.meas_pa_err[jj] < 0.1:
            f.write('{} & {:7.2f} & ${:6.1f} \pm {:6.1f}$ & ${:5.2f} \pm {:5.2f}$ & {}\\\\\n'.format(o.objname,o.obsdates[jj],o.meas_sep[jj],o.meas_sep_err[jj],o.meas_pa[jj],o.meas_pa_err[jj],'this work'))
        else:
            f.write('{} & {:7.2f} & ${:6.1f} \pm {:6.1f}$ & ${:5.1f} \pm {:5.1f}$ & {}\\\\\n'.format(o.objname,o.obsdates[jj],o.meas_sep[jj],o.meas_sep_err[jj],o.meas_pa[jj],o.meas_pa_err[jj],'this work'))
    f.close()


import wget
import re
import os

import pandas as pd

data=pd.read_csv('cim_codes.csv')

data = data[1000:2000]

cimoccs=[]
cims=[]

for d in data.to_numpy():
#for d in [['A020']]:
    cim=d[0]
    print("process code: '"+cim+"'")

    url = 'https://www.scansante.fr/applications/statistiques-activite-MCO-par-diagnostique-et-actes/outilExcel?_program=mcoprog.affiche_cataghm.sas&base=deux&typt=cim&annee=2015&code='+cim
    print(url)
    wget.download(url, 'data.xml')

    f=open('data.xml',"r")
    found=0
    for line in f.readlines():
        if found:
            linetotal = line.strip()
            break
        if "Total" in line:
            found=1
        elif "secret statistique" in line:
            cimoccs.append(0)
            cims.append(cim)
            print("\t-> secret statistique (0)")
            found=2
            break
    f.close()
    os.remove("data.xml")
    if found==0:
        print("\t-> no observation")
    elif found==1:
        m = re.search('(.*)>(\s*[ 0-9]+)</td>', linetotal)
        if not m is None:
            nb=int(m.group(2).replace(" ",""))
            cimoccs.append(nb)
            cims.append(cim)
            print("\t-> "+str(nb))
        else:
            print("\t-> parsing error ('"+linetotal+"')")

cimcounts=pd.DataFrame({"cim":cims,"count":cimoccs})

cimcounts.to_csv("cimcounts.csv")


"""NASDAQ Data Link Model"""
__docformat__ = "numpy"

import argparse
import logging
import os
from typing import List

import pandas as pd
import requests

from openbb_terminal.config_terminal import API_KEY_QUANDL
from openbb_terminal.decorators import log_start_end
from openbb_terminal.rich_config import console

logger = logging.getLogger(__name__)

@log_start_end(log=logger)
def check_country_code_type(list_of_codes: str) -> List[str]:
    """Check that codes are valid for NASDAQ API"""
    nasdaq_codes = list(
        pd.read_csv(os.path.join(os.path.dirname(__file__), "NASDAQ_CountryCodes.csv"))[
            "Code"
        ]
    )
    valid_codes = []
    for code in list_of_codes.split(","):
        if code.upper() in nasdaq_codes:
            valid_codes.append(code.upper())
    if valid_codes:
        return valid_codes
    raise argparse.ArgumentTypeError("No valid codes provided.")

@log_start_end(log=logger)
def get_country_codes() -> List[str]:
    """Get available country codes for Bigmac index

    Returns
    -------
    List[str]
        List of ISO-3 letter country codes.
    """
    file = os.path.join(os.path.dirname(__file__), "NASDAQ_CountryCodes.csv")
    codes = pd.read_csv(file, index_col=0)
    return codes

@log_start_end(log=logger)
def get_big_mac_index(country_code: str) -> pd.DataFrame:
    """Gets the Big Mac index calculated by the Economist

    Parameters
    ----------
    country_code : str
        ISO-3 letter country code to retrieve. Codes available through get_country_codes().

    Returns
    -------
    pd.DataFrame
        Dataframe with Big Mac index converted to USD equivalent.
    """
    URL = f"https://data.nasdaq.com/api/v3/datasets/ECONOMIST/BIGMAC_{country_code}"
    URL += f"?column_index=3&api_key={API_KEY_QUANDL}"
    r = requests.get(URL)

    df = pd.DataFrame()

    if r.status_code == 200:
        response_json = r.json()
        df = pd.DataFrame(response_json["dataset"]["data"])
        df.columns = response_json["dataset"]["column_names"]
        df["Date"] = pd.to_datetime(df["Date"])

    # Wrong API Key
    elif r.status_code == 400:
        console.print(r.text)
    # Premium Feature
    elif r.status_code == 403:
        console.print(r.text)
    # Catching other exception
    elif r.status_code != 200:
        console.print(r.text)

    return df

@log_start_end(log=logger)
def get_big_mac_indices(country_codes: List[str]) -> pd.DataFrame:
    """Display Big Mac Index for given countries

    Parameters
    ----------
    country_codes : List[str]
        List of country codes (ISO-3 letter country code). Codes available through get_country_codes().

    Returns
    -------
    pd.DataFrame
        Dataframe with Big Mac indices converted to USD equivalent.
    """

    df_cols = ["Date"]
    df_cols.extend(country_codes)
    big_mac = pd.DataFrame(columns=df_cols)
    for country in country_codes:
        df1 = get_big_mac_index(country)
        if not df1.empty:
            big_mac[country] = df1["dollar_price"]
            big_mac["Date"] = df1["Date"]
    big_mac.set_index("Date", inplace=True)

    return big_mac


from django.db import models
from django.db.models import Q
from django.db.models.query import QuerySet
from django.utils import timezone

class AbstractQuerySet(QuerySet):
    def update(self, **kwargs):
        if "updated_at" not in kwargs:
            kwargs["updated_at"] = timezone.now()
        return super().update(**kwargs)

    def delete(self):
        return self.update(is_active=False)

    def force_delete(self):
        return super().delete()

class AbstractManager(models.Manager):
    def get_queryset(self, enforce_filter=True):
        qs = AbstractQuerySet(self.model, using=self._db)
        return qs.filter(is_active=True) if enforce_filter else qs

    def unfiltered(self):
        return self.get_queryset(enforce_filter=False)

class AbstractModel(models.Model):
    objects = AbstractManager()

    created_at = models.DateTimeField(
        default=timezone.now,
        help_text="Time of creation of this object",
        editable=False,
    )
    updated_at = models.DateTimeField(
        auto_now=True, help_text="Time of updation of this object", editable=False
    )
    is_active = models.BooleanField(
        default=True,
        help_text="Denotes if the object is active or not. Inactive objects behave similar to how a deleted object "
        "works.",
    )

    class Meta:
        abstract = True

    def update_fields(self, **params):
        self.updated_at = timezone.now()
        update_fields = ["updated_at"]

        for key, value in params.items():
            setattr(self, key, value)
            update_fields.append(key)

        self.save(update_fields=update_fields)

    def delete(self, using=None, keep_parents=False):
        return self.update_fields(is_active=False)