from scripts.helpful_scripts import get_account, get_contract, OPENSEA_URL
from brownie import DappToken, Escrow, SimpleNFT, network, config, ANFT
from web3 import Web3
import time

import yaml
import json
import os
import shutil

sample_token_uri = (
    "ipfs://Qmd9MCGtdVz2miNumBHDbvj8bigSgTwnr4SbyH6DNnpWdt?filename=0-PUG.json"
)

KEPT_BALANCE = Web3.toWei(1000, "ether")
KEPT_LOAT_BALANCE = Web3.toWei(500000, "ether")

def update_front_end():
    # sending the build folder
    src = "./build"
    # dest = "./front_end/src/chain-info"
    dest = "../loanAgainstNFT/contracts"

    copy_folders_to_front_end(src, dest)
    # # sending the front end our config in JSON format
    # with open("brownie-config.yaml", "r") as brownie_config:
    #     config_dict = yaml.load(brownie_config, Loader=yaml.FullLoader)
    #     with open("./front_end/src/brownie-config.json", "w") as brownie_config_json:
    #         json.dump(config_dict, brownie_config_json)
    # print("front end updated")

def copy_folders_to_front_end(src, dest):
    if os.path.exists(dest):
        shutil.rmtree(dest)
    shutil.copytree(src, dest)

def deploy_escrow_and_tokens_and_nfts():
    account = get_account()
    # non_owner = get_account(index=1)  # gonachi
    non_owner = get_account(key=1)  # goerli
    dapp_token = DappToken.deploy({"from": account})  # governance token
    # loan_token = LoanToken.deploy({"from": account})  # loan token
    escrow = Escrow.deploy(  # escrow wallet
        dapp_token.address,
        {"from": account},
        publish_source=config["networks"][network.show_active()]["verify"],
    )
    tx = dapp_token.transfer(  # no approval because the account is the owner
        escrow.address,
        dapp_token.totalSupply() - KEPT_BALANCE,
        {"from": account},  # 99.9%
    )
    tx.wait(1)
    # SimpleNFT, and we have the NFT address, can we mock NFT too? no need?
    simple_nft = SimpleNFT.deploy({"from": account})
    tx = simple_nft.createNFT(sample_token_uri, {"from": account})
    tx.wait(1)
    simple_nft_id = 0
    #
    # a_nft = get_contract("a_nft")
    a_nft = ANFT.deploy({"from": account})
    tx = a_nft.createNFT(sample_token_uri, {"from": account})
    tx.wait(1)
    a_nft_id = 0

    loan_token = get_contract("loan_token")
    loan_token_price_feed = get_contract("loan_token_price_feed")
    init_amount = (
        loan_token.balanceOf(account.address) / 10
    )  # give escrow half of the loan token for test
    loan_token.approve(escrow.address, init_amount, {"from": account})
    tx = loan_token.transfer(
        escrow.address,
        init_amount,
        {"from": account},
    )
    tx.wait(1)
    init_amount = (
        loan_token.balanceOf(account.address) / 100
    )  # give non-owner half of the left loan token for test
    loan_token.approve(non_owner.address, init_amount, {"from": account})
    tx = loan_token.transfer(
        non_owner.address,
        init_amount,
        {"from": account},
    )
    tx.wait(1)
    dict_of_allowed_nfts = {
        simple_nft: get_contract("simple_nft_price_feed"),
        a_nft: get_contract("a_nft_price_feed"),
        # b_nft: get_contract("b_nft_price_feed"),
    }
    add_allowed_nfts(escrow, dict_of_allowed_nfts, account)
    # set simple and a nft collection offers and specific offers
    loan_Amount = Web3.toWei(0.0001, "ether")
    loan_Days = 3
    loan_Interest = 286
    sel_index = 1
    tx = escrow.setCollectionOffers(
        simple_nft.address, loan_Amount, loan_Days, loan_Interest, sel_index
    )
    tx.wait(1)
    sel_index = 2
    tx = escrow.setCollectionOffers(
        simple_nft.address, loan_Amount / 2, loan_Days * 2, loan_Interest, sel_index
    )
    tx.wait(1)
    sel_index = 3
    tx = escrow.setCollectionOffers(
        simple_nft.address, loan_Amount / 4, loan_Days * 5, loan_Interest, sel_index
    )
    tx.wait(1)
    tx = escrow.setOffers(
        simple_nft.address, simple_nft_id, loan_Amount * 2, loan_Days * 3, loan_Interest
    )
    tx.wait(1)
    sel_index = 1
    tx = escrow.setCollectionOffers(
        a_nft.address, loan_Amount, loan_Days, loan_Interest, sel_index
    )
    tx.wait(1)
    sel_index = 2
    tx = escrow.setCollectionOffers(
        a_nft.address, loan_Amount / 4, loan_Days * 3, loan_Interest, sel_index
    )
    tx.wait(1)
    sel_index = 3
    tx = escrow.setCollectionOffers(
        a_nft.address, loan_Amount / 8, loan_Days * 6, loan_Interest, sel_index
    )
    tx.wait(1)
    tx = escrow.setOffers(
        a_nft.address, a_nft_id, loan_Amount * 3, loan_Days * 2, loan_Interest
    )
    tx.wait(1)

    sel_index = 2
    loanProcess2(escrow, simple_nft, simple_nft_id, sel_index, account, loan_token)
    get_stats(escrow)
    sel_index = 0
    loanProcess2(escrow, a_nft, a_nft_id, sel_index, account, loan_token)
    get_stats(escrow)
    tx = simple_nft.createNFT(sample_token_uri, {"from": non_owner})
    tx.wait(1)
    simple_nft_id = 1
    sel_index = 0
    loanProcess2(escrow, simple_nft, simple_nft_id, sel_index, non_owner, loan_token)
    get_stats(escrow)
    tx = a_nft.createNFT(sample_token_uri, {"from": non_owner})
    tx.wait(1)
    a_nft_id = 1
    sel_index = 2
    loanProcess2(escrow, a_nft, a_nft_id, sel_index, non_owner, loan_token)
    get_stats(escrow)

    simple_nft_id = 0
    repayProcess2(escrow, simple_nft, simple_nft_id, account, loan_token)
    get_stats(escrow)
    a_nft_id = 1
    repayProcess2(escrow, a_nft, a_nft_id, non_owner, loan_token)
    get_stats(escrow)
    simple_nft_id = 1
    repayProcess2(escrow, simple_nft, simple_nft_id, non_owner, loan_token)
    get_stats(escrow)
    a_nft_id = 0
    repayProcess2(escrow, a_nft, a_nft_id, account, loan_token)
    get_stats(escrow)

    # print(escrow.allowedNfts(0))
    # print(escrow.allowedNfts(1))
    # print(escrow.numOfAllowedNfts())
    # tx = escrow.updateAllowedNfts(simple_nft.address, False, {"from": account})
    # tx.wait(1)
    # print(escrow.allowedNfts(0))
    # print(escrow.numOfAllowedNfts())

    # return escrow, simple_nft, dapp_token, loan_token

def get_stats(escrow):
    numOfBorrowers = int(escrow.numOfBorrowers())
    print("There are", numOfBorrowers, "borrowers")
    print("They are", escrow.borrowers)
    for borrowerIndex in range(int(numOfBorrowers)):
        borrower = escrow.borrowers(borrowerIndex)
        numOfNftStaked = escrow.numOfNftStaked(borrower)
        print(borrower, "has", numOfNftStaked, "nfts staked")
        for index in range(int(numOfNftStaked)):
            stakedNftAddress = escrow.stakedNftAddress(borrower, index)
            stakedNftId = escrow.stakedNftId(borrower, index)
            print(index, "th staked nft address is ", stakedNftAddress)
            print(index, "th staked nft id is ", stakedNftId)
            nftLoanAmount = escrow.nftLoanAmount(stakedNftAddress, stakedNftId)
            nftLoanPeriod = escrow.nftLoanPeriod(stakedNftAddress, stakedNftId)
            nftLoanInterest = escrow.nftLoanInterest(stakedNftAddress, stakedNftId)

            nftLoanRepayAmount = escrow.nftLoanRepayAmount(
                stakedNftAddress, stakedNftId
            )
            nftLoanExpireTime = escrow.nftLoanExpireTime(stakedNftAddress, stakedNftId)
            nftLoanHolderAddress = escrow.nftLoanHolderAddress(
                stakedNftAddress, stakedNftId
            )
            print(
                "nftLoanAmount=",
                nftLoanAmount,
                "nftLoanPeriod=",
                nftLoanPeriod,
                "nftLoanInterest=",
                nftLoanInterest,
                "nftLoanRepayAmount=",
                nftLoanRepayAmount,
                "nftLoanExpireTime=",
                nftLoanExpireTime,
                "nftLoanHolderAddress=",
                nftLoanHolderAddress,
            )

def loanProcess2(escrow, simple_nft, simple_nft_id, sel_index, _account, loan_token):
    simple_nft.approve(escrow.address, simple_nft_id, {"from": _account})

    loan_amount, loan_days, loan_interest = escrow.getOffers(
        simple_nft.address, simple_nft_id, sel_index
    )
    loan_token.approve(escrow.address, loan_amount, {"from": _account})
    tx = escrow.requestLoan(
        loan_token.address,
        simple_nft.address,
        simple_nft_id,
        sel_index,
        {"from": _account},
    )
    tx.wait(1)

def repayProcess2(escrow, simple_nft, simple_nft_id, account, loan_token):
    time.sleep(1)
    # holder_address, expire_time, repay_amount = escrow.getNftLockData(
    #     simple_nft.address, simple_nft_id, {"from": account}
    # )
    deposit_amount = escrow.nftLoanRepayAmount(simple_nft.address, simple_nft_id)
    loan_token.approve(escrow.address, deposit_amount, {"from": account})
    tx = escrow.redeemLoan(
        loan_token.address,
        simple_nft.address,
        simple_nft_id,
        {"from": account},
    )
    tx.wait(1)

def loanProcess(escrow, simple_nft, simple_nft_id, account, loan_token):
    simple_nft.approve(escrow.address, simple_nft_id, {"from": account})
    tx = escrow.nftStaking(
        simple_nft.address,
        simple_nft_id,
        {
            "from": account,  # it's not the approval problem that make us cannt pass here
            # "gas_price": 0,
            # "gas_limit": 120000000000,
            # "allow_revert": True,
        },
    )
    tx.wait(1)

    loan_Amount = Web3.toWei(0.001, "ether")
    loan_Days = 3
    loan_Interest = 286
    escrow.setOffers(
        simple_nft.address, simple_nft_id, loan_Amount, loan_Days, loan_Interest
    )
    loan_amount, loan_days, loan_interest = escrow.getOffers(
        simple_nft.address, simple_nft_id
    )
    loan_token.approve(escrow.address, loan_amount, {"from": account})
    tx = escrow.loanTransfer(
        loan_token.address, account, loan_amount, {"from": account}
    )
    tx.wait(1)

    initTime = time.time()
    expireTime = initTime + loan_days * 24 * 60 * 60
    repayAmount = loan_amount * (1 + loan_interest / (10000))
    tx = escrow.nftLock(
        simple_nft.address,
        simple_nft_id,
        account,
        expireTime,
        repayAmount,
        {"from": account},
    )
    tx.wait(1)

def repayProcess(escrow, simple_nft, simple_nft_id, account, loan_token):
    time.sleep(1)
    holder_address, expire_time, repay_amount = escrow.getNftLockData(
        simple_nft.address, simple_nft_id, {"from": account}
    )
    deposit_amount = repay_amount
    current_time = time.time()
    if (holder_address == account.address) & (time.time() < expire_time):
        loan_token.approve(escrow.address, deposit_amount, {"from": account})
        tx = escrow.loanRepay(
            loan_token.address,
            deposit_amount,
            {"from": account},
        )
        tx.wait(1)
        if deposit_amount >= repay_amount:
            # simple_nft.approve(account, 0, {"from": account})
            tx = escrow.nftUnStaking(
                simple_nft.address, simple_nft_id, {"from": account}
            )
            tx.wait(1)

def add_allowed_nfts(escrow, dict_of_allowed_nfts, account):
    update = True
    for nft in dict_of_allowed_nfts:
        add_tx = escrow.updateAllowedNfts(nft.address, update, {"from": account})
        add_tx.wait(1)
        set_tx = escrow.setPriceFeedContract(
            nft.address, dict_of_allowed_nfts[nft], {"from": account}
        )
        set_tx.wait(1)

def main():
    deploy_escrow_and_tokens_and_nfts()
    # update_front_end()


from django.core.mail import EmailMessage
from django.contrib.sites.shortcuts import get_current_site
from django.utils.encoding import force_bytes, force_text
from django.utils.http import urlsafe_base64_encode, urlsafe_base64_decode
from django.template.loader import render_to_string

from .tokens import user_activation_token

def send_email(user, request, mail_subject, template_name, to_email):
    current_site = get_current_site(request)
    mail_subject = mail_subject
    if current_site.domain == 'localhost':
        port = ':1337'
        protocol ='http'
    else:
        port = ''
        protocol = 'https'
    message = render_to_string(template_name, {
        'user': '%s %s' %(user.first_name, user.last_name),
        'domain': current_site.domain,
        'port': '%s' % port,
        'protocol': protocol,
        'uid': urlsafe_base64_encode(force_bytes(user.pk)),
        'token': user_activation_token.make_token(user),
        'new_email': to_email
    })
    email = EmailMessage(mail_subject, message, to=[to_email])
    email.send()


from .models import Auction
from .serializer import AuctionSerializer
from utils.utils import Utils
from django.db import connection

class AuctionService:

    def create(self, auction):
        serializer = AuctionSerializer(data=auction)

        if serializer.is_valid():
            serializer.save()
            return True

        return False

    def isNotEmpty(self):
        return Auction.objects.all().first()

    def deleteOldAuctions(self):
        cursor = connection.cursor()
        cursor.execute("DELETE FROM %s WHERE `isActive`=0" % Auction._meta.db_table)
        cursor.execute("OPTIMIZE TABLE %s" % Auction._meta.db_table)

    def getConnectedRealmActiveAucCount(self, connected_realm):
        count = 0
        for realm in connected_realm:
            count += Auction.objects.filter(ownerRealm=realm.name, isActive=1).count()
        return count

    def getActiveAuctions(self, connected_realms):
        current_auctions = []
        current_auctions_auc = {}

        for connected_realm in connected_realms:
            current_auctions += Auction.objects.filter(ownerRealm=connected_realm['name'], isActive=1)
        counter = 0
        for current_auction in current_auctions:
            counter += 1
            current_auctions_auc[current_auction.auc] = current_auction
        return current_auctions_auc

    def getRealmPriceList(self, _itemId, _realm_name):
        price_tab = []
        current_auctions = Auction.objects.filter(ownerRealm=_realm_name, isActive=1, item=_itemId)
        for current_auction in current_auctions:
            quantity = current_auction.quantity
            price = Utils.unifyPrice(current_auction.buyout)/quantity
            # If price == 0 then auction is only for bid
            if price > 0:
                while quantity > 0:
                    price_tab.append(price)
                    quantity -= 1
        return price_tab

    def getAllRealmActiveAuctionsList(self, _realm_name):
        return Auction.objects.filter(ownerRealm=_realm_name, isActive=1)

    def getCurrentAuctions(self, connected_realms):
        current_auctions = []
        current_auctions_auc = []

        for connected_realm in connected_realms:
            current_auctions += Auction.objects.filter(ownerRealm=connected_realm['name'], isActive=1)
        counter = 0
        for current_auction in current_auctions:
            counter += 1
            current_auctions_auc.append(current_auction.auc)
        return current_auctions_auc

    def unactive(self, _connected_realms, _auc):
        for connected_realm in _connected_realms:
            auction = Auction.objects.filter(ownerRealm=connected_realm['name'], auc=_auc).first()

            if auction:
                auction.isActive = False
                auction.save()
                return True

        return False

    def updateAuction(self, _auc):
        auction = Auction.objects.filter(ownerRealm=_auc['ownerRealm'],
                                         auc=_auc['auc']).first()
        if auction:
            if str(auction.bid) != str(_auc['bid']) or str(auction.buyout) != str(_auc['buyout']) or str(auction.quantity) != str(_auc['quantity']) or str(auction.timeLeft) != str(_auc['timeLeft']):
                auction.bid = _auc['bid']
                auction.buyout = _auc['buyout']
                auction.quantity = _auc['quantity']
                auction.timeLeft = _auc['timeLeft']
                auction.save()
                return True

        return False


"""
Input
-----
intensities_experimental.csv
intensities_sensitivity_unscaled.csv
intensities_singlef_unscaled.csv
intensities_multif_unscaled.csv
conf[model][reference_amplitude]
conf[experimental][reference_amplitude]

Output
------
intensities.csv :
    columns: view, Experimental, Model_SingleFreq_Max, Model_SingleFreq_Centre,

"""
import logging

import pandas as pd
import arim

from . import common

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

def finalise_intensities(dataset_name, save):
    conf = arim.io.load_conf(dataset_name)
    result_dir = conf["result_dir"]

    try:
        exp_ref_amp = conf["experimental"]["reference_amplitude"]
        model_ref_amp = conf["model"]["reference_amplitude"]
    except KeyError:
        exp_ref_amp = 1.0
        model_ref_amp = 1 / conf["model"]["scaling"]
        logger.warning(
            "Missing reference amplitude model scaling. Use model scaling instead."
        )

    df_exp = pd.read_csv(result_dir / "intensities_experimental.csv", index_col=0)
    intensities = df_exp / exp_ref_amp

    df2 = pd.read_csv(result_dir / "intensities_sensitivity_unscaled.csv", index_col=0)
    # scale and save
    intensities["Model_Sensitivity"] = df2["Model_Sensitivity"] / model_ref_amp

    try:
        df3 = pd.read_csv(result_dir / "intensities_singlef_unscaled.csv", index_col=0)
    except FileNotFoundError:
        logger.info("Could not find intensities_singlef_unscaled.csv")
    else:
        # scale and save
        intensities["Model_SingleFreq_Centre"] = (
            df3["Model_SingleFreq_Centre"] / model_ref_amp
        )
        intensities["Model_SingleFreq_Max"] = (
            df3["Model_SingleFreq_Max"] / model_ref_amp
        )

    try:
        df4 = pd.read_csv(result_dir / "intensities_multif_unscaled.csv", index_col=0)
    except FileNotFoundError:
        logger.info("Could not find intensities_multif_unscaled.csv")
    else:
        # ignore useless model Model_MultiFreq_Max
        intensities["Model_MultiFreq_Max"] = df4["Model_MultiFreq_Max"] / model_ref_amp

    if save:
        intensities.to_csv(result_dir / "intensities.csv")

    return intensities

if __name__ == "__main__":
    args = common.argparser(__doc__).parse_args()
    intensities = finalise_intensities(args.dataset_name, args.save)
    print(intensities)


from django.shortcuts import get_object_or_404

from rest_framework import viewsets
from rest_framework.permissions import IsAuthenticated

from project.models import Project

from ...models import ProjectTeamRole
from ..serializers.role import TeamRoleSerializer

class TeamRoleViewSet(
        viewsets.ModelViewSet):

    model = ProjectTeamRole
    permission_classes = (IsAuthenticated,)
    serializer_class = TeamRoleSerializer
    pagination_class = None

    @property
    def project(self):
        return get_object_or_404(Project, pk=self.kwargs.get('project_pk'))

    def get_queryset(self):
        user = self.request.user

        if user.is_superuser:
            queryset = self.model.objects.filter(
                project_id=self.kwargs.get('project_pk'))
        else:
            queryset = self.model.objects.filter(
                project_id=self.kwargs.get('project_pk'),
                project__created_by=user)

        return queryset


# Desenvolva um programa que leia o nome, idade e sexo de 4 pessoas. No final mostre:
    # A média de idade do grupo.
    # Qual o nome do homem mais velho.
    # Quantas mulheres tem menos de 20 anos.

from datetime import date

# Variáveis de controle:
idadeh = 0
nm_h = ''
nm_m = ''
sx = ''
idadem = 0
somaidade = 0

# Objetos principais:
ano_at = date.today().year

# Algoritmo:

for i in range(1, 5, 1):
    print('----- {}ª PESSOA -----'.format(i))
    nome = str(input('Diga o nome da {}ª pessoa:\n'.format(i))).upper().strip()
    sexo = str(input('Qual o sexo da {}ª pessoa (m/h):\n'.format(i))).upper().strip()
    idadei = int(input('Por fim, em que ano nasceu a pessoa?\n'))
    somaidade += (ano_at - idadei)
    if i == 1:
        if sexo == 'H':
            nm_h = nome
            idadeh = ano_at - idadei
        else:
            nm_m = nome
            idadem += 1
    else:
        if sexo == 'H' and (ano_at - idadei) > idadeh:
            nm_h = nome
            idadeh = ano_at - idadei
        if sexo == 'M':
            if (ano_at - idadei) < 20:
                idadem += 1

média_idade = somaidade/i

print('O homem mais velho é {}, com {} anos.'.format(nm_h,idadeh))
print('A média_idade do grupo é {} anos.'.format(média_idade))
print('Existem {} mulheres com menos de 20 anos.'.format(idadem))


#!/usr/bin/env python3

"""
Script to make graph for QMEE collaboration network. Edge width represents number of collaborations, node colour represents
institution type.

INPUTS:
    None

OUTPUTS:
    ../Results/QMEENet_python.svg.pdf   =   network representing QMEE collaboration network

"""

__appname__ = 'Nets.py'
__author__ = 'Sam Turner (sat19@ic.ac.uk)'
__version__ = '0.0.1'
__license__ = 'GNU public'

# imports

import numpy as np
import pandas as pd
import itertools
import networkx as nx
import matplotlib.pylab as p
import math
import matplotlib.patches as mpatches

# load edges and nodes
edges = pd.read_csv("../Data/QMEE_Net_Mat_edges.csv", header=0)
edges.index = edges.columns
nodes = pd.read_csv("../Data/QMEE_Net_Mat_nodes.csv", header=0,index_col=0)

# adjacency list
AdjL = []
for inst1,inst2 in itertools.product(edges.index, edges.columns):
    if edges.loc[inst1, inst2] > 0:
        AdjL.append((inst1,inst2,edges.loc[inst1, inst2]))

# nodes list
sps = nodes.index.to_list()

# sizes
sizeL = nodes['Pis']

# colours for legend
conv = {'University':'green','Hosting Partner':'red','Non-Hosting Partners':'blue'}
collist = [conv[ty] for ty in nodes['Type'].to_list()]

red_patch = mpatches.Patch(color='red', label='Hosting Partner')
green_patch = mpatches.Patch(color='green', label='University')
blue_patch = mpatches.Patch(color='blue', label='Non-Hosting Partner')

# get positions
pos = nx.circular_layout(sps)

# Initialise graph, add edges to graph, and produce list of edge weights
ws=np.array([])

G = nx.Graph()
G.add_nodes_from(sps)

for l in AdjL:
     G.add_edges_from([(l[0], l[1])])
     ws=np.append(ws,l[2])

ws = np.log(ws) * 2

# calculate node sizes
NodSizs= 2000 * (1+sizeL-min(sizeL))/(1+max(sizeL)-min(sizeL))

# draw network
nx.draw_networkx(G, pos, width=ws,node_size= NodSizs,edge_color='grey', arrows = True, node_color=collist)
p.legend(handles=[red_patch,green_patch,blue_patch], loc = [0,0.7])

#save network
print("Saving network to ../Results/QMEENet_python.svg")
p.savefig("../Results/QMEENet_python.svg", format = "svg")


from app_ml.functionalities.preprocessing_mongo import read_from_mongo_as_dataframe
from app_ml.functionalities.constants import SAVED_MODEL_PATH
from app_ml.models.RNN import RNN
import pymongo

def main():
    cars_db = pymongo.MongoClient('mongodb://localhost:27017')['cars']
    data, labels = read_from_mongo_as_dataframe(cars_db)
    rnn_model = RNN(data.iloc[0][0].shape, 2)
    x_train, y_train, x_test, y_test = rnn_model.split_train_test(data, labels)
    rnn_model.train(x_train, y_train)
    rnn_model.test(x_test, y_test)
    rnn_model.model.save(SAVED_MODEL_PATH)

if __name__ == '__main__':
    main()


#sql requesting in python,
import sqlite3
import pandas

conn = sqlite3.connect("database.db")
cur = conn.cursor()
cur.execute("SELECT * FROM countries WHERE area >= 2000000")
rows = cur.fetchall()
conn.close()

for i in rows:
    print(i)

#put in csv
df = pandas.DataFrame.from_records(rows)
df.columns = ["Rank", "Country", "Area", "Population"]
print(df)
df.to_csv("countries_big_area.csv", index=False)

#insert into db
data = pandas.read_csv("ten_more_countries.txt")
conn = sqlite3.connect("database.db")
cur = conn.cursor()
for index,row in data.iterrows():
    print(row["Country"],row["Area"])
    cur.execute("INSERT INTO countries VALUES(NULL,?,?,NULL)",(row["Country"],row["Area"]))
conn.commit()
conn.close()

#files counter
import glob

file_list = glob.glob1("files", "*.py")
print(len(file_list))

#recursive files counter
file_list = glob.glob("files/subdirs/**/*.py", recursive=True)
print(len(file_list))


# -*- coding: utf-8 -*-

from django.conf.urls import url

from baremetal_service.bw_views import BmSharedBandwidthsViews, BmSharedBandwidthViews, BmSharedBandwidthFipViews
from baremetal_service.views import BaremetalServiceFloatingIPViews

urlpatterns = [

    # 共享带宽实例管理
    # 功能：购买，列表，编辑，详情，删除/批量删除

    # resource: shared_bandwidths_order
    # 功能：购买
    url(r'^shared_bandwidths_order$', BmSharedBandwidthsViews.as_view(
        actions={'post': 'create'})),

    # resource: shared_bandwidths
    # 功能：列表，编辑，删除/批量删除
    url(r'^shared_bandwidths$', BmSharedBandwidthsViews.as_view(
        actions={'get': 'list', 'put': 'update', 'post': 'destroy'})),

    # resource: shared_bandwidth
    # 功能：搜索详情
    url(r'shared_bandwidth$', BmSharedBandwidthViews.as_view(
        actions={'post': 'search'})),

    # resource: floating IP
    # 功能：添加弹性公网IP，移除弹性公网IP
    url(r'^shared_bandwidth/(?P<bandwidth_id>[0-9a-fA-F-]{1,})/floating_ips$',
        BmSharedBandwidthFipViews.as_view(actions={'get': "list_bandwidth_floatingips"})),
    url(r'^floating_ip/(?P<floating_ip_id>[0-9a-fA-F-]{1,})/shared_bandwidths$',
        BmSharedBandwidthFipViews.as_view(actions={'get': "list_floatingip_bandwidths"})),

    url(r'^shared_bandwidth/(?P<bandwidth_id>[0-9a-fA-F-]{1,})/attach_floating_ips$',
        BmSharedBandwidthFipViews.as_view(actions={'post': 'attach'},
                                          name='bandwidth_attach_floatingips')),

    url(r'^shared_bandwidth/(?P<bandwidth_id>[0-9a-fA-F-]{1,})/detach_floating_ips$',
        BmSharedBandwidthFipViews.as_view(actions={'post': 'detach'},
                                          name='bandwidth_detach_floatingips')),

    # 裸金属平台中涉及的更新：弹性公网IP（floating_ip）
    # 功能：列表，删除
    url(r'^floating_ips$', BaremetalServiceFloatingIPViews.as_view(
        actions={'get': 'floating_ip_list', 'delete': 'floating_ip_delete'})),

]


import _
import config
from logging import Logger
from sklearn.metrics import *
from utils.experiments_utils import *
from utils.experiments_utils.results.tables import *
from utils.helpers.datasets import Dataset
from utils.rulekit.classification import RuleClassifier
from steps.train import TrainedModelsResults

def reduce_mean(df: pd.DataFrame) -> pd.DataFrame:
    numerical_columns = df.select_dtypes('number').columns.tolist()
    data = {}
    for column in df.columns.tolist():
        if column in numerical_columns and '(std)' not in column:
            data[column] = [df[column].mean()]
            data[f'{column} (std)'] = [df[column].std()]
        else:
            data[column] = [df[column].iloc[0]]
    return pd.DataFrame(data)

def write_rules(
    results_dir: str,
    models: TrainedModelsResults
):
    models = getattr(models, 'original')
    for i, model in enumerate(models):
        results_cv_dir = f'{results_dir}/original/cv/{i + 1}'
        os.makedirs(results_cv_dir, exist_ok=True)
        rule_file_path = f'{results_cv_dir}/rules.txt'
        with open(rule_file_path, 'w+') as file:
            for rule in model.model.rules:
                file.write(
                    f'{str(rule)} (p={int(rule.weighted_p)}, n={int(rule.weighted_n)}, P={int(rule.weighted_P)}, N={int(rule.weighted_N)})\n')

def evaluate_models(
    dataset: Dataset,
    variant_name: str,
    dataset_type: str,
    models: List[RuleClassifier],
    logger: Logger
) -> None:
    logger.info(
        f'Evalute model for dataset: "{dataset.name}" ({dataset_type}) for variant: "{variant_name}"')
    metrics: Table = Tables.get(dataset.name, variant_name, dataset_type, 'metrics')
    conditions_stats: Table = Tables.get(
        dataset.name, variant_name, dataset_type, 'conditions_stats')
    confusion_matrix_train_avg = None
    confusion_matrix_test_avg = None

    for i, model in enumerate(models):
        if len(models) == 1:
            logger.info(f'Evaluate on train_test')
            X_train, y_train, X_test, y_test = dataset.get_train_test()
        else:
            logger.info(f'Evaluate for fold: {i + 1}')
            X_train, y_train, X_test, y_test = dataset.get_cv_fold(i + 1)

        fold_metrics: Table = Tables.get(
            dataset.name, variant_name, dataset_type, 'cv', str(i + 1), 'metrics')

        prediction_test = model.predict(X_test)
        prediction_train = model.predict(X_train)

        fold_metrics.rows.append({
            'dataset': dataset.name,
            'variant': variant_name,
            'dataset type': dataset_type,

            'BAcc (test)': balanced_accuracy_score(y_test, prediction_test),
            'BAcc (train)': balanced_accuracy_score(y_train, prediction_train),
            'Acc (test)': accuracy_score(y_test, prediction_test),
            'Acc (train)': accuracy_score(y_train, prediction_train),
            'rules':  sum([
                value if 'statistics' not in key else 0 for key, value in model.model.stats.conditions_stats.stats.items()
            ]), # change conditiosn counting so that alternatives are counted as one conditions
            'conditions_count': model.model.stats.rules_count * model.model.stats.conditions_per_rule,
            'avg conditions per rule': model.model.stats.conditions_per_rule,
            'avg rule quality': model.model.stats.avg_rule_quality,
            'avg rule precision': model.model.stats.avg_rule_precision,
            'avg rule coverage': model.model.stats.avg_rule_coverage,
            'training time total (s)': model.model.stats.time_total_s,
            'training time growing (s)': model.model.stats.time_growing_s,
            'training time pruning (s)': model.model.stats.time_pruning_s,

            'induction measure': model.get_params()['induction_measure'].replace('Measures.', ''),
            'pruning measure': model.get_params()['pruning_measure'].replace('Measures.', ''),
            'voting measure': model.get_params()['voting_measure'].replace('Measures.', ''),
        })
        fold_metrics.save()

        labels_values = Dataset(dataset.name).get_full()[1].unique().tolist()
        cm = confusion_matrix(
            y_train, prediction_train, labels=labels_values)
        if confusion_matrix_train_avg is None:
            confusion_matrix_train_avg = cm
        else:
            confusion_matrix_train_avg += cm
        train_confusion_matrix = pd.DataFrame(
            cm,
            index=[f'true:{value}' for value in labels_values],
            columns=[f'pred:{value}' for value in labels_values]
        )
        train_confusion_matrix.to_csv(
            f'{os.path.dirname(fold_metrics._file_path)}/confusion_matrix_train.csv')

        cm = confusion_matrix(
            y_test, prediction_test, labels=labels_values)
        if confusion_matrix_test_avg is None:
            confusion_matrix_test_avg = cm
        else:
            confusion_matrix_test_avg += cm
        test_confusion_matrix = pd.DataFrame(
            cm,
            index=[f'true:{value}' for value in labels_values],
            columns=[f'pred:{value}' for value in labels_values]
        )
        test_confusion_matrix.to_csv(
            f'{os.path.dirname(fold_metrics._file_path)}/confusion_matrix_test.csv')

        fold_conditions_stats: Table = Tables.get(
            dataset.name, variant_name, dataset_type, 'cv', str(i + 1), 'conditions_stats')

        conditions_stats_dict: dict = model.model.stats.conditions_stats.stats
        conditions_stats_dict['dataset'] = dataset.name
        conditions_stats_dict['variant'] = variant_name
        conditions_stats_dict['dataset type'] = dataset_type
        tmp = conditions_stats_dict
        if 'Inner alternatives statistics' in conditions_stats_dict:
            for key, value in conditions_stats_dict['Inner alternatives statistics'].items():
                tmp[f'Inner alternatives - {key}'] = value
            del conditions_stats_dict['Inner alternatives statistics']
        fold_conditions_stats.rows.append(conditions_stats_dict)
        fold_conditions_stats.save()

        metrics.rows += fold_metrics.rows
        conditions_stats.rows += fold_conditions_stats.rows

    confusion_matrix_train_avg = confusion_matrix_train_avg / len(models)
    confusion_matrix_train_avg = pd.DataFrame(
        confusion_matrix_train_avg,
        index=[f'true:{value}' for value in labels_values],
        columns=[f'pred:{value}' for value in labels_values]
    )
    confusion_matrix_train_avg.to_csv(
        f'{os.path.dirname(metrics._file_path)}/confusion_matrix_train.csv')

    confusion_matrix_test_avg = confusion_matrix_test_avg / len(models)
    confusion_matrix_test_avg = pd.DataFrame(
        confusion_matrix_test_avg,
        index=[f'true:{value}' for value in labels_values],
        columns=[f'pred:{value}' for value in labels_values]
    )
    confusion_matrix_test_avg.to_csv(
        f'{os.path.dirname(metrics._file_path)}/confusion_matrix_test.csv')

    metrics.set_df(reduce_mean(metrics.as_pandas()))
    conditions_stats.set_df(reduce_mean(conditions_stats.as_pandas()))

def evaluate_all_models(
    dataset_name: str,
    variant_name: str,
    models: TrainedModelsResults,
    logger: Logger
) -> None:
    logger.info(
        f'Evalute model for dataset: "{dataset_name}" for variant: "{variant_name}"')
    Tables.configure(directory=config.RESULTS_BASE_PATH)

    logger.info('Write model rules')
    write_rules(
        f'{config.RESULTS_BASE_PATH}/{dataset_name}/{variant_name}', models)
    if len(models.original) > 0:
        dataset = Dataset(dataset_name)
        evaluate_models(dataset, variant_name, 'original',
                        models.original, logger)

@step()
def evaluate_plain_model(
    dataset_name: str,
    variant_name: str,
    models: TrainedModelsResults,
):
    evaluate_all_models(dataset_name, variant_name, models,
                        evaluate_plain_model.logger)

@step()
def evaluate_models_inner_alternatives(
    dataset_name: str,
    variant_name: str,
    models: TrainedModelsResults,
):
    evaluate_all_models(dataset_name, variant_name, models,
                        evaluate_models_inner_alternatives.logger)

@step()
def evaluate_models_complex_conditions(
    dataset_name: str,
    variant_name: str,
    models: TrainedModelsResults,
):
    evaluate_all_models(dataset_name, variant_name, models,
                        evaluate_models_complex_conditions.logger)

@step()
def evaluate_models_complex_conditions_and_alternatives(
    dataset_name: str,
    variant_name: str,
    models: TrainedModelsResults,
):
    evaluate_all_models(dataset_name, variant_name, models,
                        evaluate_models_complex_conditions_and_alternatives.logger)


from enum import Enum

from selenium_util.locator import Locator
from selenium.webdriver.common.by import By

from pages.mortgage_rates_page import MortgageRatesPage
from pages.zillow_base_page import ZillowBasePage
from utilities.mortgage_math import calculate_payment

class LoanPrograms(Enum):
    """
    A class to represent the loan programs that are available in the web select element

    Note to developer, if you change the structure of this enum, make sure you fix all references to it, there are some
    references to the 0th and 1st indexes of the tuple values
    """
    FIXED_30 = ("Fixed30Year", 30)
    FIXED_15 = ("Fixed15Year", 15)
    ARM_5 = ("ARM5", 5)

    @staticmethod
    def lookup(html_value):
        """
        If you have an html value and you need the corresponding enum, use this static helper method
        :param html_value: the html "value" attribute on a <option> that you would like the enum for
        :return: The enum that represents that value
        """
        for program in LoanPrograms:
            if program.value[0] == html_value:
                return program
        assert False, "Failed to find [" + html_value + "] in LoanPrograms enum, please add a new enum value"

class MortgageCalcPage(ZillowBasePage):
    """
    Class that represents the mortgage calculator web page, inherits from ZillowBasePage

    ...

    Attributes
    ----------
    driver : webdriver
        webdriver that this page will use to interact with the web page, defined in parent class
    various locators : Locator
        Locator objects that define ways to find web elements on this page, see decelerations below

    Methods
    -------
    See below - numerous methods for setting, and asserting different web elements on this page
    """

    '''
    ***** BEGIN LOCATORS *****
    '''

    _HOME_PRICE_INPUT = Locator(By.ID, "homePrice")

    _DOWN_PAYMENT_PERCENT_INPUT = Locator(By.ID, "form-1_downPaymentPercent")
    _DOWN_PAYMENT_AMOUNT_INPUT = Locator(By.ID, "form-1_downPayment")

    _TERM_SELECT = Locator(By.ID, "form-1_term")

    _RATE_INPUT = Locator(By.ID, 'rate')
    _RATE_HELP_BUTTON = Locator(By.XPATH, "//span[text()=\"More info on Interest rate\"]/ancestor::button")
    # TODO ask dev for an ID on this element, this is the best way to locate the element currently and it is decently
    #  fragile to future changes
    _SEE_CURRENT_RATES_LINK = Locator(By.XPATH, "//a[text()=\"See current rates\"]")
    _RATE_ERROR_MESSAGE = Locator(By.CSS_SELECTOR, "[class*=StyledFormHelp]")

    # TODO ask dev for an ID on this element, this is the best way to locate the element currently and it is very
    #  fragile to future changes
    _ADVANCED_BUTTON = Locator(By.XPATH, "//button[text() = \"Advanced\"]")

    _PMI_CHECKBOX = Locator(By.ID, "form-1_includePMI")
    _TAXES_INSURANCE_CHECKBOX = Locator(By.ID, "form-1_includeTaxesInsurance")

    _TAXES_INPUT = Locator(By.ID, "form-1_propertyTaxRateAnnualAmount")
    _INSURANCE_INPUT = Locator(By.ID, "annualHomeownersInsurance")

    # TODO ask dev for an ID on this element, this is the best way to locate the element currently and it is very
    #  fragile to future changes
    _PAYMENT_TEXT = Locator(By.CSS_SELECTOR, "[y=\"20\"]")

    '''
    ***** END LOCATORS *****
    '''

    def __init__(self, driver):
        """
        Create a new MortgageCalcPage
        Assumes you have already navigated to this page, and will wait for the mortgage rate input element to be
        clickable before returning your new page object to you
        :param driver: webdriver that this page will use to interact with the web page
        """
        super().__init__(driver)
        self.wait_for_element_to_exist(self._RATE_INPUT).wait_for_element_to_be_clickable()

    def set_interest_rate(self, rate):
        """
        Set the interest rate to the given value
        :param rate: rate to set, can be any type, eventually it will be casted to string for entry
        :return: self, this page object after any changes
        """
        # TODO if pressing enter bug on Interest rate input is fixed, we could make this method send the enter key
        #  and then we could remove the click of payment
        self.get_element(self._RATE_INPUT).set_text(rate)
        # click on something else to make the input field lose focus and cause a re-calculation
        self.get_element(self._PAYMENT_TEXT).click()
        return self

    def assert_interest_rate(self, expected_rate):
        """
        assert that the interest rate input has an expected value
        :param expected_rate: the rate you expect
        :return: self, this page object after any changes
        """
        actual_value = self.get_element(self._RATE_INPUT).get_value()

        assert actual_value == expected_rate, \
            "Expected interest rate to be [" + expected_rate + "] but it was [" + actual_value + "]"
        return self

    def assert_interest_rate_has_value(self):
        """
        Simple assertion, just verifies that the interest rate input has a value
        :return: self, this page object after any changes
        """
        actual_value = self.get_element(self._RATE_INPUT).get_value()

        assert float(actual_value) > 0, \
            "Expected interest rate to be a non empty value greater than 0 but it was [" + actual_value + "]"
        return self

    def assert_payment(self, expected_payment):
        """
        Assert the calculated payment on the web page is the expected value
        :param expected_payment: numeric value of what you expect the payment to be
        :return: self, this page object after any changes
        """

        # add a $ and strip off an decimal places in the expected
        expected_payment = "${:,.0f}".format(expected_payment)

        payment_element = self.get_element(self._PAYMENT_TEXT)
        # We might have just changed an input,
        # give the payment element a chance to update if it hasn't yet (race condition)
        payment_element.wait_for_element_to_have_text(expected_payment)
        actual_value = payment_element.get_text()

        print("Asserting payment is [" + expected_payment + "]")

        # note we are comparing strings here
        assert actual_value == expected_payment, \
            "Expected payment to be [" + expected_payment + "] but it was [" + actual_value + "]"
        return self

    def assert_payment_given_input_values(self):
        """
        Assertion method to assert that based on what value all of the inputs have, the displayed calculation is correct
        This method was created to specifically test that when the page loads, it loads with all of the inputs having
        a value, and that the displayed calculation is correct based on those values.

        This method should NOT be used in place of assert_payment to prevent false positives. It's better practice to
        fully calculate the payment yourself, based on what you inputted

        :return: self, this page object after any changes
        """
        price = float(self.get_element(self._HOME_PRICE_INPUT).get_value().replace(",", ""))
        down_payment = float(self.get_element(self._DOWN_PAYMENT_AMOUNT_INPUT).get_value().replace(",", ""))

        term_selected = self.get_select_element(self._TERM_SELECT).get_selected_value()
        loan_program = LoanPrograms.lookup(term_selected)

        rate = float(self.get_element(self._RATE_INPUT).get_value())

        calculated_payment = calculate_payment(price, down_payment, rate, loan_program)

        monthly_taxes = float(self.get_element(self._TAXES_INPUT).get_value().replace(",", "")) / 12
        monthly_insurance = float(self.get_element(self._INSURANCE_INPUT).get_value().replace(",", "")) / 12

        # our calculation above does not consider taxes and insurance, so now that we have scraped them off the page,
        # add them into to our expected total
        total_payment = calculated_payment + monthly_taxes + monthly_insurance

        self.assert_payment(total_payment)

        return self

    def set_home_price(self, home_price):
        """
        Set the home price input to the desired value
        :param home_price: value to enter into the input, will be casted to string
        :return: self, this page object after any changes
        """
        self.get_element(self._HOME_PRICE_INPUT).set_text(home_price)
        return self

    def set_down_payment_percent(self, percent):
        """
        Set the down payment percent input to the desired value (numeric or string accepted)
        Note your percent should not include a % if you choose to pass a string
        :param percent: desired percent (numeric or string accepted)
        :return: self, this page object after any changes
        """
        self.get_element(self._DOWN_PAYMENT_PERCENT_INPUT).set_text(percent, True)
        return self

    def assert_down_payment_percent(self, expected_percent):
        """
        Assert the down payment percent input is the expected value
        :param expected_percent: value to assert against the input, should be numeric
        :return: self, this page object after any changes
        """
        percent_element = self.get_element(self._DOWN_PAYMENT_PERCENT_INPUT)
        percent_element.wait_for_element_to_have_value(expected_percent, True)
        actual_value = percent_element.get_value()
        assert float(actual_value) == float(expected_percent), \
            "Expected down payment percent to be [" + str(expected_percent) + "] but it was [" + str(actual_value) + "]"
        return self

    def set_down_payment_amount(self, amount):
        """
        Set the down payment amount input to the desired value
        :param amount: desired value, will be casted to string before entered
        :return: self, this page object after any changes
        """
        self.get_element(self._DOWN_PAYMENT_AMOUNT_INPUT).set_text(amount, True)
        return self

    def assert_down_payment_amount(self, expected_amount):
        """
        Assert that the down payment amount has an expected value
        :param expected_amount: numeric value that you expect the input to have
        :return: self, this page object after any changes
        """

        # Add in commas to your expected amount and remove any decimal places
        expected_amount = "{:,.0f}".format(expected_amount)

        actual_value = self.get_element(self._DOWN_PAYMENT_AMOUNT_INPUT).get_value()

        # note comparing strings here
        assert actual_value == expected_amount, \
            "Expected down payment to be [" + str(expected_amount) + "] but it was [" + str(actual_value) + "]"
        return self

    def select_loan_program(self, loan_program: LoanPrograms):
        """
        Select the desired loan program in the drop-down
        :param loan_program: Enum for the value you would like selected
        :return: self, this page object after any changes
        """
        # 0th index of the loan program value is the html value attribute for that choice
        self.get_select_element(self._TERM_SELECT).select_by_value(loan_program.value[0])
        return self

    def _open_advanced(self):
        """
        Open the advanced drop down to expose the advanced options, smart enough to first determine if it is already
        open and do nothing
        """
        optional_element = self.get_element_if_exists(self._ADVANCED_BUTTON)

        if optional_element is not None:
            optional_element.click()
            self.wait_for_element_to_exist(self._TAXES_INSURANCE_CHECKBOX)

        # private method that does not refresh the page, does not need to return self

    def check_taxes_insurance(self, check):
        """
        Check or uncheck the taxes/insurance checkbox based on your desired value
        :param check: true to check the box, false to uncheck it
        :return: self, this page object after any changes
        """
        self._open_advanced()
        self.get_checkbox_element(self._TAXES_INSURANCE_CHECKBOX).check(check)
        return self

    def check_pmi(self, check):
        """
        Check or uncheck the PMI checkbox based on your desired value
        :param check: true to check the box, false to uncheck it
        :return: self, this page object after any changes
        """
        self._open_advanced()
        self.get_checkbox_element(self._PMI_CHECKBOX).check(check)
        return self

    def assert_interest_help_modal_opens_and_closes(self, click_x):
        """
        Assert that the interest help modal will open, be click-able and then assert that it closes if someone clicks
        off the modal, or clicks the close x button
        :param click_x: do you want to click the x button (True) or do you want to click off the modal (FALSE)
        :return: self, this page object after any changes
        """

        # create locator inline because it is very unlikely that any other method in this class will ever need it. If
        # that changes, move this to a class deceleration
        modal_p_loc = Locator(By.XPATH, "//p[contains(text(), \"Representative interest rates\")]")

        # Assert that the modal is not open, needed to prove we can tell it's not open
        assert len(self.driver.find_elements(*modal_p_loc.as_args())) == 0, \
            "Found modal in DOM before trying to open it, this means our next assertion cannot prove " \
            "that the modal is actually open"

        self.get_element(self._RATE_HELP_BUTTON).click()

        # Wait for the modal to open and be click-able, if either of these fail this test will fail before the assertion
        modal_p_element = self.wait_for_element_to_exist(modal_p_loc)
        modal_p_element.wait_for_element_to_be_clickable()

        # Technically covered by the conditions above, but a good sanity check
        assert len(self.driver.find_elements(*modal_p_loc.as_args())) == 1, \
            "Interest rate help modal did not open"

        if click_x:
            # create locator inline because it is very unlikely that any other method in this class will ever need it.
            # If that changes, move this to a class deceleration
            self.get_element(Locator(By.CSS_SELECTOR, "[class*=CloseButton]")).click()
        else:
            # click somewhere else to close the modal
            self.get_element(self._PAYMENT_TEXT).click()

        # Wait until the modal is no longer in the DOM
        modal_p_element.wait_for_element_to_be_stale()

        # Technically covered by the condition above, but a good sanity check to make sure it's actually gone
        assert len(self.driver.find_elements(*modal_p_loc.as_args())) == 0, \
            "Found modal after it was supposedly closed"

        return self

    def click_see_current_rates(self):
        """
        Click the see current rates link and navigate to that page
        :return: a new MortgageRatesPage as a result of navigation
        """
        # Note! opens in a new tab
        self.get_element(self._SEE_CURRENT_RATES_LINK).click()
        # close the current window
        self.driver.close()
        # switch to the new tab we opened
        self.driver.switch_to.window(self.driver.window_handles[0])
        return MortgageRatesPage(self.driver)

    def assert_interest_rate_error_message(self, expected_message):
        """
        Assert the interest rate error message
        :param expected_message: error message that you expect to be showing
        :return: self, this page object after any changes
        """
        message_p_element = self.wait_for_element_to_exist(self._RATE_ERROR_MESSAGE)
        actual_message = message_p_element.get_text()

        assert actual_message == expected_message, \
            "Expected down payment percent to be [" + expected_message + "] but it was [" + actual_message + "]"
        return self

    def assert_no_interest_rate_error_message(self):
        """
        Assert that the interest rate input does NOT have any error messages
        :return: self, this page object after any changes
        """
        potential_message_elements = self.get_elements(self._RATE_ERROR_MESSAGE)

        assert len(potential_message_elements) == 0, \
            "Unexpected error message found: " + potential_message_elements[0].get_text()
        return self


import feedparser
import datetime
import dateutil.parser
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404, render, redirect
from django.urls import reverse
from django.views import generic
from django_feedparser.settings import *
from .models import Story
from source.models import Sources
from django.utils import timezone
from django.template import loader
from django.db.models.aggregates import Count
from random import randint

def view_stories(request,id):
    num_post = 15
        # adding stories in database
    try:
        rss_url = Sources.objects.get(id=id).rss_text
    except Sources.DoesNotExist:
        alert("Does not Exist")
    feed = feedparser.parse(rss_url)
    for post in Story.objects.all():
        post.delete()

    story_urls = Story.objects.filter(id=id).values_list('url', flat=True).distinct()
    for entry in feed['entries']:
        url = entry.get('link')
        if url not in story_urls:
            story = Story()
            story.title = entry.get('title')
            story.url = url
            date=entry.get('published')
            story.pub_date = dateutil.parser.parse(date)
            body = entry.get('description')
            from bs4 import BeautifulSoup
            soup = BeautifulSoup(body)
            body = soup.get_text()
            story.body_text = body
            story.save()
# passing stories data to template context and render it
    context ={
    "id":id
    }
    return render(request, 'story/story_list.html', context)

def newsfeed(request):
    num_post = 20
    args={}
        # adding stories in database
    query_params = {}
    source_qs = Sources.objects.filter(**query_params).values('rss_text')
    for post in Story.objects.all():
        post.delete()
    for source in source_qs:
        feed = feedparser.parse(source.get('rss_text'))

        story_urls = Story.objects.all().values_list('url', flat=True).distinct()
        for entry in feed['entries']:
                url = entry.get('link')
                if url not in story_urls:
                    story = Story()
                    story.title = entry.get('title')
                    story.url = url
                    date=entry.get('published')
                    story.pub_date = dateutil.parser.parse(date)
                    story.save()
    args['story']=story
    return render(request, 'story/newsfeed.html', args)


import argparse
import logging
import pathlib

import matplotlib.pyplot as plt

# from train import test

plt.style.use("ggplot")
import gc
from pprint import pformat

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

# from functorch import grad, make_functional_with_buffers, vmap
from torch.utils.data import DataLoader, Subset
from torchsummary import summary
from tqdm import trange
from tqdm import tqdm

from utils.common import (
    AlexNet,
    DatasetwithIndices,
    create_config,
    get_dataset,
    get_dataset_with_indices,
    get_logger,
    get_model,
    get_optimizer,
    get_parser,
    get_test_dataset,
    get_train_dataset,
    seed_everything,
)

# def get_mean_gradients(model, loader, use_all_params=False):
#     num_params = len(
#         list(model.parameters() if use_all_params else model.fc.parameters())
#     )
#     mean_gradients = [None for i in range(num_params)]
#     num_iter = len(loader)
#     progress_bar = tqdm(
#         loader, total=num_iter, desc="Mean Gradients", leave=False, position=2
#     )
#     for batch in progress_bar:
#         images, labels, _ = batch
#         torch.cuda.empty_cache()
#         images, labels = images.to(device), labels.to(device)
#         output = model(images)
#         gradient = torch.autograd.grad(
#             F.nll_loss(output, labels),
#             model.parameters() if use_all_params else model.fc.parameters(),
#         )
#         if mean_gradients[0] is not None:
#             for j in range(num_params):
#                 mean_gradients[j] += gradient[j].detach()  # .cpu().numpy()
#         else:
#             for j in range(len(gradient)):
#                 mean_gradients[j] = gradient[j].detach()  # .cpu().numpy()

#         for j in range(len(gradient)):
#             mean_gradients[j] /= num_iter

#     return mean_gradients

# def get_similarities(model, dataset, batch_size, mean_gradients, use_all_params=False):
#     slmodel, params, buffers = make_functional_with_buffers(
#         model if use_all_params else model.fc
#     )
#     loader = DataLoader(
#         dataset, batch_size, shuffle=True, num_workers=2, pin_memory=True
#     )

#     def loss_function(params, buffers, x, y):
#         x = x.unsqueeze(0)
#         y = y.unsqueeze(0)
#         preds = slmodel(params, buffers, x)
#         return F.nll_loss(preds, y)

#     batched_loss = vmap(
#         grad(loss_function),
#         (None, None, 0, 0),
#     )

#     similarities = []
#     img_indices = []
#     progress_bar = tqdm(
#         enumerate(loader),
#         total=len(loader),
#         desc="Per Sample Gradient Similarity",
#         leave=False,
#         position=2,
#     )
#     for i, batch in progress_bar:
#         imgs, labels, inds = batch
#         torch.cuda.empty_cache()
#         imgs, labels, inds = imgs.to(device), labels.to(device), inds.numpy()
#         with torch.no_grad():  ### TODO: Add if else for if use_all_params
#             hidden_state = model.features(imgs)
#         gradient = batched_loss(params, buffers, hidden_state, labels)
#         gc.collect()
#         torch.cuda.empty_cache()
#         # gradient = torch.autograd.grad(F.nll_loss(model(imgs), labels), model.parameters())

#         sim = (
#             torch.stack(
#                 [
#                     F.cosine_similarity(a.view(a.shape[0], -1), b.view(1, -1))
#                     for a, b in zip(gradient, mean_gradients)
#                 ],
#                 dim=-1,
#             )
#             .sum(dim=-1)
#             .detach()
#             .cpu()
#             .numpy()
#         )
#         # sim = torch.stack(list(map(lambda, gradient, mean_gradients))).sum()
#         similarities.append(sim)
#         img_indices.append(inds)
#     return np.concatenate(similarities), np.concatenate(img_indices)

# def cosinesimilarity(a,b):
#     return np.divide(np.dot(a, b.T), (np.linalg.norm(a, axis=-1, keepdims=True) * np.linalg.norm(b, keepdims=True)))

def cosinesimilarity(a, b):
    return torch.divide(
        torch.mm(a, b.T),
        (
            torch.linalg.norm(a, dim=-1, keepdims=True)
            * torch.linalg.norm(b, keepdims=True)
        ),
    )

def get_sims(gradients):
    mean_gradients = gradients.mean(dim=0, keepdims=True)
    return cosinesimilarity(gradients, mean_gradients)

def get_mean_gradients(p, model, loader, criterion, optimizer):
    num_params = len(list(model.get_last_layer().parameters()))
    num_iter = len(loader)
    embedding_dim = model.get_last_layer().in_features
    sample_num = len(loader.dataset)
    gradients = torch.zeros(
        [sample_num, p.num_classes * (embedding_dim + 1)],
        requires_grad=False,
        device="cuda",
    )
    img_indices = torch.zeros([sample_num], requires_grad=False, device="cuda")
    # gradients = np.zeros([sample_num, p.num_classes * (embedding_dim + 1)])
    # img_indices = np.zeros(sample_num)
    progress_bar = tqdm(
        loader, total=num_iter, desc="Mean Gradients", leave=False, position=2
    )
    model.eval()
    model.no_grad = True
    with model.embedding_recorder:
        for i, batch in enumerate(progress_bar):
            optimizer.zero_grad(set_to_none=True)
            images, labels, inds = batch
            torch.cuda.empty_cache()
            images, labels = images.to(device), labels.squeeze().to(device)
            output = model(images).requires_grad_(True)
            # logger.info((output.shape, labels.shape, inds.shape))
            loss = criterion(output, labels).sum()
            batch_num = labels.shape[0]
            with torch.no_grad():
                bias_parameters_grads = torch.autograd.grad(
                    loss, output, retain_graph=True
                )[0].cpu()
                weight_parameters_grads = model.embedding_recorder.embedding.cpu().view(
                    batch_num, 1, embedding_dim
                ).repeat(1, p.num_classes, 1) * bias_parameters_grads.view(
                    batch_num, p.num_classes, 1
                ).repeat(
                    1, 1, embedding_dim
                )

                gradients[
                    i * p.batch_size : min((i + 1) * p.batch_size, sample_num)
                ] = torch.cat(
                    (bias_parameters_grads, weight_parameters_grads.flatten(1)), dim=1
                )
            img_indices[
                i * p.batch_size : min((i + 1) * p.batch_size, sample_num)
            ] = inds.squeeze()

    return gradients, img_indices

def train_epoch(
    loader: torch.utils.data.DataLoader,
    model: nn.Module,
    criterion: nn.NLLLoss,
    optimizer: torch.optim.Optimizer,
    device: torch.device,
) -> None:
    """Trains model for one epoch

    Args:
        loader (torch.utils.data.DataLoader): Dataloader
        model (nn.Module): model
        criterion (nn.NLLLoss): Loss Function
        optimizer (torch.optim.Optimizer): Optimizer
        device (torch.device): device
    """
    model.train()
    model.no_grad = False
    # losses, accs = [], []
    for images, labels, _ in loader:
        images, labels = images.to(device), labels.to(device)
        optimizer.zero_grad(set_to_none=True)
        output = model(images)
        loss = criterion(output, labels)
        loss.backward()
        optimizer.step()
        # losses.append(loss.item())
        # acc = output.argmax(dim=1).eq(labels).float().mean().item()
        # accs.append(acc)
    model.eval()
    # return np.mean(losses), np.mean(accs)

# def train_loop(p, model):
#     # train_loader = DataLoader(data, p.batch_size, shuffle=True)

#     train_loader = DataLoader(get_train_dataset(p), p.batch_size, shuffle=True, num_workers=2, pin_memory=True)
#     test_data = get_test_dataset(p)
#     test_loader = DataLoader(test_data, p.batch_size)

#     criterion = nn.NLLLoss()
#     optimizer = get_optimizer(p, model)

#     losses, accs = [], []
#     for epoch in trange(p.train_epochs, leave=True):
#         model.train()
#         loss, acc = train_epoch(
#             train_loader, model, criterion, optimizer, device
#         )
#         losses.append(loss)
#         accs.append(acc)
#         gc.collect()
#         torch.cuda.empty_cache()

#     test_correct = test(test_loader, model, device)
#     test_acc = test_correct / len(test_data) * 100
#     return test_acc

def gradient_mathcing(p, data, logger):
    """Calculated mean gradient for the given dataset and find per sample similarity with mean gradients

    Args:
        p (EasyDict): Hyperparameters
        data (Dataset): Dataset
        logger

    Returns:
        tuple[np.ndarray, np.ndarray]: Arrays of shape (iter, len(dataset)) for similarities calculated for each sample for every iteration and corresponding indices
    """
    iterations = p.iter
    logger.debug(len(data))
    # assert len(data) % p.batch_size == 0, "All batches are not of same shape"

    if p.per_class:
        logger.info("Finding Mean Gradients for each class individually.")
        train_labels = torch.as_tensor(data.targets)
        cls_data = [
            Subset(data, torch.argwhere(train_labels == c))
            for c in range(p.num_classes)
        ]
        logger.debug(f"len datasets: {len(data)}")
    else:
        logger.info("Finding Mean Gradients for whole dataset at once.")

    seed_everything(p.seed)
    model = get_model(p, device)
    logger.info(
        "Model Summary\n"
        + str(summary(model, (p.channel, *p.im_size), verbose=0, device=device))
    )

    train_loader = DataLoader(
        data, p.batch_size, shuffle=True, num_workers=2, pin_memory=True
    )
    criterion = nn.CrossEntropyLoss().to(device)
    optimizer = get_optimizer(p, model)

    all_similarities, all_imginds = [], []
    progressbar = trange(iterations, desc="Iterations", position=0, leave=True)
    for k in progressbar:
        # if p.with_train:
        # moving to the end of loop
        if not p.with_train:
            seed_everything(p.seed + k)
            model = get_model(p, device)
            # slmodel, params, buffers = make_functional_with_buffers(model.fc)

        if not p.per_class:
            gradients, img_indices = get_mean_gradients(
                p, model, train_loader, criterion, optimizer
            )
            # similarities, img_indices = get_similarities(
            #     model, data, p.batch_size, mean_gradients
            # )
            similarities = get_sims(gradients).cpu().numpy()
            img_indices = img_indices.cpu().numpy()
        elif p.per_class:
            similarities, img_indices = [], []
            progressbar2 = tqdm(
                cls_data, desc="Per CLass Gradient Mathcing", position=1, leave=False
            )
            for dataset in progressbar2:
                loader = DataLoader(
                    dataset,
                    p.batch_size,
                    shuffle=True,
                    num_workers=2,
                    pin_memory=True,
                    # drop_last=True,
                )
                gradients, cls_all_inds = get_mean_gradients(
                    p, model, loader, criterion, optimizer
                )
                # cls_all_sims, cls_all_inds = get_similarities(
                #     model, dataset, p.batch_size, mean_gradients,
                # )
                cls_all_sims = get_sims(gradients).cpu().numpy().squeeze()
                cls_all_inds = cls_all_inds.cpu().numpy().squeeze()
                similarities.append(cls_all_sims)
                img_indices.append(cls_all_inds)
            try:
                similarities, img_indices = np.stack(similarities), np.stack(
                    img_indices
                )
            except ValueError:
                similarities, img_indices = np.array(
                    similarities, dtype=object
                ), np.array(img_indices, dtype=object)

        all_similarities.append(similarities)
        all_imginds.append(img_indices)

        if p.with_train:
            train_epoch(train_loader, model, criterion, optimizer, device)
            gc.collect()
            torch.cuda.empty_cache()

    if p.per_class:
        logger.info(progressbar2)
    logger.info(progressbar)
    all_similarities, all_imginds = np.stack(all_similarities), np.stack(all_imginds)
    return all_similarities, all_imginds

def main(p, logger):
    # p = create_config(args.config, args)

    # logger.info("Hyperparameters\n" + pformat(vars(p)))

    global device
    if torch.cuda.is_available:
        device = torch.device("cuda")
    else:
        device = torch.device("cpu")
        logger.warning("Using CPU to run the program.")

    seed_everything(p.seed)

    # dataset
    train_data, _ = get_dataset(p)
    train_data = DatasetwithIndices(train_data)
    logger.info(f"Dataset\n{str(train_data)}")

    logger.info("Hyperparameters\n" + pformat(vars(p)))
    # p.num_classes = len(train_data.classes)
    # logger.debug(f"Num Classes: {p.num_classes}")

    all_similarities, all_imginds = gradient_mathcing(p, train_data, logger)
    logger.info(
        f"All similarities shape: {all_similarities.shape}, All imgindices shape: {all_imginds.shape}"
    )
    np.save(
        p.output_dir
        / f"all_similarities{'_perclass' if p.per_class else ''}{'_withtrain' if p.with_train else ''}.npy",
        all_similarities,
    )
    np.save(
        p.output_dir
        / f"all_imginds{'_perclass' if p.per_class else ''}{'_withtrain' if p.with_train else ''}.npy",
        all_imginds,
    )

if __name__ == "__main__":
    # parser = argparse.ArgumentParser(
    #     description="Getting gradient similarity for each sample."
    # )
    # parser.add_argument("--config", help="Location of config file", required=True)
    # parser.add_argument("--seed", default=0, help="Seed")
    # parser.add_argument("--dataset", default="cifar100", help="Dataset to use")
    # parser.add_argument("--dataset_dir", default="./data", help="Dataset directory")
    # parser.add_argument("--topn", default=1000, type=int, help="Size of Coreset")
    # parser.add_argument(
    #     "--iter", default=100, type=int, help="Number of iterations for finding coreset"
    # )
    # parser.add_argument("-bs", "--batch_size", default=1000, help="BatchSize", type=int)
    # parser.add_argument(
    #     "--per_class",
    #     action="store_true",
    #     help="Specify whether to find Mean Gradients classwise",
    # )
    # parser.add_argument(
    #     "--with_train",
    #     action="store_true",
    #     help="No. of epochs to train before finding Gmean",
    # )
    # parser.add_argument(
    #     "--temp",
    #     action="store_true",
    #     help="Specify whether to use temp folder",
    # )
    # parser.add_argument(
    #     "--use_all_params",
    #     help="Specify if all model parameters' gradients to be used. Defaults: (FC layers only)",
    #     action="store_true",
    # )
    # parser.add_argument(
    #     "--resume", default=None, help="path to checkpoint from where to resume"
    # )

    parser = get_parser()

    args = parser.parse_args()
    if args.output_dir is not None:
        args.output_dir = pathlib.Path(args.dataset.lower()) / args.output_dir
    else:
        args.output_dir = pathlib.Path(args.dataset.lower())
    if args.pretrained:
        args.output_dir = args.output_dir / "pretrained"
    if args.temp:
        args.output_dir = args.output_dir / f"temp"
    args.logdir = args.output_dir / "logs"
    args.logdir.mkdir(parents=True, exist_ok=True)

    # temporary fix
    args.class_balanced = None
    args.augment = None
    logger = get_logger(args, "gradmatch")
    try:
        main(args, logger)
    except Exception:
        logger.exception("A Error Occurred")


'''
Loads up each users config and creates the service watchers
'''
import yaml

def load_config(filename):
    '''Loads config file,
    format is in yaml

    and looks like:

    services:
        - name: openvpn
          input: 10
          output: 2
        - name: samba
          input: 10
          output: 2
    '''
    with open(filename) as f:
        config = yaml.load(f)

    assert 'services' in config, "no services in config"

    services = []
    for service in config['services']:
        assert 'name' in service, 'Missing code for one service '
        assert 'input' in service, 'Missing input pin for {}'.format(service['name'])
        assert 'output' in service, 'Missing output pin for {}'.format(service['name'])

    return config

if __name__ == '__main__':
    print(load_config('./sample.yaml'))


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import os, sys, re, time, datetime, logging, random, string, logging.handlers, gzip, paramiko
import multiprocessing, subprocess, requests, urllib3, uuid
from threading import Timer
from configparser import ConfigParser
from Crypto.Cipher import AES

from iscpy.iscpy_dns.named_importer_lib import *
import base64, hashlib, zlib, json, lxml.etree, pexpect, dns, dns.resolver

from time import sleep
import threading, binascii, xml.dom.minidom, shutil

from daemon import Daemon
from waj import *

def get_transfer_ip_and_delay(soa):
	target = 'serial ' + str(soa)
	try:
		with open(run_file) as f:
			l = f.readlines()
			for i in range(len(l)):
				if l[i].find(target) > 0:
					for v in l[i:]:
						if v.find('Transfer completed') > 0:
							res = v
							return int(1000*float(res.split(', ')[-1].split(' ')[0])) , res.split('#')[0].split(' ')[-1]

	except Exception as e:
		logger.warning('get transfer ip and delay error:'+str(e))
	return 0,'0.0.0.0'

def get_server_from_file():
	try:
		server = ''
		with open(standard_source, 'r') as f:
			data = f.read()
			named_data = MakeNamedDict(data)
			servers = named_data['orphan_zones']['.']['options']['masters']
			for ip in servers:
				server += ip + ','
		with open(exigency_source, 'r') as f:
			data = f.read()
			named_data = MakeNamedDict(data)
			servers = named_data['orphan_zones']['.']['options']['masters']
			for ip in servers:
				server += ip + ','
		return server[:-1]

	except Exception as e:
		logger.warning('get server from root source file error:'+str(e))

	return ''

def get_transfer_ip_and_delay_from_file(soa):
	try:
		with open(root_source, 'r') as f:
			data = f.read()
			named_data = MakeNamedDict(data)
			servers = named_data['orphan_zones']['.']['options']['masters']
			dns_query = dns.message.make_query('.', 'SOA')
			for ip in servers:
				begin = datetime.datetime.now()
				res = dns.query.udp(dns_query, ip, port = 53,timeout = 2)
				end = datetime.datetime.now()
				for i in res.answer:
					for j in i.items:
						if j.serial == soa:
							return (end - begin).microseconds//1000,ip
	except Exception as e:
		logger.warning('get transfer ip and delay from swotch_root.zone error:'+str(e))
	return 0,'0.0.0.0'

def get_root_file_size():
	try:
		with open(root_source, 'r') as f:
			data = f.read()
			named_data = MakeNamedDict(data)
			return os.path.getsize(named_data['orphan_zones']['.']['file'])
	except Exception as e:
		logger.warning('get root_copy file size error:'+str(e))
	return 0

def upload_root_run_data(soa):
	result = 'get source or size error'
	delay,ip = get_transfer_ip_and_delay(soa)
	if delay == 0 and ip == '0.0.0.0':
		delay,ip = get_transfer_ip_and_delay_from_file(soa)
	size = get_root_file_size()
	if delay != 0 and ip != '0.0.0.0' and size != 0:
		result = 'success'

	server = get_server_from_file()

	timestamp = time.strftime('%Y-%m-%d %H:%M:%S')

	root_soa_str_data = dns_id + '|' + root_copy_room_id + '|' + server_id + '|' + server + '|' + ip\
	+ '|' + timestamp + '|' + result + '|' + str(size) + '|' + str(soa) + '|' + str(delay)

	file_name = 'zoneOperation_full_' + dns_id + '_0_' + time.strftime('%Y%m%d%H%M%S') + '.gz'

	logger.info(root_soa_str_data)
	try:
		cache_dir = '/var/drms_toggle_data/'
		with gzip.open(cache_dir + file_name, "wb") as f:
			f.write(bytes(root_soa_str_data, 'utf-8'))
		upload_to_ftp(cache_dir,file_name,'15')
	except Exception as e:
		logger.error('upload root resove data error:'+str(e))

def get_root_copy_soa():
	try:
		dns_query = dns.message.make_query('.', 'SOA')
		res = dns.query.udp(dns_query, '127.0.0.1', port = 53,timeout = 2)
		for i in res.answer:
			for j in i.items:
				return j.serial
	except Exception as e:
		logger.warning('Server exception get soa error:'+str(e))
	return 0

def check_soa_and_upload():
	global loop_count
	now_soa,root_soa = 0,0
	while True:
		if server_type == 'root_copy' and loop_count % 60 == 0:
			now_soa = get_root_copy_soa()
			if now_soa > 0:
				if root_soa != now_soa:
					root_soa = now_soa
					upload_root_run_data(now_soa)
				if loop_count  >= 900 and int(share_delay.value) == 900:
					upload_root_run_data(now_soa)
			else:
				logger.warning('dns server can not work please check')
		if loop_count >= 900:
			loop_count = 0
		sleep(1)
		loop_count += 1

class DrmsToggle(Daemon):
	def run(self):
		logger.info('main process start at: %s' % time.ctime())

		threading._start_new_thread(xgj_main_task,())
		threading._start_new_thread(waj_main_task,())
		check_soa_and_upload()

		logger.info('main process end at: %s' % time.ctime())

if __name__ == '__main__':
	drms_toggle = DrmsToggle('/var/drms_toggle_data/drms_toggle.pid')
	drms_toggle.start()


"""Define the Autorization Manager."""
from datetime import datetime, timedelta
from typing import Optional

import jwt
from fastapi import BackgroundTasks, Depends, HTTPException, Request, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer

from app.config.settings import get_settings
from app.database.db import get_database
from app.managers.email import EmailManager
from app.models.enums import RoleType
from app.models.user import User
from app.schemas.email import EmailTemplateSchema
from app.schemas.request.auth import TokenRefreshRequest

class ResponseMessages:
    """Error strings for different circumstances."""

    CANT_GENERATE_JWT = "Unable to generate the JWT"
    CANT_GENERATE_REFRESH = "Unable to generate the Refresh Token"
    CANT_GENERATE_VERIFY = "Unable to generate the Verification Token"
    INVALID_TOKEN = "That token is Invalid"  # nosec
    EXPIRED_TOKEN = "That token has Expired"  # nosec
    VERIFICATION_SUCCESS = "User succesfully Verified"
    USER_NOT_FOUND = "User not Found"
    ALREADY_VALIDATED = "You are already validated"
    VALIDATION_RESENT = "Validation email re-sent"

class AuthManager:
    """Handle the JWT Auth."""

    @staticmethod
    def encode_token(user):
        """Create and return a JTW token."""
        try:
            payload = {
                "sub": user["id"],
                "exp": datetime.utcnow()
                + timedelta(minutes=get_settings().access_token_expire_minutes),
            }
            return jwt.encode(
                payload, get_settings().secret_key, algorithm="HS256"
            )
        except Exception as exc:
            # log the exception
            raise HTTPException(
                status.HTTP_401_UNAUTHORIZED, ResponseMessages.CANT_GENERATE_JWT
            ) from exc

    @staticmethod
    def encode_refresh_token(user):
        """Create and return a JTW token."""
        try:
            payload = {
                "sub": user["id"],
                "exp": datetime.utcnow() + timedelta(minutes=60 * 24 * 30),
                "typ": "refresh",
            }
            return jwt.encode(
                payload, get_settings().secret_key, algorithm="HS256"
            )
        except Exception as exc:
            # log the exception
            raise HTTPException(
                status.HTTP_401_UNAUTHORIZED,
                ResponseMessages.CANT_GENERATE_REFRESH,
            ) from exc

    @staticmethod
    def encode_verify_token(user):
        """Create and return a JTW token."""
        try:
            payload = {
                "sub": user["id"],
                "exp": datetime.utcnow() + timedelta(minutes=10),
                "typ": "verify",
            }
            return jwt.encode(
                payload, get_settings().secret_key, algorithm="HS256"
            )
        except Exception as exc:
            # log the exception
            raise HTTPException(
                status.HTTP_401_UNAUTHORIZED,
                ResponseMessages.CANT_GENERATE_VERIFY,
            ) from exc

    @staticmethod
    async def refresh(refresh_token: TokenRefreshRequest, database):
        """Refresh an expired JWT token, given a valid Refresh token."""
        try:
            payload = jwt.decode(
                refresh_token.refresh,
                get_settings().secret_key,
                algorithms=["HS256"],
            )

            if payload["typ"] != "refresh":
                raise HTTPException(
                    status.HTTP_401_UNAUTHORIZED, ResponseMessages.INVALID_TOKEN
                )

            user_data = await database.fetch_one(
                User.select().where(User.c.id == payload["sub"])
            )

            if not user_data:
                raise HTTPException(
                    status.HTTP_404_NOT_FOUND, ResponseMessages.USER_NOT_FOUND
                )

            # block a banned user
            if user_data["banned"]:
                raise HTTPException(
                    status.HTTP_401_UNAUTHORIZED, ResponseMessages.INVALID_TOKEN
                )
            new_token = AuthManager.encode_token(user_data)
            return new_token

        except jwt.ExpiredSignatureError as exc:
            raise HTTPException(
                status.HTTP_401_UNAUTHORIZED, ResponseMessages.EXPIRED_TOKEN
            ) from exc
        except jwt.InvalidTokenError as exc:
            raise HTTPException(
                status.HTTP_401_UNAUTHORIZED, ResponseMessages.INVALID_TOKEN
            ) from exc

    @staticmethod
    async def verify(code: str, database):
        """Verify a new User's Email using the token they were sent."""
        try:
            payload = jwt.decode(
                code,
                get_settings().secret_key,
                algorithms=["HS256"],
            )
            user_data = await database.fetch_one(
                User.select().where(User.c.id == payload["sub"])
            )

            if not user_data:
                raise HTTPException(
                    status.HTTP_404_NOT_FOUND, ResponseMessages.USER_NOT_FOUND
                )

            if payload["typ"] != "verify":
                raise HTTPException(
                    status.HTTP_401_UNAUTHORIZED, ResponseMessages.INVALID_TOKEN
                )

            # block a banned user
            if user_data["banned"]:
                raise HTTPException(
                    status.HTTP_401_UNAUTHORIZED, ResponseMessages.INVALID_TOKEN
                )

            if user_data["verified"]:
                raise HTTPException(
                    status.HTTP_401_UNAUTHORIZED, ResponseMessages.INVALID_TOKEN
                )

            await database.execute(
                User.update()
                .where(User.c.id == payload["sub"])
                .values(
                    verified=True,
                )
            )
            raise HTTPException(
                status.HTTP_200_OK, ResponseMessages.VERIFICATION_SUCCESS
            )

        except jwt.ExpiredSignatureError as exc:
            raise HTTPException(
                status.HTTP_401_UNAUTHORIZED, ResponseMessages.EXPIRED_TOKEN
            ) from exc
        except jwt.InvalidTokenError as exc:
            raise HTTPException(
                status.HTTP_401_UNAUTHORIZED, ResponseMessages.INVALID_TOKEN
            ) from exc

    @staticmethod
    async def resend_verify_code(
        user: int, background_tasks: BackgroundTasks, database
    ):  # pragma: no cover (code not used at this time)
        """Resend the user a verification email."""
        user_data = await database.fetch_one(
            User.select().where(User.c.id == user)
        )

        if not user_data:
            raise HTTPException(
                status.HTTP_404_NOT_FOUND, ResponseMessages.USER_NOT_FOUND
            )

        # block a banned user
        if user_data["banned"]:
            raise HTTPException(
                status.HTTP_401_UNAUTHORIZED, ResponseMessages.INVALID_TOKEN
            )

        if user_data["verified"]:
            raise HTTPException(
                status.HTTP_400_BAD_REQUEST,
                ResponseMessages.ALREADY_VALIDATED,
            )

        email = EmailManager()
        email.template_send(
            background_tasks,
            EmailTemplateSchema(
                recipients=[user_data["email"]],
                subject=f"Welcome to {get_settings().api_title}!",
                body={
                    "application": f"{get_settings().api_title}",
                    "user": user_data["email"],
                    "base_url": get_settings().base_url,
                    "verification": AuthManager.encode_verify_token(user_data),
                },
                template_name="welcome.html",
            ),
        )
        # await email.simple_send(
        #     EmailSchema(
        #         recipients=[user_data["email"]],
        #         subject=f"Welcome to {get_settings().api_title}!",
        #         body="Test Email",
        #     ),
        # )

        raise HTTPException(
            status.HTTP_200_OK, ResponseMessages.VALIDATION_RESENT
        )

class CustomHTTPBearer(HTTPBearer):
    """Our own custom HTTPBearer class."""

    async def __call__(
        self, request: Request, db=Depends(get_database)
    ) -> Optional[HTTPAuthorizationCredentials]:
        """Override the default __call__ function."""
        res = await super().__call__(request)

        try:
            payload = jwt.decode(
                res.credentials, get_settings().secret_key, algorithms=["HS256"]
            )
            user_data = await db.fetch_one(
                User.select().where(User.c.id == payload["sub"])
            )
            # block a banned or unverified user
            if user_data["banned"] or not user_data["verified"]:
                raise HTTPException(
                    status.HTTP_401_UNAUTHORIZED, ResponseMessages.INVALID_TOKEN
                )
            request.state.user = user_data
            return user_data
        except jwt.ExpiredSignatureError as exc:
            raise HTTPException(
                status.HTTP_401_UNAUTHORIZED, ResponseMessages.EXPIRED_TOKEN
            ) from exc
        except jwt.InvalidTokenError as exc:
            raise HTTPException(
                status.HTTP_401_UNAUTHORIZED, ResponseMessages.INVALID_TOKEN
            ) from exc

oauth2_schema = CustomHTTPBearer()

def is_admin(request: Request):
    """Block if user is not an Admin."""
    if request.state.user["role"] != RoleType.admin:
        raise HTTPException(status.HTTP_403_FORBIDDEN, "Forbidden")

def can_edit_user(request: Request):
    """Check if the user can edit this resource.

    True if they own the resource or are Admin
    """
    if request.state.user["role"] != RoleType.admin and request.state.user[
        "id"
    ] != int(request.path_params["user_id"]):
        raise HTTPException(status.HTTP_403_FORBIDDEN, "Forbidden")

def is_banned(request: Request):
    """Dont let banned users access the route."""
    if request.state.user["banned"]:
        raise HTTPException(status.HTTP_403_FORBIDDEN, "Banned!")


from pydantic import BaseModel, Field

from faststream import FastStream, Logger
from faststream.kafka import KafkaBroker

class Employee(BaseModel):
    name: str = Field(..., examples=["Mickey"], description="name example")
    surname: str = Field(..., examples=["Mouse"], description="surname example")
    email: str = Field(
        ..., examples=["mikey.mouse@mail.ai"], description="email example"
    )

broker = KafkaBroker("localhost:9092")
app = FastStream(broker)

to_notify_accounting = broker.publisher("notify_accounting")
to_notify_all_employees = broker.publisher("notify_all_employees")

@broker.subscriber("new_employee")
async def on_new_employee(msg: Employee, logger: Logger) -> None:
    logger.info(msg)

    await to_notify_accounting.publish(
        f"Please prepare all the paper work for: {msg.name} {msg.surname}"
    )
    await to_notify_all_employees.publish(
        f"Please welcome our new colleague: {msg.name} {msg.surname}"
    )


from pypaq.lipytools.files import r_json
from pypaq.lipytools.plots import two_dim_multi
from pypaq.lipytools.moving_average import MovAvg
from typing import List, Dict, Optional

from envy import RESULTS_FP
from run.functions import get_saved_dmks_names

def get_ranks(
        all_results: Optional[Dict]=    None,
        mavg_factor=                    0.3,
) -> Dict:

    if all_results is None:
        all_results = r_json(RESULTS_FP)

    all_names = []
    for lix in all_results['loops']:
        all_names += all_results['loops'][lix]
    all_names = set(all_names)

    ranks = {dn: [] for dn in all_names}
    for lix in all_results['loops']:
        names_present = []
        for r,dn in enumerate(all_results['loops'][lix]):
            names_present.append(dn)
            ranks[dn].append(r)
        for dn in ranks:
            if dn not in names_present:
                ranks[dn].append(None)

    ranks_smooth = {}
    for dn in ranks:
        ranks_smooth[dn] = []
        ma = MovAvg(mavg_factor)
        for v in ranks[dn]:
            if v is None:
                ma = MovAvg(mavg_factor)
                ranks_smooth[dn].append(None)
            else:
                ranks_smooth[dn].append(ma.upd(v))

    """
    len_ranks = max([len(v) for v in ranks.values()])
    families = set([all_results[dn]['family'] for dn in dmk_TR])
    ranks_fam = {f: [[] for _ in range(len_ranks)] for f in families}
    for dn in ranks:
        dnf = all_results[dn]['family']
        for ix,r in enumerate(ranks[dn]):
            ranks_fam[dnf][ix].append(r)
    print(families)
    print(ranks_fam)
    for f in ranks_fam:
        for ix in range(len(ranks_fam[f])):
            ranks_fam[f][ix] = sum(ranks_fam[f][ix]) / len(ranks_fam[f][ix])
    """

    return {
        'ranks':        ranks,
        'ranks_smooth': ranks_smooth,
        #'ranks_fam':    ranks_fam
    }

if __name__ == "__main__":

    rd = get_ranks()
    print(rd)

    """
    two_dim_multi(
        ys=     list(rd['ranks'].values()),
        names=  list(rd['ranks'].keys()))

    two_dim_multi(
        ys=     list(rd['ranks_smooth'].values()),
        names=  list(rd['ranks_smooth'].keys()))

    two_dim_multi(
        ys=     list(rd['ranks_fam'].values()),
        names=  list(rd['ranks_fam'].keys()))
    """


'''
Set Dataset

This file was developed as a project for DACO subject from Bioengeneering Masters at FEUP

It separates the images from a folder into a respective one according to its class
It helps to better analyse and organize the project
'''

import os
import pandas as pd
from torch.utils.data import DataLoader
import torch
from torch.utils.data import Dataset
from torchvision import transforms
import numpy as np
import shutil

TRAIN = "train_features"
TEST = "test_features"
BATCH_SIZE = 32
BASE_CSV_PATH = "features"
MODEL_PATH = "features/model.cpickle"
LE_PATH = "features/le.cpickle"
CLASSES = ['antelope_duiker',
 'bird',
 'blank',
 'civet_genet',
 'hog',
 'leopard',
 'monkey_prosimian',
 'rodent']

train_features = pd.read_csv("train_features.csv", index_col="id")
test_features = pd.read_csv("test_features.csv", index_col="id")
train_labels = pd.read_csv("train_labels.csv", index_col="id")

frac = 1

y = train_labels.sample(frac=frac, random_state=1)
x = train_features.loc[y.index].filepath.to_frame()

# This class was given in the benchmark.ipynb provided by the competition. It pre-processes the images and gets data from it
# This will be applyed to organize the images into folders
class ImagesDataset(Dataset):
    """Reads in an image, transforms pixel values, and serves
    a dictionary containing the image id, image tensors, and label.
    """

    def __init__(self, x_df, y_df=None):
        self.data = x_df
        self.label = y_df
        self.transform = transforms.Compose(
            [
                transforms.Resize((224, 224)),
                transforms.ToTensor(),
                transforms.Normalize(
                    mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)
                ),
            ]
        )

    def __getitem__(self, index):
        image = self.data.iloc[index]["filepath"]
        image_id = self.data.index[index]
        if self.label is None:
            sample = {"image_id": image_id, "image": image}
        else:
            label = torch.tensor(self.label.iloc[index].values,
                                 dtype=torch.float)
            sample = {"image_id": image_id, "image": image, "label": label}
        return sample

    def __len__(self):
        return len(self.data)

train_dataset = ImagesDataset(x, y)
train_dataloader = DataLoader(train_dataset, batch_size=32)

# Using the information obtained with the provided class (above) we save the images into different class folders
for i in range(len(train_dataset)):
    filename = train_dataset[i]['image_id']
    label = CLASSES[np.where(train_dataset[i]['label'].numpy() == 1.)[0][0]]

    dirPath = "images/train/" + label
    if not os.path.exists(dirPath):
        os.makedirs(dirPath)

    p = dirPath + '/' + filename +'.jpg'
    shutil.copy2(train_dataset[i]['image'], p)