from classes.config import Config
from .notification import Notification
from jinja2 import Template
from pytz import timezone
import socket
import logging
import boto3
import hashlib
from lib import HTML

class AWSInstanceStatusEvent(object):

    def __init__(self, instance_id, event):
        self._config = Config().get_config()
        self._instance_id = instance_id
        self._event = event
        self.set_digest()

    def is_complete(self):
        if "Completed" in self._event['Description']: self._completed = True
        else: self._completed = False
        return self._completed

    def set_event_saserver_details(self, servers):
        logging.info('START: set_event_saserver_details')
        server_details = servers.get_saserver_details()
        if ( self._instance_id in server_details.keys() ):
            self._server = server_details[self._instance_id]
            logging.info('AWSInstanceStatusEvent: set_event_saserver_details: Found server: ' + str(self._server))
        else:
            self._server = None
        logging.info('AWSInstanceStatusEvent: set_event_saserver_details: Has server: ' + str(self.has_server()))

    def has_server(self):
        return True if ( self._server ) else False

    def set_digest(self):
        self._digest = hashlib.sha224(
            self._instance_id.encode('utf-8')
            + self._event['Code'].encode('utf-8')
        ).hexdigest()
        logging.info('AWSInstanceStatusEvent: digest: ' + str(self._digest))

    def get_digest(self):
        return self._digest

    def set_is_new(self, digests):
        last_run_digests = digests.get_last_run_digests()
        if ( self._digest in last_run_digests ):
            self._new_event = False
        else:
            self._new_event = True
        logging.info('AWSInstanceStatusEvent: Is new: ' + str(self._new_event))

    def is_new(self):
        return self._new_event

    def get_owners(self):
        if ( self._server is None ):
            return []
        else:
            return self._server['owners']

    def get_owners_str(self):
        if ( self._server is None ):
            return 'N/A'
        else:
            return ', '.join(self.get_owners())

    def get_owners_email_addrs(self):
        if ( self._server is None ):
            return ['paul.leddy@jpl.nasa.gov']
        else:
            return map(lambda x: x + '@jpl.nasa.gov', self._server['owners'])

    def get_owners_email_addrs_str(self):
        return ', '.join(self.get_owners_email_addrs())

    def get_cog_admin(self):
        if ( self._server is None ):
            return 'N/A'
        else:
            return self._server['cog_admin']

    def get_html_message_to_owners(self):
        return self.get_html_template()

    def get_text_message_to_owners(self):
        return 'You got mail: ' + self._instance_id + '. STOP.'

    def get_not_before_date_str(self):
        return self._event['NotBefore'].astimezone(timezone('US/Pacific')).strftime("%Y-%m-%d %H:%M %Z") if ( 'NotBefore' in self._event ) else 'N/A'

    def get_not_after_date_str(self):
        return self._event['NotAfter'].astimezone(timezone('US/Pacific')).strftime("%Y-%m-%d %H:%M %Z") if ( 'NotAfter' in self._event ) else 'N/A'

    def set_name(self, name):
        self._name = name

    def get_vname(self):
        if ( self._server is None ):
            return self._instance_id
        if ( self._server['hostname'] is not None and self._server['domain'] is not None ):
            return self._server['hostname'] + '.' + self._server['domain']
        elif ( ( self._server['hostname'] is None or self._server['domain'] is None) and self._server['device_name'] is not None ):
            return self._server['device_name']
        elif ( self._server['aws_acct_type'] is 'GOV' and self._server['ip_address'] is not None):
             return self._server['ip_address']
        elif ( self._server['device_name'] is None and self._server['aws_acct_type'] is 'EC2' and self._server['ip_address'] is not None ):
            return self._server['ip_address']
        elif ( self._name is not None ):
            return self._name
        else:
            return self._instance_id

    def send_notification_to_owners(self):
        subject='Instance Status Event Notification: ' + self.get_vname()
        email_to = self._config['general']['sa_recipients'] if ( self._config['general']['POC'] ) else self.get_owners_email_addrs()
        email_attrs = {
            'email_type': self._config['general']['email_type'],
            'email_to': email_to,
            'email_from': self._config['general']['email_from'],
            'subject': subject,
            'html': self.get_html_message_to_owners(),
            'text': self.get_text_message_to_owners()
        }
        Notification('email', email_attrs)

    def get_html_template(self):
        template_raw = '''

DEV PHASE

To: {{ owners_email_addrs_str }}

Hello,

Your Amazon VM, {{ vname }}, is scheduled for maintenance by Amazon.  An outage will be incurred during the maintenance window below.  Please reply to this email if you would like to resolve the maintenance at a time before the scheduled maintenance window.  Resolving the maintenance requirement ahead of time will involve powering down your VM, and powering it back on, or merely a reboot, depending on the maintenance type.

Maintenance Details:

VM: {{ vname }}

Instance: {{ instance_id }}

Description: {{ maintenance_desc }}

Maintenance window start time: {{ not_before }}

Maintenance window end time: {{ not_after }}

Owners: {{ owners_str }}

Thank you,

            Infrastructure SA Team

            jplis-sa@jpl.nasa.gov

            '''

        template = Template(p_template_raw)
        html = template.render(
            owners_email_addrs_str = self.get_owners_email_addrs_str(),
            vname = self.get_vname(),
            instance_id = self._instance_id,
            not_before = self.get_not_before_date_str(),
            not_after = self.get_not_after_date_str(),
            maintenance_desc = self._event['Description'],
            owners_str = self.get_owners_str(),
        )

        return html

    def get_sa_report_row(self):
        return [self._name, self.get_cog_admin(), self._instance_id, self._event['Description'], self.get_not_before_date_str(), self.get_not_after_date_str(), self.get_owners_str()]

# END


import click
from pathlib import Path

@click.command()
@click.option('--bed', '-b', type=Path, required=True,
              help="input bed file")
@click.option('--toml', '-t', type=Path, required=True,
              help="output toml file")
def main(bed, toml):
    with open(toml, 'w') as out:
        out.write('[[jobs]]\n')
        out.write('name = "change_here"\n')
        out.write('time = [0, 6000]\n')
        out.write('ch = [1, 512]\n')
        out.write('bc = "all"\n')
        out.write('target = [\n')

        with open(bed, 'r') as f:
            for line in f:
                content = line.rstrip().split('\t')
                chrom, start, end = content[0], int(content[1]), int(content[2])
                out.write(f'\t{{region = ["{chrom}", "{start}", "{end}"], action = "stop_receiving"}},\n')

        out.write(f'\t{{region = "multi", action = "unblock"}},\n')
        out.write(f'\t{{region = "unmapped", action = "unblock"}},\n')
        out.write(f'\t{{region = "miss", action = "unblock"}},\n')
        out.write("]\n")

if __name__ == "__main__":
    main()


"""Train OT-Flow on two-dimensional toy data using the JKO scheme.

OT-Flow is trained for a predetermined number of steps. After each step,
the target is updated to the pushforward of the learned density.
"""
import argparse
import copy
import logging
import math
import os
import pathlib
from pprint import pprint
import sys

from adabelief_pytorch import AdaBelief
import matplotlib.pyplot as plt
import numpy as np
import proplot as pplt
import torch

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
import cnflows as cnf

import plotting
from utils import get_torch_device_mps
from utils import Monitor
from utils import ScriptManager

pplt.rc["grid"] = False
pplt.rc["cycle"] = "538"
pplt.rc["cmap.discrete"] = False
pplt.rc["cmap.sequential"] = "viridis"

# Parse command line arguments
# --------------------------------------------------------------------------------------
parser = argparse.ArgumentParser("ot-flow")

parser.add_argument("--save", type=int, default=1)
parser.add_argument("--outdir", type=str, default="./data_output/")
parser.add_argument("--data", type=str, default="2spirals")
parser.add_argument("--precision", type=str, default="single", choices=["single", "double"])
parser.add_argument("--gpu", type=int, default=0)

parser.add_argument("--m", type=int, default=16)
parser.add_argument("--n_layers", type=int, default=2)
parser.add_argument("--alpha_L", type=float, default=1.0)
parser.add_argument("--alpha_C", type=float, default=10.0)
parser.add_argument("--alpha_R", type=float, default=1.0)

parser.add_argument("--data_size", type=int, default=int(1.00e05))
parser.add_argument("--batch_size", type=int, default=2000)
parser.add_argument("--nt", type=int, default=8)
parser.add_argument("--seed", type=int, default=0)

parser.add_argument("--val_nt", type=int, default=8)

parser.add_argument("--n_steps", type=int, default=5)
parser.add_argument("--n_epochs", type=int, default=40)

parser.add_argument("--optim", type=str, default="adam", choices=["adam", "adabelief"])
parser.add_argument("--lr", type=float, default=1.00e-02)
parser.add_argument("--min_lr", type=float, default=1.00e-03)
parser.add_argument("--lr_drop", type=float, default=0.5)
parser.add_argument("--lr_patience", type=int, default=5)

parser.add_argument("--vis_freq", type=int, default=2)
parser.add_argument("--vis_n_points", type=int, default=int(10.00e+04))
parser.add_argument("--vis_n_bins", type=int, default=100)
parser.add_argument("--vis_cmap", type=str, default="viridis")

args = parser.parse_args()

# Settings
# --------------------------------------------------------------------------------------
device = get_torch_device_mps(gpu=args.gpu)

precision = torch.float32
if args.precision == "double":
    precision = torch.float64
torch.set_default_dtype(precision)

cvt = lambda x: x.type(precision).to(device, non_blocking=True)

man = ScriptManager(outdir=args.outdir, path=pathlib.Path(__file__))
man.save_script_copy()
pprint(man.get_info())

logger = man.get_logger(save=args.save, print=True, filename="log.txt")
for key, val in man.get_info().items():
    logger.info(f"{key}: {val}")
logger.info(args)

# Load training data.
# --------------------------------------------------------------------------------------

def get_data(size):
    data = cnf.utils.gen_toy_data(name=args.data, size=size)
    data = cvt(torch.from_numpy(data))
    return data

data = get_data(args.data_size)
dataset = torch.utils.data.TensorDataset(data)
data_loader = torch.utils.data.DataLoader(
    dataset=dataset, batch_size=args.batch_size, shuffle=False
)

val_data = get_data(args.data_size)
val_dataset = torch.utils.data.TensorDataset(val_data)
val_data_loader = torch.utils.data.DataLoader(
    dataset=val_dataset, batch_size=val_data.shape[0], shuffle=False
)

# Model
# --------------------------------------------------------------------------------------
d = 2
m = args.m
n_layers = args.n_layers
alpha = [args.alpha_L, args.alpha_C, args.alpha_R]

base_dist = cnf.distributions.DiagGaussian(d, trainable=False)
nfm = cnf.OTFlow(d=d, m=m, n_layers=n_layers, alpha=alpha, base_dist=base_dist)
nfm = nfm.to(precision).to(device)

logger.info("")
logger.info("Model")
logger.info("--------------------------------------------------------------------------")
logger.info(f"Number of trainable parameters = {nfm.get_n_params()}")
logger.info("")
for key, value in nfm.state_dict().items():
    logger.info(key)
    logger.info(value)
    logger.info("")

# Optimizer
# --------------------------------------------------------------------------------------

optimizer = None
if args.optim == "adam":
    optimizer = torch.optim.Adam(
        nfm.parameters(),
        lr=args.lr,
        weight_decay=0.0,
        betas=(0.9, 0.999),
        eps=1.00e-08,
        amsgrad=True,
        foreach=None,
        maximize=False,
        capturable=False,
        differentiable=False,
        fused=None,
    )
elif args.optim == "adabelief":
    from adabelief_pytorch import AdaBelief

    optimizer = AdaBelief(
        nfm.parameters(),
        lr=args.lr,
        eps=1.00e-16,
        betas=(0.9, 0.999),
        weight_decouple=True,
        rectify=False,
    )

lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
    optimizer,
    mode="min",
    factor=args.lr_drop,
    patience=args.lr_patience,
    threshold=0.001,
    threshold_mode="rel",
    min_lr=args.min_lr,
    verbose=False,
)

logger.info("")
logger.info("Optimizer")
logger.info("--------------------------------------------------------------------------")
logger.info(optimizer)

# Diagnostics
# --------------------------------------------------------------------------------------

def save_figure(filename, step=0, iteration=None, ext="png", verbose=True, **kwargs):
    filename = f"{filename}_{step:02.0f}"
    if iteration is not None:
        filename = f"{filename}_{iteration:05.0f}"
    filename = man.get_filename(filename)
    filename = f"{filename}.{ext}"
    kwargs.setdefault("dpi", 250)
    if verbose:
        print(f"Saving file {filename}")
    plt.savefig(filename, **kwargs)
    plt.close()

# Training loop
# --------------------------------------------------------------------------------------

def pushforward_data(data_loader=None, nfm=None, nt=None):
    x_new = torch.zeros(1)
    with torch.no_grad():
        for (x,) in data_loader:
            x_temp, _, _, _ = nfm.forward(x, nt=nt)
            if x_new.shape[0] == 1:
                x_new = x_temp
            else:
                x_new = torch.cat((x_new, x_temp), dim=0)
    dataset = torch.utils.data.TensorDataset(x_new)
    data_loader = torch.utils.data.DataLoader(
        dataset=dataset,
        batch_size=data_loader.batch_size,
        shuffle=False,
    )
    return data_loader

state_dicts = []

for step in range(args.n_steps):
    print(f"step={step}")

    best_avg_loss = best_avg_vloss = float("inf")
    best_state_dict = nfm.state_dict()

    for param_group in optimizer.param_groups:
        param_group["lr"] = args.lr
    lr_scheduler.num_bad_epochs = 0

    monitor = Monitor(filename=man.get_filename(f"history_{step:02.0f}.dat"))
    nfm.to(device)
    nfm.train()
    iteration = 0

    for epoch in range(args.n_epochs):
        running_loss = 0.0
        for batch_index, (x,) in enumerate(data_loader):
            x = cvt(x)
            optimizer.zero_grad()
            loss, costs = nfm.forward_kld(x, nt=args.nt, return_costs=True)
            loss.backward()
            optimizer.step()
            monitor.action(
                loss=loss.item(),
                loss_L=costs[0].item(),
                loss_C=costs[1].item(),
                loss_R=costs[2].item(),
                lr=optimizer.param_groups[0]["lr"],
            )
            running_loss += loss
            iteration += 1
        avg_loss = running_loss / (batch_index + 1)

        nfm.eval()
        with torch.no_grad():
            running_vloss = 0.0
            for batch_index, (x,) in enumerate(val_data_loader):
                x = cvt(x)
                vloss = nfm.forward_kld(x, nt=args.val_nt)
                running_vloss += vloss
        avg_vloss = running_vloss / (batch_index + 1)

        if avg_loss < best_avg_loss:
            best_avg_loss = avg_loss
        if avg_vloss < best_avg_vloss:
            best_avg_vloss = avg_vloss
            best_state_dict = nfm.state_dict()

        lr_scheduler.step(avg_loss)

        print(
            "Epoch {}: avg_loss={:.02e} avg_vloss={:.02e}".format(
                epoch,
                avg_loss,
                avg_vloss,
            )
        )

        if (epoch % args.vis_freq == 0) or (epoch == args.n_epochs - 1):
            with torch.no_grad():
                nfm.eval()
                current_state_dict = nfm.state_dict()
                nfm.load_state_dict(best_state_dict)

                # Plot samples.
                x = val_data[: args.vis_n_points, :]
                plotting.plot_samples(
                    x,
                    nfm,
                    nt=args.val_nt,
                    xmax=4.5,
                    bins=args.vis_n_bins,
                    cmap=args.vis_cmap,
                )
                save_figure("fig_samples", step=step, iteration=iteration)

                # Plot loss history.
                history = monitor.get_data()
                fig, ax = pplt.subplots()
                ax.plot(history["iteration"], history["loss"], color="black")
                ax.format(xlabel="Iteration", ylabel="Loss")
                save_figure("fig__loss", step=step)
                for log in [True, False]:
                    fig, ax = pplt.subplots()
                    for key in ["loss_C", "loss_L", "loss_R"]:
                        ax.plot(history["iteration"], history[key], label=key)
                    if log:
                        ax.format(yscale="log")
                    ax.legend(loc="r", ncols=1, framealpha=0.0)
                    ax.format(xlabel="Iteration", ylabel="Loss")
                    name = "fig__loss_terms_log" if log else "fig__loss_terms"
                    save_figure(name, step=step)

                # Plot learning rate history.
                fig, ax = pplt.subplots()
                ax.plot(history["iteration"], history["lr"], color="black")
                ax.format(xlabel="Iteration", ylabel="Learning rate")
                save_figure("fig__lr", step=step)

                nfm.load_state_dict(current_state_dict)
                nfm.train()

    # Flow forward.
    print("Updating target")
    nfm.load_state_dict(best_state_dict)
    data_loader = pushforward_data(data_loader, nfm, args.nt)
    val_data_loader = pushforward_data(val_data_loader, nfm, args.nt)
    val_data = next(iter(val_data_loader))[0]

    # Save the model.
    print("Saving model")
    state_dict = copy.deepcopy(nfm.state_dict())
    state_dicts.append(state_dict)
    state = {
        "state_dict": state_dict,
        "d": d,
        "m": m,
        "alpha": alpha,
        "n_layers": n_layers,
        "base_dist": nfm.base_dist,
    }
    filename = man.get_filename(f"checkpoint_{step:02.0f}.pth")
    torch.save(state, filename)

    # Evaluate the chained models.
    print("Plotting")
    with torch.no_grad():
        current_state_dict = nfm.state_dict()
        nfm.eval()

        x = get_data(args.vis_n_points)
        plotting.plot_samples_chained(
            x,
            nfm=nfm,
            state_dicts=state_dicts,
            nt=args.val_nt,
            bins=args.vis_n_bins,
            cmap=args.vis_cmap,
        )
        save_figure("fig_samples_chained", step=step)

        nfm.load_state_dict(current_state_dict)
        nfm.train()


from selenium import webdriver
from time import sleep

# Iniciando o Navegador.
driver = webdriver.Chrome('C:/Users/Jhony/Downloads/Animes/chromedriver.exe')
pagina = 'https://www.meuanime.com/baixar?file=1857'
count = 0
status = "notok"
open('anime.txt', 'a')
while count != 3:
    # Procurando o link para download.
    driver.get(pagina)

    episodio = driver.find_element_by_xpath('/html/body/div[5]/div/div[1]/p')
    episodio = episodio.text
    episodio = episodio.replace('Voçê esta baixando - ','')
    print(episodio)

    download = driver.find_element_by_xpath('//*[@id="conteudo404"]/div/a')
    download = download.get_attribute('href')
    print(download)

    proximo = driver.find_element_by_xpath('//*[@id="postados404"]/div[16]/a[1]')
    proximo = proximo.get_attribute('href')
    print(proximo)

    with open('anime.txt', 'a') as texte :
        print(f'{episodio} adicionado com sucesso')
        print(episodio, '\n',
              download, '\n', file=texte)

    pagina = proximo
    count += 1
    print()
    sleep(1)

driver.close()


#https://www.blog.pythonlibrary.org/2010/03/08/a-simple-step-by-step-reportlab-tutorial/

import time
from reportlab.lib.enums import TA_JUSTIFY
from reportlab.lib.pagesizes import A4
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, Image
from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
from reportlab.lib.units import inch
from reportlab.lib.colors import HexColor
import os as os

from tools import logger

#import os as os

def generatePdfForPerson(person):
	#path = os.path.dirname(os.path.realpath(__file__))
	#path = "/".join(path.split("/")[:-1]) # remove characters after last "/"" in filepath
	#doc = SimpleDocTemplate(path + '/form.pdf', rightMarge=72, leftMargin=72, topMargin=10, bottomMargin=18)
	doc = SimpleDocTemplate('Workout_App_Data\\form.pdf', rightMarge=72, leftMargin=72, topMargin=10, bottomMargin=18)
	Story=[]
	#logo = path + '/pdf/MSWLogo.jpg'

	logger.log("opening: " + str(os.path.dirname(os.path.realpath(__file__)) + '\\' + 'MSWLogo.jpg'))
	logo = os.path.dirname(os.path.realpath(__file__)) + '\\' + 'MSWLogo.jpg'

	im = Image(logo, 2*inch, 2*inch)
	im.hAlign = 'LEFT'
	Story.append(im)

	styles = getSampleStyleSheet()
	styles.add(ParagraphStyle(name='Justify', alignment=TA_JUSTIFY, borderWidth=1, borderColor=HexColor('#800020'), borderPadding=3))
	ptext = ""
	ptext = ptext + 'Name: %s
' % person.name
	ptext = ptext + 'Test Date: %s
' % person.testDate
	ptext = ptext + '
'
	ptext = ptext + 'Heart Rate: %s
' % person.restHR
	ptext = ptext + 'Blood Pressure: %s
' % person.restBP
	ptext = ptext + 'Height (cm): %s
' % person.height
	ptext = ptext + 'Weight (kg): %s
' % person.weight
	Story.append(Paragraph(ptext, styles["Justify"]))
	Story.append(Spacer(1, 12))

	if "BMIframe" in person.framesChecked:
		ptext = 'BMI: %s
' % str(person.getBMI())
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "Circumference" in person.framesChecked:
		ptext = 'Circumference:
'
		ptext = ptext + 'Hip Circ (cm):%s
' % str(person.hipCirc)
		ptext = ptext + 'Waist Circ (cm):%s
' % str(person.waistCirc)
		ptext = ptext + 'Arm Circ (cm):%s
' % str(person.armCirc)
		ptext = ptext + 'Thigh Circ (cm):%s
' % str(person.thighCirc)
		Story.append(Paragraph(ptext, styles["Justify"]))

		Story.append(Spacer(1, 12))

	if "ThreeSiteSkinfold" in person.framesChecked:
		ptext = 'Three Site Skinfold:
'

		if (person.threeSiteFemaleSupra.strip() != "" and person.threeSiteFemaleTricep.strip() != "" and person.threeSiteFemaleThigh.strip() != ""):
			ptext = ptext + 'Body Density: %s
' % str(person.getThreeSiteMale())

			ptext = ptext + 'Body Fat %%: %s
' % str(person.getBodyFatThreeMale())

		if (person.threeSiteMaleChest.strip() != "" and person.threeSiteMaleAb.strip() != "" and person.threeSiteMaleThigh.strip() != ""):
			ptext = ptext + 'Body Density: %s
' % str(person.getThreeSiteFemale())

			ptext = ptext + 'Body Fat %%: %s
' % str(person.getBodyFatThreeFemale())

		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "SvnSiteSkinfold" in person.framesChecked:
		ptext = 'Seven Site Skinfold:
'

		ptext = ptext + 'Body Density: %s
' % str(person.getSevenSiteDensity())

		ptext = ptext + 'Body Fat %%: %s
' % str(person.getBodyFatSeven())
		Story.append(Paragraph(ptext, styles["Justify"]))

		Story.append(Spacer(1, 12))

	if "Coopers" in person.framesChecked:
		ptext = 'Coopers Run Test:
'

		ptext = ptext + 'VO2max (ml/kg/min):%s
' % str(person.getCooperAerobic())
		Story.append(Paragraph(ptext, styles["Justify"]))

		Story.append(Spacer(1, 12))

	if "ModAst" in person.framesChecked:
		ptext = 'Modified Astrand Cycle Test:
'

		ptext = ptext + 'VO2max(ml/kg/min): %s
' % str(person.modAstAerobic)
		Story.append(Paragraph(ptext, styles["Justify"]))

		Story.append(Spacer(1, 12))

	if "Ebelling" in person.framesChecked:
		ptext = 'Ebelling Treadmill Test:
'

		ptext = ptext + 'VO2max (ml/kg/min): %s
' % str(person.getEbellingAerobic())
		Story.append(Paragraph(ptext, styles["Justify"]))

		Story.append(Spacer(1, 12))

	if "Rockport" in person.framesChecked:
		ptext = 'Rockport 1 Mile Walk Test:
'

		ptext = ptext + 'VO2max(ml/kg/min): %s
' % str(person.getRockportAerobic())
		Story.append(Paragraph(ptext, styles["Justify"]))

		Story.append(Spacer(1, 12))

	if "MBtoss" in person.framesChecked:
		ptext = 'Med. Ball Toss Distance(cm): %s' % str(person.seatMBDist)
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "VertJump" in person.framesChecked:
		ptext = 'Vertical Jump Test:
'

		ptext = ptext + 'Stand and Reach(cm): %s
' % str(person.vertJumpSR)

		ptext = ptext + 'Best Jump(cm): %s
' % str(person.getVertJumpBest())

		ptext = ptext + 'Peak Power(W): %s
' % str(person.getVertJumpPower())
		Story.append(Paragraph(ptext, styles["Justify"]))

		Story.append(Spacer(1, 12))

	if "RMtest" in person.framesChecked:
		ptext = '1 Rep Max (Tested):
'

		if (person.RMTestExA.strip() != "" and person.RMTestExAWeight.strip() != ""):
			ptext = ptext + 'Exercise: %s
' % str(person.RMTestExA)

			ptext = ptext + '1 Rep Max Weight: %s
' % str(person.RMTestExAWeight)

		if (person.RMTestExB.strip() != "" and person.RMTestExBWeight.strip() != ""):
			ptext = ptext + 'Exercise: %s
' % str(person.RMTestExB)

			ptext = ptext + '1 Rep Max Weight: %s
' % str(person.RMTestExBWeight)

		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "RMpredict" in person.framesChecked:
		ptext = '1 Rep Max (Predicted)
'

		ptext = ptext + 'Exercise: %s
' % str(person.RMPredictEx)

		ptext = ptext + '1 Rep Max Weight: %s
' % str(person.getRepMaxPredict())
		Story.append(Paragraph(ptext, styles["Justify"]))

		Story.append(Spacer(1, 12))

	if "GripStrength" in person.framesChecked:
		ptext = 'GripStrength:
'

		ptext = ptext + 'Left Hand(kg): %s
' % str(person.gripStrengthLeft)

		ptext = ptext + 'Right Hand(kg): %s
' % str(person.gripStrengthRight)
		Story.append(Paragraph(ptext, styles["Justify"]))

		Story.append(Spacer(1, 12))

	if "PushUps" in person.framesChecked:
		ptext = 'Push Ups: %s' % str(person.pushUpNum)
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "CurlUps" in person.framesChecked:
		ptext = 'Curl Ups: %s' % str(person.curlUpNum)
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "PlankEnd" in person.framesChecked:
		ptext = 'Front Plank Time(sec.): %s' % str(person.plankTime)
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "WallSit" in person.framesChecked:
		ptext = 'Wall Sit Time(sec.): %s' % str(person.wallSitTime)
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "FlexTests" in person.framesChecked:
		ptext = 'Sit & Reach Distance(cm): %s' % str(person.sitReachDist)
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "SLstance" in person.framesChecked:
		ptext = 'Single Leg Stance Test:
'

		ptext = ptext + 'Left Leg Eyes Open Time(sec.): %s
' % str(person.SLOpenLeft)

		ptext = ptext + 'Right Leg Eyes Open Time(sec.): %s
' % str(person.SLOpenRight)

		ptext = ptext + 'Left Leg Eyes Closed Time(sec.): %s
' % str(person.SLCloseLeft)

		ptext = ptext + 'Right Leg Eyes Closed Time(sec.): %s
' % str(person.SLCloseRight)
		Story.append(Paragraph(ptext, styles["Justify"]))

		Story.append(Spacer(1, 12))

	if "DeepSquat" in person.framesChecked:
		ptext = 'Deep Squat Assessment Rating(0-3): %s' % str(person.deepSquatRate)
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "WallSlide" in person.framesChecked:
		ptext = ' Wall Slide Assessment Rating(0-3):%s' % str(person.wallSlideRate)
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "HipHinge" in person.framesChecked:
		ptext = 'Hip Hinge Assessment Rating(0-3): %s' % str(person.hipHingeRate)
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	if "FrontPlank" in person.framesChecked:
		ptext = 'Front Plank Assessment Rating(0-3): %s' % str(person.frontPlankRate)
		Story.append(Paragraph(ptext, styles["Justify"]))
		Story.append(Spacer(1, 12))

	doc.build(Story)
	print("PDF created")


import cv2
import numpy as np
from shapely.geometry import LineString, Point

import mmocr.utils as utils
from .box_utils import sort_vertex

def box_jitter(points_x, points_y, jitter_ratio_x=0.5, jitter_ratio_y=0.1):
    """Jitter on the coordinates of bounding box.

    Args:
        points_x (list[float | int]): List of y for four vertices.
        points_y (list[float | int]): List of x for four vertices.
        jitter_ratio_x (float): Horizontal jitter ratio relative to the height.
        jitter_ratio_y (float): Vertical jitter ratio relative to the height.
    """
    assert len(points_x) == 4
    assert len(points_y) == 4
    assert isinstance(jitter_ratio_x, float)
    assert isinstance(jitter_ratio_y, float)
    assert 0 <= jitter_ratio_x < 1
    assert 0 <= jitter_ratio_y < 1

    points = [Point(points_x[i], points_y[i]) for i in range(4)]
    line_list = [
        LineString([points[i], points[i + 1 if i < 3 else 0]])
        for i in range(4)
    ]

    tmp_h = max(line_list[1].length, line_list[3].length)

    for i in range(4):
        jitter_pixel_x = (np.random.rand() - 0.5) * 2 * jitter_ratio_x * tmp_h
        jitter_pixel_y = (np.random.rand() - 0.5) * 2 * jitter_ratio_y * tmp_h
        points_x[i] += jitter_pixel_x
        points_y[i] += jitter_pixel_y

def warp_img(src_img,
             box,
             jitter_flag=False,
             jitter_ratio_x=0.5,
             jitter_ratio_y=0.1):
    """Crop box area from image using opencv warpPerspective w/o box jitter.

    Args:
        src_img (np.array): Image before cropping.
        box (list[float | int]): Coordinates of quadrangle.
    """
    assert utils.is_type_list(box, float) or utils.is_type_list(box, int)
    assert len(box) == 8

    h, w = src_img.shape[:2]
    points_x = [min(max(x, 0), w) for x in box[0:8:2]]
    points_y = [min(max(y, 0), h) for y in box[1:9:2]]

    points_x, points_y = sort_vertex(points_x, points_y)

    if jitter_flag:
        box_jitter(
            points_x,
            points_y,
            jitter_ratio_x=jitter_ratio_x,
            jitter_ratio_y=jitter_ratio_y)

    points = [Point(points_x[i], points_y[i]) for i in range(4)]
    edges = [
        LineString([points[i], points[i + 1 if i < 3 else 0]])
        for i in range(4)
    ]

    pts1 = np.float32([[points[i].x, points[i].y] for i in range(4)])
    box_width = max(edges[0].length, edges[2].length)
    box_height = max(edges[1].length, edges[3].length)

    pts2 = np.float32([[0, 0], [box_width, 0], [box_width, box_height],
                       [0, box_height]])
    M = cv2.getPerspectiveTransform(pts1, pts2)
    dst_img = cv2.warpPerspective(src_img, M,
                                  (int(box_width), int(box_height)))

    return dst_img

def crop_img(src_img, box):
    """Crop box area to rectangle.

    Args:
        src_img (np.array): Image before crop.
        box (list[float | int]): Points of quadrangle.
    """
    assert utils.is_type_list(box, float) or utils.is_type_list(box, int)
    assert len(box) == 8

    h, w = src_img.shape[:2]
    points_x = [min(max(x, 0), w) for x in box[0:8:2]]
    points_y = [min(max(y, 0), h) for y in box[1:9:2]]

    left = int(min(points_x))
    top = int(min(points_y))
    right = int(max(points_x))
    bottom = int(max(points_y))

    dst_img = src_img[top:bottom, left:right]

    return dst_img


#Python file to fetch specific jobs, cvs, categories in command line
# Example usage:
# python3 -u fetch.py 0 542 "description" > output.txt
from pathlib import Path
import sys
import json
import os
def process(jeysan, property_name):
    if property_name == "all":
        print(json.dumps(jeysan))
        os._exit(0)
    else:
        try:
            print(jeysan[property_name])
            os._exit(0)
        except:
            print("invalid property name")
            os._exit(1)

def main():
    if len(sys.argv) != 4:
        print('Usage: python3 fetch.py type id json_property_name')
        os._exit(1)
    job_type = sys.argv[1] #0 for job, 1 for cv, 2 for category
    resource_id = sys.argv[2]
    property_name = sys.argv[3] #can also be 'all'
    try:
        id_string, fullpath = "", ""
        if(job_type == '0'):
            id_string = "jobId"
            fullpath = 'alljobs4rdd/alljobs.jsonl'
        elif(job_type == '1'):
            id_string = "cvid"
            fullpath = 'allcvs4rdd/allcvs.jsonl'
        elif(job_type == '2'):
            id_string = "id"
            fullpath = 'allcategories4rdd/allcategories.jsonl'
        else:
            print('invalid job type')
            os._exit(1)
        contents = Path(fullpath).read_text().strip()
        line_arr = contents.split('\n')
        for line in line_arr:
            try:
                jeysan = json.loads(line)
                if jeysan[id_string] == int(resource_id):
                    process(jeysan, property_name)
            except ValueError:
                print('ValueError')
                continue
    except BaseException as e:
        print('exceptionda')
        print(str(e))
        os._exit(1)

if __name__ == '__main__':
    main()


import glob
import json
import os
import re
import sys
import time

def set_env_var(env_var, value, overwrite_existing=False):
    """
    Sets the specified environment variable.

    If overwrite_existing is False, it will only set the new env var value
    if the environment variable is not already set.

    If overwrite_existing is True, the environment variable will always be
    set to the specified value.
    """
    if overwrite_existing or not os.environ.get(env_var):
        os.environ[env_var] = str(value)

class BaseModelInitializer(object):

    def __init__(self, args, custom_args=[], platform_util=None):
        self.args = args
        self.custom_args = custom_args
        self.platform_util = platform_util

        # Set default values for TCMalloc and convert string value to a boolean
        if self.args.disable_tcmalloc is None:
            # Set to False for int8 and True for other precisions
            self.args.disable_tcmalloc = self.args.precision != "int8"
        elif isinstance(self.args.disable_tcmalloc, str):
            self.args.disable_tcmalloc = self.args.disable_tcmalloc == "True"

        # Ensure that we are using the proper version of python to run the benchmarking script
        self.python_exe = os.environ["PYTHON_EXE"]

        if not platform_util:
            raise ValueError("Did not find any platform info.")

        # use case: bare-metal with openmpi, horovod and multi-node
        if os.environ["MPI_HOSTNAMES"] != "None" and ("DOCKER" not in os.environ or os.environ["DOCKER"] == "False"):
            if os.environ["MPI_NUM_PROCESSES"] != "None":
                try:
                    # slots per host calculation using MPI_NUM_PROCESSES and number of hosts
                    host_names = os.environ["MPI_HOSTNAMES"]
                    number_of_hosts = len(host_names.split(','))
                    slots_per_host = int(int(os.environ["MPI_NUM_PROCESSES"]) / number_of_hosts)
                    host_names = ",".join([host + ":" + str(slots_per_host) for host in host_names.split(',')])
                    # see the [examples](https://horovod.readthedocs.io/en/latest/mpirun.html) for the mca flags
                    self.python_exe = "mpirun " + " -x LD_LIBRARY_PATH " + " -x PYTHONPATH " \
                        + " --allow-run-as-root -n " + os.environ["MPI_NUM_PROCESSES"] + " -H " + host_names \
                        + " -mca pml ob1 -mca btl ^openib -mca btl_tcp_if_exclude " \
                          "lo,docker0 --bind-to none --map-by slot " \
                        + self.python_exe
                except Exception as exception:
                    raise ValueError("Caught exception calculating slots per host {}".format(str(exception)))
            else:
                raise ValueError("MPI_NUM_PROCESSES is required for MPI_HOSTNAMES and will be split evenly across the "
                                 "hosts.")
        # use case: docker with openmpi, single-node, multi-instance
        elif os.environ["MPI_NUM_PROCESSES"] != "None":
            if os.environ["MPI_NUM_PROCESSES_PER_SOCKET"] == "1":
                # Map by socket using OpenMPI by default (PPS=1).
                self.python_exe = "mpirun --allow-run-as-root -n " + os.environ["MPI_NUM_PROCESSES"] \
                    + " --map-by socket " + self.python_exe
            else:
                # number of processes per socket (pps)
                pps = int(os.environ["MPI_NUM_PROCESSES_PER_SOCKET"])
                split_a_socket = str(platform_util.num_cores_per_socket // pps)
                # Launch pps MPI processes over one socket
                self.python_exe = "mpirun --allow-run-as-root -n " + os.environ["MPI_NUM_PROCESSES"] \
                    + " --map-by ppr:" + str(pps) + ":socket:pe=" + split_a_socket + " --cpus-per-proc " \
                    + split_a_socket + " " + self.python_exe

    def run_command(self, cmd, replace_unique_output_dir=None):
        """
        Prints debug messages when verbose is enabled, and then runs the
        specified command.

        If the replace_unique_output_dir arg is set, multi-instance runs will
        swap out that path for a path with the instance number in the folder name
        so that each instance uses a unique output folder.
        """
        if self.args.verbose:
            print("Received these standard args: {}".format(self.args))
            print("Received these custom args: {}".format(self.custom_args))
            print("Current directory: {}".format(os.getcwd()))

        if self.args.numa_cores_per_instance:
            num_numas = self.platform_util.num_numa_nodes

            if not num_numas:
                print("Warning: Unable to run multiple instances using numactl, "
                      "because no numa nodes were found.")
            elif not self.platform_util.cpu_core_list:
                print("Warning: Unable to run multiple instances using numactl, "
                      "the list of cpu nodes could not be retrieved. Please ensure "
                      "that your system has numa nodes and numactl is installed.")
            else:
                self.run_numactl_multi_instance(
                    cmd, replace_unique_output_dir=replace_unique_output_dir)
        else:
            if self.args.verbose:
                print("Running: {}".format(str(cmd)))

            os.system(cmd)

    def group_cores(self, cpu_cores_list, cores_per_instance):
        """
        Group cores based on the number of cores we want per instance.
        Returns a 2D array with the list of cores for each instance.
        """
        list_of_groups = zip(*(iter(cpu_cores_list),) * cores_per_instance)
        end_list = [list(i) for i in list_of_groups]
        count = len(cpu_cores_list) % cores_per_instance
        end_list.append(cpu_cores_list[-count:]) if count != 0 else end_list
        return end_list

    def run_numactl_multi_instance(self, cmd, replace_unique_output_dir=None):
        """
        Generates a series of commands that call the specified cmd with multiple
        instances, where each instance uses the a specified number of cores. The
        number of cores used per instance is specified by args.numa_cores_per_instance.

        The command for each instance uses numactl and the --physcpubind arg with
        the appropriate core list. Each instance writes output to it's own log file,
        and a combined log file is created after everything has executed.

        If the replace_unique_output_dir arg is set, multi-instance runs will
        swap out that path for a path with the instance number in the folder name
        so that each instance uses a unique output folder.
        """

        # Find LD_PRELOAD vars, remove them from the cmd, and save them to add on to the prefix
        ld_preload_strs = re.findall(r'\bLD_PRELOAD=\S*', cmd)
        ld_preload_prefix = ""
        for ld_preload_str in ld_preload_strs:
            cmd = cmd.replace(ld_preload_str, "")
            ld_preload_prefix += ld_preload_str + " "

        # Remove leading/trailing whitespace
        cmd = cmd.strip()

        if self.args.numa_cores_per_instance != "socket":
            # Get the cores list and group them according to the number of cores per instance
            cores_per_instance = int(self.args.numa_cores_per_instance)
            cpu_cores_list = self.platform_util.cpu_core_list

            if self.args.socket_id != -1:
                # If it's specified to just use a single socket, then only use the cores from that socket
                if len(cpu_cores_list) > self.args.socket_id:
                    cpu_cores_list = cpu_cores_list[self.args.socket_id]
                else:
                    raise ValueError("Error while trying to get the core list for socket {0}. "
                                     "The core list does not have cores for socket {0}.\n "
                                     "Core list: {1}\n".format(self.args.socket_id, str(cpu_cores_list)))
            else:
                # Using cores from all sockets
                combined_core_list = []
                for socket_cores in cpu_cores_list:
                    combined_core_list += socket_cores
                cpu_cores_list = combined_core_list

            instance_cores_list = self.group_cores(cpu_cores_list, cores_per_instance)
        else:
            instance_cores_list = []
            cores_per_instance = "socket"
            # Cores should be grouped based on the cores for each socket
            if self.args.socket_id != -1:
                # Only using cores from one socket
                instance_cores_list[0] = self.platform_util.cpu_core_list[self.args.socket_id]
            else:
                # Get the cores for each socket
                instance_cores_list = self.platform_util.cpu_core_list

        # Setup the log file name with the model name, precision, mode, batch size (if there is one),
        # number of cores per instance. An extra {} is intentionally left in the log_filename_format
        # string, because this value is filled in with the instance number later on.
        batch_size = ""
        if self.args.batch_size and self.args.batch_size > 0:
            batch_size = "bs{}_".format(self.args.batch_size)
        log_filename_format = os.path.join(
            self.args.output_dir, "{}_{}_{}_{}cores{}_".format(
                self.args.model_name, self.args.precision, self.args.mode, batch_size, cores_per_instance))
        log_filename_format += "{}.log"
        instance_logfiles = []

        # Loop through each instance and add that instance's command to a string
        multi_instance_command = ""
        for instance_num, core_list in enumerate(instance_cores_list):
            if cores_per_instance != "socket" and len(core_list) < int(cores_per_instance):
                print("NOTE: Skipping remainder of {} cores for instance {}"
                      .format(len(core_list), instance_num))
                continue

            if len(core_list) == 0:
                continue

            prefix = ("{0}OMP_NUM_THREADS={1} "
                      "numactl --localalloc --physcpubind={2}").format(
                ld_preload_prefix, len(core_list), ",".join(core_list))
            instance_logfile = log_filename_format.format("instance" + str(instance_num))

            unique_command = cmd
            if replace_unique_output_dir:
                # Swap out the output dir for a unique dir
                unique_dir = os.path.join(replace_unique_output_dir,
                                          "instance_{}".format(instance_num))
                unique_command = unique_command.replace(replace_unique_output_dir, unique_dir)

            instance_command = "{} {}".format(prefix, unique_command)
            multi_instance_command += "{} >> {} 2>&1 & \\\n".format(
                instance_command, instance_logfile)
            instance_logfiles.append(instance_logfile)

            # write the command to the instance's log file
            with open(instance_logfile, "w") as log:
                log.write(instance_command)
                log.write("\n\n")

        multi_instance_command += "wait"

        # Run the multi-instance command
        print("\nMulti-instance run:\n" + multi_instance_command)
        sys.stdout.flush()
        os.system(multi_instance_command)

        # Wait to ensure that log files have been written
        max_retries = 20
        retry_counter = 0
        while retry_counter < max_retries:
            if all([os.path.exists(log) for log in instance_logfiles]):
                break
            retry_counter += 1
            if retry_counter >= max_retries:
                print("Warning: Log files for all instances were not found after "
                      "rechecking and waiting for {} seconds. The combined log file "
                      "may not have output from all instances.".format(retry_counter))
                break
            time.sleep(1)

        # Generate the combined log file
        all_instance_log = log_filename_format.format("all_instances")
        os.environ["LOG_FILENAME"] = os.path.basename(all_instance_log)
        with open(all_instance_log, mode="w") as combined_file:
            for instance_logfile in instance_logfiles:
                if not os.path.exists(instance_logfile):
                    print("Skipping {} when generating the combined log file, because "
                          "it doesn't exist".format(os.path.basename(instance_logfile)))
                    continue

                with open(instance_logfile) as individual_file:
                    for line in individual_file:
                        combined_file.write(line)

        # Print out lists of log files
        print("\nThe following log files were saved to the output directory:")
        print("\n".join([os.path.basename(log_path) for log_path in instance_logfiles
                         if os.path.exists(log_path)]))
        if os.path.exists(all_instance_log):
            print("\nA combined log file was saved to the output directory:\n"
                  "{}\n".format(os.path.basename(all_instance_log)))

    def get_command_prefix(self, socket_id, numactl=True):
        """
        Returns the command prefix with:
         - LD_PRELOAD for int8 models (if tcmalloc is not disabled)
         - The numactl command with --cpunodebind and --membind set to the specified socket_id (if numactl=True)

        Should be used only for single instance.
        """
        command = ""
        ld_preload = ""

        if not self.args.disable_tcmalloc:
            # Try to find the TCMalloc library file
            matches = glob.glob("/usr/lib/libtcmalloc.so*")

            if len(matches) == 0:
                matches = glob.glob("/usr/lib64/libtcmalloc.so*")

            if len(matches) == 0:
                matches = glob.glob("/usr/lib/*/libtcmalloc.so*")

            if len(matches) == 0:
                matches = glob.glob("/usr/lib64/*/libtcmalloc.so*")

            if len(matches) > 0:
                ld_preload += "LD_PRELOAD={} ".format(matches[0])
            else:
                # Unable to find the TCMalloc library file
                print("Warning: Unable to find the TCMalloc library file (libtcmalloc.so) in /usr/lib, /usr/lib64, "
                      "/usr/lib/*, or /usr/lib64/* so the LD_PRELOAD environment variable will not be set.")

        num_numas = self.platform_util.num_numa_nodes
        if num_numas and socket_id != -1 and numactl and not self.args.numa_cores_per_instance:
            if self.args.num_cores == -1:
                # Running on the whole socket
                command += "numactl --cpunodebind={0} --membind={0} ".format(
                    str(socket_id))
            else:
                # Running on specific number of cores
                first_physical_core = self.platform_util.cpuset_cpus[0][0]
                num_sockets = len(self.platform_util.cpuset_cpus.keys())
                num_cores_in_socket0 = len(self.platform_util.cpuset_cpus[0])
                for i in range(num_sockets):
                    if num_cores_in_socket0 != len(
                            self.platform_util.cpuset_cpus[i]):
                        raise ValueError(
                            "Error: Identifying logical core id assumes all sockets have same number of cores"
                        )
                first_logical_core = num_cores_in_socket0 * num_sockets
                if self.platform_util.num_threads_per_core == 1:
                    # HT is off
                    cpus_range = "{0}-{1}".format(
                        first_physical_core,
                        first_physical_core + self.args.num_cores - 1)
                else:
                    # HT is on.
                    cpus_range = "{0}-{1},{2}-{3}".format(
                        first_physical_core,
                        first_physical_core + self.args.num_cores - 1,
                        first_logical_core,
                        first_logical_core + self.args.num_cores - 1)
                command += "numactl -C{0} --membind=0 ".format(cpus_range)

        # Add LD_PRELOAD to the front of the command
        command = ld_preload + command

        return command

    def add_args_to_command(self, command, arg_list):
        """
        Add args that are specified in the arg list to the command.  batch_size
        is a special case, where it's not added if it's set to -1 (undefined).
        Returns the command string with args.
        """
        for arg in vars(self.args):
            arg_value = getattr(self.args, arg)
            if arg == "batch_size" and arg_value == -1:
                continue
            if arg_value and (arg in arg_list):
                command = "{cmd} --{param}={value}".format(
                    cmd=command, param=arg, value=arg_value)
        return command

    def set_num_inter_intra_threads(self, num_inter_threads=None, num_intra_threads=None):
        """
        Sets default values for self.args.num_inter_threads and
        self.args.num_intra_threads, only if they are not already set.

        If num_inter_threads and/or num_intra_threads are specified, then those
        are the values that will be used. Otherwise, if they are None, then the
        following criteria applies:

        If multiple instances are being used (specified with numa_cores_per_instance),
        then each instance should have:
        * num_inter_threads = 1
        * num_intra_threads = number of cores per instance

        If a single socket is being used:
         * num_inter_threads = 1
         * num_intra_threads = The number of cores on a single socket, or
           self.args.num_cores if a specific number of cores was defined.

        If all sockets are being used:
         * num_inter_threads = The number of sockets
         * num_intra_threads = The total number of cores across all sockets, or
           self.args.num_cores if a specific number of cores was defined.
         * in case MPI_NUM_PROCESSES is used
           * num_inter_threads = 1
           * num_intra_threads = the number of cores on a single socket minus 2
        """
        # if num_inter_threads is specified, use that value as long as the arg isn't set
        if num_inter_threads and not self.args.num_inter_threads:
            self.args.num_inter_threads = num_inter_threads

        # if num_intra_threads is specified, use that value as long as the arg isn't set
        if num_intra_threads and not self.args.num_intra_threads:
            self.args.num_intra_threads = num_intra_threads

        if self.args.numa_cores_per_instance:
            # Set default num inter/intra threads if the user didn't provide specific values
            if self.args.numa_cores_per_instance == "socket":
                if self.args.socket_id != -1:
                    inter_threads = len(self.platform_util.cpu_core_list[self.args.socket_id])
                else:
                    # since we can only have one value for inter threads and the number of cores
                    # per socket can vary, if the cpuset is limited, get the lowest core count
                    # per socket and use that as the num inter threads
                    inter_threads = min([len(i) for i in self.platform_util.cpu_core_list if len(i) > 0])
            else:
                inter_threads = self.args.numa_cores_per_instance

            if not self.args.num_inter_threads:
                self.args.num_inter_threads = 1
            if not self.args.num_intra_threads:
                self.args.num_intra_threads = inter_threads
            if not self.args.data_num_inter_threads:
                self.args.data_num_inter_threads = 1
            if not self.args.data_num_intra_threads:
                self.args.data_num_intra_threads = inter_threads
        elif self.args.socket_id != -1:
            if not self.args.num_inter_threads:
                self.args.num_inter_threads = 1
            if not self.args.num_intra_threads:
                if self.args.num_cores != -1:
                    self.args.num_intra_threads = self.args.num_cores
                elif self.platform_util.cpuset_cpus and \
                        self.args.socket_id in self.platform_util.cpuset_cpus.keys():
                    self.args.num_intra_threads = len(self.platform_util.cpuset_cpus[self.args.socket_id])
                else:
                    self.args.num_intra_threads = self.platform_util.num_cores_per_socket
        else:
            if not self.args.num_inter_threads:
                self.args.num_inter_threads = self.platform_util.num_cpu_sockets
                if os.environ["MPI_NUM_PROCESSES"] != "None":
                    self.args.num_inter_threads = 1
            if not self.args.num_intra_threads:
                if self.args.num_cores == -1:
                    if self.platform_util.cpuset_cpus and len(self.platform_util.cpuset_cpus.keys()) > 0:
                        # Total up the number of cores in the cpuset
                        self.args.num_intra_threads = sum([len(self.platform_util.cpuset_cpus[socket_id])
                                                           for socket_id in self.platform_util.cpuset_cpus.keys()])
                    else:
                        self.args.num_intra_threads = \
                            int(self.platform_util.num_cores_per_socket *
                                self.platform_util.num_cpu_sockets)
                    if os.environ["MPI_NUM_PROCESSES"] != "None":
                        self.args.num_intra_threads = self.platform_util.num_cores_per_socket - 2
                else:
                    self.args.num_intra_threads = self.args.num_cores

        if self.args.verbose:
            print("num_inter_threads: {}\nnum_intra_threads: {}".format(
                self.args.num_inter_threads, self.args.num_intra_threads))

    def set_kmp_vars(self, config_file_path, kmp_settings=None, kmp_blocktime=None, kmp_affinity=None):
        """
        Sets KMP_* environment variables to the specified value, if the environment variable has not already been set.
        The default values in the json file are the best known settings for the model.
        """
        if os.path.exists(config_file_path):
            with open(config_file_path, 'r') as config:
                config_object = json.load(config)

            # First sets default from config file
            for param in config_object.keys():
                for env in config_object[param].keys():
                    set_env_var(env, config_object[param][env])

        else:
            print("Warning: File {} does not exist and \
            cannot be used to set KMP environment variables".format(config_file_path))

        # Override user provided envs
        if kmp_settings:
            set_env_var("KMP_SETTINGS", kmp_settings, overwrite_existing=True)
        if kmp_blocktime:
            set_env_var("KMP_BLOCKTIME", kmp_blocktime, overwrite_existing=True)
        if kmp_affinity:
            set_env_var("KMP_AFFINITY", kmp_affinity, overwrite_existing=True)


""" THIS CODE BELOW CAN BE COPY PASTED INTO A NEW PROJECT THAT
 REQUIRED HAND TRACKING - USING HandTrackingModule.py """

import cv2
import mediapipe as mp
import time
import HandTrackingModule as htm

pTime = 0
cTime = 0
cap = cv2.VideoCapture(0)
detector = htm.handDetector()

while True:
    success, img = cap.read()
    img = detector.findHands(img)
    lmList = detector.findPosition(img)

    cTime = time.time()
    fps = 1 / (cTime - pTime)
    pTime = cTime

    cv2.putText(img, str(int(fps)), (10, 70), cv2.FONT_HERSHEY_PLAIN, 3, (0, 0, 255), 3)

    cv2.imshow("Video", img)
    if cv2.waitKey(1) == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()


'''
    Mooc 的虚基类：用于派生所有Mooc子类
'''

import os
from abc import ABC, abstractmethod
from Mooc.Mooc_Config import *
from Mooc.Mooc_Download import *
from Mooc.Mooc_Request import *
import m3u8
from urllib.parse import urljoin
from shutil import rmtree
__all__ = [
    "Mooc_Base"
]

class Mooc_Base(ABC):
    def __init__(self):
        self.__mode = None
        self.__cid = None
        self.__title = None
        self.__infos = None
        self.__rootDir = None

    @property
    def mode(self):
        '''下载模式: 用于选择性下载'''
        return self.__mode

    @mode.setter
    def mode(self, mode):
        self.__mode = mode

    @property
    def cid(self):
        '''课程的 ID'''
        return self.__cid

    @cid.setter
    def cid(self, cid):
        self.__cid = cid

    @property
    def title(self):
        '''课程的标题'''
        return self.__title

    @title.setter
    def title(self, title):
        self.__title = title

    @property
    def infos(self):
        '''解析后的课程信息'''
        return self.__infos

    @property
    def rootDir(self):
        return self.__rootDir

    @rootDir.setter
    def rootDir(self, rootDir):
        self.__rootDir = rootDir

    @infos.setter
    def infos(self, infos):
        self.__infos = infos

    @abstractmethod
    def _get_cid(self):
        pass

    @abstractmethod
    def _get_title(self):
        pass

    @abstractmethod
    def _get_infos(self):
        pass

    @abstractmethod
    def _download(self):
        pass

    @abstractmethod
    def set_mode(self):
        pass

    @abstractmethod
    def prepare(self, url):
        pass

    @abstractmethod
    def download(self):
        pass

    @classmethod
    def download_video(cls, video_url, video_name, video_dir, video_size=None):
        '''下载 MP4 视频文件'''
        succeed = True
        if not cls.judge_file_existed(video_dir, video_name, '.mp4'):
            if 'm3u8' in video_url:
                cls.download_m3u8_video(
                    video_url, video_name, video_dir, video_size=video_size)
                print(
                    "\r  |-{}  [mp4]下载完成".format(cls.align(video_name, LENGTH)))
            else:
                try:
                    header = request_head(video_url)
                    size = float(header['Content-Length']) / (1024*1024)
                    print(
                        "  |-{}  [mp4] 大小: {:.2f}M".format(cls.align(video_name, LENGTH), size))
                    aria2_download_file(
                        video_url, video_name+'.mp4', video_dir)
                except DownloadFailed:
                    print(
                        "  |-{}  [mp4] 资源无法下载！".format(cls.align(video_name, LENGTH)))
                    succeed = False
        else:
            print(
                "  |-{}  [mp4] 已经成功下载！".format(cls.align(video_name, LENGTH)))
        return succeed

    @classmethod
    def download_m3u8_video(cls, video_url, video_name, video_dir, video_size=None):
        tmpdir = os.path.join(video_dir, 'tmp')
        tsURLsFile = 'ts.txt'

        succeed = True

        if not (os.path.exists(tmpdir)):
            os.mkdir(tmpdir)
        os.chdir(tmpdir)

        m3u8Str = request_get(video_url)
        m3u8_obj = m3u8.loads(m3u8Str)

        with open(tsURLsFile, 'w') as f:
            for url in m3u8_obj.files:
                # m3u8文件获取各个ts流的真正下载地址，它实际是m3u8文件的url更改路径得到的
                # 另外，由于实际下载下来的文件名称可能和url中的名称不一致，所以需要强制指定
                # 下载文件名称
                f.write(urljoin(video_url, url) + '\n' +
                        '    out=' + url + '\n')
            f.close()

        try:
            print("  |-{}  [mp4] 大小: {:.2f}M".format(cls.align(video_name,
                                                               LENGTH), video_size / (1024*1024)))
            aria2_download_m3u8_video(
                video_name, video_dir, tsURLsFile, video_size)
            # 一定要记得先跳出tmp目录再删除，否则无法删除！！！
            os.chdir(video_dir)
            rmtree(tmpdir)
        except DownloadFailed:
            print(
                "  |-{}  [mp4] 资源无法下载！".format(cls.align(video_name, LENGTH)))
            succeed = False
        return succeed

    @classmethod
    def download_pdf(cls, pdf_url, pdf_name, pdf_dir):
        '''下载 PDF '''
        succeed = True
        if not cls.judge_file_existed(pdf_dir, pdf_name, '.pdf'):
            try:
                aria2_download_file(pdf_url, pdf_name+'.pdf', pdf_dir)
                print("  |-{}  (pdf) 已经成功下载！".format(cls.align(pdf_name, LENGTH)))
            except DownloadFailed:
                print("  |-{}  (pdf) 资源无法下载！".format(cls.align(pdf_name, LENGTH)))
                succeed = False
        else:
            print("  |-{}  (pdf) 已经成功下载！".format(cls.align(pdf_name, LENGTH)))
        return succeed

    @classmethod
    def download_sub(cls, sub_url, sub_name, sub_dir):
        '''下载字幕'''
        succeed = True
        if not cls.judge_file_existed(sub_dir, sub_name, '.srt'):
            try:
                aria2_download_file(sub_url, sub_name+'.srt', sub_dir)
            except DownloadFailed:
                succeed = False
        return succeed

    @staticmethod
    def judge_file_existed(dirname, filename, fmt):
        '''
        judge_file_existed(dirname, filename, fmt)
        判断在 dirname 目录下是否存在已下载成功的格式为 fmt 且文件名为 filename 的文件
        '''
        filepath = os.path.join(dirname, filename)
        exist1 = os.path.exists(filepath+fmt)
        exist2 = os.path.exists(filepath+fmt+'.aria2')
        return exist1 and not exist2

    @staticmethod
    def align(string, width):  # 对齐汉字字符窜，同时截断多余输出
        '''
        align(string, width) 根据width宽度居中对齐字符窜 string，主要用于汉字居中
        '''
        res = ""
        size = 0
        for ch in string:
            if (size+3 > width):
                break
            size += 1 if ord(ch) <= 127 else 2
            res += ch
        res += (width-size)*' '
        return res


from django.conf.urls import url

from .views import *

urlpatterns=[
    #首页
    url(r'^home/',home,name='home'),
    # 闪购
    url(r'^market/$', market, name='market'),
    url(r'^marketwithparams/(\d+)/(\d+)/(\d+)/$', market_width_params, name='market_with_params'),

]


from django.conf.urls import url
from django.urls import path,include
from Annotation import views

urlpatterns=[
	path('article_upload',views.articleUploadAPI),
	url('all_article',views.getArticle),
	path('get_sentence/<int:aid>',views.getSentence_ArticleID),
	path('postlabels',views.postLabels),
	path('getSentence_sid/<int:sid>',views.getSentence_sid),
	path('addFrame',views.addFrame),
	path('getHistorySentence/<int:sid>',views.getWordLabel),
	url('getFinishArticle',views.getFinishArticle),
	path('getSentence_HistoryID/<int:aid>',views.getSentence_HistoryID),
	path('updateSentence',views.updateSentence),
	path('getArticlePage/<int:pagenumber>/<int:s>',views.getArticlePage),
	path('getPageNumber/<int:s>',views.getPageNumber),
	path('draftLabels',views.draftLabels)
]


# 2021-04-25(일) 수정

from urllib.request import urlopen
from bs4 import BeautifulSoup
import pandas
#import 색 보이기 Unresolved references

# 1. urlopen
html = urlopen('http://www.naver.com')
bsob = BeautifulSoup(html, "html.parser")

# 2. 출력 시행
print(bsob) # 전체 출력
print(bsob.head.title) # 태그로 구성된 트리에서 title 태그만 출력

# 3. tag a와 href속성 출력
bsob.find_all('a')[0].text.strip()
bsob.find_all('a')[0].get('href')

for link in bsob.find_all('a'):
    print(link.text.strip(), link.get('href')) # a 태그로 둘러싸인 텍스트와 a태그의 href속성 출력

# 4. 교보문고 주간순위 책 정보 가져오기
html = urlopen('http://www.kyobobook.co.kr/bestSellerNew/bestseller.laf')
bs = BeautifulSoup(html,'html.parser')

# 5. div에서 class: detail의 href 가져오기
#

# <a href="http://www.kyobobook.co.kr/.....

book_page_urls = []
for cover in bs.find_all('div',{'class' : 'detail'}):
    link = cover.select('a')[0].get('href')
    book_page_urls.append(link) #url list 만들기

len(book_page_urls)

# 6. enumerate로 index반환 및 확인
for index, urls in enumerate(book_page_urls):
    print(index, urls)

# ===================================================
# 7. 정보 크롤링
list = []
for index, urls in enumerate(book_page_urls):
    html = urlopen(urls)
    bs = BeautifulSoup(html, 'html.parser')

    title = bs.find('meta', {'property':'rb:itemName'}).get('content')
    author = bs.select('span.name')[0].text
    url = bs.find('meta',{'property':'rb:salePrice'}).get('content')

    print(index+1,title, author, url)

    list.append(index+1)
    list.append(title)
    list.append(author)
    list.append(url)

list2 = [list[i * 4: (i +1) * 4]for i in range((len(list)+4-1) // 4)]
df = pandas.DataFrame(list2)
df = pandas.DataFrame(list2, columns=['index', 'title', 'author', 'url'])
df.title[2] = "코로나 투자 전쟁"

for i in range(df):
    df.author[i] = df.author[i].replace()

for i in range(1,6):
    try:
        print('kk' + str(i) +'hj'/0)
    except:
        print('kk' + str(i) + 'hj')

df = pd.DataFrame(columns=['a','b','c'])
df.a = list_2 # 칼럼에 리스트 삽입
df.b = list_1_2
list_2 + list_2_2 # 일자로 결합

df[0:10] #행 가져오기

News_data_analysis/Python_code(정리필요)/정리완료/20-06-10 기본 크롤링 연습.py


from src import layers
import torch.nn as nn
import torch.nn.functional as F
import torch

# Capsule model
class CapsModel(nn.Module):
    def __init__(self,
                 image_dim_size,
                 params,
                 backbone,
                 dp,
                 num_routing,
                 sequential_routing=True):

        super(CapsModel, self).__init__()
        #### Parameters
        self.sequential_routing = sequential_routing

        ## Primary Capsule Layer
        self.pc_num_caps = params['primary_capsules']['num_caps']
        self.pc_caps_dim = params['primary_capsules']['caps_dim']
        self.pc_output_dim = params['primary_capsules']['out_img_size']
        ## General
        self.num_routing = num_routing # >3 may cause slow converging

        #### Building Networks
        ## Backbone (before capsule)
        if backbone == 'simple':
            self.pre_caps = layers.simple_backbone(params['backbone']['input_dim'],
                                            params['backbone']['output_dim'],
                                            params['backbone']['kernel_size'],
                                            params['backbone']['stride'],
                                            params['backbone']['padding'])
        elif backbone == 'resnet':
            self.pre_caps = layers.resnet_backbone(params['backbone']['input_dim'],
                                           params['backbone']['output_dim'],
                                           params['backbone']['stride'])

        ## Primary Capsule Layer (a single CNN)
        self.pc_layer = nn.Conv2d(in_channels=params['primary_capsules']['input_dim'],
                                     out_channels=params['primary_capsules']['num_caps'] *\
                                                          params['primary_capsules']['caps_dim'],
                                     kernel_size=params['primary_capsules']['kernel_size'],
                                     stride=params['primary_capsules']['stride'],
                                     padding=params['primary_capsules']['padding'],
                                     bias=False)

        #self.pc_layer = nn.Sequential()

        self.nonlinear_act = nn.LayerNorm(params['primary_capsules']['caps_dim'])

        ## Main Capsule Layers
        self.capsule_layers = nn.ModuleList([])
        for i in range(len(params['capsules'])):
            if params['capsules'][i]['type'] == 'CONV':
                in_n_caps = params['primary_capsules']['num_caps'] if i==0 else \
                                                               params['capsules'][i-1]['num_caps']
                in_d_caps = params['primary_capsules']['caps_dim'] if i==0 else \
                                                               params['capsules'][i-1]['caps_dim']
                self.capsule_layers.append(
                    layers.CapsuleCONV(in_n_capsules=in_n_caps,
                                in_d_capsules=in_d_caps,
                                out_n_capsules=params['capsules'][i]['num_caps'],
                                out_d_capsules=params['capsules'][i]['caps_dim'],
                                kernel_size=params['capsules'][i]['kernel_size'],
                                stride=params['capsules'][i]['stride'],
                                matrix_pose=params['capsules'][i]['matrix_pose'],
                                dp=dp,
                                coordinate_add=False
                            )
                )
            elif params['capsules'][i]['type'] == 'FC':
                if i == 0:
                    in_n_caps = params['primary_capsules']['num_caps'] * params['primary_capsules']['out_img_size'] *\
                                                                                params['primary_capsules']['out_img_size']
                    in_d_caps = params['primary_capsules']['caps_dim']
                elif params['capsules'][i-1]['type'] == 'FC':
                    in_n_caps = params['capsules'][i-1]['num_caps']
                    in_d_caps = params['capsules'][i-1]['caps_dim']
                elif params['capsules'][i-1]['type'] == 'CONV':
                    in_n_caps = params['capsules'][i-1]['num_caps'] * params['capsules'][i-1]['out_img_size'] *\
                                                                                params['capsules'][i-1]['out_img_size']
                    in_d_caps = params['capsules'][i-1]['caps_dim']
                self.capsule_layers.append(
                    layers.CapsuleFC(in_n_capsules=in_n_caps,
                          in_d_capsules=in_d_caps,
                          out_n_capsules=params['capsules'][i]['num_caps'],
                          out_d_capsules=params['capsules'][i]['caps_dim'],
                          matrix_pose=params['capsules'][i]['matrix_pose'],
                          dp=dp
                          )
                )

        ## Class Capsule Layer
        if not len(params['capsules'])==0:
            if params['capsules'][-1]['type'] == 'FC':
                in_n_caps = params['capsules'][-1]['num_caps']
                in_d_caps = params['capsules'][-1]['caps_dim']
            elif params['capsules'][-1]['type'] == 'CONV':
                in_n_caps = params['capsules'][-1]['num_caps'] * params['capsules'][-1]['out_img_size'] *\
                                                                                params['capsules'][-1]['out_img_size']
                in_d_caps = params['capsules'][-1]['caps_dim']
        else:
            in_n_caps = params['primary_capsules']['num_caps'] * params['primary_capsules']['out_img_size'] *\
                                                                               params['primary_capsules']['out_img_size']
            in_d_caps = params['primary_capsules']['caps_dim']
        self.capsule_layers.append(
            layers.CapsuleFC(in_n_capsules=in_n_caps,
                  in_d_capsules=in_d_caps,
                  out_n_capsules=params['class_capsules']['num_caps'],
                  out_d_capsules=params['class_capsules']['caps_dim'],
                  matrix_pose=params['class_capsules']['matrix_pose'],
                  dp=dp
                  )
        )

        ## After Capsule
        # fixed classifier for all class capsules
        self.final_fc = nn.Linear(params['class_capsules']['caps_dim'], 1)
        # different classifier for different capsules
        #self.final_fc = nn.Parameter(torch.randn(params['class_capsules']['num_caps'], params['class_capsules']['caps_dim']))

    def forward(self, x, lbl_1=None, lbl_2=None):
        #### Forward Pass
        ## Backbone (before capsule)
        c = self.pre_caps(x)

        ## Primary Capsule Layer (a single CNN)
        u = self.pc_layer(c) # torch.Size([100, 512, 14, 14])
        u = u.permute(0, 2, 3, 1) # 100, 14, 14, 512
        u = u.view(u.shape[0], self.pc_output_dim, self.pc_output_dim, self.pc_num_caps, self.pc_caps_dim) # 100, 14, 14, 32, 16
        u = u.permute(0, 3, 1, 2, 4) # 100, 32, 14, 14, 16
        init_capsule_value = self.nonlinear_act(u)#capsule_utils.squash(u)

        ## Main Capsule Layers
        # concurrent routing
        if not self.sequential_routing:
            # first iteration
            # perform initilialization for the capsule values as single forward passing
            capsule_values, _val = [init_capsule_value], init_capsule_value
            for i in range(len(self.capsule_layers)):
                _val = self.capsule_layers[i].forward(_val, 0)
                capsule_values.append(_val) # get the capsule value for next layer

            # second to t iterations
            # perform the routing between capsule layers
            for n in range(self.num_routing-1):
                _capsule_values = [init_capsule_value]
                for i in range(len(self.capsule_layers)):
                    _val = self.capsule_layers[i].forward(capsule_values[i], n,
                                    capsule_values[i+1])
                    _capsule_values.append(_val)
                capsule_values = _capsule_values
        # sequential routing
        else:
            capsule_values, _val = [init_capsule_value], init_capsule_value
            for i in range(len(self.capsule_layers)):
                # first iteration
                __val = self.capsule_layers[i].forward(_val, 0)
                # second to t iterations
                # perform the routing between capsule layers
                for n in range(self.num_routing-1):
                    __val = self.capsule_layers[i].forward(_val, n, __val)
                _val = __val
                capsule_values.append(_val)

        ## After Capsule
        out = capsule_values[-1]
        out = self.final_fc(out) # fixed classifier for all capsules
        out = out.squeeze() # fixed classifier for all capsules
        #out = torch.einsum('bnd, nd->bn', out, self.final_fc) # different classifiers for distinct capsules

        return out


import numpy as np
import matplotlib.pyplot as plt
import os
import csv
import argparse as ap
import matplotlib.animation as manimation
from tqdm import tqdm
import sys

def open_densities(filename, y_min, y_max):
    csv_data = np.genfromtxt(filename, dtype=float, delimiter=',')[y_min:y_max]

    return csv_data

plt.rc('text', usetex=True)
plt.rc('font', family='serif')

data_0 = open_densities("data/densityTest2/test_0/densities.csv", 0, 3000)
label_0 = r'$\epsilon = 0.1$'
data_1 = open_densities("data/densityTest2/test_1/densities.csv", 0, 3000)
label_1 = r'$\epsilon = 1.25$'
data_2 = open_densities("data/densityTest2/test_2/densities.csv", 0, 3000)
label_2 = r'$\epsilon = 2.5$'
data_3 = open_densities("data/densityTest2/test_3/densities.csv", 0, 900)
label_3 = r'$\epsilon = 10.0$'

fig, ax = plt.subplots()
ax.plot(data_0, 'c-', label=label_0)
ax.plot(data_1, 'm--', label=label_1)
ax.plot(data_2, 'y-.', label=label_2)
ax.plot(data_3, color='forestgreen', linestyle=':', label=label_3)
ax.legend()

ax.grid(which='both', zorder=1)
fig.set_tight_layout(True)

ttl = ax.set_title("Density")
ax.set_xlabel(r'$t$')
ax.set_ylabel(r'$ \rho (t)$')

fig.savefig("densities.png", dpi=200)


import os
import tempfile

from jadio import Hibiki
from jadio.util import check_dict_deep

def test_login_without_user_info():
    with Hibiki():
        pass

def test_api_get_programs():
    with Hibiki() as radio:
        raw_program = radio._get("programs")[0]
        need_keys = [
            "id",
            "name",
            "description",
            "pc_image_url",
            "onair_information",
            "email",
            "copyright",
            "share_url",
            "cast",
            "episode",
            ["episode", "id"],
            ["episode", "name"],
            ["episode", "video", "id"],
            ["episode", "video", "duration"],
            ["episode", "updated_at"],
        ]
        for key in need_keys:
            if isinstance(key, list):
                assert check_dict_deep(raw_program, key)
            else:
                assert key in raw_program

def test_get_programs():
    with Hibiki() as radio:
        programs = radio.get_programs()
        assert len(programs) > 0

def test_get_programs_with_filters():
    with Hibiki() as radio:
        target_id = radio._get("programs")[0]["access_id"]
        programs = radio.get_programs(filters=[target_id])
        assert len(programs) > 0

def test_download():
    with Hibiki() as radio, tempfile.TemporaryDirectory() as tmp_dir:
        target_id = radio._get("programs")[0]["access_id"]
        programs = radio.get_programs(filters=[target_id])
        filename = os.path.join(tmp_dir, "hoge.mp4")
        radio.download(programs[0], filename)
        assert os.path.exists(filename)


# -*- coding: utf-8 -*-

from pyspark.sql.window import Window
from pyspark.sql import SparkSession
from pyspark.sql.types import *
from pyspark.sql.functions import *
import pyspark.sql.functions as F
from pyspark.sql.types import LongType
import pandas as pd

# File location and type
file_location ="C:/Practica_Bosonit/Ejercicios_Spark/reto2/udfs.csv"
file_type = "csv"

infer_schema = "true"
first_row_is_header = "true"
delimiter = ";"

"""Importamos el csv"""

if __name__ == "__main__":
 # Create a SparkSession
    spark = (SparkSession
              .builder
              .appName("Reto_2")
              .getOrCreate())

    """DF"""

# The applied options are for CSV files. For other file types, these will be ignored.
    df = spark.read.format(file_type) \
        .option("inferSchema", infer_schema) \
        .option("header", first_row_is_header) \
        .option("sep", delimiter) \
        .load(file_location)

   # df.show()

    """Primer pivot (string)"""

    pivotDF = (df.groupBy("nb")
               .pivot("udf_name", ["M_CLIENT", "M_SELLER", "M_CCY", "M_SUCURSAL"])
               .agg(first("string_value"))
               .withColumnRenamed("M_SELLER", "M_seller"))

    """Segundo pivot (numerico)"""

    df2 = (df.groupBy("nb")
               .pivot("udf_name", ["M_DISCMARGIN", "M_DIRECTIAV", "M_LIQDTYCHRG", "M_CRDTCHRG", "M_MVA", "M_RVA"])
               .agg(first("num_value"))
               .withColumnRenamed("M_DISCMARGIN", "M_discmargin")
               .withColumnRenamed("M_DIRECTIAV", "M_directiav")
               .withColumnRenamed("M_LIQDTYCHRG", "M_liqdtychrg")
               .withColumnRenamed("M_CRDTCHRG", "M_crdtchrg")
               .withColumnRenamed("M_MVA", "M_mva")
               .withColumnRenamed("M_RVA", "M_rva")

               )

    DF2_final= df2.filter((df2.M_discmargin != 0) & (df2.M_discmargin.isNotNull()) | (df2.M_directiav != 0) & (df2.M_directiav.isNotNull())
                    & (df2.M_liqdtychrg != 0) & (df2.M_liqdtychrg.isNotNull()) & (df2.M_crdtchrg != 0) & (df2.M_crdtchrg.isNotNull())
                    & (df2.M_mva != 0) & (df2.M_mva.isNotNull()) & (df2.M_rva != 0) & (df2.M_rva.isNotNull()))

    DF1_final = pivotDF.filter(pivotDF.M_seller != "")

    df_final = DF1_final.join(DF2_final, DF1_final.nb == DF2_final.nb).show()