import os, datetime, time
import torch
import torch.optim as optim
import numpy as np
import math
import cv2
import tqdm
import config

import constants
from utils.trainer_utils import (
    AverageMeter,
    get_HHMMSS_from_second,
    save_checkpoint,
    save_all_img,
    save_joints3d_img,
    save_mesh,
    save_templates_info,

    train_only_3task_network,
    train_hmr_using_3task,
    train_hmr_using_joints,
    train_texture_net,
    train_hmr_using_adv_loss,
)

from utils.imutils import uncrop

from lib.utils.eval_utils import (
    batch_compute_similarity_transform_torch,
)
from lib.utils.geometry import batch_rodrigues
from lib.models.smpl import SMPL, SMPL_MODEL_DIR, H36M_TO_J14, SMPL_MEAN_PARAMS

# import soft_renderer as sr
# from soft_renderer.mesh import Mesh
# from soft_renderer.renderer import SoftRenderer
# import soft_renderer.cuda.load_textures as load_textures_cuda
from lib.models.smpl import get_smpl_faces

class Trainer():
    def __init__(
        self,
        model_name,

        HMR,
        context_encoder_net,
        discriminator,
        jigsaw_puzzle_net,
        rotation_net,
        texture_net,
        img_renderer,
        seg_renderer,
        texture_discriminator,

        train_dataloader,
        test_dataloader,
        test_dataloader_h36m,
        test_dataloader_3dpw,
        test_dataloader_lsp,

        loss_fn_BCE,
        loss_fn_CE,
        loss_fn_MSE,
        loss_fn_keypoints,
        loss_fn_mask,

        HMR_optimizer_all,
        HMR_scheduler_all,
        discriminator_optimizer,
        context_encoder_optimizer,
        jigsaw_puzzle_optimizer,
        rotation_optimizer,
        texture_net_optimizer,
        texture_discriminator_optimizer,

        device,
        num_epoch,
        args
        ):
        self.model_name = model_name
        self.args = args

        # model
        self.HMR = HMR
        self.context_encoder_net = context_encoder_net
        self.jigsaw_puzzle_net = jigsaw_puzzle_net
        self.rotation_net = rotation_net
        self.discriminator = discriminator
        self.texture_net = texture_net
        self.img_renderer = img_renderer
        self.seg_renderer = seg_renderer
        self.texture_discriminator = texture_discriminator

        # device
        self.device = device

        # dataloader
        self.train_dataloader = train_dataloader
        if test_dataloader:
            self.test_dataloader = test_dataloader
        if test_dataloader_h36m:
            self.test_dataloader_h36m = test_dataloader_h36m
        if test_dataloader_3dpw:
            self.test_dataloader_3dpw = test_dataloader_3dpw
        if test_dataloader_lsp:
            self.test_dataloader_lsp = test_dataloader_lsp
        # loss
        self.loss_fn_BCE = loss_fn_BCE
        self.loss_fn_CE = loss_fn_CE
        self.loss_fn_MSE = loss_fn_MSE
        self.loss_fn_keypoints = loss_fn_keypoints
        self.loss_fn_mask = loss_fn_mask

        # optimizer
        self.HMR_optimizer_all = HMR_optimizer_all
        self.HMR_scheduler_all = HMR_scheduler_all
        self.discriminator_optimizer = discriminator_optimizer
        self.context_encoder_optimizer = context_encoder_optimizer
        self.jigsaw_puzzle_optimizer = jigsaw_puzzle_optimizer
        self.rotation_optimizer = rotation_optimizer
        self.texture_net_optimizer = texture_net_optimizer
        self.texture_discriminator_optimizer = texture_discriminator_optimizer

        # Valiable
        self.num_epoch = num_epoch
        self.freq_print = args.freq_print
        self.num_patch = 4
        self.tex_size = args.tex_size
        today = datetime.datetime.now()
        self.today = datetime.datetime.strftime(today, "%y%m%d_%H%M%S")
        self.output_dir = self.args.output_dir if self.args.output_dir else self.today

        # smpl and J_regressor
        self.smpl_neutral = SMPL("data/vibe_data",
                        create_transl=False).to(self.device)
        self.smpl_male = SMPL("data/vibe_data",
                     gender='male',
                     create_transl=False).to(self.device)
        self.smpl_female = SMPL("data/vibe_data",
                       gender='female',
                       create_transl=False).to(self.device)
        self.J_regressor = np.load("data/vibe_data/J_regressor_h36m.npy")
        self.J_regressor_torch = torch.from_numpy(self.J_regressor).float()
        parts_texture = np.load("data/vertex_texture.npy")
        self.parts_texture = torch.from_numpy(parts_texture).to(self.device).float()
        self.cube_parts = torch.FloatTensor(np.load("data/cube_parts.npy")).to(self.device)

    def train(self):
        print("===================Train===================\nEpoch {} Start".format(self.epoch+1))
        train_template = \
            'Epoch: {}/{} | Batch_idx: {}/{} | ' \
            'loss_DC: {losses_DC.val:.4f} ({losses_DC.avg:.4f}) | loss_CE: {losses_CE.val:.4f} ({losses_CE.avg:.4f}) | ' \
            'loss_JP: {losses_JP.val:.4f} ({losses_JP.avg:.4f}) | acc_JP: {acces_JP.val:.4f} ({acces_JP.avg:.4f}) | ' \
            'loss_ROT: {losses_ROT.val:.4f} ({losses_ROT.avg:.4f}) | acc_ROT: {acces_ROT.val:.4f} ({acces_ROT.avg:.4f}) | ' \
            'loss_Texture: {losses_texture_ori_img.val:.4f} ({losses_texture_ori_img.avg:.4f}) | ' \
            'loss_Seg: {losses_seg.val:.4f} ({losses_seg.avg:.4f}) | ' \
            'loss_Texture_Total: {losses_texture_total.val:.4f} ({losses_texture_total.avg:.4f}) | ' \
            'loss_disc_e: {losses_disc_e.val:.4f} ({losses_disc_e.avg:.4f}) | ' \
            'loss_disc_d: {losses_disc.val:.4f} ({losses_disc.avg:.4f}) | ' \
            'loss_disc_real: {losses_disc_real.val:.4f} ({losses_disc_real.avg:.4f}) | ' \
            'loss_disc_fake: {losses_disc_fake.val:.4f} ({losses_disc_fake.avg:.4f}) | ' \
            'loss_HMR_3task: {losses_HMR_3task.val:.4f} ({losses_HMR_3task.avg:.4f}) | ' \
            'loss_HMR_joints3d: {losses_HMR_joints3d.val:.4f} ({losses_HMR_joints3d.avg:.4f}) | ' \
            'loss_joints: {losses_joints.val:.4f} ({losses_joints.avg:.4f}) | ' \
            'MPJPE: {train_MPJPE.val:.4f} ({train_MPJPE.avg:.4f}) | ' \
            'PA_MPJPE: {train_PA_MPJPE.val:.4f} ({train_PA_MPJPE.avg:.4f}) | ' \
            'loss_total: {losses_total.val:.4f} ({losses_total.avg:.4f}) | ' \
            'Batch duration: {duration}'

        batch_start = time.time()

        self.losses_DC = AverageMeter() # Discriminator Loss
        self.losses_CE = AverageMeter() # Context Encoder Loss
        self.losses_JP = AverageMeter() # Jigsaw Puzzle Loss
        self.acces_JP = AverageMeter() # Jigsaw Puzzle Accuracy
        self.losses_ROT = AverageMeter() # Rotation Loss
        self.acces_ROT = AverageMeter() # Rotation Accuracy
        self.losses_texture_ori_img = AverageMeter() # Texture Loss
        self.losses_seg = AverageMeter() # Segmentation Loss
        self.losses_texture_total = AverageMeter() # Texture Loss + Segmentation Loss
        self.losses_HMR_3task = AverageMeter() # Discriminator Loss + Context Encoder Loss + Roation Loss
        self.losses_disc_e = AverageMeter() # Encoder Discriminator Loss for rendering img
        self.losses_disc = AverageMeter() # Real + Fake Discriminator Loss
        self.losses_disc_real = AverageMeter() # Discriminator Loss for Real
        self.losses_disc_fake = AverageMeter() # Discriminator Loss for Fake
        self.losses_total = AverageMeter() # Total Sum Loss
        self.losses_joints = AverageMeter() # 2d + 3d joints loss
        self.losses_HMR_joints3d = AverageMeter() # 3D joints loss
        self.train_MPJPE = AverageMeter() # MPJPE
        self.train_PA_MPJPE = AverageMeter() # PA_MPJPE

        len_train_dataloader = len(self.train_dataloader)

        for batch_idx, item in tqdm.tqdm(enumerate(self.train_dataloader), desc='Train {}/{}'.format(self.epoch+1, self.num_epoch), total=len(self.train_dataloader)):
            img = item['img'].to(self.device)
            black_img = item['black_img'].to(self.device)
            context_encoder_input = item['context_encoder_input'].to(self.device)
            center_crop_img = item['center_crop_img'].to(self.device)
            jigsaw_input = item['jigsaw_input'].to(self.device)
            rotation_img = item['rotation_input'].to(self.device)
            jigsaw_order = item['jigsaw_order'].to(self.device)
            rotation_idx = item['rotation_idx'].to(self.device)
            joints3d = item['pose_3d'].to(self.device)
            has_joints3d = item['has_pose_3d'].to(self.device)
            joints2d = item['keypoints'].to(self.device)
            batch_size = img.shape[0]

            gt_mask = item['gt_mask'].to(self.device)
            has_mask = item['has_mask'].to(self.device)

            self.zeros = torch.zeros([batch_size, 1]).to(self.device)
            self.ones = torch.ones([batch_size, 1]).to(self.device)

            faces = get_smpl_faces().astype(np.int32)
            faces = torch.from_numpy(faces).to(self.device)
            faces = faces.expand((batch_size, -1, -1))

            joint_mapper_gt = constants.J24_TO_J17 if self.args.test_dataset == 'mpi-inf-3dhp' else constants.J24_TO_J14

            if self.epoch < self.args.first_stage_nEpoch: # Epoch 0~9
                # Training 3 Task net
                # Discriminator, Context Encoder, Jigsaw Puzzle, Rotation net
                self.HMR.eval()
                self.context_encoder_net.train()
                self.discriminator.train()
                self.jigsaw_puzzle_net.train()
                self.rotation_net.train()
                output_ce_224 = \
                    train_only_3task_network(
                        self.HMR,
                        self.context_encoder_net,
                        self.discriminator,
                        self.jigsaw_puzzle_net,
                        self.rotation_net,

                        self.loss_fn_BCE,
                        self.loss_fn_MSE,
                        self.loss_fn_CE,

                        self.losses_CE,
                        self.losses_DC,
                        self.acces_JP,
                        self.losses_JP,
                        self.acces_ROT,
                        self.losses_ROT,

                        self.discriminator_optimizer,
                        self.context_encoder_optimizer,
                        self.jigsaw_puzzle_optimizer,
                        self.rotation_optimizer,

                        img,
                        context_encoder_input,
                        center_crop_img,
                        jigsaw_input,
                        jigsaw_order,
                        rotation_img,
                        rotation_idx,

                        self.num_patch,
                        self.ones,
                        self.zeros,
                        batch_size,
                    )

                # Training Texture Net
                self.texture_net.train()
                output_train_texture_net = \
                    train_texture_net(
                        self.HMR,
                        self.texture_net,
                        self.img_renderer,

                        self.loss_fn_MSE,
                        self.loss_fn_mask,

                        self.losses_texture_ori_img,
                        self.losses_seg,
                        self.losses_texture_total,

                        self.texture_net_optimizer,

                        img,
                        black_img,

                        batch_size,
                        self.args,
                        gt_mask,
                        has_mask,
                        train_first_stage=True
                    )

                mask = output_train_texture_net[0]
                detach_images = output_train_texture_net[1]
                rendering = output_train_texture_net[2]
                vertices = output_train_texture_net[3]

            # train hmr & texture net using 3task, rendering, segmentation and gan loss (or joints)
            else: # Epoch 10~19
                self.HMR.eval()
                self.context_encoder_net.eval()
                self.discriminator.eval()
                self.jigsaw_puzzle_net.eval()
                self.rotation_net.eval()
                self.texture_net.train()
                self.texture_discriminator.train()

                # Training Mesh network (HMR) using 3 Task Loss
                loss_HMR, output_ce_224 = \
                    train_hmr_using_3task(
                        self.HMR,
                        self.context_encoder_net,
                        self.discriminator,
                        self.jigsaw_puzzle_net,
                        self.rotation_net,

                        self.loss_fn_BCE,
                        self.loss_fn_MSE,
                        self.loss_fn_CE,

                        self.losses_CE,
                        self.acces_JP,
                        self.losses_JP,
                        self.acces_ROT,
                        self.losses_ROT,
                        self.losses_HMR_3task,

                        img,
                        context_encoder_input,
                        center_crop_img,
                        jigsaw_input,
                        jigsaw_order,
                        rotation_img,
                        rotation_idx,

                        self.num_patch,
                        self.ones,
                        self.zeros,
                        batch_size,
                        self.args
                    )

                loss_all = loss_HMR

                # Trainign Texture net
                output_train_texture_net = \
                    train_texture_net(
                        self.HMR,
                        self.texture_net,
                        self.img_renderer,

                        self.loss_fn_MSE,
                        self.loss_fn_mask,

                        self.losses_texture_ori_img,
                        self.losses_seg,
                        self.losses_texture_total,

                        self.texture_net_optimizer,

                        img,
                        black_img,

                        batch_size,
                        self.args,
                        gt_mask,
                        has_mask,
                        train_first_stage=False
                    )

                texture_loss = output_train_texture_net[0]
                loss_all += texture_loss

                mask = output_train_texture_net[1]
                detach_images = output_train_texture_net[2]
                rendering = output_train_texture_net[3]
                vertices = output_train_texture_net[4]

                # Trining HMR using adversarial loss
                e_disc_loss, d_disc_loss, rendering_bg = \
                    train_hmr_using_adv_loss(
                        self.HMR,
                        self.texture_discriminator,
                        self.texture_net,
                        self.img_renderer,

                        self.losses_disc_e,
                        self.losses_disc,
                        self.losses_disc_real,
                        self.losses_disc_fake,

                        img,

                        batch_size,
                    )

                loss_all += e_disc_loss
                self.texture_discriminator_optimizer.zero_grad()
                d_disc_loss.backward()
                self.texture_discriminator_optimizer.step()

                if not self.args.self_supervised:
                    # Training Mesh network (HMR) using joints
                    joints_loss, mpjpe, pa_mpjpe, num_data = train_hmr_using_joints(
                        self.HMR,

                        self.loss_fn_keypoints,

                        self.losses_HMR_joints3d,

                        img,
                        joints2d,
                        joints3d,
                        has_joints3d,

                        joint_mapper_gt,
                        batch_size,
                        self.device,
                        self.args,
                    )

                    loss_all += joints_loss

                    self.losses_joints.update(joints_loss.item(), num_data)
                    self.train_MPJPE.update(mpjpe.item(), num_data)
                    self.train_PA_MPJPE.update(pa_mpjpe.item(), num_data)

                self.HMR_optimizer_all.zero_grad()
                self.losses_total.update(loss_all.item(), batch_size)
                loss_all.backward()
                self.HMR_optimizer_all.step()

            if (batch_idx==0 or (batch_idx+1)%self.args.freq_print==0):
                for i in range(10 if batch_size > 10 else batch_size):
                    img_dict = dict()
                    ### original img ###
                    img_dict["orig_img.jpg"] = img[i]

                    ### context encoder input img ###
                    img_dict["ce_input_img.jpg"] = context_encoder_input[i].clone().detach()
                    ### center crop img for CE  ###
                    img_dict["center_crop_img.jpg"] = center_crop_img[i].clone().detach()
                    ### output img of CE ###
                    img_dict["reconst_img.jpg"] = output_ce_224[i].clone().detach()
                    ### jigsaw input img ###
                    img_dict["jigsaw_input_img.jpg"] = jigsaw_input[i].clone().detach()
                    ### ratation input img ###
                    img_dict["rotation_input_img.jpg"] = rotation_img[i].clone().detach()

                    ### texture img ###
                    img_dict["rendering.jpg"] = rendering[i].clone().detach()
                    ### segmentation img ###
                    img_dict["mask.jpg"] = mask[i].clone().detach()
                    ### detach img ###
                    img_dict["detach.jpg"] = detach_images[i].clone().detach()
                    ### Segmentation gt ###
                    img_dict["seg_gt.jpg"] = gt_mask[i].clone().detach()
                    if self.epoch >= self.args.first_stage_nEpoch:
                        ### rendering background ###
                        img_dict["rendering_bg.jpg"] = rendering_bg[i].clone().detach()

                    save_all_img(img_dict, self.output_dir,
                                self.epoch+1, i+batch_idx)

                    ### save mesh ###
                    if self.epoch >= self.args.first_stage_nEpoch:
                        _faces = faces[i].clone().detach()
                        _vertices = vertices[i].clone().detach()
                        save_mesh(
                            _vertices, _faces,
                            self.output_dir,
                            self.epoch+1,
                            i,
                        )
            ### print train info while running in batch loop ###
            if (batch_idx+1) % self.freq_print == 0 or (batch_idx+1) == len_train_dataloader:
                train_template_filled = train_template.format(
                    self.epoch+1, self.num_epoch,
                    batch_idx+1, len(self.train_dataloader),
                    losses_DC=self.losses_DC,
                    losses_CE=self.losses_CE,
                    losses_JP=self.losses_JP,
                    acces_JP=self.acces_JP,
                    losses_ROT=self.losses_ROT,
                    acces_ROT=self.acces_ROT,
                    losses_texture_ori_img=self.losses_texture_ori_img,
                    losses_seg=self.losses_seg,
                    losses_texture_total=self.losses_texture_total,
                    losses_disc_e=self.losses_disc_e,
                    losses_disc=self.losses_disc,
                    losses_disc_real=self.losses_disc_real,
                    losses_disc_fake=self.losses_disc_fake,
                    losses_HMR_3task=self.losses_HMR_3task,
                    losses_HMR_joints3d=self.losses_HMR_joints3d,
                    losses_joints=self.losses_joints,
                    losses_total=self.losses_total,
                    train_MPJPE=self.train_MPJPE,
                    train_PA_MPJPE=self.train_PA_MPJPE,
                    duration=get_HHMMSS_from_second(seconds=(time.time()-batch_start))
                )
                print(train_template_filled)
                self.train_templates.append(train_template_filled)
                if (batch_idx+1) == len_train_dataloader:
                    self.train_templates.append("======================================================================")
                batch_start = time.time()

        ### save train info when one epoch is completed ###
        save_templates_info(self.train_templates, self.output_dir, "train_templates.txt")

        print("Train Time: {train_time}, Total Time: {total_time}".format(
                train_time=get_HHMMSS_from_second(seconds=(time.time()-self.epoch_start)),
                total_time=get_HHMMSS_from_second(seconds=(time.time()-self.total_start))))

    ### Evaluate ###
    def evaluate(self, test_dataloader, test_dataset_name, is_save_pth=True):
        joint_mapper_h36m = constants.H36M_TO_J17 if self.args.test_dataset == 'mpi-inf-3dhp' else constants.H36M_TO_J14
        joint_mapper_gt = constants.J24_TO_J17 if self.args.test_dataset == 'mpi-inf-3dhp' else constants.J24_TO_J14
        test_start = time.time()
        self.HMR.eval()

        if self.args.train == 0:
            if test_dataset_name == "h36m" or test_dataset_name == "3dpw":
                test_template_batch = \
                    '===================Test===================\n' \
                    'Batch: {}/{} | ' \
                    'Mpjpe: {mpjpe_average_meter.avg:.2f} ' \
                    'Rec_error: {pa_mpjpe_average_meter.avg:.2f} ' \
                    '\n=========================================='

            elif test_dataset_name == "lsp":
                test_template_batch = \
                    '===================Test===================\n' \
                    'Batch: {}/{} | ' \
                    'Part acc: {part_Acc_average_meter:.2f} ' \
                    'Part F1: {part_F1_average_meter:.2f} ' \
                    'FG-BG Acc: {Acc_average_meter:.2f} ' \
                    'FG-BG F1: {F1_average_meter:.2f} ' \
                    '\n=========================================='

        if test_dataset_name == "h36m":
            test_template = \
                '===================Test===================\n' \
                'Test Data: {} | ' \
                'Epoch: {}/{} | ' \
                'Mpjpe: {mpjpe_average_meter.avg:.2f} | ' \
                'Rec_error: {pa_mpjpe_average_meter.avg:.2f} | ' \
                'loss: {losses.avg:.5f} | ' \
                'Test Time: {test_time} | Epoch Time: {epoch_time} | ' \
                'Total Time: {total_time}' \
                '\n=========================================='

        elif test_dataset_name == "3dpw":
            test_template = \
                '===================Test===================\n' \
                'Test Data: {} | ' \
                'Epoch: {}/{} | ' \
                'Mpjpe: {mpjpe_average_meter.avg:.2f} | ' \
                'Rec_error: {pa_mpjpe_average_meter.avg:.2f} | ' \
                'Test Time: {test_time} | Epoch Time: {epoch_time} | ' \
                'Total Time: {total_time}' \
                '\n=========================================='

        elif test_dataset_name == "lsp":
            test_template = \
                '===================Test===================\n' \
                'Test Data: {} | ' \
                'Epoch: {}/{} | ' \
                'Part acc: {part_Acc_average_meter:.2f} | ' \
                'Part F1: {part_F1_average_meter:.2f} | ' \
                'FG-BG Acc: {Acc_average_meter:.2f} | ' \
                'FG-BG F1: {F1_average_meter:.2f} | ' \
                'Test Time: {test_time} | Epoch Time: {epoch_time} | ' \
                'Total Time: {total_time}' \
                '\n=========================================='

        self.mpjpe_average_meter = AverageMeter()
        self.pa_mpjpe_average_meter = AverageMeter()
        self.losses = AverageMeter()
        self.part_Acc_average_meter = 0
        self.part_F1_average_meter = 0
        self.Acc_average_meter = 0
        self.F1_average_meter = 0

        if self.args.train != 0:
            if test_dataset_name == "h36m":
                current_mpjpe = self.current_mpjpe_h36m
            elif test_dataset_name == "3dpw":
                current_mpjpe = self.current_mpjpe_3dpw
            elif test_dataset_name == "lsp":
                current_acc = self.current_acc_lsp

        batch_num = len(test_dataloader)

        accuracy = 0.
        parts_accuracy = 0.

        pixel_count = 0
        parts_pixel_count = 0

        tp = np.zeros((2,1))
        fp = np.zeros((2,1))
        fn = np.zeros((2,1))

        parts_tp = np.zeros((7,1))
        parts_fp = np.zeros((7,1))
        parts_fn = np.zeros((7,1))

        with torch.no_grad():
            for batch_idx, item in tqdm.tqdm(enumerate(test_dataloader), desc='{} Eval'.format(test_dataset_name), total=len(test_dataloader)):
                # Validation for early stopping
                if test_dataset_name == "h36m":
                    img = item["img"]
                    img = img.to(self.device)
                    batch_size = img.shape[0]

                    output = self.HMR(img, J_regressor=self.J_regressor)
                    output = output[-1]

                    pred_j3ds = output['kp_3d']
                    pred_pelvis = (pred_j3ds[:,[2],:] + pred_j3ds[:,[3],:]) / 2.0
                    pred_j3ds -= pred_pelvis

                    target_j3ds = item["pose_3d"]
                    target_j3ds = target_j3ds[:, joint_mapper_gt, :-1]
                    target_j3ds = target_j3ds.float().to(self.device)
                    target_pelvis = (target_j3ds[:,[2],:] + target_j3ds[:,[3],:]) / 2.0
                    target_j3ds -= target_pelvis

                    loss = self.loss_fn_MSE(pred_j3ds, target_j3ds)
                    self.losses.update(loss.item(), batch_size)

                if test_dataset_name == "h36m" or test_dataset_name == "3dpw":
                    img = item["img"]
                    img = img.to(self.device)
                    batch_size = img.shape[0]

                    output = self.HMR(img, J_regressor=self.J_regressor)
                    output = output[-1]

                    pred_j3ds = output['kp_3d']
                    pred_pelvis = (pred_j3ds[:,[2],:] + pred_j3ds[:,[3],:]) / 2.0
                    pred_j3ds -= pred_pelvis
                    pred_vertices = output["verts"]

                    faces = get_smpl_faces().astype(np.int32)
                    faces = torch.from_numpy(faces).to(self.device)
                    faces = faces.expand((batch_size, -1, -1))

                    J_regressor_batch = self.J_regressor_torch[None, :].expand(pred_vertices.shape[0], -1, -1).to(self.device)
                    if test_dataset_name == 'h36m':
                        target_j3ds = item["pose_3d"]
                        target_j3ds = target_j3ds[:, joint_mapper_gt, :-1]
                    else:
                        gt_pose = item["pose"].to(self.device)
                        gt_betas = item["betas"].to(self.device)
                        gender = item["gender"].to(self.device)
                        gt_vertices = self.smpl_male(global_orient=gt_pose[:,:3], body_pose=gt_pose[:,3:], betas=gt_betas).vertices
                        gt_vertices_female = self.smpl_female(global_orient=gt_pose[:,:3], body_pose=gt_pose[:,3:], betas=gt_betas).vertices
                        gt_vertices[gender==1, :, :] = gt_vertices_female[gender==1, :, :]
                        target_j3ds = torch.matmul(J_regressor_batch, gt_vertices)
                        gt_pelvis = target_j3ds[:, [0],:].clone()
                        target_j3ds = target_j3ds[:, joint_mapper_h36m, :]
                        target_j3ds = target_j3ds - gt_pelvis

                    target_j3ds = target_j3ds.float().to(self.device)
                    target_pelvis = (target_j3ds[:,[2],:] + target_j3ds[:,[3],:]) / 2.0
                    target_j3ds -= target_pelvis
                    errors = torch.sqrt(((pred_j3ds - target_j3ds) ** 2).sum(dim=-1)).mean(dim=-1).cpu().numpy()
                    S1_hat = batch_compute_similarity_transform_torch(pred_j3ds, target_j3ds)
                    errors_pa = torch.sqrt(((S1_hat - target_j3ds) ** 2).sum(dim=-1)).mean(dim=-1).cpu().numpy()
                    m2mm = 1000

                    mpjpe = np.mean(errors) * m2mm
                    pa_mpjpe = np.mean(errors_pa) * m2mm
                    self.mpjpe_average_meter.update(mpjpe, batch_size)
                    self.pa_mpjpe_average_meter.update(pa_mpjpe, batch_size)

                elif test_dataset_name == "lsp":
                    annot_path = config.DATASET_FOLDERS['upi-s1h']
                    img = item["img"]
                    img = img.to(self.device)
                    batch_size = img.shape[0]
                    orig_shape = item["orig_shape"].cpu().numpy()
                    scale = item["scale"].cpu().numpy()
                    center = item["center"].cpu().numpy()

                    output = self.HMR(img, J_regressor=self.J_regressor)
                    output = output[-1]
                    pred_vertices = output["verts"]
                    cam = output['theta'][:, :3]

                    faces = get_smpl_faces().astype(np.int32)
                    faces = torch.from_numpy(faces).to(self.device)
                    faces = faces.expand((batch_size, -1, -1))

                    mask, parts = self.seg_renderer(pred_vertices, cam)
                    save_gt_parts = []
                    save_gt_seg = []

                    for i in range(batch_size):
                        # After rendering, convert imate back to original resolution
                        pred_mask = uncrop(mask[i].cpu().numpy(), center[i], scale[i], orig_shape[i]) > 0
                        # Load gt mask
                        gt_mask = cv2.imread(os.path.join(annot_path, item['maskname'][i]), 0) > 0
                        if batch_idx == 0:
                            save_gt_seg.append(gt_mask)
                        # Evaluation consistent with the original UP-3D code
                        accuracy += (gt_mask == pred_mask).sum()
                        pixel_count += np.prod(np.array(gt_mask.shape))
                        for c in range(2):
                            cgt = gt_mask == c
                            cpred = pred_mask == c
                            tp[c] += (cgt & cpred).sum()
                            fp[c] +=  (~cgt & cpred).sum()
                            fn[c] +=  (cgt & ~cpred).sum()
                        f1 = 2 * tp / (2 * tp + fp + fn)

                    for i in range(batch_size):
                        pred_parts = uncrop(parts[i].cpu().numpy().astype(np.uint8), center[i], scale[i], orig_shape[i])
                        # Load gt part segmentation
                        gt_parts = cv2.imread(os.path.join(annot_path, item['partname'][i]), 0)
                        if batch_idx == 0:
                            save_gt_parts.append(gt_parts)
                        # Evaluation consistent with the original UP-3D code
                        # 6 parts + background
                        for c in range(7):
                            cgt = gt_parts == c
                            cpred = pred_parts == c
                            cpred[gt_parts == 255] = 0
                            parts_tp[c] += (cgt & cpred).sum()
                            parts_fp[c] +=  (~cgt & cpred).sum()
                            parts_fn[c] +=  (cgt & ~cpred).sum()
                        gt_parts[gt_parts == 255] = 0
                        pred_parts[pred_parts == 255] = 0
                        parts_f1 = 2 * parts_tp / (2 * parts_tp + parts_fp + parts_fn)
                        parts_accuracy += (gt_parts == pred_parts).sum()
                        parts_pixel_count += np.prod(np.array(gt_parts.shape))

                    self.part_Acc_average_meter = (parts_accuracy / parts_pixel_count) * 100
                    self.part_F1_average_meter = parts_f1[[0,1,2,3,4,5,6]].mean()

                    self.Acc_average_meter = (accuracy / pixel_count) * 100
                    self.F1_average_meter = f1.mean()

                if batch_idx == 0:
                    for i in range(10 if batch_size > 10 else batch_size):
                        img_dict = dict()
                        img_dict["orig_img.jpg"] = img[i]

                        if test_dataset_name == "h36m" or test_dataset_name == "3dpw":
                            save_joints3d_img(
                                target_j3ds[i],
                                pred_j3ds[i],
                                self.output_dir,
                                self.epoch+1,
                                test_dataset=test_dataset_name,
                                test_idx=i
                            )
                            save_mesh(
                                pred_vertices[i],
                                faces[i],
                                self.output_dir,
                                self.epoch+1,
                                test_dataset=test_dataset_name,
                                test_idx=i
                            )

                        elif test_dataset_name == "lsp":
                            img_dict["mask.jpg"] = mask[i].cpu().numpy()
                            img_dict["parts.jpg"] = parts[i].cpu().numpy()
                            img_dict["gt_parts.jpg"] = save_gt_parts[i]
                            img_dict["gt_seg.jpg"] = save_gt_seg[i]

                        save_all_img(img_dict, self.output_dir, self.epoch+1, test_dataset=test_dataset_name, test_idx=i)

                if self.args.train == 0 and (batch_idx+1) % self.args.freq_print_test == 0:
                    if test_dataset_name == "h36m" or test_dataset_name == "3dpw":
                        test_template_batch_filled = test_template_batch.format(
                            batch_idx+1,
                            batch_num,
                            mpjpe_average_meter=self.mpjpe_average_meter,
                            pa_mpjpe_average_meter=self.pa_mpjpe_average_meter,
                        )

                    elif test_dataset_name == "lsp":
                        test_template_batch_filled = test_template_batch.format(
                            batch_idx+1,
                            batch_num,
                            part_Acc_average_meter=self.part_Acc_average_meter,
                            part_F1_average_meter=self.part_F1_average_meter,
                            Acc_average_meter=self.Acc_average_meter,
                            F1_average_meter=self.F1_average_meter,
                        )
                    print(test_template_batch_filled)

            if test_dataset_name == "h36m" or test_dataset_name == "3dpw":
                test_template_filled = test_template.format(
                    test_dataset_name,
                    self.epoch+1,
                    self.num_epoch,
                    mpjpe_average_meter=self.mpjpe_average_meter,
                    pa_mpjpe_average_meter=self.pa_mpjpe_average_meter,
                    losses=self.losses,
                    test_time=get_HHMMSS_from_second(seconds=(time.time()-test_start)),
                    epoch_time=get_HHMMSS_from_second(seconds=(time.time()-self.epoch_start)),
                    total_time=get_HHMMSS_from_second(seconds=(time.time()-self.total_start))
                )

            elif test_dataset_name == "3dpw":
                test_template_filled = test_template.format(
                    test_dataset_name,
                    self.epoch+1,
                    self.num_epoch,
                    mpjpe_average_meter=self.mpjpe_average_meter,
                    pa_mpjpe_average_meter=self.pa_mpjpe_average_meter,
                    test_time=get_HHMMSS_from_second(seconds=(time.time()-test_start)),
                    epoch_time=get_HHMMSS_from_second(seconds=(time.time()-self.epoch_start)),
                    total_time=get_HHMMSS_from_second(seconds=(time.time()-self.total_start))
                )

            elif test_dataset_name == "lsp":
                test_template_filled = test_template.format(
                    test_dataset_name,
                    self.epoch+1,
                    self.num_epoch,
                    part_Acc_average_meter=self.part_Acc_average_meter,
                    part_F1_average_meter=self.part_F1_average_meter,
                    Acc_average_meter=self.Acc_average_meter,
                    F1_average_meter=self.F1_average_meter,
                    test_time=get_HHMMSS_from_second(seconds=(time.time()-test_start)),
                    epoch_time=get_HHMMSS_from_second(seconds=(time.time()-self.epoch_start)),
                    total_time=get_HHMMSS_from_second(seconds=(time.time()-self.total_start))
                )
            print(test_template_filled)

            # save test templates info txt file
            if self.args.train != 0:
                if test_dataset_name == "h36m":
                    self.test_templates_h36m.append(test_template_filled)
                    templates_filename = "test_templates_h36m.txt"
                    save_templates_info(self.test_templates_h36m, self.output_dir, templates_filename)

                elif test_dataset_name == "3dpw":
                    self.test_templates_3dpw.append(test_template_filled)
                    templates_filename = "test_templates_3dpw.txt"
                    save_templates_info(self.test_templates_3dpw, self.output_dir, templates_filename)

                elif test_dataset_name == "lsp":
                    self.test_templates_lsp.append(test_template_filled)
                    templates_filename = "test_templates_lsp.txt"
                    save_templates_info(self.test_templates_lsp, self.output_dir, templates_filename)

            else:
                self.test_templates.append(test_template_filled)
                templates_filename = "test_templates.txt"
                save_templates_info(self.test_templates, self.output_dir, templates_filename)

        # save pth file
        if is_save_pth:
            if self.epoch >= self.args.first_stage_nEpoch:
                if test_dataset_name == "h36m" or test_dataset_name == "3dpw":
                    # Save best pth
                    if self.mpjpe_average_meter.avg < current_mpjpe:
                        print("MPJPE changes from {:.4f} to {:.4f}".format(current_mpjpe, self.mpjpe_average_meter.avg))
                        self.save_checkpoint_all(test_dataset_name, best=True)
                    else:
                        print("MPJPE doesn't change {:.4f}".format(current_mpjpe))
                elif test_dataset_name == "lsp":
                    # Save best pth
                    if self.Acc_average_meter > current_acc:
                        print("ACC changes from {:.4f} to {:.4f}".format(current_acc, self.Acc_average_meter))
                        self.save_checkpoint_all(test_dataset_name, best=True)
                    else:
                        print("ACC doesn't change {:.4f}".format(current_acc))

            # Save lastest pth
            if self.args.save_pth_all_epoch:
                self.save_checkpoint_all(test_dataset_name, save_all_epoch=True)
            else:
                self.save_checkpoint_all(test_dataset_name)

    def save_checkpoint_all(self, test_dataset_name, best=False, save_all_epoch=False):
        """
            save pth file
        """
        filename = "best_{}_{}.pth" if best else "{}.pth"
        if save_all_epoch and not best:
            filename = "{}_{}_epoch"+str(self.epoch+1)+".pth"

        ### 3task network save pth, texture net save pth ###
        if self.epoch < self.args.first_stage_nEpoch:
            save_checkpoint({
                "state_dict": self.context_encoder_net.state_dict(),
                "loss": self.losses_CE.avg,
                "optimizer": self.context_encoder_optimizer.state_dict()
            },
            self.output_dir,
            filename.format("context_encoder_net", test_dataset_name))

            save_checkpoint({
                "state_dict": self.discriminator.state_dict(),
                "loss": self.losses_DC.avg,
                "optimizer": self.discriminator_optimizer.state_dict()
            },
            self.output_dir,
            filename.format("discriminator", test_dataset_name))

            save_checkpoint({
                "state_dict": self.jigsaw_puzzle_net.state_dict(),
                "accuracy": self.acces_JP.avg,
                "optimizer": self.jigsaw_puzzle_optimizer.state_dict()
            },
            self.output_dir,
            filename.format("jigsaw_puzzle_net", test_dataset_name))

            save_checkpoint({
                "state_dict": self.rotation_net.state_dict(),
                "accuracy": self.acces_ROT.avg,
                "optimizer": self.rotation_optimizer.state_dict()
            },
            self.output_dir,
            filename.format("rotation_net", test_dataset_name))

            save_checkpoint({
                "state_dict": self.texture_net.state_dict(),
                "loss": self.losses_texture_ori_img.avg,
                "optimizer": self.texture_net_optimizer.state_dict(),
            },
            self.output_dir,
            filename.format("texture_net", test_dataset_name))

        ### hmr save pth ###
        else:
            save_checkpoint({
                "state_dict": self.HMR.state_dict(),
                "loss_joints3d": self.losses_HMR_joints3d.avg,
                "loss_3task": self.losses_HMR_3task.avg,
                "optimizer_joints": self.HMR_optimizer_all.state_dict(),
            },
            self.output_dir,
            filename.format("hmr", test_dataset_name))

            save_checkpoint({
                "state_dict": self.texture_discriminator.state_dict(),
                "loss": self.losses_disc.avg,
                "optimizer": self.texture_discriminator_optimizer.state_dict()
            },
            self.output_dir,
            filename.format("texture_discriminator", test_dataset_name))

        if best:
            if test_dataset_name == "h36m":
                self.current_mpjpe_h36m = self.mpjpe_average_meter.avg
            elif test_dataset_name == "3dpw":
                self.current_mpjpe_3dpw = self.mpjpe_average_meter.avg
            elif test_dataset_name == "lsp":
                self.current_acc_lsp = self.Acc_average_meter

    def fit(self):
        if self.args.train != 0:
            self.current_mpjpe_h36m = math.inf
            self.current_mpjpe_3dpw = math.inf
            self.current_acc_lsp = 0

            self.train_loss = math.inf

            self.total_start = time.time()
            self.train_templates = list()
            self.test_templates_h36m = list()
            self.test_templates_3dpw = list()
            self.test_templates_lsp = list()

            for epoch in range(self.num_epoch):
                self.epoch = epoch
                self.epoch_start = time.time()
                if epoch == 0 and self.args.first_eval:
                    self.epoch = -1
                    self.evaluate(self.test_dataloader_h36m, test_dataset_name="h36m", is_save_pth=False)
                    # self.evaluate(self.test_dataloader_3dpw, test_dataset_name="3dpw", is_save_pth=False)
                    # self.evaluate(self.test_dataloader_lsp, test_dataset_name="lsp", is_save_pth=False)
                    self.epoch = epoch
                print("HMR_optimizer_joints lr:", self.HMR_scheduler_all.get_lr())

                if self.epoch == self.args.first_stage_nEpoch:
                    texture_discriminator_checkpoint = torch.load(os.path.join("results", self.args.output_dir, "save_pth", "discriminator.pth"), map_location=self.device)
                    self.texture_discriminator.load_state_dict(texture_discriminator_checkpoint["state_dict"])

                self.train()

                if self.epoch >= self.args.first_stage_nEpoch:
                    self.HMR_scheduler_all.step()
                if (epoch+1)%self.args.freq_eval == 0 or (epoch+1) == self.num_epoch:
                    self.evaluate(self.test_dataloader_h36m, test_dataset_name="h36m")
                    # self.evaluate(self.test_dataloader_3dpw, test_dataset_name="3dpw")
                    # self.evaluate(self.test_dataloader_lsp, test_dataset_name="lsp")

        else:
            self.epoch = 0
            self.num_epoch = 1
            self.total_start = time.time()
            self.epoch_start = time.time()
            self.test_templates = list()

            self.evaluate(self.test_dataloader, test_dataset_name=self.args.test_dataset.replace("-p2", ""), is_save_pth=False)


from django.core.management.base import BaseCommand
from import_data.models import OuraMember, FitbitMember, GoogleFitMember
from retrospective.tasks import (
    update_fitbit_data,
    update_oura_data,
    update_googlefit_data,
)
import time
import requests

class Command(BaseCommand):
    help = "Updates all data for all members"

    def handle(self, *args, **options):
        # cheat to wake up sleeping worker
        requests.get("https://oh-oura-connect.herokuapp.com/")

        oura_users = OuraMember.objects.all()
        for o in oura_users:
            update_oura_data.delay(o.id)
            print("submitted oura update for {}".format(o.id))
            time.sleep(2)

        fitbit_users = FitbitMember.objects.all()
        for f in fitbit_users:
            update_fitbit_data.delay(f.id)
            print("submitted fitbit update for {}".format(f.id))
            time.sleep(2)

        gf_users = GoogleFitMember.objects.all()
        for g in gf_users:
            update_googlefit_data.delay(g.user.oh_id, g.user.user.id)
            print("submitted googlefit update for {}".format(g.id))
            time.sleep(2)


from hashlib import sha1
from json import dump
from os import makedirs

apps = {
    'apps': [
        'club.postdata.covid19cuba',
        'com.codestrange.www.cuba_weather',
        'com.cubanopensource.todo',
    ]
}

def main():
    result = {}
    makedirs('api', exist_ok=True)
    with open(f'api/apps.json', mode='w', encoding='utf-8') as file:
        dump(apps, file, ensure_ascii=False)
    with open('api/apps.json', encoding='utf-8') as file:
        text = file.read()
        cache = sha1(text.encode())
        result['hash'] = cache.hexdigest()
    with open(f'api/apps_hash.json', mode='w', encoding='utf-8') as file:
        dump(result, file, ensure_ascii=False)

if __name__ == '__main__':
    main()


import array
import binascii
import configparser
import datetime
import io
import logging
import os
import signal
import sys
import time

try:
	import serial
except ImportError:
	pass
	ModulSerialMissing = True

################################################################################
# Constants
BUILDVERSION 	= "V1.0.0"
BUILDDATE 		= "2017-10-22"

################################################################################
# classes / structs

class mondata:
	def __init__(self):
		self.viewmode=0
		self.Lights="0"
		self.SC1TX="0"
		self.percAssist="000"
		self.AWD="0"
		self.C10="0"
		self.Voltage="000"
		self.Current="000"
		self.SC1RX="0"
		self.SC2="00"
		self.Speed="000"
		self.D1618="000"
		self.D1921="000"
		self.D2224="000"
		self.D2527="000"
		self.wsize="0"
		self.TX=""
		self.RX=""
		self.PLIST=["---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---","---"]

################################################################################
# Import external functions

import lib.message	as msg
import lib.config	as cfg
import lib.osys		as osy
import arg		as arg

#	TODO:
#	- TX RX Handlers
#	- TX RX Parsers
#	- Basic Logging
#	- Basic TX Construction mode (Parameter Reading P00 P59)
#	- PDCURSES?

################################################################################
# Functions

def signal_handler(signal, frame):
	sys.exit(0)

################################################################################
# beginn main

# init global vars
monitordata= mondata()
mondata.viewmode=0
signal.signal(signal.SIGINT, signal_handler)

cfg.read(cfg)

if len(sys.argv) == 1:
	print('\npye-motion ' + BUILDVERSION + ' - ' + BUILDDATE)

	# check for modules which might not be part of the standard python 3 installation
	if 'ModulSerialMissing' in locals():
		print('Missing Module pyserial. Install by typing pye-motion -install')

	print('No command line argument given. type pye-motion -help for valid arguments')

if len(sys.argv) != 1:
	if (sys.argv[1] in ("-help")):
		arg.help()
		exit()

	elif (sys.argv[1] in ( "-install")):
		arg.install()
		exit()

	elif (sys.argv[1] in ( "-listen")):
		msg.serialOpen(cfg)
		arg.listen(monitordata)

	elif (sys.argv[1] in ( "-plisten")):
		print("warning: This modus requires to set the LCD into settings mode first. ")
		print("Hold + and - simultaneously to enter settings. ")
		rawtx = input("press enter to continue")
		msg.serialOpen(cfg)
		arg.plisten(monitordata)

	elif (sys.argv[1] in ( "-pquery")):
		print("warning: This modus requires to set the LCD into settings mode first. ")
		print("Hold + and - simultaneously to enter settings. ")
		rawtx = input("press enter to continue")
		msg.serialOpen(cfg)
		arg.pquery(monitordata)

	elif (sys.argv[1] in ( "-speedlimit")):
		print("warning: This modus requires to set the LCD into settings mode first. ")
		print("Hold + and - simultaneously to enter settings. ")
		rawtx = input("press enter to continue")
		msg.serialOpen(cfg)
		if len(sys.argv) == 3:
			arg.speedlimit(monitordata, sys.argv[2])
		else:
			arg.speedlimit(monitordata, 0)
		exit()

	else:
		print('Invalid command line argument given. type pye-motion - help for valid arguments')

# sample code for opening, sending, receiving and closing comport
#ser = serial.Serial(port_A, py pybaudrate=baud_A, timeout=1)  # open first serial port
#print ("Port opened: " + ser.portstr)       # check which port was really used
#ser.write("hello world".encode("utf-8"))      # write a string
#receive = ser.read(11)
#print (receive.decode("utf-8"))
#ser.close()             # close port


import logging
import sys

import click
import requests
from bs4 import BeautifulSoup
from telegram.ext import CommandHandler, Filters, MessageHandler, Updater
from tinydb import Query, TinyDB

db = TinyDB("db.json")
Job = Query()

TELEGRAM_BOT_TOKEN = None

class JobExistsException(Exception):
    pass

def parse_result_item(item):
    """
    Takes a li item containing one search result and parses id, url and price from it.
    Returns a dict containing the results.
    """
    main = item.find_all("div", {"aditem-main"})
    price = item.find_all("p", {"aditem-main--middle--price"})
    article = item.find_all("article")

    if len(main) != 1 or len(article) != 1 or len(price) != 1:
        return
    main = main[0]
    article = article[0]
    price = price[0]

    result = {
        "ad_id": article["data-adid"],
        "price": price.text.strip(),
    }

    a = main.find_all("a")[0]
    result["url"] = "https://www.ebay-kleinanzeigen.de" + a["href"]

    return result

def execute_search(search_term):
    """
    Runs the search for one search term.
    Returns a list containing all parsed search results.
    """
    headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"
    }
    url = f"https://www.ebay-kleinanzeigen.de/s-79249/{search_term}/k0l9364r20"

    response = requests.get(url, headers=headers)

    soup = BeautifulSoup(response.content, features="html.parser")

    ul = soup.find_all("ul", {"id": "srchrslt-adtable"})
    assert len(ul) == 1

    ul = ul[0]

    items = ul.find_all("li")

    results = []
    for i in items:
        data = parse_result_item(i)
        if data is not None:
            results.append(data)
    if len(results) == 0:
        logging.warning(
            f"No results found for search term '{search_term}'. Check if parser works correctly."
        )
    return results

def init_search(search_term, chat_id):
    """
    Initialize a new search term.
    Executes one search and marks all current results as known.
    """
    result = db.search(Job.search_term == search_term)

    if result:
        raise JobExistsException

    initial_results = execute_search(search_term)
    ids = [_["ad_id"] for _ in initial_results]
    db.insert({"search_term": search_term, "chat_id": chat_id, "known_ads": ids})

def echo(update, context):
    context.bot.send_message(chat_id=update.effective_chat.id, text=update.message.text)

def start_watching(update, context):
    """
    Command handler for starting to watch a new search term.
    """
    search_target = "".join(context.args)
    try:
        init_search(search_target, update.effective_chat.id)
    except JobExistsException:
        reply = "Hm, looks like I'm watching that already."
    else:
        reply = f"Ok, I'll start watching '{search_target}'"
    context.bot.send_message(chat_id=update.effective_chat.id, text=reply)

def stop_watching(update, context):
    """
    Command handler for stopping to watch a search term
    """
    search_term = "".join(context.args)
    result = db.search(Job.search_term == search_term)

    if not result:
        reply = "I don't think I am watching that."
    else:
        db.remove(Job.search_term == search_term)
        reply = "Ok. I'll no longer watch " + search_term
    context.bot.send_message(chat_id=update.effective_chat.id, text=reply)

def look_for_stuff(context):
    """
    Command handler to peridically check all active search jobs.
    """
    for job in db.all():
        known_ads = set(job["known_ads"])
        results = execute_search(job["search_term"])
        something_new = False
        for r in results:
            if r["ad_id"] not in known_ads:
                message = (
                    f"New item for {job['search_term']} ({r['price']}): {r['url']}"
                )
                context.bot.send_message(chat_id=job["chat_id"], text=message)
                known_ads.add(r["ad_id"])
                something_new = True

        if something_new:
            db.update(
                {"known_ads": list(known_ads)}, Job.search_term == job["search_term"]
            )
        else:
            # context.bot.send_message(chat_id=job["chat_id"], text=f"Nothing new for {job['search_term']}")
            pass

def status(update, context):
    message = "I'm currently watching: \n"
    for job in db.all():
        message += "- " + job["search_term"] + "\n"
    context.bot.send_message(chat_id=update.effective_chat.id, text=message)

@click.group()
def cli():
    pass

@cli.command()
@click.option("--token", prompt=True, help="The telegram bot api token")
def run(token):
    TELEGRAM_BOT_TOKEN = token

    updater = Updater(token=TELEGRAM_BOT_TOKEN, use_context=True)
    dispatcher = updater.dispatcher
    job_queue = updater.job_queue

    job_minute = job_queue.run_repeating(look_for_stuff, interval=5 * 60, first=0)

    echo_handler = MessageHandler(Filters.text & (~Filters.command), echo)
    dispatcher.add_handler(echo_handler)

    start_watching_handler = CommandHandler("start", start_watching)
    dispatcher.add_handler(start_watching_handler)

    stop_handler = CommandHandler("stop", stop_watching)
    dispatcher.add_handler(stop_handler)

    status_handler = CommandHandler("status", status)
    dispatcher.add_handler(status_handler)

    updater.start_polling()

@cli.command()
@click.argument("searchterm")
def search(searchterm):
    data = execute_search(searchterm)
    click.echo(data)

if __name__ == "__main__":
    cli()


""" Image editing class for head to bot, time-trail, obstacle where
there is only single agent
"""
import datetime
import logging
import rospy
import cv2

from markov.log_handler.logger import Logger
from markov.utils import get_racecar_idx
from mp4_saving import utils
from mp4_saving.constants import (RaceCarColorToRGB,
                                  IconographicImageSize,
                                  TrackAssetsIconographicPngs, RACE_COMPLETE_Y_OFFSET,
                                  RACE_TYPE_TO_VIDEO_TEXT_MAPPING, XYPixelLoc, AWS_DEEPRACER_WATER_MARK,
                                  SCALE_RATIO, FrameQueueData)
from mp4_saving.image_editing_interface import ImageEditingInterface
from mp4_saving.top_view_graphics import TopViewGraphics

LOG = Logger(__name__, logging.INFO).get_logger()

class SingleAgentImageEditing(ImageEditingInterface):
    """ Image editing class for head to bot, time-trail, obstacle where
    there is only single agent
    """
    def __init__(self, racecar_name, racecar_info, race_type):
        """ Initializing the required data for the head to bot, time-trail. This is used for single agent
        Arguments:
            racecars_info (list): list of dict having information of the agent
            race_type (str): Since this class is reused for all the different race_type
        """
        self.racecar_info = racecar_info
        self.race_type = race_type
        racecar_index = get_racecar_idx(racecar_name)
        self.racecar_index = racecar_index if racecar_index else 0
        # Store the font which we will use to write the phase with
        self.amazon_ember_regular_20px = utils.get_font('AmazonEmber-Regular', 20)
        self.amazon_ember_regular_16px = utils.get_font('AmazonEmber-Regular', 16)
        self.amazon_ember_heavy_30px = utils.get_font('AmazonEmber-Heavy', 30)
        self.amazon_ember_light_18px = utils.get_font('AmazonEmber-Light', 18)
        self.amazon_ember_light_20px = utils.get_font('AmazonEmber-Light', 20)
        self.amazon_ember_light_italic_20px = utils.get_font('AmazonEmber-LightItalic', 20)

        self.is_racing = rospy.get_param("VIDEO_JOB_TYPE", "") == "RACING"
        self.is_league_leaderboard = rospy.get_param("LEADERBOARD_TYPE", "") == "LEAGUE"
        self.leaderboard_name = rospy.get_param("LEADERBOARD_NAME", "")
        self._total_laps = int(rospy.get_param("NUMBER_OF_TRIALS", 0))

        # The track image as iconography
        self.track_icongraphy_img = utils.get_track_iconography_image()

        # Track image offset
        self.track_loc_offset = XYPixelLoc.TRACK_IMG_WITH_OFFSET_LOC.value if self.is_league_leaderboard \
            else XYPixelLoc.TRACK_IMG_WITHOUT_OFFSET_LOC.value

        # Gradient overlay image
        gradient_img_path = TrackAssetsIconographicPngs.OBSTACLE_OVERLAY_PNG_LEAGUE_LEADERBOARD.value \
            if self.is_league_leaderboard else TrackAssetsIconographicPngs.OBSTACLE_OVERLAY_PNG.value
        self.gradient_img = self._plot_track_on_gradient(gradient_img_path)
        self.gradient_alpha_rgb_mul, self.one_minus_gradient_alpha = utils.get_gradient_values(self.gradient_img)

        # Top camera information
        top_camera_info = utils.get_top_camera_info()
        self.top_view_graphics = TopViewGraphics(top_camera_info.horizontal_fov, top_camera_info.padding_pct,
                                                 top_camera_info.image_width, top_camera_info.image_height,
                                                 racecar_info)

    def _edit_major_cv_image(self, major_cv_image, mp4_video_metrics_info):
        """ Apply all the editing for the Major 45degree camera image
        Args:
            major_cv_image (Image): Image straight from the camera
        Returns:
            Image: Edited main camera image
        """
        # Applying gradient to whole major image and then writing text
        major_cv_image = utils.apply_gradient(major_cv_image, self.gradient_alpha_rgb_mul,
                                              self.one_minus_gradient_alpha)

        # Top left location of the picture
        loc_x, loc_y = XYPixelLoc.SINGLE_AGENT_DISPLAY_NAME_LOC.value

        # Display name (Racer name/Model name)
        display_name = self.racecar_info[self.racecar_index]['display_name']
        display_name_txt = display_name if len(display_name) < 15 else "{}...".format(display_name[:15])
        major_cv_image = utils.write_text_on_image(image=major_cv_image, text=display_name_txt,
                                                   loc=(loc_x, loc_y), font=self.amazon_ember_regular_20px,
                                                   font_color=RaceCarColorToRGB.White.value,
                                                   font_shadow_color=RaceCarColorToRGB.Black.value)
        # Lap Counter
        loc_y += 30
        current_lap = min(int(mp4_video_metrics_info[self.racecar_index].lap_counter) + 1, self._total_laps)
        lap_counter_text = "{}/{}".format(current_lap, self._total_laps)
        major_cv_image = utils.write_text_on_image(image=major_cv_image, text=lap_counter_text,
                                                   loc=(loc_x, loc_y), font=self.amazon_ember_heavy_30px,
                                                   font_color=RaceCarColorToRGB.White.value,
                                                   font_shadow_color=RaceCarColorToRGB.Black.value)
        # total_evaluation_time (Race time)
        loc_y += 45
        total_eval_milli_seconds = mp4_video_metrics_info[self.racecar_index].total_evaluation_time
        time_delta = datetime.timedelta(milliseconds=total_eval_milli_seconds)
        total_eval_time_text = "Race | {}".format(utils.milliseconds_to_timeformat(time_delta))
        major_cv_image = utils.write_text_on_image(image=major_cv_image, text=total_eval_time_text,
                                                   loc=(loc_x, loc_y), font=self.amazon_ember_light_18px,
                                                   font_color=RaceCarColorToRGB.White.value,
                                                   font_shadow_color=RaceCarColorToRGB.Black.value)
        # Reset counter
        loc_y += 25
        reset_counter_text = "Reset | {}".format(mp4_video_metrics_info[self.racecar_index].reset_counter)
        major_cv_image = utils.write_text_on_image(image=major_cv_image, text=reset_counter_text,
                                                   loc=(loc_x, loc_y), font=self.amazon_ember_light_18px,
                                                   font_color=RaceCarColorToRGB.White.value,
                                                   font_shadow_color=RaceCarColorToRGB.Black.value)
        # Speed
        loc_x, loc_y = XYPixelLoc.SPEED_EVAL_LOC.value
        if self.is_league_leaderboard:
            loc_x, loc_y = XYPixelLoc.SPEED_LEADERBOARD_LOC.value
        speed_text = "{} m/s".format(utils.get_speed_formatted_str(mp4_video_metrics_info[self.racecar_index].throttle))
        major_cv_image = utils.write_text_on_image(image=major_cv_image, text=speed_text,
                                                   loc=(loc_x, loc_y), font=self.amazon_ember_light_20px,
                                                   font_color=RaceCarColorToRGB.White.value,
                                                   font_shadow_color=RaceCarColorToRGB.Black.value)
        # Leaderboard name
        if self.is_league_leaderboard:
            loc_x, loc_y = XYPixelLoc.LEADERBOARD_NAME_LOC.value
            major_cv_image = utils.write_text_on_image(image=major_cv_image, text=self.leaderboard_name,
                                                       loc=(loc_x, loc_y), font=self.amazon_ember_regular_16px,
                                                       font_color=RaceCarColorToRGB.White.value,
                                                       font_shadow_color=RaceCarColorToRGB.Black.value)
        # Evaluation type
        loc_x, loc_y = XYPixelLoc.RACE_TYPE_EVAL_LOC.value
        if self.is_league_leaderboard:
            loc_x, loc_y = XYPixelLoc.RACE_TYPE_RACE_LOC.value
        race_text = "race" if self.is_racing else "evaluation"
        evaluation_type_txt = "{} {}".format(RACE_TYPE_TO_VIDEO_TEXT_MAPPING[self.race_type], race_text)
        major_cv_image = utils.write_text_on_image(image=major_cv_image, text=evaluation_type_txt,
                                                   loc=(loc_x, loc_y), font=self.amazon_ember_light_italic_20px,
                                                   font_color=RaceCarColorToRGB.White.value,
                                                   font_shadow_color=RaceCarColorToRGB.Black.value)

        # AWS Deepracer logo at the bottom for the community leaderboard
        if self.is_league_leaderboard:
            major_cv_image = utils.write_text_on_image(image=major_cv_image, text=AWS_DEEPRACER_WATER_MARK,
                                                       loc=XYPixelLoc.AWS_DEEPRACER_WATER_MARK_LOC.value,
                                                       font=self.amazon_ember_regular_16px,
                                                       font_color=RaceCarColorToRGB.White.value,
                                                       font_shadow_color=RaceCarColorToRGB.Black.value)

        # Check if the done flag is set and set the banner appropriately
        if mp4_video_metrics_info[self.racecar_index].done and (int(self._total_laps) >= current_lap):
            # When the cv2 text is written, it automatically drops the alpha value of the image
            rel_y_offset = XYPixelLoc.TRACK_IMG_WITH_OFFSET_LOC.value[1] if self.is_league_leaderboard else 0
            racecomplete_image = utils.get_image(TrackAssetsIconographicPngs.RACE_COMPLETE_OVERLAY_PNG.value,
                                                 IconographicImageSize.RACE_COMPLETE_IMAGE_SIZE.value)
            x_offset = major_cv_image.shape[1] - racecomplete_image.shape[1]//2
            y_offset = major_cv_image.shape[0] - RACE_COMPLETE_Y_OFFSET - rel_y_offset - racecomplete_image.shape[0]//2
            major_cv_image = utils.plot_rectangular_image_on_main_image(
                major_cv_image, racecomplete_image, (x_offset, y_offset))
        major_cv_image = cv2.cvtColor(major_cv_image, cv2.COLOR_RGB2BGRA)
        return major_cv_image

    def _plot_track_on_gradient(self, gradient_img_path):
        """ For the given gradient apply the track iconographic image and use this to apply gradient
        on each camera frame. Previously this was done on the top camera which changed every frame. But
        with the track iconographic image set static, adding the track on gradient is more optimized.

        Arguments:
            gradient_img_path (str): Gradient image path

        Returns:
            (Image): Edited gradient image with track image
        """
        gradient_img = utils.get_image(gradient_img_path, IconographicImageSize.FULL_IMAGE_SIZE.value)
        gradient_img = cv2.cvtColor(gradient_img, cv2.COLOR_RGBA2BGRA)

        track_icongraphy_scaled = utils.resize_image(self.track_icongraphy_img, SCALE_RATIO)
        track_icongraphy_alpha = track_icongraphy_scaled[:, :, 3]/255.0

        # Track image is placed at the bottom right with some offset (only in leaderboard tracks)
        x_min = -(self.track_loc_offset[1] + track_icongraphy_scaled.shape[0])
        x_max = gradient_img.shape[0] - self.track_loc_offset[1]
        y_min = -(self.track_loc_offset[0] + track_icongraphy_scaled.shape[1])
        y_max = gradient_img.shape[1] - self.track_loc_offset[0]

        # This is used as the offset for plotting the agent dots
        self.track_start_loc = (gradient_img.shape[1] + y_min, gradient_img.shape[0] + x_min)

        for channel in range(0, 4):
            gradient_img[x_min:x_max, y_min:y_max, channel] =\
                (track_icongraphy_alpha * track_icongraphy_scaled[:, :, channel]) + \
                (1 - track_icongraphy_alpha) * (gradient_img[x_min:x_max, y_min:y_max, channel])
        return gradient_img

    def _plot_agents_on_major_cv_image(self, major_cv_image, mp4_video_metrics_info):
        """ Add the agents, obstacles on the track.

        Arguments:
            major_cv_image (Image): Edited image having gradient, text, track
            mp4_video_metrics_info (List): List of ROS metric values of each agent

        Returns:
            Image: Edited image with gradient, text, track and agents with dots
        """
        agents_loc = [(metric.x, metric.y) for metric in mp4_video_metrics_info]
        objects_loc = []
        if mp4_video_metrics_info[0].object_locations:
            objects_loc = [(object_loc.x, object_loc.y) for object_loc in mp4_video_metrics_info[0].object_locations]
        return self.top_view_graphics.plot_agents_as_circles(
            major_cv_image, agents_loc, objects_loc, self.track_start_loc)

    def edit_image(self, major_cv_image, metric_info):
        mp4_video_metrics_info = metric_info[FrameQueueData.AGENT_METRIC_INFO.value]
        major_cv_image = self._edit_major_cv_image(major_cv_image, mp4_video_metrics_info)
        major_cv_image = self._plot_agents_on_major_cv_image(major_cv_image, mp4_video_metrics_info)
        return cv2.cvtColor(major_cv_image, cv2.COLOR_BGRA2RGB)


import logging
import tqdm
from multiprocessing import Pool

from dsrt.config.defaults import DataConfig

class Padder:
    def __init__(self, properties, parallel=True, config=DataConfig()):
        self.properties = properties
        self.config = config
        self.parallel = parallel

        self.max_ulen = self.properties['max-utterance-length']
        self.max_dlen = self.properties['max-dialogue-length']

        self.init_logger()

    def init_logger(self):
        self.logger = logging.getLogger()
        self.logger.setLevel(self.config['logging-level'])

    def transform(self, dialogues):
        self.log('info', 'Padding the dialogues (using max utterance length={} tokens) ...'.format(self.max_ulen))

        self.empty_turn = [self.config['pad-d']] * (self.properties['max-utterance-length'] + 1)

        chunksize=self.config['chunksize']
        p = Pool() if self.parallel else Pool(1)
        res = []
        total = len(dialogues)

        self.log('info', '[padder running on {} cores]'.format(p._processes))

        for d in tqdm.tqdm(p.imap(self.pad_dialogue, dialogues, chunksize=chunksize), total=total):
            res.append(d)

        p.close()
        p.join()

        return res

    def pad_dialogues(self, dialogues):
        """
        Pad the entire dataset.
        This involves adding padding at the end of each sentence, and in the case of
        a hierarchical model, it also involves adding padding at the end of each dialogue,
        so that every training sample (dialogue) has the same dimension.
        """
        self.log('info', 'Padding the dialogues ...')

        return [self.pad_dialogue(d) for d in dialogues]

    def pad_dialogue(self, dialogue):
        for i, u in enumerate(dialogue):
            dif = self.max_ulen - len(u) + 1
            dialogue[i] += [self.config['pad-u']] * dif

        # only pad the dialogue if we're training a hierarchical model
        if self.config['hierarchical']:
            dif = self.max_dlen - len(dialogue)
            dialogues += [self.empty_turn] * dif

        return dialogue

    ####################
    #     UTILITIES    #
    ####################

    def log(self, priority, msg):
        """
        Just a wrapper, for convenience.
        NB1: priority may be set to one of:
        - CRITICAL     [50]
        - ERROR        [40]
        - WARNING      [30]
        - INFO         [20]
        - DEBUG        [10]
        - NOTSET       [0]
        Anything else defaults to [20]
        NB2: the levelmap is a defaultdict stored in Config; it maps priority
             strings onto integers
        """
        self.logger.log(logging.CRITICAL, msg)


import io
import picamera
import cv2
import numpy
import serial
import time
import RPi.GPIO as gp

####### Servo Motor Contol #######
gp.setmode(gp.BOARD)
gp.setup(11, gp.OUT)
pwm=gp.PWM(11, 50)
pwm.start(3)

port = '/dev/ttyACM0'
Face = 0

turn=1
while(turn):

    i=3
    while(i):

        #Create a memory stream so photos doesn't need to be saved in a file
        stream = io.BytesIO()

        #Get the picture (low resolution, so it should be quite fast)
        #Here you can also specify other parameters (e.g.:rotate the image)
        with picamera.PiCamera() as camera:
            camera.resolution = (320, 240)
            camera.capture(stream, format='jpeg')
            print("Captured......................")

        #Convert the picture into a numpy array
        buff = numpy.fromstring(stream.getvalue(), dtype=numpy.uint8)

        #Now creates an OpenCV image
        image = cv2.imdecode(buff, 1)

        #Load a cascade file for detecting faces
        face_cascade = cv2.CascadeClassifier('/home/pi/Desktop/Buddy/haarcascade_frontalface_alt.xml')

        #Convert to grayscale
        gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)

        #Look for faces in the image using the loaded cascade file
        faces = face_cascade.detectMultiScale(gray, 1.1, 5)

        #print "Found "+str(len(faces))+" face(s)"

        #Draw a rectangle around every found face
        for (x,y,w,h) in faces:
            Face = 1
            cv2.rectangle(image,(x,y),(x+w,y+h),(255,255,0),2)
            print("Detected")
            ser = serial.Serial(port, 9600, timeout=1)

            t=0
            while(t<2000):
                if(t%10 == 0):
                    print(t)
                t+=1

            ser.write(b'0') ## Stop_Detected

            with picamera.PiCamera() as camera:

                print("Start Video")
                camera.start_recording('examplevid.h264')
                time.sleep(5)
                camera.stop_recording()
                print("Stop Video")

        #Save the result image
        if(i==3):
            cv2.imwrite('result1.jpg',image)

        if(i==2):
            cv2.imwrite('result2.jpg',image)

        if(i==1):
            cv2.imwrite('result3.jpg',image)

        i=i-1

        if(Face == 1):
            Face = 2
            break

    ################   Move Servo   ##################

        if(i==0):
            pwm.ChangeDutyCycle(3)
            #ser.write(b'1')  ## Move_Servo_pos1
            print("First Pos__________________________")

        if(i==2):
            pwm.ChangeDutyCycle(5)
            #ser.write(b'2')  ## Move_Servo_pos2
            print("Second Pos__________________________")

        if(i==1):
            pwm.ChangeDutyCycle(7)
            #ser.write(b'3')  ## Move_Servo_pos3
            print("Third Pos__________________________")

        t=0
        while(t<200):
            if(t%10 == 0):
                print(t)
            t+=1

    print("###############################################");

    t=0
    while(t<500):
        if(t%10 == 0):
            print(t)
        t+=1

#    turn = 0

    if(cv2.waitKey(1) & 0xFF == ord('q')):
        break
    if(Face == 2):
        Face = 0
        ser = serial.Serial(port, 9600, timeout=1)
        t=0
        while(t<2000):
            if(t%10 == 0):
                print(t)
            t+=1
        ser.write(b'1');
        break


from torch import optim
from torch.distributions import Categorical
import importlib

class Model():
    def __init__(self, config, modelParam, env):
        self.update_counter = 0

        if modelParam['cuda']['use_cuda']:
            self.device = f"cuda:{modelParam['cuda']['device_idx']}"
        else:
            self.device = "cpu"

        self.config         = config
        self.modelParam     = modelParam

        self.policyNet = self.selectPolicyNet(config, env.size_of_state_space, env.size_of_action_space)
        self.policyNet.to(self.device)

        self.optimizer = self.selectOptimizer(config)

        return

    def selectPolicyNet(self, config, size_of_state_space, size_of_action_space):
        #Importing the network class based on the config[network] key
        module = importlib.import_module("networks." + config['network'])
        net    = getattr(module, config['network'])(size_of_state_space, size_of_action_space)
        return net

    def selectOptimizer(self, config):
        if config['optimizer'] == 'adam':
            optimizer = optim.Adam(self.policyNet.parameters(), lr=config['learningRate']['lr'], weight_decay=config['weight_decay'])
        elif config['optimizer'] == 'SGD':
            optimizer = optim.SGD(self.policyNet.parameters(), lr=config['learningRate']['lr'],weight_decay=config['weight_decay'])
        elif config['optimizer'] == 'RMSprop':
            optimizer = optim.RMSprop(self.policyNet.parameters(), lr=config['learningRate']['lr'],weight_decay=config['weight_decay'])
        else:
            raise Exception('invalid optimizer')
        return optimizer

    def select_action(self, state):
        state = state.to(self.device)
        probs = self.policyNet(state)
        m = Categorical(probs)
        action = m.sample()
        log_probs = m.log_prob(action)
        return action.item(), log_probs


import argparse

import logging

log_debug = logging.getLogger("debugLog")

_available_commands = ["list"]

def get_parser(parent=None):
    # Anomaly commands
    conf_file_parser = argparse.ArgumentParser(add_help=False)
    conf_file_parser.add_argument('--config_file', '--config_path', help='Path to config file', metavar='[path]',
                                  dest="config_file")

    if not parent:
        admin = argparse.ArgumentParser(description='Deployment control', prog='deployment control',
                                        parents=[conf_file_parser])
        admin.add_argument("--debug", help="Run command in debug mode", dest="debug", action='store_true')

    else:
        admin = parent.add_parser('admin', help='Deployment control')

    # Admin commands
    admin_parser = argparse.ArgumentParser(add_help=False)
    admin_parser.add_argument("list", help="List %(prog)ss")
    admin_parser.add_argument('--host', help='Hostname or ip of target', metavar='[hostname]',
                              dest='target_host', default='all')
    admin_parser.add_argument('--config-path', help='Path to config file', metavar='[path]',
                              dest="config_path", action='store')
    # add more admin commands here

    # Admin parser
    admin_subcommands = admin.add_subparsers(dest="target")
    admin_container = admin_subcommands.add_parser('container', prog='Container', parents=[admin_parser])
    admin_node = admin_subcommands.add_parser('node', prog='Node', parents=[admin_parser])
    admin_network = admin_subcommands.add_parser('network', prog='Network', parents=[admin_parser])
    admin_network.add_argument('--interface', help='Name of interface', type=str, metavar='[NAME]',
                               dest="target_interface")

    admin_deployment = admin_subcommands.add_parser('deployment', prog='Deployment', parents=[admin_parser])

    if parent:
        return parent
    else:
        return admin

def parse_arguments(args):
    args = vars(args)
    unpacked = unpack_targets(args)
    unpacked.update(unpack_params(args))
    log_debug.debug("Unpacked arguments" + str(unpacked))
    return unpacked

def unpack_targets(args):
    _unpacked = dict()
    for arg in args:
        if "target" in arg and args[arg]:
            param_split = arg.split("_")
            if len(param_split) > 1:
                _unpacked[param_split[1]] = args[arg]
            else:
                _unpacked[arg] = args[arg]
    return {"target": _unpacked}

def unpack_params(args):
    _unpacked = dict()
    for arg in args:
        if arg in _available_commands:
            return {"action": arg}


import serial
import serial
from time import sleep
import threading
import time
# sudo chmod 666 /dev/ttyACM0
device_port = "/dev/ttyACM0"
from multiprocessing.pool import ThreadPool
import settings

class uwb_data(threading.Thread):
    def __init__(self,file_name,device_port):
        threading.Thread.__init__(self)
        self.file_name = file_name
        self.serial = serial.Serial(device_port)
        self.running = True
        self.myval = []

    def create_csv_file(self):
        self.f = open(self.file_name, 'w+')
        self.f.write("timestamp,x,y,z \n")
        sleep(1)

    def store_uwb_data(self):
        val = str(self.serial.readline().decode().strip(' \r\n'))
        if val.startswith('+DPOS:'):
            val = val.strip('+DPOS:')
            val = val.split(',')
            self.myval =  [int(float(val[2])),int(float(val[3]))]

    def get_uwb_data(self):
        return self.myval

    def run(self):
        while self.running:
            self.store_uwb_data()
            settings.myList = self.get_uwb_data()

    def terminate(self):
         """clean stop"""
         self.running = False

if __name__ == "__main__":
    uwb_get_way = uwb_data('IDRdata.csv',"/dev/ttyACM0")
    uwb_get_way.start()
    pool = ThreadPool(processes=1)
    try:
        while True:
            async_result = pool.apply_async(uwb_get_way.get_uwb_data)
            return_val = async_result.get()
            print(settings.myList)

    except (KeyboardInterrupt, SystemExit):
        uwb_get_way.terminate()
    print("killed")


from __future__ import print_function
import boto3

#This module creates a table with the table constraints as well

dynamodb = boto3.resource('dynamodb', region_name='us-west-2', endpoint_url='http://localhost:8000', aws_access_key_id='Secret', aws_secret_access_key='Secret')

table = dynamodb.create_table(
        TableName = 'Movies',
        KeySchema=[
        {
            'AttributeName': 'year',
            'KeyType': 'HASH'  #Partition key
        },
        {
            'AttributeName': 'title',
            'KeyType': 'RANGE'  #Sort key
        }
    ],
    AttributeDefinitions=[
        {
            'AttributeName': 'year',
            'AttributeType': 'N'
        },
        {
            'AttributeName': 'title',
            'AttributeType': 'S'
        },

    ],
    ProvisionedThroughput={
        'ReadCapacityUnits': 10,
        'WriteCapacityUnits': 10
    }
    )


"""Train a model on Treebank"""

import random
import json
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.optim.lr_scheduler as sched
import torch.utils.data as data
import utils

from collections import OrderedDict
from tqdm import tqdm
from torch.utils.tensorboard import SummaryWriter
from args import get_train_args
from models import investorConferenceAnalyzer
from utils import Treebank, collate_fn

def main(args):
    # Set up logging and devices
    args.save_dir = utils.get_save_dir(args.save_dir, args.name, training=True)
    log = utils.get_logger(args.save_dir, args.name)
    tbx = SummaryWriter(args.save_dir)
    device, args.gpu_ids = utils.get_available_devices()
    log.info(f'Args: {json.dumps(vars(args), indent=4, sort_keys=True)}')
    args.batch_size *= max(1, len(args.gpu_ids))

    # Set random seed
    log.info(f'Using random seed {args.seed}...')
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)

    # Get model
    log.info('Building model...')
    model = investorConferenceAnalyzer(args.pce_model, args.num_labels)
    model = nn.DataParallel(model, args.gpu_ids)
    if args.load_path:
        log.iofo(f'Loading checkpoint from {args.load_path}...')
        model, step = utils.load_model(model, args.load_path, args.gpu_ids)
    else:
        step = 0
    model = model.to(device)
    model.train()
    ema = utils.EMA(model, args.ema_decay)

    # Get saver
    saver = utils.CheckpointSaver(args.save_dir,
                                  max_checkpoints=args.max_checkpoints,
                                  metric_name=args.metric_name,
                                  maximize_metric=args.maximize_metric,
                                  log=log)

    # Get optimizer and scheduler
    optimizer_grouped_params = [
        {'params': model.module.classifier.albert.parameters()},
        {'params': model.module.classifier.classifier.parameters(), 'lr': args.lr_c}
    ]
    optimizer = optim.AdamW(optimizer_grouped_params, args.lr,
                            weight_decay=args.l2_wd)
    scheduler = sched.LambdaLR(optimizer, lambda s: 1.)

    # Get data loader
    log.info('Building dataset...')
    train_dataset = Treebank(args.train_record_file)
    train_loader = data.DataLoader(train_dataset,
                                   batch_size=args.batch_size,
                                   shuffle=True,
                                   num_workers=args.num_workers,
                                   collate_fn=collate_fn)
    dev_dataset = Treebank(args.dev_record_file)
    dev_loader = data.DataLoader(dev_dataset,
                                 batch_size=args.batch_size,
                                 shuffle=False,
                                 num_workers=args.num_workers,
                                 collate_fn=collate_fn)

    # Train
    log.info('Training...')
    steps_till_eval = args.eval_steps
    epoch = step // len(train_dataset)
    while epoch != args.num_epochs:
        epoch += 1
        log.info(f'Starting epoch {epoch}...')
        with torch.enable_grad(), \
                tqdm(total=len(train_dataset)) as progress_bar:
            for input_idxs, token_type_idxs, attention_masks, ys, ids in train_loader:
                # Set up for forward
                input_idxs = input_idxs.to(device)
                token_type_idxs = token_type_idxs.to(device)
                attention_masks = attention_masks.to(device)
                batch_size = input_idxs.size(0)
                optimizer.zero_grad()

                # Forward
                log_p = model(input_idxs, token_type_idxs, attention_masks)
                ys = ys.to(device)
                if args.smoothing:
                    loss = utils.nll_loss_label_smoothing(log_p, ys, args.eps)
                else:
                    loss = F.nll_loss(log_p, ys)
                loss_val = loss.item()

                # Backward
                loss.backward()
                nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
                optimizer.step()
                scheduler.step()
                ema(model, step // batch_size)

                # Log info
                step += batch_size
                progress_bar.update(batch_size)
                progress_bar.set_postfix(epoch=epoch, NLL=loss_val)
                tbx.add_scalar('train/NLL', loss_val, step)
                tbx.add_scalar('train/LR', optimizer.param_groups[0]['lr'], step)

                steps_till_eval -= batch_size
                if steps_till_eval <= 0:
                    steps_till_eval = args.eval_steps

                    # Evaluate and save checkpoint
                    log.info(f'Evaluating at step {step}...')
                    ema.assign(model)
                    results, pred_dict = evaluate(model, dev_loader,
                                                  device, args.dev_eval_file)

                    saver.save(step, model, results[args.metric_name], device)
                    ema.resume(model)

                    # Log to console
                    results_str = ', '.join(f'{k}: {v:05.2f}' for k, v in results.items())
                    log.info(f'Dev {results_str}')

                    # Log to TensorBoard
                    log.info('Visualizing in TensorBoard...')
                    for k, v in results.items():
                        tbx.add_scalar(f'dev/{k}', v, step)
                    utils.visualize(tbx,
                                    pred_dict=pred_dict,
                                    eval_path=args.dev_eval_file,
                                    step=step,
                                    split='dev',
                                    num_visuals=args.num_visuals)

def evaluate(model, data_loader, device, eval_file):
    nll_meter = utils.AverageMeter()

    model.eval()
    pred_dict = {}

    # Load eval info
    with open(eval_file, 'r') as fh:
        gold_dict = json.load(fh)

    with torch.no_grad(), \
            tqdm(total=len(data_loader.dataset)) as progress_bar:
        for input_idxs, token_type_idxs, attention_masks, ys, ids in data_loader:
            # Set up for forward
            input_idxs = input_idxs.to(device)
            token_type_idxs = token_type_idxs.to(device)
            attention_masks = attention_masks.to(device)
            batch_size = input_idxs.size(0)

            # Forward
            log_p = model(input_idxs, token_type_idxs, attention_masks)
            ys = ys.to(device)
            loss = F.nll_loss(log_p, ys)
            nll_meter.update(loss.item(), batch_size)

            # Log info
            progress_bar.update(batch_size)
            progress_bar.set_postfix(NLL=nll_meter.avg)

            # Get accuracy
            p = log_p.exp()
            labels = torch.argmax(p, dim=-1)
            preds = utils.predict_sentiments(ids.tolist(), labels.tolist())
            pred_dict.update(preds)

    model.train()

    results = utils.eval_dicts(gold_dict, pred_dict)
    results_list = [('NLL', nll_meter.avg),
                    ('Acc', results['Acc'])]
    results = OrderedDict(results_list)

    return results, pred_dict

if __name__ == '__main__':
    main(get_train_args())


import sys
import seaborn as sns
import pandas as pd
import numpy as np
import scipy.stats
from collections import defaultdict
from matplotlib import pyplot as plt
from sklearn.metrics import r2_score, mean_absolute_error
#plt.style.use('seaborn-whitegrid')
#sns.set_theme()

#Function for creating a dictionary from the epiAneufinder data
def createDictionaryFromTable(table):
    snu_dict=table.set_index(['seq', 'start', 'end']).T.to_dict('list')
    return(snu_dict)

def calculatePopulationSomies(atac_dict, density_dict):
    gain_atac = []
    loss_atac = []
    base_atac = []
    common_keys = set(density_dict).intersection(atac_dict) #filtering for the common CNV locations between the two datasets
    sort_common_keys=sorted(common_keys)
    filtered_density_dict = {k: v for k, v in density_dict.items() if k in sort_common_keys}
    #print(sort_common_keys)
    counts=0
    for k in sort_common_keys:
        #if k[0]!=0: #selecting for all chromosomes
        if k[0]!=0:  # selecting for all chromosomes
            counts=counts+1
            #Calculating pseudobulk representation for the scATAC. 0 is loss, 1 is disomic and 2 is gain
            #If the user changes notation it should be changed here as well
            loss_atac.append(atac_dict[k].count(0) / len(atac_dict[k]))
            base_atac.append(atac_dict[k].count(1) / len(atac_dict[k]))
            gain_atac.append(atac_dict[k].count(2) / len(atac_dict[k]))
    print("Count Bins:",counts)
    return(loss_atac, base_atac, gain_atac, filtered_density_dict)

#Function for calculating different metrics between the two datasets and creating a line plot of the pseudoibulk data
def createLinePlot(density_dict, loss_atac, base_atac, gain_atac):
    new_base_atac = [x * 2 for x in base_atac]
    new_gain_atac = [x * 3 for x in gain_atac]
    atac_plot = [sum(x) for x in zip(new_gain_atac, new_base_atac, loss_atac)]
    atac_array=np.array(atac_plot)
    density_array=[x for x in density_dict.values()]
    x = list(range(len(atac_plot)))
    plt.plot(x,density_array)
    plt.plot(x, atac_plot, color='orange', label="ATAC")
    plt.show()
    #print(density_array)
    print("Pearson Correlation : ",scipy.stats.pearsonr(atac_array, density_array))
    print("Spearman Correlation : ", scipy.stats.spearmanr(atac_array, density_array))
    print("Kendall Correlation : ", scipy.stats.kendalltau(atac_array, density_array))

if __name__ =="__main__":
    density_table=pd.read_csv("/home/katia/Helmholz/epiAneufinder/Hg38_geneDensity.csv", sep="\t")
    snu_full=pd.read_csv("/home/katia/Helmholz/epiAneufinder/revisions/SNU601_br15/epiAneufinder_results/results_table.tsv", sep=" ")
    snu_dict=createDictionaryFromTable(snu_full)
    density_dict=createDictionaryFromTable(density_table)
    loss_atac, base_atac, gain_atac , filtered_density_dict= calculatePopulationSomies(snu_dict,density_dict)
    #print(filtered_density_dict)
    createLinePlot(filtered_density_dict, loss_atac, base_atac, gain_atac)