import torch
import torch.nn as nn
from network.resnetfpn_simple import *
from network.pooling import *
import MinkowskiEngine as ME

from network.general_minkfpn import GeneralMinkFPN

from models.minkloc import MinkLoc

from network.later_cloud_branch import LaterCloudBranch
import spconv.pytorch as spconv
from layers.pooling import MinkGeM as MinkGeM
try:
    from viz_lidar_mayavi_open3d import *
except:
    None
from network.general_imagefes import GeneralImageFE

from network.ffb import FFB
from network.gatt_block import GattBlock
# from network.pointnet_simple import PointNetSimple
# from pointnet2 import pointnet2_utils

from network.grids import Grids
from network.later_image_branch import LaterImageBranch

import torchvision

import matplotlib.pyplot as plt

from datasets.oxford import transform_pts_in_camsystem, project_onto_image

from network.ffb_local import FFBLocal
from network.ffb_glocal import FFBGLocal

from torch_geometric.nn import voxel_grid

from network.resnetfpn_simple import ImageGeM

from tools.options import Options
from tools.utils import set_seed
set_seed(7)
args = Options().parse()

def convert_ME_to_spconv_tensor(cloud_feat, tensor_stride):
    assert isinstance(cloud_feat, ME.SparseTensor)
    # test spconv
    spconv_feats = cloud_feat.F
    spconv_coords = cloud_feat.C
    spconv_coords[:,1:] = spconv_coords[:,1:] // tensor_stride
    spatial_shape = [
        torch.max(spconv_coords[:,1]) - torch.min(spconv_coords[:,1]),
        torch.max(spconv_coords[:,2]) - torch.min(spconv_coords[:,2]),
        torch.max(spconv_coords[:,3]) - torch.min(spconv_coords[:,3]),
    ]
    batch_size = len(cloud_feat.decomposed_coordinates)
    cloud_feat_spconv = spconv.SparseConvTensor(spconv_feats, spconv_coords, spatial_shape, batch_size=batch_size)

    return cloud_feat_spconv

def obtain_same_length_cloud_features(cloud_feat, sample_mode='rand', num_cloud_points=None):
    assert isinstance(cloud_feat, ME.SparseTensor) or isinstance(cloud_feat, spconv.SparseConvTensor)

    if isinstance(cloud_feat, ME.SparseTensor):
        # # -- obtain same length cloud features
        # -- fps ball query
        # clouds, cloud_feat = self.coordsclouds(cloud_feat, data_dict)
        # clouds, cloud_feat = self.pointnet2mlp(clouds, cloud_feat) # [b,npoint,3]  [b,c,npoint]
        # cloud_feat_gem = self.cloud_imagegem(cloud_feat.unsqueeze(-1))
        # --
        cloud_feat_list = cloud_feat.decomposed_features

    elif isinstance(cloud_feat, spconv.SparseConvTensor):
        # test spconv same length
        cloud_feat_list = []
        for i in range(cloud_feat.batch_size):
            rows_of_this_batch = cloud_feat.features[cloud_feat.indices[:,0]==i]
            cloud_feat_list.append(rows_of_this_batch)

    if sample_mode == 'rand':
        cloud_feat_avg = ME.MinkowskiGlobalAvgPooling()(cloud_feat)
        cloud_feat_avg = cloud_feat_avg.F.unsqueeze(1) # [b,1,c]
        # lengths_list = [len(row) for row in cloud_feat_list]
        # # # -- 1
        # cloud_feat = torch.nn.utils.rnn.pad_sequence(cloud_feat_list, batch_first=True)
        # # -- 2
        cloud_feat_same_length = []
        for i in range(len(cloud_feat_list)):
            this_x = cloud_feat_list[i]
            if len(this_x) < num_cloud_points:
                is_replace = True
            else:
                is_replace = False

            selected_indices = np.random.choice(len(this_x), size=num_cloud_points, replace=is_replace)
            new_x = this_x[selected_indices]
            cloud_feat_same_length.append(new_x)
        cloud_feat_same_length = torch.stack(cloud_feat_same_length) # [b,npts,c]
        # # -- 3, with map function
        # def _rand_sample(x):
        #     n = len(x)
        #     is_replace = True if n < num_cloud_points else False
        #     selected_indices = np.random.choice(len(x), size=num_cloud_points, replace=is_replace)
        #     output = x[selected_indices]
        #     return output
        # cloud_feat_same_length = map(_rand_sample, cloud_feat_list)
        # cloud_feat_same_length = list(cloud_feat_same_length)
        # cloud_feat_same_length = torch.stack(cloud_feat_same_length) # [b,npts,c]

        cloud_feat_same_length = torch.cat([cloud_feat_same_length, cloud_feat_avg], dim=1)
    elif sample_mode == 'globalavg':
        cloud_feat_same_length = ME.MinkowskiGlobalAvgPooling()(cloud_feat)
        cloud_feat_same_length = cloud_feat_same_length.F # [b,c]
        cloud_feat_same_length = cloud_feat_same_length.unsqueeze(1) # [b,1,c]
    elif sample_mode == 'globalmax':
        cloud_feat_same_length = ME.MinkowskiGlobalMaxPooling()(cloud_feat)
        cloud_feat_same_length = cloud_feat_same_length.F # [b,c]
        cloud_feat_same_length = cloud_feat_same_length.unsqueeze(1) # [b,1,c]
    elif sample_mode == 'globalavgmax':
        cloud_feat_avg = ME.MinkowskiGlobalAvgPooling()(cloud_feat)
        cloud_feat_max = ME.MinkowskiGlobalMaxPooling()(cloud_feat)
        cloud_feat_avg = cloud_feat_avg.F.unsqueeze(1) # [b,1,c]
        cloud_feat_max = cloud_feat_max.F.unsqueeze(1) # [b,1,c]
        cloud_feat_same_length = torch.cat([cloud_feat_avg, cloud_feat_max], dim=1) # [b,2,c]

    return cloud_feat_same_length

def obtain_same_length_image_features(image_feat, sample_mode, kernel_size=None, stride=None, num_image_points=None):
    '''
    sample image features, also have the same length
    input: image_feat: [b,c,h,w]
    output: [b,hw,c]
    '''
    assert len(image_feat.shape) == 4
    b,c,h,w = image_feat.shape

    if sample_mode == 'max':
        image_feat_same_length = F.max_pool2d(image_feat, kernel_size=kernel_size, stride=stride)
        image_feat_same_length = image_feat_same_length.flatten(2).permute(0,2,1) # [b,hw,c]
    elif sample_mode == 'avg':
        image_feat_same_length = F.avg_pool2d(image_feat, kernel_size=kernel_size, stride=stride)
        image_feat_same_length = image_feat_same_length.flatten(2).permute(0,2,1) # [b,hw,c]
    elif sample_mode == 'rand':
        image_feat_avg = F.adaptive_avg_pool2d(image_feat, output_size=(1,1))
        image_feat_avg = image_feat_avg.flatten(2).permute(0,2,1)
        image_feat = image_feat.flatten(2).permute(0,2,1) # [b,hw,c]
        # # --1
        selected_indices = np.random.randint(0, h*w, size=[b,num_image_points]) # [b,npts]
        image_feat_same_length = image_feat[torch.arange(b).unsqueeze(-1), selected_indices, :] # [b,npts,c]
        # image_feat_same_length = image_feat[torch.arange(b), selected_indices, :]
        # image_feat_same_length = image_feat[:, selected_indices, :]
        # # --2
        # image_feat_same_length = []
        # for _i in range(b):
        #     this_x = image_feat[_i]
        #     this_selected_indices = selected_indices[_i]
        #     this_x = this_x[this_selected_indices]
        #     image_feat_same_length.append(this_x)
        # image_feat_same_length = torch.stack(image_feat_same_length) # [b,npts,c]
        # -- 3
        # image_feat_same_length = []
        # for _i in range(b):

        image_feat_same_length = torch.cat([image_feat_same_length, image_feat_avg], dim=1)
    elif sample_mode == None:
        image_feat_same_length = image_feat.flatten(2).permute(0,2,1)
    elif sample_mode == 'globalavg':
        image_feat_same_length = F.adaptive_avg_pool2d(image_feat, output_size=(1,1))
        image_feat_same_length = image_feat_same_length.flatten(2).permute(0,2,1)
    elif sample_mode == 'globalmax':
        image_feat_same_length = F.adaptive_max_pool2d(image_feat, output_size=(1,1))
        image_feat_same_length = image_feat_same_length.flatten(2).permute(0,2,1)
    elif sample_mode == 'globalavgmax':
        image_feat_avg = F.adaptive_avg_pool2d(image_feat, output_size=(1,1))
        image_feat_max = F.adaptive_max_pool2d(image_feat, output_size=(1,1))
        image_feat_avg = image_feat_avg.flatten(2).permute(0,2,1)
        image_feat_max = image_feat_max.flatten(2).permute(0,2,1)
        image_feat_same_length = torch.cat([image_feat_avg, image_feat_max], dim=1)
    else:
        raise Exception

    return image_feat_same_length

class UniVPRV2(nn.Module):
    def __init__(self):
        super(UniVPRV2, self).__init__()

        self.image_fe = GeneralImageFE(image_fe=args.image_fe,
                                       num_other_stage_blocks=args.num_other_stage_blocks,
                                       num_stage3_blocks=args.num_stage3_blocks,
                                       input_type='image',
                                       image_fe_dim=args.image_fe_dim,
                                       )

        if args.cloud_fe == 'general_minkfpn':
            self.cloud_fe = GeneralMinkFPN(in_channels=1, out_channels=None, num_top_down=None, conv0_kernel_size=5,
                                        layers=[1,1,1], planes=[32,64,64])
        elif args.cloud_fe == 'minkloc':
            self.cloud_fe = MinkLoc(in_channels=1, feature_size=args.cloud_fe_dim, output_dim=args.cloud_fe_dim,
                            planes=[32, 64, 64], layers=[1, 1, 1], num_top_down=1,
                            conv0_kernel_size=5, block='ECABasicBlock', pooling_method='GeM')

        self.ffbs = nn.ModuleList()
        self.laterimagebranches = nn.ModuleList()
        self.latercloudbranches = nn.ModuleList()
        self.ffblocals = nn.ModuleList()
        self.gem = ImageGeM()

        for i in range(args.num_ffbs):
            ffb = FFB(in_features=args.fusion_dim, hidden_features=args.fusion_dim//8, n_heads=8,
                      num_blocks=args.num_blocks_in_ffb,
                        ffb_type=args.ffb_type,
                        num_image_points=args.num_image_points, num_cloud_points=args.num_cloud_points,
                        image_fe_last_dim=self.image_fe.last_dim)
            laterimagebranch = LaterImageBranch(block_type=args.later_image_branch_type,num_heads=8,
                                            in_features=self.image_fe.last_dim,
                                            num_blocks=args.num_blocks_in_later_image,
                                            window_size=[7,7], shift_size=[0,0])
            latercloudbranch = LaterCloudBranch(in_features=128,
                                                num_blocks=args.num_blocks_in_later_cloud)
            # ffblocal = FFBLocal(in_image_feat_dim=self.image_fe.last_dim, embed_dim=args.fusion_dim, num_heads=8, num_blocks=1, window_size=4)
            ffblocal = FFBLocal(in_image_feat_dim=self.image_fe.last_dim, embed_dim=args.fusion_dim, num_heads=8, num_blocks=1, window_size=args.ffblocal_windowsize)

            self.ffbs.append(ffb)
            self.laterimagebranches.append(laterimagebranch)
            self.latercloudbranches.append(latercloudbranch)
            self.ffblocals.append(ffblocal)

    def forward(self, data_dict):
        # feed_dict{
        #     'positives_mask(train)',
        #     'negatives_mask(train)',
        #     'coords',
        #     'clouds',
        #     'features',
        #     'uv',
        #     'colors',
        #     'maskselect',
        #     'images',
        #     'P0_camera',
        #     'T_camera_lidar_basedon_pose',
        # }

        # -- image part
        image_feat_128, image_feat_gem_org, image_feat_256, _ = self.image_fe(data_dict)

        # -- cloud part
        cloud_feat_dict = self.cloud_fe(data_dict)
        cloud_feat = cloud_feat_dict['x_feat']
        cloud_feat_gem_org = cloud_feat_dict['embedding']
        cloud_feat_gem_extra = cloud_feat_dict['embedding_extra']

        # _coords = cloud_feat.coordinates[cloud_feat.coordinates[:,0]==0][:,1:]
        # T_camera_lidar_basedon_pose = data_dict['T_camera_lidar_basedon_pose'][0]
        # P0_camera = data_dict['P0_camera'][0]
        # _pts_in_image, _pts_uv_in_image, _pts_colors_in_image, _pts_out_image = convert_pts_to_uv(_coords, T_camera_lidar_basedon_pose, P0_camera)
        # _image = data_dict['images'][0]
        # _pts_uv_in_image = _pts_uv_in_image * 0.125 / 1 # 16 is the downsample ratio of the image feature
        # _pts_window_indices, _image, _image_window_indices = get_window_indices(_pts_uv_in_image, _image, window_size=[64,64])

        # _pts_in_image = _pts_in_image.cpu().numpy()
        # _pts_uv_in_image = _pts_uv_in_image.cpu().numpy()
        # _pts_colors_in_image = _pts_colors_in_image.cpu().numpy()

        # # _pts_all = torch.cat([_pts_in_image, _pts_out_image], dim=0).cpu().numpy()
        # # viz_lidar_open3d(_pts_all)
        # # _coords = cloud_feat.coordinates[cloud_feat.coordinates[:,0]==0][:,1:]
        # # _coords = _coords.cpu().numpy()
        # # # _clouds = _clouds.cpu().numpy()
        # # # viz_lidar_open3d(_clouds)
        # # _T = data_dict['T_camera_lidar_basedon_pose'][0].cpu().numpy()
        # # _P0 = data_dict['P0_camera'][0].cpu().numpy()
        # # _coordsincamsystem = transform_pts_in_camsystem(_coords,T=_T)
        # # # viz_lidar_open3d(_coordsincamsystem)
        # # _uv,_colors,_mask = project_onto_image(_coordsincamsystem,P=_P0)
        # # _selected_indices = (_coords[:,0]>0) & (_coords[:,0]<1000) \
        # #             & (_coords[:,1]>0) & (_coords[:,1]<1000) \
        # #             & (_coords[:,2]>-10) & (_coords[:,2]<50)
        # # _maskselect = _mask & _selected_indices
        # # # _uv = _uv[_maskselect]
        # # _uv = _uv * 0.125 / 16
        # # viz projected points
        # plt.figure(dpi=200)
        # # _image = image_feat_128[0,:3].cpu()
        # # _image = image_feat_256[0,:3].cpu()
        # _image = data_dict['images'][0].cpu()
        # _mean = torch.tensor([0.485, 0.456, 0.406]).unsqueeze(-1).unsqueeze(-1)
        # _std = torch.tensor([0.229, 0.224, 0.225]).unsqueeze(-1).unsqueeze(-1)
        # _image = _image*_std + _mean
        # _image = torchvision.transforms.ToPILImage()(_image)
        # _image = np.array(_image)
        # # _uv = _uv[_maskselect]
        # # _colors = _colors[_maskselect]
        # _uv = _pts_uv_in_image  * 0.125
        # h_max = _uv[:,1].max()
        # w_max = _uv[:,0].max()

        # _colors = _pts_colors_in_image
        # plt.scatter(_uv[:,0], _uv[:,1], c=_colors, s=1)
        # plt.imshow(_image)
        # plt.show()
        # # plt.close()

        a=1

        if args.num_ffbs > 0:

            for i in range(args.num_ffbs):
                ffb = self.ffbs[i]
                laterimagebranch = self.laterimagebranches[i]
                latercloudbranch = self.latercloudbranches[i]
                ffblocal = self.ffblocals[i]

                # GFM
                image_feat_same_length = obtain_same_length_image_features(image_feat_128, sample_mode=args.sample_mode, num_image_points=args.num_image_points)
                cloud_feat_same_length = obtain_same_length_cloud_features(cloud_feat, sample_mode=args.sample_mode, num_cloud_points=args.num_cloud_points)

                output_dict = ffb(in_image_feat=image_feat_same_length,  # image_feat_128
                                  in_cloud_feat=cloud_feat_same_length)
                ffb_output_gem_128 = output_dict['output_feat_gem_128']
                ffb_output_gem_256 = output_dict['output_feat_gem_256']

                image_feat_128, image_feat_256_gem_later, image_feat_256 = laterimagebranch(image_feat_256, ffb_output_gem_256)
                cloud_feat, cloud_feat_gem_later = latercloudbranch(cloud_feat, ffb_output_gem_128)

                # local_ffb + NDM
                if args.use_ffblocal and i<args.num_ffbs-1:
                    image_feat_256, cloud_feat = ffblocal(image_feat_256, cloud_feat, data_dict['T_camera_lidar_basedon_pose'], data_dict['P0_camera'],
                                                        downsample_ratio=16, image=data_dict['images'])

            if args.final_embedding_type == 'imageorg_cloudorg_ffb':
                embedding = torch.cat([image_feat_gem_org, cloud_feat_gem_org, ffb_output_gem_128], dim=1)
            elif args.final_embedding_type == 'imagelater_cloudlater_ffb':
                embedding = torch.cat([image_feat_256_gem_later, cloud_feat_gem_later, ffb_output_gem_128], dim=1)
            elif args.final_embedding_type == 'imagelater_cloudlater':
                embedding = torch.cat([image_feat_256_gem_later, cloud_feat_gem_later], dim=1)
            elif args.final_embedding_type == 'imageorg':
                embedding = image_feat_gem_org
            elif args.final_embedding_type == 'cloudorg':
                embedding = cloud_feat_gem_org
            # elif args.final_embedding_type == 'imagelater_cloudlater_ffb_sphcloud':
            #     embedding = torch.cat([image_feat_256_gem_later, cloud_feat_gem_later, ffb_output_gem_128, sph_cloud_feat_gem_org], dim=1)
            else:
                raise Exception

        elif args.num_ffbs == 0:
            image_feat_same_length = obtain_same_length_image_features(image_feat_128, sample_mode=args.sample_mode, num_image_points=args.num_image_points)
            cloud_feat_same_length = obtain_same_length_cloud_features(cloud_feat, sample_mode=args.sample_mode, num_cloud_points=args.num_cloud_points)
            fusion_feat = torch.cat([image_feat_same_length, cloud_feat_same_length], dim=1) # [b,n,c]
            fusion_feat = fusion_feat.permute(0,2,1) # [b,c,n]
            fusion_feat = fusion_feat.unsqueeze(-1) # [b,c,n,1]
            fusion_gem = self.gem(fusion_feat) # [b,c]

            if args.final_embedding_type == 'imageorg_cloudorg':
                embedding = torch.cat([image_feat_gem_org, cloud_feat_gem_org], dim=1)
            elif args.final_embedding_type == 'imageorg_cloudorg_ffb':
                embedding = torch.cat([image_feat_gem_org, cloud_feat_gem_org, fusion_gem], dim=1)

        # print(f'beforfusion_{args.use_l2norm_before_fusion}_outputfeat_{args.use_l2norm_for_output_feat}')

        output_dict = {
            'image_embedding':image_feat_gem_org,
            'cloud_embedding':cloud_feat_gem_org,
            'embedding':embedding,
        }

        return output_dict


# coding=utf-8
from __future__ import division

import logging

from nrnutils import Section, PROXIMAL
from model.mechanisms import pas_mechanism
from model.base import BaseNeuron

logger = logging.getLogger('morphology')

# geometry and topology

def create_soma(self, soma_L=15, soma_diam=15, soma_nseg=1, mechanisms=None, **kwargs):
    if mechanisms is None:
        mechanisms = self.mechanisms
    self.soma = Section(name='soma', L=soma_L, diam=soma_diam, nseg=soma_nseg, Ra=self.Ra, cm=self.cm,
                        mechanisms=mechanisms)
    self.sections.append(self.soma)

def create_axon(self, axon_L=100, axon_diam=0.2, mechanisms=None, **kwargs):
    if mechanisms is None:
        mechanisms = self.mechanisms
    self.axon = Section(name='axon', L=axon_L, diam=axon_diam, Ra=self.Ra, cm=self.cm,
                        mechanisms=mechanisms,
                        parent=self.soma,
                        connection_point=PROXIMAL)
    self.sections.append(self.axon)

def create_radial_dends(self, num_dendrites=1, parent=None, **kwargs):
    if parent is None:
        parent = self.soma
    self.radial_dends = []
    for i in range(num_dendrites):
        self.radial_dends.append(Section(name='radial_dends_{}'.format(i + 1),
                                         L=self.radial_L,
                                         diam=self.radial_diam,
                                         nseg=self.radial_nseg,
                                         Ra=self.Ra, cm=self.cm,
                                         mechanisms=self.mechanisms,
                                         parent=parent))
    self.sections += self.radial_dends

def create_sinks(self, num_sinks=1, parent=None, **kwargs):
    if parent is None:
        parent = self.soma
    self.sinks = []
    for i in range(num_sinks):
        self.sinks.append(Section(name='sink_{}'.format(i + 1),
                                         L=self.sink_L,
                                         diam=self.sink_diam,
                                         nseg=self.sink_nseg,
                                         Ra=self.Ra, cm=self.cm,
                                         mechanisms=self.mechanisms,
                                         parent=parent))
    self.sections += self.sinks

# noinspection PyAttributeOutsideInit
class MultiDend(BaseNeuron):
    """
    Soma with [num_dendrites] projecting from it.
    """

    def __init__(self, name="MultiDend", num_dendrites=1,
                 radial_l=50, radial_diam=2, radial_nseg=9,
                 reduced=False, call_geom_nseg=False,
                 *args, **kwargs):
        self.num_dendrites = num_dendrites
        self.radial_L = radial_l
        self.radial_diam = radial_diam
        self.radial_nseg = radial_nseg
        super(MultiDend, self).__init__(name, call_geom_nseg=call_geom_nseg, *args, **kwargs)

    def build_sections(self, soma_L=0.01, soma_diam=2, **kwargs):
        super(MultiDend, self).build_sections(soma_L=soma_L, soma_diam=soma_diam, **kwargs)
        self.soma.Ra = 0.2

        create_radial_dends(self, num_dendrites=self.num_dendrites)

        self.dend = self.radial_dends

    create_mechanisms = pas_mechanism

# noinspection PyAttributeOutsideInit
class MultiDendWithSink(MultiDend):
    def __init__(self, name="MultiDendWithSink",
                 sink_num=1, sink_l=50, sink_diam=2, sink_nseg=9,
                 *args, **kwargs):
        self.num_sinks = sink_num
        self.sink_L = sink_l
        self.sink_diam = sink_diam
        self.sink_nseg = sink_nseg
        super(MultiDendWithSink, self).__init__(name, *args, **kwargs)

    def build_sections(self, **kwargs):
        super(MultiDendWithSink, self).build_sections(**kwargs)

        create_sinks(self, num_sinks=self.num_sinks)

        self.dend = self.radial_dends + self.sinks

# noinspection PyAttributeOutsideInit,PyPep8Naming
class SingleDend(BaseNeuron):
    """
    Soma with a single dendrite and (optionally) an axon
    don't auto-calculate nseg
    """

    def __init__(self, name="SingleDend", call_geom_nseg=False, *args, **kwargs):
        super(SingleDend, self).__init__(name, call_geom_nseg=call_geom_nseg, *args, **kwargs)
        # reset soma area: set L and diam to 2.8012206 , so that area = 24.651565 (from rho = 0.1)
        # self.soma.L = 2.8012206
        # self.soma.diam = 2.8012206

    create_mechanisms = pas_mechanism

    def build_sections(self, dend_l=707, dend_diam=1., dend_nseg=81, axon=False, **kwargs):
        super(SingleDend, self).build_sections(**kwargs)
        if axon:
            create_axon(self, **kwargs)
        self.dend.append(Section(name='dend_1',
                                 L=dend_l, diam=dend_diam, nseg=dend_nseg, Ra=self.Ra, cm=self.cm,
                                 mechanisms=self.mechanisms,
                                 parent=self.soma))
        self.sections += self.dend


#!/usr/bin/env python3

import os
import rospy
import rospkg

from qt_gui.plugin import Plugin
from python_qt_binding import loadUi
from python_qt_binding.QtWidgets import QWidget, QMainWindow

from motor_testbench.msg import *
# import rosbag
import time

class ControlCenter(Plugin):

    def __init__(self, context):
        # rospy.init_node('QtGUI_node', anonymous = True)
        super(ControlCenter, self).__init__(context)
        self.setObjectName('ControlCenter')

        # Create window
        self._widget = QMainWindow()
        # Load the UI resource
        ui_file = os.path.join(rospkg.RosPack().get_path('control_center'), 'resource', 'motor_testbench.ui')
        loadUi(ui_file, self._widget)
        self._widget.setObjectName('ControlCenterUi')

        # Add plugin pane name/number
        if context.serial_number() > 1:
            self._widget.setWindowTitle(self._widget.windowTitle() + (' (%d)' % context.serial_number()))
        context.add_widget(self._widget)

        # Connect the slider
        self.sl_joint_pos = self._widget.sliderj1
        self.sl_joint_pos.setMinimum(-180)
        self.sl_joint_pos.setMaximum(180)

        self.sl_kd_val = self._widget.kdSlider
        self.sl_kd_val.setMinimum(0)
        self.sl_kd_val.setMaximum(5)

        self.sl_kp_val = self._widget.kpSlider
        self.sl_kp_val.setMinimum(0)
        self.sl_kp_val.setMaximum(500)

        # Connect to the motorID input box
        self.motorID_input = self._widget.motorID_input

        # Connect the buttons
        self._widget.motor_connect.clicked[bool].connect(self.on_click_connect)
        self._widget.enable.clicked[bool].connect(self.on_click_enable)
        self._widget.disable.clicked[bool].connect(self.on_click_disable)
        self._widget.setZero.clicked[bool].connect(self.on_click_setZero)
        self._widget.sendControl.clicked[bool].connect(self.on_click_move)
        self.cb_enable_control = self._widget.enableControl
        self.cb_enable_control.toggled.connect(self.on_enable_control)

        # Message config
        self.state_pub_duration = 0.4 # s
        self.command_pub_duration = 0.15 # s
        self.rate = rospy.Rate(50)

        # Setup useful variables
        self.enable_control = True

    def on_click_connect(self):
        # # Make the motor number a slot into which the plugin connects to, that way we can yse the same plugin multiple times
        # Select the motorID upon an connect button is clicked
        self.motorID = int(self.motorID_input.toPlainText())

        # Initiate the corresponding state publishers
        self.state_channel = rospy.Publisher('motor_state/'+str(self.motorID)+'/', tmotor_cmd, queue_size=10)
        # Initiate control publishers
        self.command_channel = rospy.Publisher('motor_command/'+str(self.motorID)+'/', tmotor_cmd, queue_size = 10)

        # Subscribe to the response on CAN
        rospy.Subscriber('can_response/motor_'+str(self.motorID), tmotor_data, self.motor_response_callback)

    # Modify sliders based on if the control mode is enabled or not
    def on_enable_control(self):
        self.enable_control = self.cb_enable_control.isChecked()

    def on_click_enable(self):
        cmd = tmotor_cmd()
        cmd.status = True

        self.send_multiple_messages(cmd, self.state_channel, self.state_pub_duration)
        cmd = tmotor_cmd()
        cmd.position = 0
        cmd.velocity = 0
        cmd.torque = 0
        cmd.kp = 0
        cmd.kd = 0
        self.send_multiple_messages(cmd, self.command_channel, self.command_pub_duration)

    def on_click_disable(self):
        cmd = tmotor_cmd()
        cmd.position = 0
        cmd.velocity = 0
        cmd.torque = 0
        cmd.kp = 0
        cmd.kd = 0
        self.send_multiple_messages(cmd, self.command_channel, self.command_pub_duration)
        cmd = tmotor_cmd()
        cmd.status = False
        self.send_multiple_messages(cmd, self.state_channel, self.state_pub_duration)

    def on_click_setZero(self):
        cmd = tmotor_cmd()
        cmd.status = True
        cmd.setzero = True
        self.send_multiple_messages(cmd, self.state_channel, self.state_pub_duration)
        cmd = tmotor_cmd()
        cmd.position = 0
        cmd.velocity = 0
        cmd.torque = 0
        cmd.kp = 0
        cmd.kd = 0
        self.send_multiple_messages(cmd, self.command_channel, self.command_pub_duration)
        # Come out of set zero by re-arming the motor again
        # self.on_click_enable()

    def on_click_move(self):
        # Check if control is enabled
        if self.enable_control:
            cmd = tmotor_cmd()
            # Read the values from the sliders
            cmd.position = self.sl_joint_pos.value()
            cmd.velocity = 0
            cmd.torque = 0
            cmd.kp = self.sl_kp_val.value()
            cmd.kd = self.sl_kd_val.value()
            self.send_multiple_messages(cmd, self.command_channel, self.command_pub_duration)
        else:
            raise Exception('Control not enabled')

    # This is to ensure that the messages are reached even
    # with the constant rtr pings for data
    def send_multiple_messages(self, cmd, channel, pub_duration):
        # Send only one message, since RTR is off this should work as predicted
        channel.publish(cmd)
        # For continuous publish
        # # get current time
        # now = time.time()
        # # publish message for a preset duration of time
        # while ((time.time()-now) < pub_duration):
        #     channel.publish(cmd)
        #     self.rate.sleep()

    def motor_response_callback(self, data):
        # Also use the status to show color bulbs
        # The updated values pos, vel and torque can be stored here if required
        # If control is disabled
        self._widget.ui_pos.setText(str(data.position))
        self._widget.ui_vel.setText(str(data.velocity))
        self._widget.ui_tau.setText(str(data.torque))
        if not self.enable_control:
            # Set slider value based on the incoming encoder value
            self.sl_joint_pos.setValue(data.position)

    # TODO implementation of necessary routines
    def shutdown_plugin(self):
        # TODO unregister all publishers here
        pass

    def save_settings(self, plugin_settings, instance_settings):
        # TODO save intrinsic configuration, usually using:
        # instance_settings.set_value(k, v)
        pass

    def restore_settings(self, plugin_settings, instance_settings):
        # TODO restore intrinsic configuration, usually using:
        # v = instance_settings.value(k)
        pass


'''
Loop over this for a row like this:

for c in range(1, 27):
        sheet.merge_cells(start_col=c+1, start_row=2, end_col=c+2, end_row=2)
        '''

nl = '\n'
print(nl)
# sheet.unmerge_cells('A1:A5') # HELP: error unmerge

# imports
import os
import openpyxl
from openpyxl.workbook import Workbook
from openpyxl.styles.colors import Color
from openpyxl.styles import PatternFill, Border, Side, Alignment, Protection, Font, GradientFill, NamedStyle

# Set directory
print(os.getcwd())
os.chdir('C:\\Users\\pcurtis7\\Desktop\\_myScripts\\python_excel')
print(os.getcwd(), nl)

# create workbook
wb = openpyxl.Workbook()
print('type(workbook):', type(wb))
print('all.sheetnames:', wb.sheetnames)

# set sheets
sheet = wb.active
sheet.title = 'Daily_Rounds'
s0 = sheet
s1 = wb.create_sheet('Colors')
print('all.sheetnames:', wb.sheetnames)
print('wb.index(s0/Daily_Rounds) =', wb.index(s0))
print('wb.index(s1/Colors) =', wb.index(s1))

# help: how do you change the active sheet?
wb.active = wb.index(s0)
print('sheet/wb.active =', sheet)
wb.active = wb.index(s1)
print('sheet/wb.active =', sheet)
print('s1[1] =', s1[1])

wb.save('daily_log_v2_loops.xlsx')

# General styles
sheet.page_setup.orientation = sheet.ORIENTATION_PORTRAIT
highlight = NamedStyle(name="highlight")
highlight.font = Font(bold=True, size=12)
bd = Side(style='thick', color="000000")
highlight.border = Border(left=bd, top=bd, right=bd, bottom=bd)
# wb.add_named_style(highlight) # confirm if this adds to entire workbook
sheet['A1'].style = highlight
sheet['A41'].style = 'highlight'

# Merge Cells
# sheet.merge_cells(start_row=, start_column=, end_row=, end_column=)
# Rows 6-21

for c in range(1, 27):
        sheet.merge_cells(start_col=c+1, start_row=2, end_col=c+2, end_row=2)

''
start_col = [2, 4, 6, 8]
end_col = [x + 1 for x in start_col]

for i in range (1, 6):
  if i in [1, 4]:
    sheet.merge_cells(start_row=i, start_column=1, end_row=i, end_column=9)
  else:
    for col1, col2 in zip(start_col, end_col):
      sheet.merge_cells(start_row=i, start_column=col1, end_row=i, end_column=col2)
'''
Create a 'start' list with 2, 4, 6, & 8 integers
create an 'end' list that adds +1 for every integer in the 'start' list
For every value (named i) within the range 1 to 6, but not including 6:
  if i is either 1 or 4:
    merge cells starting with row range from 1 to 5, going from columns 1 to 9
  or else:
    for variables col1 & col2 in the zip() function calling start_col & end_col:
      merge cells within row i (range from 1 to 5), going from col1 to col2

I'm not exactly clear on everything here, specifically the objective of the zip() function and what value the col1 and col2 have.

I see the zip() defination but it's still not clear (must be 02:30am!):
  "The zip() function take iterables (zero or more), makes iterator that aggregates elements based on the iterables passed, and returns an iterator of tuples.
  Return Value from zip()
The zip() function returns an iterator of tuples based on the iterable object.
If no parameters are passed, zip() returns an empty iterator
If a single iterable is passed, zip() returns an iterator of 1-tuples. Meaning, the number of elements in each tuple is 1.
If multiple iterables are passed, ith tuple contains ith Suppose, two iterables are passed; one iterable containing 3 and other containing 5 elements. Then, the returned iterator has 3 tuples. It's because iterator stops when shortest iterable is exhaused."

Is my description correct?  What am I missing?
'''

'''
sheet.merge_cells(start_row=6, start_column=2, end_row=6, end_column=3) # SR1 CW Loop
sheet.merge_cells(start_row=6, start_column=4, end_row=6, end_column=5) # SR1 CW Loop
sheet.merge_cells(start_row=6, start_column=6, end_row=6, end_column=7) # SR1 CW Loop
sheet.merge_cells(start_row=6, start_column=8, end_row=6, end_column=9) # SR1 CW Loop
sheet.merge_cells(start_row=10, start_column=2, end_row=10, end_column=3) # PDU 11
sheet.merge_cells(start_row=10, start_column=4, end_row=10, end_column=5) # PDU 11
sheet.merge_cells(start_row=10, start_column=6, end_row=10, end_column=7) # PDU 11
sheet.merge_cells(start_row=10, start_column=8, end_row=10, end_column=9) # PDU 11
sheet.merge_cells(start_row=11, start_column=2, end_row=11, end_column=3) # PDU 9
sheet.merge_cells(start_row=11, start_column=4, end_row=11, end_column=5) # PDU 9
sheet.merge_cells(start_row=11, start_column=6, end_row=11, end_column=7) # PDU 9
sheet.merge_cells(start_row=11, start_column=8, end_row=11, end_column=9) # PDU 9
sheet.merge_cells(start_row=12, start_column=2, end_row=12, end_column=3) # PDU 2
sheet.merge_cells(start_row=12, start_column=4, end_row=12, end_column=5) # PDU 2
sheet.merge_cells(start_row=12, start_column=6, end_row=12, end_column=7) # PDU 2
sheet.merge_cells(start_row=12, start_column=8, end_row=12, end_column=9) # PDU 2
sheet.merge_cells(start_row=13, start_column=2, end_row=13, end_column=3) # PDU 4
sheet.merge_cells(start_row=13, start_column=4, end_row=13, end_column=5) # PDU 4
sheet.merge_cells(start_row=13, start_column=6, end_row=13, end_column=7) # PDU 4
sheet.merge_cells(start_row=13, start_column=8, end_row=13, end_column=9) # PDU 4
sheet.merge_cells(start_row=17, start_column=2, end_row=17, end_column=3) # PDU 1
sheet.merge_cells(start_row=17, start_column=4, end_row=17, end_column=5) # PDU 1
sheet.merge_cells(start_row=17, start_column=6, end_row=17, end_column=7) # PDU 1
sheet.merge_cells(start_row=17, start_column=8, end_row=17, end_column=9) # PDU 1
sheet.merge_cells(start_row=18, start_column=2, end_row=18, end_column=3) # PDU 8
sheet.merge_cells(start_row=18, start_column=4, end_row=18, end_column=5) # PDU 8
sheet.merge_cells(start_row=18, start_column=6, end_row=18, end_column=7) # PDU 8
sheet.merge_cells(start_row=18, start_column=8, end_row=18, end_column=9) # PDU 8
'''
wb.save('daily_log_v2_loops.xlsx')


import pandas as pd
from django.shortcuts import render
from rest_framework.generics import ListCreateAPIView, UpdateAPIView
from rest_framework.response import Response
from rest_framework import authentication, permissions
from rest_framework.response import Response
from sklearn.preprocessing import MinMaxScaler
import random
from django.views.decorators.csrf import ensure_csrf_cookie

from datetime import datetime
from Seasonality import daily_seasonality, weekly_seasonality
from timeseries.models import *
from .Serializers import *
from rest_framework.decorators import action
from Seasonality.daily_seasonality import Daily_Seasonality
from Seasonality.weekly_seasonality import Weekly_Seasonality
from .models import *
from Cycles_and_trends.cycle import cycle
from Cycles_and_trends.trends import trends

from Generator.Time_Series import Time_Series
from Noise.missing_values import missing_values
from Noise.noise import noise
from Noise.outliers import outliers
from Save.csv_file import csv_file
# Create your views here.
class simulator(ListCreateAPIView, UpdateAPIView):
    queryset = simulator_details.objects.all()
    serializer_class = timeseries_serializer

    def create(self, request, *args, **kwargs):
        data=request.data
        ts_serial = timeseries_serializer(data=data)
        configuration=data.pop('configuration')[0]
        seaonality=configuration.pop('seasonality')[0]
        #ts_serial=timeseries_serializer(data=data)
        if ts_serial.is_valid():
            ts_serial.save()
        configurationserializer=configuration_serializer(data=configuration)
        if configurationserializer.is_valid():
            configurationserializer.save()
        seasonalityserializer=seasonality_serializer(data=seaonality)
        if seasonalityserializer.is_valid():
            seasonalityserializer.save()

        return Response("Time series added",status=200)

    @action(detail=True, methods=['GET'])

    def run(self,request,*args, **kwargs):
        id = request.Get.get('id',-1)

        data = simulator_details.objects.filter(id=id)[0]

        #ts_serial = timeseries_serializer(data=data)
        configuration = data.pop('configuration')[0]
        seaonality = configuration.pop('seasonality')[0]
        Start_date = data["start_date"]
        End_date = data["end_date"]
        use_case_name = data['use_case_name']
        frequencies = configuration["Frequency"]

        frequency_type = seaonality["frequency_type"]
        phase_shift = seaonality["phase_shift"]
        amplitude = seaonality["amplitude"]
        noise_levels = [configuration["Noise_level"]]
        trend_levels = [configuration["Trend_coefficient"]]
        cyclic_periods = [configuration["Cycle_frequency"]]
        data_types = data["time_series_type"]
        percentage_outliers_options = [configuration["Outlier_percentage"]]
        Start_date = datetime.strptime(Start_date, "%Y-%m-%d")
        End_date = datetime.strptime(End_date, "%Y-%m-%d")
        data_size = (End_date - Start_date).days
        counter = 0
        meta_data = []
        for noise_level in noise_levels:
            for trend in trend_levels:
                for cyclic_period in cyclic_periods:
                    for percentage_outliers in percentage_outliers_options:
                        for data_type in data_types:
                            for _ in range(13):

                                counter += 1

                                # Generate time index

                                series = Time_Series(Start_date, End_date, frequencies)
                                date_rng = series.create_time_series()
                                daily_seasonal_component = 1
                                weekly_seasonal_component = 1
                                # Create components
                                if frequency_type == "Daily":

                                    Daily_seasonality = Daily_Seasonality()
                                    daily_seasonal_component = Daily_seasonality.add_daily_seasonality(date_rng)
                                elif frequency_type == "Weekly":
                                    Weekly_seasonality = Weekly_Seasonality()
                                    weekly_seasonal_component = Weekly_seasonality.add_weekly_seasonality(date_rng)
                                trnd = trends(trend, data_size=data_size,
                                              data_type=data_type)
                                trend_component = trnd.add_trend(date_rng)
                                cyclic_period = "exist"
                                Cycle = cycle(cyclic_period, season_type=data_type)
                                cyclic_component = Cycle.add_cycles(date_rng)

                                # Combine components and add missing values and outliers
                                if data_type == 'multiplicative':
                                    data = daily_seasonal_component * weekly_seasonal_component * trend_component * cyclic_component
                                else:
                                    data = daily_seasonal_component + weekly_seasonal_component + trend_component + cyclic_component
                                    # Create a MinMaxScaler instance
                                    scaler = MinMaxScaler(feature_range=(-1, 1))
                                    data = scaler.fit_transform(data.values.reshape(-1, 1))
                                    Noise = noise(noise_level)
                                    data = Noise.add_noise(data)
                                    Outliers = outliers(percentage_outliers)
                                    data, anomaly = Outliers.add_outliers(data)
                                    Missing_Value = missing_values(0.05)
                                    data = Missing_Value.add_missing_values(data)
                                    df = pd.DataFrame({'value': data, 'timestamp': date_rng, 'anomaly': anomaly})
                                    file = csv_file('sample_datasets/' + str(counter) + '.csv', df)
                                    file.save()
                                    meta_data.append({'id': str(counter) + '.csv',
                                                      'data_type': data_type,
                                                      'daily_seasonality': daily_seasonality,
                                                      'weekly_seasonality': weekly_seasonality,
                                                      'noise (high 30% - low 10%)': noise_level,
                                                      'trend': trend,
                                                      'cyclic_period (3 months)': cyclic_period,

                                                      'percentage_outliers': percentage_outliers,
                                                      'percentage_missing': 0.05,
                                                      'freq': frequencies})
                                    # generate_csv(list(zip(date_rng, data)), file_name)

                                    meta_data_df = pd.DataFrame.from_records(meta_data)
                                    meta_data_file = csv_file('sample_datasets/meta_data.csv', meta_data_df)
                                    meta_data_file.save()

        return Response("Time series created",status=200)

    @action(detail=True, methods=['GET'])
    def stop(self,request):
        self.get_object()
        return


import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')

import django
django.setup()

from blog.models import Blog
from django.contrib.auth.models import User

user = User.objects.first()

def add_data(title, main_image, content, user, status):
    data, created = Blog.objects.get_or_create(title=title,
                                            main_image=main_image,
                                            content=content,
                                            status=status,
                                            author=user)
    print(f'Data: {data}, Created: {created}')

def populate():
    for i,_ in enumerate(range(20)):
        add_data(f'test data-{i}', f'{i}.jpg', f'test content-{i}', user, 'PU')

if __name__ == '__main__':
    populate()


#!/usr/bin/env python
#
#===- add_new_check.py - cpp-clippy check generator ---------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===-----------------------------------------------------------------------===#

from __future__ import print_function
from __future__ import unicode_literals

import argparse
import io
import os
import re
import sys

# Adapts the module's CMakelist file. Returns 'True' if it could add a new
# entry and 'False' if the entry already existed.
def adapt_cmake(module_path, check_name_camel):
  filename = os.path.join(module_path, 'CMakeLists.txt')

  # The documentation files are encoded using UTF-8, however on Windows the
  # default encoding might be different (e.g. CP-1252). To make sure UTF-8 is
  # always used, use `io.open(filename, mode, encoding='utf8')` for reading and
  # writing files here and elsewhere.
  with io.open(filename, 'r', encoding='utf8') as f:
    lines = f.readlines()

  cpp_file = check_name_camel + '.cpp'

  # Figure out whether this check already exists.
  for line in lines:
    if line.strip() == cpp_file:
      return False

  print('Updating %s...' % filename)
  with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
    cpp_found = False
    file_added = False
    for line in lines:
      cpp_line = line.strip().endswith('.cpp')
      if (not file_added) and (cpp_line or cpp_found):
        cpp_found = True
        if (line.strip() > cpp_file) or (not cpp_line):
          f.write('  ' + cpp_file + '\n')
          file_added = True
      f.write(line)

  return True

# Adds a header for the new check.
def write_header(module_path, module, namespace, check_name, check_name_camel):
  filename = os.path.join(module_path, check_name_camel) + '.h'
  print('Creating %s...' % filename)
  with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
    header_guard = ('CPP_CLIPPY_SRC_' + module.upper() + '_'
                    + check_name_camel.upper() + '_H_')
    f.write('//===--- ')
    f.write(os.path.basename(filename))
    f.write(' - cpp-clippy ')
    f.write('-' * max(0, 42 - len(os.path.basename(filename))))
    f.write('*- C++ -*-===//')
    f.write("""
//
// Part of the Cpp-Clippy Project, under the Apache License v2.0.
// See https://github.com/cpp-clippy/cpp-clippy/LICENSE.txt for
// license information.
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//

#ifndef %(header_guard)s
#define %(header_guard)s

#include "../ClangTidyCheck.h"

namespace clang {
namespace tidy {
namespace %(namespace)s {

/// FIXME: Write a short description.
///
/// For the user-facing documentation see:
/// https://cpp-clippy.gitbook.io/book/rules/%(module)s/%(check_name)s
class %(check_name_camel)s : public ClangTidyCheck {
public:
  %(check_name_camel)s(StringRef Name, ClangTidyContext *Context)
      : ClangTidyCheck(Name, Context) {}
  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
};

} // namespace %(namespace)s
} // namespace tidy
} // namespace clang

#endif // %(header_guard)s
""" % {'header_guard': header_guard,
       'check_name_camel': check_name_camel,
       'check_name': check_name,
       'module': module,
       'namespace': namespace})

# Adds the implementation of the new check.
def write_implementation(module_path, module, namespace, check_name_camel):
  filename = os.path.join(module_path, check_name_camel) + '.cpp'
  print('Creating %s...' % filename)
  with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
    f.write('//===--- ')
    f.write(os.path.basename(filename))
    f.write(' - cpp-clippy ')
    f.write('-' * max(0, 51 - len(os.path.basename(filename))))
    f.write('-===//')
    f.write("""
//
// Part of the Cpp-Clippy Project, under the Apache License v2.0.
// See https://github.com/cpp-clippy/cpp-clippy/LICENSE.txt for
// license information.
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//

#include "%(check_name)s.h"
#include "clang/AST/ASTContext.h"
#include "clang/ASTMatchers/ASTMatchFinder.h"

using namespace clang::ast_matchers;

namespace clang {
namespace tidy {
namespace %(namespace)s {

void %(check_name)s::registerMatchers(MatchFinder *Finder) {
  // FIXME: Add matchers.
  Finder->addMatcher(functionDecl().bind("x"), this);
}

void %(check_name)s::check(const MatchFinder::MatchResult &Result) {
  // FIXME: Add callback implementation.
  const auto *MatchedDecl = Result.Nodes.getNodeAs<FunctionDecl>("x");
  if (!MatchedDecl->getIdentifier() || MatchedDecl->getName().startswith("awesome_"))
    return;
  diag(MatchedDecl->getLocation(), "function %%0 is insufficiently awesome")
      << MatchedDecl;
  diag(MatchedDecl->getLocation(), "insert 'awesome'", DiagnosticIDs::Note)
      << FixItHint::CreateInsertion(MatchedDecl->getLocation(), "awesome_");
}

} // namespace %(namespace)s
} // namespace tidy
} // namespace clang
""" % {'check_name': check_name_camel,
       'module': module,
       'namespace': namespace})

# Returns the source filename that implements the module.
def get_module_filename(module_path, module):
  modulecpp = list(filter(
      lambda p: p.lower() == module.lower() + 'clippymodule.cpp',
      os.listdir(module_path)))[0]
  return os.path.join(module_path, modulecpp)

# Modifies the module to include the new check.
def adapt_module(module_path, module, check_name, check_name_camel):
  filename = get_module_filename(module_path, module)
  with io.open(filename, 'r', encoding='utf8') as f:
    lines = f.readlines()

  print('Updating %s...' % filename)
  with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
    header_added = False
    header_found = False
    check_added = False
    check_fq_name = module + '-' + check_name
    check_decl = ('    CheckFactories.registerCheck<' + check_name_camel +
                  '>(\n        "' + check_fq_name + '");\n')

    lines = iter(lines)
    try:
      while True:
        line = next(lines)
        if not header_added:
          match = re.search('#include "(.*)"', line)
          if match:
            header_found = True
            if match.group(1) > check_name_camel:
              header_added = True
              f.write('#include "' + check_name_camel + '.h"\n')
          elif header_found:
            header_added = True
            f.write('#include "' + check_name_camel + '.h"\n')

        if not check_added:
          if line.strip() == '}':
            check_added = True
            f.write(check_decl)
          else:
            match = re.search('registerCheck<(.*)> *\( *(?:"([^"]*)")?', line)
            prev_line = None
            if match:
              current_check_name = match.group(2)
              if current_check_name is None:
                # If we didn't find the check name on this line, look on the
                # next one.
                prev_line = line
                line = next(lines)
                match = re.search(' *"([^"]*)"', line)
                if match:
                  current_check_name = match.group(1)
              if current_check_name > check_fq_name:
                check_added = True
                f.write(check_decl)
              if prev_line:
                f.write(prev_line)
        f.write(line)
    except StopIteration:
      pass

# Adds a release notes entry.
def add_release_notes(module_path, module, check_name):
  check_name_dashes = module + '-' + check_name
  filename = os.path.normpath(os.path.join(module_path,
                                           '../../docs/ReleaseNotes.rst'))
  with io.open(filename, 'r', encoding='utf8') as f:
    lines = f.readlines()

  lineMatcher = re.compile('New checks')
  nextSectionMatcher = re.compile('New check aliases')
  checkMatcher = re.compile('- New :doc:`(.*)')

  print('Updating %s...' % filename)
  with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
    note_added = False
    header_found = False
    add_note_here = False

    for line in lines:
      if not note_added:
        match = lineMatcher.match(line)
        match_next = nextSectionMatcher.match(line)
        match_check = checkMatcher.match(line)
        if match_check:
          last_check = match_check.group(1)
          if last_check > check_name_dashes:
            add_note_here = True

        if match_next:
          add_note_here = True

        if match:
          header_found = True
          f.write(line)
          continue

        if line.startswith('^^^^'):
          f.write(line)
          continue

        if header_found and add_note_here:
          if not line.startswith('^^^^'):
            f.write("""- New :doc:`%s
  <clang-tidy/checks/%s/%s>` check.

  FIXME: add release notes.

""" % (check_name_dashes, module, check_name))
            note_added = True

      f.write(line)

# Adds a test for the check.
def write_test(module_path, module, check_name, test_extension):
  check_name_dashes = module + '-' + check_name
  filename = os.path.normpath(os.path.join(
    module_path, '..', '..', 'test',
    module, check_name + '.' + test_extension))
  print('Creating %s...' % filename)
  with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
    f.write("""// RUN: %%check_cpp_clippy %%s %(check_name_dashes)s %%t

// FIXME: Add something that triggers the check here.
void f();
// CHECK-MESSAGES: :[[@LINE-1]]:6: warning: function 'f' is insufficiently awesome [%(check_name_dashes)s]

// FIXME: Verify the applied fix.
//   * Make the CHECK patterns specific enough and try to make verified lines
//     unique to avoid incorrect matches.
//   * Use {{}} for regular expressions.
// CHECK-FIXES: {{^}}void awesome_f();{{$}}

// FIXME: Add something that doesn't trigger the check here.
void awesome_f2();
""" % {'check_name_dashes': check_name_dashes})

def get_actual_filename(dirname, filename):
  if not os.path.isdir(dirname):
    return ''
  name = os.path.join(dirname, filename)
  if (os.path.isfile(name)):
    return name
  caselessname = filename.lower()
  for file in os.listdir(dirname):
    if (file.lower() == caselessname):
      return os.path.join(dirname, file)
  return ''

# Recreates the list of checks in the book/src/SUMMARY.md.
def update_checks_list(clang_tidy_path):
  filename = os.path.normpath(os.path.join(clang_tidy_path, '../book/src/SUMMARY.md'))
  docs_dir = os.path.join(clang_tidy_path, '../book/src/rules')
  # Read the content of the current list.rst file
  with io.open(filename, 'r', encoding='utf8') as f:
    lines = f.readlines()
  # Get all existing docs
  doc_files = []
  a = os.listdir(docs_dir)
  for subdir in list(filter(lambda s: not s.endswith('.md') and not s.endswith('.py'),
                     os.listdir(os.path.join(docs_dir)))):
    for file in filter(lambda s: s.endswith('.md'), os.listdir(os.path.join(docs_dir, subdir))):
      doc_files.append([subdir, file])
  doc_files.sort()

  print('Updating %s...' % filename)
  with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
    for line in lines:
      if '(rules/' in line:
        continue

      f.write(line)
      if line.strip() == '## Rules':
        for doc_file in doc_files:
          module = doc_file[0]
          check_file = doc_file[1].replace('.md', '')

          if check_file == 'README':
            f.write('\n* [%(module_name)s](rules/%(module)s/%(check_file)s.md)' % {
              'module_name': module.capitalize(),
              'module': module,
              'check_file': check_file,
            })
          else:
            f.write('\n  * [Rule %(rule_name)s](rules/%(module)s/%(check_file)s.md)' % {
              'rule_name': check_file.upper(),
              'module': module,
              'check_file': check_file,
            })

# Adds a documentation for the check.
def write_docs(module_path, module, check_name):
  check_name_dashes = module + '-' + check_name
  filename = os.path.normpath(os.path.join(
      module_path, '../../book/src/rules/', module, check_name + '.md'))
  print('Creating %s...' % filename)
  with io.open(filename, 'w', encoding='utf8', newline='\n') as f:
    f.write("""---
description: FIXME
---

# Rule %(check_name_dashes)s

* **Classification:** FIXME [_Required_ / _Advisory_]
* **Offers Fixes:** FIXME [_Yes_ / _No_]

## What it does?

FIXME

## Why is this bad?

FIXME

## Example

```cpp
FIXME
```

### See also

FIXME
""" % {'check_name_dashes': check_name_dashes})

def get_camel_name(check_name):
  return ''.join(map(lambda elem: elem.capitalize(),
                     check_name.split('-')))

def get_camel_check_name(check_name):
  return get_camel_name(check_name) + 'Check'

def main():
  language_to_extension = {
      'c': 'c',
      'c++': 'cpp',
      'objc': 'm',
      'objc++': 'mm',
  }
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--update-docs',
      action='store_true',
      help='just update the list of documentation files, then exit')
  parser.add_argument(
      '--language',
      help='language to use for new check (defaults to c++)',
      choices=language_to_extension.keys(),
      default='c++',
      metavar='LANG')
  parser.add_argument(
      'module',
      nargs='?',
      help='module directory under which to place the new tidy check (e.g., misc)')
  parser.add_argument(
      'check',
      nargs='?',
      help='name of new tidy check to add (e.g. foo-do-the-stuff)')
  args = parser.parse_args()

  if args.update_docs:
    update_checks_list(os.path.dirname(sys.argv[0]))
    return

  if not args.module or not args.check:
    print('Module and check must be specified.')
    parser.print_usage()
    return

  module = args.module
  check_name = args.check
  check_name_camel = get_camel_check_name(check_name)
  if check_name.startswith(module):
    print('Check name "%s" must not start with the module "%s". Exiting.' % (
        check_name, module))
    return
  clang_tidy_path = os.path.dirname(sys.argv[0])
  module_path = os.path.join(clang_tidy_path, module)

  if not adapt_cmake(module_path, check_name_camel):
    return

  # Map module names to namespace names that don't conflict with widely used top-level namespaces.
  if module == 'llvm':
    namespace = module + '_check'
  else:
    namespace = module

  write_header(module_path, module, namespace, check_name, check_name_camel)
  write_implementation(module_path, module, namespace, check_name_camel)
  adapt_module(module_path, module, check_name, check_name_camel)
  # add_release_notes(module_path, module, check_name)
  test_extension = language_to_extension.get(args.language)
  write_test(module_path, module, check_name, test_extension)
  write_docs(module_path, module, check_name)
  update_checks_list(clang_tidy_path)
  print('Done. Now it\'s your turn!')

if __name__ == '__main__':
  main()


#!/usr/bin/env python
# encoding: utf-8

from multiprocessing import Process
import time
import datetime
# from server_link import LinkInfo
from server_limit import Limit
from server_rand import generateRandom
# from own_exception import dbEmpty
from logger import logger
from params import dataScheme
logger = logger('server_control', file_level=None, stream_level='DEBUG')

class ControlProcess(Process, dataScheme):
    """ """
    def __init__(self, *args, **kwargs):
        """ """
        dataScheme.__init__(self)
        Process.__init__(self)
        self.loop_interval = args[0]
        self.time_lvl2 = int(args[1])
        self.set_lvl1 = args[2]
        self.loop_number = args[3]
        self.nbr_client = args[4]
        self.db_process_profile = kwargs['process_profile']
        self.db_process_link = kwargs['process_link']
        self.db_process_tweet = kwargs['process_tweet']
        self.db_current_prof = kwargs['current_profile']
        self.db_current_link = kwargs['current_link']
        self.db_rand_2 = kwargs['rand_lvl2']
        self.db_rand_3 = kwargs['rand_lvl3']
        self.do_limit = Limit(self.loop_interval, max_user=True)
        self.limit = self.do_limit.calculing_limit()
        self.create_lock_file()

    def create_lock_file(self):
        """
        Create a lock file with a false value
        """
        with open('.lock_server_control', 'w') as f:
            f.write('false')

    def process_limit(self, nbr_client, size):
        """ """
        limit = self.do_limit.calculing_limit(nbr_client, size)
        logger.info('Max size sample. lvl1: {} - lvl2: {} - lvl3 - {}'.format(limit['lvl1'],
                                                                              limit['lvl2'],
                                                                              limit['lvl3']))
        return limit['lvl2'], limit['lvl3']

    def record_sample(self, loop_number, lvl2_size, lvl3_size):
        """
        Record the sample size taken for the loop
        """
        with open('sample_size', 'a') as f:
            f.write('{}::{}::LVL2 size - {} :: LVL3 size - {}'.format(datetime.datetime.now(),
                                                                      loop_number,
                                                                      lvl2_size,
                                                                      lvl3_size))
            f.write('\n')

    def check_lock(self):
        """
        Try to open the file with the client_id name, Check if a value
        "true" is in it, which means it has to pause. It helps to kill the
        running instance or doing other modification somewhere else
        """
        try:
            with open('.lock_server_control', 'r') as f:
                lock = f.readline().rstrip()
                if lock == 'true':
                    logger.info('Lock file is true, pause for 1 minute')
                    time.sleep(50)
                    logger.info('Start again in 10 sec')
                    time.sleep(5)
                    logger.info('Start again in 5 sec')
                    time.sleep(5)
        except FileNotFoundError:
            pass

    def keep_looking(self, db, query, lap_time=60):
        """
        Loop to keep checking until no record are return
        then exit
        """
        def check_todo(db, query):
            """ Check if there is still some profile or link to do """
            if query == 'todo':
                query = {'doing': {'$exists': False}, 'extra': {'$exists': False}}
            elif query == 'doing':
                query = {}
            return bool(db.find_one(query))

        lock = True
        while lock is True:
            lock = check_todo(db, query)
            time.sleep(lap_time)
        return

    def remove_extra_remaining(self, db):
        """ """
        query = query = {'doing': {'$exists': False}, 'extra': {'$exists': True}}
        db.delete_many(query)

    def recreate_lvl1(self):
        """
        Function to copy the set of lvl1 id_str created from the beggining
        to the new loop. Be sure to have the right one all the time instead of
        loosing it with some twitter errors
        """

        def generate_query(result):
            """
            Need to recreate the query after every insert otherwise some
            field are removed after the insertion in a db
            """
            query = result
            return query

        for id_str in self.set_lvl1:
            info_user = dict()
            info_user['id_str'] = int(id_str)
            info_user['loop_number'] = self.loop_number
            update ={'$unset': {'extra': ''}}

            search = generate_query(info_user)
            self.db_process_profile.update_one(search, update, upsert=True)
            search = generate_query(info_user)
            self.db_process_tweet.update_one(search, update, upsert=True)

            info_user['type_link'] = 'followers'
            search = generate_query(info_user)
            self.db_process_link.update_one(search, update, upsert=True)

            info_user['type_link'] = 'friends'
            search = generate_query(info_user)
            self.db_process_link.update_one(search, update, upsert=True)

    def remove_previous_loop(self, db):
        """ """
        db.delete_many({self.loop: self.loop_number})

    def run(self):
        """ """
        while True:
            # Continue to do activity and updating storing while client
            # Still have to do twitter call
            logger.info('New loop: {}'.format(self.loop_number))

            # logger.info('Clean the Rand_lvl2 db')
            # self.remove_previous_loop(self.db_rand_2)

            # logger.info('Clean the Rand_lvl3 db')
            # self.remove_previous_loop(self.db_rand_3)

            logger.info('Get lock until all links are finished')
            self.keep_looking(self.db_process_link, 'todo')
            self.check_lock()

            logger.info('Get lock until all profiles are finished')
            self.keep_looking(self.db_process_profile, 'todo')

            logger.info('Remove extra links')
            self.remove_extra_remaining(self.db_process_link)
            self.check_lock()

            logger.info('Remove extra profile')
            self.remove_extra_remaining(self.db_process_profile)
            self.check_lock()

            logger.info('Remove extra tweet')
            self.remove_extra_remaining(self.db_process_tweet)
            self.check_lock()

            logger.info('Wait until all links are done')
            self.keep_looking(self.db_process_link, 'doing')
            self.check_lock()

            logger.info('Wait until all profile are done')
            self.keep_looking(self.db_process_profile, 'doing')
            self.check_lock()

            logger.info('Calculate the limit of lvl2 and lvl3')
            lvl2_sample_size, lvl3_sample_size = self.process_limit(len(self.nbr_client), len(self.set_lvl1))

            logger.info('Record the sample size')
            self.record_sample(self.loop_number + 1, lvl2_sample_size, lvl3_sample_size)
            self.check_lock()

            logger.info('Pause until all tweets are completely finished')
            self.keep_looking(self.db_process_tweet, 'todo')

            logger.info('Get lock until all tweets are finished')
            self.keep_looking(self.db_process_tweet, 'doing')

            logger.info('Start the rand process')
            rand_list = []
            rand_list.append(generateRandom(2, self.db_process_profile,
                                            self.db_process_link,
                                            self.db_process_tweet,
                                            self.db_rand_2, lvl2_sample_size,
                                            self.loop_number))
            rand_list.append(generateRandom(3, self.db_process_profile,
                                            self.db_process_link,
                                            self.db_process_tweet,
                                            self.db_rand_3, lvl3_sample_size,
                                            self.loop_number))

            for rand_process in rand_list:
                rand_process.start()
                rand_process.join()
            self.check_lock()

            self.loop_number += 1
            # reset the number of client
            self.nbr_client = set()

            logger.info('Update the lvl1 to todo')
            self.recreate_lvl1()
            self.check_lock()

def main():
    """ """
    pass

if __name__ == '__main__':
    main()


import os
import unittest

import carla
import srunner.challenge.utils.route_configuration_parser as parser
from srunner.challenge.challenge_evaluator_routes import ChallengeEvaluator
from srunner.challenge.utils.route_manipulation import interpolate_trajectory
from srunner.scenariomanager.carla_data_provider import CarlaActorPool
from srunner.scenariomanager.carla_data_provider import CarlaDataProvider

"""
The idea of this test is to check if sampling is able to sample random sequencial images
inside a batch

"""

class Arguments():
    def __init__(self):
        self.agent = None
        self.use_docker = False
        self.host = '127.0.0.1'
        self.port = 2000
        self.split = 'dev_track_1'

class TestRouteGenerator(unittest.TestCase):

    def __init__(self, name='runTest'):
        unittest.TestCase.__init__(self, name)
        self.root_route_file_position = 'srunner/challenge/'

    def test_route_parser(self):

        args = Arguments()
        client = carla.Client(args.host, int(args.port))
        client.set_timeout(25.0)
        challenge = ChallengeEvaluator(args)

        filename = os.path.join(self.root_route_file_position, 'all_towns_traffic_scenarios.json')
        world_annotations = parser.parse_annotations_file(filename)
        # retrieve routes
        # Which type of file is expected ????

        filename = os.path.join(self.root_route_file_position, 'routes_training.xml')
        list_route_descriptions = parser.parse_routes_file(filename)

        # For each of the routes to be evaluated.
        for route_description in list_route_descriptions:
            if route_description['town_name'] == 'Town03' or route_description['town_name'] == 'Town01':
                continue
            print (" TOWN: ", route_description['town_name'])
            challenge.world = client.load_world(route_description['town_name'])
            # Set the actor pool so the scenarios can prepare themselves when needed
            CarlaActorPool.set_world(challenge.world)

            CarlaDataProvider.set_world(challenge.world)
            # find and filter potential scenarios
            # Returns the interpolation in a different format

            challenge.world.wait_for_tick()

            gps_route, route_description['trajectory'] = interpolate_trajectory(challenge.world,
                                                                                route_description['trajectory'])

            potential_scenarios_definitions, existent_triggers = parser.scan_route_for_scenarios(route_description,
                                                                                world_annotations)

            for trigger_id, possible_scenarios in potential_scenarios_definitions.items():

                print ("  For trigger ", trigger_id, " --  ", possible_scenarios[0]['trigger_position'])
                for scenario in possible_scenarios:
                    print ("      ", scenario['name'])

            challenge.cleanup(ego=True)


import threading
import os

from ..utils.print import _print
from serial import Serial

FORWARD = "TF"
BACKWARD = "TB"
LEFT = "TL"
RIGHT = "TR"
TESTCO = "TG"
SENS = "SE"
RESP = "RE"
START = "ST"
LOAD_SEN = "LO"
REQUEST_CONFIG = "RC"
STOP_UPDATING = "SO";
ALLOW_UPDATING = "AL";

class ControlVer2:
  FILE = "./static/project.txt"

  def __init__(self, arduino : Serial):
    self.arduino = arduino

    self.srv = True
    self.ard = False
    threading.Thread(name="Arduino service", target=self.read) \
      .start()

    # Câu hiện tại
    self.current_sens = ""
    self.ip = None

    # Cập nhật câu đã lưu
    self.allow_update = True
    self.load_sen()

  @staticmethod
  def write(arduino, command):
    _print("Lệnh gửi sang arduino {}".format(command))
    arduino.write(bytes(command, "utf-8"))

  def read(self):
    while self.srv:
      command = self.arduino.readline().decode("utf-8") \
        .replace("\r\n", "")

      if command == RESP:
        # Gửi chuỗi câu sang cho thiết bị
        _print("Đang nạp câu mới")
        self.write(self.arduino, self.current_sens + ' #')

      if command == STOP_UPDATING:
        _print("Thiết bị đang hoạt động, không thể nạp câu mới vào thiết bị. Thông báo này có thể có trong quá trình thiết bị hoạt động")
        self.allow_update = False

      if command == ALLOW_UPDATING:
        _print("Quá trình đọc đã hoàn tất")
        self.allow_update = True

        # Gọi hàm này để nạp câu đã chạy
        self.update_sen(self.current_sens)

      if command == START:
        _print("Arduino đã sẵn sàng")
        self.ard = True

      if command == LOAD_SEN:
        if self.current_sens != "":
          _print("Tải câu đã lưu trữ {}".format(self.current_sens))
          self.update_sen(self.current_sens)

      if command == REQUEST_CONFIG:
        _print("Yêu cầu lấy cài đặt")
        if self.ard and self.ip != None:
          self.write(self.arduino, self.ip + '#')

      if command[0] == 'D':
        _print("Phản hồi từ thiết bị {}".format(command[1:]))

    # _print("Kết thúc dịch vụ Arduino")

  def update_sen(self, sen : str):
    '''
      Gọi phương thức này để cập nhật câu mới
    '''
    self.current_sens = sen.upper()

    if self.allow_update:
      self.write(self.arduino, SENS + '#')

    with open(self.FILE, "w", encoding="utf-8") as f:
      f.write(self.current_sens)

    return self.allow_update

  def sh_config(self, ip):
    '''
      Gọi hàm này để hiển thị dữ liệu trên LCD
    '''
    self.ip = ip

  def load_sen(self):
    if os.path.exists(self.FILE):
      with open(self.FILE, "r", encoding="utf-8") as f:
        sen = f.readline().replace("\r\n", "")
        # self.update_sen(sen)
        self.current_sens = sen
        _print("Đọc câu đã lưu trữ: {}".format(sen))


from datetime import datetime
import logging

import utility

#
# Scanner for MeteoVenezia weather stations alike stations
#
def scan_meteovenezia_alike(last_seen_timestamp, server, save=True, log=True):

  location_id=server["location_id"]
  server_name=server["name"]
  weather_station_url=server["url"]

  tree, _ =utility.get_tree(weather_station_url, location_id, server_name)
  if tree is None:
    return last_seen_timestamp

  timestamp_string=None
  timestamp_string_date=None
  timestamp_string_time=None
  try:
    timestamp_list = tree.xpath('/html/body/div[2]/table[2]/tbody/tr[1]/td[1]')
    timestamp_ele=timestamp_list[0].text.split('\xa0\xa0\xa0')
    timestamp_ele_1=timestamp_ele[0].strip()
    timestamp_ele_2=timestamp_ele[1].strip()

    timestamp_string=timestamp_ele_1+" "+timestamp_ele_2
    timestamp_obj=datetime.strptime(timestamp_string, "%d.%m.%Y %H:%M")

    timestamp_string=timestamp_obj.strftime("%d/%m/%Y %H:%M:%S")
    if last_seen_timestamp and last_seen_timestamp==timestamp_string:
      return last_seen_timestamp

    timestamp_string_date=timestamp_obj.strftime("%d/%m/%Y")
    timestamp_string_time=timestamp_obj.strftime("%H:%M:%S")

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting timestamp: "{e}"!')

  if timestamp_string==last_seen_timestamp:
    # Weather station is not updating data
    logging.info(f'{utility.get_identification_string(location_id, server_name)}, timestamp_string: {timestamp_string}, last_seen_timestamp: {last_seen_timestamp}, skip saving!')
    # TODO Raise an alert
    return last_seen_timestamp

  wind_speed_knots=None
  try:
    wind_speed_elem = tree.xpath('/html/body/div/table[2]/tbody/tr[3]/td[1]')
    wind_speed=wind_speed_elem[0].text.strip()
    if wind_speed:
      wind_speed_knots=float(wind_speed)

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting wind_speed_knots: "{e}"!')

  wind_gust_knots=None
  try:
    wind_gust_elem = tree.xpath('/html/body/div/table[2]/tbody/tr[3]/td[3]')
    wind_gust=wind_gust_elem[0].text.strip()
    if wind_gust:
      wind_gust_knots=float(wind_gust)

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting wind_gust_knots: "{e}"!')

  wind_direction_deg=None
  try:
    wind_direction_ele = tree.xpath('/html/body/div/table[2]/tbody/tr[3]/td[2]')
    wind_direction=wind_direction_ele[0].text.strip()
    wind_direction=wind_direction.split('°')[0].strip()
    if wind_direction:
      wind_direction_deg=float(wind_direction)

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting wind_direction_deg: "{e}"!')

  barometric_pressure_ssl_hPa=None
  try:
    barometric_pressure_ssl_ele = tree.xpath('/html/body/div/table[2]/tbody/tr[11]/td[2]')
    barometric_pressure_ssl=barometric_pressure_ssl_ele[0].text.split('hPa')[0].strip()
    if barometric_pressure_ssl:
      barometric_pressure_ssl_hPa=float(barometric_pressure_ssl)

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting barometric_pressure_ssl_hPa: "{e}"!')

  rain_today_mm=None
  try:
    rain_today_ele = tree.xpath('/html/body/div/table[2]/tbody/tr[12]/td[3]')
    rain_today=rain_today_ele[0].text.split(' ')[1].strip()
    if rain_today:
      rain_today_mm=float(rain_today)

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting rain_today_mm: "{e}"!')

  rain_rate_mmh=None
  try:
    rain_rate_ele = tree.xpath('/html/body/div/table[2]/tbody/tr[12]/td[2]')
    rain_rate=rain_rate_ele[0].text.split('mm/h')[0].strip()
    if rain_rate:
      rain_rate_mmh=float(rain_rate)

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting rain_rate_mmh: "{e}"!')

  temperature_cels=None
  try:
    temperature_ele=tree.xpath('/html/body/div/table[2]/tbody/tr[7]/td[2]')
    temperature=temperature_ele[0].text
    temperature=temperature.split('°')[0].strip()
    if temperature:
      temperature_cels=float(temperature)

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting temperature_cels: "{e}"!')

  rel_humidity=None
  try:
    humidity_ele = tree.xpath('/html/body/div/table[2]/tbody/tr[9]/td[2]')
    humidity=humidity_ele[0].text.split("%")[0].strip()
    if humidity:
      rel_humidity=float(humidity)/100

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting rel_humidity: "{e}"!')

  perceived_temperature_cels=None
  try:
    perceived_temperature_ele=tree.xpath("/html/body/div/table[2]/tbody/tr[8]/td[2]")
    perceived_temperature=perceived_temperature_ele[0].text.split('°')[0]
    if perceived_temperature:
      perceived_temperature_cels=float(perceived_temperature)

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting perceived_temperature_cels: "{e}"!')

  dew_point_cels=None
  try:
    dew_point_ele = tree.xpath('/html/body/div/table[2]/tbody/tr[10]/td[2]')
    dew_point=dew_point_ele[0].text.split('°')[0]
    if dew_point:
      dew_point_cels=float(dew_point)

  except Exception as e:
    logging.exception(f'{utility.get_identification_string(location_id, server_name)}, exception getting dew_point_cels: "{e}"!')

  meteo_data_dict={}
  meteo_data_dict["timestamp_string"]=timestamp_string
  meteo_data_dict["timestamp_string_date"]=timestamp_string_date
  meteo_data_dict["timestamp_string_time"]=timestamp_string_time
  meteo_data_dict["wind_speed_knots"]=wind_speed_knots
  meteo_data_dict["wind_direction_deg"]=wind_direction_deg
  meteo_data_dict["barometric_pressure_ssl_hPa"]=barometric_pressure_ssl_hPa
  meteo_data_dict["rain_today_mm"]=rain_today_mm
  meteo_data_dict["rain_rate_mmh"]=rain_rate_mmh
  meteo_data_dict["temperature_cels"]=temperature_cels
  meteo_data_dict["rel_humidity"]=rel_humidity
  meteo_data_dict["perceived_temperature_cels"]=perceived_temperature_cels
  meteo_data_dict["wind_gust_knots"]=wind_gust_knots
  meteo_data_dict["dew_point_cels"]=dew_point_cels

  if log:
    utility.log_sample(location_id, server_name, meteo_data_dict)

  if not utility.check_minimum_data(location_id, server_name, meteo_data_dict):
    return last_seen_timestamp

  utility.save(location_id, server_name, meteo_data_dict)
  return timestamp_string

if __name__=="__main__":
  utility.test_starter(10) # Location id


from airflow import DAG
from airflow.operators.bash import BashOperator
from datetime import datetime, timedelta

default_args = {
    'owner': 'RecSys-02',
    'depends_on_past': False,  # 이전 DAG의 Task가 성공, 실패 여부에 따라 현재 DAG 실행 여부가 결정. False는 과거의 실행 결과 상관없이 매일 실행한다
    'start_date': datetime(2023, 2, 1),
    'retires': 1,  # 실패시 재시도 횟수
    'retry_delay': timedelta(minutes=1)  # 만약 실패하면 5분 뒤 재실행
    # 'priority_weight': 10 # DAG의 우선 순위를 설정할 수 있음
    # 'end_date': datetime(2022, 4, 24) # DAG을 마지막으로 실행할 Date
    # 'execution_timeout': timedelta(seconds=300), # 실행 타임아웃 : 300초 넘게 실행되면 종료
    # 'on_failure_callback': some_function # 만약에 Task들이 실패하면 실행할 함수
    # 'on_success_callback': some_other_function
    # 'on_retry_callback': another_function
}

# with 구문으로 DAG 정의
with DAG(
        dag_id='Train',
        default_args=default_args,
        schedule_interval='0 15 * * *',
        tags=['RecSys-02-Dags']
) as dag:
    # BashOperator 사용
    train = BashOperator(
        task_id='Train',  # task의 id
        bash_command='python /opt/ml/git/final-project-level3-recsys-02/bentoml/run_finetune_full.py'  # 실행할 bash command
    )

    shell = BashOperator(
        task_id='Shell',
        bash_command='cd /opt/ml/git/final-project-level3-recsys-02/bentoml && poetry shell',
    )

    pakage = BashOperator(
        task_id='Pakage',
        bash_command='python /opt/ml/git/final-project-level3-recsys-02/bentoml/my_bento_packer.py',
    )

    termination = BashOperator(
        task_id='Termination',
        bash_command='pkill -9 bentoml'
    )

    server = BashOperator(
        task_id='Server',
        bash_command='bentoml serve --port 30001 MyService:latest',
        depends_on_past=False
    )

    train >> shell >> pakage >> termination >> server


from base64 import b64encode
from logging import getLogger
from os import remove, makedirs
from os.path import join, exists
from re import search

from decouple import config
from requests import get, post
from bs4 import BeautifulSoup

logger = getLogger(__name__)

class CapturarDados:
    """
    Classe responsável por capturar e processar dados do site do desafio, incluindo o download de anexos
    e conversão para Base64.
    """

    def __init__(self):
        """
        Inicializa a classe com o token e cabeçalhos padrão para as requisições.
        """
        self._token = None
        self._headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                          "Chrome/114.0.0.0 Safari/537.36 OPR/100.0.0.0"
        }

    def _capturar_token(self):
        """
        Captura o token CSRF do site e armazena internamente na classe.
        """
        response = get(url=config('BASE_URL'), headers=self._headers)
        conteudo_html = response.text
        try:
            self._token = search(pattern=r'(?<=csrf\"\svalue=\")(.*?)(?=")', string=conteudo_html).group()
        except AttributeError:
            logger.error("Não conseguiu capturar o token do site!")
            self._token = None

    def _consultar_codigo(self, codigo: int):
        """
        Consulta um código específico no site e retorna o conteúdo da página.
        """
        payload = {
            "csrf": self._token,
            "codigo": codigo,
        }
        response = post(url=config('BASE_URL'), headers=self._headers, data=payload)
        if response.status_code == 200:
            return response.text
        else:
            logger.info(f"Código {codigo} inexistente!")
            return None

    def _parser_dados(self, codigo: int, html_page: str):
        """
        Processa o conteúdo HTML da página, captura informações sobre arquivos e anexos.
        """
        dados_parseados = []
        soup = BeautifulSoup(markup=html_page, features='html.parser', parse_only=None)

        # Encontrar todas as divs que contêm informações sobre arquivos
        body = soup.find('body')
        divs = body.find_all('div', recursive=False)

        for div in divs[1:]:
            if 'arquivo' in div.get_text().lower():
                nome_particao_arquivo = div.find_next('div').get_text(strip=True)
                conteudo_arquivos = [
                    {
                        "nome_arquivo": tag_a.get_text(),
                        "href": tag_a['href'],
                        "base64": self._capturar_anexo(url_anexo=tag_a['href'])
                    }
                    for tag_a in div.find_all('a')
                ]

                dados_parseados.append({
                    "nome_particao_arquivo": nome_particao_arquivo,
                    "anexos": conteudo_arquivos,
                    "codigo_consultado": codigo,
                })
        return dados_parseados

    def _capturar_anexo(self, url_anexo: str):
        """
        Captura o conteúdo de um anexo e converte para Base64.
        """
        response = get(url=f"{config('BASE_URL')}{url_anexo}", headers=self._headers)
        return self._gerar_base64_anexo(nome=url_anexo, conteudo=response.content)

    @staticmethod
    def _gerar_base64_anexo(nome: str, conteudo: bytes):
        """
        Gera o conteúdo Base64 a partir de um arquivo.
        """
        try:
            nome_diretorio_temporario = 'tmp_files'

            if not exists(nome_diretorio_temporario):
                makedirs(nome_diretorio_temporario)

            caminho_completo_arquivo = join(nome_diretorio_temporario, nome)

            with open(caminho_completo_arquivo, 'wb') as file:
                file.write(conteudo)

            # Leitura do conteúdo do arquivo temporário
            with open(caminho_completo_arquivo, 'rb') as file:
                file_content = file.read()

            # Conversão para base64
            base64_content = b64encode(file_content).decode('utf-8')

            return base64_content

        except Exception as e:
            logger.error(f"Erro ao gerar base64 do arquivo {nome} | Erro: {e}")
            return None

        finally:
            try:
                remove(caminho_completo_arquivo)
            except Exception as e:
                logger.warning(f"Erro ao remover arquivo temporário {caminho_completo_arquivo}: | Erro: {e}")

    def capturar_dados_existentes(self, codigo: int):
        """
        Captura os dados existentes para um código específico.
        """
        if not self._token:
            self._capturar_token()
        conteudo_codigo = self._consultar_codigo(codigo=codigo)
        if conteudo_codigo:
            dados_parseados = self._parser_dados(codigo=codigo, html_page=conteudo_codigo)
            return dados_parseados

        return None


from handlers.base import BaseHandler
from tornado.concurrent import run_on_executor
from concurrent.futures import ThreadPoolExecutor
import time

class AAA:
    def __init__(self):
        self.executor = ThreadPoolExecutor()

    @run_on_executor
    def aaa(self):
        time.sleep(1)

class TestHandler(BaseHandler):
    async def get(self):
        print('get')
        # res = await self.get_external('http://www.baidu.com')
        # print('end-baidu')
        a = AAA()
        await a.aaa()
        print('end')
        self.write("Hello, world!")

class TestESHandler(BaseHandler):
    """
    ES实现
    """
    async def get(self):
        t = self.get_argument('t', 1)
        # es = AsyncES()
        print('start')
        info = await self.es.info()
        print('end info')
        # g = await self.es.search(index='ecommerce', doc_type='products')
        # print('end')
        self.success_response({'data': len(info)})


import os
from flask import Flask, flash, redirect, request
from flask import (url_for, current_app as app)
from werkzeug.utils import secure_filename

import casparser

app = Flask(__name__)
UPLOAD_FOLDER = './uploads/'
ALLOWED_EXTENSIONS = {'pdf'}
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route("/")
def hello():
    return "Hello, World!"

@app.route("/parse", methods=['POST'])
def parse_cas():
    if 'file' not in request.files:
        flash('No file part')
        return redirect(request.url)
    file = request.files['file']
    # If the user does not select a file, the browser submits an
    # empty file without a filename.
    if file.filename == '':
        flash('No selected file')
        return redirect(request.url)
    if file and allowed_file(file.filename):
        filename = secure_filename(file.filename)
        upload_path = os.path.join(app.config['UPLOAD_FOLDER'])
        file.save(upload_path, filename)
        return casparser.read_cas_pdf(upload_path + filename, "password", output="json")

# favicon
@app.route('/favicon.ico')
def favicon():
    return url_for('static', filename='/assets/favicon.ico')


#!/usr/bin/env python
import os.path as osp
import xml.etree.ElementTree as ET
import os
import cv2
import numpy as np
import argparse
import pickle as pkl
"""
 Annotation format:
    [
        {
            'filename': 'a.jpg',
            'width': 1280,
            'height': 720,
            'ann': {
                'bboxes': <np.ndarray> (n, 4),
                'labels': <np.ndarray> (n, ),
                'bboxes_ignore': <np.ndarray> (k, 4), (optional field)
                'labels_ignore': <np.ndarray> (k, 4) (optional field)
            }
        },
        ...
    ]

"""

classes = ["holothurian","echinus","scallop","starfish"]
cat2label = {cat: i + 1 for i, cat in enumerate(classes)}
def convert(data_root):
    img_infos = []
    filenames = os.listdir(osp.join(data_root, "train", "image"))
    img_ids = [id.split('.')[0] for id in filenames]
    for img_id in img_ids:
        filename = osp.join(data_root, 'train', "image",
                            '{}.jpg'.format(img_id))
        # 需要打开才知道图片宽高,需要做一个annotation.json快速读取
        img_cv = cv2.imread(filename)

        img_info = dict(id=img_id, filename=filename, width=img_cv.shape[1], height=img_cv.shape[0])

        del img_cv

        #打开xml 获取object bbox信息
        xml_path = osp.join(data_root, 'train', "box",
                            '{}.xml'.format(img_id))
        tree = ET.parse(xml_path)
        root = tree.getroot()
        bboxes = []
        labels = []
        bboxes_ignore = []
        labels_ignore = []
        for obj in root.findall('object'):
            name = obj.find('name').text
            if name not in cat2label:
                continue
            label = cat2label[name]
            # difficult = int(obj.find('difficult').text)
            difficult = False
            bnd_box = obj.find('bndbox')
            bbox = [
                int(bnd_box.find('xmin').text),
                int(bnd_box.find('ymin').text),
                int(bnd_box.find('xmax').text),
                int(bnd_box.find('ymax').text)
            ]
            ignore = False
            # if self.min_size:
            #     assert not self.test_mode
            #     w = bbox[2] - bbox[0]
            #     h = bbox[3] - bbox[1]
            #     if w < self.min_size or h < self.min_size:
            #         ignore = True
            if difficult or ignore:
                bboxes_ignore.append(bbox)
                labels_ignore.append(label)
            else:
                bboxes.append(bbox)
                labels.append(label)
        if not bboxes:
            bboxes = np.zeros((0, 4))
            labels = np.zeros((0,))
        else:
            bboxes = np.array(bboxes, ndmin=2) - 1
            labels = np.array(labels)
        if not bboxes_ignore:
            bboxes_ignore = np.zeros((0, 4))
            labels_ignore = np.zeros((0,))
        else:
            bboxes_ignore = np.array(bboxes_ignore, ndmin=2) - 1
            labels_ignore = np.array(labels_ignore)
        ann = dict(
            bboxes=bboxes.astype(np.float32),
            labels=labels.astype(np.int64),
            bboxes_ignore=bboxes_ignore.astype(np.float32),
            labels_ignore=labels_ignore.astype(np.int64))
        img_info.setdefault("ann",ann)
        img_infos.append(img_info)
    return img_infos
def parse_args():
    parser = argparse.ArgumentParser(description='underwaterdataset train dataset to mmdetction ann json')
    parser.add_argument('--dir', help='dataset root dir path')
    parser.add_argument('--out', help='output result file(pkl file)')
    args = parser.parse_args()
    return args
if __name__ == '__main__':
    args = parse_args()
    assert args.out is not None and args.dir is not None
    import json
    img_infos = convert(args.dir)
    with open(args.out,"wb") as file_handler:
        pkl.dump(img_infos, file_handler)