import serial
import time
# message
class RoboteqHandler:
    """
    Create a roboteq device object for communication, read the README for more information
    """

    def __init__(self, exit_on_interrupt = False, debug_mode = False):
        self.is_alive = False
        self.port = ""
        self.baudrate = 115200
        self.ser = None
        self.exit_on_interrupt = exit_on_interrupt
        self.debug_mode = debug_mode

    def connect(self, port: str, baudrate: int = 115200) -> bool:
        """
        Attempt to establish connection with the controller
        """
        self.port = port
        self.baudrate = baudrate

        if self.debug_mode == True:
            print(f"DEBUG MODE: {self.debug_mode}")
            print(f"EXIT ON INTERRUPT: {self.exit_on_interrupt}")
            time.sleep(1)

        try:
            self.ser = serial.Serial(
                port = self.port,
                baudrate = self.baudrate,
                parity = serial.PARITY_NONE,
                stopbits = serial.STOPBITS_ONE,
                bytesize= serial.EIGHTBITS
            )
            if self.ser.isOpen():
                self.ser.close()
            self.ser.open()
            self.is_alive = True

        except Exception as e:
            if self.debug_mode == True:
                print("DEBUG MODE: ERROR: Failed to connect to the roboteq device, read the exception error below:")
                print(e)
                print("\n")
            self.is_alive = False

        return self.is_alive

    def send_raw_command(self, str_command: str = "") -> None:
        """
        Send a raw string command, the library will handle sending the command, but how you write it
        is up to you.
        """
        raw_command = f"{str_command}+\r"
        try:
            if self.debug_mode == True:
                print(f"DEBUG MODE: Tx:{raw_command}")
            self.ser.write(raw_command.encode())

        except Exception as e:
            if self.debug_mode == True:
                print("DEBUG MODE: Failed to send command to the controller, read the exception error below:")
                print(e)
                print("\n")
            if self.exit_on_interrupt == True:
                quit()

    def request_handler(self, request: str = "") -> str:
        """
        Sends a command and a parameter,
        """
        def get_data(serial):
            raw_data = b''
            while raw_data == b'':
                try:
                    raw_data = serial.read_all()
                except Exception as e:
                    if self.debug_mode == True:
                        print("DEBUG MODE: Failed to read from the controller, read the exception error below:")
                        print(e)
                        print("\n")
                    if self.exit_on_interrupt == True:
                        quit()
                    raw_data = b' '

            if self.debug_mode == True:
                print(f"DEBUG MODE: Rx:{raw_data}")
            return raw_data

        self.send_raw_command(request)
        result = get_data(self.ser)
        result = result.decode()
        result = result.split("\r")
        try:
            return_result = result[1]
            return result[1]
        except IndexError as e:
            debug_return = "DEBUG MODE: Received faulty message, ignoring.."
            if self.exit_on_interrupt == True:
                quit()
            if self.debug_mode == True:
                print(debug_return)
            return debug_return

    def dual_motor_control(self, left_motor: int = 0, right_motor: int = 0) -> None:
        """
        Controlling the motor using a Dual Drive mode
        Send speed for the left, and right side of the robot/vehicle seperately
        Effective for doing Pivot drive and running track based robots
        left_motor: integer from -1000 to 1000
        right_motor: integer from -1000 to 1000
        """
        raw_command = f"!M {left_motor} {right_motor} "
        self.request_handler(raw_command)

    def send_command(self, command: str, first_parameter = "", second_parameter = "") -> None:
        if first_parameter != "" and second_parameter != "":
            message = f"{command} {first_parameter} {second_parameter} "
        if first_parameter != "" and second_parameter == "":
            message = f"{command} {first_parameter} "
        if first_parameter == "" and second_parameter == "":
            message = f"{command} "

        try:
            response = self.request_handler(message)
        except Exception as e:
            if self.debug_mode == True:
                print("DEBUG MODE: Failed to construct a message, read the exception error below:")
                print(e)
                print("\n")
            if self.exit_on_interrupt == True:
                quit()

    def read_value(self, command: str = "", parameter = "") -> str:
        request = f"{command} [{parameter}]"
        response = self.request_handler(request)
        return response


import torch
from mlp import MLP1
import CoverageControlTorch.utils.coverage_system as CoverageSystemUtils
from CoverageControlTorch.data_loaders import data_loader_utils as dl_utils

if torch.cuda.is_available():
    device = torch.device('cpu')
    print('Using GPU')

# nlayers = 2
# K = 2
# num_nodes = 3
# nfeatures = 2
# latent_size = 3
class GNN(torch.nn.Module):
    def __init__(self):
        super(GNN, self).__init__()
        self.add_module("graph_conv_0", TAGConv1(in_channels = nfeatures, out_channels = latent_size, K = 2))
        self.add_module("graph_conv_1", TAGConv1(in_channels = latent_size, out_channels = latent_size, K = 2))

    def forward(self, x, edge_index, edge_weight = None):
        x = self._modules["graph_conv_0"](x, edge_index, edge_weight)
        x = torch.relu(x)
        x = self._modules["graph_conv_1"](x, edge_index, edge_weight)
        x = torch.relu(x)
        return x

# model = GNN()
nlayers = 5
K = 3
num_nodes = 10
nfeatures = 34
latent_size = 256

model = torch.load('model_k3_1024.pt')
model.eval()
model.to(device)

print(model)
gnn = model
gnn = model.gnn_backbone
print(gnn)

mlp_layer_custom = MLP1([256,32,32])
mlp_layer_orig = model.gnn_mlp
mlp_layer_custom.load_state_dict(mlp_layer_orig.state_dict())
mlp_layer = mlp_layer_custom
print(mlp_layer.state_dict().keys())
mlp_layer.eval()
# Turn grad off
for param in mlp_layer.parameters():
    param.requires_grad = False
mlp_layer.to('cpu')

# print(mlp_layer.norms[0].module.weight)
random_input = torch.rand((1,256))
# print(random_input)

output1 = mlp_layer(random_input)
# print(f'output1: {output1}')
# print(mlp_layer.norms[0].module.weight)

x = random_input
l0_wts = mlp_layer.lins[0].weight
l0_bias = mlp_layer.lins[0].bias
x = torch.matmul(x, l0_wts.t()) + l0_bias
# print(f'x after l0: {x}')
batch_norm = torch.nn.BatchNorm1d(32)
batch_state_dict = mlp_layer.norms[0].module.state_dict()
print(batch_state_dict)
batch_state_dict['num_batches_tracked'] = batch_state_dict['num_batches_tracked']*0
# batch_state_dict['running_mean'] = batch_state_dict['running_mean']*0
# batch_state_dict['running_var'] = batch_state_dict['running_var']*0

batch_norm.load_state_dict(batch_state_dict)
print(batch_norm.state_dict())
batch_norm.eval()

# n0_wts = mlp_layer.norms[0].module.weight
# n0_bias = mlp_layer.norms[0].module.bias
# x = x*n0_wts + n0_bias
x = batch_norm(x)
# print(f'norm layer: ', {mlp_layer.norms[0].module})
# print(f'x after n0: {x}')
x = torch.relu(x)

l1_wts = mlp_layer.lins[1].weight
l1_bias = mlp_layer.lins[1].bias
x = torch.matmul(x, l1_wts.t()) + l1_bias

output2 = x
# print(f'output2: {output2}')
print(torch.allclose(output1, output2))
# print(x)

output3 = model.output_linear(mlp_layer_orig(random_input))
outlayer_wts = model.output_linear.weight
outlayer_bias = model.output_linear.bias
output4 = torch.matmul(output2, outlayer_wts.t()) + outlayer_bias

print(torch.allclose(output3, output4))


# Course: Data Analysis Tools
# Week3
#
import pandas as pd
import numpy as np
import seaborn
import scipy
import matplotlib.pyplot as plt

df = pd.read_csv("gapminder.csv", low_memory = False, index_col = 0)
maleemployrate = pd.read_csv("maleemployrate_sub.csv", low_memory = False, index_col = 0)

df["incomeperperson"] = df["incomeperperson"].convert_objects(convert_numeric=True)
df["femaleemployrate"] = df["femaleemployrate"].convert_objects(convert_numeric=True)
df["polityscore"] = df["polityscore"].convert_objects(convert_numeric=True)
df['employrate'] = df['employrate'].convert_objects(convert_numeric=True)
df['internetuserate'] = df['internetuserate'].convert_objects(convert_numeric=True)
df['urbanrate'] = df['urbanrate'].convert_objects(convert_numeric=True)

maleemployrate['2007'] = maleemployrate['2007'].convert_objects(convert_numeric=True)

df['incomeperperson']=df['incomeperperson'].replace(' ', np.nan)
df['polityscore']=df['polityscore'].replace(' ', np.nan)
df['internetuserate']=df['internetuserate'].replace(' ', np.nan)
df['urbanrate']=df['urbanrate'].replace(' ', np.nan)

# Scatterplot
scat1 = seaborn.regplot(x="polityscore", y="incomeperperson", fit_reg=True, data=df)
plt.xlabel('polity score')
plt.ylabel('income per person')
plt.title('Scatterplot for the Association Between polity score and income per person')
plt.show()

scat2 = seaborn.regplot(x="polityscore", y="femaleemployrate", fit_reg=True, data=df)
plt.xlabel('polity score')
plt.ylabel('female employ rate')
plt.title('Scatterplot for the Association Between female employ rate and polity score')
plt.show()

scat3 = seaborn.regplot(x="internetuserate", y="urbanrate", fit_reg=True, data=df)
plt.xlabel('internetuserate')
plt.ylabel('urbanrate ')
plt.title('Scatterplot for the Association Between internetuserate and urbanrate')
plt.show()

data_clean = df.dropna()
print ('association between polity score and income per person')
print (scipy.stats.pearsonr(data_clean['polityscore'], data_clean['incomeperperson']))

print ('association Between female employ rate and polity score')
print (scipy.stats.pearsonr(data_clean['polityscore'], data_clean['femaleemployrate']))

print ('association Between internetuserate and urbanrate')
print (scipy.stats.pearsonr(data_clean['internetuserate'], data_clean['urbanrate']))


import time
import random
import matplotlib.pyplot as plt
import graphviz

class Node:
    def __init__(self,key=None,next=None):
        self.key = key
        self.next = next

class PriorityQueue:
    def __init__(self):
        self.head = None
        self.tail = None
        self.size = 0

    def searchNode(self,i):
        cur = self.head
        while(i > 0):
            if(cur == None): return None
            cur = cur.next
            i = i - 1
        return cur

    # Get parent
    def get_parent(self,i):
        if i > 0:
            return self.searchNode((i-1)//2)
        else:
            return None

    # Get left child
    def get_left_child(self,i):
        return self.searchNode(2*i + 1)

    # Get right child
    def get_right_child(self,i):
        return self.searchNode(2*i + 2)

    def insert(self, key):
        node = Node(key)
        if self.tail:
            self.tail.next = node
        else:
            self.head = node

        self.tail = node

        index = self.size
        self.size = self.size + 1

        while(index != 0):
            parent = self.get_parent(index)
            if(parent.key <= key):break
            tmp = parent.key
            parent.key = key
            node.key = tmp
            index = (index-1)//2
            node = parent

    def _has_left(self,index):
        # determine if there is a left child
        return index * 2 + 1 < self.size

    def _has_right(self,index):
        # determine if there is a left child
        return index * 2 + 2 < self.size

    def delMin(self):
        if self.head is None:
            return None

        self.head.key = self.tail.key

        cur = self.head
        index = 0

        self.size = self.size - 1
        self.tail = self.searchNode(self.size-1)
        self.tail.next = None

        while(self._has_left(index)):
            left = self.get_left_child(index)
            small_child = left
            small_child_index = index * 2 + 1
            if(self._has_right(index)):
                right = self.get_right_child(index)
                if(left.key > right.key):
                    small_child = right
                    small_child_index = index * 2 + 2
            if(small_child.key >= cur.key):
                return
            else:
                tmp = small_child.key
                small_child.key = cur.key
                cur.key = tmp
            cur = small_child
            index = small_child_index

def Q4():
    data = []
    for i in range(1000):
        data.append(random.randint(0, 100000))

    priority_queue = PriorityQueue()

    start = time.time()
    for i in data:
        priority_queue.insert(i)
    end = time.time()
    insert_time = end - start

    start = time.time()
    for i in range(len(data)):
        priority_queue.delMin()
    end = time.time()
    delMin_time = end - start

    x = ['Insert', 'DelMin']
    y = [insert_time, delMin_time]

    plt.bar(x,y)
    plt.title('Linked List based PriorityQueue Performance')
    plt.xlabel('Operation')
    plt.ylabel('Time (s)')
    plt.show()

def dfs_add_edge(priority_queue,g,cur,index):
    left = priority_queue.get_left_child(index)
    if(left == None): return
    g.edge(str(cur.key), str(left.key))
    right = priority_queue.get_right_child(index)
    if(right == None): return
    g.edge(str(cur.key), str(right.key))

    dfs_add_edge(priority_queue,g,left,index*2+1)
    dfs_add_edge(priority_queue,g,right,index*2+2)

def Q5():

    priority_queue = PriorityQueue()
    for i in range(10):
        priority_queue.insert(9-i)

    # Create graph
    g = graphviz.Digraph('PriorityQueue', filename='PriorityQueue.gv', format='png')

    g.node_attr['shape'] = 'circle'
    g.node_attr['style'] = 'filled'

    cur = priority_queue.head
    while cur:
        g.node(str(cur.key),fillcolor='green')
        cur = cur.next

    cur = priority_queue.head
    index = 0

    dfs_add_edge(priority_queue,g,cur,index)

    g.render()


from PIL import Image
import os
import os.path
import numpy as np
import sys
from typing import Any, Callable, Dict, List, Optional, Tuple
from torchvision.datasets.vision import VisionDataset
from torchvision.datasets.utils import *
from torchvision.datasets.mnist import *
from urllib.error import URLError
import torchvision
from math import *

if sys.version_info[0] == 2:
    import cPickle as pickle
else:
    import pickle

import torch.utils.data as data
from .utils import noisify_x, noisify_overlap, noisify_y, add_trigger

cifar10_classes = ['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck',]
cifar10_templates = [
    'a photo of a {}.',
    'a blurry photo of a {}.',
    'a black and white photo of a {}.',
    'a low contrast photo of a {}.',
    'a high contrast photo of a {}.',
    'a bad photo of a {}.',
    'a good photo of a {}.',
    'a photo of a small {}.',
    'a photo of a big {}.',
    'a photo of the {}.',
    'a blurry photo of the {}.',
    'a black and white photo of the {}.',
    'a low contrast photo of the {}.',
    'a high contrast photo of the {}.',
    'a bad photo of the {}.',
    'a good photo of the {}.',
    'a photo of the small {}.',
    'a photo of the big {}.',
]

cifar100_classes = [
    'apple',
    'aquarium fish',
    'baby',
    'bear',
    'beaver',
    'bed',
    'bee',
    'beetle',
    'bicycle',
    'bottle',
    'bowl',
    'boy',
    'bridge',
    'bus',
    'butterfly',
    'camel',
    'can',
    'castle',
    'caterpillar',
    'cattle',
    'chair',
    'chimpanzee',
    'clock',
    'cloud',
    'cockroach',
    'couch',
    'crab',
    'crocodile',
    'cup',
    'dinosaur',
    'dolphin',
    'elephant',
    'flatfish',
    'forest',
    'fox',
    'girl',
    'hamster',
    'house',
    'kangaroo',
    'keyboard',
    'lamp',
    'lawn mower',
    'leopard',
    'lion',
    'lizard',
    'lobster',
    'man',
    'maple tree',
    'motorcycle',
    'mountain',
    'mouse',
    'mushroom',
    'oak tree',
    'orange',
    'orchid',
    'otter',
    'palm tree',
    'pear',
    'pickup truck',
    'pine tree',
    'plain',
    'plate',
    'poppy',
    'porcupine',
    'possum',
    'rabbit',
    'raccoon',
    'ray',
    'road',
    'rocket',
    'rose',
    'sea',
    'seal',
    'shark',
    'shrew',
    'skunk',
    'skyscraper',
    'snail',
    'snake',
    'spider',
    'squirrel',
    'streetcar',
    'sunflower',
    'sweet pepper',
    'table',
    'tank',
    'telephone',
    'television',
    'tiger',
    'tractor',
    'train',
    'trout',
    'tulip',
    'turtle',
    'wardrobe',
    'whale',
    'willow tree',
    'wolf',
    'woman',
    'worm',
]

cifar100_templates = [
    'a photo of a {}.',
    'a blurry photo of a {}.',
    'a black and white photo of a {}.',
    'a low contrast photo of a {}.',
    'a high contrast photo of a {}.',
    'a bad photo of a {}.',
    'a good photo of a {}.',
    'a photo of a small {}.',
    'a photo of a big {}.',
    'a photo of the {}.',
    'a blurry photo of the {}.',
    'a black and white photo of the {}.',
    'a low contrast photo of the {}.',
    'a high contrast photo of the {}.',
    'a bad photo of the {}.',
    'a good photo of the {}.',
    'a photo of the small {}.',
    'a photo of the big {}.',
]

class CIFAR10Noise_clip(VisionDataset):
    """`CIFAR10 <https://www.cs.toronto.edu/~kriz/cifar.html>`_ Dataset.
    """
    base_folder = 'cifar-10-batches-py'
    url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
    filename = "cifar-10-python.tar.gz"
    tgz_md5 = 'c58f30108f718f92721af3b95e74349a'
    train_list = [
        ['data_batch_1', 'c99cafc152244af753f735de768cd75f'],
        ['data_batch_2', 'd4bba439e000b95fd0a9bffe97cbabec'],
        ['data_batch_3', '54ebc095f3ab1f0389bbae665268c751'],
        ['data_batch_4', '634d18415352ddfa80567beed471001a'],
        ['data_batch_5', '482c414d41f54cd18b22e5b47cb7c3cb'],
    ]

    test_list = [
        ['test_batch', '40351d587109b95175f43aff81a1287e'],
    ]
    nb_classes = 10
    template = cifar10_templates
    classnames = cifar10_classes
    meta = {
        'filename': 'batches.meta',
        'key': 'label_names',
        'md5': '5ff9c542aee3614f3951f8cda6e48888',
    }

    def __init__(
            self,
            root: str = '/data/LargeData/Regular/cifar',
            train: bool = True,
            transform: Optional[Callable] = None,
            target_transform: Optional[Callable] = None,
            download: bool = False,
            xnoise_type='gaussian', xnoise_arg=1, xnoise_rate=0,
            ynoise_type='symmetric', ynoise_rate=0,
            trigger_size=3, trigger_rate=0.0,
            random_state=0
    ) -> None:

        super(CIFAR10Noise_clip, self).__init__(root, transform=transform,
                                      target_transform=target_transform)

        self.train = train  # training set or test set

        if download:
            self.download()

        if not self._check_integrity():
            raise RuntimeError('Dataset not found or corrupted.' +
                               ' You can use download=True to download it')

        if self.train:
            downloaded_list = self.train_list
        else:
            downloaded_list = self.test_list

        self.data: Any = []
        self.targets = []

        # now load the picked numpy arrays
        for file_name, checksum in downloaded_list:
            file_path = os.path.join(self.root, self.base_folder, file_name)
            with open(file_path, 'rb') as f:
                entry = pickle.load(f, encoding='latin1')
                self.data.append(entry['data'])
                if 'labels' in entry:
                    self.targets.extend(entry['labels'])
                else:
                    self.targets.extend(entry['fine_labels'])

        self.data = np.vstack(self.data).reshape(-1, 3, 32, 32)
        self.data = self.data.transpose((0, 2, 3, 1))  # convert to HWC

        if self.train:
            # if trigger_rate > 0.0:
            #     self.data, self.targets, self.poison_or_not= add_trigger(trainset=self.data, targets = self.targets, trigger_size=trigger_size, trigger_rate=trigger_rate, random_state=random_state)
            # else:
            #     self.poison_or_not = np.zeros(self.data.shape[0])
            if xnoise_rate > 0.0:
                self.data, self.xnoisy_or_not= noisify_x(dataset='cifar', data=self.data, noise_type=xnoise_type, noise_arg=xnoise_arg, noise_rate=xnoise_rate, random_state=random_state)
            else:
                self.xnoisy_or_not = np.zeros(self.data.shape[0])
            self.xnoisy_or_not = (self.xnoisy_or_not == 1)
            if ynoise_rate > 0.0:
                self.targets = np.asarray([[self.targets[i]] for i in range(len(self.targets))])
                # self.noise_targets = noisify_y(train_labels=self.targets, noise_type=ynoise_type, noise_rate=ynoise_rate, random_state=random_state, nb_classes=self.nb_classes,xnoisy_or_not=self.xnoisy_or_not)
                self.noise_targets = noisify_y(train_labels=self.targets, noise_type=ynoise_type, noise_rate=ynoise_rate, random_state=random_state, nb_classes=self.nb_classes)
                self.noise_targets = np.asarray([i[0] for i in self.noise_targets])
                self.targets =  np.asarray([i[0] for i in self.targets])
                self.ynoisy_or_not = (self.targets != self.noise_targets)
            else:
                self.noise_targets = self.targets
                self.ynoisy_or_not = np.zeros(self.data.shape[0]).astype(np.int64)
            self.report_noise()
        self._load_meta()

    def get_noise(self):
        xy_noise = np.logical_and(self.xnoisy_or_not, self.ynoisy_or_not)
        x_noise = np.logical_and(self.xnoisy_or_not, ~self.ynoisy_or_not)
        y_noise = np.logical_and(~self.xnoisy_or_not, self.ynoisy_or_not)
        # poison = np.logical_and(self.poison_or_not, ~self.xnoisy_or_not)
        clean = np.logical_and(~self.xnoisy_or_not, ~self.ynoisy_or_not)
        return {
            'xy_noise': xy_noise,
            'x_noise': x_noise,
            'y_noise': y_noise,
            # 'poison': poison,
            'clean': clean,
            'xnoisy': self.xnoisy_or_not,
            'ynoisy': self.ynoisy_or_not
        }
    def report_noise(self):
        noise_stat = self.get_noise()
        print('Noise Stat:')
        for key, val in noise_stat.items():
            print(key, val.sum())

    def _load_meta(self) -> None:
        path = os.path.join(self.root, self.base_folder, self.meta['filename'])
        if not check_integrity(path, self.meta['md5']):
            raise RuntimeError('Dataset metadata file not found or corrupted.' +
                               ' You can use download=True to download it')
        with open(path, 'rb') as infile:
            data = pickle.load(infile, encoding='latin1')
            self.classes = data[self.meta['key']]
        self.class_to_idx = {_class: i for i, _class in enumerate(self.classes)}

    def __getitem__(self, index: int) -> Tuple[Any, Any]:
        """
        Args:
            index (int): Index

        Returns:
            tuple: (image, target) where target is index of the target class.
        """
        if self.train:
            img, noise_tar, true_tar, xnoisy = self.data[index], int(self.noise_targets[index]), int(self.targets[index]), self.xnoisy_or_not[index]
        else:
            img, target = self.data[index], int(self.targets[index])
        # doing this so that it is consistent with all other datasets
        # to return a PIL Image
        img = Image.fromarray(img)

        if self.transform is not None:
            img = self.transform(img)

        # if self.target_transform is not None:
        #     target = self.target_transform(target)
        if self.train:
            return ((img, xnoisy), (noise_tar, true_tar))
        else:
            return (img, target)

    def __len__(self) -> int:
        return len(self.data)

    def _check_integrity(self) -> bool:
        root = self.root
        for fentry in (self.train_list + self.test_list):
            filename, md5 = fentry[0], fentry[1]
            fpath = os.path.join(root, self.base_folder, filename)
            if not check_integrity(fpath, md5):
                return False
        return True

    def download(self) -> None:
        if self._check_integrity():
            print('Files already downloaded and verified')
            return
        download_and_extract_archive(self.url, self.root, filename=self.filename, md5=self.tgz_md5)

    def extra_repr(self) -> str:
        return "Split: {}".format("Train" if self.train is True else "Test")

class CIFAR100Noise_clip(CIFAR10Noise_clip):
    """`CIFAR100 <https://www.cs.toronto.edu/~kriz/cifar.html>`_ Dataset.

    This is a subclass of the `CIFAR10` Dataset.
    """
    base_folder = 'cifar-100-python'
    url = "https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
    filename = "cifar-100-python.tar.gz"
    tgz_md5 = 'eb9058c3a382ffc7106e4002c42a8d85'
    train_list = [
        ['train', '16019d7e3df5f24257cddd939b257f8d'],
    ]

    test_list = [
        ['test', 'f0ef6b0ae62326f3e7ffdfab6717acfc'],
    ]
    meta = {
        'filename': 'meta',
        'key': 'fine_label_names',
        'md5': '7973b15100ade9c7d40fb424638fde48',
    }
    nb_classes = 100
    template = cifar100_templates
    classnames = cifar100_classes

class IMBALANCECIFAR10(torchvision.datasets.CIFAR10):
    nb_classes = 10
    template = cifar10_templates
    classnames = cifar10_classes
    decay_stride = 2.1971

    def __init__(self,root: str = '/data/LargeData/Regular/cifar',imb_type='exp',train=True,transform=None,target_transform=None,download=False):
        super(IMBALANCECIFAR10,self).__init__(root, train, transform, target_transform, download)
        if train:
            img_num_list = self.get_img_num_per_cls(self.nb_classes,imb_type)
            self.gen_imbalanced_data(img_num_list)

    def get_img_num_per_cls(self,num_class,imb_type):
        img_max = len(self.data)/num_class
        img_num_per_cls = []
        if imb_type == 'exp':
            for cls_idx in range(num_class):
                num = img_max*exp(-cls_idx/self.decay_stride)
                img_num_per_cls.append(int(num+0.5))
        else:
            img_num_per_cls.extend([int(img_max)]*num_class)
        return img_num_per_cls

    def gen_imbalanced_data(self,img_num_per_cls):
        img_max = len(self.data)/self.nb_classes
        new_data,new_targets = [],[]
        targets_np = np.array(self.targets,dtype=np.int64)
        classes = np.arange(self.nb_classes)

        self.num_per_cls = np.zeros(self.nb_classes)
        for class_i,volume_i in zip(classes,img_num_per_cls):
            self.num_per_cls[class_i] = volume_i
            idx = np.where(targets_np==class_i)[0]
            np.random.shuffle(idx)
            keep_num = volume_i+1
            selec_idx = idx[:keep_num]
            new_data.append(self.data[selec_idx,...])
            new_targets.extend([class_i]*keep_num)
        new_data = np.vstack(new_data)
        self.data = new_data
        self.targets = new_targets

    def get_cls_num_list(self):
        return self.num_per_cls.tolist()

class IMBALANCECIFAR100(IMBALANCECIFAR10):
    base_folder = 'cifar-100-python'
    url = "https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
    filename = "cifar-100-python.tar.gz"
    tgz_md5 = 'eb9058c3a382ffc7106e4002c42a8d85'
    train_list = [
        ['train', '16019d7e3df5f24257cddd939b257f8d'],
    ]
    test_list = [
        ['test', 'f0ef6b0ae62326f3e7ffdfab6717acfc'],
    ]
    meta = {
        'filename': 'meta',
        'key': 'fine_label_names',
        'md5': '7973b15100ade9c7d40fb424638fde48',
    }
    nb_classes = 100
    decay_stride = 21.9714
    template = cifar100_templates
    classnames = cifar100_classes

if __name__ == '__main__':
    dataset = CIFAR10Noise_clip( train=True, xnoise_rate=0.1, ynoise_rate=0.1)
    print(dataset.get_noise())


import cv2
import math
import numpy as np

from .points import sort_left, get_midpoint, project_point, get_rotated_rect, perpendicular_point
from .text import draw_text
from .utils import get_coords

# colors
WHITE = (255, 255, 255)
BLUE = (193, 87, 21)
YELLOW = (66, 185, 244)

def draw_edge(image, edge, color, point_radius=10, edge_thickness=2):
    """Draw edge on image.
    Args:
        image: image array as opened by cv2
        edge: list of points [p0, p1], where each point is np.array in format [x, y]
        color:
        point_radius:
        edge_thickness:
    """
    # draw edge
    p0, p1 = sort_left(edge)
    cv2.circle(image, tuple(p0), point_radius, color, -1, lineType=cv2.LINE_AA)
    cv2.circle(image, tuple(p1), point_radius, color, -1, lineType=cv2.LINE_AA)
    cv2.line(image, tuple(p0), tuple(p1), color, edge_thickness, lineType=cv2.LINE_AA )
    return image

def draw_rounded_rect(image, color, center, theta=0, b=22, w=70):
    """Draw a rounded rect on image
    Args:
        image: image as opened by cv2
        color:
        center: coord of center of rect, np.array in [x, y] format
        theta: angle of rotation, in radians, counter-clockwise.
        b: distance from center to top edge of rectangle, or height / 2
        w: distance from center to left edge of rectangle, or width / 2
    """
    rect = get_rotated_rect(center, theta, b, w)
    pts = rect.reshape((-1,1,2))
    cv2.fillPoly(image, [pts], color, lineType=cv2.LINE_AA)

    # draw rounded-rect
    r0 = project_point(center, w, theta)
    r1 = project_point(center, -w, theta)
    cv2.circle(image, tuple(r0), b, color, -1, lineType=cv2.LINE_AA)
    cv2.circle(image, tuple(r1), b, color, -1, lineType=cv2.LINE_AA)
    return image

def draw_label(image, edge, text, color=YELLOW, text_color=WHITE):
    """Draw label on image.
    Args:
        image: image as opened via cv2.
        edge: list of points [p0, p1], where each point is np.array in format [x, y]
        text: str
    """
    image = draw_edge(image, edge, color, point_radius=10, edge_thickness=2)
    # get midpoint
    c, angle = get_midpoint(edge)
    image = draw_rounded_rect(image, color, c, angle, b=22, w=70)
    # draw text
    image = draw_text(image, text, c, angle, fontScale=1, thickness=4, color=text_color)
    return image

def draw_t_label(image, points, text, color=BLUE, text_color=WHITE):
    """Draw a T-label on image.
    For given points (p1, p2, p3) draws perpendicular edges (p1,p2),
    and (p3,p4) where p4 is intersection of (p1,p2). Draws label on edge (p3,p4).

    Args:
        image: image as opened via cv2.
        points: list of points [p1,p2,p3] where each point is np.array in format [x, y]
        text: str
    """
    p1, p2, p3 = points
    edge = [p1, p2]

    image = draw_edge(image, edge, color, point_radius=10, edge_thickness=2)
    perp_point = perpendicular_point(edge[0], edge[1], p3)

    t_edge = [perp_point, p3]
    image = draw_edge(image, t_edge, color, point_radius=10, edge_thickness=2)

    # get midpoint
    c, angle = get_midpoint(t_edge, b=0.3)
    image = draw_rounded_rect(image, color, c, angle, b=22, w=70)
    # draw text
    image = draw_text(image, text, c, angle, fontScale=1, thickness=4, color=text_color)
    return image

def draw_labels(image, labels):
    """Draw labels on image.

    Args:
        image: image opened via cv2.
        labels: list containing [edge, text], where edge is a array of points.
            Each point should be np.array integer coords [x, y] on image, where
            origin is image top-left.
    """

    for label in labels:
        points = [get_coords(p) for p in label['points']]
        text = label['text']
        if len(points) > 2:
            image = draw_t_label(image, points, text)
        else:
            image = draw_label(image, points, text)
    return image


from pathlib import PurePath, Path
from subprocess import check_output
from time import time, strftime, gmtime
from typing import Optional, Callable
from shutil import disk_usage

from .abs_module import Module

from interutils import pr, cyan, pause, human_bytes, count_lines, choose_file, file_volume, IterationTimer

def show_ebt(algos: dict, tlc: int):
    """
    Show estimated burn time
    tlc -> Total lines count
    """
    assert tlc > 0

    for algo, elps in algos.items():
        # Calculate EBT
        ebt = ((tlc * 2) / elps)
        ebt = strftime('%H:%M:%S', gmtime(ebt))
        pr(f'Estimated {cyan(algo)} time: {cyan(ebt)} (assuming elps={elps})')

class Combination(Module):
    def __init__(self, stargen):
        super().__init__(stargen, 'comb')

    def menu(self) -> tuple:
        return {
            'intermix': (self.mix, 'Mix two wordlists'),
            'concat': (self.concat, 'Concatenate two wordlists')
        }

    def ask_two_wl(self, _total_calc=Callable[[int, int], int], _write_action=Callable[[Path, Path, Path, IterationTimer], None]):
        def _select_wordlist(title: str):
            pr(f'Select {title} wordlist:')
            f = choose_file(self.workspace)
            if not f:
                raise KeyboardInterrupt
            fsb, flc, ftxt = file_volume(f)
            pr(f'  {ftxt}')
            return f, fsb, flc

        # Get wordlists
        try:
            f1, f1sb, f1lc = _select_wordlist('first')
            f2, f2sb, f2lc = _select_wordlist('secund')
        except KeyboardInterrupt:
            print()
            return pr('Interrupted', '!')

        # Calculate impact and let the user accept the facts
        tsb = _total_calc(f1sb, f2sb)
        tlc = _total_calc(f1lc, f2lc)
        pr(f'Mixing will allocate {cyan(human_bytes(tsb))} for {cyan("{:,}".format(tlc))} lines')

        free = disk_usage(self.workspace.resolve()).free
        pr(f"Available space on workspace's disk: " + cyan(human_bytes(free)))
        if tsb > free:
            pr('Not enough space on the workspace disk for allocation!', '!')
            return
        max_size = self.config['max_created_file_size']
        if tsb > max_size:
            return pr(f'Calculation resulted in an oversized file (>{human_bytes(max_size)}), aborting!', '!')
        if not pause(cancel=True):
            return

        out_path = self.dest_dir.joinpath(f'{f1.stem}_{f2.stem}')
        with out_path.open('w', encoding='utf-8') as out_file:
            itmr = IterationTimer(tlc, init_interval=1, max_interval=15)
            _write_action(f1, f2, out_file, itmr)

        # Finalize
        pr('Wordlist written into: ' + cyan(out_path.name))
        show_ebt({  # TODO Move to config
            'WPA2': 57000
        }, tlc)

    def mix(self, args: tuple) -> None:
        def mix_action(f1: Path, f2: Path, out_file: Path, itmr: IterationTimer) -> None:
            with f1.open(encoding='utf-8') as f1d:
                for l1 in f1d:
                    if '# ' in l1 or '##' in l1:
                        continue
                    l1 = l1.strip()

                    with f2.open(encoding='utf-8') as f2d:
                        for l2 in f2d:
                            if '# ' in l2 or '##' in l2:
                                continue
                            l2 = l2.strip()

                            out_file.write(f'{l1}{l2}\n{l2}{l1}\n')
                            itmr.tick()  # O(n^2)

        if self.ask_two_wl((lambda a, b: int(a * b * 2)), mix_action) is None:
            return  # Currently redundant - might be of use later

    def concat(self, args: tuple) -> None:
        def concat_action(f1: Path, f2: Path, out_file: Path, itmr: IterationTimer) -> None:
            with f1.open(encoding='utf-8') as f1d:
                for l1 in f1d:
                    if '# ' in l1 or '##' in l1:
                        continue

                    out_file.write(l1)
                    itmr.tick()  # O(n)

            with f2.open(encoding='utf-8') as f2d:
                for l2 in f2d:
                    if '# ' in l2 or '##' in l2:
                        continue

                    out_file.write(l2)
                    itmr.tick()  # O(n)

        if self.ask_two_wl((lambda a, b: a + b), concat_action) is None:
            return  # Currently redundant - might be of use later


"""
884. 两句话中的不常见单词
给定两个句子 A 和 B 。 （句子是一串由空格分隔的单词。每个单词仅由小写字母组成。）

如果一个单词在其中一个句子中只出现一次，在另一个句子中却没有出现，那么这个单词就是不常见的。

返回所有不常用单词的列表。

您可以按任何顺序返回列表。

示例 1：

输入：A = "this apple is sweet", B = "this apple is sour"
输出：["sweet","sour"]
示例 2：

输入：A = "apple apple", B = "banana"
输出：["banana"]

提示：

0 <= A.length <= 200
0 <= B.length <= 200
A 和 B 都只包含空格和小写字母

链接：https://leetcode-cn.com/problems/uncommon-words-from-two-sentences
"""
from typing import List

class Solution:
    def uncommonFromSentences(self, A: str, B: str) -> List[str]:
        aa = " ".join([A, B]).split()
        cc = dict()
        for a in aa:
            cc[a] = cc.get(a, 0) + 1

        return map(lambda x: x[0], filter(lambda x:x[1]==1, cc.items()))

哈希表/884. 两句话中的不常见单词.py

884. 两句话中的不常见单词.py


import numpy as np
from matplotlib import pyplot as plt
import torch
from tqdm import tqdm
import os  # To create a folder for each run
import src
#import src.plot
import datetime
import json  # To save simulation parameters in a text file
#import pdb
import pickle
import sys

orig_stdout = sys.stdout
#f = open('out.txt', 'w')
#sys.stdout = f

# set device
dtype = torch.float
cpu = torch.device("cpu")
gpu = torch.device("cuda:0")

device = cpu
# Uncomment the line below to run on GPU
#device = gpu

torch.manual_seed(0)
np.random.seed(0)

# Set parameters
args = {'thres': -50,
        'u_rest': -60,
        'tau_mem': 1e-2,
        'tau_syn': 5e-3,
        'tau_ref': 5e-3,
        't_rise': 5e-3,  # the pre-synaptic double exponential kernel rise time
        't_decay': 1e-2,  # the pre-synaptic double exponential kernel decay time
        'timestep_size': 1e-4,  # 0.1 msec timesteps
        't_rise_alpha': 5e-3,
        't_decay_alpha': 1e-2,
        'nb_steps': 5000,
        'tau_rms': 5e-4,  # this is a guess and might need changing
        'nb_inputs': 100,
        'nb_outputs': 1,
        'device': device,  # for src in different modules
        'dtype': dtype,
        'nb_epochs': 1000,
        'epsilon': 1e-4,  # noise term for learning rate
        'p': 0.5
        }

# constants
nb_inputs = args['nb_inputs']
nb_outputs = args['nb_outputs']

nb_trials = 10
nb_epochs = args['nb_epochs']

nb_steps = args['nb_steps']
dt = args['timestep_size']

tau_syn = args['tau_syn']
tau_mem = args['tau_mem']

alpha = np.exp(-dt/tau_syn)
beta = np.exp(-dt/tau_mem)

args['alpha'] = alpha
args['beta'] = beta

# input trains
# not sure about this, but assuming it since the paper uses 10 Hz frequency as the target output frequency (actually, 5 equidistant spikes over 500 ms)
spk_freq = 10
input_trains = src.poisson_trains(100, spk_freq*np.ones(100), args)

# Create Target Train
target = torch.zeros(nb_steps, device=device, dtype=dtype)
target[500:: nb_steps//5] = 1

# weights = src.initialize_weights(nb_inputs, nb_outputs, args, scale=80) # initialize weights

# v_ij = 1e-2*torch.zeros((nb_inputs, nb_outputs), device=device, dtype=dtype)
#learning_rates = np.array([10, 5, 1, 0.5, 0.1]) * 1e-3

#learning_rates = np.array([5, 1, 10, 0.5 , 0.1]) * 1e-3
#learning_rates = np.array([30]) * 1e-3
learning_rates = np.array([40, 60]) * 1e-3
#learning_rates = learning_rates[::-1]

for r_0 in learning_rates:
    # r_0 = 5e-3 # basal learning rate
    print(args)
    print("Learning rate =", r_0)
    loss_rec = np.zeros((nb_trials, nb_epochs))
    plt.figure(dpi=150)

    recordings_list = []

    for i in range(nb_trials):
        weights = src.new_initialize_weights(nb_inputs, nb_outputs, args)
        new_weights, loss_rec[i], recordings = src.train_single_neuron(
            input_trains, target, weights, r_0, args)
        recordings_list.append(recordings)
        #r_ij, v_ij, g_ij2 = learning_rate_params
        plt.plot(loss_rec[i], alpha=0.6)

    plt.plot(np.mean(loss_rec, axis=0), alpha=1,
             color='black', label="Avg. Loss")
    plt.legend()
    plt.title("Loss, learning-rate = " + str(r_0) + ", p = " +
              str(args['p']) + "spike freq = " + str(spk_freq))
    plt.xlabel("Epochs")
 #   plt.show()

    data_folder = "data/" + \
        str(datetime.datetime.today())[:13] + ' rate = ' + str(r_0) + '/'
    location = os.path.abspath(data_folder)
    location = os.path.join(os.getcwd(), location)
    os.makedirs(location)

    plt.savefig(location + "/loss over epochs" + "learning-rate = " + str(r_0) +
                ", epsilon = " + str(args['epsilon']) + "spike freq = " + str(spk_freq) + ".png")

    loss_file_name = location + "/loss_rec epsilon= " + \
        str(args['epsilon']) + "learning_rate = " + \
        str(r_0) + "spike freq = " + str(spk_freq)
    loss_file = open(loss_file_name, 'wb')
    pickle.dump(loss_rec, loss_file)
    loss_file.close()

    # Store args:
    file_name = location + "/args epsilon = " + \
        str(args['epsilon']) + "learning_rate = " + \
        str(r_0) + "spike freq = " + str(spk_freq)
   # args_file = open(file_name, 'w')
   # json.dump(args, args_file)
    args_file = open(file_name, 'a')
    args_file.write(str(args))
    args_file.close()

    recordings_filename = location + "/recordings epsilon= " + \
        str(args['epsilon']) + "learning_rate = " + \
        str(r_0) + "spike freq = " + str(spk_freq)
    recordings_file = open(recordings_filename, 'wb')
    pickle.dump(recordings_list, recordings_file)
    recordings_file.close()

sys.stdout = orig_stdout
# f.close()

"""
    data_folder = "data/" h+ str(datetime.datetime.today())[:10] + ' rate = ' + str(r_0) + '/'
    #os.makedirs(location)
    location = os.path.abspath(data_folder)
    location = os.path.join(os.getcwd(), location)
    print(location)
#    data_folder = "/data/rate = " + str(r_0) + ' /'
#    location = os.path.join(os.path.abspath(), data_folder)
#    print(location)
#    break
    os.makedirs(location)
    #location = ''
    #location = os.path.cudir()
    # Store parameters (in dict args) in the given location

    src.plot.plot_loss(loss_rec, location, args) # Saves the loss-over-epochs plot in the given location
    src.plot.plot_learning_rate_params(learning_rate_params, location, args)

    # Store weights
    file_name = location + "/weights" #+ str(datetime.datetime.now())[:10]
    weight_file = open(file_name, 'wb')
    pickle.dump(new_weights, weight_file)

    # Store loss_rec
    file_name = location + "/loss_rec" #+ str(datetime.datetime.now())
    weight_file = open(file_name, 'wb') # output file in binary mode
    pickle.dump(loss_rec, weight_file)

    # Store learning_rate_params:
    file_name = location + "/learning_rate_params" #+ str(datetime.datetime.now())
    param_file = open(file_name, 'wb')
    pickle.dump(learning_rate_params, param_file)

    """

"""plt.plot(loss_rec)
plt.title("Loss over epochs")
plt.show()

plt.plot(torch.flatten(torch.mean(r_ij, dim=0)[0]), label='Avg. Learning Rate')
plt.plot(torch.flatten(torch.median(r_ij, dim=0)[0]), label='Median Learning Rate')
plt.title("Learning Rate over Epochs")
plt.show()

"""

""" for i in range(nb_epochs):

    print("\n Iteration:", i+1)
    mem_rec, spk_rec, error_rec, eligibility_rec, pre_trace_rec = src.run_single_neuron(input_trains, weights,
                                                                                target, args)
    # loss = van_rossum_loss(spk_rec, target, args)
    # print("Loss = ", loss)
    norm_loss = src.van_rossum_loss(spk_rec, target, args)
    print("Normalized Loss =", norm_loss)
    loss_rec.append(norm_loss)

    norm_factor, _ = torch.max(torch.abs(eligibility_rec), dim=2) # take max along time dimension for each i-jth synapse
    norm_factor = torch.unsqueeze(norm_factor, 2)

    eligibility_rec = eligibility_rec / norm_factor # shape: (nb_inputs, nb_outputs, nb_steps)
    eligibility_rec[eligibility_rec != eligibility_rec] = 0
    assert eligibility_rec.shape == (nb_inputs, nb_outputs, nb_steps), "eligibility_rec shape incorrect"

    # normalizing the error signal:
    norm_factor = torch.max(torch.abs(error_rec))
    error_rec = error_rec / norm_factor  # shape: shape: (nb_steps,)
    assert error_rec.shape == (nb_steps,), "error_rec shape incorrect"
    #print(error_rec)

    # Weight update
    weight_updates = torch.sum(error_rec * eligibility_rec, dim=2)
    assert weight_updates.shape == (nb_inputs, nb_outputs), "wegiht_updates shape incorrect"

    # per-parameter learning rate

    gamma = float(np.exp(-dt/args['tau_rms']))
#   g_ij2 = (error_rec * eligibility_rec)[:, :, -1]**2 # this has a problem
    g_ij2 = torch.sum((error_rec*eligibility_rec)**2, dim=2)
    assert g_ij2.shape == (nb_inputs, nb_outputs), "g_ij2 shape incorrect"
    # Question 1: Whether to take the value of g_ij at the last timestep in each epoch or to take the sum of its values over all timesteps in the epoch?
    # Question 2: How to do normalized convolution for error_signal and eligibility_trace?

    v_ij = torch.max(gamma*v_ij, g_ij2)

    # Store learning rate information for this epoch for xth weight
    g_ij2_rec.append(g_ij2[x])
    v_ij_rec.append(v_ij[x])

    # Evaluate learning rate for this epoch
    r_ij = r_0 / torch.sqrt(v_ij)

    r_ij_rec.append(r_ij[x])

    rate_med = torch.median(r_ij)
    print("Median Learning Rate:", rate_med)
    rate_mean = torch.mean(r_ij)
    print("Avg. Learning Rate:", rate_mean)

    print("Rate of 20th weight:", r_ij[19])
    print("Rate of 67th weight:", r_ij[66])

    weights += r_ij * weight_updates
 """

""" fig, ax = plt.subplots(2, sharex=True)

ax[0].plot(loss_rec)
ax[0].set_title("Loss over epochs")
ax[0].set_ylabel("Loss")

#print("Plotting learning rate parameters")
ax[1].plot(v_ij_rec, label='v_ij')
ax[1].plot(g_ij2_rec, label='g_ij2')
ax[1].plot(r_ij_rec, label='r_ij')
ax[1].set_title("Learning rate parameters for the " + str(x) + 'th neuron') """


import sys
from collections import deque
input= sys.stdin.readline

N, M = map(int, input().strip().split())

map_list=[]
for i in range(N):
    line = input().strip()
    line_list=[]
    for c in line:
        line_list.append(int(c))
    map_list.append(line_list)

visited = [[[0, 0] for j in range(M)] for i in range(N)]

dx = [-1, 0, 1, 0]
dy = [0, -1, 0, 1]

# data = [pos, is_one, cnt]

def bfs():
    q = deque()
    q.append([[0,0], 0])
    visited[0][0][0] = 1
    while q:
        data = q.popleft()
        pos, iscrash = data[0], data[1]
        r, c = pos[0], pos[1]
        # print(cnt, pos)

        if r == N-1 and c == M-1:
            return visited[r][c][iscrash]

        for i in range(4):
            new_r = r + dx[i]
            new_c = c + dy[i]

            if new_r < 0 or new_r >= N or new_c < 0 or new_c >= M:
                continue

            if map_list[new_r][new_c] == 0 and visited[new_r][new_c][iscrash] == 0:
                q.append([[new_r, new_c], iscrash])
                visited[new_r][new_c][iscrash] = visited[r][c][iscrash] + 1

            if map_list[new_r][new_c] == 1 and iscrash == 0:
                q.append([[new_r, new_c], iscrash+1])
                visited[new_r][new_c][iscrash+1] = visited[r][c][iscrash] + 1

    return -1

print(bfs())

# 못해먹겠다,, 어려워,,, 시작이반 블로그 참고


#!/usr/bin/env python

'''
Module that update the software and its databases
'''

import os
import shutil
from sys import exit
import os.path
import tarfile
import requests
from bs4 import BeautifulSoup

from ...base import *
from ...sentry import sentry
from ...clint import progress

class Updater(object):
    def __init__(self, path, ver, url):
        self.inst_path = path
        self.repo_url = url
        self.version = ver

    def update_all(self):
        '''
        Upgrade BigBrother completely
        '''
        print(color.info.info("Fetching version from Github..."))
        # Retrieving github releases
        try:
            response = requests.get(self.repo_url)
        except requests.exceptions.RequestException as e:
            print(color.info.error(e))
            return
        # Getting latest release
        soup = BeautifulSoup(response.content, 'html.parser')
        try:    # Parsing info from page
            version = soup.select("ul.tag-references > li > a > span")[0].text
            download_url = "https://github.com" + \
                soup.select(".release-downloads > li > a")[1]['href']
        except Exception as e:
            sentry.client.captureException()
            print(color.info.error(e))
            return
        # check version
        if version == self.version:
            print(color.info.info("You have already the latest version"))
        else:
            print(color.info.info("New version " + color.bold(
                "{ver}".format(ver=version)) + " found"))
            # install
            if self.install(self.inst_path, download_url):
                print(color.info.info("Need to be restarted for changes to be effective"))
                exit()

    def install(self, path, url):
        try:
            # downloaded file name
            dl_file = self.download(url, path)
            # change directory
            os.chdir(path)
            # extract in path directory
            inst_module = self.extract(dl_file)
            # normalize name
            inst_module_norm = inst_module[:inst_module.find('-')]
            if inst_module_norm in os.listdir():
                shutil.rmtree(inst_module_norm)
            shutil.move(inst_module, inst_module_norm)
            print(color.info.info(color.info.success("Installation completed")))
            return 1
        except Exception as e:
            print(color.info.info(color.info.fail("Installation failed")))
            print(color.info.error(e))
            return 0

    def download(self, url, path):
        '''
        Download module from [url] to [path]
        '''
        # get name of file to downaload
        local_filename = url.split('/')[-1]
        try:
            stream = requests.get(url, stream=True)
            total_length = int(stream.headers['Content-Length'])
        except requests.exceptions.RequestException as e:
            print(color.info.error(e))
            return
        # change to downlaod dir
        try:
            os.chdir(path)
        except Exception as e:
            print(color.info.error(e))
            return
        # write on file
        with open(local_filename, 'wb') as f:
            for chunk in progress.bar(stream.iter_content(chunk_size=1024),
                label=local_filename, expected_size=(total_length/1024)):
                if chunk:
                    f.write(chunk)
                    f.flush()
        return local_filename

    def extract(self, filename):
        try:
            tar = tarfile.open(filename)
            repo = tar.getnames()[0]
            # remove old repo
            if repo in os.listdir():
                shutil.rmtree(repo)
            # extract in current directory
            tar.extractall()
            return repo
        except Exception as e:
            print(color.info.error(e))
            return
        finally:
            tar.close()
            os.remove(filename)


import hashlib
import json
import os
import requests

from tqdm.contrib.concurrent import process_map

import torch
from torch.utils.data import DataLoader

from transformers import PreTrainedTokenizer

import pytorch_lightning as pl

class CoQA(pl.LightningDataModule):
    class CoQADataset(torch.utils.data.Dataset):
        def __init__(
            self,
            src=None,
            tgt=None,
            vocab_size=None,
            max_input_length=None,
            max_output_length=None,
            filename=None,
        ):
            if filename is not None:
                self.uncompressed_type, self.input_ids, self.labels = torch.load(
                    filename
                )
                self.compressed_type = self.input_ids[0].type()
                self.labels = type_list(self.labels, self.uncompressed_type)
            else:
                self.uncompressed_type = torch.long

                if vocab_size is not None:
                    self.compressed_type = get_int_type(vocab_size)
                else:
                    self.compressed_type = self.uncompressed_type

                self.input_ids = [
                    torch.tensor(sample, dtype=self.compressed_type) for sample in src
                ]
                self.labels = [
                    torch.tensor(sample, dtype=self.uncompressed_type) for sample in tgt
                ]

            if max_input_length is not None:
                self.input_ids = [
                    sample[: min(len(sample), max_input_length)]
                    for sample in self.input_ids
                ]
            if max_output_length is not None:
                self.labels = [
                    sample[: min(len(sample), max_output_length)]
                    for sample in self.labels
                ]

        def __getitem__(self, idx):
            item = {"input_ids": self.input_ids[idx], "labels": self.labels[idx]}
            return item

        def __len__(self):
            return len(self.input_ids)

        def save(self, filename):
            torch.save(
                (
                    self.uncompressed_type,
                    self.input_ids,
                    type_list(self.labels, self.compressed_type),
                ),
                filename,
            )

    def __init__(self, hparams, tokenizer: PreTrainedTokenizer):
        super().__init__()
        self.hparams = hparams
        self.tokenizer = tokenizer

    def prepare_data(self):
        datasets = {
            "validate": [
                self.hparams.val_dataset,
                "https://nlp.stanford.edu/data/coqa/coqa-dev-v1.0.json",
            ],
            "train": [
                self.hparams.train_dataset,
                "https://nlp.stanford.edu/data/coqa/coqa-train-v1.0.json",
            ],
        }

        for mode, (dataset_path, dataset_url) in datasets.items():
            if not os.path.isfile(dataset_path):
                print(f"File {dataset_path} not found. Downloading from {dataset_url}")
                download_from_url(dataset_url, dataset_path)

            tokenized_path = self.get_tokenized_path(dataset_path)

            if os.path.isfile(tokenized_path):
                print(f"Found {dataset_path} tokenized. Loading from {tokenized_path}")
                dataset = self.CoQADataset(
                    max_input_length=self.hparams.max_input_length,
                    max_output_length=self.hparams.max_output_length,
                    filename=tokenized_path,
                )
            else:
                print(f"Tokenizing {dataset_path}. This might take a while...")

                dataset = self.process_data(dataset_path)
                tokenized = {}

                src = [sentence.replace("\n", "<n>") for sentence in dataset["src"]]
                tokenized["src"] = self.tokenizer(src)["input_ids"]
                print(f"Source: {len(tokenized['src'])} samples")

                tgt = [sentence.replace("\n", "<n>") for sentence in dataset["tgt"]]
                tokenized["tgt"] = self.tokenizer(tgt)["input_ids"]
                print(f"Target: {len(tokenized['tgt'])} samples")

                dataset = self.CoQADataset(
                    tokenized["src"],
                    tokenized["tgt"],
                    vocab_size=self.tokenizer.vocab_size,
                    max_input_length=self.hparams.max_input_length,
                    max_output_length=self.hparams.max_output_length,
                )

                dataset.save(tokenized_path)
                print(f"Saved tokenized dataset to {tokenized_path}")

            if mode == "train":
                self.train_dataset = dataset
            elif mode == "validate":
                self.val_dataset = dataset
            else:
                print("Unrecognized mode. Only supports train and validate.")

    def train_dataloader(self):
        return DataLoader(
            self.train_dataset,
            batch_size=self.hparams.batch_size,
            shuffle=True,
            num_workers=os.cpu_count(),
            collate_fn=AdaptiveBatch(self.tokenizer.pad_token_id),
            pin_memory=bool(torch.cuda.device_count()),
        )

    def val_dataloader(self):
        return DataLoader(
            self.val_dataset,
            batch_size=self.hparams.batch_size,
            shuffle=False,
            num_workers=os.cpu_count(),
            collate_fn=AdaptiveBatch(self.tokenizer.pad_token_id),
            pin_memory=bool(torch.cuda.device_count()),
        )

    def test_dataloader(self):
        return self.val_dataloader()

    @staticmethod
    def get_tokenized_path(filename):
        hash_value = hash_file(filename)[:8]
        tokenized_path = (
            os.path.splitext(filename)[0] + "_tokenized_" + hash_value + ".pt"
        )
        return tokenized_path

    @staticmethod
    def process_data(dataset_path):
        separator = "\n"

        with open(dataset_path, "r") as f:
            data = json.load(f)

        dataset = {"src": [], "tgt": []}

        for sample in data["data"]:
            context = sample["story"]

            questions = sample["questions"]
            answers = sample["answers"]

            for question, answer in zip(questions, answers):
                if question["turn_id"] != answer["turn_id"]:
                    print(
                        "question and answer turn ids don't match for sample",
                        sample["id"],
                    )

                question = question["input_text"]
                answer = answer["input_text"]

                dataset["src"].append(question + separator + context)
                dataset["tgt"].append(answer)

        return dataset

def download_from_url(url, filename):
    r = requests.get(url, allow_redirects=True)
    with open(filename, "wb") as f:
        f.write(r.content)
    return

# From https://www.geeksforgeeks.org/compare-two-files-using-hashing-in-python/
def hash_file(filename):
    # A arbitrary (but fixed) buffer
    # size (change accordingly)
    # 65536 = 65536 bytes = 64 kilobytes
    BUF_SIZE = 65536

    # Initializing the sha256() method
    sha256 = hashlib.sha256()

    # Opening the file provided as
    # the first commandline arguement
    with open(filename, "rb") as f:
        while True:
            # reading data = BUF_SIZE from
            # the file and saving it in a
            # variable
            data = f.read(BUF_SIZE)

            # True if eof = 1
            if not data:
                break

            # Passing that data to that sh256 hash
            # function (updating the function with
            # that data)
            sha256.update(data)

    # sha256.hexdigest() hashes all the input
    # data passed to the sha256() via sha256.update()
    # Acts as a finalize method, after which
    # all the input data gets hashed hexdigest()
    # hashes the data, and returns the output
    # in hexadecimal format
    return sha256.hexdigest()

def get_int_type(vocab_size):
    if vocab_size <= 2 ** 8:
        return torch.uint8
    else:
        for int_size, int_type in {
            16: torch.int16,
            32: torch.int32,
            64: torch.int64,
        }.items():
            if vocab_size < 2 ** (int_size - 1):
                return int_type
        return torch.int64

def type_list(list_tensors, new_type):
    return [x.type(new_type) for x in list_tensors]

class AdaptiveBatch:
    def __init__(self, pad_token_id):
        self.pad_token_id = pad_token_id

    def __call__(self, samples):
        return {
            k: torch.nn.utils.rnn.pad_sequence(
                [sample[k].squeeze(0) for sample in samples],
                batch_first=True,
                padding_value=self.pad_token_id,
            )
            for k in samples[0]
        }


# -*- encoding: utf-8 -*-
import collections
import email.parser
import json
import platform
import sys
import threading
import time

import pytest
import six
from six.moves import BaseHTTPServer

import ddtrace
from ddtrace.internal import compat
from ddtrace.internal import gitmetadata
from ddtrace.internal.processor.endpoint_call_counter import EndpointCallCounterProcessor
from ddtrace.internal.utils.formats import parse_tags_str
from ddtrace.profiling import exporter
from ddtrace.profiling.exporter import http

from . import test_pprof

# Skip this test on Windows:
# they add little value and the HTTP server shutdown seems unreliable and crashes
# TODO: rewrite those test with another HTTP server that works on win32?
if sys.platform == "win32":
    pytestmark = pytest.mark.skip

_API_KEY = "my-api-key"
_ENDPOINT_COUNTS = {"a": 1, "b": 2}

class _APIEndpointRequestHandlerTest(BaseHTTPServer.BaseHTTPRequestHandler):
    error_message_format = "%(message)s\n"
    error_content_type = "text/plain"
    path_prefix = "/profiling/v1"

    @staticmethod
    def log_message(format, *args):  # noqa: A002
        pass

    @staticmethod
    def _check_tags(tags):
        tags.split(",").sort()
        return (
            len(tags) == 6
            and tags[0].startswith("host:")
            and tags[1] == "language:python"
            and tags[2] == ("profiler_version:%s" % ddtrace.__version__)
            and tags[3].startswith("runtime-id:")
            and tags[4] == "runtime:CPython"
            and tags[5].startswith("service:")
            and tags[6] == platform.python_version(),
        )

    @staticmethod
    def _check_endpoints(endpoints):
        return endpoints == _ENDPOINT_COUNTS

    def _check_event(self, event_json):
        event = json.loads(event_json.decode())
        for key, check in {
            "start": lambda x: x == "1970-01-01T00:00:00Z",
            "end": lambda x: x.startswith("20"),
            "family": lambda x: x == "python",
            "version": lambda x: x == "4",
            "tags_profiler": self._check_tags,
            "endpoint_counts": self._check_endpoints,
        }.items():
            if not check(event[key]):
                return False
        return True

    def do_POST(self):
        assert self.path.startswith(self.path_prefix)
        api_key = self.headers["DD-API-KEY"]
        if api_key != _API_KEY:
            self.send_error(400, "Wrong API Key")
            return
        length = int(self.headers["Content-Length"])
        body = self.rfile.read(length)
        mmpart = b"Content-Type: " + self.headers["Content-Type"].encode() + b"\r\n" + body
        if six.PY2:
            msg = email.parser.Parser().parsestr(mmpart)
        else:
            msg = email.parser.BytesParser().parsebytes(mmpart)
        if not msg.is_multipart():
            self.send_error(400, "No multipart")
            return
        items = collections.defaultdict(list)
        for part in msg.get_payload():
            items[part.get_param("name", header="content-disposition")].append(part.get_payload(decode=True))
        for key, check in {
            "event": lambda x: self._check_event(x[0]),
            "auto": lambda x: x[0].startswith(b"\x1f\x8b\x08\x00"),
            "code-provenance": lambda x: x[0].startswith(b"\x1f\x8b\x08\x00"),
        }.items():
            if not check(items[key]):
                self.send_error(400, "Wrong value for %s: %r" % (key, items[key]))
                return

        self.send_error(200, "OK")

class _TimeoutAPIEndpointRequestHandlerTest(_APIEndpointRequestHandlerTest):
    def do_POST(self):
        # This server sleeps longer than our timeout
        time.sleep(5)
        self.send_error(500, "Argh")

class _ResetAPIEndpointRequestHandlerTest(_APIEndpointRequestHandlerTest):
    def do_POST(self):
        return

class _UnknownAPIEndpointRequestHandlerTest(_APIEndpointRequestHandlerTest):
    def do_POST(self):
        self.send_error(404, "Argh")

_PORT = 8992
_TIMEOUT_PORT = _PORT + 1
_RESET_PORT = _PORT + 2
_UNKNOWN_PORT = _PORT + 3
_ENDPOINT = "http://localhost:%d" % _PORT
_TIMEOUT_ENDPOINT = "http://localhost:%d" % _TIMEOUT_PORT
_RESET_ENDPOINT = "http://localhost:%d" % _RESET_PORT
_UNKNOWN_ENDPOINT = "http://localhost:%d" % _UNKNOWN_PORT

def _make_server(port, request_handler):
    server = BaseHTTPServer.HTTPServer(("localhost", port), request_handler)
    t = threading.Thread(target=server.serve_forever)
    # Set daemon just in case something fails
    t.daemon = True
    t.start()
    return server, t

@pytest.fixture(scope="module")
def endpoint_test_server():
    server, thread = _make_server(_PORT, _APIEndpointRequestHandlerTest)
    try:
        yield thread
    finally:
        server.shutdown()
        thread.join()

@pytest.fixture(scope="module")
def endpoint_test_timeout_server():
    server, thread = _make_server(_TIMEOUT_PORT, _TimeoutAPIEndpointRequestHandlerTest)
    try:
        yield thread
    finally:
        server.shutdown()
        thread.join()

@pytest.fixture(scope="module")
def endpoint_test_reset_server():
    server, thread = _make_server(_RESET_PORT, _ResetAPIEndpointRequestHandlerTest)
    try:
        yield thread
    finally:
        server.shutdown()
        thread.join()

@pytest.fixture(scope="module")
def endpoint_test_unknown_server():
    server, thread = _make_server(_UNKNOWN_PORT, _UnknownAPIEndpointRequestHandlerTest)
    try:
        yield thread
    finally:
        server.shutdown()
        thread.join()

def _get_span_processor():
    endpoint_call_counter_span_processor = EndpointCallCounterProcessor()
    endpoint_call_counter_span_processor.endpoint_counts = _ENDPOINT_COUNTS
    return endpoint_call_counter_span_processor

def test_wrong_api_key(endpoint_test_server):
    # This is mostly testing our test server, not the exporter
    exp = http.PprofHTTPExporter(
        endpoint=_ENDPOINT,
        api_key="this is not the right API key",
        max_retry_delay=2,
        endpoint_call_counter_span_processor=_get_span_processor(),
    )
    with pytest.raises(exporter.ExportError) as t:
        exp.export(test_pprof.TEST_EVENTS, 0, 1)
    assert str(t.value) == "Server returned 400, check your API key"

def test_export(endpoint_test_server):
    exp = http.PprofHTTPExporter(
        endpoint=_ENDPOINT, api_key=_API_KEY, endpoint_call_counter_span_processor=_get_span_processor()
    )
    exp.export(test_pprof.TEST_EVENTS, 0, compat.time_ns())

def test_export_server_down():
    exp = http.PprofHTTPExporter(
        endpoint="http://localhost:2",
        api_key=_API_KEY,
        max_retry_delay=2,
        endpoint_call_counter_span_processor=_get_span_processor(),
    )
    with pytest.raises(exporter.ExportError):
        exp.export(test_pprof.TEST_EVENTS, 0, 1)

def test_export_timeout(endpoint_test_timeout_server):
    exp = http.PprofHTTPExporter(
        endpoint=_TIMEOUT_ENDPOINT,
        api_key=_API_KEY,
        timeout=1,
        max_retry_delay=2,
        endpoint_call_counter_span_processor=_get_span_processor(),
    )
    with pytest.raises(exporter.ExportError):
        exp.export(test_pprof.TEST_EVENTS, 0, 1)

def test_export_reset(endpoint_test_reset_server):
    exp = http.PprofHTTPExporter(
        endpoint=_RESET_ENDPOINT,
        api_key=_API_KEY,
        timeout=1,
        max_retry_delay=2,
        endpoint_call_counter_span_processor=_get_span_processor(),
    )
    with pytest.raises(exporter.ExportError):
        exp.export(test_pprof.TEST_EVENTS, 0, 1)

def test_export_404_agent(endpoint_test_unknown_server):
    exp = http.PprofHTTPExporter(endpoint=_UNKNOWN_ENDPOINT, endpoint_call_counter_span_processor=_get_span_processor())
    with pytest.raises(exporter.ExportError) as t:
        exp.export(test_pprof.TEST_EVENTS, 0, 1)
    assert str(t.value) == (
        "Datadog Agent is not accepting profiles. " "Agent-based profiling deployments require Datadog Agent >= 7.20"
    )

def test_export_404_agentless(endpoint_test_unknown_server):
    exp = http.PprofHTTPExporter(
        endpoint=_UNKNOWN_ENDPOINT, api_key="123", timeout=1, endpoint_call_counter_span_processor=_get_span_processor()
    )
    with pytest.raises(exporter.ExportError) as t:
        exp.export(test_pprof.TEST_EVENTS, 0, 1)
    assert str(t.value) == "HTTP Error 404"

def test_export_tracer_base_path(endpoint_test_server):
    # Base path is prepended to the endpoint path because
    # it does not start with a slash.
    exp = http.PprofHTTPExporter(
        endpoint=_ENDPOINT + "/profiling/",
        api_key=_API_KEY,
        endpoint_path="v1/input",
        endpoint_call_counter_span_processor=_get_span_processor(),
    )
    exp.export(test_pprof.TEST_EVENTS, 0, compat.time_ns())

def test_export_tracer_base_path_agent_less(endpoint_test_server):
    # Base path is ignored by the profiling HTTP exporter
    # because the endpoint path starts with a slash.
    exp = http.PprofHTTPExporter(
        endpoint=_ENDPOINT + "/profiling/",
        api_key=_API_KEY,
        endpoint_path="/profiling/v1/input",
        endpoint_call_counter_span_processor=_get_span_processor(),
    )
    exp.export(test_pprof.TEST_EVENTS, 0, compat.time_ns())

def test_get_tags():
    tags = parse_tags_str(http.PprofHTTPExporter(env="foobar", endpoint="")._get_tags("foobar"))
    assert len(tags) == 8
    assert tags["service"] == "foobar"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["env"] == "foobar"
    assert tags["runtime"] == "CPython"
    assert tags["profiler_version"] == ddtrace.__version__
    assert "version" not in tags

def test_get_malformed(monkeypatch):
    monkeypatch.setenv("DD_TAGS", "mytagfoobar")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    assert len(tags) == 7
    assert tags["service"] == "foobar"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["profiler_version"] == ddtrace.__version__

    monkeypatch.setenv("DD_TAGS", "mytagfoobar,")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    assert len(tags) == 7
    assert tags["service"] == "foobar"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["profiler_version"] == ddtrace.__version__

    monkeypatch.setenv("DD_TAGS", ",")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    assert len(tags) == 7
    assert tags["service"] == "foobar"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["profiler_version"] == ddtrace.__version__

    monkeypatch.setenv("DD_TAGS", "foo:bar,")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    assert len(tags) == 8
    assert tags["service"] == "foobar"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["foo"] == "bar"
    assert tags["profiler_version"] == ddtrace.__version__

def test_get_tags_override(monkeypatch):
    monkeypatch.setenv("DD_TAGS", "mytag:foobar")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    assert len(tags) == 8
    assert tags["service"] == "foobar"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["mytag"] == "foobar"
    assert tags["profiler_version"] == ddtrace.__version__
    assert "version" not in tags

    monkeypatch.setenv("DD_TAGS", "mytag:foobar,author:jd")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    assert len(tags) == 9
    assert tags["service"] == "foobar"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["mytag"] == "foobar"
    assert tags["author"] == "jd"
    assert tags["profiler_version"] == ddtrace.__version__
    assert "version" not in tags

    monkeypatch.setenv("DD_TAGS", "")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    assert len(tags) == 7
    assert tags["service"] == "foobar"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["profiler_version"] == ddtrace.__version__
    assert "version" not in tags

    monkeypatch.setenv("DD_TAGS", "foobar:baz,service:mycustomservice")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    assert len(tags) == 8
    assert tags["service"] == "mycustomservice"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["foobar"] == "baz"
    assert tags["profiler_version"] == ddtrace.__version__
    assert "version" not in tags

    monkeypatch.setenv("DD_TAGS", "foobar:baz,service:�不")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    assert len(tags) == 8
    assert tags["service"] == "�不"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["foobar"] == "baz"
    assert tags["profiler_version"] == ddtrace.__version__
    assert "version" not in tags

    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="", version="123")._get_tags("foobar"))
    assert len(tags) == 9
    assert tags["service"] == "�不"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["foobar"] == "baz"
    assert tags["profiler_version"] == ddtrace.__version__
    assert tags["version"] == "123"
    assert "env" not in tags

    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="", version="123", env="prod")._get_tags("foobar"))
    assert len(tags) == 10
    assert tags["service"] == "�不"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["foobar"] == "baz"
    assert tags["profiler_version"] == ddtrace.__version__
    assert tags["version"] == "123"
    assert tags["env"] == "prod"

    tags = parse_tags_str(
        http.PprofHTTPExporter(endpoint="", version="123", env="prod", tags={"mytag": "123"})._get_tags("foobar")
    )
    assert len(tags) == 11
    assert tags["service"] == "�不"
    assert len(tags["host"])
    assert len(tags["runtime-id"])
    assert tags["language"] == "python"
    assert tags["runtime"] == "CPython"
    assert tags["foobar"] == "baz"
    assert tags["profiler_version"] == ddtrace.__version__
    assert tags["version"] == "123"
    assert tags["env"] == "prod"
    assert tags["mytag"] == "123"

@pytest.mark.skip(reason="Needs investigation about the segfaulting")
@pytest.mark.subprocess(env=dict(DD_PROFILING_TAGS="mytag:baz"))
def test_get_tags_legacy():
    from ddtrace.internal.utils.formats import parse_tags_str  # noqa
    from ddtrace.profiling.exporter import http  # noqa

    # REVERTME: Investigating segfaults on CI
    # tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    # assert tags["mytag"] == "baz"

@pytest.mark.subprocess(env=dict(DD_PROFILING_TAGS="mytag:val2,ddptag:lo", DD_TAGS="mytag:val1,ddtag:hi"))
def test_get_tags_precedence():
    from ddtrace.internal.utils.formats import parse_tags_str
    from ddtrace.profiling.exporter import http

    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))
    assert tags["mytag"] == "val2"
    assert tags["ddtag"] == "hi"
    assert tags["ddptag"] == "lo"

def test_gitmetadata_ddtags(monkeypatch):
    gitmetadata._GITMETADATA_TAGS = None

    monkeypatch.setenv("DD_TAGS", "git.commit.sha:12345,git.repository_url:github.com/user/tag_repo")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))

    # must be from env variables
    assert tags["git.commit.sha"] == "12345"
    assert tags["git.repository_url"] == "github.com/user/tag_repo"

def test_gitmetadata_env(monkeypatch):
    gitmetadata._GITMETADATA_TAGS = None

    monkeypatch.setenv("DD_TAGS", "git.commit.sha:12345,git.repository_url:github.com/user/tag_repo")
    monkeypatch.setenv("DD_GIT_COMMIT_SHA", "123456")
    monkeypatch.setenv("DD_GIT_REPOSITORY_URL", "github.com/user/env_repo")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))

    # must be from env variables
    assert tags["git.commit.sha"] == "123456"
    assert tags["git.repository_url"] == "github.com/user/env_repo"

def test_gitmetadata_disabled(monkeypatch):
    gitmetadata._GITMETADATA_TAGS = None

    monkeypatch.setenv("DD_TAGS", "git.commit.sha:12345,git.repository_url:github.com/user/tag_repo")
    monkeypatch.setenv("DD_GIT_COMMIT_SHA", "123456")
    monkeypatch.setenv("DD_GIT_REPOSITORY_URL", "github.com/user/env_repo")
    monkeypatch.setenv("DD_TRACE_GIT_METADATA_ENABLED", "false")
    tags = parse_tags_str(http.PprofHTTPExporter(endpoint="")._get_tags("foobar"))

    # must not present
    assert "git.commit.sha" not in tags
    assert "git.repository_url" not in tags


# -*- coding: utf-8 -*-
# @Time    : 2022/8/23 10:34
# @Author  : HongFei Wang
import os.path

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import torch
import tqdm
import xlwt
from osgeo import gdal

from main import hyper_parameter
from main import transform
from model import UNet

mdl_index = 40  # 模型索引
modle = UNet(hyper_parameter['num_class'])

# 预测图像
file_paths = [
		r'E:\taobao\unet_1200\datas\image\10.tif',
		r'E:\taobao\unet_1200\datas\image\11.tif',
		r'E:\taobao\unet_1200\datas\image\12.tif'
]

colors = ['red', 'orange', 'yellow', 'green', 'blue', 'purple', 'grey', 'pink']

def ConfusionMatrix(numClass, imgPredict, Label):
	"""

	:param numClass:
	:param imgPredict:
	:param Label:
	:return: numclass*numclass的混淆矩阵
	"""
	# 计算混淆矩阵
	mask = (Label >= 0) & (Label < numClass)
	label = numClass * Label[mask] + imgPredict[mask]
	count = np.bincount(label, minlength=numClass ** 2)
	confusionMatrix = count.reshape(numClass, numClass)
	return confusionMatrix

def OverallAccuracy(confusionMatrix):
	#  返回所有类的整体像素精度OA
	# acc = (TP + TN) / (TP + TN + FP + TN)
	"""
		:param confusionMatrix:
		:return: OA 值
		"""
	acc = np.diag(confusionMatrix).sum()
	alls = confusionMatrix.sum()
	OA = acc / alls
	return OA

def write_sheet(sheet, arrays):
	rows, cols = arrays.shape
	for row in range(rows):
		for col in range(cols):
			sheet.write(row, col, int(arrays[row][col]))

def readTif(fileName):
	dataset = gdal.Open(fileName)
	if dataset == None:
		print(fileName + "文件无法打开")
		return
	im_width = dataset.RasterXSize  # 栅格矩阵的列数
	im_height = dataset.RasterYSize  # 栅格矩阵的行数
	im_data = dataset.ReadAsArray(0, 0, im_width, im_height)  # 获取数据
	im_geotrans = dataset.GetGeoTransform()  # 获取仿射矩阵信息
	im_proj = dataset.GetProjection()  # 获取投影信息
	return im_data, im_proj, im_geotrans, im_width, im_height

def writeTiff(im_data, im_width, im_height, im_geotrans, im_proj, path):
	if 'int8' in im_data.dtype.name:
		datatype = gdal.GDT_Byte
	elif 'int16' in im_data.dtype.name:
		datatype = gdal.GDT_UInt16
	else:
		datatype = gdal.GDT_Float32
	im_data = np.array(im_data)
	driver = gdal.GetDriverByName("GTiff")
	dataset = driver.Create(path, im_width, im_height, 1, datatype)
	if dataset is not None:
		dataset.SetGeoTransform(im_geotrans)  # 写入仿射变换参数
		dataset.SetProjection(im_proj)  # 写入投影
	dataset.GetRasterBand(1).WriteArray(im_data)
	del dataset

def main(file_paths, nums_class, save_excle: str, save_path: str):
	"""

	:param file_paths:
	:param nums_class:
	:param save_path: 输出路径
	:return:
	"""
	# 分类数据的tiff
	wook = xlwt.Workbook()
	sheet_index = 0
	label_out = None
	for file_path in tqdm.tqdm(file_paths):
		# 读取数据
		img, im_proj, im_geotrans, im_width, im_height = readTif(file_path)
		maxs_ = np.max(img)
		img_file = img / maxs_  # 归一化
		img = transform(torch.from_numpy(img_file.astype(np.float32)))
		with torch.no_grad():
			pred = modle(img.unsqueeze(0))
			out = torch.argmax(pred, dim=1).squeeze().numpy()
		# 标签读取
		label = gdal.Open(file_path.replace('image', 'label'))
		label = label.ReadAsArray()
		plt.figure(figsize=(16, 9))

		# 标签绘制
		maxs_label, mins_label = label.max(), label.min()
		colors_temp_label = colors[mins_label:maxs_label + 1]
		plt.subplot(221)
		plt.title('label')
		plt.imshow(label, cmap=mpl.colors.ListedColormap(colors_temp_label))

		# 原图绘制
		plt.subplot(222)
		plt.title('image')
		img_file = np.transpose(img_file[:3, :, :], (1, 2, 0))
		plt.imshow(img_file)
		# 输出绘制
		plt.subplot(223)
		plt.title('out')
		maxs_out, mins_out = out.max(), out.min()
		colors_temp_out = colors[mins_out:maxs_out + 1]
		plt.imshow(out, cmap=mpl.colors.ListedColormap(colors_temp_out))
		writeTiff(out, im_width, im_height, im_geotrans, im_proj,
		          save_path + file_path.split('\\')[-1].replace('.tif', 'out.tif'))

		plt.subplot(224)
		plt.title('superposition')
		plt.imshow(img_file)
		plt.imshow(out, alpha=0.2, cmap=mpl.colors.ListedColormap(colors_temp_out))
		if label_out is not None:
			sheet_index += 1
			sheet = wook.add_sheet(str(sheet_index))
			comix = ConfusionMatrix(numClass=nums_class, imgPredict=out, Label=label_out)
			write_sheet(sheet, comix)
		else:
			label_out = out
	wook.save(save_excle)
	plt.show()

if __name__ == '__main__':
	if not os.path.exists(f'./Mdl_Data/unet/{mdl_index}.pth'):
		print(f'File not found  “./Mdl_Data/unet/{mdl_index}.pth”')
	else:
		print(f'load modle {mdl_index}.pth')
		modle.load_state_dict(torch.load(f'./Mdl_Data/unet/{mdl_index}.pth'))
	main(file_paths=file_paths, nums_class=hyper_parameter['num_class'], save_excle='result.xls', save_path='')


#BRIEF, ORB feature
import cv2
import numpy as np

img = cv2.imread('D:\TestImage\SK-1_FRAME_IMAGE\F3_F1.png', cv2.IMREAD_COLOR)

orb = cv2.ORB_create()
orb.setMaxFeatures(200)

keyPoints = orb.detect(img, None)
keyPoints, descriptors = orb.compute(img, keyPoints)

show_img = cv2.drawKeypoints(img, keyPoints, None, (0, 0, 255),
                             cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)

cv2.imshow('ORB descriptors', show_img)
cv2.waitKey()
cv2.destroyAllWindows()

brief = cv2.xfeatures2d.BriefDescriptorExtractor_create(32, True)

keyPoints, descriptors = brief.compute(img, keyPoints)

show_img = cv2.drawKeypoints(img, keyPoints, None, (0, 255, 0),
                             cv2. DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
cv2.imshow('BRIEF descriptors', show_img)
cv2.waitKey()
cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
"""
@Date:   2020/3/11 10:26
@Author: Pangpd
@FileName: show_maps.py
@IDE: PyCharm
@Description:  展示
"""
import numpy as np
import matplotlib.pyplot as plt
import sklearn.preprocessing
import matplotlib.patches as mpatches

def color_list(class_nums):
    """
    :param class_nums: 地物类数
    :return: 返回实际的地物数量的颜色版
    """
    colors = np.array([[0, 0, 0], [254, 0, 0], [0, 255, 1], [2, 2, 255],
                       [255, 255, 2], [2, 255, 255], [255, 2, 255], [193, 193, 193],
                       [129, 129, 129], [129, 2, 2], [128, 129, 2], [2, 129, 3],
                       [130, 2, 128], [2, 129, 131], [3, 2, 129], [255, 166, 2],
                       [255, 216, 2]])
    return colors[:class_nums + 1]

def show_label(data, labels, class_nums, save_path, class_names=None):
    '''
    只画出标签图
    :param data: loadData函数中返回的data（经过标准化后的）
    :param labels: 地物标签值
    :param class_nums: 地物总类数
    :param save_path: gt图的保存路径
    :param class_names: 地物标签名
    :return: None
    '''
    number_of_rows = int(data.shape[0])
    number_of_columns = int(data.shape[1])
    gt = labels.reshape(-1, 1)  # 拉成一列
    colors = color_list(class_nums)
    colors = sklearn.preprocessing.minmax_scale(colors, feature_range=(0, 1))
    gt_thematic_map = np.zeros(shape=(number_of_rows, number_of_columns, 3))
    cont = 0
    for i in range(number_of_rows):
        for j in range(number_of_columns):
            gt_thematic_map[i, j, :] = colors[gt[cont, 0]]
            cont += 1
    plt.axis('off')
    height, width = labels.shape

    plt.gcf().set_size_inches(width / 100.0, height / 100.0)  # 输出width*height像素
    plt.gca().xaxis.set_major_locator(plt.NullLocator())
    plt.gca().yaxis.set_major_locator(plt.NullLocator())
    plt.subplots_adjust(top=1, bottom=0, right=0.999, left=0, hspace=0, wspace=0)
    #plt.margins(0, 0)
    plt.xticks([])  # 关闭刻度
    plt.yticks([])
    plt.imshow(gt_thematic_map)  # 标签图
    plt.savefig(save_path, dpi=300)

def show_gt(data, labels, class_nums, save_path, class_names=None, only_color=False):
    '''
    画出标签图的RGB图和假彩色图(任意非R,G,B波段的合成图，)
    :param data: loadData函数中返回的data（经过标准化后的）
    :param labels: 地物标签值
    :param class_nums: 地物总类数
    :param save_path: gt图的保存路径
    :param class_names: 地物标签名
    :return: None
    '''
    number_of_rows = int(data.shape[0])
    number_of_columns = int(data.shape[1])
    data = data.reshape(-1, data.shape[-1])
    gt = labels.reshape(-1, 1)  # 拉成一列
    colors = color_list(class_nums)
    # visualizing the indian pines dataset hyperspectral cube in RGB

    colors = sklearn.preprocessing.minmax_scale(colors, feature_range=(0, 1))
    pixels_normalized = sklearn.preprocessing.minmax_scale(data, feature_range=(0, 1))

    gt_thematic_map = np.zeros(shape=(number_of_rows, number_of_columns, 3))
    rgb_hyperspectral_image = np.zeros(shape=(number_of_rows, number_of_columns, 3))
    cont = 0
    for i in range(number_of_rows):
        for j in range(number_of_columns):
            rgb_hyperspectral_image[i, j, 0] = pixels_normalized[cont, 29]  # 10
            rgb_hyperspectral_image[i, j, 1] = pixels_normalized[cont, 42]  # 24
            rgb_hyperspectral_image[i, j, 2] = pixels_normalized[cont, 89]  # 44
            gt_thematic_map[i, j, :] = colors[gt[cont, 0]]
            cont += 1
    # fig = plt.figure(figsize=(15, 15))
    if only_color == True:  # 仅获取假彩色图像

        plt.axis('off')
        height, width =labels.shape

        plt.gcf().set_size_inches(width / 100.0, height / 100.0)  # 输出width*height像素
        plt.gca().xaxis.set_major_locator(plt.NullLocator())
        plt.gca().yaxis.set_major_locator(plt.NullLocator())
        plt.subplots_adjust(top=1, bottom=0, right=0.999, left=0, hspace=0, wspace=0)
        plt.margins(0, 0)

        plt.xticks([])  # 关闭刻度
        plt.yticks([])

        plt.imshow(rgb_hyperspectral_image)  # 假彩色
        plt.savefig(save_path, dpi=300)
        return

    fig = plt.figure()
    columns = 2
    rows = 1

    ax1 = fig.add_subplot(rows, columns, 1)
    plt.xticks([])  # 关闭刻度
    plt.yticks([])
    ax1.set_xlabel("(a)", fontsize=15)
    plt.imshow(rgb_hyperspectral_image)  # 假彩色

    ax2 = fig.add_subplot(rows, columns, 2)
    fig.subplots_adjust(left=0.01, top=0.96, right=0.96, bottom=0.04, wspace=0.02, hspace=0.04)
    plt.xticks([])  # 关闭刻度
    plt.yticks([])
    ax2.set_xlabel("(b)", fontsize=15)
    plt.imshow(gt_thematic_map)  # 标签图

    patches = [mpatches.Patch(color=colors[i + 1], label=class_names[i]) for i in range(len(colors) - 1)]
    # plt.legend(handles=patches, loc=4, borderaxespad=0.)
    ax2.legend(handles=patches, bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0, frameon=False)
    # plt.tight_layout()
    fig.savefig(save_path, dpi=300, bbox_inches='tight')
    plt.show()

def show_pred(pred_test, labels, class_nums, dataset_name, save_path, removeZeroLabels=True):
    '''
    画出预测后的结果图，带背景和不带背景,默认使用不带背景
    :param pred_test: 预测后的结果（是行元素）
    :param labels: 真实标签值
    :param class_nums: 地物种类数
    :param dataset_name: 数据集名字
    :param save_path: 保存路径 数据集名字
    :param removeZeroLabels: 是否移除背景，True表示显示不带背景图，False表示显示背景图
    :return:  None
    '''
    pred_map = np.zeros(shape=(labels.shape[0], labels.shape[1], 3))
    x = np.ravel(pred_test).reshape(-1, 1)  # 预测的结果也拉成一列
    colors = color_list(class_nums)  # 获取染色板
    colors = sklearn.preprocessing.minmax_scale(colors, feature_range=(0, 1))
    cont = 0
    for i in range(labels.shape[0]):
        for j in range(labels.shape[1]):
            pred_map[i, j, :] = colors[x[cont, 0]]  # 染色
            cont += 1
    fig = plt.figure(figsize=(15, 15))

    height, width = labels.shape
    plt.axis('off')
    plt.gcf().set_size_inches(width / 100.0, height / 100.0)  # 输出width*height像素
    plt.gca().xaxis.set_major_locator(plt.NullLocator())
    plt.gca().yaxis.set_major_locator(plt.NullLocator())
    plt.subplots_adjust(top=1, bottom=0, right=0.999, left=0, hspace=0, wspace=0)
    plt.margins(0, 0)

    plt.xticks([])  # 关闭刻度
    plt.yticks([])
    plt.imshow(pred_map)  # 带背景的预测结果图
    fig.savefig(save_path + '/maps_withZeros' + '_' + dataset_name + '.png', dpi=300)

    if removeZeroLabels == True:  # 显示不带背景的预测结果图
        pred_test_2D = pred_test.reshape(labels.shape[0], labels.shape[1])  # 转换成2维
        for i in range(labels.shape[0]):
            for j in range(labels.shape[1]):
                if labels[i, j] == 0:
                    pred_test_2D[i, j] = 0
        pred_test_1D = pred_test_2D.reshape(-1, 1)  # 再次将结果拉成一列
        pred_map_removeZeroLabels = np.zeros(shape=(labels.shape[0], labels.shape[1], 3))
        cont = 0
        for i in range(labels.shape[0]):
            for j in range(labels.shape[1]):
                pred_map_removeZeroLabels[i, j, :] = colors[pred_test_1D[cont, 0]]  # 染色
                cont += 1
        fig = plt.figure(figsize=(15, 15))

        plt.axis('off')
        plt.gcf().set_size_inches(width / 100.0, height / 100.0)  # 输出width*height像素
        plt.gca().xaxis.set_major_locator(plt.NullLocator())
        plt.gca().yaxis.set_major_locator(plt.NullLocator())
        plt.subplots_adjust(top=1, bottom=0, right=0.999, left=0, hspace=0, wspace=0)
        plt.margins(0, 0)
        plt.xticks([])  # 关闭刻度
        plt.yticks([])
        plt.imshow(pred_map_removeZeroLabels)  # 不带背景的预测结果图
        fig.savefig(save_path + '/maps_pred' + '_' + dataset_name + '.png', dpi=300)


#!/usr/bin/python

# Author: Ivy Aiwei Zhang
# Last updated: 7-10-2023
# Purpose: An updated version for unit testing
# Pipeline: imports FrameExtraction to get real-time images, write to txt file

# import visual odometry model --> change hypothesis to visual_odometry when finalized
from hypothesis import VisualOdometry
from frame_extraction import FrameExtraction

# ROS node messages
import rospy
from sensor_msgs.msg import CompressedImage
from stag_ros.msg import StagMarkers
from geometry_msgs.msg import PoseStamped

# other packages
import numpy as np
import cv2 as cv
import roslib
from cv_bridge import CvBridge, CvBridgeError
import sys
import os as os
import transformations as transf
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import yaml
from yaml.loader import SafeLoader

# GLOBAL VARIABLES
PREVIOUS_IMAGE = None
CURRENT_IMAGE = None

class UnitTest:
    def __init__(self):
        # first extract the images
        self.adjacent_image_extraction = FrameExtraction()

        self.bridge = CvBridge()

        # initialize visual odometry node
        self.vo_node = VisualOdometry()
        self.vo_node.parse_camera_intrinsics()

        self.prev_transformation_matrix = self.vo_node.make_transform_mat(translation=[0, 0, 0], euler=[0, 0, 0])
        print("Previous transform matrix at initialization: {}".format(self.prev_transformation_matrix))
        self.robot_curr_position = self.vo_node.make_transform_mat(translation=[0, 0, 0], euler=[0, 0, 0])
        print("robot_curr_position at initialization: {}".format(self.robot_curr_position))

        self.detected_marker = False
        self.start_feature_matching = False

        self.ground_truth_full_list = []

        self.previous_frame = self.adjacent_image_extraction.frame_one
        self.current_frame = self.adjacent_image_extraction.frame_two
        print("on initialization: are the frames the same? ", self.previous_frame==self.current_frame)

    """THIS PART OF THE CODE CONTAINS MISCELLANEOUS HELPER FUNCTIONS"""

    def show_two_frames(self):
        pass

    """THIS PART OF THE CODE COMPUTES THE GROUND TRUTH COORDINATES AND CALCULATES THE TRANSFORMATION"""

    def get_ground_truth_coordinates_of_group_of_frames(self, frames):
        for frame in frames:
            self.vo_node.marker_callback(msg=frame)
            self.ground_truth_full_list.append(self.vo_node.ground_truth_list)
        print("Length of ground truth full list: ", len(self.ground_truth_full_list))
        print("Ground truth full list: {}".format(self.ground_truth_full_list))

    def compute_ground_truth_of_one_single_frame(self):
        frames = [self.previous_frame]
        self.get_ground_truth_coordinates_of_group_of_frames(frames)

    def compute_ground_truth_of_two_frames(self):
        frames = [self.previous_frame, self.current_frame]
        self.get_ground_truth_coordinates_of_group_of_frames(frames)

    def are_they_the_same(self):  # note: only two frames to compare
        print("Are the two frames the same? {}".format(self.previous_frame == self.current_frame))
        print("Are the two frames the same format? {}, {}".format(type(self.previous_frame), type(self.current_frame)))

        cv.imshow("previous",self.previous_frame)
        cv.imshow("current", self.current_frame)

    """THIS PART OF THE CODE COMPUTES THE VISUAL ODOMETRY DATA AND CALCULATE THE PREDICTED TRANSFORMATION"""

    # def get_matches_and_transformations(self):
    #     previous_key_points, previous_descriptors, previous_image_with_keypoints_drawn = self.vo_node.compute_current_image_elements(
    #         self.vo_node.previous_image)
    #
    #     print("typeeee", type(previous_image_with_keypoints_drawn))
    #
    #
    #     current_key_points, current_descriptors, current_image_with_keypoints_drawn = self.vo_node.compute_current_image_elements(
    #         self.current_image)
    #     print("Current key points: {}".format(current_key_points))
    #     print("Current descriptors: {}".format(current_descriptors))
    #
    #     self.vo_node.previous_current_matching(previous_image_with_keypoints_drawn, current_key_points, current_descriptors,
    #                                            current_image_with_keypoints_drawn)
    # print("The essential matrix for image 0 and image 1: {}".format(self.vo_node.essential_matrix))
    # print("The essential matrix for image 0 and image 1: {}".format(self.vo_node.essential_matrix))

def main(args):
    rospy.init_node('VisualOdometryNode', anonymous=True)
    two_images = UnitTest()
    print("visual odometry activated")
    rospy.sleep(1)

    """UNCOMMENT the following line to compute ground truth of one single frame"""
    # two_images.compute_ground_truth_of_one_single_frame()

    """UNCOMMENT the following line to compute ground truth of two frames"""
    two_images.compute_ground_truth_of_two_frames()

    """UNCOMMENT the following line if you want to check if ground truths labelled are exactly the same"""
    print(two_images.are_they_the_same())
    # print(two_images.adjacent_image_extraction.are_they_the_same())

    # two_images.get_matches_and_transformations()

# create the name function
if __name__ == '__main__':
    try:
        main(sys.argv)
    except rospy.ROSInterruptException:
        pass