from tensorflow.examples.tutorials.mnist import input_data
import matplotlib.pyplot as plt
import numpy as np
from scipy.misc import imread, imresize

import config as cfg
from utils.layers import *
from utils.layer_utils import *
from utils.fast_layers import *
from utils.vis_utils import *
from utils.gradient_check import *
from utils.solver import *

def rel_error(x, y):
	""" returns relative error """
	return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))

def imshow_noax(img, normalize=True):
	""" Tiny helper to show images as uint8 and remove axis labels """
	if normalize:
		img_max, img_min = np.max(img), np.min(img)
		img = 255.0 * (img - img_min) / (img_max - img_min)
	if len(img.shape) == 2:
		out_img = img[:, :, np.newaxis].repeat([3], axis=2)
	else: out_img = img
	plt.imshow(out_img.astype('uint8'))
	plt.gca().axis('off')

def test(data):
	model = ThreeLayerConvNet(weight_scale=0.001, hidden_dim=500, reg=0.001, use_batchnorm=False)

	solver = Solver(model, data,
	                num_epochs=1, batch_size=50,
	                update_rule='adam',
	                optim_config={
	                  'learning_rate': 1e-3,
	                },
	                verbose=True, print_every=20)
	solver.train()

	grid = visualize_grid(model.params['W1'].transpose(0, 2, 3, 1))
	plt.imshow(grid.astype('uint8'))
	plt.axis('off')
	plt.gcf().set_size_inches(5, 5)

	plt.show()

def main(data_set="cifar10"):

	if data_set == "cifar10":
		# load cifar10 data set
		train_images, train_labels, val_images, val_labels, test_images, test_labels = cfg.load_cifar10()
		classes = cfg.CIFAR10_classes
	elif data_set == "mnist":
		mnist = input_data.read_data_sets(cfg.MNIST_PATH)
		train_images = mnist.train.images
		train_labels = mnist.train.labels
		test_images = mnist.test.images
		test_labels = mnist.test.labels
		classes = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']

	train_images = train_images.reshape(-1, 3, 32, 32)
	val_images = val_images.reshape(-1, 3, 32, 32)
	test_images = test_images.reshape(-1, 3, 32, 32)

	data = {
		  'X_train': train_images,
		  'y_train': train_labels,
		  'X_val': val_images,
		  'y_val': val_labels,
		}

	np.random.seed(231)

	test(data)

class ThreeLayerConvNet(object):
	"""
    A three-layer convolutional network with the following architecture:

    conv - relu - 2x2 max pool - affine - relu - affine - softmax

    The network operates on minibatches of data that have shape (N, C, H, W)
	"""
	def __init__(self, input_dim=(3, 32, 32), num_filters=32, filter_size=7,
				 hidden_dim=100, num_classes=10, weight_scale=1e-3, reg=0.0,
				 use_batchnorm=False, dtype=np.float32):
		"""
		Initialize a new network.

		Inputs:
		- input_dim: Tuple (C, H, W) giving size of input data
		- num_filters: Number of filters to use in the convolutional layer
		- filter_size: Size of filters to use in the convolutional layer
		- hidden_dim: Number of units to use in the fully-connected hidden layer
		- num_classes: Number of scores to produce from the final affine layer.
		- weight_scale: Scalar giving standard deviation for random initialization
		  of weights.
		- reg: Scalar giving L2 regularization strength
		- dtype: numpy datatype to use for computation.
		"""
		self.params = {}
		self.reg = reg
		self.dtype = dtype
		self.use_batchnorm = use_batchnorm

		# network. Weights should be initialized from a Gaussian with standard
		# deviation equal to weight_scale; biases should be initialized to zero.
		# All weights and biases should be stored in the dictionary self.params.
		# Store weights and biases for the convolutional layer using the keys 'W1'
		# and 'b1'; use keys 'W2' and 'b2' for the weights and biases of the
		# hidden affine layer, and keys 'W3' and 'b3' for the weights and biases
		# of the output affine layer.
		self.params['W1'] = weight_scale * np.random.randn(num_filters, input_dim[0], filter_size, filter_size)
		self.params['b1'] = np.zeros(num_filters)
		self.flatten_dim  = num_filters * np.power(((input_dim[-1] - 1) // 2 + 1), 2)
		self.params['W2'] = weight_scale * np.random.randn(self.flatten_dim, hidden_dim)
		self.params['b2'] = np.zeros(hidden_dim)
		self.params['W3'] = weight_scale * np.random.randn(hidden_dim, num_classes)
		self.params['b3'] = np.zeros(num_classes)

		self.bn_params = []
		if self.use_batchnorm:
			self.bn_params = [{'mode': 'train'}]
			self.params['gamma1'] = np.ones(num_filters)
			self.params['beta1'] = np.zeros(num_filters)

		for k, v in self.params.items():
			self.params[k] = v.astype(dtype)

	def loss(self, X, y=None):
		"""
		Evaluate loss and gradient for the three-layer convolutional network.
		"""
		mode = 'test' if y is None else 'train'
		if self.use_batchnorm:
			gamma1, beta1 = self.params['gamma1'], self.params['beta1']
			for bn_param in self.bn_params:
				bn_param['mode'] = mode

		W1, b1 = self.params['W1'], self.params['b1']
		W2, b2 = self.params['W2'], self.params['b2']
		W3, b3 = self.params['W3'], self.params['b3']

		# pass conv_param to the forward pass for the convolutional layer
		filter_size = W1.shape[2]
		conv_param = {'stride': 1, 'pad': (filter_size - 1) // 2}

		# pass pool_param to the forward pass for the max-pooling layer
		pool_param = {'pool_height': 2, 'pool_width': 2, 'stride': 2}

		cache_hist = []
		out_hist = []
		# forward pass

		# conv - (batchnorm) - relu
		if self.use_batchnorm:
			out, cache = conv_bn_relu_forward(X, W1, b1, gamma1, beta1, conv_param, self.bn_params[0])
		else:
			out, cache = conv_relu_forward(X, W1, b1, conv_param)
		cache_hist.append(cache)
		out_hist.append(out)

		# 2x2 max pool
		out, cache = max_pool_forward_fast(out, pool_param)
		cache_hist.append(cache)
		out_hist.append(out)

		# affine - relu
		#out = out.reshape(X.shape[0], self.flatten_dim)
		out, cache = affine_relu_forward(out, W2, b2)
		cache_hist.append(cache)
		out_hist.append(out)

		# affine - softmax
		out, cache = affine_forward(out, W3, b3)
		out_shift = np.exp(out - np.max(out, axis=1, keepdims=True))
		scores = out_shift / np.sum(out_shift, axis=1, keepdims=True)

		if y is None:
			return scores

		# calc loss with L2
		loss, dout = softmax_loss(out, y)
		loss = loss + 0.5 * self.reg * (np.sum(W1**2) + np.sum(W2**2)+ np.sum(W3**2))

		# calc gradiants
		grads = {}
		dout, grads['W3'], grads['b3'] = affine_backward(dout, cache)
		dout, grads['W2'], grads['b2'] = affine_relu_backward(dout, cache_hist[2])
		dout = max_pool_backward_fast(dout, cache_hist[1])
		if self.use_batchnorm:
			dout, grads['W1'], grads['b1'], grads['gamma1'], grads['beta1'] = conv_bn_relu_backward(dout, cache_hist[0])
		else:
			dout, grads['W1'], grads['b1'] = conv_relu_backward(dout, cache_hist[0])

		grads['W3'] += self.reg * W3
		grads['W2'] += self.reg * W2
		grads['W1'] += self.reg * W1

		return loss, grads

if __name__ == '__main__':
	main()


import numpy as np
from numba.pycc import CC

cc = CC('resamper')

@cc.export('compiled_spline_prefilter', '(f8[:], float64)')
def spline_prefilter(input_array: np.ndarray, tau: float) -> None:
    """Apply a recursive prefilter for spline interpolation.

    A recursive high-pass filter necessary for spline interpolation. The
    filter is applied in-place, make a copy external to this function.

    Args:
        input_array (ndarray): one-dimensional numpy array
        tau (float): constant used in the recursive filter

    Returns:
        None

    Notes:
        A numba compiled version exists "compiled_spline_prefilter"
    """
    initialization_length = min(int(-12 / np.log10(abs(tau))),
                                input_array.size)

    initial_condition = 0.0
    tau_power = 1.0

    # Initialization of recursive filter
    for ndx in range(initialization_length):
        initial_condition += input_array[ndx] * tau_power
        tau_power *= tau

    input_array[0] = initial_condition

    # Forward recursive filter
    for ndx in range(1, input_array.size-1):
        input_array[ndx] += tau * input_array[ndx-1]

    # Initialization of reverse filter
    input_array[-1] = (tau / (1 - tau * tau)
                       * (input_array[-1] +  tau * input_array[-2]))

    # Reverse recursive filter
    for ndx in range(input_array.size-2, -1, -1):
        input_array[ndx] = tau * (input_array[ndx+1] - input_array[ndx])

@cc.export('compiled_linear_prefilter', '(f8[:],)')
def linear_prefilter(input_array: np.ndarray) -> None:
    """Apply recursive filter for offset linear interpolation.

    A recursive high-pass filter necessary for shifted linear interpolation.
    The filter is applied in-place, make a copy external to this function.

    Args:
        input_array (ndarray): one-dimensional numpy array

    Returns:
        None

    Notes:
        A numba compiled version exists "compiled_linear_prefilter"
    """
    initialization_length = 18
    tau = 0.21

    initial_condition = 0.0
    tau_power = 1.0

    for ndx in range(initialization_length):
        initial_condition += input_array[ndx] * tau_power
        tau_power *= tau

    input_array[0] = initial_condition

    for ndx in range(1, input_array.size-1):
        input_array[ndx] += tau * (input_array[ndx] - input_array[ndx-1])

if __name__ == "__main__":
    cc.compile()


'''
Create a 1D/2D grid object as a connected network.
'Mask' marks any obstacles on the grid.
It is an array of the same size as the grid, where a location is True if clear and False is blocked.
'Wrap' marks whether the grid is wrapped in a direction so the sensor can move directly from one end to the other.
Internally always uses index (ind) instead of coordinate (loc).
'''

import numpy as np
from scipy.linalg import qr, svd
from collections import deque

class Grid():
    dest = False

    def bfs(self, src_ind, max_dist):
        '''Use Breadth-first search to get the shortest path from x to all nodes within a max distance.'''
        queue = deque()
        visited = [False for ind in range(self.size)]
        dist = [10**9 for ind in range(self.size)]
        visited[src_ind] = True
        dist[src_ind] = 0
        queue.append(src_ind)

        while len(queue) != 0:
            ind = queue.popleft()
            if dist[ind] >= max_dist:
                continue

            for i in range(len(self.adj[ind])):
                temp = self.adj[ind][i] # adjacent node index
                if not visited[temp]:
                    visited[temp] = True
                    dist[temp] = dist[ind] + 1
                    queue.append(temp)

        return dist

    def set_destination(self, inds, max_dist=None):
        '''Set the ending sensor positions, construct shortest path lists for sensors.
           Max distance is the time step to reach the ending position.
           If the ending position is the same at where it starts, it is a cycle.
           Max distance is half of the cycle period.
        '''
        # self.start = inds
        self.dist_to_dest = np.zeros((len(inds), self.size))
        self.dest = True

        for i in range(len(inds)):
            self.dist_to_dest[i] = self.bfs(inds[i], max_dist)

    def reset_destination(self):
        self.dist_to_dest = None
        self.dest = False

    def set_radius(self, r):
        self.adj = dict()
        for ind in range(self.size):
            self.adj[ind] = self.get_adj(ind)

    # def set_start(self, inds, max_dist=None):
    #     '''Set the ending sensor positions, construct shortest path lists for sensors.
    #        Max distance is the time step to reach the ending position.
    #        If the ending position is the same at where it starts, it is a cycle.
    #        Max distance is half of the cycle period.
    #     '''
    #     # self.start = inds
    #     self.dist_from_start = np.zeros((len(inds), self.size))
    #     self.cycle = True

    #     for i in range(len(inds)):
    #         self.dist_from_start[i] = self.bfs(inds[i], max_dist)

    def get_neighbors(self, inds, nbg_dist_to_start=None):
        if nbg_dist_to_start is None:
            nbg_dist_to_start = 1e9
        neighbors = set()
        neighbor_dict = dict()
        for i in range(len(inds)):
            if self.dest:
                ns_ind = [j for j in self.adj[inds[i]] if self.dist_to_dest[i, j] <= nbg_dist_to_start]
            else:
                ns_ind = self.adj[inds[i]]

            if len(ns_ind) == 0:
                raise ValueError("No valid neighbor is found for sensor {}.".format(i+1))

            neighbors.update(ns_ind)
            for n in ns_ind:
                if n in neighbor_dict:
                    neighbor_dict[n].update({inds[i]})
                else:
                    neighbor_dict[n] = {inds[i]}

        return list(neighbors), neighbor_dict

    def get_adj(self, ind):
        pass

class Grid1D(Grid):
    def __init__(self, length, r=None, mask=None, wrap=False):
        '''Initialize. Build an adjacency dictionary. '''
        if mask is not None:
            assert length == mask.size
        self.length = length
        self.mask = mask
        self.wrap = wrap
        self.size = length if mask is None else np.sum(mask)
        self.r = r
        if r is not None:
            self.set_radius(r)

    def ind_to_loc(self, ind):
        if self.mask is not None:
            x = np.where(self.mask==True)[0]
            return x[ind]
        else:
            return ind

    def loc_to_ind(self, loc):
        if self.mask is not None:
            return np.sum(self.mask[:loc])
        else:
            return loc

    def get_adj(self, ind):
        x = self.ind_to_loc(ind)
        x_min = x-self.r if self.wrap else max(0,x-self.r)
        x_max = x+self.r if self.wrap else min(self.length-1,x+self.r)

        neighbors = [(i % self.length) for i in range(x_min,x_max+1)]
        if self.mask is not None:
            neighbors = [self.loc_to_ind(loc) for loc in neighbors if self.mask[loc]]
        return neighbors

class Grid2D(Grid):
    '''
    Indexing the grid from the top left corner.
    For example, a grid with width 3 and height 2 is
        1,2,3,
        4,5,6.

    '''
    def __init__(self, width, height, r=None, mask=None,
                 wrap_x=False, wrap_y=False, dist_type='euclidean'):
        if mask is not None:
            assert width * height == mask.size
        self.width = width # x
        self.height = height # y
        self.mask = mask
        self.wrap_x = wrap_x
        self.wrap_y = wrap_y
        self.dist_type = dist_type
        self.size = width * height if mask is None else int(np.sum(mask))
        self.r = r
        if r is not None:
            self.set_radius(r)
        else:
            self.adj = dict()
            for ind in range(self.size):
                self.adj[ind] = list(range(self.size))

    def ind_to_loc(self, ind):
        if self.mask is not None:
            y, x = np.where(self.mask==True)
            return (x[ind], y[ind])
        else:
            return (ind % self.width, ind // self.width)

    def loc_to_ind(self, loc):
        x = loc[0] % self.width
        y = loc[1] % self.height
        val = x + y * self.width
        if self.mask is not None:
            if self.mask[y,x]:
                return int(np.sum(self.mask.flatten()[:val]))
            else:
                return None
        else:
            return val

    def dist(self, loc1, loc2):
        if self.dist_type=='euclidean':
            dx = abs(loc1[0]-loc2[0])
            if self.wrap_x:
                dx = min(dx, self.width-dx)
            dy = abs(loc1[1]-loc2[1])
            if self.wrap_y:
                dy = min(dy, self.height-dy)
            return np.sqrt(dx**2+dy**2)
        elif self.dist_type=='sphere':
            lat1 = (90-loc1[1]) * np.pi / 180
            lat2 = (90-loc2[1]) * np.pi / 180
            dx = abs(loc1[0]-loc2[0])
            dx = min(dx, self.width-dx)
            dlong = dx * np.pi / 180
            R = 6371
            d = np.arccos(np.sin(lat1) * np.sin(lat2) + np.cos(lat1) * np.cos(lat2) * np.cos(dlong)) * R
            return d

        return None

    def has_obs(self, loc1, loc2):
        if self.mask is None:
            return False

        x_range = np.sort([loc1[0], loc2[0]])
        y_range = np.sort([loc1[1], loc2[1]])
        dist_x = x_range[1]-x_range[0]
        dist_y = y_range[1]-y_range[0]
        if self.wrap_x and dist_x > (self.width-dist_x):
            x_range = np.array([x_range[1],x_range[0]+self.width])
            dist_x = x_range[1]-x_range[0]
        if self.wrap_y and dist_y > (self.height-dist_y):
            y_range = np.array([y_range[1],y_range[0]+self.height])
            dist_y = y_range[1]-y_range[0]
        if dist_x > dist_y:
            for i in range(dist_x):
                x = int(x_range[0] + i + 1)
                y = int(y_range[0] + np.round((i+1)/dist_x*dist_y))
                if not self.mask[y%self.height, x%self.width]:
                    return True
        else:
            for i in range(dist_y):
                y = int(y_range[0] + i + 1)
                x = int(x_range[0] + np.round((i+1)/dist_y*dist_x))
                if not self.mask[y%self.height, x%self.width]:
                    return True
        return False

    def get_adj(self, ind):
        origin = self.ind_to_loc(ind)
        x,y = origin
        x_min = x-self.r if self.wrap_x else max(0,x-self.r)
        x_max = x+self.r if self.wrap_x else min(self.width-1,x+self.r)
        y_min = y-self.r if self.wrap_y else max(0,y-self.r)
        y_max = y+self.r if self.wrap_y else min(self.height-1,y+self.r)

        neighbors = list()
        for i in range(x_min,x_max+1):
            for j in range(y_min,y_max+1):
                loc = (i%self.width, j%self.height)

                # within radius r
                if self.dist(loc, origin) > self.r:
                    continue

                # mask/obstacle
                if (self.mask is not None) and (not self.mask[loc[1],loc[0]]):
                    continue

                if (self.mask is not None) and (self.has_obs(origin, loc)):
                    continue

                neighbors.append(loc)

        ns_ind = [self.loc_to_ind(n) for n in neighbors]

        return ns_ind


from bs4 import BeautifulSoup
import requests
import json
import query

FROM_URL = 'http://friendorfollow.com/twitter/most-followers/'

# order tweets by activity
def sort_tweets_by_activity(tweets):
    def activityKey(tweet):
        return int(tweet.likes) + int(tweet.retweets) + int(tweet.replies)

    return sorted(tweets, key=activityKey, reverse=True)

# gets 500 most recent tweets from a user post-2010 (the text), sorted
def get_user_tweets_text(username):
    tweets = sort_tweets_by_activity(query.query_tweets_once("from:" + username, 500))
    ret = []
    for tweet in tweets:
        if len(tweet.text) > 0 and tweet.text[0] != '@':
            ret.append(tweet.text)
    for text in ret:
        list = text.split()
        for word in list:
            if len(word) > 4 and word[0:3] == "http":
                list.remove(word)
        " ".join(list)
    return ret

# gets a list of strings representing the twitter handles of the
# celebs with the 100 most visited twitter pages
def get_celeb_usernames():
    webpage = requests.get(FROM_URL).content
    soup = BeautifulSoup(webpage, 'html.parser')
    links = soup.find_all("a", target='f', class_='tUser')
    users = []
    for a in links:
        users.append(a.contents[0][1:])
    return users[:100]

# updates celebs.json, which holds the data on celebrity twitter names
def update_db(filename="celebs.json", users=get_celeb_usernames()):
    with open(filename, 'r+') as f:
        data = f.read()
        cur_json_dict = {}
        if(data != ''):
            cur_json_dict = json.loads(data)
        f.seek(0)
        for name in users:
            if name not in cur_json_dict:
                print("Fetching tweets for " + name)
                cur_json_dict[name] = get_user_tweets_text(name)
            else:
                print("Already have tweets for " + name)
        f.write(json.dumps(cur_json_dict))
        f.truncate()

# gets the link to a given user's profile picture
def get_prof_pic(user):
    profile = requests.get('http://twitter.com/' + user).content
    soup = BeautifulSoup(profile, 'html.parser')
    link = soup.find("img", class_='ProfileAvatar-image')['src']
    return link

if __name__ == '__main__':
    quoatable_users = ["colesprouse","rickygervais","bjnovak","mindykaling","VancityReynolds","chrissyteigen",
                       "AnnaKendrick47","prattprattpratt","StephenAtHome","ConanOBrien"]
    update_db("quotable.json", users=quoatable_users)


#!/usr/bin/python3
import tkinter as tk
from cv2 import VideoCapture,resize,cvtColor,COLOR_BGR2RGBA,CAP_DSHOW,CAP_PROP_FRAME_HEIGHT,CAP_PROP_FRAME_WIDTH
import threading
from requests import get,post
from base64 import b64encode
from PIL import Image, ImageTk#图像控件
from io import BytesIO
import pandas as pd
import datetime
import numpy as np
import matplotlib
from matplotlib import pyplot as plt
matplotlib.use('Agg')
import openpyxl

#excel processor
def file_save(df):
    lots_info=pd.read_excel('data\\data.xlsx',engine='openpyxl',sheet_name='lots_info')
    writer=pd.ExcelWriter('data\\data.xlsx',engine='openpyxl')
    df.to_excel(writer,sheet_name='cars_info',index=False)
    lots_info.to_excel(writer,sheet_name='lots_info',index=False)
    writer.save()

def car_check(car_id):
    if car_id==False:
        return False
    df = pd.read_excel('data\\data.xlsx',engine='openpyxl')
    #筛选status为True的car_id车辆
    t=datetime.datetime.now()
    df_in=df['status']
    df_id=df['car_id']==car_id
    df_filter=df_id & df_in
    if df_filter.sum()==0:
        if get_lots()==0:
            return -2,t.strftime("%Y-%m-%d %H:%M:%S"),car_id,-1
        add=pd.Series((car_id,True,t.strftime("%Y-%m-%d %H:%M:%S"),'','',''),index=df.columns)
        df=df.append(add,ignore_index=True)
        file_save(df)
        counter()
        return -1,t.strftime("%Y-%m-%d %H:%M:%S"),car_id,-1
    else:
        df.loc[df_filter,'status']=False
        df.loc[df_filter,'out_time']=t.strftime("%Y-%m-%d %H:%M:%S")
        t_in=str(df.loc[df_filter,'in_time'].values[0])
        try:
            t_in=datetime.datetime.strptime(t_in,"%Y-%m-%d %H:%M:%S")
        except:
            t_in=datetime.datetime.strptime(t_in,"%Y-%m-%dT%H:%M:%S.000000000")
        delta=t-t_in
        span=(delta.seconds//3600,(delta.seconds-delta.seconds//3600*3600)//60)
        df.loc[df_filter,'span']=str(span)
        global config
        fee=delta.seconds//1800*config['priceper30m']+config['priceper30m']#时间提取 费用计算
        df.loc[df_filter,'fee']=fee
        file_save(df)
        return fee,t_in.strftime("%Y-%m-%d %H:%M:%S"),car_id,span[0],span[1]

def car_sort():
    df = pd.read_excel('data\\data.xlsx',engine='openpyxl')
    df.sort_values(by=['status','in_time','out_time'],inplace=True,ascending=False)
    file_save(df)

def get_longest():
    t=datetime.datetime.now()
    df = pd.read_excel('data\\data.xlsx',engine='openpyxl')
    df.sort_values(by='in_time',inplace=True,ascending=False)
    df_in=df['status']
    t_in=df.loc[df_in,['car_id','in_time']]
    t_in.sort_values(by='in_time',inplace=True)
    try:
        car_id=t_in.iloc[[0],[0]].values[0][0]
        t_in=str(t_in.iloc[[0],[1]].values[0][0])
        try:
            t_in=datetime.datetime.strptime(t_in,"%Y-%m-%d %H:%M:%S")
        except:
            t_in=datetime.datetime.strptime(t_in,"%Y-%m-%dT%H:%M:%S.000000000")
        delta=t-t_in
        span=(delta.days*24+delta.seconds//3600,(delta.seconds-delta.seconds//3600*3600)//60)
        return car_id,span[0],span[1]
    except:
        return -1
    #df.to_excel('data\\data.xlsx',engine='openpyxl',index=False)

def get_lots():
    df = pd.read_excel('data\\data.xlsx',engine='openpyxl')
    lots_in=df['status'].sum()
    return 100-lots_in

def get_parkers():
    df = pd.read_excel('data\\data.xlsx',engine='openpyxl')
    df.sort_values(by='in_time',inplace=True,ascending=False)
    df_in=df['status']
    parkers=df.loc[df_in,['car_id','in_time']].values
    lenth=len(parkers)
    for i in range(lenth):
        parkers[i][1]=str(parkers[i][1])
    return parkers

def file_create():
    try:
        df=pd.read_excel('data\\data.xlsx',engine='openpyxl')
    except:
        writer=pd.ExcelWriter('data\\data.xlsx',engine='openpyxl')
        df=pd.DataFrame(columns=['car_id','status','in_time','out_time','fee','span'])
        df2=pd.DataFrame(columns=['date','weekday','total','max'])
        df.to_excel(writer,sheet_name='cars_info',index=False)
        df2.to_excel(writer,sheet_name='lots_info',index=False)
        writer.save()

def counter():
    lots_in=100-get_lots()
    df = pd.read_excel('data\\data.xlsx',engine='openpyxl')
    lots_info=pd.read_excel('data\\data.xlsx',engine='openpyxl',sheet_name='lots_info')
    date=datetime.datetime.now()
    lots_filter=lots_info['date']==str(date.date())
    if lots_filter.sum()==0:
        add=pd.Series((str(date.date()),date.weekday()+1,1,lots_in),index=lots_info.columns)
        lots_info=lots_info.append(add,ignore_index=True)
    else:
        lots_max=lots_info.loc[lots_filter,'max'].values[0]
        lots_info.loc[lots_filter,'total']+=1
        if lots_max<lots_in:
            lots_info.loc[lots_filter,'max']=lots_in
        pass
    writer=pd.ExcelWriter('data\\data.xlsx',engine='openpyxl')
    df.to_excel(writer,sheet_name='cars_info',index=False)
    lots_info.to_excel(writer,sheet_name='lots_info',index=False)
    writer.save()
    pass

def lots_warning():
    lots_info=pd.read_excel('data\\data.xlsx',engine='openpyxl',sheet_name='lots_info')
    date=datetime.datetime.now().date()
    date+=datetime.timedelta(1)
    date_list=[]
    delta=datetime.timedelta(-7)
    global config
    span=config['warn_source_span']
    if span==0:
        return False
    for i in range(span):
        date=date+delta
        date_list.append(str(date))
    date_filter=lots_info['date']==date_list[0]
    for j in range(span):
        date_filter|=lots_info['date']==date_list[j]
    num=date_filter.sum()
    if num==0:
        return False
    total_expect=lots_info.loc[date_filter,'total'].sum()/num
    max_expect=lots_info.loc[date_filter,'max'].sum()/num
    if total_expect>config['warn_total'] or max_expect>config['warn_max']:
        return True
    else:
        return False

def draw_chart():
    df = pd.read_excel('data\\data.xlsx',engine='openpyxl')
    df.sort_values(by='in_time',inplace=True,ascending=False)
    df_in=df['status']==False
    parkers=df.loc[df_in,['fee','in_time']].values
    lenth=len(parkers)
    for i in range(lenth):
        parkers[i][1]=str(parkers[i][1])
        parkers[i][1]=datetime.datetime.strptime(parkers[i][1],"%Y-%m-%d %H:%M:%S").month
    profit=[0,0,0,0,0,0,0,0,0,0,0,0]
    months=['一月','\n二月','三月','\n四月','五月','\n六月','七月','\n八月','九月','\n十月','十一月','\n十二月']
    for j in range(lenth):
        profit[parkers[j][1]-1]+=parkers[j][0]
    #指定默认字体
    matplotlib.rcParams['font.sans-serif'] = ['SimHei']
    matplotlib.rcParams['font.family']='sans-serif'
    #解决负号’-‘显示为方块的问题
    matplotlib.rcParams['axes.unicode_minus'] = False
    fig, ax = plt.subplots(figsize=(4,5))
    ax.bar(x=months,height=profit,width=0.9)
    ax.set_title("月销售量统计图",fontsize=15)
    xticks = ax.get_xticks()
    m=max(profit)
    for i in range(len(profit)):
        xy = (xticks[i], profit[i]+m*1.01-m)
        s = str(profit[i])
        ax.annotate(
            text=s,  # 要添加的文本
            xy=xy,  # 将文本添加到哪个位置
            fontsize=8,  # 标签大小
            color="black",  # 标签颜色
            ha="center",  # 水平对齐
            va="baseline"  # 垂直对齐
        )
    buffer = BytesIO()
    fig.canvas.print_png(buffer)
    return buffer

def config_get():
    def raise_error(err_str):
        top = tk.Tk()
        top.geometry('0x0+999999+0')
        tk.messagebox.showerror(title='错误', message=err_str)
        top.destroy()
        quit()
    def cam_test(cam_id):
        try:
            cap = VideoCapture(cam_id,CAP_DSHOW)
        except:
            cap = VideoCapture(0,CAP_DSHOW)
        ret,img = cap.read()
        if type(img)!=np.ndarray:
            return False
        else:
            return True
    try:#文件是否存在
        f=open('data\\config.txt')
    except:
        raise_error('丢失config.txt或无权读取')
    try:#文件格式是否正确
        config=eval(f.read())
    except:
        f.close()
        raise_error('config.txt格式不正确')
    f.close()
    if type(config==dict):#文件格式是否正确
        dict_check=[['warn_source_span',2,int],
                    ['warn_max',80,int,float],
                    ['warn_total',200,int,float],
                    ['priceper30m',1.5,float,int]]
        for i in dict_check:#检查除camid外的数值项，若非法直接替换为默认值
            try:
                if config[i[0]]<0 or (type(config[i[0]]) not in i):
                    config[i[0]]=i[1]
            except:
                config[i[0]]=i[1]
        try:#检查camid
            temp=config['camera_id']
        except:
            config['camera_id']==0
        if not cam_test(config['camera_id']):
            raise_error('您的设备没有摄像头\n或者您输入了错误的camera_id')
        try:#检查key
            host = 'https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id='+config['API_KEY']+'&client_secret='+config['SECRET_KEY']
            response = get(host)
            access_token = False
            if response:
                access_token=eval(response.text)
                access_token=access_token['access_token']
            if access_token==False:
                trash=1/0
        except:
            raise_error('无效的API_KEY或SECRET_KEY')
    else:
        raise_error('config.txt格式不正确')
    return config,access_token#返回config及token

#config get
config,access_token=config_get()
#open/create files
file_create()

def ocr(img):
    request_url = "https://aip.baidubce.com/rest/2.0/ocr/v1/license_plate"
    # 二进制方式打开图片文件
    img = b64encode(img)
    params = {"image":img}
    request_url = request_url + "?access_token=" + access_token
    headers = {'content-type': 'application/x-www-form-urlencoded'}
    response0 = post(request_url, data=params, headers=headers)
    car_id=False
    try:
        if response0:
            car_id=eval(response0.text)
            car_id=car_id['words_result']['number']
    except:
        car_id=False
    return car_id

def cam():
    global config
    try:
        cap = VideoCapture(config['camera_id'],CAP_DSHOW)
    except:
        cap = VideoCapture(0,CAP_DSHOW)
    cap.set(CAP_PROP_FRAME_WIDTH,3000)
    cap.set(CAP_PROP_FRAME_HEIGHT,3000)
    ret,img = cap.read()
    py=img.shape[0]
    px=img.shape[1]
    if px/700 > py/500:
        fy=fx=500/py
        cutterx=(px*fx-700)/2
        cuttery=0
    else:
        fy=fx=700/px
        cuttery=(py*fy-500)/2
        cutterx=0
    dsize=(int(px*fx),int(py*fy))
    global CAP_FLAG
    CAP_FLAG=CAP_FLAG
    notend=True
    while notend:
        ret, img = cap.read()#翻转 0:上下颠倒 大于0水平颠倒
        img = resize(img,dsize)
        img = cvtColor(img, COLOR_BGR2RGBA)
        img = Image.fromarray(img)
        img = img.crop((cutterx,cuttery,cutterx+700,cuttery+500))
        if CAP_FLAG:
            tr_cp=threading.Thread(target=capture,args=(img,))
            tr_cp.start()
            CAP_FLAG=False
        img=ImageTk.PhotoImage(img)
        id=camera.create_image(0,0,anchor='nw',image=img)
        camera.delete(id-1)
        camera.image=img
        global NOT_END
        notend=NOT_END
        #no_flash=img
    NOT_END=-1
    cap.release()

def capture(img):
    cap.config(state=tk.DISABLED)
    bytesIO = BytesIO()
    img.save(bytesIO, format='png')
    img=bytesIO.getvalue()
    car_id=ocr(img)##input('input car_id:')##
    renew_msg(car_check(car_id))
    renew_lots()
    renew_msg0()
    renew_parkers()
    car_sort()
    global WIN_FLAG
    if WIN_FLAG==1:
        WIN_FLAG=0
        show_profit()
    cap.config(state=tk.NORMAL)
def cap_():
    global CAP_FLAG
    CAP_FLAG=True

def show_profit():
    profit_info.config(state=tk.DISABLED)
    global WIN_FLAG
    if WIN_FLAG==0:
        win.geometry('1400x502')
        src=draw_chart()
        img=Image.open(src)
        img=img.resize((400, 500))
        #img.save('test.png')
        img=ImageTk.PhotoImage(image=img)
        profit_canvas.create_image(0,0,anchor='nw',image=img)
        profit_canvas.image=img
    else:
        win.geometry('1000x502')
        profit_canvas.delete('all')
    WIN_FLAG=1-WIN_FLAG
    profit_info.config(state=tk.NORMAL)

def renew_lots():
    for widget in lots_frame.winfo_children():
        widget.destroy()
    all_lots=100
    lots=get_lots()
    lots_str='共有车位：{}，现存车位：{}'
    remain_lots=tk.Label(lots_frame,text=lots_str.format(all_lots,lots),font=('微软雅黑',14),bg='#6699ff')
    remain_lots.pack()

def renew_msg0():
    for widget in msg_frame0.winfo_children():
        widget.destroy()
    temp=get_longest()
    if temp!=-1:
        msg0_str='停车时间最长车辆：{}\n已停车时长：{}小时{}分'
        msg0=tk.Label(msg_frame0,text=msg0_str.format(temp[0],temp[1],temp[2]),font=('微软雅黑',10),bg='#6699ff')
    else:
        msg0=tk.Label(msg_frame0,text='停车场暂无车辆',font=('微软雅黑',10),bg='#6699ff')
    msg0.pack()
def auto_renew_msg0():
    renew_msg0()
    win.after(30000,auto_renew_msg0)

def renew_msg(tuple):
    for widget in msg_frame.winfo_children():
        widget.destroy()
    if tuple==False:
        msg_str='未识别到有效车牌信息！'
        msg=tk.Label(msg_frame,text=msg_str,font=('微软雅黑',10),bg='#6699ff')
    else:
        msg_str='车牌号：{}\n'
        if tuple[0]==-2:
            msg_str+='本停车场已无空余车位！'
            msg=tk.Label(msg_frame,text=msg_str.format(tuple[2]),font=('微软雅黑',10),bg='#6699ff')
        elif tuple[0]==-1:
            msg_str+='有空余车位，可以进场停车\n进入停车场时间：{}'
            msg=tk.Label(msg_frame,text=msg_str.format(tuple[2],tuple[1]),font=('微软雅黑',10),bg='#6699ff')
        else:
            msg_str+='停车时长：{}小时{}分钟\n停车费：{}元\n离开停车场时间：{}\n'
            msg=tk.Label(msg_frame,text=msg_str.format(tuple[2],tuple[3],tuple[4],tuple[0],tuple[1]),font=('微软雅黑',10),bg='#6699ff')
    msg.pack()

def renew_parkers():
    parkers_body.delete('all')
    #获取车辆信息
    id=0
    for i in get_parkers():
        parkers_body.create_text(10, id*30+2, anchor=tk.NW, text=i[0],font=('微软雅黑',10))#window=info1)
        parkers_body.create_text(115, id*30+2, anchor=tk.NW, text=i[1],font=('微软雅黑',10))#window=info2)
        id+=1
    parkers_body.config(scrollregion=(0, 0, 280, 30*id+2))
    pass

def scroll_event(event):
    number = int(-event.delta / 120)
    parkers_body.yview_scroll(number, 'units')

def closeWindow():
    global NOT_END
    NOT_END=False
    def run():
        while NOT_END!=-1:
            pass
        win.quit()
    ex=threading.Thread(target=run)
    ex.start()

#window init
win = tk.Tk()
win.title('Smart Parking System')
win.geometry('1000x502')
win.resizable(0,0)
win.protocol('WM_DELETE_WINDOW', closeWindow)
#global flag
WIN_FLAG=0
NOT_END=True
CAP_FLAG=False
#camara panel init
camera_frame=tk.Frame(win,width=700,height=500,bd=-1)
camera=tk.Canvas(camera_frame,bg='white',width=700,height=500)
cap=tk.Button(camera_frame,text='捕获车牌号',command=cap_)
camera_frame.pack_propagate(0)
camera_frame.place(x=0,y=0,anchor='nw')
camera.pack(side='top')
cap.place(x=700,y=500,anchor='se')
if lots_warning():
    lots_warning=tk.Label(camera_frame,bg='white',text='明日车位情况可能比较紧张，请提前做好调度准备！',font=('微软雅黑',18))
    lots_warning.place(x=20,y=20,anchor=tk.NW)
#camera thread
tr_cam=threading.Thread(target=cam)
tr_cam.start()
#Info panel init
info_frame=tk.Frame(win,width=300,height=500,bg='#6699ff',bd=0)
info_frame.pack_propagate(0)
info_frame.place(x=1000,y=0,anchor='ne')
lots_frame=tk.Frame(info_frame,width=280,height=50,bg='#6699ff')
lots_frame.pack_propagate(0)
lots_frame.pack()
parkers_frame=tk.Frame(info_frame,width=280,height=250,bg='#6699ff')
parkers_frame.pack_propagate(0)
parkers_frame.pack()
parkers_head=tk.Frame(parkers_frame,width=280,height=30,bg='#6699ff')
parkers_head.grid_propagate(0)
parkers_head.pack()
header1=tk.Label(parkers_head,text='车牌号',font=('微软雅黑',13,'bold'),padx=20,bg='#6699ff')
header2=tk.Label(parkers_head,text='停入时间',font=('微软雅黑',13,'bold'),padx=54,bg='#6699ff')
header1.grid(row=0,column=0)
header2.grid(row=0,column=1)
parkers_scroll=tk.Scrollbar(parkers_frame,width=16,bg='#6699ff',orient='vertical',bd=0)
parkers_scroll.grid_propagate(0)
parkers_scroll.pack(side='right',fill=tk.Y)
parkers_body=tk.Canvas(parkers_frame,bg='#6699ff',bd=0)
parkers_body.pack(expand=True,fill='both')
parkers_body.config(yscrollcommand=parkers_scroll.set)
parkers_body.bind("<MouseWheel>", scroll_event)
parkers_scroll.config(command=parkers_body.yview)
msg_frame0=tk.Frame(info_frame,width=290,height=50,bg='#6699ff')
msg_frame0.pack_propagate(0)
msg_frame0.pack()
msg_frame=tk.Frame(info_frame,width=280,height=130,bg='#6699ff')
msg_frame.pack_propagate(0)
msg_frame.pack()
profit_info=tk.Button(info_frame,text='收入统计',command=show_profit)
profit_info.place(x=298,y=500,anchor='se')
profit_canvas=tk.Canvas(win,width=400,height=500,bg='white')
profit_canvas.place(x=1401,y=0,anchor='ne')
#info init
car_sort()
renew_lots()
auto_renew_msg0()
renew_parkers()
#main loop start
win.mainloop()


# ******************************************************************************************
# Author: Luis Ernesto Colchado Soncco
# Email: luis.colchado@ucsp.edu.pe / luisernesto.200892@gmail.com
# Description: Match meteorological condition, satellite products and pollution data for SAO PAULO
# ******************************************************************************************
import os, sys
currentdir = os.path.dirname(os.path.realpath(__file__))
parentdir = os.path.dirname(currentdir)
sys.path.append(parentdir)

import numpy as np
import pandas as pd
from datetime import timedelta
from tqdm import tqdm
import json
from point import Point

class setting:
    DIR_SP = '../Data/SaoPaulo/'
    DIR_CETESB = '../Data/SaoPaulo/CETESB'
    DIR_POINTS = f'{DIR_CETESB}/points.csv'
    start_date = pd.to_datetime('01-01-2017')
    end_date = pd.to_datetime('12-31-2019')
    NDVI_DIR = f'{DIR_SP}/Satellite_data/MOD13A2/DATA_CROPPED'
    VNP_DIR = f'{DIR_SP}/Satellite_data/VNP46A1/DATA_CROPPED'
    DEM_DIR = f'{DIR_SP}/Satellite_data/DEM/DATA_CROPPED'
    #selected_columns = ['lat', 'lon', 'temp', 'pres', 'dewp', 'wd', 'ws', 'ndvi', 'ntl', 'dem', 'pm25']

def get_data(DATA_DIR):
    data = {}
    with open(DATA_DIR + '/info.txt', 'r') as f:
        nrows, ncolumns = f.readline().split('x')
    print('getting data from {}'.format(DATA_DIR))
    with tqdm(total=len(os.listdir(DATA_DIR))) as bar:
        for file in os.listdir(DATA_DIR):
            if file.endswith('.dat'):
                year_days = file.split('_')[3][1:-4]
                year = int(year_days[:4])
                days = int(year_days[4:])
                current_date = pd.to_datetime(str(year - 1) + '-12-31')
                current_date += timedelta(days=days)
                array = np.fromfile(DATA_DIR + '/' + file, dtype=float)
                array = array.reshape(int(len(array) / 3), 3)
                #print(str(current_date)[:-9])
                data.update({str(current_date)[:-9]: array})
                bar.update(1)
    return data

def match_data(points, df_pm25, df_temp, df_hum, df_press, df_wd, data_ndvi, data_ntl, array_dem):
    current_date = setting.start_date
    data = pd.DataFrame()
    count_error = 0
    while current_date <= setting.end_date:
        string_date = str(current_date)[:10]
        current_temp = df_temp[df_temp['date']==string_date]
        current_hum = df_hum[df_hum['date']==string_date]
        current_press = df_press[df_press['date']==string_date]
        current_wd = df_wd[df_wd['date']==string_date]

        current_pm25 = df_pm25[df_pm25['date']==string_date]
        if string_date in data_ndvi:
            current_ndvi = data_ndvi[string_date]
            #print(current_ndvi)
            prev_ndvi = current_ndvi
            print(string_date)
        else:
            current_ndvi = prev_ndvi

        if string_date in data_ntl:
            current_ntl = data_ntl[string_date]
            prev_ntl = current_ntl
        else:
            count_error += 1
            current_ntl = prev_ntl

        with tqdm(total=len(points)) as bar:
            bar.set_description(string_date)
            for p in points.index:
                current_station = points.loc[p, 'station']
                lat, lon = points.loc[p, 'lat'], points.loc[p, 'long']
                lat_left_top, lon_left_top = Point(points.loc[p, 'left'],
                                                    points.loc[p, 'top']).convertToCoord()
                lat_right_bottom, lon_right_bottom = Point(points.loc[p, 'right'],
                                                    points.loc[p, 'bottom']).convertToCoord()
                lat_left_top, lon_left_top = round(lat_left_top, 6), round(lon_left_top, 6)
                lat_right_bottom, lon_right_bottom = round(lat_right_bottom,6 ), round(lon_right_bottom, 6)
                #!print(lat_left_top, lon_left_top, ' - ', lat_right_bottom, lon_right_bottom)
                point_knn = json.loads(points.loc[p, 'knn'])
                stations = [k for k in point_knn.keys()]
                dist = [k for k in point_knn.values()]

                s = 0
                temp = hum = press = wd = None
                while s < len(stations) and (temp is None or hum is None or press is None or wd is None):
                    station = stations[s]
                    station_temp = current_temp[current_temp['stationname']==station].reset_index(drop=True)
                    station_hum = current_hum[current_hum['stationname']==station].reset_index(drop=True)
                    station_press = current_press[current_press['stationname']==station].reset_index(drop=True)
                    station_wd = current_wd[current_wd['stationname']==station].reset_index(drop=True)

                    if not station_temp.empty and temp is None:
                        temp = station_temp.loc[0, 'conc']
                    if not station_hum.empty and hum is None:
                        hum = station_hum.loc[0, 'conc']
                    if not station_press.empty and press is None:
                        #print(station_press)
                        press = station_press.loc[0, 'conc']
                    if not station_wd.empty and wd is None:
                        wd = station_wd.loc[0, 'conc']
                    s += 1

                d_ndvi = current_ndvi[(current_ndvi[:, 0] <= lat_left_top) & (current_ndvi[:, 1] >= lon_left_top) &
                                    (current_ndvi[:, 0] >= lat_right_bottom) & (current_ndvi[:, 1] <= lon_right_bottom)][:, 2]
                d_ntl = current_ntl[(current_ntl[:, 0] <= lat_left_top) & (current_ntl[:, 1] >= lon_left_top) &
                                    (current_ntl[:, 0] >= lat_right_bottom) & (current_ntl[:, 1] <= lon_right_bottom)][:, 2]

                d_dem = array_dem[(array_dem[:, 0] <= lat_left_top) & (array_dem[:, 1] >= lon_left_top) &
                                (array_dem[:, 0] >= lat_right_bottom) & (array_dem[:, 1] <= lon_right_bottom)][:, 2]

                if len(d_ndvi) > 0:
                    mean_ndvi = d_ndvi.mean()
                    mean_ntl = d_ntl.mean()
                else:
                    mean_ndvi = np.nan
                    mean_ntl = np.nan
                mean_dem = d_dem.mean()

                row = {'station': current_station, 'current_date': string_date,'lat': lat, 'lon': lon, 'temp': temp, 'hum': hum, 'press': press, 'wd': wd, 'ndvi': mean_ndvi,
                        'ntl': mean_ntl, 'dem': mean_dem}

                k = 0
                for station in stations:
                    current_neighbors = current_pm25[current_pm25['stationname']==station].reset_index(drop=True)
                    if not current_neighbors.empty:
                        value_pm25 = current_neighbors.loc[0, 'conc']
                        key_pm25 = f'pm25_{k}'
                        key_dist = f'dist_{k}'

                        value_dist = dist[k]
                        row.update({key_pm25: value_pm25})
                        row.update({key_dist: value_dist})
                        k += 1

                pm25_ = None
                if str(current_station) != 'nan':
                    curr_data = current_pm25[current_pm25['stationname']==current_station].reset_index(drop=True)
                    pm25_ = curr_data.loc[0, 'conc']
                row.update({'PM25': pm25_})
                data = data.append(row, ignore_index=True)
                bar.update(1)

            # get temperature and humidity

        current_date += timedelta(days=1)
    return data

if __name__ == '__main__':
    pm25 = pd.read_csv(setting.DIR_CETESB+'/data_pm25.csv')
    temperature = pd.read_csv(setting.DIR_CETESB+'/data_temperature.csv')
    humidity = pd.read_csv(setting.DIR_CETESB+'/data_humidity.csv')
    pressure = pd.read_csv(setting.DIR_CETESB+'/data_pressure.csv')
    wind_direction = pd.read_csv(setting.DIR_CETESB+'/data_wind_direction.csv')
    points = pd.read_csv(setting.DIR_POINTS)

    print(f'Loading data from {setting.DEM_DIR} ...')
    dem_data = np.zeros(shape=(0, 3))
    for file in os.listdir(setting.DEM_DIR):
        if file.endswith('.dat'):
            dem_array = np.fromfile(f'{setting.DEM_DIR}/{file}', dtype=float)
            dem_array = dem_array.reshape(int(len(dem_array) / 3), 3)
            dem_data = np.concatenate([dem_data, dem_array], axis=0)
    print('Loaded!')

    data_ndvi = get_data(setting.NDVI_DIR)

    data_ntl = get_data(setting.VNP_DIR)

    data_train = match_data(points, pm25, temperature, humidity, pressure, wind_direction, data_ndvi, data_ntl, dem_array)

    data_train.to_csv('Results/data_train_sp_.csv')


import torch
import torch.nn.functional as F
from torch import nn

from modules.blocks import Hourglass, AntiAliasInterpolation2d

class MRAARegionPredictor(nn.Module):
    """
    Region predictor from MRAA, which is used for mask prediction.
    """

    def __init__(self,
                 num_regions=10,
                 block_expansion=32,
                 max_features=1024,
                 num_blocks=5,
                 temperature=0.1,
                 scale_factor=0.25,
                 num_channels=3,
                 multipart=False,
                 threshold=0.0001):
        """
        :param num_regions: number of regions (should correspond to number of parts)
        :param block_expansion: size of the first conv block
        :param max_features: maximum number of features
        :param num_blocks: number of blocks in the network
        :param temperature: softmax temperature for regions
        :param num_channels: number of channels in the frame
        :param multipart: return mask with multiple parts or single part
        :param threshold: threshold for quantize soft mask to hard mask
        :param scale_factor: resize the image by this factor, before input
        """
        super(MRAARegionPredictor, self).__init__()
        self.predictor = Hourglass(block_expansion=block_expansion,
                                   in_features=num_channels,
                                   max_features=max_features,
                                   num_blocks=num_blocks)

        self.regions = nn.Conv2d(in_channels=self.predictor.out_filters,
                                 out_channels=num_regions,
                                 kernel_size=(7, 7),
                                 padding=3)

        self.temperature = temperature
        self.scale_factor = scale_factor
        self.multipart = multipart
        self.threshold = threshold

        if self.scale_factor != 1:
            self.down = AntiAliasInterpolation2d(num_channels, self.scale_factor)

        for param in self.parameters():
            param.requires_grad = False

    def load_state_dict(self, state_dict, strict=False):
        """
        Load state dict, but rename properly and don't load AntiAliasInterpolation2d parameters.
        """
        with torch.no_grad():
            self_state = self.state_dict()
            for name, param in state_dict.items():
                if name.startswith('down'):
                    continue
                name = name.replace('encoder.', '').replace('decoder.', '')
                if isinstance(param, nn.Parameter):
                    param = param.data
                self_state[name].copy_(param)

    def convert_regions_to_segmentation_mask(self, regions):
        """
        Convert soft regions to hard segmentation mask with background.

        :param regions: soft segmentation mask[bs, n_parts, h, w]
        :return: hard segmentation mask[bs, n_parts, h, w]
        """
        hard_mask, max_index = regions.max(dim=1, keepdim=True)
        hard_mask = hard_mask > self.threshold
        hard_mask = hard_mask.float()

        if self.multipart:
            num_classes = regions.shape[1]
            regions = torch.argmax(regions, dim=1)
            regions = F.one_hot(regions, num_classes=num_classes)
            regions = torch.permute(regions, (0, 3, 1, 2))
            result_mask = torch.cat([hard_mask * regions, (1 - hard_mask)], dim=1)
        else:
            result_mask = torch.cat([hard_mask, (1 - hard_mask)], dim=1)

        class_weights = 1 / torch.clamp(result_mask.mean(axis=(0, 2, 3)), min=1e-3)
        return result_mask, class_weights

    def forward(self, x):
        if self.scale_factor != 1:
            x = self.down(x)

        feature_map = self.predictor(x)
        prediction = self.regions(feature_map)

        final_shape = prediction.shape
        region = prediction.view(final_shape[0], final_shape[1], -1)
        region = F.softmax(region / self.temperature, dim=2)
        region = region.view(*final_shape)

        return self.convert_regions_to_segmentation_mask(region)


from __future__ import unicode_literals

from django.db import migrations

from django_changeset.models.mixins import RevisionModelMixin

from eric.core.models import disable_permission_checks

def create_maintenance_text(apps, schema_editor):
    RevisionModelMixin.set_enabled(False)
    db_alias = schema_editor.connection.alias
    Content = apps.get_model("cms", "Content")

    with disable_permission_checks(Content):
        # first check if an entry exists
        maintenance_text = Content.objects.using(db_alias).filter(
            slug="maintenance"
        ).first()

        # if no entry exists, we create an "empty" one
        if not maintenance_text:
            Content.objects.using(db_alias).create(
                slug="maintenance",
                title="Maintenance"
            )
    RevisionModelMixin.set_enabled(True)

def remove_maintenance_text(apps, schema_editor):
    db_alias = schema_editor.connection.alias
    Content = apps.get_model("cms", "Content")

    with disable_permission_checks(Content):
        # first check if an entry exists
        maintenance_text = Content.objects.using(db_alias).filter(
            slug="maintenance"
        ).first()

        # if an entry exists, we delete it
        if maintenance_text:
            maintenance_text.delete()

class Migration(migrations.Migration):

    dependencies = [
        ('cms', '0001_initial'),
    ]

    operations = [
        migrations.RunPython(create_maintenance_text, remove_maintenance_text),
    ]


import unittest
from src.controller import WebScrapingController
from src.model import WebScraper
import requests

class UnitTest(unittest.TestCase):
    def test_url_string(self):
        self._model = WebScraper()
        self.expected = True
        self.assertEqual(self._model.is_valid("http://www.google.com"),
                         self.expected)
        self.assertEqual(self._model.is_valid("https://www.google.com"),
                         self.expected)
        self.assertEqual(self._model.is_valid("https://google.com"),
                         self.expected)
        self.assertEqual(self._model.is_valid("http://WWW.GoOgLe.cOm"),
                         self.expected)
        self.assertEqual(self._model.is_valid("http://www.google.com/maps"),
                         self.expected)
        self.assertEqual(self._model.is_valid("http://maps.google.com"),
                         self.expected)
        self.assertEqual(self._model.is_valid("http://www.google.domain.com"),
                         self.expected)
        self.expected = False
        self.assertEqual(self._model.is_valid("www.google.com"),
                         self.expected)
        self.assertEqual(self._model.is_valid("google.com"),
                         self.expected)
        self.assertEqual(self._model.is_valid("//www.google.com"),
                         self.expected)
        self.assertEqual(self._model.is_valid("google.http://www.com"),
                         self.expected)
        self.assertEqual(self._model.is_valid("www.http://.com"),
                         self.expected)
        self.assertEqual(self._model.is_valid("http:\\\\www.google.com"),
                         self.expected)
        self.assertEqual(self._model.is_valid("htps://www.google.com"),
                         self.expected)

    def test_url_whitespace(self):
        self._model = WebScraper()
        self.expected = "http://www.google.com"
        self.assertEqual(self.expected, self._model.is_correct(
            "http://www.g o o g l e.com"))
        self.assertEqual(self.expected, self._model.is_correct(
            "http : / / www.google . com"))

    def test_url_missing_scheme(self):
        self._model = WebScraper()
        self.expected = "http://www.google.com"
        self.actual = self._model.is_correct("www.google.com")
        self.assertEqual(self.expected,
                         self.actual)

    def test_url_https_scheme(self):
        self._model = WebScraper()
        self.expected = "https://www.google.com"
        self.actual = self._model.is_correct("https://www.google.com")
        self.assertEqual(self.expected,
                         self.actual)

    def test_fetch_data_from_file(self):
        self._model = WebScraper()
        self.expected = ['
One
']
        self.actual = self._model.fetch(maximum=1, path='data.txt')

        self.assertEqual(self.expected,
                         self.actual)

    # Requires internet connection
    def test_url_fetch_data(self):
        self._model = WebScraper()
        self.expected = 1
        self.actual = len(self._model.fetch(maximum=1))
        self.assertEqual(self.expected,
                         self.actual)

    def test_url_found(self):
        self._model = WebScraper()
        self.expected = True
        self.actual = self._model.is_connected("http://www.google.com")
        self.assertEqual(self.expected,
                         self.actual)

    def test_url_not_found_1(self):
        self._model = WebScraper()
        self.expected = False
        self.actual = self._model.is_connected("http:\\qaz.wsx.edc.rfv")
        self.assertEqual(self.expected,
                         self.actual)

    def test_url_not_found_2(self):
        self._model = WebScraper()
        self.assertRaises(requests.ConnectionError,
                          self._model.is_connected, "http://qaz.wsx.edc.rfv")

    def test_pickle(self):
        self.controller = WebScrapingController(WebScraper())
        self.controller.get_data()
        self.controller.save_pickle_data()
        self.controller._container = {}
        self.controller.load_pickle_data()
        self.expected = 10
        self.assertEqual(self.expected,
                         len(self.controller.get_container(0)))

if __name__ == '__main__':
    unittest.main()


from itertools import permutations

# users_input = 'aasdfg'
# for i in list(map("".join, permutations(users_input, 3))):
#     print(i)

# for item in permutations('aasdfg', 3):
#     print(''.join(item))

# def names_combinations(users_input, words_amount):
#     if words_amount <= len(users_input):
#         for name in list(map("".join, permutations(users_input, words_amount))):
#             print(name.capitalize())
#     else:
#         print(f'The number of letters should be no more than {len(users_input)}')
#
#
# names_combinations('Alexander', 5)

# Create a function to sort a word with one given length.
def letters_permutation(users_input, words_amount):
    names_variants = sorted(list(set([''.join(name_variant).capitalize()
                                      for name_variant
                                      in permutations(users_input, words_amount)])))
    return names_variants

# Create a function to sort words with a range of preset lengths.
def letters_permutation_range(users_input, min_amount, max_amount):
    names_variants = [letters_permutation(users_input, words_amount)
                      for words_amount in range(min_amount, max_amount + 1)]
    return names_variants

print(letters_permutation('bfFf', 4))
print(letters_permutation_range('axXx', 3, 4))


# ISLR Ch 9 by Carol Cui

import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import seaborn as sns
from mlxtend.plotting import plot_decision_regions
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.decomposition import PCA
from sklearn.metrics import confusion_matrix

# ----------------------------------------------------------------------------
# Q4
np.random.seed(1)
n = 100
p = 2
y = np.concatenate((np.repeat(1,30), np.repeat(1,30), np.repeat(2,40)))
x = np.random.rand(n,p)
x[1:30,:]  = x[1:30,:]  + 2
x[31:60,:] = x[31:60,:] - 2
plt.scatter(x[:,0], x[:,1], c=y, cmap=mpl.cm.Paired)
plt.xlabel('x1')
plt.ylabel('x2')

xtrain, xtest, ytrain, ytest = train_test_split(x, y, train_size=0.5, random_state=2)

# SVC
svc = SVC(C=10, kernel='linear')
svc.fit(xtrain, ytrain)
print(confusion_matrix(ytrain, svc.predict(xtrain))) # training error = 8/50 = 16%
plot_decision_regions(X=xtrain, y=ytrain, clf=svc, legend=2)
plt.xlabel('x1')
plt.ylabel('x2')

# SVM poly
svmp = SVC(C=10, kernel='poly', degree=4)
svmp.fit(xtrain, ytrain)
print(confusion_matrix(ytrain, svmp.predict(xtrain))) # training error = 0%
plot_decision_regions(X=xtrain, y=ytrain, clf=svmp, legend=2)
plt.xlabel('x1')
plt.ylabel('x2')

# SVM radial
svmr = SVC(C=10, kernel='rbf', gamma=1.0)
svmr.fit(xtrain, ytrain)
print(confusion_matrix(ytrain, svmr.predict(xtrain))) # training error = 0%
plot_decision_regions(X=xtrain, y=ytrain, clf=svmr, legend=2)
plt.xlabel('x1')
plt.ylabel('x2')

# SVM radial and poly outperform SVC w.r.t. training error.

# test errors
print('SVC test error: %.2f' %(1-svc.score(xtest,ytest)))
print('SVM poly test error: %.2f' %(1-svmp.score(xtest,ytest)))
print('SVM radial test error: %.2f' %(1-svmr.score(xtest,ytest)))
# SVM poly and radial perform eqaully well on test data.

# ----------------------------------------------------------------------------
# Q5
# (a)
np.random.seed(1)
n = 500
p = 2

x1 = np.random.uniform(0, 1, n) - 0.5
x2 = np.random.uniform(0, 1, n) - 0.5
y  = 1*(x1**2-x2**2 > 0)
df = pd.DataFrame({'x1':x1, 'x2':x2, 'y':y})

# (b)
fig1 = sns.lmplot(x='x1', y='x2', data=df, fit_reg=False, hue='y', legend=False)

# (c)
glm1     = LogisticRegression()
glm1_fit = glm1.fit(df.iloc[:,0:2], df['y'])

# (d)
glm1_pred = glm1_fit.predict(df.iloc[:,0:2])
fig2, (ax1, ax2) = plt.subplots(1, 2)
ax1.scatter(df['x1'], df['x2'], c=glm1_pred, cmap=mpl.cm.Paired)
ax1.set_xlabel('x1')
ax1.set_ylabel('x2')
ax1.set_title('linear logit')

# (e)
df['x12'] = df['x1']**2
df['x22'] = df['x2']**2
glm2      = LogisticRegression()
glm2_fit  = glm2.fit(df.drop('y', axis=1), df['y'])

# (f)
glm2_pred = glm2_fit.predict(df.drop('y', axis=1))
ax2.scatter(df['x1'], df['x2'], c=glm2_pred, cmap=mpl.cm.Paired)
ax2.set_xlabel('x1')
ax2.set_ylabel('x2')
ax2.set_title('non-linear logit')

# (g)
svc = SVC(C=10, kernel='linear')
svc.fit(df.iloc[:,0:2], df['y'])

# (h)
svmp = SVC(C=10, kernel='poly', degree=2)
svmp.fit(df.iloc[:,0:2], df['y'])

# (i)
fig3   = plt.figure()
labels = ['SVC', 'SVM poly']
gs     = gridspec.GridSpec(1, 2)
for clf, lab, grd in zip([svc, svmp], labels, ([0,0], [0,1])):
    clf.fit(np.stack((x1, x2), axis=-1), y)
    ax   = plt.subplot(gs[grd[0], grd[1]])
    fig3 = plot_decision_regions(X=np.stack((x1, x2), axis=-1), y=y, clf=clf, legend=2)
    plt.title(lab)
plt.show()

# While SVC and linear logit are similar, SVM poly and non-linear logit are similar.

# ----------------------------------------------------------------------------
# Q6
# (a)
np.random.seed(1)
n = 100
p = 2
y = np.concatenate((np.repeat(1,30), np.repeat(2,70)))
x = np.random.rand(n,p)
x[y==1,:] = x[y==1,:] + 0.9
df = pd.DataFrame({'x1':x[:,0], 'x2':x[:,1], 'y':y})

sns.lmplot(x='x1', y='x2', data=df, fit_reg=False, hue='y', legend=False)

# (b)
tune_param = [{'C': [0.1, 1, 10, 100, 1000]}]

tune_out = GridSearchCV(SVC(kernel='linear'), tune_param, cv=10, scoring='accuracy', return_train_score=True)
tune_out.fit(x, y)
tune_out.best_params_

print('mean CV error: %s' %(1-tune_out.cv_results_['mean_test_score']))
# best cost: 1
# CV error can go up as cost rises.

print('mean train error: %s' %(1-tune_out.cv_results_['mean_train_score']))
# best cost: 1000
# Training error never rises in cost.

# (c)
np.random.seed(10)
ytest = np.concatenate((np.repeat(1,30), np.repeat(2,70)))
xtest = np.random.rand(n,p)
xtest[ytest==1,:] = xtest[ytest==1,:] + 0.9
dftest = pd.DataFrame({'x1':xtest[:,0], 'x2':xtest[:,1], 'y':ytest})
sns.lmplot(x='x1', y='x2', data=dftest, fit_reg=False, hue='y', legend=False)

test_err = np.zeros(5)
for i in range(0,5):
    svc = SVC(C=tune_param[0]['C'][i], kernel='linear')
    svc.fit(x, y)
    test_err[i] = 1-svc.score(xtest,ytest)

print('test error: %s' %test_err)
# best cost: 1
# Test error selection agrees with CV selection, not training error selection.

# (d) The claim at the end of Section 9.6.1 is true.

# ----------------------------------------------------------------------------
# Q7
Auto = pd.read_csv('C:\\Users\\Carol\\Desktop\\Auto.csv', na_values='?').dropna()

# (a)
Auto['mpg01'] = np.where(Auto['mpg'] > np.median(Auto['mpg']), 1, 0)
Auto          = Auto.drop(['mpg', 'name'], axis=1)
# scale continuous variables
var_no_scale  = ['cylinders', 'year', 'origin', 'mpg01']
var_to_scale  = ['displacement', 'horsepower', 'weight', 'acceleration']
scaled_var    = StandardScaler().fit_transform(Auto[var_to_scale])
temp1         = pd.DataFrame(scaled_var, columns=var_to_scale)
temp2         = Auto[var_no_scale].reset_index(drop=True)
df            = pd.concat([temp1, temp2], axis=1)
x             = df.iloc[:,:-1]
y             = df['mpg01']

# (b)
tune_param = [{'C': [0.01, 0.1, 1, 10]}]
tune_out   = GridSearchCV(SVC(kernel='linear'), tune_param, cv=5, scoring='accuracy', n_jobs=-1)
tune_out.fit(x, y)
best_svc   = tune_out.best_estimator_
print('best cost for SVC %s' %tune_out.best_params_) # best C=10

# (c)
tune_param = [{'C': [0.01, 0.1, 1, 10], 'degree': [2, 3, 4]}]
tune_out   = GridSearchCV(SVC(kernel='poly'), tune_param, cv=5, scoring='accuracy', n_jobs=-1)
tune_out.fit(x, y)
best_svmp  = tune_out.best_estimator_
print('best cost for SVM poly %s' %tune_out.best_params_) # best C=10, degree=2

tune_param = [{'C': [0.01, 0.1, 1, 10], 'gamma': [0.5, 1, 2]}]
tune_out   = GridSearchCV(SVC(kernel='rbf'), tune_param, cv=5, scoring='accuracy', n_jobs=-1)
tune_out.fit(x, y)
best_svmr  = tune_out.best_estimator_
print('best cost for SVM poly %s' %tune_out.best_params_) # best C=1, gamma=0.5

# (d)
pca      = PCA(n_components=2)
xreduced = pca.fit_transform(x)

model1 = SVC(C=10, kernel='linear')
clf1   = model1.fit(xreduced, y)

model2 = SVC(C=10, kernel='poly', degree=2)
clf2   = model2.fit(xreduced, y)

model3 = SVC(C=1, kernel='rbf', gamma=0.5)
clf3   = model3.fit(xreduced, y)

fig    = plt.figure()
fig.suptitle('decison surface using PCA transformed/projected features')
labels = ['SVC', 'SVM poly', 'SVM radial']
gs     = gridspec.GridSpec(3, 1)
for clf, lab, grd in zip([clf1, clf2, clf3], labels, ([0,0], [1,0], [2,0])):
    clf.fit(np.stack((xreduced[:,0], xreduced[:,1]), axis=-1), np.array(y))
    ax  = plt.subplot(gs[grd[0], grd[1]])
    fig = plot_decision_regions(X=np.stack((xreduced[:,0], xreduced[:,1]), axis=-1), y=np.array(y), clf=clf, legend=2)
    plt.title(lab)
plt.show()

err_name = ['SVC', 'SVM poly', 'SVM radial']
err_rate = [1-best_svc.score(x, y), 1-best_svmp.score(x, y), 1-best_svmr.score(x, y)]
print(pd.DataFrame({'training error rate': err_rate}, index=err_name))

# SVM radial fits the training data best.

# ----------------------------------------------------------------------------
# Q8
# (a)
OJ = pd.read_csv('C:\\Users\\Carol\\Desktop\\OJ.csv').drop('Unnamed: 0', axis=1)
OJ['Purchase'] = pd.factorize(OJ.Purchase)[0]
OJ['Store7']   = OJ.Store7.map({'No':0, 'Yes':1})

x = OJ.drop(['Purchase'], axis=1)
y = OJ['Purchase']
xtrain, xtest, ytrain, ytest = train_test_split(x, y, train_size=800, random_state=1)

# scale continuous variables
x_no_scale    = ['StoreID', 'SpecialCH', 'SpecialMM', 'Store7', 'STORE']
x_to_scale    = xtrain[xtrain.columns.difference(x_no_scale)]
scaler        = StandardScaler().fit(x_to_scale)
temp1         = pd.DataFrame(scaler.transform(x_to_scale), columns=xtrain.columns.difference(x_no_scale))
temp2         = xtrain[x_no_scale].reset_index(drop=True)
scaled_xtrain = pd.concat([temp1, temp2], axis=1)

x_to_scale    = xtest[xtest.columns.difference(x_no_scale)]
temp1         = pd.DataFrame(scaler.transform(x_to_scale), columns=xtest.columns.difference(x_no_scale))
temp2         = xtest[x_no_scale].reset_index(drop=True)
scaled_xtest  = pd.concat([temp1, temp2], axis=1)

# (b)
svc = SVC(C=0.01, kernel='linear')
svc.fit(scaled_xtrain, ytrain)

# (c)
svc_train_err = 1-svc.score(scaled_xtrain, ytrain)
svc_test_err  = 1-svc.score(scaled_xtest, ytest)
print('SVC training error: %.2f | SVC test error: %.2f' %(svc_train_err, svc_test_err))

# (d) & (e)
tune_param = [{'C': [0.01, 0.1, 1, 10]}]

tune_out = GridSearchCV(SVC(kernel='linear'), tune_param, cv=10, scoring='accuracy')
tune_out.fit(scaled_xtrain, ytrain)
best_svc = tune_out.best_estimator_

best_svc_train_err = 1-best_svc.score(scaled_xtrain, ytrain)
best_svc_test_err  = 1-best_svc.score(scaled_xtest, ytest)
print('best SVC training error: %.2f | best SVC test error: %.2f' %(best_svc_train_err, best_svc_test_err))

# (f)
svmr = SVC(C=0.01, kernel='rbf', gamma='auto')
svmr.fit(scaled_xtrain, ytrain)

svmr_train_err = 1-svmr.score(scaled_xtrain, ytrain)
svmr_test_err  = 1-svmr.score(scaled_xtest, ytest)
print('SVM radial training error: %.2f | SVM radial test error: %.2f' %(svmr_train_err, svmr_test_err))

tune_out   = GridSearchCV(SVC(kernel='rbf', gamma='auto'), tune_param, cv=10, scoring='accuracy')
tune_out.fit(scaled_xtrain, ytrain)
best_svmr  = tune_out.best_estimator_

best_svmr_train_err = 1-best_svmr.score(scaled_xtrain, ytrain)
best_svmr_test_err  = 1-best_svmr.score(scaled_xtest, ytest)
print('training error: %.2f | test error: %.2f' %(best_svmr_train_err, best_svmr_test_err))

# (g)
svmp = SVC(C=0.01, kernel='poly', degree=2)
svmp.fit(scaled_xtrain, ytrain)

svmp_train_err = 1-svmp.score(scaled_xtrain, ytrain)
svmp_test_err  = 1-svmp.score(scaled_xtest, ytest)
print('SVM poly training error: %.2f | SVM poly test error: %.2f' %(svmp_train_err, svmp_test_err))

tune_out   = GridSearchCV(SVC(kernel='poly', degree=2), tune_param, cv=10, scoring='accuracy')
tune_out.fit(scaled_xtrain, ytrain)
best_svmp  = tune_out.best_estimator_

best_svmp_train_err = 1-best_svmp.score(scaled_xtrain, ytrain)
best_svmp_test_err  = 1-best_svmp.score(scaled_xtest, ytest)
print('training error: %.2f | test error: %.2f' %(best_svmp_train_err, best_svmp_test_err))

# (h)
x_label = np.arange(6)
plt.bar(x_label, [svc_test_err, svmr_test_err, svmp_test_err, best_svc_test_err, best_svmr_test_err, best_svmp_test_err])
plt.xticks(x_label, ('linear', 'radial', 'poly', 'best linear', 'best radial', 'best poly'))
plt.ylabel('test error')
# SVC performs the best.


"""
Handles the authentication of Slack API and the use it for inviting and removing of the
participants.

The code presume that the slack bot is already created and the token is stored in the
environment variable USER_TOKEN.

To communicate with the slack api and add or remove the participants from specific
channel, we need to have following user token scope in the slack bot:
1. channels:read
2. groups:read
3. groups:write
4. users:read
5. users:read.email
6. admin.conversations:write
"""
import os
import slack

client = slack.WebClient(token=os.environ["USER_TOKEN"])

def get_all_user_ids(emails: list[str]) -> set[str]:
    """
    Get all the user Ids from the Workspace
    that match the email addresses in the input list.
    Return a set of IDs.
    Args:
        email (str): The email to use

    Returns:
        list[str]:  Gets all the user Ids from the Workspace
    """
    id_set = set()
    for email in emails:
        response = client.users_lookupByEmail(email=email)
        id_set.add(response["user"]["id"])
    return id_set

def get_user_ids_from_channel(channel_id: str) -> set[str]:
    """
    Get all the user IDs from a specific channel.
    Return a set of user IDs.
    Args:
        channelId (str): The Channel Id.

    Returns:
        list[str]:  Return lists of user Ids that are already invited to the channel
    """
    response = client.conversations_members(channel=channel_id)
    member_ids = response["members"]
    return set(member_ids)

def invite_members(emails: list[str], channel_id: str):
    """
    Invite all the users with their email addresses to a specific channel.
    Args:
        emailList list[str]: The list of the email address.
        channelId (str): The Channel Id.

    Returns:
        list[str]:  Return lists of user Ids that are already invited to the channel
    """
    all_id_set = get_all_user_ids(emails)
    members_in_channel_set = get_user_ids_from_channel(channel_id)

    new_members_set = all_id_set - members_in_channel_set
    kicked_members_set = members_in_channel_set - all_id_set

    for member in new_members_set:
        client.admin_conversations_invite(channel=channel_id, user=member)

    for member in kicked_members_set:
        client.conversations_kick(channel=channel_id, user=member)

if __name__ == "__main__":
    invite_members(["email1@example.com", "email2@example.com"], "ABC123")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import random
import numpy as np
from collections import deque
import os
import sys

import torch
import torch.nn.functional as F
import torch.optim as optim

from agent import Agent
from dqn_model import DQN
from collections import deque
"""
you can import any package and define any extra function as you need
"""

torch.manual_seed(595)
np.random.seed(595)
random.seed(595)

class Agent_DQN(Agent):
    def __init__(self, env, args):
        """
        Initialize everything you need here.
        For example:
            paramters for neural network
            initialize Q net and target Q net
            parameters for repaly buffer
            parameters for q-learning; decaying epsilon-greedy
            ...
        """
        super(Agent_DQN,self).__init__(env)
        ###########################
        # YOUR IMPLEMENTATION HERE #
        self.replay_buffer_size = 10000
        self.start_to_learn = 5000
        self.update_target_net = 5000
        self.learning_rate = 1.5e-4
        self.batch_size = 32
        self.buffer_ = deque(maxlen=self.replay_buffer_size)
        self.epsilon = 0.99
        self.epsilon_ = 0.2

        if args.test_dqn:
            #you can load your model here
            print('loading trained model')
            ###########################
            # YOUR IMPLEMENTATION HERE #
            self.Net = DQN()

    def init_game_setting(self):
        """
        Testing function will call this function at the begining of new game
        Put anything you want to initialize if necessary.
        If no parameters need to be initialized, you can leave it as blank.
        """
        ###########################
        # YOUR IMPLEMENTATION HERE #

        ###########################
        pass

    def make_action(self, observation, epsilon, device='cpu', test=True):
        """
        Return predicted action of your agent
        Input:
            observation: np.array
                stack 4 last preprocessed frames, shape: (84, 84, 4)
        Return:
            action: int
                the predicted action from trained model
        """
        ###########################
        # YOUR IMPLEMENTATION HERE #
        if random.random() < epsilon:
            action = self.env.action_space.sample()
        else:
            observation = torch.tensor(observation).to(device)
            q_values = self.Net(observation)
            _,action_value = torch.max(q_values,dim=-1)
            action = int(action_value.item())
        ###########################
        return action

    def push(self, sample):
        """ You can add additional arguments as you need.
        Push new data to buffer and remove the old one if the buffer is full.

        Hints:
        -----
            you can consider deque(maxlen = 10000) list
        """
        ###########################
        # YOUR IMPLEMENTATION HERE #
        self.buffer_.append(sample)
        ###########################

    def replay_buffer(self,batch_size):
        """ You can add additional arguments as you need.
        Select batch from buffer.
        """
        ###########################
        # YOUR IMPLEMENTATION HERE #
        idx_s = random.sample(range(len(self.buffer_)),batch_size)
        minibatch = []
        for _,i in enumerate(idx_s):
            minibatch.append(i)
        ###########################
        return np.array(minibatch)

    def get_reward(self,action):
        next_state,reward,terminal,_ = env.step(action)
        return next_state,reward,terminal

    def train(self):
        """
        Implement your training algorithm here
        """
        ###########################
        # YOUR IMPLEMENTATION HERE #
        optimizer = optim.Adam(net.parameters(),lr=self.learning_rate)
        total_r = []
        best_r = None
        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

        while True:
            observation = env.reset()
            sum_reward = 0
            i = 0
            self.epsilon = max(epsilon_,self.epsilon-0.1/i)
            while True:
                action_training = self.make_action(observation,self.epsilon,device)
                next_state,reward,terminal = get_reward(action_training)
                sum_reward += reward
                if terminal:
                    break
                else:
                    observation = next_state
            total_r.append(sum_reward)
            if total_r is not None:
                mean_reward = np.mean(total_r)
                if mean_reward > 40:
                    print("DOne")
                    break

        ###########################


from datetime import datetime
import sys
import tensorflow as tf
import numpy as np

from model import Model
from dataset import Dataset
from network import *
import tensorflow.contrib.slim as slim

def main():
    if len(sys.argv) != 4:
        print('Usage: python3 finetune.py train_file test_file weight_file')
        return

    train_file = sys.argv[1]
    test_file = sys.argv[2]
    weight_file = sys.argv[3]

    # checkpoint save path
    checkpoint_path = '/path/to/saved/model/vggface16_finetuned.ckpt'
    # writer path
    train_writer_path = '/path/to/logs/train'
    test_writer_path = '/path/to/logs//test'

    # Learning params
    learning_rate_init = 0.001   # 0.001
    decay_steps = 10000
    decay_rate = 0.5

    # Train and dispaly params
    training_iters = 60000
    batch_size = 50         #36
    display_step = 20
    test_step = 1000
    save_step = 1000

    # Network params
    n_classes = 10575
    keep_rate = 0.5  # 1.0

    # Graph input
    x = tf.placeholder(tf.float32, [batch_size, 224, 224, 3])    #input of vgg is 224*224, input of alexnet is 227*227
    y = tf.placeholder(tf.float32, [None, n_classes])
    keep_var = tf.placeholder(tf.float32)

    # Model
    # tf.reset_default_graph()
    pred = Model.vgg16(x, keep_var, n_classes)

    # Loss
    with tf.name_scope('loss'):
        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))

    # Optimizer
    global_step = tf.Variable(0, trainable=False, name='global_step')
    with tf.name_scope('learning_rate'):
        learning_rate = tf.train.exponential_decay(learning_rate_init, global_step, decay_steps, decay_rate, staircase=True)
    optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(loss, global_step=global_step)

    # Evaluation
    correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

    # Init
    init = tf.global_variables_initializer()

    # Create a summary to monitor learning_rate, loss, accuracy
    tf.summary.scalar('learning_rate', learning_rate)
    tf.summary.scalar("loss", loss)
    tf.summary.scalar("accuracy", accuracy)

    merged_summary_op = tf.summary.merge_all()

    # Load dataset
    dataset = Dataset(train_file, test_file)

    # Create a saver
    saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=3)  # ADD

    # Launch the graph
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    with tf.Session() as sess:
        print('Init variable')
        sess.run(init)

        print("Model restored from file: %s" % weight_file)
        load_with_skip(weight_file, sess, ['fc8'])  # Skip weights from fc8
        print('Start training')

        # write logs to Tensorboard
        train_writer = tf.summary.FileWriter(train_writer_path, sess.graph)
        test_writer = tf.summary.FileWriter(test_writer_path, sess.graph)

        step = 1
        while step < training_iters:
            batch_xs, batch_ys = dataset.next_batch(batch_size, 'train')
            sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys, keep_var: keep_rate})

            summary = sess.run(merged_summary_op, feed_dict={x: batch_xs, y: batch_ys, keep_var: keep_rate})
            # Write logs at every iteration
            train_writer.add_summary(summary, step)

            # Display testing status
            if step % test_step == 0:
                test_acc = 0.
                test_count = 0
                t_loss = 0
                for _ in range(dataset.test_size // batch_size):
                    batch_tx, batch_ty = dataset.next_batch(batch_size, 'test')
                    acc = sess.run(accuracy, feed_dict={x: batch_tx, y: batch_ty, keep_var: 1.})
                    b_loss = sess.run(loss, feed_dict={x: batch_tx, y: batch_ty, keep_var: 1.})
                    summary = sess.run(merged_summary_op, feed_dict={x: batch_tx, y: batch_ty, keep_var: 1.})    # ADD
                    test_writer.add_summary(summary, step)                                              # ADD
                    test_acc += acc
                    t_loss += b_loss
                    test_count += 1
                test_acc /= test_count
                t_loss/= test_count
                print('{} Iter {}: Testing Accuracy = {:.4f}, Test loss = {:.4f}'.format(datetime.now(), step, test_acc,t_loss), file=sys.stderr)

            # Display training status
            if step % display_step == 0:
                acc = sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys, keep_var: 1.})
                batch_loss = sess.run(loss, feed_dict={x: batch_xs, y: batch_ys, keep_var: 1.})
                lr = sess.run(learning_rate)
                print('{} Iter {}: Training Loss = {:.4f}, Accuracy = {:.4f}, lr = {:.4f}'.format(datetime.now(), step, batch_loss, acc, lr), file=sys.stderr)

            # Save model
            if step % save_step == 0:
                saver.save(sess, checkpoint_path, global_step=step)

            step += 1
        print('Finish!')

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd
from  sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import joblib
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.ensemble import RandomForestRegressor
import json

# data cleaning

data_file = 'immo.csv'
df = pd.read_csv(data_file)

# drop column has nan missing values for more than 60%
df = pd.DataFrame(df)
df = df.drop(['ID','PostCode', 'Sub type', 'Floor', 'IsIsolated', 'HasSeaView', 'TotalRoomCount', 'HasAttic',  'HasDiningRoom',    'GardenArea', 'LivingRoomArea', 'NetHabitableSurface(msq)', 'SchoolDistance', 'ShopDistance', 'TransportDistance', 'BuildingCondition','RegionCode','street'], axis=1)

# age of building
list_years = []
for year in df['ConstructionYear']:
    years = 2022 - year
    list_years.append(years)
df = df.assign(ConstructionYear = list_years)

# replace & drop nan values
df[['NetHabitableSurface', 'ConstructionYear']] = df[['NetHabitableSurface','ConstructionYear']].fillna(df[['NetHabitableSurface', 'ConstructionYear']].mean())
columns_to_n = ['Type', 'locality', 'KitchekType']
df[columns_to_n] = df[columns_to_n].replace(np.nan, 'No Value')
df = df.drop_duplicates()
df = df.dropna(subset=['BedroomCount', 'Price', 'Province'])

# replace nan value in heating type with most frequent value
heating_type = df['HeatingType'].mode()[0]
facade_count = df['FacadeCount'].mode()[0]
flood_zone_type = df['FloodZoneType'].mode()[0]

df = df.fillna({'HasLift': False, 'HasBalcony': False,  'HasGarden': False, 'HasBasement': False,  'IsDoubleGlaze': False, 'HeatingType':heating_type, 'FacadeCount': facade_count, 'FloodZoneType':flood_zone_type})

#turn data type to int
df['NetHabitableSurface'] = df['NetHabitableSurface'].astype('int')
df['Price'] = df['Price'].astype('int')
df['BedroomCount'] = df['BedroomCount'].astype('int')
df['ConstructionYear'] = df['ConstructionYear'].astype('int')
df['FacadeCount'] = df['FacadeCount'].astype('int')

df = df.dropna(subset=['Price', 'NetHabitableSurface', 'BedroomCount', 'Province', 'Region'])

df.columns

# pipeline normorlize ohe
num_feature = ['BedroomCount', 'NetHabitableSurface']
cate_feature = ['Type', 'Province', 'locality', 'Region', 'FloodZoneType', 'HeatingType', 'KitchekType']

num_trans = Pipeline([('scaler', StandardScaler())])
cate_trans = Pipeline([('ohe', OneHotEncoder(handle_unknown='ignore', sparse=False))])

preprocessor = ColumnTransformer([('num_transformer', num_trans, num_feature),
                                ('cate_transformer', cate_trans, cate_feature)])

# split data
X = df.drop('Price', axis=1)
y = df['Price']

X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0,  test_size=0.2)
X.columns

trans_train = preprocessor.fit_transform(X_train)
trans_test = preprocessor.fit_transform(X_test)

# random forest
rf = RandomForestRegressor()

rf.fit(trans_train, y_train)
y_predict = rf.predict(trans_test)

#
model = joblib.load('rt_model.pkl')

predictions = model.predict(trans_test)
predictions


from tensorflow.keras.models import load_model
import data_tools
import os
import numpy as np
import soundfile as sf
import config_params

def createpath(path):
    if not os.path.exists(path):
        os.makedirs(path)

def prediction():

    createpath(config_params.PATH_DIR_TEST_NOISE_dB)
    createpath(config_params.PATH_DIR_TEST_VOICE_dB)
    createpath(config_params.PATH_DIR_TEST_SNR_BASED)
    createpath(config_params.PATH_DIR_TEST_NOISY)
    createpath(config_params.PATH_DIR_TEST_IMAGE_NOISY)
    createpath(config_params.PATH_DIR_TEST_IMAGE_VOICE)
    createpath(config_params.PATH_DIR_TEST_IMAGE_DENOISE)

    list_noise_files = []
    list_voice_files = []
    list_noise_snr_files = []

    print("Loading...")
    # Load noise file paths
    for root, dirs, files in os.walk(config_params.PATH_DIR_TEST_NOISE_PREDICT):
        if len(files) == 0:
            continue

        for f in files:
            files = os.path.join(root, f)
            list_noise_files.append(files)

    # Load voice filenames
    for root, dirs, files in os.walk(config_params.PATH_DIR_TEST_VOICE_PREDICT):
        if len(files) == 0:
            continue

        for f in files:
            files = os.path.join(root, f)
            list_voice_files.append(files)

    list_dB_noise_file = data_tools.scale_dB(
        list_noise_files, config_params.PATH_DIR_TEST_NOISE_dB)
    list_dB_voice_file = data_tools.scale_dB(
        list_voice_files, config_params.PATH_DIR_TEST_VOICE_dB)

    list_noise_files.clear()
    list_voice_files.clear()

    print("Setting snr...")
    SNR = [15, 10, 5, 0, -5, -10]
    for snr in SNR:
        createpath(config_params.PATH_DIR_TEST_SPLIT_NOISY + str(snr) + '/')
        createpath(config_params.PATH_DIR_TEST_SPLIT_VOICE + str(snr) + '/')
        snr_base_noise_file = data_tools.set_snr(
            list_dB_voice_file, list_dB_noise_file, snr, config_params.PATH_DIR_TEST_SNR_BASED)
        list_noise_snr_files.append(snr_base_noise_file)

    print(f'Loading Model: {config_params.PATH_WEIGHTS}')
    loaded_model = load_model(config_params.PATH_WEIGHTS)

    for snr_noise_data in list_noise_snr_files:
        cropped_list_noisy = []
        output_audio_list = []
        snr = snr_noise_data.split('/')[-1].split('_')[0]
        print('SNR:', snr)

        noise = data_tools.audio_files_to_numpy(snr_noise_data)
        voice = data_tools.audio_files_add_to_numpy(list_dB_voice_file)

        print("Mixing...")
        prod_voice, prod_noise, prod_noisy = data_tools.mix_noise_voice(
            voice, noise)

        # sf.write(f'{config_params.PATH_DIR_TEST_NOISY}/{snr}_voice.wav',
        #          prod_voice, config_params.SAMPLE_RATE, 'PCM_24')
        # sf.write(f'{config_params.PATH_DIR_TEST_NOISY}/{snr}_noise.wav',
        #          prod_noise, config_params.SAMPLE_RATE, 'PCM_24')
        sf.write(f'{config_params.PATH_DIR_TEST_NOISY}/{snr}_noisy.wav',
                 prod_noisy, config_params.SAMPLE_RATE, 'PCM_24')

        print("Cropping raw data...")
        cropped_list_noisy.extend(data_tools.split_into_one_second(
            prod_noisy, config_params.PATH_DIR_TEST_SPLIT_NOISY, snr, False))

        cropped_array_noisy = np.array(cropped_list_noisy)

        print("cropped_array_noisy:", cropped_array_noisy.shape)
        fix_length = cropped_array_noisy.shape[1]
        print(config_params.HOP_LENGTH_FFT, fix_length)

        mag_amp_db_noisy,  mag_phase_noisy = data_tools.numpy_audio_to_matrix_spectrogram(
            cropped_array_noisy, config_params.PATH_DIR_TEST_IMAGE_NOISY)

        X_in = data_tools.scaled_in(mag_amp_db_noisy)
        X_pred = loaded_model.predict(X_in)
        inv_sca_X_pred = data_tools.inv_scaled_ou(X_pred)
        X_denoise = mag_amp_db_noisy - inv_sca_X_pred

        print("X_in:", X_in.shape)
        print("X_pred:", X_pred.shape)
        print("inv_sca_X_pred:", inv_sca_X_pred.shape)
        print("X_denoise:", X_denoise.shape)

        audio_denoise_recons = data_tools.matrix_spectrogram_to_numpy_audio(
            X_denoise, mag_phase_noisy, fix_length, config_params.PATH_DIR_TEST_IMAGE_DENOISE)

        sf.write(
            f'{config_params.PATH_DIR_PREDICT_ROOT}/{snr}_{config_params.PATH_PREDICT_OUTPUT_NAME}', audio_denoise_recons, config_params.SAMPLE_RATE, 'PCM_24')

        output_audio_list.append(
            f'{config_params.PATH_DIR_PREDICT_ROOT}/{snr}_{config_params.PATH_PREDICT_OUTPUT_NAME}')

        data_tools.scale_dB(output_audio_list, config_params.PATH_DIR_PREDICT_ROOT, purepath=True)

        print("")

if __name__ == '__main__':
    prediction()


import numpy as np
import pickle
import torch
import matplotlib.pyplot as plt
import os
import torch.nn as nn
from torchvision.transforms import RandomRotation
from PIL import Image

class trainLoader():

	def __init__(self, config=None, batch_size=50, epoch_len = 100, seq_len = 8, transform=None, training_path = '/home/mayank/Desktop/GitRepos/crnn-pytorch_mnist/data/processed/training.pt', ip_channels = 1, size=(20,200), Type='train', profiler = None, target_transform = None):

		self.training_path = training_path
		self.abc = '0123456789'
		self.seq_len = seq_len
		self.epoch_len = epoch_len
		self.transform = transform

		self.train_data, self.train_labels = torch.load(training_path)
		self.num_total = len(self.train_labels)
		self.final_size = size
		self.normal_mean = 7
		self.clip = (1,40)
		self.ip_channels = ip_channels
		self.resized_shape = (*size,ip_channels)

		self.target_aspect_ratio = 10

		self.out_dir = 'out'
		self.rotate = RandomRotation(10)

		self.batch_size = batch_size
		self.encoding_to_char = {1: '0', 2:'1', 3:'2', 4:'3', 5:'4', 6:'5', 7:'6', 8:'7', 9:'8', 10:'9'}

	def seed(self):

		np.random.seed(self.config['seed'])
		random.seed(self.config['seed'])
		torch.manual_seed(self.config['seed'])
		torch.cuda.manual_seed(self.config['seed'])
		torch.backends.cudnn.deterministic = True

	def _update_config(self, config):

		self.config = config

	def get_abc(self):
		return self.abc

	def set_mode(self, mode='train'):
		return

	def generate_string(self):
		return ''.join(random.choice(self.abc) for _ in range(self.seq_len))

	def __len__(self):

		return self.epoch_len

	def instance(self, num):

		choices = np.random.randint(0, self.num_total, num)
		sample = self.train_data[choices]

		# Variable Spacing

		final = np.zeros([28, 28*num])

		if self.training_path == 'data/processed/training.pt':
			final[:, 0:28] = self.rotate(Image.fromarray(sample[0].numpy()).convert('L'))
		else:
			final[:, 0:28] = sample[0]
		till_now = 28

		for i in sample[1:]:

			i = Image.fromarray(i.numpy()).convert('L')
			if self.training_path == 'data/processed/training.pt':
				CENTER_DIST = 27 - np.random.randint(0, 14)
				rotated = self.rotate(i)
			else:
				CENTER_DIST = 27
				rotated = i

			final[:, till_now-(28 - CENTER_DIST):till_now-(28 - CENTER_DIST)+28] += rotated
			final[:, till_now-(28 - CENTER_DIST):till_now-(28 - CENTER_DIST)+28][final[:, till_now-(28 - CENTER_DIST):till_now-(28 - CENTER_DIST)+28]>255] = 255
			till_now += CENTER_DIST

		final = final[:, :till_now]

		labels = self.train_labels[choices]+1

		return final.astype(np.uint8), labels

	def batch(self, size):

		# http://www.ravi.io/language-word-lengths

		sizes = np.array([0.1, 0.6, 2.6, 5.2, 8.5, 12.2, 14, 14, 12.6, 10.1, 7.5, 5.2, 3.2, 2, 1, 0.6, 0.3, 0.2, 0.1, 0.1])
		sizes = sizes/np.sum(sizes)
		sizes = np.random.choice(np.arange(2, len(sizes)+2), size=size, p=sizes)

		images, targets = [], []
		for size in sizes:
			sample, target = self.instance(size)
			images.append(sample)
			targets.append(target)

		return images, targets

	def resize(self, to_resize_batch):

		#to_resize_batch: (examples, width, height, channels)
		#upsample requires (batch_size, input_features, width, height)

		final = np.zeros([len(to_resize_batch), self.ip_channels, self.resized_shape[0], self.resized_shape[1]])

		for i, sample in enumerate(to_resize_batch):
			# print(sample.shape)
			c_aspect_ratio = sample.shape[1]//sample.shape[0]
			# print(c_aspect_ratio)

			if c_aspect_ratio > self.target_aspect_ratio:
				#Add along height
				new_h = int(sample.shape[0]*c_aspect_ratio/self.target_aspect_ratio)
				temp = np.zeros((new_h, sample.shape[1]))
				temp[(new_h-sample.shape[0])//2:(new_h-sample.shape[0])//2+sample.shape[0], :] = sample

			elif c_aspect_ratio < self.target_aspect_ratio:
				#Add along width

				new_w = int(sample.shape[1]*self.target_aspect_ratio/c_aspect_ratio)
				temp = np.zeros((sample.shape[0], new_w))
				temp[:, (new_w-sample.shape[1])//2:(new_w-sample.shape[1])//2+sample.shape[1]] = sample
			else:
				temp = sample

			sample = temp[None, None, :, :]
			final[i] = nn.functional.interpolate(torch.FloatTensor(sample), size=(self.resized_shape[0], self.resized_shape[1]), mode='nearest', align_corners=None)

		return final

	def getitem(self, index):

		size = self.batch_size
		images, targets = self.batch(size)
		images = [torch.FloatTensor(np.array(Image.fromarray(img).convert('RGB').resize(size=(int(32*img.shape[1]/img.shape[0]), 32))).transpose(2, 0, 1)) for img in images]

		seq_len = [torch.IntTensor([target_i.shape[0]]) for target_i in targets]

		seq = []
		for i in range(len(targets)):

			seq.append(targets[i].int())

		images = [image.unsqueeze(0) for image in images]

		sample = {"img": images, "seq": seq, "seq_len": seq_len, "aug": True}

		return sample


from typing import Any, Optional

import torch
import torch.distributed as dist
from torch.distributed import ReduceOp

from pipegoose.distributed._p2p import _P2P
from pipegoose.distributed.parallel_context import ParallelContext
from pipegoose.distributed.parallel_mode import ParallelMode

def scatter(
    tensor: torch.Tensor,
    dim: int,
    parallel_context: Optional[ParallelContext] = None,
    parallel_mode: Optional[ParallelMode] = None,
) -> torch.Tensor:
    """Scatter tensors to all ranks in parallel group."""
    world_size = parallel_context.get_world_size(parallel_mode)
    rank = parallel_context.get_local_rank(parallel_mode)

    if world_size == 1:
        return tensor

    assert tensor.size(dim) % world_size == 0

    tensor_list = torch.chunk(tensor, world_size, dim=dim)
    return tensor_list[rank]

def reduce(
    tensor: torch.Tensor,
    dst: int,
    op: ReduceOp = ReduceOp.SUM,
    async_op: bool = False,
    parallel_context: Optional[ParallelContext] = None,
    parallel_mode: Optional[ParallelContext] = None,
) -> torch.Tensor:
    """Reduce tensors from all processes in parallel group."""
    world_size = parallel_context.get_world_size(parallel_mode)

    if world_size == 1:
        return tensor

    group = parallel_context.get_group(parallel_mode)
    work = dist.reduce(tensor, dst=dst, op=op, group=group)

    if async_op:
        return tensor, work
    else:
        return tensor

def broadcast(
    tensor: torch.Tensor,
    src: int,
    async_op: bool = False,
    parallel_context: Optional[ParallelContext] = None,
    parallel_mode: Optional[ParallelMode] = None,
) -> torch.Tensor:
    """Broadcast a tensor from src rank to all processes in parallel group."""
    world_size = parallel_context.get_world_size(parallel_mode)

    if world_size == 1:
        return tensor

    group = parallel_context.get_group(parallel_mode)
    work = dist.broadcast(tensor, src=src, group=group, async_op=async_op)

    if async_op:
        return tensor, work
    else:
        return tensor

def all_gather(
    tensor: torch.Tensor,
    dim: int = 0,
    async_op: bool = False,
    parallel_context: Optional[ParallelContext] = None,
    parallel_mode: Optional[ParallelMode] = None,
) -> torch.Tensor:
    """All gather tensors from all processes in parallel group.

    Args:
        input (torch.Tensor): The tensor you want to gather.
        dim (int, optional): The dimension along which to gather the tensor.. Defaults to 0.
        async_op (bool, optional): _description_. Defaults to False.
        parallel_context (Optional[ParallelContext], optional): _description_. Defaults to None.
        parallel_mode (Optional[ParallelMode], optional): _description_. Defaults to None.

    Returns:
        torch.Tensor: _description_
    """
    world_size = parallel_context.get_world_size(parallel_mode)
    group = parallel_context.get_group(parallel_mode)

    if world_size == 1:
        return tensor

    tensor_list = [torch.empty_like(tensor) for _ in range(world_size)]
    work = dist.all_gather(tensor_list=tensor_list, tensor=tensor, async_op=async_op, group=group)

    if tensor.dim() == 0:
        tensor_list = [tensor.unsqueeze(dim=0) for tensor in tensor_list]

    tensor_list = torch.cat(tensor_list, dim=dim)

    if async_op:
        return tensor_list, work
    else:
        return tensor_list

def all_reduce(
    tensor: torch.Tensor,
    op: ReduceOp = ReduceOp.SUM,
    async_op: bool = False,
    parallel_context: Optional[ParallelContext] = None,
    parallel_mode: Optional[ParallelMode] = None,
) -> torch.Tensor:
    """All reduce a tensor from all ranks in parallel group."""
    world_size = parallel_context.get_world_size(parallel_mode)

    if world_size == 1:
        return tensor

    group = parallel_context.get_group(parallel_mode)
    work = dist.all_reduce(tensor, op=op, group=group, async_op=async_op)

    if async_op:
        return tensor, work
    else:
        return tensor

def reduce_scatter():
    pass

def send(
    data: Any,
    src: int,
    dst: int,
    parallel_context: ParallelContext,
    parallel_mode: ParallelMode = ParallelMode.PIPELINE,
):
    """P2P communication: Send an object from src rank to dst rank."""
    if src == parallel_context.get_local_rank(parallel_mode):
        _P2P().send(data, dst, parallel_context, parallel_mode)

def recv(
    src: int, dst: int, parallel_context: ParallelContext, parallel_mode: ParallelMode = ParallelMode.PIPELINE
) -> Optional[Any]:
    """P2P communication: Receive an object from src rank to dst rank."""
    if dst == parallel_context.get_local_rank(parallel_mode):
        return _P2P().recv(src, parallel_context, parallel_mode)

def barrier(parallel_context: ParallelContext, parallel_mode: ParallelMode):
    """Wait until all processes in the parallel mode reach this barrier."""
    process_group = parallel_context.get_group(parallel_mode)
    dist.barrier(group=process_group)