"""Support for the Mastertherm Sensors."""
from decimal import Decimal
from datetime import date, datetime
import logging

from homeassistant.core import HomeAssistant
from homeassistant.components.sensor import SensorEntity, SensorDeviceClass
from homeassistant.config_entries import ConfigEntry
from homeassistant.const import CONF_ENTITIES, UnitOfTemperature, Platform
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.typing import StateType

from .const import DOMAIN, MasterthermSensorEntityDescription
from .coordinator import MasterthermDataUpdateCoordinator
from .entity import MasterthermEntity

_LOGGER = logging.getLogger(__name__)

async def async_setup_entry(
    hass: HomeAssistant,
    entry: ConfigEntry,
    async_add_entities: AddEntitiesCallback,
):
    """Load Sensors from the config settings."""
    coordinator: MasterthermDataUpdateCoordinator = hass.data[DOMAIN][entry.entry_id]

    entities: list[SensorEntity] = []
    for entity_key, entity_description in coordinator.entity_types[
        Platform.SENSOR
    ].items():
        for module_key, module in coordinator.data["modules"].items():
            if entity_key in module[CONF_ENTITIES]:
                entities.append(
                    MasterthermSensor(
                        coordinator, module_key, entity_key, entity_description
                    )
                )

    async_add_entities(entities, True)
    coordinator.remove_old_entities(Platform.SENSOR)

class MasterthermSensor(MasterthermEntity, SensorEntity):
    """Representation of a MasterTherm Sensor, e.g. ."""

    def __init__(
        self,
        coordinator: MasterthermDataUpdateCoordinator,
        module_key: str,
        entity_key: str,
        entity_description: MasterthermSensorEntityDescription,
    ):
        """Initialize the sensor."""
        self._icon_state_map = entity_description.icon_state_map

        if entity_description.device_class == SensorDeviceClass.TEMPERATURE:
            entity_description.native_unit_of_measurement = UnitOfTemperature.CELSIUS
            entity_description.suggested_unit_of_measurement = UnitOfTemperature.CELSIUS

        self._attr_state_class = entity_description.state_class

        super().__init__(
            coordinator=coordinator,
            module_key=module_key,
            entity_key=entity_key,
            entity_type=Platform.SENSOR,
            entity_description=entity_description,
        )

    @property
    def icon(self) -> str | None:
        """Set dynamic icons if available."""
        if self._icon_state_map:
            return self._icon_state_map[self.native_value]
        else:
            return self.entity_description.icon

    @property
    def native_value(self) -> StateType | date | datetime | Decimal:
        """Return the sensor value."""
        return self.coordinator.data["modules"][self._module_key]["entities"][
            self._entity_key
        ]


from django.shortcuts import render
from .models import cities, city
from django.http import Http404
from datetime import datetime
import pytz
import folium

now = datetime.now(pytz.timezone('Europe/Warsaw')).strftime("%H")

def weather_view(request, *args, **kwargs):
    cities_ = cities.objects.all()
    city_ = city.objects.all()
    ci_date = city.objects.get(city_id=1)
    if int(now) >= 6 and int(now) < 20:
        for c in city_:
            c.pic = c.pic.replace("n","d")
    context = {
        'cities': cities_,
        'city': city_,
        'c_date':ci_date,
    }
    return render(request, "weather.html", context)

def weather_view_detail(request,pk):
    city_ = city.objects.get(city_id=pk)
    city_name = cities.objects.get(id=pk)
    city_name_n = city_name.name.replace(" ","_")

    timezone = int(city_.timezone) // 3600
    timezone = f"UTC {('+' if timezone >= 0 else '')}{timezone}"
    sunset = city_.sunset
    sunrise = datetime.utcfromtimestamp(city_.sunrise).strftime("%H:%M")
    sunset = datetime.utcfromtimestamp(sunset)
    sunset = sunset.strftime("%H:%M")
    if int(now) >= 6 and int(now) < 20:
        city_.pic = city_.pic.replace("n","d")

    m = folium.Map(location=[city_.lat, city_.lon], zoom_start=12)
    folium.Marker([city_.lat, city_.lon], popup=city_name.name).add_to(m)
    map_html = m._repr_html_()

    context = {
        'map_html': map_html,
        'city': city_,
        'name_space': city_name.name,
        'name' : city_name_n,
        'timezone': timezone,
        'sunrise': sunrise,
        'sunset': sunset,

    }
    if city_ is not None:
        return render(request, 'detail.html', context)
    else:
        raise Http404('Something gone wrong')

# def index(request):
#     cities_ = cities.objects.all()
#     return render(request, 'weather/index.html', {'cities': cities_})
#
# def city(request, city_id):
#     city_ = city.objects.get(id=city_id)
#     return render(request, 'weather/city.html', {'city': city_})


from setuptools import setup

with open('requirements.txt') as fp:
    install_requires = fp.read()

setup(
    name='mapped_config',
    packages=['mapped_config'],
    version='2.36',
    description='Mapped config loader for python for secure, easy and modular configuration management',
    author='Alvaro Garcia Gomez',
    author_email='maxpowel@gmail.com',
    url='https://github.com/maxpowel/mapped_config',
    download_url='https://github.com/maxpowel/mapped_config/archive/master.zip',
    keywords=['config', 'configuration', 'yml', 'json'],
    classifiers=['Topic :: Adaptive Technologies', 'Topic :: Software Development', 'Topic :: System',
                 'Topic :: Utilities'],
    install_requires=install_requires
)


import csv
import random
import numpy as np
from sklearn import preprocessing
import matplotlib.pyplot as plt
from keras.models import Sequential, Model
from keras.layers import Dense, LSTM, TimeDistributed, Conv2D, Flatten
from keras.optimizers import Adam
import utm

max_latitude = 330414.05273900216
max_longitude = 3463503.3311055717
min_latitude = 328787.97245886514
min_longitude = 3462203.7096695383

la_size = 16*2
lo_size = 13*2
la = (max_latitude - min_latitude)/la_size
lo = (max_longitude - min_longitude)/lo_size
print(lo)
print(la)

def coordinate_to_label(coordinate):
    la_label = int((coordinate[0] - min_latitude - 1e-12)/la)
    lo_label = int((coordinate[1] - min_longitude - 1e-12)/lo)
    tmp = np.zeros(la_size * lo_size, dtype=int)
    tmp[la_label + lo_label*la_size] = 1
    return tmp

def shuffle_train_test(X, Y, LEN, ratio):
    random.seed(1)
    random_list = random.sample(range(LEN), k=int(ratio*LEN))
    X_Train = []
    Y_Train = []
    X_Test = []
    Y_Test = []
    for i in range(LEN):
        if i in random_list:
            X_Test.append(X[i])
            Y_Test.append(Y[i])
        else:
            X_Train.append(X[i])
            Y_Train.append(Y[i])
    return np.array(X_Train), np.array(Y_Train), np.array(X_Test), np.array(Y_Test)

# lat, lon, 51 'R'
def check_validation(value_dict):
    validate_list = []
    validate_data = {
        'Latitude': value_dict['Latitude'],
        'Longitude': value_dict['Longitude'],
        'MRTime': value_dict['MRTime'],
        'TrajID': value_dict['TrajID']
    }
    if float(value_dict['Accuracy']) > 30:
        return False, validate_data

    for i in range(6):
        if value_dict['RNCID_' + str(i+1)] != '-999' and value_dict['CellID_' + str(i+1)] != '-999' and value_dict['Dbm_' + str(i+1)] != '-999' \
            and value_dict['AsuLevel_' + str(i+1)] != '-1' and value_dict['SignalLevel_' + str(i+1)] != '-1':

            validate_list.append(i)
            validate_data['RNCID_' + str(i+1)] = value_dict['RNCID_' + str(i+1)]
            validate_data['CellID_' + str(i+1)] = value_dict['CellID_' + str(i+1)]
            validate_data['Dbm_' + str(i+1)] = value_dict['Dbm_' + str(i+1)]
            validate_data['AsuLevel_' + str(i+1)] = value_dict['AsuLevel_' + str(i+1)]
            validate_data['SignalLevel_' + str(i+1)] = value_dict['SignalLevel_' + str(i+1)]

    if len(validate_list) < 3: # 我认为基站数量少于3，即理论上不可能得出手机坐标
        return False, validate_data
    else:
        for i in range(6):
            if i not in validate_list:
                np.random.seed(1)
                k = np.random.choice(validate_list)
                validate_data['RNCID_' + str(i+1)] = value_dict['RNCID_' + str(k+1)]
                validate_data['CellID_' + str(i+1)] = value_dict['CellID_' + str(k+1)]
                validate_data['Dbm_' + str(i+1)] = value_dict['Dbm_' + str(k+1)]
                validate_data['AsuLevel_' + str(i+1)] = value_dict['AsuLevel_' + str(k+1)]
                validate_data['SignalLevel_' + str(i+1)] = value_dict['SignalLevel_' + str(k+1)]
        return True, validate_data

# 按照轨迹ID进行切割
def cut_to_slices(X, Y, slice_length, LEN):
    tmp_count = 1
    tmp_X = []
    tmp_Y = []
    x_slice_list = []
    y_slice_list = []
    tmp_traj_label = int(X[0][1])
    for i in range(LEN):
        traj_id = int(X[i][1])
        if tmp_count == slice_length:
            if traj_id == tmp_traj_label:
                x_slice_list.append(X[i])
                y_slice_list.append(Y[i])
                tmp_X += x_slice_list
                tmp_Y += y_slice_list
                x_slice_list = []
                y_slice_list = []
                tmp_count = 1
                if (i+1) < LEN:
                    tmp_traj_label = int(X[i+1][1])
            else:
                x_slice_list.insert(0, X[i-slice_length])
                y_slice_list.insert(0, Y[i-slice_length])
                tmp_X += x_slice_list
                tmp_Y += y_slice_list
                x_slice_list = [X[i]]
                y_slice_list = [Y[i]]
                tmp_count = 2
                tmp_traj_label = traj_id
        else:
            if traj_id == tmp_traj_label:
                tmp_count += 1
                x_slice_list.append(X[i])
                y_slice_list.append(Y[i])
            else:
                x_slice_list = []
                y_slice_list = []
                for k in range(slice_length):
                    x_slice_list.append(X[i+k-slice_length])
                    y_slice_list.append(Y[i+k-slice_length])
                tmp_X += x_slice_list
                tmp_Y += y_slice_list
                x_slice_list = [X[i]]
                y_slice_list = [Y[i]]
                tmp_count = 2
                tmp_traj_label = traj_id
    return np.array(tmp_X), np.array(tmp_Y)

signal_tower_dict = {}
with open('../data/gongcan.csv', 'r') as file:
    reader = csv.DictReader(file)
    for row in reader:
        coordinate = utm.from_latlon(float(row['Latitude']), float(row['Longitude']))[:2]
        signal_tower_dict[row['RNCID'] + '|' + row['CellID']] = coordinate

X = []
Y = []
features = 32
with open('../data/train_2g.csv', 'r') as file:
    reader = csv.DictReader(file)
    for row in reader:
        validation, parsed_row = check_validation(row)
        if(not validation):
            continue
        coordinate = utm.from_latlon(float(parsed_row['Latitude']), float(parsed_row['Longitude']))[:2]
        Y.append([coordinate[0], coordinate[1]])
        mr_sample = np.zeros(features)
        mr_sample[0] = parsed_row['MRTime']
        mr_sample[1] = parsed_row['TrajID']
        for i in range(6):
            coordinate = signal_tower_dict[parsed_row['RNCID_' + str(i+1)] + '|' + parsed_row['CellID_' + str(i+1)]]
            mr_sample[i*5 + 2] = coordinate[0] # 转换成信号站的经纬度坐标
            mr_sample[i*5 + 3] = coordinate[1]
            mr_sample[i*5 + 4] = parsed_row['Dbm_' + str(i+1)]
            mr_sample[i*5 + 5] = parsed_row['AsuLevel_' + str(i+1)]
            mr_sample[i*5 + 6] = parsed_row['SignalLevel_' + str(i+1)]
        X.append(np.array(mr_sample))

LEN = len(X)
slice_length = 6

X = np.array(X).astype('float64')
Y = np.array(Y).astype('float64')

X, Y = cut_to_slices(X, Y, slice_length, LEN)
LEN = X.shape[0]

scalerX = preprocessing.StandardScaler()
X = scalerX.fit_transform(X)
X = X.reshape(round(LEN/slice_length), slice_length, features)
Y = Y.reshape(round(LEN/slice_length), slice_length, 2)

X_CNN, Y_CNN, X_LSTM, Y_LSTM = shuffle_train_test(X, Y, round(LEN/slice_length), 0.5)

X_CNN = X_CNN.reshape(X_CNN.shape[0]*X_CNN.shape[1], features)
X_LSTM = X_LSTM.reshape(X_LSTM.shape[0]*X_LSTM.shape[1], features)
Y_CNN = Y_CNN.reshape(Y_CNN.shape[0]*Y_CNN.shape[1], 2)
Y_LSTM = Y_LSTM.reshape(Y_LSTM.shape[0]*Y_LSTM.shape[1], 2)

X_CNN = np.delete(X_CNN, [1], axis=1) # 删除轨迹ID
X_LSTM = np.delete(X_LSTM, [1], axis=1)
X_CNN = np.delete(X_CNN, [0], axis=1) # CNN训练时，不需要时间戳
X_LSTM = np.delete(X_LSTM, [0], axis=1) # CNN训练时，不需要时间戳
X_CNN = X_CNN.reshape(X_CNN.shape[0], 6, 5)
X_LSTM = X_LSTM.reshape(X_LSTM.shape[0], 6, 5)
X_LSTM = X_LSTM[:, :, :, np.newaxis]
X_CNN = X_CNN[:, :, :, np.newaxis]

tmp_Y = []
for i in range(Y_CNN.shape[0]):
    tmp_Y.append(coordinate_to_label(Y_CNN[i]))
Y_CNN = np.array(tmp_Y)

adam_cnn = Adam(lr=2e-4, beta_1=0.9, beta_2=0.999, epsilon=None, decay=2e-9, amsgrad=False)
model_cnn = Sequential()

model_cnn.add(Conv2D(128, kernel_size=3, activation='relu', input_shape=(6, 5, 1)))
model_cnn.add(Conv2D(256, kernel_size=3, activation='relu'))
model_cnn.add(Flatten())
model_cnn.add(Dense(512, activation='relu'))
model_cnn.add(Dense(256, activation='relu'))
model_cnn.add(Dense(la_size*lo_size, activation='softmax'))

model_cnn.compile(optimizer=adam_cnn, loss='categorical_crossentropy', metrics=['accuracy'])
model_cnn.fit(X_CNN, Y_CNN, epochs=300, batch_size=64)
cnn_result = model_cnn.predict(X_LSTM)
print(cnn_result.shape) # 4710, 832

# def label_to_coordinate(label_list): # error!
#     label = np.argmax(label_list)
#     la_label = label % la_size
#     lo_label = (label - la_label) / la_size
#     return ((0.5+la_label)*la + min_latitude, (0.5+lo_label)*lo + min_longitude)

# def calcu_distance(true_latitude, true_longitude, pred_latitude, pred_longitude):
#     vector1 = np.array([true_latitude, true_longitude])
#     vector2 = np.array([pred_latitude, pred_longitude])
#     return np.sqrt(np.sum(np.square(vector1 - vector2)))

# error_list = []
# coordinate_result = []
# for i in range(cnn_result.shape[0]):
#     tmp = label_to_coordinate(cnn_result[i])
#     coordinate_result.append(tmp)
#     error = calcu_distance(Y_LSTM[i][0], Y_LSTM[i][1], tmp[0], tmp[1])
#     error_list.append(error)

# coordinate_result = np.array(coordinate_result)
# print(np.median(error_list)) # 这就是中位误差？
# print(np.mean(error_list)) # 这就是中位误差？

# plt.figure()
# plt.scatter(Y_LSTM[:,0], Y_LSTM[:,1], c='blue', s=5)
# plt.scatter(coordinate_result[:,0], coordinate_result[:,1], c='red', s=3)

# plt.show()

lstm_input = cnn_result
lstm_input = lstm_input.reshape(round(lstm_input.shape[0]/slice_length), slice_length, la_size*lo_size)
print(lstm_input.shape)

Y_LSTM = Y_LSTM.reshape(round(Y_LSTM.shape[0]/slice_length), slice_length, 2)
X_Train, Y_Train, X_Test, Y_Test = shuffle_train_test(lstm_input, Y_LSTM, lstm_input.shape[0], 0.1)

Y_Train = Y_Train.reshape(Y_Train.shape[0]*Y_Train.shape[1], 2)
tmp_Y = []
for i in range(Y_Train.shape[0]):
    tmp_Y.append(coordinate_to_label(Y_Train[i]))
Y_Train = np.array(tmp_Y)

Y_Train = Y_Train.reshape(round(Y_Train.shape[0]/slice_length), slice_length, la_size*lo_size)

adam_lstm = Adam(lr=5e-4, beta_1=0.9, beta_2=0.999, epsilon=None, decay=1e-9, amsgrad=False)
model_lstm = Sequential()

model_lstm.add(LSTM(slice_length*20, input_shape=(X_Train.shape[1], X_Train.shape[2]), return_sequences=True))
model_lstm.add(TimeDistributed(Dense(2*la_size*lo_size, activation='relu')))
model_lstm.add(TimeDistributed(Dense(la_size*lo_size, activation='softmax')))
model_lstm.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=adam_lstm)
model_lstm.fit(X_Train, Y_Train, epochs=100, batch_size=6)
lstm_result = model_lstm.predict(X_Test)
print(lstm_result.shape)
print(Y_Test.shape)

lstm_result = lstm_result.reshape(lstm_result.shape[0]*slice_length, lo_size*la_size)
Y_Test = Y_Test.reshape(Y_Test.shape[0]*slice_length, 2)

def label_to_coordinate(label_list): # error!
    label = np.argmax(label_list)
    # print(label)
    la_label = label % la_size
    lo_label = (label - la_label) / la_size
    return ((0.5+la_label)*la + min_latitude, (0.5+lo_label)*lo + min_longitude)

def calcu_distance(true_latitude, true_longitude, pred_latitude, pred_longitude):
    vector1 = np.array([true_latitude, true_longitude])
    vector2 = np.array([pred_latitude, pred_longitude])
    return np.sqrt(np.sum(np.square(vector1 - vector2)))

error_list = []
coordinate_result = []
error_dict = {
    '0': 0,
    '1': 0,
    '2': 0,
    '3': 0,
    '4': 0
}
for i in range(Y_Test.shape[0]):
    tmp = label_to_coordinate(lstm_result[i])
    coordinate_result.append(tmp)
    error = calcu_distance(Y_Test[i][0], Y_Test[i][1], tmp[0], tmp[1])
    if error <= 20:
        error_dict['0'] += 1
    elif error <= 40:
        error_dict['1'] += 1
    elif error <= 60:
        error_dict['2'] += 1
    elif error <= 80:
        error_dict['3'] += 1
    elif error <= 100:
        error_dict['4'] += 1
    error_list.append(error)

coordinate_result = np.array(coordinate_result)
error_list.sort()
print(np.median(error_list)) # 这就是中位误差？
print(np.mean(error_list)) # 这就是中位误差？
print(error_list[int(len(error_list)*0.9)]) # 90%误差

error_ratio = []
for i in range(5):
    if i > 0:
        error_dict[str(i)] += error_dict[str(i-1)]
    error_ratio.append(error_dict[str(i)]/len(error_list))

plt.figure('figure 1')
plt.scatter(Y_Test[:,0], Y_Test[:,1], c='blue', s=5)
plt.scatter(coordinate_result[:,0], coordinate_result[:,1], c='red', s=3)

plt.figure('figure 2')
plt.plot([20, 40, 60, 80, 100], error_ratio, marker='o')

plt.show()


from typing import List
from tensorflow import keras
from sklearn.model_selection import train_test_split
import librosa
import numpy as np
from tqdm.notebook import tqdm
import os
import data_augmentation
import random

RATE = 8000

def load_recordings(paths=["recordings"], label_type="number", sr=RATE):
    """
    Load the recordings in the given directories
    :param paths: List containing the path(s) where audio files are stored
    :param label_type: whether the audio tracks will be used for number or speaker recognition
    :param sr: Sample Rate of the given tracks. Default is 8000
    :return:
    """
    res = []
    for path in paths:
        print(f"Loading from {path}")
        for f in tqdm(sorted(os.listdir(path))):
            if f.endswith('.wav'):
                if "pitch" in f and label_type == "speaker":
                    # do not consider pitch alteration
                    next
                else:
                    # Load file and extract features
                    audio, sample_rate = librosa.load(path + "/" + f, sr=sr)
                    res.append(audio)
    return np.array(res)

def mfcc(track, rate=8000, sampling=1, n_mfcc=20, flatten=True):
    """
    Compute MFCC of the given track
    :param track: input audio
    :param rate: sampling rate
    :param min_len: minimum length of the resulting mfcc
    :param sampling:
    :param n_mfcc: number of mfcc to include
    :param flatten: whether to flatten the output mfcc or not (useful for SVM)
    :return:
    """
    # Campiona i valori
    signal = track[::sampling]
    # Calcola coefficienti MFCC
    mfcc_coefs = librosa.feature.mfcc(signal * 1.0, sr=int(rate / sampling), n_mfcc=n_mfcc)
    if flatten:
        # Appiattisci rappresentazione per uso con SVM
        mfcc_coefs = mfcc_coefs.flatten()
    return mfcc_coefs

def load_labels(paths=["recordings"], label_type="number"):
    """
    Load labels (a.k.a Y) of the recordings in the given inputs
    :param paths: List containing the path(s) where audio files are stored
    :param label_type: whether the audio tracks will be used for number or speaker recognition
    :return:
    """
    labels = []
    for path in paths:
        for f in sorted(os.listdir(path)):
            if f.endswith('.wav'):
                if "pitch" in f and label_type == "speaker":
                    next
                else:
                    if label_type.startswith("n"):
                        label = f.split('_')[0]
                    else:
                        label = f.split('_')[1]
                    labels.append(label)
    return labels

def pad_zeros(recordings, compute_max_rec_length=True, max_rec_length=0):
    """
    Add zeros, at the beginning and at the end, of the given recordings
    :param recordings: List of recordings to preprocess
    :param compute_max_rec_length:
    :param max_rec_length:
    :return:
    """
    if compute_max_rec_length:
        max_rec_length = max(map(np.shape, recordings))[0]
    res = [padding(max_rec_length, rec) for rec in recordings]
    return np.array(res)

def padding(max_rec_length, rec):
    """
    Add zeros at the start and end of the given recording if length(recording) < max_rec_length
    :param max_rec_length: length that all recordings must have
    :param rec: current recording
    :return:
    """
    diff_in_rec_length = max_rec_length - rec.shape[0]
    if diff_in_rec_length > 0:
        half_diff = int(diff_in_rec_length / 2)
        remaining_diff = diff_in_rec_length - half_diff
        v = np.pad(rec, (half_diff, remaining_diff), 'constant', constant_values=0)
        return v
    else:
        return rec

def compute_spectrogram(audio, rate=8000, normalize=False, paper_data=False):
    """
    Compute spectrogram of the given recording
    :param audio: Input audio track
    :param rate: sampling rate of the input audio track
    :param normalize: whether to apply dynamic range compression of the spectrograms or not
    :param paper_data: whether to trasform the input recordings according to the paper specs
    :return:
    """
    if paper_data:
        spectrogram = librosa.feature.melspectrogram(y=np.array(audio),
                                                     sr=rate,
                                                     n_fft=1024,
                                                     hop_length=160)
    else:
        spectrogram = librosa.feature.melspectrogram(y=np.array(audio),
                                                     sr=rate)
    if normalize:
        spectrogram = np.log10(1000 * spectrogram + 1)
    return spectrogram

def prepare_data_nn(X_train, X_val, X_test, y_train, y_val, y_test, number_mode):
    """
    Transform recordings, in MFCC or spectrograms form, and their label for NN training
    :param X_train:
    :param X_val:
    :param X_test:
    :param y_train:
    :param y_val:
    :param y_test:
    :param number_mode:
    :return:
    """
    # Change shape of X for model training purpose
    X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2], 1)
    X_val = X_val.reshape(X_val.shape[0], X_val.shape[1], X_val.shape[2], 1)
    X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2], 1)
    input_shape = (X_train.shape[1], X_train.shape[2], 1)
    target_names = []
    if number_mode:
        y_train = keras.utils.to_categorical(y_train, 10)
        y_val = keras.utils.to_categorical(y_val, 10)
        y_test = keras.utils.to_categorical(y_test, 10)
    else:
        enc, y_train, target_names = transform_categorical_y(y_train)
        y_val = enc.fit_transform(np.array(y_val).reshape(-1, 1)).toarray()
        y_test = enc.fit_transform(np.array(y_test).reshape(-1, 1)).toarray()
    X_data = [np.array(X_train), np.array(X_val), np.array(X_test)]
    y_data = [np.array(y_train), np.array(y_val), np.array(y_test)]
    return X_data, y_data, input_shape, target_names

def transform_categorical_y(labels):
    """
    Perform one hot encoding and return transformed y and labels for building the classification report
    :param labels: Original labels
    :return:
    """
    from sklearn.preprocessing import OneHotEncoder
    enc = OneHotEncoder()
    y = enc.fit_transform(np.array(labels).reshape(-1, 1)).toarray()
    label_0 = enc.inverse_transform(np.array([1, 0, 0, 0, 0, 0, 0, 0]).reshape(1, -1))[0][0]
    label_1 = enc.inverse_transform(np.array([0, 1, 0, 0, 0, 0, 0, 0]).reshape(1, -1))[0][0]
    label_2 = enc.inverse_transform(np.array([0, 0, 1, 0, 0, 0, 0, 0]).reshape(1, -1))[0][0]
    label_3 = enc.inverse_transform(np.array([0, 0, 0, 1, 0, 0, 0, 0]).reshape(1, -1))[0][0]
    label_4 = enc.inverse_transform(np.array([0, 0, 0, 0, 1, 0, 0, 0]).reshape(1, -1))[0][0]
    label_5 = enc.inverse_transform(np.array([0, 0, 0, 0, 0, 1, 0, 0]).reshape(1, -1))[0][0]
    label_6 = enc.inverse_transform(np.array([0, 0, 0, 0, 0, 0, 1, 0]).reshape(1, -1))[0][0]
    label_7 = enc.inverse_transform(np.array([0, 0, 0, 0, 0, 0, 0, 1]).reshape(1, -1))[0][0]
    target_names = [label_0, label_1, label_2, label_3, label_4, label_5, label_6, label_7]
    return enc, y, target_names

def get_pattern_indexes(lst: List, pattern: str, split_index: int):
    """
    Return the indexes of elements, in the given list, that contain the given pattern, surrounded by _
    :param lst: List with elements of interest
    :param pattern: pattern we are interested in finding
    :param split_index: which element, after splitting by _ , is of interest
    :return:
    """
    occurrences = [i for i, x in enumerate(lst) if pattern in "_" + x.split('_')[split_index] + "_"]
    return occurrences

def split_and_augment_dataset(audio_dir: str,
                              y_type: str,
                              n_category_audio_to_pick_test: int,
                              include_pitch: bool,
                              max_length: int,
                              recordings_made_by_us: bool):
    """
    Augment and split in train, validation and test the given recordings
    :param audio_dir: path where the recordings of interest are stored
    :param y_type: whether we are interest in speakers or digits
    :param n_category_audio_to_pick_test: how many sample, for each unique Y value, should be put in the test set
    :param include_pitch: whether to include audio with modified pitch or not
    :param max_length: maximum length a given recording should have in order to be included
    :param recordings_made_by_us: whether the recs are made by us or not
    :return:
    """
    print("split_and_augment_dataset >>>")
    n_noise = 5
    n_pitch = 0
    if y_type == "speakers_us":
        categories = ['_gian_', '_alinda_', '_khaled_', '_ale_']
        # Used later on for getting y label
        split_index = 1
    elif y_type == "speakers_default":
        categories = ['jackson', 'nicolas', 'theo', 'yweweler']
        split_index = 1
    else:
        categories = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
        split_index = 0
        n_pitch = 5
    augmented_tracks = data_augmentation.enrich_dataset(audio_dir,
                                                        mode="normal",
                                                        n_noise=n_noise,
                                                        n_pitch=n_pitch,
                                                        recordings_made_by_us=recordings_made_by_us,
                                                        max_length=max_length)

    all_keys = [k for k in augmented_tracks.keys()]
    test_labels = []
    # Get the list of recordings name that will compose our test set
    for c in categories:
        indexes_of_c = get_pattern_indexes(all_keys, c, split_index)
        records_to_pick = random.sample(indexes_of_c, n_category_audio_to_pick_test)
        current_test_categories = [all_keys[i] for i in records_to_pick]
        test_labels = test_labels + current_test_categories
    # Get the recordings for the test set
    test_recordings = []
    for k in test_labels:
        # store original recording
        current_key = augmented_tracks[k]
        test_recordings.append(current_key['original'][0])
        # eliminate the original + augmented tracks from the dataset
        del augmented_tracks[k]
    train_recordings = []
    train_labels = []
    for k in augmented_tracks.keys():
        # Original track
        train_labels.append(k)
        train_recordings.append(augmented_tracks[k]['original'][0])
        # Noise track
        noise_recordings = augmented_tracks[k]['noise']
        noise_labels = len(noise_recordings) * [k]
        train_labels = train_labels + noise_labels
        train_recordings = train_recordings + noise_recordings
        # Pitch tracks
        if include_pitch:
            pitch_recordings = augmented_tracks[k]['pitch']
            pitch_labels = len(pitch_recordings) * [k]
            train_labels = train_labels + pitch_labels
            train_recordings = train_recordings + pitch_recordings
    # Get final label. The file format is number_speaker_n.wav
    train_labels = [label.split('_')[split_index] for label in train_labels]
    test_labels = [label.split('_')[split_index] for label in test_labels]
    train_recordings, val_recordings, train_labels, val_labels = train_test_split(train_recordings,
                                                                                train_labels,
                                                                                test_size=0.2,
                                                                                random_state=1)
    print("split_and_augment_dataset <<<")
    return train_recordings, train_labels, val_recordings, val_labels, test_recordings, test_labels

def prepare_augmented_recordings(audio_dirs: List[str],
                                 y_type: List[str],
                                 n_category_test: int,
                                 include_pitch: bool,
                                 max_length: int,
                                 recordings_source: List[bool],
                                 transform_function="spectrogram",
                                 paper_data=False):
    """
    Augment, split in train-val-test and compute spectrograms of the given recordings
    :param audio_dirs: list of path where the recordings of interest are stored
    :param y_type: category (digits, baseline speakers and "us" speakers) of each directory
    :param n_category_test: how many sample, for each unique Y value, should be put in the test set
    :param include_pitch: whether to include audio with modified pitch or not
    :param max_length: maximum length a given recording should have in order to be included
    :param recordings_source: whether the current recordings are made by us(useful for setting data augmentation params)
    :param transform_function: whether to transform recordings using MFCC or spectrograms
    :param paper_data: whether to transform the data as specified in the paper or not
    :return:
    """
    X_train = []
    y_train = []
    X_val = []
    y_val = []
    X_test = []
    y_test = []
    for i, dir_path in enumerate(audio_dirs):
        train_recordings, train_labels, val_recordings, val_labels, test_recordings, test_labels = split_and_augment_dataset(
            dir_path,
            y_type[i],
            n_category_test,
            include_pitch,
            max_length,
            recordings_source[i])
        X_train = X_train + train_recordings
        y_train = y_train + train_labels
        X_val = X_val + val_recordings
        y_val = y_val + val_labels
        X_test = X_test + test_recordings
        y_test = y_test + test_labels
    X_train = [np.array(x) for x in X_train]
    y_train = [np.array(x) for x in y_train]
    X_val = [np.array(x) for x in X_val]
    y_val = [np.array(x) for x in y_val]
    X_test = [np.array(x) for x in X_test]
    y_test = [np.array(x) for x in y_test]
    X_train, X_val, X_test = transform_recordings(X_train, X_val, X_test, transform_function, paper_data)
    X_data = [np.array(X_train), np.array(X_val), np.array(X_test)]
    y_data = [np.array(y_train), np.array(y_val), np.array(y_test)]
    return X_data, y_data

def transform_recordings(X_train, X_val, X_test, transform_function, paper_data):
    """
    Normalize through padding and compute the spectrograms of train, validation and test recordings
    :param X_train: train recordings
    :param X_val: validation recordings
    :param X_test: test recordings
    :param transform_function: whether to apply spectrogram or mfcc
    :param paper_data: whether to transform the recordings as specified in the paper or not
    :return:
    """
    print("transform_recordings >>>")
    # In order to normalise the length of recordings we have to define the maximum length of the various recordings
    max_length_rec = max(map(np.shape, X_train + X_val + X_test))[0]
    X_train = pad_zeros(X_train, compute_max_rec_length=False, max_rec_length=max_length_rec)
    X_val = pad_zeros(X_val, compute_max_rec_length=False, max_rec_length=max_length_rec)
    X_test = pad_zeros(X_test, compute_max_rec_length=False, max_rec_length=max_length_rec)
    # Now let's transform our recordings the spectrograms
    if transform_function == "spectrogram":
        X_train = [compute_spectrogram(x, normalize=True, paper_data=paper_data) for x in X_train]
        X_val = [compute_spectrogram(x, normalize=True, paper_data=paper_data) for x in X_val]
        X_test = [compute_spectrogram(x, normalize=True, paper_data=paper_data) for x in X_test]
    else:
        X_train = [mfcc(x, flatten=False) for x in X_train]
        X_val = [mfcc(x, flatten=False) for x in X_val]
        X_test = [mfcc(x, flatten=False) for x in X_test]
    print("transform_recordings <<<")
    return X_train, X_val, X_test

def balanced_train_val_test_split(X, y, train_size=0.6):
    """
    Split the current X and y in balanced (all labels have the same n of recordings) train, val and test set
    :param X:
    :param y:
    :param train_size:
    :return:
    """
    X_train = []
    X_val = []
    X_test = []
    y_train = []
    y_val = []
    y_test = []
    # Find out unique values and their occurences
    unique, counts = np.unique(y, return_counts=True)
    # Occurences of the least frequent class
    min_len = np.min(counts)
    # How many samples should train, val and test have:
    train_freq = int(min_len * train_size)
    val_freq = (min_len - train_freq) // 2
    test_freq = min_len - train_freq - val_freq
    for c in unique:
        current_indexes = np.where(y == c)[0]
        np.random.shuffle(current_indexes)
        train_indexes = current_indexes[0:train_freq]
        val_indexes = current_indexes[train_freq:train_freq + val_freq]
        test_indexes = current_indexes[train_freq + val_freq: train_freq + val_freq + test_freq]
        X_train = X_train + [X[i] for i in train_indexes]
        y_train = y_train + [y[i] for i in train_indexes]
        X_val = X_val + [X[i] for i in val_indexes]
        y_val = y_val + [y[i] for i in val_indexes]
        X_test = X_test + [X[i] for i in test_indexes]
        y_test = y_test + [y[i] for i in test_indexes]
    X_data = [np.array(X_train), np.array(X_val), np.array(X_test)]
    y_data = [np.array(y_train), np.array(y_val), np.array(y_test)]
    return X_data, y_data

def balanced_train_val_split(X, y, train_size=0.75):
    """
    Split the current X and y in balanced (all labels have the same n of recordings) train and validation set
    :param X:
    :param y:
    :param train_size: how big should the train set be, compared to the validation set
    :return:
    """
    X_train = []
    X_val = []
    y_val = []
    y_train = []
    # Find out unique values and their occurences
    unique, counts = np.unique(y, return_counts=True)
    # Occurences of the least frequent class
    min_len = np.min(counts)
    # How many samples should train, val and test have:
    train_freq = int(min_len * train_size)
    val_freq = min_len - train_freq
    print(train_freq, val_freq)
    for c in unique:
        print(c)
        current_indexes = np.where(y == c)[0]
        np.random.shuffle(current_indexes)
        train_indexes = current_indexes[0:train_freq]
        val_indexes = current_indexes[train_freq:train_freq + val_freq]
        X_train = X_train + [X[i] for i in train_indexes]
        y_train = y_train + [y[i] for i in train_indexes]
        X_val = X_val + [X[i] for i in val_indexes]
        y_val = y_val + [y[i] for i in val_indexes]
    X_data = [np.array(X_train), np.array(X_val)]
    y_data = [np.array(y_train), np.array(y_val)]
    return X_data, y_data


from typing import Optional, TYPE_CHECKING

from django.db import models

from django.contrib.auth.models import Group, User

from rest_framework import serializers
from rest_framework.exceptions import NotFound

from processes.exception import UnprocessableEntity

from .uuid_model import UuidModel

if TYPE_CHECKING:
    from .run_environment import RunEnvironment

class NamedWithUuidModel(UuidModel):
    class Meta:
        abstract = True

    name = models.CharField(max_length=200)
    description = models.CharField(max_length=5000, blank=True)
    created_by_user = models.ForeignKey(User, on_delete=models.SET_NULL,
            null=True, editable=False)
    created_by_group = models.ForeignKey(Group, on_delete=models.CASCADE,
            editable=False)

    @classmethod
    def find_by_uuid_or_name(cls, obj_dict,
            required_group: Optional[Group] = None,
            required_run_environment: 'Optional[RunEnvironment]' = None,
            check_conflict: bool = True,
            allowed_run_environment: 'Optional[RunEnvironment]' = None,
            allow_any_run_environment: Optional[bool] = None):
        uuid = obj_dict.get('uuid')
        name = obj_dict.get('name')

        if uuid is not None:
            entity = cls.objects.get(uuid=uuid)

            if check_conflict and (name is not None) and (entity.name != name):
                raise UnprocessableEntity(
                        f"{cls.__name__} {uuid} is named '{entity.name}', not '{name}'")
        else:
            if name is None:
                raise serializers.ValidationError('Neither uuid or name found in request')

            entity = cls.objects.get(name=name, created_by_group=required_group)

        if required_group and (entity.created_by_group != required_group):
            raise NotFound()

        if required_run_environment and (
                entity.run_environment != required_run_environment):
            raise NotFound()

        # So that allowed_run_environment can be omitted if
        # required_run_environment is set
        allowed_run_environment = allowed_run_environment or \
                required_run_environment

        if allow_any_run_environment is None:
            allow_any_run_environment = (allowed_run_environment is None)

        if (not allow_any_run_environment) and \
                hasattr(entity, 'run_environment') and \
                entity.run_environment:

            if allowed_run_environment:
                if entity.run_environment != allowed_run_environment:
                    raise NotFound()
            else:
                raise NotFound()

        return entity

    def __str__(self) -> str:
        return (self.name or 'Unnamed') + ' / ' + str(self.uuid)


from django.shortcuts import render
from operator import attrgetter

# Para a paginação
from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage, InvalidPage

# Importamos a função criada anteriormente
from post.views import get_post_queryset

POSTS_PER_PAGE = 2  # Pode ser qualquer valor acima de 0, coloquei 1 pois não tenho muitas postagens cadastradas
LIMIT_OF_PAGES = 10

def home_view(request):
    context = {}

    # inicialmente inicializamos a query com texto vazio
    query = ''

    # Caso no request.GET não sejá vazio, no caso seja diferente de <QueryDict: {}>
    # Lembre que qualquer ideia de vazio é considerado falso pelo Django
    if request.GET:
        # Coleta-se a query
        query = request.GET.get('q', '')  # Tenta pegar valor de 'q' caso não tenha retorna ''
        # Envia o valor da query para o template, para se manter a query da busca no search bar
        context['query'] = query

    # Faz a busca dos posts usando a query pra filtrar
    posts = get_post_queryset(query)
    # Ordena as postagens
    posts = sorted(posts, key=attrgetter('date_published'), reverse=True)

    # Pagination
    # Tenta pegar o valor de page do template, caso não encontre retorna o valor 1 que representa a primeira página
    page = request.GET.get('page', 1)
    # Cria-se um paginator, para isso se passa uma lista de objetos e a quantidade de páginas
    posts_paginator = Paginator(posts, POSTS_PER_PAGE)

    # Vai fazer toda a lógica de separação e navegação entre páginas, e já fazendo o tratamento das exceptions
    try:
        # Tenta acessar a página que foi passada pelo template, caso dê erro vai para as exceptions
        posts = posts_paginator.page(page)
    except InvalidPage:
        posts = posts_paginator.page(1)

    # Passa as postagens para o context do template
    context['posts'] = posts
    context['limit_of_pages'] = LIMIT_OF_PAGES
    return render(request, 'personal/home_view.html', context)


import asyncio
import atexit
import dataclasses
import multiprocessing
import time
from typing import Any, Dict

from aiogram import Bot, Dispatcher
from aiogram.contrib.fsm_storage.memory import MemoryStorage
from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters import Text
from aiogram.dispatcher.filters.state import State, StatesGroup
from aiogram.types import BotCommand, InlineKeyboardButton, InlineKeyboardMarkup, CallbackQuery, Message
from aiogram.utils import executor
from aiogram.utils.exceptions import MessageToForwardNotFound, ChatNotFound
from aiogram.contrib.middlewares.i18n import I18nMiddleware

from tg_filtering_bot.async_queue import AsyncQueue
from tg_filtering_bot.config import settings
from tg_filtering_bot.crud.crud import (
    create_or_update_user_chat,
    add_filter,
    get_user_filters,
    disable_filter
)
from tg_filtering_bot.crud.dto import UserFilterDTO, UserChatDTO, ForwardMessageDTO
from tg_filtering_bot.logger import get_logger

I18N = I18nMiddleware(settings.I18N_DOMAIN, settings.LOCALES_DIR)
_ = I18N.gettext

class UserState(StatesGroup):
    adding = State()
    deleting = State()

@dataclasses.dataclass
class DeleteFilterData:
    order: int
    button: InlineKeyboardButton
    filter_: UserFilterDTO

def build_delete_filters_markup(
    storage_data: Dict[str, DeleteFilterData]
) -> InlineKeyboardMarkup:
    inline_kb = InlineKeyboardMarkup()

    delete_filters_data = [d for d in storage_data.values() if isinstance(d, DeleteFilterData)]
    for data in sorted(delete_filters_data, key=lambda d: d.order):
        inline_kb.add(data.button)
    return inline_kb

class FilteringBot:
    LOGGER = get_logger("FilteringBot")
    _BTN_DELETE_PREFIX = "delete_"

    def __init__(self, message_queue: AsyncQueue) -> None:
        self._message_queue = message_queue

        self.storage = MemoryStorage()
        self.bot = Bot(settings.BOT_TOKEN)
        self.dispatcher = Dispatcher(self.bot, storage=self.storage)
        self.dispatcher.middleware.setup(I18N)

        self.LOGGER.info("Bot created")

        self.dispatcher.register_message_handler(
            self.start_command,
            commands=['start'],
            state="*"
        )

        self.dispatcher.register_message_handler(
            self.add_command,
            commands=['add'],
            state="*"
        )

        self.dispatcher.register_message_handler(
            self.delete_command,
            commands=['delete'],
            state="*"
        )

        self.dispatcher.register_message_handler(
            self.list_command,
            commands=['list'],
            state="*"
        ),

        self.dispatcher.register_message_handler(
            self.cancel_command,
            commands=['cancel'],
            state="*"
        ),
        self.dispatcher.register_message_handler(
            self.add_address,
            state=UserState.adding
        ),

        self.dispatcher.register_callback_query_handler(
            self.delete_address,
            Text(startswith=self._BTN_DELETE_PREFIX),
            state=UserState.deleting
        )

    async def _set_base_commands(self) -> None:
        await self.bot.delete_my_commands()
        await self.bot.set_my_commands(commands=[
            BotCommand(
                command="add",
                description=_("Add new address")
            ),
            BotCommand(
                command="list",
                description=_("List addresses")
            ),
            BotCommand(
                command="delete",
                description=_("Delete address")
            ),
            BotCommand(
                command="cancel",
                description=_("Cancel previous command")
            ),
        ])

    async def _register_user(self, message: Any) -> None:
        if message:
            self.LOGGER.info("Adding or updating user %s, chat %s", message.from_user.id, message.chat.id)

            user_chat_dto = UserChatDTO(
                user_id=message.from_user.id,
                chat_id=message.chat.id,
                name=message.from_user.full_name,
                username=message.from_user.username,
                language_code=message.from_user.language_code
            )
            await create_or_update_user_chat(user_chat_dto)

    async def _process_message_queue(self) -> None:
        message = await self._message_queue.get()
        if isinstance(message, ForwardMessageDTO):
            return await self.forward_message(message)

        raise NotImplementedError(f"Processing for {message} is not implemented")

    async def _periodic(self) -> None:
        while True:
            try:
                await self._process_message_queue()
            except Exception as e:
                self.LOGGER.exception(e)

    def start_bot(self) -> None:
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        loop.create_task(self._set_base_commands())
        loop.create_task(self._periodic())
        executor.start_polling(self.dispatcher, loop=loop)

    def stop_bot(self) -> None:
        self.dispatcher.stop_polling()

    async def _clear_state(self, state: FSMContext) -> None:
        await state.reset_state()
        await state.reset_data()
        await self._set_base_commands()

    async def start_command(self, message: Message, state: FSMContext) -> None:
        await self._register_user(message)
        await self._clear_state(state)

        await message.reply(_("""Hi there,
I am a filtering bot for '{channel}' channel.
Please use command {command} to add a new address.""").format(
            channel=settings.LISTENER_CHANNEL_NAME, command="/add"
        ))

    async def add_command(self, message: Message, state: FSMContext) -> None:
        await self._register_user(message)

        await state.set_state(UserState.adding)
        await self.bot.send_message(message.chat.id, _("Please type street name to create a filter"))

    async def delete_command(self, message: Message, state: FSMContext) -> None:
        await self._register_user(message)

        filters = await get_user_filters(user_id=message.from_user.id)

        if not filters:
            return await message.reply(_("You have no addresses to delete"))

        await state.set_state(UserState.deleting)

        storage_data = {}

        for i, f in enumerate(filters, start=1):
            key = f"{self._BTN_DELETE_PREFIX}{f.filter_id}"
            storage_data[key] = DeleteFilterData(
                order=i,
                filter_=f,
                button=InlineKeyboardButton(
                    _("{i}. Delete {address}").format(i=i, address=f.filter_),
                    callback_data=key
                )
            )
        await state.set_data(storage_data)
        inline_kb = build_delete_filters_markup(storage_data)
        await message.reply(_("Click addresses to delete"), reply_markup=inline_kb)

    async def list_command(self, message: Message, state: FSMContext) -> Message:
        await self._register_user(message)

        filters = await get_user_filters(user_id=message.from_user.id)
        if not filters:
            return await message.reply(_("You have no addresses"))

        response = _("You addresses are:") + "\r\n" + "\r\n".join(
            f"{i}. {f.filter_}" for i, f in enumerate(filters, start=1)
        )

        return await message.reply(response)

    async def cancel_command(self, message: Message, state: FSMContext) -> None:
        await self._register_user(message)
        await self._clear_state(state)

    async def add_address(self, message: Message, state: FSMContext) -> None:
        await self._register_user(message)

        address = message.text
        self.LOGGER.info("Adding new address: %s", message)

        user_filter = UserFilterDTO(user_id=message.from_user.id, filter_=address, filter_id=-1)
        await add_filter(user_filter)

#        await self.bot.delete_state(message.from_user.id, message.chat.id)
        await self.bot.send_message(
            message.chat.id, _("Filter '{address}' was added for monitoring").format(address=address)
        )
        await self._clear_state(state)

    async def delete_address(self, event: CallbackQuery, state: FSMContext) -> Message:
        self.LOGGER.warning(event)

        storage_data = await state.get_data()

        if not storage_data or event.data not in storage_data:
            return await event.message.answer(_("No address to delete"))

        delete_filter_data = storage_data.pop(event.data)
        await disable_filter(delete_filter_data.filter_)

        await state.set_data(storage_data)

        await event.message.edit_reply_markup(build_delete_filters_markup(storage_data))
        return await event.message.answer(
            _("Filter '{address}' was deleted").format(address=delete_filter_data.filter_.filter_)
        )

    async def send_message_to_user(self, forward_message: ForwardMessageDTO) -> None:
        self.LOGGER.debug(
            "Sending message %s to user %s", forward_message.message.message_id, forward_message.user_chat.user_id
        )

        await self.bot.send_message(
            forward_message.user_chat.chat_id,
            forward_message.message.message
        )

    async def forward_message(self, forward_message: ForwardMessageDTO) -> None:
        self.LOGGER.debug(
            "Forwarding message %s to user %s", forward_message.message.message_id, forward_message.user_chat.user_id
        )

        try:
            await self.bot.forward_message(
                forward_message.user_chat.chat_id,
                from_chat_id=forward_message.message.channel_id,
                message_id=forward_message.message.message_id
            )
        except (MessageToForwardNotFound, ChatNotFound):
            self.LOGGER.warning(
                "Cannot forward message %s", forward_message.message.message_id
            )
            return await self.send_message_to_user(forward_message)

def start_filtering_bot() -> AsyncQueue[ForwardMessageDTO]:
    queue: multiprocessing.Queue[ForwardMessageDTO] = multiprocessing.Queue(maxsize=settings.QUEUE_SIZE)
    message_queue = AsyncQueue(queue)

    bot = FilteringBot(
        message_queue=message_queue
    )

    process = multiprocessing.Process(target=bot.start_bot)
    process.start()
    atexit.register(bot.stop_bot)
    atexit.register(process.join)

    time.sleep(1)
    if process.exitcode is not None:
        raise RuntimeError("FilteringBot closed after launch")

    return message_queue


from selenium import webdriver
import time
from selenium.webdriver.common.by import By

class clickSendkey():

    def test(self):
        driver = webdriver.Firefox(executable_path='/Users/Jatin Singh/Downloads/geckodriver')
        driver.get('https://google.com/')
        driver.maximize_window()

        texttype = driver.find_element_by_xpath("//input[@type='text']")
        texttype.send_keys('letskodeitselenium')
        time.sleep(3)

        search = driver.find_element_by_xpath("//div[@class='FPdoLc tfB0Bf']//input[@class='gNO89b']")
        search.click()
        time.sleep(3)

        driver.close()

o = clickSendkey()
o.test()


from abc import ABC, abstractmethod
from dataclasses import dataclass
from functools import partial
from typing import Any, Dict, List, Optional, Tuple

import torch

from shap_e.models.nn.utils import sample_pmf
from shap_e.models.volume import Volume, VolumeRange
from shap_e.util.collections import AttrDict

from .model import NeRFModel, Query

def render_rays(
    rays: torch.Tensor,
    parts: List["RayVolumeIntegral"],
    void_model: NeRFModel,
    shared: bool = False,
    prev_raw_outputs: Optional[List[AttrDict]] = None,
    render_with_direction: bool = True,
    importance_sampling_options: Optional[Dict[str, Any]] = None,
) -> Tuple["RayVolumeIntegralResults", List["RaySampler"], List[AttrDict]]:
    """
    Perform volumetric rendering over a partition of possible t's in the union
    of rendering volumes (written below with some abuse of notations)

        C(r) := sum(
            transmittance(t[i]) *
            integrate(
                lambda t: density(t) * channels(t) * transmittance(t),
                [t[i], t[i + 1]],
            )
            for i in range(len(parts))
        ) + transmittance(t[-1]) * void_model(t[-1]).channels

    where

    1) transmittance(s) := exp(-integrate(density, [t[0], s])) calculates the
       probability of light passing through the volume specified by [t[0], s].
       (transmittance of 1 means light can pass freely)
    2) density and channels are obtained by evaluating the appropriate
       part.model at time t.
    3) [t[i], t[i + 1]] is defined as the range of t where the ray intersects
       (parts[i].volume \\ union(part.volume for part in parts[:i])) at the surface
       of the shell (if bounded). If the ray does not intersect, the integral over
       this segment is evaluated as 0 and transmittance(t[i + 1]) :=
       transmittance(t[i]).
    4) The last term is integration to infinity (e.g. [t[-1], math.inf]) that
       is evaluated by the void_model (i.e. we consider this space to be empty).

    :param rays: [batch_size x ... x 2 x 3] origin and direction.
    :param parts: disjoint volume integrals.
    :param void_model: use this model to integrate over the empty space
    :param shared: All RayVolumeIntegrals are calculated with the same model.
    :param prev_raw_outputs: Raw outputs from the previous rendering step

    :return: A tuple of
        - AttrDict containing the rendered `channels`, `distances`, and the `aux_losses`
        - A list of importance samplers for additional fine-grained rendering
        - A list of raw output for each interval
    """
    if importance_sampling_options is None:
        importance_sampling_options = {}

    origin, direc = rays[..., 0, :], rays[..., 1, :]

    if prev_raw_outputs is None:
        prev_raw_outputs = [None] * len(parts)

    samplers = []
    raw_outputs = []
    t0 = None
    results = None

    for part_i, prev_raw_i in zip(parts, prev_raw_outputs):

        # Integrate over [t[i], t[i + 1]]
        results_i = part_i.render_rays(
            origin,
            direc,
            t0=t0,
            prev_raw=prev_raw_i,
            shared=shared,
            render_with_direction=render_with_direction,
        )

        # Create an importance sampler for (optional) fine rendering
        samplers.append(
            ImportanceRaySampler(
                results_i.volume_range, results_i.raw, **importance_sampling_options
            )
        )
        raw_outputs.append(results_i.raw)

        # Pass t[i + 1] as the start of integration for the next interval.
        t0 = results_i.volume_range.next_t0()

        # Combine the results from [t[0], t[i]] and [t[i], t[i+1]]
        results = results_i if results is None else results.combine(results_i)

    # While integrating out [t[-1], math.inf] is the correct thing to do, this
    # erases a lot of useful information. Also, void_model is meant to predict
    # the channels at t=math.inf.

    # # Add the void background over [t[-1], math.inf] to complete integration.
    # results = results.combine(
    #     RayVolumeIntegralResults(
    #         output=AttrDict(
    #             channels=void_model(origin, direc),
    #             distances=torch.zeros_like(t0),
    #             aux_losses=AttrDict(),
    #         ),
    #         volume_range=VolumeRange(
    #             t0=t0,
    #             t1=torch.full_like(t0, math.inf),
    #             intersected=torch.full_like(results.volume_range.intersected, True),
    #         ),
    #         # Void space extends to infinity. It is assumed that no light
    #         # passes beyond the void.
    #         transmittance=torch.zeros_like(results_i.transmittance),
    #     )
    # )

    results.output.channels = results.output.channels + results.transmittance * void_model(
        Query(origin, direc)
    )

    return results, samplers, raw_outputs

@dataclass
class RayVolumeIntegralResults:
    """
    Stores the relevant state and results of

        integrate(
            lambda t: density(t) * channels(t) * transmittance(t),
            [t0, t1],
        )
    """

    # Rendered output and auxiliary losses
    # output.channels has shape [batch_size, *inner_shape, n_channels]
    output: AttrDict

    """
    Optional values
    """

    # Raw values contain the sampled `ts`, `density`, `channels`, etc.
    raw: Optional[AttrDict] = None

    # Integration
    volume_range: Optional[VolumeRange] = None

    # If a ray intersects, the transmittance from t0 to t1 (e.g. the
    # probability that the ray passes through this volume).
    # has shape [batch_size, *inner_shape, 1]
    transmittance: Optional[torch.Tensor] = None

    def combine(self, cur: "RayVolumeIntegralResults") -> "RayVolumeIntegralResults":
        """
        Combines the integration results of `self` over [t0, t1] and
        `cur` over [t1, t2] to produce a new set of results over [t0, t2] by
        using a similar equation to (4) in NeRF++:

            integrate(
                lambda t: density(t) * channels(t) * transmittance(t),
                [t0, t2]
            )

          = integrate(
                lambda t: density(t) * channels(t) * transmittance(t),
                [t0, t1]
            ) + transmittance(t1) * integrate(
                lambda t: density(t) * channels(t) * transmittance(t),
                [t1, t2]
            )
        """
        assert torch.allclose(self.volume_range.next_t0(), cur.volume_range.t0)

        def _combine_fn(
            prev_val: Optional[torch.Tensor],
            cur_val: Optional[torch.Tensor],
            *,
            prev_transmittance: torch.Tensor,
        ):
            assert prev_val is not None
            if cur_val is None:
                # cur_output.aux_losses are empty for the void_model.
                return prev_val
            return prev_val + prev_transmittance * cur_val

        output = self.output.combine(
            cur.output, combine_fn=partial(_combine_fn, prev_transmittance=self.transmittance)
        )

        combined = RayVolumeIntegralResults(
            output=output,
            volume_range=self.volume_range.extend(cur.volume_range),
            transmittance=self.transmittance * cur.transmittance,
        )
        return combined

@dataclass
class RayVolumeIntegral:
    model: NeRFModel
    volume: Volume
    sampler: "RaySampler"
    n_samples: int

    def render_rays(
        self,
        origin: torch.Tensor,
        direction: torch.Tensor,
        t0: Optional[torch.Tensor] = None,
        prev_raw: Optional[AttrDict] = None,
        shared: bool = False,
        render_with_direction: bool = True,
    ) -> "RayVolumeIntegralResults":
        """
        Perform volumetric rendering over the given volume.

        :param position: [batch_size, *shape, 3]
        :param direction: [batch_size, *shape, 3]
        :param t0: Optional [batch_size, *shape, 1]
        :param prev_raw: the raw outputs when using multiple levels with this model.
        :param shared: means the same model is used for all RayVolumeIntegral's
        :param render_with_direction: use the incoming ray direction when querying the model.

        :return: RayVolumeIntegralResults
        """
        # 1. Intersect the rays with the current volume and sample ts to
        # integrate along.
        vrange = self.volume.intersect(origin, direction, t0_lower=t0)
        ts = self.sampler.sample(vrange.t0, vrange.t1, self.n_samples)

        if prev_raw is not None and not shared:
            # Append the previous ts now before fprop because previous
            # rendering used a different model and we can't reuse the output.
            ts = torch.sort(torch.cat([ts, prev_raw.ts], dim=-2), dim=-2).values

        # Shape sanity checks
        batch_size, *_shape, _t0_dim = vrange.t0.shape
        _, *ts_shape, _ts_dim = ts.shape

        # 2. Get the points along the ray and query the model
        directions = torch.broadcast_to(direction.unsqueeze(-2), [batch_size, *ts_shape, 3])
        positions = origin.unsqueeze(-2) + ts * directions

        optional_directions = directions if render_with_direction else None
        mids = (ts[..., 1:, :] + ts[..., :-1, :]) / 2
        raw = self.model(
            Query(
                position=positions,
                direction=optional_directions,
                t_min=torch.cat([vrange.t0[..., None, :], mids], dim=-2),
                t_max=torch.cat([mids, vrange.t1[..., None, :]], dim=-2),
            )
        )
        raw.ts = ts

        if prev_raw is not None and shared:
            # We can append the additional queries to previous raw outputs
            # before integration
            copy = prev_raw.copy()
            result = torch.sort(torch.cat([raw.pop("ts"), copy.pop("ts")], dim=-2), dim=-2)
            merge_results = partial(self._merge_results, dim=-2, indices=result.indices)
            raw = raw.combine(copy, merge_results)
            raw.ts = result.values

        # 3. Integrate the raw results
        output, transmittance = self.integrate_samples(vrange, raw)

        # 4. Clean up results that do not intersect with the volume.
        transmittance = torch.where(
            vrange.intersected, transmittance, torch.ones_like(transmittance)
        )

        def _mask_fn(_key: str, tensor: torch.Tensor):
            return torch.where(vrange.intersected, tensor, torch.zeros_like(tensor))

        def _is_tensor(_key: str, value: Any):
            return isinstance(value, torch.Tensor)

        output = output.map(map_fn=_mask_fn, should_map=_is_tensor)

        return RayVolumeIntegralResults(
            output=output,
            raw=raw,
            volume_range=vrange,
            transmittance=transmittance,
        )

    def integrate_samples(
        self,
        volume_range: VolumeRange,
        raw: AttrDict,
    ) -> Tuple[AttrDict, torch.Tensor]:
        """
        Integrate the raw.channels along with other aux_losses and values to
        produce the final output dictionary containing rendered `channels`,
        estimated `distances` and `aux_losses`.

        :param volume_range: Specifies the integral range [t0, t1]
        :param raw: Contains a dict of function evaluations at ts. Should have

            density: torch.Tensor [batch_size, *shape, n_samples, 1]
            channels: torch.Tensor [batch_size, *shape, n_samples, n_channels]
            aux_losses: {key: torch.Tensor [batch_size, *shape, n_samples, 1] for each key}
            no_weight_grad_aux_losses: an optional set of losses for which the weights
                                       should be detached before integration.

            after the call, integrate_samples populates some intermediate calculations
            for later use like

            weights: torch.Tensor [batch_size, *shape, n_samples, 1] (density *
                transmittance)[i] weight for each rgb output at [..., i, :].
        :returns: a tuple of (
            a dictionary of rendered outputs and aux_losses,
            transmittance of this volume,
        )
        """

        # 1. Calculate the weights
        _, _, dt = volume_range.partition(raw.ts)
        ddensity = raw.density * dt

        mass = torch.cumsum(ddensity, dim=-2)
        transmittance = torch.exp(-mass[..., -1, :])

        alphas = 1.0 - torch.exp(-ddensity)
        Ts = torch.exp(torch.cat([torch.zeros_like(mass[..., :1, :]), -mass[..., :-1, :]], dim=-2))
        # This is the probability of light hitting and reflecting off of
        # something at depth [..., i, :].
        weights = alphas * Ts

        # 2. Integrate all results
        def _integrate(key: str, samples: torch.Tensor, weights: torch.Tensor):
            if key == "density":
                # Omit integrating the density, because we don't need it
                return None
            return torch.sum(samples * weights, dim=-2)

        def _is_tensor(_key: str, value: Any):
            return isinstance(value, torch.Tensor)

        if raw.no_weight_grad_aux_losses:
            extra_aux_losses = raw.no_weight_grad_aux_losses.map(
                partial(_integrate, weights=weights.detach()), should_map=_is_tensor
            )
        else:
            extra_aux_losses = {}
        output = raw.map(partial(_integrate, weights=weights), should_map=_is_tensor)
        if "no_weight_grad_aux_losses" in output:
            del output["no_weight_grad_aux_losses"]
        output.aux_losses.update(extra_aux_losses)

        # Integrating the ts yields the distance away from the origin; rename the variable.
        output.distances = output.ts
        del output["ts"]
        del output["density"]

        assert output.distances.shape == (*output.channels.shape[:-1], 1)
        assert output.channels.shape[:-1] == raw.channels.shape[:-2]
        assert output.channels.shape[-1] == raw.channels.shape[-1]

        # 3. Reduce loss
        def _reduce_loss(_key: str, loss: torch.Tensor):
            return loss.view(loss.shape[0], -1).sum(dim=-1)

        # 4. Store other useful calculations
        raw.weights = weights

        output.aux_losses = output.aux_losses.map(_reduce_loss)

        return output, transmittance

    def _merge_results(
        self, a: Optional[torch.Tensor], b: torch.Tensor, dim: int, indices: torch.Tensor
    ):
        """
        :param a: [..., n_a, ...]. The other dictionary containing the b's may
            contain extra tensors from earlier calculations, so a can be None.
        :param b: [..., n_b, ...]
        :param dim: dimension to merge
        :param indices: how the merged results should be sorted at the end
        :return: a concatted and sorted tensor of size [..., n_a + n_b, ...]
        """
        if a is None:
            return None

        merged = torch.cat([a, b], dim=dim)
        return torch.gather(merged, dim=dim, index=torch.broadcast_to(indices, merged.shape))

class RaySampler(ABC):
    @abstractmethod
    def sample(self, t0: torch.Tensor, t1: torch.Tensor, n_samples: int) -> torch.Tensor:
        """
        :param t0: start time has shape [batch_size, *shape, 1]
        :param t1: finish time has shape [batch_size, *shape, 1]
        :param n_samples: number of ts to sample
        :return: sampled ts of shape [batch_size, *shape, n_samples, 1]
        """

class StratifiedRaySampler(RaySampler):
    """
    Instead of fixed intervals, a sample is drawn uniformly at random from each
    interval.
    """

    def __init__(self, depth_mode: str = "linear"):
        """
        :param depth_mode: linear samples ts linearly in depth. harmonic ensures
            closer points are sampled more densely.
        """
        self.depth_mode = depth_mode
        assert self.depth_mode in ("linear", "geometric", "harmonic")

    def sample(
        self,
        t0: torch.Tensor,
        t1: torch.Tensor,
        n_samples: int,
        epsilon: float = 1e-3,
    ) -> torch.Tensor:
        """
        :param t0: start time has shape [batch_size, *shape, 1]
        :param t1: finish time has shape [batch_size, *shape, 1]
        :param n_samples: number of ts to sample
        :return: sampled ts of shape [batch_size, *shape, n_samples, 1]
        """
        ones = [1] * (len(t0.shape) - 1)
        ts = torch.linspace(0, 1, n_samples).view(*ones, n_samples).to(t0.dtype).to(t0.device)

        if self.depth_mode == "linear":
            ts = t0 * (1.0 - ts) + t1 * ts
        elif self.depth_mode == "geometric":
            ts = (t0.clamp(epsilon).log() * (1.0 - ts) + t1.clamp(epsilon).log() * ts).exp()
        elif self.depth_mode == "harmonic":
            # The original NeRF recommends this interpolation scheme for
            # spherical scenes, but there could be some weird edge cases when
            # the observer crosses from the inner to outer volume.
            ts = 1.0 / (1.0 / t0.clamp(epsilon) * (1.0 - ts) + 1.0 / t1.clamp(epsilon) * ts)

        mids = 0.5 * (ts[..., 1:] + ts[..., :-1])
        upper = torch.cat([mids, t1], dim=-1)
        lower = torch.cat([t0, mids], dim=-1)
        t_rand = torch.rand_like(ts)

        ts = lower + (upper - lower) * t_rand
        return ts.unsqueeze(-1)

class ImportanceRaySampler(RaySampler):
    """
    Given the initial estimate of densities, this samples more from
    regions/bins expected to have objects.
    """

    def __init__(
        self, volume_range: VolumeRange, raw: AttrDict, blur_pool: bool = False, alpha: float = 1e-5
    ):
        """
        :param volume_range: the range in which a ray intersects the given volume.
        :param raw: dictionary of raw outputs from the NeRF models of shape
            [batch_size, *shape, n_coarse_samples, 1]. Should at least contain

            :param ts: earlier samples from the coarse rendering step
            :param weights: discretized version of density * transmittance
        :param blur_pool: if true, use 2-tap max + 2-tap blur filter from mip-NeRF.
        :param alpha: small value to add to weights.
        """
        self.volume_range = volume_range
        self.ts = raw.ts.clone().detach()
        self.weights = raw.weights.clone().detach()
        self.blur_pool = blur_pool
        self.alpha = alpha

    @torch.no_grad()
    def sample(self, t0: torch.Tensor, t1: torch.Tensor, n_samples: int) -> torch.Tensor:
        """
        :param t0: start time has shape [batch_size, *shape, 1]
        :param t1: finish time has shape [batch_size, *shape, 1]
        :param n_samples: number of ts to sample
        :return: sampled ts of shape [batch_size, *shape, n_samples, 1]
        """
        lower, upper, _ = self.volume_range.partition(self.ts)

        batch_size, *shape, n_coarse_samples, _ = self.ts.shape

        weights = self.weights
        if self.blur_pool:
            padded = torch.cat([weights[..., :1, :], weights, weights[..., -1:, :]], dim=-2)
            maxes = torch.maximum(padded[..., :-1, :], padded[..., 1:, :])
            weights = 0.5 * (maxes[..., :-1, :] + maxes[..., 1:, :])
        weights = weights + self.alpha
        pmf = weights / weights.sum(dim=-2, keepdim=True)
        inds = sample_pmf(pmf, n_samples)
        assert inds.shape == (batch_size, *shape, n_samples, 1)
        assert (inds >= 0).all() and (inds < n_coarse_samples).all()

        t_rand = torch.rand(inds.shape, device=inds.device)
        lower_ = torch.gather(lower, -2, inds)
        upper_ = torch.gather(upper, -2, inds)

        ts = lower_ + (upper_ - lower_) * t_rand
        ts = torch.sort(ts, dim=-2).values
        return ts


#!/usr/bin/python
import sys, glob

from sys import platform
if platform == "linux" or platform == "linux2":
    sys.path.insert(0, glob.glob('/home/yaoliu/src_code/local/lib/lib/python2.7/site-packages/')[0])

from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer

from thrift.Thrift import TType, TMessageType, TException, TApplicationException
from thrift.Thrift import TProcessor
from thrift.protocol import TProtocol

try:
    from thrift.protocol import fastbinary
except:
    fastbinary = None

import re
import hashlib

class NodeID:
    """
  Attributes:
   - id
   - ip
   - port
  """

    thrift_spec = (
        None,  # 0
        (1, TType.STRING, 'id', None, None,),  # 1
        (2, TType.STRING, 'ip', None, None,),  # 2
        (3, TType.I32, 'port', None, None,),  # 3
    )

    def __init__(self, id=None, ip=None, port=None, ):
        self.id = id
        self.ip = ip
        self.port = port

    def read(self, iprot):
        if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans,
                                                                                TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
            fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
            return
        iprot.readStructBegin()
        while True:
            (fname, ftype, fid) = iprot.readFieldBegin()
            if ftype == TType.STOP:
                break
            if fid == 1:
                if ftype == TType.STRING:
                    self.id = iprot.readString()
                else:
                    iprot.skip(ftype)
            elif fid == 2:
                if ftype == TType.STRING:
                    self.ip = iprot.readString()
                else:
                    iprot.skip(ftype)
            elif fid == 3:
                if ftype == TType.I32:
                    self.port = iprot.readI32()
                else:
                    iprot.skip(ftype)
            else:
                iprot.skip(ftype)
            iprot.readFieldEnd()
        iprot.readStructEnd()

    def write(self, oprot):
        if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
            oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
            return
        oprot.writeStructBegin('NodeID')
        if self.id is not None:
            oprot.writeFieldBegin('id', TType.STRING, 1)
            oprot.writeString(self.id)
            oprot.writeFieldEnd()
        if self.ip is not None:
            oprot.writeFieldBegin('ip', TType.STRING, 2)
            oprot.writeString(self.ip)
            oprot.writeFieldEnd()
        if self.port is not None:
            oprot.writeFieldBegin('port', TType.I32, 3)
            oprot.writeI32(self.port)
            oprot.writeFieldEnd()
        oprot.writeFieldStop()
        oprot.writeStructEnd()

    def validate(self):
        return

    def __repr__(self):
        L = ['%s=%r' % (key, value)
             for key, value in self.__dict__.items()]
        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))

    def __eq__(self, other):
        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__

    def __ne__(self, other):
        return not (self == other)

class Iface:
    def setReplicationtable(self, node_list):
        """
        Parameters:
         - node_list
        """
        pass

class Client(Iface):
    def __init__(self, iprot, oprot=None):
        self._iprot = self._oprot = iprot
        if oprot is not None:
            self._oprot = oprot
        self._seqid = 0

    def setReplicationtable(self, node_list):
        """
        Parameters:
         - node_list
        """
        self.send_setReplicationtable(node_list)
        self.recv_setReplicationtable()

    def send_setReplicationtable(self, node_list):
        self._oprot.writeMessageBegin('setReplicationtable', TMessageType.CALL, self._seqid)
        args = setReplicationtable_args()
        args.node_list = node_list
        args.write(self._oprot)
        self._oprot.writeMessageEnd()
        self._oprot.trans.flush()

    def recv_setReplicationtable(self):
        (fname, mtype, rseqid) = self._iprot.readMessageBegin()
        if mtype == TMessageType.EXCEPTION:
            x = TApplicationException()
            x.read(self._iprot)
            self._iprot.readMessageEnd()
            raise x
        result = setReplicationtable_result()
        result.read(self._iprot)
        self._iprot.readMessageEnd()
        return

class setReplicationtable_args:
    """
  Attributes:
   - node_list
  """

    thrift_spec = (
        None,  # 0
        (1, TType.LIST, 'node_list', (TType.STRUCT, (NodeID, NodeID.thrift_spec)), None,),  # 1
    )

    def __init__(self, node_list=None, ):
        self.node_list = node_list

    def read(self, iprot):
        if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans,
                                                                                TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
            fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
            return
        iprot.readStructBegin()
        while True:
            (fname, ftype, fid) = iprot.readFieldBegin()
            if ftype == TType.STOP:
                break
            if fid == 1:
                if ftype == TType.LIST:
                    self.node_list = []
                    (_etype3, _size0) = iprot.readListBegin()
                    for _i4 in range(_size0):
                        _elem5 = NodeID()
                        _elem5.read(iprot)
                        self.node_list.append(_elem5)
                    iprot.readListEnd()
                else:
                    iprot.skip(ftype)
            else:
                iprot.skip(ftype)
            iprot.readFieldEnd()
        iprot.readStructEnd()

    def write(self, oprot):
        if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
            oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
            return
        oprot.writeStructBegin('setReplicationtable_args')
        if self.node_list is not None:
            oprot.writeFieldBegin('node_list', TType.LIST, 1)
            oprot.writeListBegin(TType.STRUCT, len(self.node_list))
            for iter6 in self.node_list:
                iter6.write(oprot)
            oprot.writeListEnd()
            oprot.writeFieldEnd()
        oprot.writeFieldStop()
        oprot.writeStructEnd()

    def validate(self):
        return

    def __repr__(self):
        L = ['%s=%r' % (key, value)
             for key, value in self.__dict__.items()]
        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))

    def __eq__(self, other):
        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__

    def __ne__(self, other):
        return not (self == other)

class setReplicationtable_result:
    thrift_spec = (
    )

    def read(self, iprot):
        if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans,
                                                                                TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
            fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
            return
        iprot.readStructBegin()
        while True:
            (fname, ftype, fid) = iprot.readFieldBegin()
            if ftype == TType.STOP:
                break
            else:
                iprot.skip(ftype)
            iprot.readFieldEnd()
        iprot.readStructEnd()

    def write(self, oprot):
        if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
            oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
            return
        oprot.writeStructBegin('setReplicationtable_result')
        oprot.writeFieldStop()
        oprot.writeStructEnd()

    def validate(self):
        return

    def __repr__(self):
        L = ['%s=%r' % (key, value)
             for key, value in self.__dict__.items()]
        return '%s(%s)' % (self.__class__.__name__, ', '.join(L))

    def __eq__(self, other):
        return isinstance(other, self.__class__) and self.__dict__ == other.__dict__

    def __ne__(self, other):
        return not (self == other)

def create_node(s):
    sha256 = hashlib.sha256()
    sha256.update(s.encode())
    k = sha256.hexdigest()

    m = re.match('([^:]+):([^:]+)', s)
    return NodeID(k, m.group(1), int(m.group(2)))

def createReplicas(nid, lst):
    print('Creation replica for', nid)
    index = lst.index(nid) // 4
    start = 4 * index
    end = 4 * (index + 1) if 4 * (index + 1) < len(lst) else len(lst)
    replicas = lst[start:end]
    # print(replicas)
    return replicas

def init_replicatiom(nid, lst, cdct=None):
    replicaTable = createReplicas(nid, lst)

    if cdct == None:
        transport = TSocket.TSocket(nid.ip, nid.port)
        transport = TTransport.TBufferedTransport(transport)
        protocol = TBinaryProtocol.TBinaryProtocol(transport)
        client = Client(protocol)
        transport.open()
        try:
            client.setReplicationtable(replicaTable)
        finally:
            transport.close()
    else:
        # buffered clients
        client = cdct['%s:%d' % (nid.ip, nid.port)]
        client.setReplicationtable(replicaTable)

if __name__ == '__main__':
    fn = sys.argv[1]
    lst = sorted([create_node(l[:-1]) for l in open(fn)], key=lambda x: x.id)

    print ('starting replicas')
    for l in lst:
        print('replica for' , l.port)
        init_replicatiom(l, lst)


from skimage.io import imread, imsave
from numpy import clip

img = imread('img.png')

pixels = []
for row in img:
    for pixel in row:
        pixels.append(pixel)
pixels.sort()

k = round(len(pixels) * 0.05)
mn, mx = pixels[k], pixels[-k]

img = img.astype('float')
img = (img - mn) / (mx - mn) * 255
img = clip(img, 0, 255)
img = img.astype('uint8')

imsave('out_img.png', img)

week2/3. Устойчивый автоконтраст черно-белого изображения.py


'''
Descripttion: 两个数组的交集 II
version: 1
Author: Jason
Date: 2020-11-22 16:58:35
LastEditors: Jason
LastEditTime: 2020-11-22 16:59:35
'''

import random
from typing import List

def GenerateRandomList(number, size):
    temp = list()
    random_legth = random.randint(0, size)
    current_length = 0
    while current_length < random_legth:
        temp.append(random.randint(1, number))
        current_length += 1
    return temp

class Solution:
    def intersect(self, nums1: List[int], nums2: List[int]) -> List[int]:
        res = list()
        nums1.sort()
        nums2.sort()
        p1 = 0
        p2 = 0
        while p1 < len(nums1) and p2 < len(nums2):
            if nums1[p1] == nums2[p2]:
                res.append(nums1[p1])
                p1 += 1
                p2 += 1
            elif nums1[p1] < nums2[p2]:
                p1 += 1
            else:
                p2 += 1
        return res

    def intersect2(self, nums1: List[int], nums2: List[int]) -> List[int]:
        from collections import defaultdict
        dict1 = defaultdict(int)
        dict2 = defaultdict(int)
        res = list()
        for each in nums1:
            dict1[each] += 1

        for each in nums2:
            dict2[each] += 1

        for each_key in dict1:
            if dict2[each_key] != 0:
                nums = min(dict1[each_key], dict2[each_key])
                res.extend([each_key] * nums)
        return res

s = Solution()
for _ in range(10):
    nums1 = GenerateRandomList(4, 12)
    nums2 = GenerateRandomList(4, 12)
    print(s.intersect2(nums1, nums2))
    print(s.intersect(nums1, nums2))


from setuptools import setup, find_packages
import sys, os

here = os.path.abspath(os.path.dirname(__file__))
README = open(os.path.join(here, 'README.rst')).read()
NEWS = open(os.path.join(here, 'NEWS.txt')).read()

version = '0.1'

install_requires = [
    # List your project dependencies here.
    # For more details, see:
    # http://packages.python.org/distribute/setuptools.html#declaring-dependencies
    "scipy>=1.4",
    "seaborn>=0.10",
    "statsmodels>=0.11",
    "matplotlib>=3.2",
    "numpy>=1.18",
    "pandas>=1.0"
]

setup(name='starplot',
    version=version,
    description="Create barplots or boxplots with significant level annotations.",
    long_description=README + '\n\n' + NEWS,
    classifiers=[
      # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
    ],
    keywords='python numpy scipy statsmodels matplotlib seaborn t-test f-test mann-whitney non-parametric boxplots barplots significance-stars',
    author='yufongpeng',
    author_email='sciphypar@gmail.com',
    url='https://github.com/yufongpeng/starplot',
    license='',
    packages=find_packages('src'),
    package_dir = {'': 'src'},include_package_data=True,
    zip_safe=False,
    install_requires=install_requires,
    entry_points={
        'console_scripts':
            ['starplot=starplot:main']
    }
)


#!/usr/bin/env python3

import sys
import mrcfile
args = sys.argv
from IsoNet.util.filter import maxmask,stdmask
import numpy as np
#import cupy as cp
import os

def make_mask_dir(tomo_dir,mask_dir,side = 8, density_percentage=30,std_percentage=1,surface=None):
    tomo_list = ["{}/{}".format(tomo_dir,f) for f in os.listdir(tomo_dir)]
    try:
        os.makedirs(mask_dir)
    except FileExistsError:
        import shutil
        shutil.rmtree(mask_dir)
        os.makedirs(mask_dir)
    mask_list = ["{}/{}_mask.mrc".format(mask_dir,f.split('.')[0]) for f in os.listdir(tomo_dir)]

    for i,tomo in enumerate(tomo_list):
        print('tomo and mask',tomo, mask_list[i])
        make_mask(tomo, mask_list[i],side = side,density_percentage=density_percentage,std_percentage=std_percentage,surface=surface)

def make_mask(tomo_path, mask_name, mask_boundary = None, side = 5, density_percentage=50., std_percentage=50., surface=None):
    from scipy.ndimage.filters import gaussian_filter
    from skimage.transform import resize

    with mrcfile.open(tomo_path, permissive=True) as n:
        header_input = n.header
        #print(header_input)
        pixel_size = n.voxel_size
        tomo = n.data.astype(np.float32)
    sp=np.array(tomo.shape)
    sp2 = sp//2
    bintomo = resize(tomo,sp2,anti_aliasing=True)

    gauss = gaussian_filter(bintomo, side/2)
    if density_percentage <=99.8:
        mask1 = maxmask(gauss,side=side, percentile=density_percentage)
    else:
        mask1 = np.ones(sp2)

    if std_percentage <=99.8:
        mask2 = stdmask(gauss,side=side, threshold=std_percentage)
    else:
        mask2 = np.ones(sp2)

    out_mask_bin = np.multiply(mask1,mask2)

    if mask_boundary is not None:
        from IsoNet.util.filter import boundary_mask
        mask3 = boundary_mask(bintomo, mask_boundary)
        out_mask_bin = np.multiply(out_mask_bin, mask3)

    if (surface is not None) and surface < 1:
        for i in range(int(surface*sp2[0])):
            out_mask_bin[i] = 0
        for i in range(int((1-surface)*sp2[0]),sp2[0]):
            out_mask_bin[i] = 0

    out_mask = np.zeros(sp)
    out_mask[0:-1:2,0:-1:2,0:-1:2] = out_mask_bin
    out_mask[0:-1:2,0:-1:2,1::2] = out_mask_bin
    out_mask[0:-1:2,1::2,0:-1:2] = out_mask_bin
    out_mask[0:-1:2,1::2,1::2] = out_mask_bin
    out_mask[1::2,0:-1:2,0:-1:2] = out_mask_bin
    out_mask[1::2,0:-1:2,1::2] = out_mask_bin
    out_mask[1::2,1::2,0:-1:2] = out_mask_bin
    out_mask[1::2,1::2,1::2] = out_mask_bin
    out_mask = (out_mask>0.5).astype(np.uint8)

    with mrcfile.new(mask_name,overwrite=True) as n:
        n.set_data(out_mask)

        n.header.extra2 = header_input.extra2
        n.header.origin = header_input.origin
        n.header.nversion = header_input.nversion
        n.voxel_size = pixel_size
        #print(n.header)
    # with mrcfile.new('./test_mask1.rec',overwrite=True) as n:
    #     n.set_data(mask1.astype(np.float32))
    # with mrcfile.new('./test_mask2.rec',overwrite=True) as n:
    #     n.set_data(mask2.astype(np.float32))
if __name__ == "__main__":
# the first arg is tomo name the second is mask name
    make_mask(args[1],args[2])


import copy
import math
import numpy as np
import random as rd
from typing import List

class Layer:
    def __init__(self, size: int, next_size: int):
        self.size = size
        self.neurons = np.zeros((size,))
        self.biases = np.zeros((size,))
        self.weights = np.zeros((size, next_size))

class Point:
    x: int
    y: int
    type: int

    def __init__(self, x, y, type) -> None:
        self.x = x
        self.y = y
        self.type = type

class NeuralNetwork:
    learningRate: float
    layers: List[Layer]

    def __init__(self, learningRate, sizes) -> None:
        self.learningRate = learningRate
        self.layers = copy.copy(sizes)
        self.epoch = 0
        for i, _ in enumerate(sizes):
            next_size = 0
            if (i < len(sizes) - 1):
                next_size = sizes[i + 1]
            self.layers[i] = Layer(sizes[i], next_size)
            for j in range(sizes[i]):
                self.layers[i].biases[j] = rd.random() * 2.0 - 1.0
                for k in range(next_size):
                    self.layers[i].weights[j][k] = rd.random() * 2.0 - 1.0

    def sigmoid(self, x):
        return 1 / (1 + math.exp(-x))

    def dsigmoid(self, y):
        return y * (1 - y)

    def feed_forfard(self, inputs: List[float]):
        layers = self.layers

        for i, v in enumerate(inputs):
            layers[0].neurons[i] = v

        for i in range(1, len(layers)):
            l = layers[i-1]
            l1 = layers[i]
            for j in range(l1.size):
                l1.neurons[j] = 0
                for k in range(l.size):
                    l1.neurons[j] += l.neurons[k] * l.weights[k][j]

                l1.neurons[j] += l1.biases[j]
                l1.neurons[j] = self.sigmoid(l1.neurons[j])

        return layers[len(layers)-1].neurons

    def back_propagation(self, targets: List[float]):
        layers = self.layers
        errors = np.zeros((layers[len(layers)-1].size,))
        for i in range(layers[len(layers)-1].size):
            errors[i] = targets[i] - layers[len(layers)-1].neurons[i]

        for k in range(len(layers)-2, -1, -1):
            l = layers[k]
            l1 = layers[k+1]
            errors_next = np.zeros((l.size,))
            gradients = np.zeros((l1.size,))

            for i in range(l1.size):

                gradients[i] = errors[i] * \
                    self.dsigmoid(layers[k + 1].neurons[i])
                gradients[i] *= self.learningRate

            deltas = np.zeros((l1.size, l.size))
            for i in range(l1.size):
                for j in range(l.size):
                    deltas[i][j] = gradients[i] * l.neurons[j]

            for i in range(l.size):
                errors_next[i] = 0
                for j in range(l1.size):
                    errors_next[i] += l.weights[i][j] * errors[j]

            errors = np.zeros((l.size,))
            for i, v in enumerate(errors_next):
                errors[i] = v

            weights_new = np.zeros((len(l.weights), len(l.weights[0])))
            for i in range(l1.size):
                for j in range(l.size):
                    weights_new[j][i] = l.weights[j][i] + deltas[i][j]

            l.weights = weights_new
            for i in range(l1.size):
                l1.biases[i] += gradients[i]

        return None