import pandas as pd

import numpy as np

import os

import sys

import random

import torch

import matplotlib.pyplot as plt

from pathlib import Path

import math

import plotly

import plotly_express as px

import seaborn as sns

import itertools

from functools import partial

import tqdm.notebook as tqdm

from scipy.stats import kurtosis, skew

import time

import pydicom

import scipy

import scipy.ndimage as ndimage

from skimage import measure, morphology, segmentation

from scipy.ndimage.interpolation import zoom

from PIL import Image

# from pytorch_tabnet.tab_network import TabNet

import warnings

warnings.filterwarnings('ignore')

import sys

sys.path.append('../input/pytorchtabnet/tabnet-develop')

from pytorch_tabnet.tab_network import TabNet

## Code from https://www.kaggle.com/aadhavvignesh/lung-segmentation-by-marker-controlled-watershed

def generate_markers(image):

    """

    Generates markers for a given image.

    Parameters: image

    Returns: Internal Marker, External Marker, Watershed Marker

    """

    #Creation of the internal Marker

    marker_internal = image < -400

    marker_internal = segmentation.clear_border(marker_internal)

    marker_internal_labels = measure.label(marker_internal)

    areas = [r.area for r in measure.regionprops(marker_internal_labels)]

    areas.sort()

    if len(areas) > 2:

        for region in measure.regionprops(marker_internal_labels):

            if region.area < areas[-2]:

                for coordinates in region.coords:

                       marker_internal_labels[coordinates[0], coordinates[1]] = 0

    marker_internal = marker_internal_labels > 0

    # Creation of the External Marker

    external_a = ndimage.binary_dilation(marker_internal, iterations=10)

    external_b = ndimage.binary_dilation(marker_internal, iterations=55)

    marker_external = external_b ^ external_a

    # Creation of the Watershed Marker

    marker_watershed = np.zeros(image.shape, dtype=np.int)

    marker_watershed += marker_internal * 255

    marker_watershed += marker_external * 128

    return marker_internal, marker_external, marker_watershed

def seperate_lungs(image,iterations = 1):

    """

    Segments lungs using various techniques.

    Parameters: image (Scan image), iterations (more iterations, more accurate mask)

    Returns:

        - Segmented Lung

        - Lung Filter

        - Outline Lung

        - Watershed Lung

        - Sobel Gradient

    """

    # Store the start time

    # start = time.time()

    marker_internal, marker_external, marker_watershed = generate_markers(image)

    '''

    Creation of Sobel Gradient

    '''

    # Sobel-Gradient

    sobel_filtered_dx = ndimage.sobel(image, 1)

    sobel_filtered_dy = ndimage.sobel(image, 0)

    sobel_gradient = np.hypot(sobel_filtered_dx, sobel_filtered_dy)

    sobel_gradient *= 255.0 / np.max(sobel_gradient)

    '''

    Using the watershed algorithm

    We pass the image convoluted by sobel operator and the watershed marker

    to morphology.watershed and get a matrix matrix labeled using the

    watershed segmentation algorithm.

    '''

    watershed = morphology.watershed(sobel_gradient, marker_watershed)

    '''

    Reducing the image to outlines after Watershed algorithm

    '''

    outline = ndimage.morphological_gradient(watershed, size=(3,3))

    outline = outline.astype(bool)

    '''

    Black Top-hat Morphology:

    The black top hat of an image is defined as its morphological closing

    minus the original image. This operation returns the dark spots of the

    image that are smaller than the structuring element. Note that dark

    spots in the original image are bright spots after the black top hat.

    '''

    # Structuring element used for the filter

    blackhat_struct = [[0, 0, 1, 1, 1, 0, 0],

                       [0, 1, 1, 1, 1, 1, 0],

                       [1, 1, 1, 1, 1, 1, 1],

                       [1, 1, 1, 1, 1, 1, 1],

                       [1, 1, 1, 1, 1, 1, 1],

                       [0, 1, 1, 1, 1, 1, 0],

                       [0, 0, 1, 1, 1, 0, 0]]

    blackhat_struct = ndimage.iterate_structure(blackhat_struct, iterations)

    # Perform Black Top-hat filter

    outline += ndimage.black_tophat(outline, structure=blackhat_struct)

    '''

    Generate lung filter using internal marker and outline.

    '''

    lungfilter = np.bitwise_or(marker_internal, outline)

    lungfilter = ndimage.morphology.binary_closing(lungfilter, structure=np.ones((5,5)), iterations=3)

    '''

    Segment lung using lungfilter and the image.

    '''

    segmented = np.where(lungfilter, image, -1000)

    #return segmented, lungfilter, outline, watershed, sobel_gradient

    return segmented
def load_scan(path):

    paths = os.listdir(path)

    paths = sorted(paths, key=lambda x: int(str(x).split('/')[-1].split('.')[0]))

    slices = [pydicom.read_file(path + '/' + s) for s in paths]

    print(len(slices), 'slices')

    try:

        slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))

    except:

        pass

    try:

        slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])

    except:

        slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))

        slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)

    for s in slices:

        s.SliceThickness = slice_thickness

    return slices

def get_pixels_hu(slices):

    image = np.stack([s.pixel_array for s in slices])

    # Convert to int16 (from sometimes int16),

    # should be possible as values should always be low enough (<32k)

    image = image.astype(np.int16)

    # Set outside-of-scan pixels to 0

    # The intercept is usually -1024, so air is approximately 0

    # Convert to Hounsfield units (HU)

    for slice_number in range(len(slices)):

        intercept = slices[slice_number].RescaleIntercept

        slope = slices[slice_number].RescaleSlope

        if slope != 1:

            image[slice_number] = slope * image[slice_number].astype(np.float64)

            image[slice_number] = image[slice_number].astype(np.int16)

        image[slice_number] += np.int16(intercept)

    image[image <= -1900] = -1000

    return np.array(image, dtype=np.int16)

def resample(image, scan, new_spacing=[1,1,1]):

    st = time.time()

    slice_thickness = scan[0].SliceThickness

    spacing = np.array([slice_thickness] + list(scan[0].PixelSpacing), dtype=np.float32)

    resize_factor = spacing / new_spacing

    new_real_shape = image.shape * resize_factor

    new_shape = np.round(new_real_shape)

    real_resize_factor = new_shape / image.shape

    new_spacing = spacing / real_resize_factor

    print('resample factor', time.time() - st, real_resize_factor)

    image = scipy.ndimage.zoom(image, real_resize_factor, mode='nearest')

    print('resample time', time.time() - st)

    return image, new_spacing

def torch_resample(image, scan, new_spacing=[1,1,1]):

    st = time.time()

    slice_thickness = scan[0].SliceThickness

    spacing = np.array([slice_thickness] + list(scan[0].PixelSpacing), dtype=np.float32)

    resize_factor = spacing / new_spacing

    new_real_shape = image.shape * resize_factor

    new_shape = np.round(new_real_shape)

    real_resize_factor = new_shape / image.shape

    new_spacing = spacing / real_resize_factor

    #print('resample factor', time.time() - st, real_resize_factor)

    image = torch.tensor(image).unsqueeze(0).unsqueeze(0)

    #print(real_resize_factor)

    #size = [int(x*y) for x, y in zip(real_resize_factor, image.shape[-3:])]

    #print(image.shape)

    #print('using resize')

    image = torch.nn.functional.interpolate(image, scale_factor=tuple(real_resize_factor), mode='nearest')

    image = image.squeeze(0).squeeze(0).numpy()

    #print(image.shape)

    #print('resample time', time.time() - st)

    return image, new_spacing

def get_3d_resampled_array(patient_path):

    start_time = time.time()

    patient_slices = load_scan(str(patient_path))

    patient_slices_hu = get_pixels_hu(patient_slices)

    print('HU loaded', time.time() - start_time)

    print(patient_slices_hu.shape)

    #lungmask_3d = np.apply_over_axes(seperate_lungs, patient_slices_hu, 0)

    idx = np.ndindex(patient_slices_hu.shape[0])

    patient_slices_hu_masked = np.zeros(patient_slices_hu.shape)

    for i in idx:

        patient_slices_hu_masked[i] = seperate_lungs(patient_slices_hu[i])

        #patient_slices_hu_masked[i, :, :] = np.where(lungmask, patient_slices_hu[i, :, :], -1000)

    #patient_slices_hu_masked = np.where(lungmask_3d, patient_slices_hu, -1000)

    print('mask generated', time.time() - start_time)

    resampled_array, spacing = torch_resample(patient_slices_hu_masked, patient_slices, [1,1,1])

    print('after resample', time.time() - start_time)

    return resampled_array, spacing

def get_features_from_3d_array(resampled_array, spacing):

    features = {}

    # volume of lungs

    cube_volume = spacing[0] * spacing[1] * spacing[2]

    total_lung_volume = (resampled_array[resampled_array > -900].shape[0] * cube_volume)

    lung_volume_in_liters = total_lung_volume / (1000*1000)

    features['lung_volume_in_liters'] = lung_volume_in_liters

    #HU unit binning

    bins_threshold = (resampled_array <= 300) & (resampled_array >= -900)

    total_hu_units_bin = resampled_array[bins_threshold].flatten().shape[0]

    bin_values, bins = np.histogram(resampled_array[bins_threshold].flatten(), bins=range(-900, 400, 100))

    features['total_hu_units_bin'] = total_hu_units_bin

    for i, _bin in enumerate(bins[:-1]):

        features[f'bin_{_bin}'] = bin_values[i] / total_hu_units_bin

    #mean, skew, kurtosis

    lung_threshold = (resampled_array <= -320) & (resampled_array >= -900)

    histogram_values, _ = np.histogram(resampled_array[lung_threshold].flatten(), bins=100)

    features['lung_mean_hu'] = np.mean(resampled_array[lung_threshold].flatten())

    features['lung_skew'] = skew(histogram_values)

    features['lung_kurtosis'] = kurtosis(histogram_values)

    #height_of_lung

    n_lung_pixels = lung_threshold.sum(axis=1).sum(axis=1)

    height_start = np.argwhere(n_lung_pixels > 1000).min()

    height_end = np.argwhere(n_lung_pixels > 1000).max()

    features['height_of_lung_cm'] = (height_end - height_start)/10

    return features
data_dir = Path('../input/osic-pulmonary-fibrosis-progression')
def seed_everything(seed=42):

    random.seed(seed)

    torch.manual_seed(seed)

    torch.cuda.manual_seed_all(seed)

    np.random.seed(seed)

    os.environ['PYTHONHASHSEED'] = str(seed)

    torch.backends.cudnn.deterministic = True

    torch.backends.cudnn.benchmark = False
seed_everything()
def process_submission(submission):

    submission['Weeks'] = submission['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))

    submission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])

    return submission
train = pd.read_csv(data_dir/'train.csv')

test = pd.read_csv(data_dir/'test.csv')

# dicom_meta = pd.read_pickle(data_dir/'train_dicom_df')/

submission = process_submission(pd.read_csv(data_dir/'sample_submission.csv'))

image_feature_df = pd.read_csv('../input/lung-image-features/patient_feature2_df.csv')
image_feature_df =image_feature_df.drop(['Unnamed: 0'], axis=1)
image_feature_df.head()
import torch

from torch.utils.data import Dataset, DataLoader
from sklearn.model_selection import StratifiedKFold, train_test_split, KFold

from sklearn.preprocessing import LabelEncoder

def frequency_binning(x, nbin):

    nlen = len(x)

    return np.unique(np.interp(np.linspace(0, nlen, nbin + 1),

                     np.arange(nlen),

                     np.sort(x)))

def replace_with_frequency_bins(arr, nbins=5):

    bins = frequency_binning(arr, nbins)

    out = pd.cut(arr, bins)

    return [inter.left if (type(inter) is not float) else inter for inter in out]

train['expected_fvc'] = np.round((100 * train['FVC']) / train['Percent'], 2)

test['expected_fvc'] = np.round((100 * test['FVC']) / test['Percent'], 2)

patient_df = train[['Patient', 'Weeks', 'Age', 'Sex', 'expected_fvc', 'SmokingStatus']]

patient_df['num_weeks'] = train[['Patient', 'Weeks']].groupby('Patient').transform('count')

patient_df['min_week'] = train[['Patient', 'Weeks']].groupby('Patient').transform('min')

_le = LabelEncoder()

patient_df['SmokingStatus'] = _le.fit_transform(train['SmokingStatus'])

patient_df['Sex'] = (patient_df['Sex'] == 'Male').astype(int)

patient_df['Age'] = replace_with_frequency_bins(patient_df['Age'], 5)

patient_df['num_weeks'] = replace_with_frequency_bins(patient_df['num_weeks'], 5)

patient_df['min_week'] = replace_with_frequency_bins(patient_df['min_week'], 5)

patient_df['expected_fvc'] = pd.Series(replace_with_frequency_bins(patient_df['expected_fvc'], 6)).astype(str)

patient_df = patient_df.drop('Weeks',axis=1).drop_duplicates()

patient_df.index = list(range(len(patient_df)))
patient_df = patient_df.merge(image_feature_df, left_on='Patient', right_on='patient_id').drop('patient_id', axis=1)
class LungDataset(Dataset):

    def __init__(self, mode, data, image_features, preprocessing_params=None, expand=True):

        data = data.drop_duplicates(subset=['Patient', 'Weeks'])

        self.inference_data = None

        self.ohe = False

        self.preprocessing_params = preprocessing_params if preprocessing_params is not None else {

            'min_week': -12,

            'max_week': 133

        }

        self.expand = expand

        self.raw_data = self.process_data(data.copy(), image_features, mode, self.preprocessing_params)

        self.data = self.raw_data[self.features]

        self.target = self.raw_data['FVC'] if mode != 'test' else None

        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'

        self.inference = False

        self.pass_y = True if mode != 'test' else False

    def expand_train_data(self, train):

        train_expanded = pd.DataFrame()

        for patient, patient_df in train.groupby('Patient'):

            patient_expanded = pd.DataFrame()

            expected_fvc = np.mean((100 * patient_df['FVC']) / patient_df['Percent'])

            for week in patient_df['Weeks'].sort_values():

                week_df = patient_df.copy()

                week_df['weeks_from_first_visit'] = week_df['Weeks'] - week

                week_df['first_test_fvc'] = patient_df.loc[patient_df['Weeks'] == week, 'FVC'].mean()

                week_df['first_test_week'] = week

                week_df['predict_week'] = patient_df['Weeks']

                week_df.drop(['Percent', 'Weeks'], axis=1, inplace=True)

                patient_expanded = pd.concat([patient_expanded, week_df], ignore_index=True)

            patient_expanded['expected_fvc'] = expected_fvc

            train_expanded = pd.concat([train_expanded, patient_expanded], ignore_index=True)

            train_expanded = train_expanded[(train_expanded['weeks_from_first_visit'] >= -12) & (train_expanded['weeks_from_first_visit'] <= 133)]

        return train_expanded

    def process_data(self, data, img_data, mode, params):

        if mode == 'train' and self.expand:

            data = self.expand_train_data(data)

        else:

            # expected_FVC

            data['expected_fvc'] = (100 * data['FVC']) / data['Percent']

            data['first_test_week'] = data[['Patient', 'Weeks']].groupby('Patient').transform('min')

            data['weeks_from_first_visit'] = data['Weeks'] - data['first_test_week']

            data['predict_week'] = data['Weeks']

            min_fvc_df = data.sort_values('first_test_week').groupby('Patient').head(1)[['Patient', 'FVC']]

            min_fvc_df.columns = ['Patient', 'first_test_fvc']

            data = data.merge(min_fvc_df, on='Patient', how='left')

        data['percent'] = data['first_test_fvc'] / data['expected_fvc']

        # age

        min_age = params.get('min_age', data['Age'].min())

        max_age = params.get('max_age', data['Age'].max())

        data['age'] = (data['Age'] - data['Age'].min()) / (data['Age'].max() - data['Age'].min())

        # OHE

        if self.ohe:

            sex_df = pd.get_dummies(data['Sex']).reset_index(drop=True)

            sex_columns = ['Male', 'Female']

            for col in sex_columns:

                if col not in sex_df:

                    sex_df[col] = 0

            smoke_df = pd.get_dummies(data['SmokingStatus']).reset_index(drop=True)

            smoke_columns = ['Currently smokes','Ex-smoker','Never smoked']

            for col in smoke_columns:

                if col not in smoke_df.columns:

                    smoke_df[col] = 0

        else:

            sex_map = {'Male': 0, 'Female': 1}

            smoke_map = {'Ex-smoker': 0, 'Never smoked': 1, 'Currently smokes': 2}

            data['sex'] = data['Sex'].map(sex_map)

            data['smoke'] = data['SmokingStatus'].map(smoke_map)

        # base week

        min_week = params.get('min_week')

        max_week = params.get('max_week')

        data['first_test_week'] = (data['first_test_week'] - min_week) / (max_week - min_week)

        data['weeks_from_first_visit'] = (data['weeks_from_first_visit'] - min_week) / (max_week - min_week)

        data['predict_week'] = (data['predict_week'] - min_week) / (max_week - min_week)

        # Opting min_FVC value

        min_FVC = params.get('min_fvc', min(data['expected_fvc'].min(), data['FVC'].min()))

        max_FVC = params.get('max_fvc', max(data['expected_fvc'].max(), data['FVC'].max()))

        data['expected_fvc'] = (data['expected_fvc'] - min_FVC) / (max_FVC - min_FVC)

        data['first_test_fvc'] = (data['first_test_fvc'] - min_FVC) /(max_FVC - min_FVC)

        data['FVC'] = (data['FVC'] - min_FVC) /(max_FVC - min_FVC)

        #print(data.shape, sex_df.shape, smoke_df.shape)

        data.reset_index(drop=True, inplace=True)

        if self.ohe:

            data = pd.concat([data, sex_df, smoke_df], axis=1)

        #print(data.shape)

        # update params

        self.preprocessing_params = {

            'min_age': min_age,

            'max_age': max_age,

            'min_week': min_week,

            'max_week': max_week,

            'min_fvc': min_FVC,

            'max_fvc': max_FVC

        }

        self.features = ['first_test_fvc',

                         'age', 'predict_week', 'percent', 'expected_fvc', 'weeks_from_first_visit'] #'first_test_week']

        #Image features

        if img_data is not None:

            # lung_volume

            missing_img_data = img_data[img_data['missing'] == 1].copy()

            img_data = img_data[img_data['missing'] == 0].copy()

            min_lung_vol = params.get('min_lung_vol', img_data['lung_volume_in_liters'].min())

            max_lung_vol = params.get('max_lung_vol', img_data['lung_volume_in_liters'].max())

            img_data['lung_volume'] = (img_data['lung_volume_in_liters'] - min_lung_vol) / (max_lung_vol - min_lung_vol)

            # lung height

            min_lung_height = params.get('min_lung_height', img_data['height_of_lung_cm'].min())

            max_lung_height = params.get('max_lung_height', img_data['height_of_lung_cm'].max())

            img_data['lung_height'] = (img_data['height_of_lung_cm'] - min_lung_height) / (max_lung_height - min_lung_height)

            #mean HU value

            min_hu_mean = params.get('min_hu_mean', img_data['lung_mean_hu'].min())

            max_hu_mean = params.get('max_hu_mean', img_data['lung_mean_hu'].max())

            img_data['hu_mean'] = (img_data['lung_mean_hu'] - min_hu_mean) / (max_hu_mean - min_hu_mean)

            #skew

            min_hu_skew = params.get('min_hu_skew', img_data['lung_skew'].min())

            max_hu_skew = params.get('max_hu_skew', img_data['lung_skew'].max())

            img_data['hu_skew'] = (img_data['lung_skew'] - min_hu_skew) / (max_hu_skew - min_hu_skew)

            #kurtosis

            min_hu_kurtosis = params.get('min_hu_kurtosis', img_data['lung_kurtosis'].min())

            max_hu_kurtosis = params.get('max_hu_kurtosis', img_data['lung_kurtosis'].max())

            img_data['hu_kurtosis'] = (img_data['lung_kurtosis'] - min_hu_kurtosis) / (max_hu_kurtosis - min_hu_kurtosis)

            img_features = ['lung_volume', 'lung_height', 'hu_mean', 'hu_skew', 'hu_kurtosis', 'missing']

            self.features += img_features

            self.preprocessing_params.update({

                'min_lung_vol': min_lung_vol,

                'max_lung_vol': max_lung_vol,

                'min_lung_height': min_lung_height,

                'max_lung_height': max_lung_height,

                'min_hu_mean': min_hu_mean,

                'max_hu_mean': max_hu_mean,

                'min_hu_skew': min_hu_skew,

                'max_hu_skew': max_hu_skew,

                'min_hu_kurtosis': min_hu_kurtosis,

                'max_hu_kurtosis': max_hu_kurtosis,

            })

            bin_features = [x for x in img_data.columns if 'bin_' in x]

            self.features += bin_features

            img_feature_df = img_data[[*img_features, *bin_features, 'patient_id']]

            for col in img_feature_df.columns:

                if col not in missing_img_data:

                    missing_img_data[col] = 0

            img_feature_df = pd.concat([img_feature_df, missing_img_data], ignore_index=True)

            img_feature_df = img_feature_df.fillna(0)

            data = data.merge(img_feature_df, left_on='Patient', right_on='patient_id')

        if self.ohe:

            self.features += [*sex_df.columns, *smoke_df.columns]

        else:

            self.features += ['sex', 'smoke']

        data.index = list(range(len(data)))

        # construct inference data

        if mode != 'train':

            inference_df = pd.DataFrame(

                itertools.product(data['Patient'].unique(), range(-12, 134)), columns=['Patient', 'Weeks']

            )

            min_week_df = data[data['weeks_from_first_visit'] == 12 / (133 + 12)].drop('Weeks', axis=1)

            inference_df = inference_df.merge(min_week_df, on='Patient', how='left')

            first_test_week = (inference_df['first_test_week'] * (max_week - min_week)) + min_week

            inference_df['weeks_from_first_visit'] = inference_df['Weeks'] - first_test_week

            inference_df['predict_week'] = inference_df['Weeks']

            inference_df['weeks_from_first_visit'] = (inference_df['weeks_from_first_visit'] - min_week) / (max_week - min_week)

            inference_df['predict_week'] = (inference_df['predict_week'] - min_week) / (max_week - min_week)

            self.inference_data = inference_df.reset_index(drop=True)

        return data

    def get_params(self):

        return self.preprocessing_params

    def get_inference_df(self, last_3_visits=False):

        return self.inference_data

    def set_inference(self, value):

        self.inference = value

    def get_y(self, value):

        self.pass_y = value

    def __len__(self):

        if not self.inference:

            return len(self.data)

        return len(self.inference_data)

    def __getitem__(self, idx):

        if not self.inference:

            data = self.data

        else:

            data = self.inference_data[self.features]

        if (self.target is None) or self.inference or (not self.pass_y):

            return torch.tensor(data.loc[idx].values, dtype=torch.float32).to(self.device)

        return torch.tensor(data.loc[idx].values, dtype=torch.float32).to(self.device), torch.tensor(self.target[idx], dtype=torch.float32).to(self.device)
train_dataset = LungDataset('train', train, image_feature_df, expand=False)
train_dataset.get_params()
def get_splits(stratify_df, train_df, image_df, kfolds, stratify_col=None, expand=True):

    splits = kfolds.split(stratify_df) if not stratify_col else kfolds.split(stratify_df, stratify_df[stratify_col])

    for train_index, valid_index in splits:

        train_patients = patient_df.loc[train_index, 'Patient']

        valid_patients = patient_df.loc[valid_index, 'Patient']

        batch_train_df = train_df[train_df['Patient'].isin(train_patients)]

        batch_valid_df = train_df[train_df['Patient'].isin(valid_patients)]

        batch_train_image_df = image_df[image_df['patient_id'].isin(train_patients)]

        batch_valid_image_df = image_df[image_df['patient_id'].isin(valid_patients)]

        train_dataset = LungDataset('train', batch_train_df, batch_train_image_df, expand=expand)

        params = train_dataset.get_params()

        valid_dataset = LungDataset('valid', batch_valid_df, batch_valid_image_df, params)

        yield train_dataset, valid_dataset
from pytorch_lightning.callbacks import Callback

class CVResultTracker(Callback):

    def __init__(self, result_file, cv_result_file):

        super().__init__()

        self.result_file = result_file

        self_cv_result_file = cv_result_file

    def write_file(self, path, df):

        if path.is_file():

            existing_df = pd.read_csv(path)

            df = pd.concat([existing_df, df], ignore_index=True)

        df.to_csv(path, index=False)

        return df

    def teardown(self, trainer, pl_module, stage):

        logger = None

        for logr in trainer.logger:

            logger = logr if 'CSVLogger' in str(logr.__class__) else None

        if logger:

            metrics_df = pd.read_csv(f'./{logger.save_dir}/{logger.name}/{logger.version}/metrics.csv')

            val_df = metrics_df.dropna(subset=['val_loss'])

            val_df['name'] = logger.name + '_' + logger.version

            val_df['model_name'] = logger.name.split('fold')[0]

            val_df_last = val_df[val_df['epoch'] == val_df['epoch'].max()]

            val_df_best = val_df[val_df['val_nll'] == val_df['val_nll'].min()]

            val_df_last = val_df_last.dropna(axis=1, how='all')

            val_df_best = val_df_best.dropna(axis=1, how='all')

            fold_results_best_path = Path(f'./{logger.save_dir}/fold_results_best.csv')

            fold_results_last_path = Path(f'./{logger.save_dir}/fold_results_last.csv')

            model_results_best_path = Path(f'./{logger.save_dir}/model_results_best.csv')

            model_results_last_path = Path(f'./{logger.save_dir}/model_results_last.csv')

            fold_results_best_df = self.write_file(fold_results_best_path, val_df_best)

            fold_results_last_df = self.write_file(fold_results_last_path, val_df_last)

            model_best_df = fold_results_best_df.drop('name', axis=1)

            model_last_df = fold_results_last_df.drop('name', axis=1)

            model_best_df = model_best_df.groupby('model_name', as_index=False).mean()

            model_last_df = model_last_df.groupby('model_name', as_index=False).mean()

            model_best_df.to_csv(model_results_best_path, index=False)

            model_last_df.to_csv(model_results_last_path, index=False)

import torch.nn as nn

import torch.optim as optim

import torch.nn.functional as F

import pytorch_lightning as pl

from pytorch_lightning.metrics.metric import TensorMetric

from pytorch_lightning.callbacks.early_stopping import EarlyStopping

from pytorch_lightning.callbacks import ModelCheckpoint

from pytorch_lightning.callbacks import LearningRateLogger

from pytorch_lightning import loggers

class NLL(TensorMetric):

    name = 'NLL'

    def __init__(self, min_fvc, max_fvc):

        super().__init__(name='nll')

        self.min_fvc = min_fvc

        self.max_fvc = max_fvc

    def forward(self, preds, target):

        preds = (preds * (self.max_fvc - self.min_fvc)) + self.min_fvc

        target = (target * (self.max_fvc - self.min_fvc)) + self.min_fvc

        sigma = preds[:, 2] - preds[:, 0]

        fvc_pred = preds[:, 1]

        #sigma_clip = sigma + C1

        sigma_clip = torch.max(sigma, torch.tensor(70.).to(self.device))

        delta = torch.abs(target - fvc_pred)

        delta = torch.min(delta, torch.tensor(1000.).to(self.device))

        sq2 = torch.sqrt(torch.tensor(2.).to(device))

        metric = (delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)

        # print(torch.mean(metric))

        return torch.mean(metric)

class MAE(TensorMetric):

    name = 'MAE'

    def __init__(self, min_fvc, max_fvc):

        super().__init__(name='MAE')

        self.min_fvc = min_fvc

        self.max_fvc = max_fvc

    def forward(self, preds, target):

        preds = (preds * (self.max_fvc - self.min_fvc)) + self.min_fvc

        target = (target * (self.max_fvc - self.min_fvc)) + self.min_fvc

        return torch.mean(torch.abs(target - preds[:, 1]))

class LQuantModel(pl.LightningModule):

    def __init__(self, learning_rate, total_steps, train_params, in_tabular_features=8,

                 out_quantiles=3):

        super(LQuantModel, self).__init__()

        self.learning_rate = learning_rate

        self.total_steps = total_steps

        self.fc1 = nn.Linear(in_tabular_features, 512)

        self.fc2 = nn.Linear(512, 256)

        self.fc3 = nn.Linear(256, out_quantiles)

        self.fc4 = nn.Linear(256, out_quantiles)

        self.train_params = train_params

        self.save_hyperparameters()

        self.metrics = [

            NLL(

                train_params.get('min_fvc'),

                train_params.get('max_fvc'),

            ),

            MAE(

                train_params.get('min_fvc'),

                train_params.get('max_fvc'),

            )

        ]

        self.eval_tracker = {}

    def forward(self, x):

        x = F.relu(self.fc1(x))

        x = F.relu(self.fc2(x))

        x1 = self.fc3(x)

        x2 = F.relu(self.fc4(x))

        preds = x1 + torch.cumsum(x2, axis=1)

        return preds

    def configure_optimizers(self):

        optimizer = optim.SGD(self.parameters(), lr=(self.learning_rate), weight_decay=0.01)

        scheduler = optim.lr_scheduler.OneCycleLR(

            optimizer,

            max_lr=self.learning_rate,

            total_steps=self.total_steps

        )

        return [optimizer], [{

            'scheduler': scheduler,

            'interval': 'step',

            'frequency': 1

        }]

    def set_loss_fn(self, loss_fn):

        self.loss_fn = loss_fn

    def training_step(self, batch, batch_idx):

        x, y = batch

        y_hat = self(x)

        loss = self.loss_fn(y_hat, y)

        result = pl.TrainResult(minimize=loss)

        result.log('train_loss', loss, prog_bar=True)

        for metric in self.metrics:

            metric.to(self.device)

            metric_value = metric(y_hat, y)

            result.log(f'train_{metric.name}', metric_value, on_epoch=True, on_step=False)

        return result

    def validation_step(self, batch, batch_idx):

        x, y = batch

        y_hat = self(x)

        loss = self.loss_fn(y_hat, y)

        result = pl.EvalResult(checkpoint_on=loss)

        result.log('val_loss', loss, on_epoch=True, on_step=False)

        for metric in self.metrics:

            metric.to(self.device)

            metric_value = metric(y_hat, y)

            result.log(f'val_{metric.name}', metric_value, on_epoch=True, on_step=False)

        return result

#     def validation_epoch_end(self, validation_results):

#         self.eval_tracker = {k: v[-1] for k, v in validation_results.items() if k[0:4] == 'val_'}

#         return validation_results

    def test_step(self, batch, batch_idx):

        y_hat = self(batch)

        result = pl.EvalResult()

        result.predictions = y_hat

        return result

class LTabnet(pl.LightningModule):

    def __init__(self, learning_rate, total_steps, train_params, in_tabular_features=8,

                 out_quantiles=3):

        super(LTabnet, self).__init__()

        self.learning_rate = learning_rate

        self.total_steps = total_steps

        self.tabnet = TabNet(

            input_dim=26,

            output_dim=3,

            n_a=64,

            n_d=64,

            cat_idxs=[24,25],

            cat_dims=[2,3],

            cat_emb_dim=[4,4],

        ).to(self.device)

        self.train_params = train_params

        self.save_hyperparameters()

        self.metrics = [

            NLL(

                train_params.get('min_fvc'),

                train_params.get('max_fvc'),

            ),

            MAE(

                train_params.get('min_fvc'),

                train_params.get('max_fvc'),

            ),

        ]

    def forward(self, x):

        preds, _ = self.tabnet(x)

        return preds

    def configure_optimizers(self):

        optimizer = optim.SGD(self.parameters(), lr=(self.learning_rate), weight_decay=0.01)

        scheduler = optim.lr_scheduler.OneCycleLR(

            optimizer,

            max_lr=self.learning_rate,

            total_steps=self.total_steps

        )

        return [optimizer], [{

            'scheduler': scheduler,

            'interval': 'step',

            'frequency': 1

        }]

    def set_loss_fn(self, loss_fn):

        self.loss_fn = loss_fn

    def training_step(self, batch, batch_idx):

        x, y = batch

        y_hat = self(x)

        loss = self.loss_fn(y_hat, y)

        result = pl.TrainResult(minimize=loss)

        result.log('train_loss', loss, prog_bar=True)

        for metric in self.metrics:

            metric.to(self.device)

            metric_value = metric(y_hat, y)

            result.log(f'train_{metric.name}', metric_value, on_epoch=True, on_step=False)

        return result

    def validation_step(self, batch, batch_idx):

        x, y = batch

        y_hat = self(x)

        loss = self.loss_fn(y_hat, y)

        result = pl.EvalResult(checkpoint_on=loss)

        result.log('val_loss', loss, on_epoch=True, on_step=False)

        for metric in self.metrics:

            metric.to(self.device)

            metric_value = metric(y_hat, y)

            result.log(f'val_{metric.name}', metric_value, on_epoch=True, on_step=False)

        return result

    def test_step(self, batch, batch_idx):

        y_hat = self(batch)

        result = pl.EvalResult()

        result.predictions = y_hat

        return result
class LungDataModule(pl.LightningDataModule):

    def __init__(self, train_dataset, valid_dataset, test_dataset, batch_size=128):

        super().__init__()

        self.train_dataset = train_dataset

        self.valid_dataset = valid_dataset

        self.test_dataset = test_dataset

        self.batch_size = batch_size

    def train_dataloader(self):

        return DataLoader(train_dataset, shuffle=True, batch_size=self.batch_size)

    def val_dataloader(self):

        return DataLoader(valid_dataset, shuffle=False, batch_size=self.batch_size)

    def test_dataloader(self):

        return DataLoader(test_dataset, shuffle=False, batch_size=self.batch_size)

def predict_dataloader(model, dataloader):

    predictions = []

    model.eval()

    for batch in dataloader:

        predictions.append(model(batch).detach().cpu().numpy())

    return np.vstack(predictions)
def quantile_loss(preds, target, quantiles, device):

    #assert not target.requires_grad

    #assert preds.size(0) == target.size(0)

    losses = []

    q = torch.tensor(quantiles).to(device).repeat(preds.shape[0], 1)

    errors = target.unsqueeze(1).repeat(1, 3) - preds

    losses = torch.max((q - 1) * errors, q * errors)

    return torch.mean(losses).unsqueeze(0)

def metric_score(preds, target, max_fvc, min_fvc, device):

    preds = (preds * (max_fvc - min_fvc)) + min_fvc

    target = (target * (max_fvc - min_fvc)) + min_fvc

    sigma = preds[:, 2] - preds[:, 0]

    fvc_pred = preds[:, 1]

    #sigma_clip = sigma + C1

    sigma_clip = torch.max(sigma, torch.tensor(70.).to(device))

    delta = torch.abs(target - fvc_pred)

    delta = torch.min(delta, torch.tensor(1000.).to(device))

    sq2 = torch.sqrt(torch.tensor(2.).to(device))

    metric = (delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)

    # print(torch.mean(metric))

    return torch.mean(metric)

def metric_loss(preds, target, _lambda, quantiles, max_fvc, min_fvc, device):

    return ((_lambda * quantile_loss(preds, target, quantiles, device)) +

            ((1 - _lambda) * metric_score(preds, target, max_fvc, min_fvc, device)))

seed_everything(2020)

device = 'cuda' if torch.cuda.is_available() else 'cpu'

_skf = StratifiedKFold(n_splits=5)

_kf = KFold()

lr_monitor_callback = LearningRateLogger(logging_interval='step')
seed_everything(2020)

name = 'tabnet_img_featuresall_skf_mloss1_lr5e-3_faster_missing'

trained_models = []

cv_predictions = []

epochs = 350

batch_size = 128

for i, (train_dataset, valid_dataset) in enumerate(get_splits(patient_df, train, image_feature_df, _skf, 'expected_fvc',expand=True)):

    total_steps = epochs * math.ceil((len(train_dataset) / batch_size))

    train_params = train_dataset.get_params()

    loss_fn = partial(

        metric_loss,

        _lambda=1,

        quantiles=(0.3, 0.5, 0.7),

        max_fvc=train_params['max_fvc'],

        min_fvc=train_params['min_fvc'],

        device=device

    )

    pathdir = Path(f'./models/{name}/fold_{i}/')

    pathdir.mkdir(parents=True, exist_ok=True)

    checkpoint_callback = ModelCheckpoint(

        filepath=pathdir,

        save_top_k=1,

        save_last=True,

        verbose=False,

        monitor='val_nll',

        mode='min',

        prefix=''

    )

    cv_result_tracker = CVResultTracker('./fold_results.csv', './model_results.csv')

    model = LTabnet(5e-3, total_steps, train_params, in_tabular_features=29).to(device)

    model.set_loss_fn(loss_fn)

    test_dataset = LungDataset('test', test, None, train_params)

    test_dataset.set_inference(True)

    data_module = LungDataModule(train_dataset, valid_dataset, test_dataset, batch_size=batch_size)

    tb_logger = loggers.TensorBoardLogger('lightning_logs/', name=f'{name}', version=f'fold_{i}')

    csv_logger = loggers.CSVLogger('results', name=name, version=f'fold_{i}')

    trainer = pl.Trainer(

        max_epochs=epochs,

        check_val_every_n_epoch=5,

        logger=[tb_logger, csv_logger],

        checkpoint_callback=checkpoint_callback,

        gpus=1,

        callbacks=[lr_monitor_callback, cv_result_tracker]

    )

    trainer.fit(model, data_module)

    trained_models.append(model)


#!/usr/bin/env python
__author__ = 'Sergei F. Kliver'
import bz2
import gzip
import sys

import argparse

from pandas import read_csv

if sys.version_info[0] == 3:
    from io import TextIOWrapper as file

def metaopen(filename, flags, buffering=None):
    if not isinstance(filename, str):  # or isinstance(filename, gzip.GzipFile) or isinstance(filename, bz2.BZ2File):
        if isinstance(filename, file):
            return filename
        else:
            raise ValueError("ERROR!!! Not file object or str: {}".format(str(filename)))
    elif filename[-3:] == ".gz":
        return gzip.open(filename, flags + ("t" if "b" not in flags else ""))
    elif filename[-4:] == ".bz2":
        return bz2.open(filename, flags + ("t" if "b" not in flags else ""))
    else:
        if buffering is not None:
            return open(filename, flags, buffering=buffering)
        else:
            return open(filename, flags)

parser = argparse.ArgumentParser()

parser.add_argument("-f", "--fai", action="store", dest="fai", required=True,
                    help=".fai file ")
parser.add_argument("-w", "--scaffold_whitelist", action="store", dest="scaffold_whitelist", required=True,
                    type=lambda path: read_csv(path, sep="\t", squeeze=True, header=None),
                    help="File or comma-separated list with scaffolds from white list")
parser.add_argument("-o", "--output", action="store", dest="output",
                    help="Output file with intervals")

args = parser.parse_args()

with metaopen(args.fai, "r") as in_fd, metaopen(args.output, "w") as out_fd:
    for line in in_fd:
        scaf, end = line.split("\t")[:2]
        if scaf in args.scaffold_whitelist.values:
            out_fd.write("{0}:1-{1}\n".format(scaf, end))


from django.urls import path, include
from django.contrib import admin
from main import views
from django.contrib.auth.views import LogoutView,LoginView

urlpatterns = [

    path('admin/', admin.site.urls),
    path('partner/', include('partner.urls')),
    path('user/', include('user.urls')),

    path('', views.home_view, name=''),
    path('logout', LogoutView.as_view(template_name='main/logout.html'), name='logout'),
    path('aboutus', views.aboutus_view),
    path('afterlogin', views.afterlogin_view, name='afterlogin'),

    path('adminclick', views.adminclick_view),
    path('adminlogin', LoginView.as_view(template_name='main/adminlogin.html'), name='adminlogin'),
    path('admin-dashboard', views.admin_dashboard_view, name='admin-dashboard'),
    path('admin-partner', views.admin_partner_view, name='admin-partner'),
    path('admin-view-partner', views.admin_view_partner_view, name='admin-view-partner'),
    path('update-partner/<int:pk>', views.update_partner_view, name='update-partner'),
    path('delete-partner/<int:pk>', views.delete_partner_view, name='delete-partner'),
    path('admin-view-pending-partner', views.admin_view_pending_partner_view, name='admin-view-pending-partner'),
    path('approve-partner/<int:pk>', views.approve_partner_view, name='approve-partner'),
    path('reject-partner/<int:pk>', views.reject_partner_view, name='reject-partner'),

    path('admin-user', views.admin_user_view, name='admin-user'),
    path('admin-view-user', views.admin_view_user_view, name='admin-view-user'),
    path('update-user/<int:pk>', views.update_user_view, name='update-user'),
    path('delete-user/<int:pk>', views.delete_user_view, name='delete-user'),

    path('admin-course', views.admin_course_view, name='admin-course'),
    path('admin-add-course', views.admin_add_course_view, name='admin-add-course'),
    path('admin-view-course', views.admin_view_course_view, name='admin-view-course'),
    path('delete-course/<int:pk>', views.delete_course_view, name='delete-course'),

    ]


import os

from iplstats.models import Match, Delivery
from django.core.management.base import BaseCommand

class Command(BaseCommand):

    def handle(self, *args, **kwargs):
        basedir = os.path.abspath(os.path.dirname('matches.csv'))
        matches_csv = os.path.join(basedir, 'matches.csv')
        deliveries_csv = os.path.join(basedir, 'deliveries.csv')
        # Since the CSV headers match the model fields,
        # you only need to provide the file's path (or a Python file object)
        matches_count = Match.objects.from_csv(matches_csv)
        deliveries_count = Delivery.objects.from_csv(deliveries_csv)
        print (f"inserted {matches_count} matches and {deliveries_count} deliveries")


from typing import Union

from django.core.exceptions import ValidationError
from django.db import IntegrityError

from . import models
from .selectors import product_get

def stock_update(
    product: Union[models.Product, str],
    quantity: int = 0,
    validate: bool = True,
    replace: bool = False,
):
    """
    Update the stock of a product if has enought stock.
    Otherwise, raise ValidationError.

    :param: product: Product id or object to update.
    :param: quantity: Quantity to update.
    :param: validate: If True, validate the stock is >0.
    :param: replace: If True, replace the stock, otherwise, add the quantity.
    """
    if not isinstance(product, models.Product):
        product = product_get(product)

    if not replace:
        product.stock += quantity
    else:
        product.stock = quantity

    if product.stock < 0 and validate:
        raise ValidationError(f"Product {product.name} has no stock left.")

    product.save()
    return product

def product_delete(product: str):
    """
    Delete a product. Prevent deleting a product with orders.

    :param: product: Product id to delete.
    """
    product = product_get(product)
    try:
        product.delete()
    except IntegrityError:
        raise ValidationError(f"Product {product.name} has orders.")


from django.db import models
from django.contrib.auth.models import AbstractUser
from phonenumber_field.modelfields import PhoneNumberField
from courses.models import Course

# Create your models here.
class User(AbstractUser):
    STUDENT = 1
    TUTOR = 2
    STAFF = 3
    ROLE_CHOICES = (
        (STUDENT, 'student'),
        (TUTOR, 'tutor'),
        (STAFF, 'staff'),
    )
    date_of_birth = models.DateField(null=True)
    contact_no = PhoneNumberField(null=True)
    place = models.CharField(max_length=20, null=True)
    role = models.PositiveSmallIntegerField(choices=ROLE_CHOICES, null=True)

class Staff(models.Model):
    user_profile = models.OneToOneField(User, on_delete=models.CASCADE)
    designation = models.CharField(max_length=20)

class Tutor(models.Model):
    user_profile = models.OneToOneField(User, on_delete=models.CASCADE)
    course = models.ForeignKey(Course, on_delete=models.CASCADE, related_name='course_study')


# ============================================
# Implementando a interface via Streamlit
# ============================================
import streamlit as st
import numpy as np
import matrix_operations
import check_forms

st.title("Calculadora de Matrizes")

# =================================================
# Importação da classe de operações e formulários
# =================================================
cm = matrix_operations.Calculadora_Matriz()
cf = check_forms.Forms()

# =================================================
# Página para operações com escalar - Início
# =================================================
A, B = st.columns([6, 1])
A.write("Matriz A")
B.write("Escalar k")

# Primeira linha com todas as colunas
(
    lin1col1,
    lin1col2,
    lin1col3,
    lin1col4,
    lin1col5,
    lin1col6,
    lin1col7,
) = st.columns(7)
A11 = lin1col1.text_input("A11")
A12 = lin1col2.text_input("A12")
A13 = lin1col3.text_input("A13")
A14 = lin1col4.text_input("A14")
A15 = lin1col5.text_input("A15")
separador = lin1col6.write(" ")
B11 = lin1col7.text_input("k")

# Segunda linha com todas as colunas
(
    lin2col1,
    lin2col2,
    lin2col3,
    lin2col4,
    lin2col5,
    lin2col6,
    lin2col7,
) = st.columns(7)
A21 = lin2col1.text_input("A21")
A22 = lin2col2.text_input("A22")
A23 = lin2col3.text_input("A23")
A24 = lin2col4.text_input("A24")
A25 = lin2col5.text_input("A25")
separador = lin2col6.write(" ")
B21 = lin2col7.write("")

# Terceira linha com todas as colunas
(
    lin3col1,
    lin3col2,
    lin3col3,
    lin3col4,
    lin3col5,
    lin3col6,
    lin3col7,
) = st.columns(7)
A31 = lin3col1.text_input("A31")
A32 = lin3col2.text_input("A32")
A33 = lin3col3.text_input("A33")
A34 = lin3col4.text_input("A34")
A35 = lin3col5.text_input("A35")
separador = lin3col6.write(" ")
B31 = lin3col7.write("")

# Quarta linha com todas as colunas
(
    lin4col1,
    lin4col2,
    lin4col3,
    lin4col4,
    lin4col5,
    lin4col6,
    lin4col7,
) = st.columns(7)
A41 = lin4col1.text_input("A41")
A42 = lin4col2.text_input("A42")
A43 = lin4col3.text_input("A43")
A44 = lin4col4.text_input("A44")
A45 = lin4col5.text_input("A45")
separador = lin4col6.write(" ")
B41 = lin4col7.write("")

# Quinta linha com todas as colunas
(
    lin5col1,
    lin5col2,
    lin5col3,
    lin5col4,
    lin5col5,
    lin5col6,
    lin5col7,
) = st.columns(7)
A51 = lin5col1.text_input("A51")
A52 = lin5col2.text_input("A52")
A53 = lin5col3.text_input("A53")
A54 = lin5col4.text_input("A54")
A55 = lin5col5.text_input("A55")
separador = lin5col6.write(" ")
B51 = lin5col7.write("")

# ================================
# Área com os botões e métodos
# ================================
inputs_A = [
    [A11, A12, A13, A14, A15],
    [A21, A22, A23, A24, A25],
    [A31, A32, A33, A34, A35],
    [A41, A42, A43, A44, A45],
    [A51, A52, A53, A54, A55],
]

scalar = B11

# Botões
botao1, botao2, botao3, botao4 = st.columns([2, 3, 3, 3])
if botao1.button("Soma (A + k)"):
    # Se o escalar não for válido
    if not cf.check_scalar(scalar):
        st.write("ATENÇÃO - O escalar deve ter formato numérico válido.")
    # Se os dados da matriz A não estiverem corretos
    elif len(cf.get_matrix_A(inputs_A)) < 1:
        st.write("ATENÇÃO - A matriz A deve ter formato válido.")
    else:
        # Calcula o resultado chamando a classe em matrix_operations.py
        result = cm.soma_escalar(cf.get_matrix_A(inputs_A), scalar)

        # Transforma o resultado em Latex
        lines = str(result).replace("[", "").replace("]", "").splitlines()
        rv = [r"\begin{bmatrix}"]
        rv += ["  " + " & ".join(l.split()) + r"\\" for l in lines]
        rv += [r"\end{bmatrix}"]
        st.write("A soma da matriz A pelo escalar k é:")
        st.latex("".join(rv))

if botao2.button("Subtração (A - k)"):
    # Se o escalar não for válido
    if not cf.check_scalar(scalar):
        st.write("ATENÇÃO - O escalar deve ter formato numérico válido.")
    # Se os dados da matriz A não estiverem corretos
    elif len(cf.get_matrix_A(inputs_A)) < 1:
        st.write("ATENÇÃO - A matriz A deve ter formato válido.")
    else:
        # Calcula o resultado chamando a classe em matrix_operations.py
        result = cm.subtrai_escalar(cf.get_matrix_A(inputs_A), scalar)

        # Transforma o resultado em Latex
        lines = str(result).replace("[", "").replace("]", "").splitlines()
        rv = [r"\begin{bmatrix}"]
        rv += ["  " + " & ".join(l.split()) + r"\\" for l in lines]
        rv += [r"\end{bmatrix}"]
        st.write("A subtração da matriz A pelo escalar k é:")
        st.latex("".join(rv))

if botao3.button("Multiplicação (A * k)"):
    # Se o escalar não for válido
    if not cf.check_scalar(scalar):
        st.write("ATENÇÃO - O escalar deve ter formato numérico válido.")
    # Se os dados da matriz A não estiverem corretos
    elif len(cf.get_matrix_A(inputs_A)) < 1:
        st.write("ATENÇÃO - A matriz A deve ter formato válido.")
    else:
        # Calcula o resultado chamando a classe em matrix_operations.py
        result = cm.multiplica_escalar(cf.get_matrix_A(inputs_A), scalar)

        # Transforma o resultado em Latex
        lines = str(result).replace("[", "").replace("]", "").splitlines()
        rv = [r"\begin{bmatrix}"]
        rv += ["  " + " & ".join(l.split()) + r"\\" for l in lines]
        rv += [r"\end{bmatrix}"]
        st.write("A multiplicação da matriz A pelo escalar k é:")
        st.latex("".join(rv))

if botao4.button("Divisão (A / k)"):
    if B11 == "0":
        st.write("ATENÇÃO - DIVISÃO POR ZERO")
        st.write("Operação impossível!")
    else:
        # Se o escalar não for válido
        if not cf.check_scalar(scalar):
            st.write("ATENÇÃO - O escalar deve ter formato numérico válido.")
        # Se os dados da matriz A não estiverem corretos
        elif len(cf.get_matrix_A(inputs_A)) < 1:
            st.write("ATENÇÃO - A matriz A deve ter formato válido.")
        else:
            # Calcula o resultado chamando a classe em matrix_operations.py
            result = cm.divide_escalar(cf.get_matrix_A(inputs_A), scalar)

            # Transforma o resultado em Latex
            lines = str(result).replace("[", "").replace("]", "").splitlines()
            rv = [r"\begin{bmatrix}"]
            rv += ["  " + " & ".join(l.split()) + r"\\" for l in lines]
            rv += [r"\end{bmatrix}"]
            st.write("A divisão da matriz A pelo escalar k é:")
            st.latex("".join(rv))

# =================================================
# Página para operações com escalar - Fim
# =================================================
# ============================================
# Menu lateral - Início
# ============================================
st.sidebar.title("Instruções")

# Using "with" notation
with st.sidebar:
    st.text(
        "Para preencher a matriz corretamente\n"
        + "insira os valores nas posições dos\n"
        + "elementos desejados e deixe os \n"
        + "demais em branco (sem preencher).\n"
        + "\n"
        + "Exemplo 1:\n"
        + "Para somar duas matrizes quadradas\n"
        + "2x2 preencha os elementos\n"
        + "A11, A12, A21, A22 (Matriz A)\n"
        + "B11, B12, B21, B22 (Matriz B)\n"
        + "e pressione o botão 'Soma (A + B)'\n\n"
        + "Exemplo 2:\n"
        + "Para calcular calcular a transposta\n"
        + "de uma matriz 3x2 preencha os elem\n"
        + "A11, A12, A21, A22, A31, A32\n"
        + "e pressione o botão 'Transposta'"
    )
# ============================================
# Menu lateral - Fim
# ============================================


from flask import render_template, redirect, url_for, request, jsonify
from app import app
from app.forms import MainForm
from imdb import IMDb
import collections

@app.route('/', methods=['GET'])
@app.route('/search', methods=['GET'])
def search():
    matching_tv_series = list()
    ia = IMDb()
    query = request.args.get("seriesName")
    series_list = ia.search_movie(query)
    for series in series_list:
        if 'tv' in series['kind']:
            tv_obj = dict()
            series_name = series['smart long imdb canonical title']
            tv_obj['id'] = series.getID()
            tv_obj['URL'] = series.get_fullsizeURL()
            tv_obj['name'] = series_name;
            matching_tv_series.append(tv_obj)
    response = jsonify(matching_tv_series)
    # https://stackoverflow.com/questions/26980713/solve-cross-origin-resource-sharing-with-flask
    response.headers.add('Access-Control-Allow-Origin', '*')
    return response

@app.route('/index')
def index():
    user = {'username': 'Nithin'}
    return render_template('index.html', title='Home', user=user)

@app.route('/ratings')
def ratings():
    ia =IMDb()
    series = ia.get_movie(request.args.get('series_id'))
    ia.update(series, 'episodes')
    episodes_dict = dict()
    episodes_dict['title']       = series.get('original title')
    episodes_dict['rating']      = series.get('rating')
    episodes_dict['num_seasons'] = series.get('number of seasons')
    episodes_dict['coverURL']    = series.get_fullsizeURL()
    episodes_dict['URL']         = 'http://www.imdb.com/title/tt{0}/'.format(request.args.get('series_id'))

    show_ratings = dict()
    for season, episodes in series['episodes'].items():
        if season not in show_ratings:
            show_ratings[season] = dict()
        for episode_num, episode in episodes.items():
            ep_data = dict()
            rating_val = episode.get('rating')
            title_val = episode.get('title')
            if rating_val is not None:
                ep_data['rating'] = "{:.2f}".format(rating_val)
                if title_val is not None:
                    ep_data['title'] = title_val
                else:
                    ep_data['title'] = "Unknown"
                show_ratings[season][episode_num] = ep_data
        if not show_ratings[season]:
            del show_ratings[season]
    ratings_list = []
    od = collections.OrderedDict(sorted(show_ratings.items()))
    for k, v in od.items():
        if k >= 0:
            ratings_list.append(v)
    episodes_dict['results'] = ratings_list

    response = jsonify(episodes_dict)
    response.headers.add('Access-Control-Allow-Origin', '*')
    return response


import torch
import torch.nn as nn
import numpy as np
from torch.optim import Adam
from torch.utils.data import DataLoader
from torch.utils.data import Dataset
from torch.utils.data import TensorDataset

from torchvision.datasets import MNIST
from torchvision.transforms import ToTensor
from tqdm import tqdm

class LeNet(nn.Module):
    def __init__(self):
        super(LeNet, self).__init__()
        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=6, kernel_size=5, padding=2)
        self.cnn1_act = nn.Tanh()
        self.avgpool1 = nn.AvgPool2d(kernel_size=2, stride=2)

        self.cnn2 = nn.Conv2d(in_channels=6, out_channels=16, kernel_size=5)
        self.cnn2_act = nn.Tanh()
        self.avgpool2 = nn.AvgPool2d(kernel_size=2, stride=2)

        self.cnn3 = nn.Conv2d(in_channels=16, out_channels=120, kernel_size=5)
        self.cnn3_act = nn.Tanh()

        self.fc1 = nn.Linear(in_features=120, out_features=84)
        self.fc1_act = nn.Tanh()

        self.fc2 = nn.Linear(in_features=84, out_features=10)
        # self.out = nn.Softmax(dim=-1)

    def forward(self, x):
        x = self.cnn1(x)
        x = self.cnn1_act(x)
        x = self.avgpool1(x)
        x = self.cnn2(x)
        x = self.cnn2_act(x)
        x = self.avgpool2(x)
        x = self.cnn3(x)
        x = self.cnn3_act(x)

        x = x.view(x.size(0), -1)
        x = self.fc1(x)
        x = self.fc1_act(x)
        x = self.fc2(x)
        return x

def random_test_lenet():
    model = LeNet()
    # B, C, H, W
    rand_test = torch.randn((10, 1, 28, 28))
    rst = model(rand_test)

    if rst.shape == (10, 10):
        print("test passed")

def run_lenet():
    # random test
    # random_test_lenet()

    # config
    DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    # print(model)
    EPOCHS = 3
    LR = 0.001
    BATCH_SIZE = 64
    N_SAMPLES = 60000

    # MNIST config
    dataset = MNIST(root='data', train=True, download=True, transform=ToTensor())
    dataloader = DataLoader(dataset, batch_size=BATCH_SIZE)

    model = LeNet().to(DEVICE)
    loss_fn = nn.CrossEntropyLoss()
    optimizer = Adam(model.parameters(), lr=LR)

    losses, accs = list(), list()

    for e in range(EPOCHS):
        epoch_loss, n_corrects = 0., 0

        for X_, y_ in tqdm(dataloader):
            X_, y_ = X_.to(DEVICE), y_.to(DEVICE)
            pred = model(X_)
            loss = loss_fn(pred, y_)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            epoch_loss += loss
            pred_cls = torch.argmax(pred, dim=1)
            n_corrects += (pred_cls == y_).sum().item()

        epoch_loss /= N_SAMPLES
        epoch_accr = n_corrects / N_SAMPLES

        print(f"epoch {e} : loss={epoch_loss.item():.4f}, accr={epoch_accr}")

    losses.append(epoch_loss)
    accs.append(epoch_accr)
    print(losses)
    print(accs)

if __name__ == '__main__':
    run_lenet()


import torch

from fastai import *
from fastai.vision import *
from torch.autograd import Variable

import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np

class Network(nn.Module):
    def __init__(self):
        super(Network, self).__init__()
        self.lin1 = nn.Linear(1025, 200)
        self.lin2 = nn.Linear(200, 150)
        self.lin3 = nn.Linear(150, 2)

        # Do Xavier initialization
        nn.init.xavier_normal_(self.lin1.weight)
        nn.init.xavier_normal_(self.lin2.weight)
        nn.init.xavier_normal_(self.lin3.weight)

    def forward(self, x):
        x = F.relu(self.lin1(x))
        x = F.relu(self.lin2(x))
        x = self.lin3(x)
        return x

def process_batch(batch):
    batch_size = batch[0].shape[0]
    inputs=np.array(batch[0])
    inputs=inputs[:,0,:,:] # just take the first color (grayscale image anyway)
    inputs=np.reshape(inputs,(inputs.shape[0],-1))
    inputs=np.concatenate((inputs,np.ones((batch_size,1))), axis=1)
    labels=np.array(batch[1]).transpose()

    # Re-orient the inputs
    return inputs, labels

def calculate_error_rate(batch, predicted, labels):
    correct = 0
    total = batch[0].shape[0]
    for i in range(total):
        correct += (predicted[i] == labels[i]).sum().item()
    err_rate = 1 - correct/total
    return correct, err_rate

if __name__ == "__main__":
    # Load data from MNIST dataset
    mnist = untar_data(URLs.MNIST_TINY)
    tfms = get_transforms(do_flip=False)

    # List the data in training, validation and test sets
    data = (ImageList.from_folder(mnist)
            .split_by_folder()
            .label_from_folder()
            .add_test_folder(f'{mnist}' +'/test/')
            .transform(tfms, size=32)
            .databunch()
            .normalize(imagenet_stats))

    # Show the data
    # data.show_batch()

    # Get the first MNIST batch from fastai
    batch = data.one_batch()
    batch_size, no_channels, dimx, dimy = batch[0].shape

    # Set up the PyTorch model
    net = Network()

    # Set up the optimization system
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(net.parameters(), lr=0.0005)

    # Reset the gradients
    optimizer.zero_grad()

    # Train the PyTorch model
    for epoch in range(50):  # loop over the dataset multiple times
        x, labels = process_batch(batch)

        # Convert inputs and labels to PyTorch variable
        x = Variable(torch.from_numpy(x).float())
        labels = Variable(torch.from_numpy(labels))

        # Run a forward pass
        outputs = net(x)

        # Compute loss and backpropagate
        loss = criterion(outputs, labels)
        loss.backward()

        # Optimize parameters
        optimizer.step()

        # Calculate error rate
        _, predicted = torch.max(outputs, 1)
        _, err_rate = calculate_error_rate(batch, predicted, labels)

        # Print statistics
        print(f"Cost: {loss.item()}, Error rate: {err_rate}")

        # Get the next MNIST batch from fastai
        batch = data.one_batch()

    print('Training Complete!')

    print('Running on the validation set...')
    num_correct = 0
    total = 0
    for i in range(50):
        # Test the PyTorch model on the validation data
        # Get the test batch
        valid_batch = data.one_batch(DatasetType.Valid)
        x, labels = process_batch(valid_batch)

        # Convert inputs and labels to PyTorch variable
        x = Variable(torch.from_numpy(x).float())
        labels = Variable(torch.from_numpy(labels))

        # Run on the test set and find the error
        outputs = net(x)
        _, predicted = torch.max(outputs, 1)

        # Calculate error rate
        correct, err_rate = calculate_error_rate(valid_batch, predicted, labels)
        num_correct += correct
        total += valid_batch[0].shape[0]

        print(f"Error rate: {err_rate}")

    err_rate = 1 - num_correct / total

    # Display statistics
    print('Validation Results')
    print(f"Total error rate: {err_rate}")


import os
import glob
import pandas as pd
import cv2
import sys
import tqdm
sys.path.append('../')

import Config

class GetTrainTestCSV:
    """
    img                 txt
    img abspath         txt abspath or None
    """
    def __init__(self, dataset_path=Config.READ_PATH_DATASET, csv_name=Config.CSV_NAME):
        self.data_path = dataset_path
        self.img_formats = Config.IMG_FORMATS
        self.save_path_csv = r'generate_dep_info'
        if not os.path.exists(self.save_path_csv):
            os.makedirs(self.save_path_csv)
        self.csv_name = csv_name

    def get_csv(self):
        data_information = {'img': [], 'label': []}
        for idx, data_dir in enumerate(self.data_path):
            if len(self.data_path) == len(self.img_formats):
                img_format = self.img_formats[idx]
            else:
                img_format = self.img_formats[0]
            assert os.path.exists(data_dir), 'No dir ' + data_dir
            img_file_list = glob.glob(
                os.path.join(data_dir, '*{0}'.format(img_format)))
            assert len(img_file_list), 'No data in DATASET_PATH!'
            pbar = tqdm.tqdm(total=len(img_file_list))
            for img_file in img_file_list:
                pbar.update(1)
                img = cv2.imread(img_file)
                if img is None:
                    continue
                data_information['img'].append(img_file)
                txt_file = img_file.replace(img_format, '.txt')
                if not os.path.exists(txt_file):
                    txt_file = None
                data_information['label'].append(txt_file)
            pbar.close()
        data_annotation = pd.DataFrame(data_information)
        writer_name = os.path.join(self.save_path_csv, '{0}.csv'.format(self.csv_name))
        data_annotation.to_csv(writer_name)
        print(os.path.basename(writer_name) + ' file saves successfully!')

if __name__ == '__main__':
    getTrainTestCSV = GetTrainTestCSV()
    getTrainTestCSV.get_csv()


import argparse
import base64
import hashlib
import json
import time
from datetime import datetime

from flask import request, jsonify, Blueprint

from config import Config
from predict import predict
from flask import Flask

app = Flask(__name__)
app.secret_key = '1234567'

api = Blueprint('changhai', __name__)

@api.route('/')
def show():
    return 'This is changhai api.'

@api.route('/ner/', methods=['POST'], strict_slashes=False)
def recognize():
    if request.method == 'POST':
        # print(request.date)
        data = json.loads(request.get_data())
        txt = data['txt']
        timestamp = data['timestamp']
        sgt = data['signiture']
        if txt is None or txt is []:
            return jsonify({'success': False, 'description': {'error msg': 'invalid post body fields'}}), 500
        elif type(txt) == list:
            valid, msg = signiture(txt, timestamp, sgt)
            # print(valid, msg)
            if not valid:
                return jsonify({'success': False, 'description': {'error msg': msg}}), 500
            else:
                try:
                    result = processt_text(txt)
                    result = {'success': True,
                              'data': result,
                              'description': "success"}
                    return jsonify(result), 200

                except Exception as e:
                    print("/n******ERROR SRART******/n")
                    print(e)
                    print("----------txt----------")
                    print(txt)
                    print("/n*******ERROR END*******/n")
                    return jsonify({'success': False, 'description': {'error msg': str(e)}}), 500
        else:
            return jsonify({'success': False, 'description': {'error msg': "invalid type"}}), 500
    else:
        return jsonify({'success': False, 'description': {'error msg': 'Invalid methods'}}), 404

def signiture(txt, timestamp, signiture):
    ts1 = int(timestamp)
    ts2 = int(round(time.time()))
    dt1 = datetime.utcfromtimestamp(ts1)
    dt2 = datetime.utcfromtimestamp(ts2)
    # print(dt1)
    # print(dt2)
    # print((dt2 - dt1).total_seconds())
    if float((dt2 - dt1).total_seconds()) < 120.0:
        base64_encode = base64.b64encode(("".join([i[0] for i in txt]) + timestamp).encode("utf-8"))
        md5 = hashlib.md5()
        md5.update(str(base64_encode, 'UTF-8').encode('utf-8'))
        md5_encode = md5.hexdigest().upper()
        # print(md5_encode)
        # print(signiture)
        if signiture.upper() == md5_encode:
            return True, ""
        else:
            return False, "invalid signiture"
    else:
        return False, "Timestamp exceed"

def processt_text(txts):
    parser = argparse.ArgumentParser()
    parser.add_argument('--config', type=str, default='./config/changhai.json')
    parser.add_argument('--device', type=int, default=0)

    parser.add_argument('--dist_emb_size', type=int)
    parser.add_argument('--type_emb_size', type=int)
    parser.add_argument('--lstm_hid_size', type=int)
    parser.add_argument('--conv_hid_size', type=int)
    parser.add_argument('--bert_hid_size', type=int)
    parser.add_argument('--ffnn_hid_size', type=int)
    parser.add_argument('--biaffine_size', type=int)

    parser.add_argument('--dilation', type=str, help="e.g. 1,2,3")

    parser.add_argument('--emb_dropout', type=float)
    parser.add_argument('--conv_dropout', type=float)
    parser.add_argument('--out_dropout', type=float)

    parser.add_argument('--epochs', type=int)
    parser.add_argument('--batch_size', type=int)

    parser.add_argument('--clip_grad_norm', type=float)
    parser.add_argument('--learning_rate', type=float)
    parser.add_argument('--weight_decay', type=float)

    parser.add_argument('--bert_name', type=str)
    parser.add_argument('--bert_learning_rate', type=float)
    parser.add_argument('--warm_factor', type=float)

    parser.add_argument('--use_bert_last_4_layers', type=int, help="1: true, 0: false")

    parser.add_argument('--seed', type=int)

    args = parser.parse_args()
    config = Config(args)
    result = json.loads(predict(txts,
                                lambda text: list(text),
                                config,
                                batch_size=255))

    for i, item in enumerate(result):
        entities = []
        for j, _ in enumerate(result[i]["entities"]):
            e = {"text": ''.join(result[i]["entities"][j][0]),
                 "category": result[i]["entities"][j][1]}
            if e not in entities:
                entities.append(e)
        result[i]["entities"] = entities
        result[i]["text"] = ''.join(result[i]["text"])
    return result

app.register_blueprint(api, url_prefix='/disctn')

if __name__ == '__main__':
    # from werkzeug.contrib.fixers import ProxyFix
    # app.wsgi_app = ProxyFix(app.wsgi_app)
    app.run(debug=True, port=5020, host='0.0.0.0')


from mock import patch, MagicMock
from nose.tools import assert_raises
import json
from pybossa.s3_client import S3Client, NoSuchBucket, PrivateBucket

class TestS3Client(object):

    def make_response(self, text, status_code=200):
        fake_response = MagicMock()
        fake_response.text = text
        fake_response.status_code = status_code
        return fake_response

    bucket_with_content = (
     """<?xml version="1.0" encoding="UTF-8"?>
        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
            <Name>test-pybossa</Name>
            <Prefix></Prefix>
            <Marker></Marker>
            <MaxKeys>1000</MaxKeys>
            <IsTruncated>false</IsTruncated>
            <Contents>
                <Key>16535035993_1080p.mp4</Key>
                <LastModified>2016-01-29T08:55:41.000Z</LastModified>
                <ETag>"10055dfebe62cf30e34d87fd27b28efc"</ETag>
                <Size>11801468</Size>
                <StorageClass>STANDARD</StorageClass>
            </Contents>
            <Contents>
                <Key>BFI-demo.mp4</Key>
                <LastModified>2016-01-29T08:55:38.000Z</LastModified>
                <ETag>"b24442a1484b6b8f2b4e08c43e0abd3f"</ETag>
                <Size>27063915</Size>
                <StorageClass>STANDARD</StorageClass>
            </Contents>
        </ListBucketResult>
        """)

    empty_bucket = (
     """<?xml version="1.0" encoding="UTF-8"?>
        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
            <Name>test-pybossa</Name>
            <Prefix></Prefix>
            <Marker></Marker>
            <MaxKeys>1000</MaxKeys>
            <IsTruncated>false</IsTruncated>
        </ListBucketResult>
        """)

    no_such_bucket = (
     """<?xml version="1.0" encoding="UTF-8"?>
        <Error>
            NoSuchBucket
            <Message>The specified bucket does not exist</Message>
            <BucketName>test-pybosa</BucketName>
            <RequestId>5DB95818E2273F2A</RequestId>
            <HostId>2xqg6pMK20zocCIN0DpqzDVEmbNkqKdTrp0BT/K2EUBbSIek5+7333DjDVuvpN0fFR/Pp/+IkM8=</HostId>
        </Error>
        """)

    bucket_with_folder = (
     """<?xml version="1.0" encoding="UTF-8"?>
        <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
            <Name>test-pybossa</Name>
            <Prefix></Prefix>
            <Marker></Marker>
            <MaxKeys>1000</MaxKeys>
            <IsTruncated>false</IsTruncated>
            <Contents>
                <Key>myfolder/</Key>
                <LastModified>2016-01-29T08:56:15.000Z</LastModified>
                <ETag>"d41d8cd98f00b204e9800998ecf8427e"</ETag>
                <Size>0</Size>
                <StorageClass>STANDARD</StorageClass>
            </Contents>
        </ListBucketResult>
        """)

    private_bucket = (
     """<?xml version="1.0" encoding="UTF-8"?>
        <Error>
            AccessDenied
            <Message>Access Denied</Message>
            <RequestId>0C189C667703869B</RequestId>
            <HostId>e6HNleTSx+vQHCXsjphJNLumbwd2YfYfZMrEBEkGOF/0jCMDZf6RIrgUAooa+HT86f0Azr27/h4=</HostId>
        </Error>
        """)

    @patch('pybossa.s3_client.requests')
    def test_objects_return_empty_list_for_an_empty_bucket(self, requests):
        resp = self.make_response(self.empty_bucket, 200)
        requests.get.return_value = resp

        objects = S3Client().objects('test-pybossa')

        assert objects == [], objects

    @patch('pybossa.s3_client.requests')
    def test_objects_return_list_of_object_names_in_a_bucket(self, requests):
        resp = self.make_response(self.bucket_with_content, 200)
        requests.get.return_value = resp

        objects = S3Client().objects('test-pybossa')

        assert objects == ['16535035993_1080p.mp4', 'BFI-demo.mp4'], objects

    @patch('pybossa.s3_client.requests')
    def test_objects_not_returns_folders_inside_bucket(self, requests):
        resp = self.make_response(self.bucket_with_folder, 200)
        requests.get.return_value = resp

        objects = S3Client().objects('test-pybossa')

        assert objects == [], objects

    @patch('pybossa.s3_client.requests')
    def test_objects_raises_NoSuchBucket_if_bucket_does_not_exist(self, requests):
        resp = self.make_response(self.no_such_bucket, 404)
        requests.get.return_value = resp

        assert_raises(NoSuchBucket, S3Client().objects, 'test-pybossa')

    @patch('pybossa.s3_client.requests')
    def test_objects_raises_PrivateBucket_if_bucket_is_private(self, requests):
        resp = self.make_response(self.no_such_bucket, 403)
        requests.get.return_value = resp

        assert_raises(PrivateBucket, S3Client().objects, 'test-pybossa')


import json
import random
import urllib.request
import urllib.parse

from app.db.base_class import Base
from app.db.session import db1, db2
from sqlalchemy.orm import Session

from app import crud, schemas
from app.db.models.relationship import Relationship
from app.core.config import settings

def generate_base_url(service_name: str, api_prefix: str) -> str:
    return f"http://{service_name}:80{api_prefix}/"

def init_db(db: Session) -> None:
    for dbb in (db1, db2):
        Base.metadata.drop_all(dbb)
        Base.metadata.create_all(dbb)

    users_resp = urllib.request.urlopen(generate_base_url(settings.USERS_SERVICE_NAME, settings.API_USERS))
    user_ids = list(map(
        lambda u: u['id'],
        json.loads(users_resp.read().decode('utf-8'))['users']
    ))

    user_count = len(user_ids)
    followers_per_user = min(2, user_count)

    # db.query(Relationship).delete()
    for user_id in user_ids:

        # Randomize users followers until not following themselves
        follower_ids = []
        while True:
            follower_ids = random.sample(user_ids, followers_per_user)
            if user_id not in follower_ids:
                break

        for follower_id in follower_ids:
            relationship = schemas.RelationshipCreate(
                followed_user_id=user_id,
                user_id=follower_id
            )
            crud.relationship.create(db, obj_in=relationship)

    db.commit()


import sys, os, argparse
from rawkit.raw import Raw
import numpy as np
from PIL import Image

def raw_to_jpg(file):
	try:
		with Raw(file) as raw_image:
			buffered_image = np.array(raw_image.to_buffer())
			im = Image.frombuffer('RGB', (raw_image.metadata.width, raw_image.metadata.height), buffered_image, 'raw', 'RGB', 0, 1)
			im.save(os.path.split(file)[1] + '.jpg')
			print('Successfully saved file as JPG.')
	except OSError:
		print('Invalid file or location. Check your data. File: %s' % file)

def check_if_raw(file):
	filename = file
	try:
		filename_ext = filename.split('.')

		if filename_ext[1] == 'CR2':
			print('PASS: \'%s\' is CR2 file.' % filename)
			raw_to_jpg(filename)
		else:
			print('FAIL: \'%s\' is \'%s\', not CR2.' % (filename, filename_ext[1]))

	except IndexError:
		print('FAIL: \'%s\' is not a valid file.' % filename)

photo = sys.argv[1]
if os.path.exists(photo):
	raw_to_jpg(photo)
else:
	print("Not a valid file")


import numpy as np
import glob
import cv2
import os
import matplotlib.pyplot as plt
import argparse
import yaml, json
import math
import pandas as pd
import seaborn as sns

def deg2rad(deg):
    return deg * math.pi / 180

def rad2deg(rad):
    return rad * 180 / math.pi

def mse(v1, v2):
    v = v1 - v2
    return np.linalg.norm(v, axis=-1)

def cosErr(v1, v2):
    nv1 = np.linalg.norm(v1, axis=-1)
    nv2 = np.linalg.norm(v2, axis=-1)
    costheta = (v1*v2).sum(-1) / (nv1*nv2)
    return np.arccos(costheta)

class Evaluator(object):
    """docstring for Evaluator"""

    def __init__(self, conf, labels, results, out, maskfile=None, infofile=None, loadnpy=False):
        super(Evaluator, self).__init__()
        self.labels = labels
        self.results = results
        self.out = out
        self.maskfile = maskfile
        self.infofile = infofile
        self.loadnpy = loadnpy

        fconf = open(conf)
        self.p_cfg = json.load(fconf)
        fconf.close()

        # get config info
        self.c_patt = self.p_cfg['pattern']
        self.eval()

    def eval(self):
        if self.loadnpy:
            print("loading labels & results from .npy")
            self.gt = np.load(self.labels)
            self.re = np.load(self.results)
            self.info = np.load(self.infofile)
            self.mask = np.load(self.maskfile)
        else:
            self.loaddata()
            self.saveLabel()
            self.saveResult()

        self.invalid_rate = np.logical_not(self.mask).sum() / self.mask.size

        self.fig = plt.figure(figsize=(6,6))

        # pitch matter
        self.pitchMatter()

        plt.show()

    def pitchMatter(self):
        print("performing pitch matter evaluation...")
        print("section size: {} (degree)".format(5))
        # each section has 5 degree
        section = deg2rad(5)

        # calculate pitch
        re_p = self.info[:, 0]
        re_p = (re_p // section).astype('int')
        re_px = np.copy(re_p)
        re_px[np.logical_not(self.mask)] = -1

        # calculate error
        self.p_counts = []
        self.p_success = []
        self.p_terrs = []
        self.p_rerrs_i = []
        self.p_rerrs_j = []
        self.p_rerrs_k = []
        for i in range(re_p.min(), re_p.max()+1):
            print("section #{} ...".format(i))

            # mask of selected condition and its success samples
            sec_mask = re_p == i
            sec_mask_succ = re_px == i

            # samples that match the selected condition and its success rate
            self.p_counts.append(sec_mask.sum())
            self.p_success.append(sec_mask_succ.sum()/sec_mask.sum())

            if self.p_success[-1] > 0:
                sec_gt = self.gt[sec_mask_succ]
                sec_re = self.re[sec_mask_succ]

                # change cv2's detection of chessboard to our defined coordination
                sec_re = sec_re if self.c_patt == 'two_circle' else sec_re * \
                    np.array([1, -1, -1, 1, -1, -1, 1, -1, -1, 1, -1, -1])

                terr = mse(sec_re[:,0:3], sec_gt[:,0:3])
                rerrs_i = cosErr(sec_re[:,3:6],  sec_gt[:,3:6])
                rerrs_j = cosErr(sec_re[:,6:9],  sec_gt[:,6:9])
                rerrs_k = cosErr(sec_re[:,9:12], sec_gt[:,9:12])

                self.p_terrs.append(terr)# / self.p_success[i])
                self.p_rerrs_i.append(rerrs_i)# / self.p_success[i])
                self.p_rerrs_j.append(rerrs_j)# / self.p_success[i])
                self.p_rerrs_k.append(rerrs_k)# / self.p_success[i])
            else:
                self.p_terrs.append([np.nan, np.nan, np.nan, np.nan, np.nan])
                self.p_rerrs_i.append([np.nan, np.nan, np.nan, np.nan, np.nan])
                self.p_rerrs_j.append([np.nan, np.nan, np.nan, np.nan, np.nan])
                self.p_rerrs_k.append([np.nan, np.nan, np.nan, np.nan, np.nan])
        self.showPitchMatter()

    def showPitchMatter(self):
        print("preparing the result")
        # Add one subplot
        na = len(self.fig.axes)
        for i in range(na):
            self.fig.axes[i].change_geometry(1, na+1, i+1)

        section = deg2rad(5)
        re_p = self.info[:, 0]
        re_p = (re_p // section).astype('int')

        n_sec = [len(i) for i in self.p_rerrs_i]
        p_terrs = np.concatenate(self.p_terrs)
        p_rerrs_i = np.concatenate(self.p_rerrs_i)
        p_rerrs_j = np.concatenate(self.p_rerrs_j)
        p_rerrs_k = np.concatenate(self.p_rerrs_k)

        pit_label = []
        for i,n in enumerate(range(re_p.min(), re_p.max()+1)):
            for nn in range(n_sec[i]):
                pit_label.append("{:.3f} - {:.3f}".format(n*section, (n+1)*section))
        # pit_label = ["{:.3f} - {:.3f}".format(i*section, (i+1)*section) for i in range(re_p.min(), re_p.max()+1)]
        type_label = ["Translation Error (m)", "Rotation Error (rad)"]
        tra_label = ["camera position"]
        rot_label = ["axis_i", "axis_j", "axis_k"]

        # create DataFrame for rotation error
        rot_labels = pd.MultiIndex.from_product([[type_label[1]], rot_label, pit_label],
            names=['type', 'station', 'measurement'])
        rot_data = pd.DataFrame({'value': np.concatenate((p_rerrs_i, p_rerrs_j, p_rerrs_k))}, index=rot_labels)
        rot_data = rot_data.reset_index()

        # create DataFrame for translation error
        tra_labels = pd.MultiIndex.from_product([[type_label[0]], tra_label, pit_label],
            names=['type', 'station', 'measurement'])
        tra_data = pd.DataFrame({'value': p_terrs}, index=tra_labels)
        tra_data = tra_data.reset_index()

        # plot rotation error
        g1 = sns.FacetGrid(rot_data, col="measurement", row="type", height=3, aspect=2, sharey=True, sharex=True)
        g1.map(sns.boxplot, "station", "value", palette=['lightcoral', 'lightgreen', 'lightblue'])
        g1.set_axis_labels("", "val").set_titles("{col_name}").despine(bottom=True, left=True)
        g1.axes[0,0].set_ylabel(type_label[1])
        g1.set(ylim=(0, math.pi/2))
        g1.fig.suptitle('Pitch Matter Rotation error', y=0.99)
        g1.fig.set_size_inches(15, 4)
        for ax in g1.axes:
            for a in ax:
                a.grid(True)
        g1.savefig(os.path.join(self.out, "rotErr.png"))

        # plot translation error
        g2 = sns.FacetGrid(tra_data, col="measurement", row="type", height=3, aspect=1, sharey=True, sharex=True)
        g2.map(sns.boxplot, "station", "value", palette=['orange'])
        g2.set_axis_labels("", "val").set_titles("{col_name}").despine(bottom=True, left=True)
        g2.axes[0,0].set_ylabel(type_label[0])
        g2.set(ylim=(0, 30))
        g2.fig.suptitle('Pitch Matter Translation error', y=0.99)
        g2.fig.set_size_inches(15, 4)
        for ax in g2.axes:
            for a in ax:
                a.grid(True)
        g2.savefig(os.path.join(self.out, "traErr.png"))

        # Plot success rate
        self.ax = self.fig.add_subplot(1, na+1, na+1)
        self.ax.set_title("Pitch Matter Detection ({})".format(self.c_patt))
        self.ax.set_xlabel("Pitch (rad)")
        self.ax.set_ylabel("Success rate")
        self.ax.bar(np.arange(len(self.p_success)), self.p_success, 0.35, label='success rate')
        self.ax.set_xticks(np.arange(len(self.p_success)))
        self.ax.set_xticklabels(pit_label, rotation=45)
        self.ax.legend(loc="best")
        self.fig.tight_layout()
        self.fig.savefig(os.path.join(self.out, "succ.png"))

    def loaddata(self):
        print("loading labels & results...")
        assert len(self.labels) == len(self.results), 'len(self.labels) != len(self.results)'

        self.samples = len(self.labels)
        # shape: (samples, (pitch, tx, ty, distance, size, space))
        self.info = np.zeros((self.samples, 6))
        # shape: (samples, (trans(3), rotation(9)))
        self.gt = np.zeros((self.samples, 3+9))
        self.re = np.zeros((self.samples, 3+9))
        self.mask = np.zeros((self.samples), bool)

        for i, (l, r) in enumerate(zip(self.labels, self.results)):
            # Read label file
            with open(l, 'r') as fl:
                label_data = yaml.load(fl, Loader=yaml.FullLoader)

            # Load infomation
            self.info[i][0] = label_data['extrinsic']['rotation'][2]
            self.info[i][1] = label_data['extrinsic']['translation'][0]
            self.info[i][2] = label_data['extrinsic']['translation'][1]
            self.info[i][3] = label_data['extrinsic']['init_distance']
            if self.c_patt == "two_circle":
                self.info[i][4] = label_data['radius']
                self.info[i][5] = label_data['circle_shift']
            elif self.c_patt == "chessboard":
                self.info[i][4] = label_data['cube_size']
                self.info[i][5] = 0

            # Load label
            self.gt[i][0:3] = label_data['spcs']['cam_center']
            self.gt[i][3:6] = label_data['spcs']['axis'][0]
            self.gt[i][6:9] = label_data['spcs']['axis'][1]
            self.gt[i][9:12] = label_data['spcs']['axis'][2]

            # Read result file
            fr = cv2.FileStorage(r, cv2.FILE_STORAGE_READ)

            # Check detection
            if fr.getNode('success').real() == 1.:
                # Load result
                self.mask[i] = True
                self.re[i][0:3] = fr.getNode('cam_position_spcs').mat().reshape(-1)
                self.re[i][3:6] = fr.getNode('cam_pose_i_spcs').mat().reshape(-1)
                self.re[i][6:9] = fr.getNode('cam_pose_j_spcs').mat().reshape(-1)
                self.re[i][9:12] = fr.getNode('cam_pose_k_spcs').mat().reshape(-1)
            else:
                # Faild detection
                self.mask[i] = False

    def saveLabel(self):
        print("saving labels to npy...")
        name = self.labels[0].split(os.sep)
        name = os.sep.join(name[:-1])
        name_gt = os.path.join(name, 'label.npy')
        with open(name_gt, 'wb') as f:
            np.save(f, self.gt)
        name_info = os.path.join(name, 'info.npy')
        with open(name_info, 'wb') as f:
            np.save(f, self.info)

    def saveResult(self):
        print("saving results to npy...")
        name = self.results[0].split(os.sep)
        name = os.sep.join(name[:-2])
        name_data = os.path.join(name, 'data.npy')
        name_mask = os.path.join(name, 'mask.npy')
        with open(name_data, 'wb') as f:
            np.save(f, self.re)
        with open(name_mask, 'wb') as f:
            np.save(f, self.mask)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Camera Pose Estimation')
    parser.add_argument('--conf', type=str, required=True,
                        help='path to generate config file')
    parser.add_argument('--label', type=str, required=True,
                        help='path to input image files')
    parser.add_argument('--result', type=str, required=True,
                        help='path to pose estimation output files')
    parser.add_argument('--mask', type=str,
                        help='path to success estimation mask npy files')
    parser.add_argument('--info', type=str,
                        help='path to success label info npy files')
    parser.add_argument('--out', type=str, required=True,
                        help='path to save output figures')
    args = parser.parse_args()
    print(args)

    if args.label.endswith('npy') and args.result.endswith('npy') and args.mask != None:
        ev = Evaluator(args.conf, args.label, args.result, out=args.out, maskfile=args.mask, infofile=args.info, loadnpy=True)
    else:
        labels = glob.glob(os.path.join(args.label,  'label_*.yml'))
        results = glob.glob(os.path.join(args.result, '*'))
        results = [os.path.join(r, 'pose.yml') for r in results if os.path.isdir(r)]

        print('{} label files'.format(len(labels)))
        print('{} result files'.format(len(results)))
        ev = Evaluator(args.conf, labels, results, out=args.out)


# Generic
from datetime import datetime

from baking.main import db

class Genre(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(90))

    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return '<Genre %r>' % self.name

class Artist(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(160))
    clean_name = db.Column(db.String(160))
    cover = db.Column(db.Boolean, nullable=True)
    formation_date = db.Column(db.DateTime)
    country = db.Column(db.String(120))
    genre_id = db.Column(db.Integer, db.ForeignKey(Genre.id))
    genre = db.relationship(Genre, backref=db.backref('artist', lazy='dynamic'))

    def __init__(self, name, clean_name, country, genre, formation_date=None, cover=False):
        self.name = name
        self.clean_name = clean_name
        self.country = country

        if formation_date is None:
            formation_date = datetime.utcnow()

        self.cover = cover
        self.formation_date = formation_date

        self.genre = genre

    def __repr__(self):
        return '<Artist %r %r>' % (self.clean_name, self.name)

class Song(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.String(150))
    lyrics = db.Column(db.Text)
    publication_date = db.Column(db.DateTime)
    artist_id = db.Column(db.Integer, db.ForeignKey(Artist.id))
    artist = db.relationship(Artist, backref=db.backref('song', lazy='dynamic'))

    def __init__(self, title, lyrics, artist, publication_date=None):
        self.title = title
        self.lyrics = lyrics
        self.artist = artist

        if publication_date is None:
            publication_date = datetime.utcnow()
        self.publication_date = publication_date

    def __repr__(self):
        return '<Song %r>' % self.title

    def as_dict(self):
        return {
            'id': self.id,
            'title': self.title,
            'lyrics': self.lyrics,
            'publication_date': self.publication_date,
            'artist': self.artist.name
        }

    def __str__(self):
        return str(self.as_dict())

class ArtificialTitle(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.String(180))
    creation_date = db.Column(db.DateTime)

    def __init__(self, title, creation_date=None):
        self.title = title

        if creation_date is None:
            creation_date = datetime.utcnow()
        self.creation_date = creation_date

    def __repr__(self):
        return '<ArtificialTitle %r>' % self.title

class ArtificialSong(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    artificial_title_id = db.Column(db.Integer, db.ForeignKey(ArtificialTitle.id))
    artificial_title = db.relationship(ArtificialTitle, backref=db.backref('artificialSong', lazy='dynamic'))
    lyrics = db.Column(db.Text)
    model = db.Column(db.String(150))
    creation_date = db.Column(db.DateTime)
    # user_id = db.Column(db.Integer, db.ForeignKey(User.id))
    # user = db.relationship(User, backref=db.backref('artificialSong', lazy='dynamic'))
    base_artist_id = db.Column(db.Integer, db.ForeignKey(Artist.id))
    base_artist = db.relationship(Artist, backref=db.backref('artificialSong', lazy='dynamic'))

    def __init__(self, artificial_title, lyrics, model, base_artist, creation_date=None):
        self.artificial_title = artificial_title
        self.lyrics = lyrics
        self.model = model
        self.base_artist = base_artist

        if creation_date is None:
            creation_date = datetime.utcnow()
        self.creation_date = creation_date

    def __repr__(self):
        return '<ArtificialSong named: %r>' % self.artificial_title.title


import yaml
from datetime import datetime, timedelta
from . import directories
from . import ingredients
from . import recipes
import os
from typing import List

PARTY_MAX_DURATION = timedelta(days=1)
PARTY_MIN_ORDER_COUNT = 5
STATISTICS_MAX_DISTANCE = timedelta(weeks=52)
class Party:
    def __init__(self, start: datetime):
        self.orders = []
        self.start = start

_prefix = "orders "
_extension = ".yaml"
_timeformat = "%Y-%m-%d %H-%M-%S"
_filename = datetime.now().strftime(_prefix + _timeformat + _extension)
_filepath = directories.join(directories.orders, _filename)

def add_order(recipe: recipes.Recipe):
    global _filepath
    data = {}
    data["recipe"] = recipe.name
    data["date"] = datetime.now()
    data["items"] = []
    for item in recipe.items:
        data_item = {}
        data_item["amount"] = item.amount
        data_item["ingredient"] = item.ingredient.identifier if item.ingredient is not None else None
        data["items"].append(data_item)
    with open(_filepath, "a") as file:
        # append as part of list
        yaml.dump([data], file)

def get_parties() -> List[Party]:
    global _prefix
    all_parties:List[Party] = []
    current_party = None
    for file in sorted(os.listdir(directories.orders)):
        if not file.endswith(_extension):
            continue
        if not file.startswith(_prefix):
            continue
        full_path = directories.join(directories.orders, file)
        # ignore empty files
        if os.path.getsize(full_path) == 0:
            continue
        str_datetime = file[len(_prefix):-len(_extension)]
        file_datetime = datetime.strptime(str_datetime, _timeformat)
        # ignore dates that are too long ago
        if datetime.now() - file_datetime > STATISTICS_MAX_DISTANCE:
            continue
        with open(full_path, "r") as file:
            orders_data = yaml.safe_load(file)
        # start a new party
        if current_party == None or file_datetime - current_party.start > PARTY_MAX_DURATION:
            # add the previous one if there is one
            if current_party is not None:
                all_parties.append(current_party)
            # create a new party
            current_party = Party(start=file_datetime)
        # add all orders from the file
        for order in orders_data:
            current_party.orders.append(order)
    # also add the last party
    if current_party is not None:
        all_parties.append(current_party)
    result = [p for p in all_parties if len(p.orders) >= PARTY_MIN_ORDER_COUNT]
    return result

def _increase_entry(item: dict, key, increment=1):
    if key in item.keys():
        item[key] += increment
    else:
        item[key] = increment

def get_statistics(party: Party):
    statistics = {}
    ingredients_amount = {}
    cocktail_count = {}
    cocktails_by_time = {}
    for order in party.orders:
        _increase_entry(cocktail_count, order["recipe"])
        hour = datetime(
            order["date"].year,
            order["date"].month,
            order["date"].day,
            order["date"].hour,
        )
        _increase_entry(cocktails_by_time, hour)
        for item in order["items"]:
            ing = ingredients.by_identifier(item["ingredient"])
            if ing.type != ingredients.IngredientType.Stirr:
                _increase_entry(ingredients_amount, ing, item["amount"])
        order["recipe"]
    statistics["ingredients_amount"] = ingredients_amount
    statistics["cocktail_count"] = cocktail_count
    statistics["cocktails_by_time"] = cocktails_by_time
    statistics["total_cocktails"] = len(party.orders)

    return statistics