import pandas as pd

import numpy as np

import os

import sys

import random

import torch

import matplotlib.pyplot as plt

from pathlib import Path

import math

import plotly

import plotly_express as px

import seaborn as sns

import itertools

from functools import partial

import tqdm.notebook as tqdm

from scipy.stats import kurtosis, skew

import time

import pydicom

import scipy

import scipy.ndimage as ndimage

from skimage import measure, morphology, segmentation

from scipy.ndimage.interpolation import zoom

from PIL import Image

# from pytorch_tabnet.tab_network import TabNet

import warnings

warnings.filterwarnings('ignore')

import sys

sys.path.append('../input/pytorchtabnet/tabnet-develop')

from pytorch_tabnet.tab_network import TabNet

## Code from https://www.kaggle.com/aadhavvignesh/lung-segmentation-by-marker-controlled-watershed

def generate_markers(image):

    """

    Generates markers for a given image.

    Parameters: image

    Returns: Internal Marker, External Marker, Watershed Marker

    """

    #Creation of the internal Marker

    marker_internal = image < -400

    marker_internal = segmentation.clear_border(marker_internal)

    marker_internal_labels = measure.label(marker_internal)

    areas = [r.area for r in measure.regionprops(marker_internal_labels)]

    areas.sort()

    if len(areas) > 2:

        for region in measure.regionprops(marker_internal_labels):

            if region.area < areas[-2]:

                for coordinates in region.coords:

                       marker_internal_labels[coordinates[0], coordinates[1]] = 0

    marker_internal = marker_internal_labels > 0

    # Creation of the External Marker

    external_a = ndimage.binary_dilation(marker_internal, iterations=10)

    external_b = ndimage.binary_dilation(marker_internal, iterations=55)

    marker_external = external_b ^ external_a

    # Creation of the Watershed Marker

    marker_watershed = np.zeros(image.shape, dtype=np.int)

    marker_watershed += marker_internal * 255

    marker_watershed += marker_external * 128

    return marker_internal, marker_external, marker_watershed

def seperate_lungs(image,iterations = 1):

    """

    Segments lungs using various techniques.

    Parameters: image (Scan image), iterations (more iterations, more accurate mask)

    Returns:

        - Segmented Lung

        - Lung Filter

        - Outline Lung

        - Watershed Lung

        - Sobel Gradient

    """

    # Store the start time

    # start = time.time()

    marker_internal, marker_external, marker_watershed = generate_markers(image)

    '''

    Creation of Sobel Gradient

    '''

    # Sobel-Gradient

    sobel_filtered_dx = ndimage.sobel(image, 1)

    sobel_filtered_dy = ndimage.sobel(image, 0)

    sobel_gradient = np.hypot(sobel_filtered_dx, sobel_filtered_dy)

    sobel_gradient *= 255.0 / np.max(sobel_gradient)

    '''

    Using the watershed algorithm

    We pass the image convoluted by sobel operator and the watershed marker

    to morphology.watershed and get a matrix matrix labeled using the

    watershed segmentation algorithm.

    '''

    watershed = morphology.watershed(sobel_gradient, marker_watershed)

    '''

    Reducing the image to outlines after Watershed algorithm

    '''

    outline = ndimage.morphological_gradient(watershed, size=(3,3))

    outline = outline.astype(bool)

    '''

    Black Top-hat Morphology:

    The black top hat of an image is defined as its morphological closing

    minus the original image. This operation returns the dark spots of the

    image that are smaller than the structuring element. Note that dark

    spots in the original image are bright spots after the black top hat.

    '''

    # Structuring element used for the filter

    blackhat_struct = [[0, 0, 1, 1, 1, 0, 0],

                       [0, 1, 1, 1, 1, 1, 0],

                       [1, 1, 1, 1, 1, 1, 1],

                       [1, 1, 1, 1, 1, 1, 1],

                       [1, 1, 1, 1, 1, 1, 1],

                       [0, 1, 1, 1, 1, 1, 0],

                       [0, 0, 1, 1, 1, 0, 0]]

    blackhat_struct = ndimage.iterate_structure(blackhat_struct, iterations)

    # Perform Black Top-hat filter

    outline += ndimage.black_tophat(outline, structure=blackhat_struct)

    '''

    Generate lung filter using internal marker and outline.

    '''

    lungfilter = np.bitwise_or(marker_internal, outline)

    lungfilter = ndimage.morphology.binary_closing(lungfilter, structure=np.ones((5,5)), iterations=3)

    '''

    Segment lung using lungfilter and the image.

    '''

    segmented = np.where(lungfilter, image, -1000)

    #return segmented, lungfilter, outline, watershed, sobel_gradient

    return segmented
def load_scan(path):

    paths = os.listdir(path)

    paths = sorted(paths, key=lambda x: int(str(x).split('/')[-1].split('.')[0]))

    slices = [pydicom.read_file(path + '/' + s) for s in paths]

    print(len(slices), 'slices')

    try:

        slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))

    except:

        pass

    try:

        slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])

    except:

        slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))

        slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)

    for s in slices:

        s.SliceThickness = slice_thickness

    return slices

def get_pixels_hu(slices):

    image = np.stack([s.pixel_array for s in slices])

    # Convert to int16 (from sometimes int16),

    # should be possible as values should always be low enough (<32k)

    image = image.astype(np.int16)

    # Set outside-of-scan pixels to 0

    # The intercept is usually -1024, so air is approximately 0

    # Convert to Hounsfield units (HU)

    for slice_number in range(len(slices)):

        intercept = slices[slice_number].RescaleIntercept

        slope = slices[slice_number].RescaleSlope

        if slope != 1:

            image[slice_number] = slope * image[slice_number].astype(np.float64)

            image[slice_number] = image[slice_number].astype(np.int16)

        image[slice_number] += np.int16(intercept)

    image[image <= -1900] = -1000

    return np.array(image, dtype=np.int16)

def resample(image, scan, new_spacing=[1,1,1]):

    st = time.time()

    slice_thickness = scan[0].SliceThickness

    spacing = np.array([slice_thickness] + list(scan[0].PixelSpacing), dtype=np.float32)

    resize_factor = spacing / new_spacing

    new_real_shape = image.shape * resize_factor

    new_shape = np.round(new_real_shape)

    real_resize_factor = new_shape / image.shape

    new_spacing = spacing / real_resize_factor

    print('resample factor', time.time() - st, real_resize_factor)

    image = scipy.ndimage.zoom(image, real_resize_factor, mode='nearest')

    print('resample time', time.time() - st)

    return image, new_spacing

def torch_resample(image, scan, new_spacing=[1,1,1]):

    st = time.time()

    slice_thickness = scan[0].SliceThickness

    spacing = np.array([slice_thickness] + list(scan[0].PixelSpacing), dtype=np.float32)

    resize_factor = spacing / new_spacing

    new_real_shape = image.shape * resize_factor

    new_shape = np.round(new_real_shape)

    real_resize_factor = new_shape / image.shape

    new_spacing = spacing / real_resize_factor

    #print('resample factor', time.time() - st, real_resize_factor)

    image = torch.tensor(image).unsqueeze(0).unsqueeze(0)

    #print(real_resize_factor)

    #size = [int(x*y) for x, y in zip(real_resize_factor, image.shape[-3:])]

    #print(image.shape)

    #print('using resize')

    image = torch.nn.functional.interpolate(image, scale_factor=tuple(real_resize_factor), mode='nearest')

    image = image.squeeze(0).squeeze(0).numpy()

    #print(image.shape)

    #print('resample time', time.time() - st)

    return image, new_spacing

def get_3d_resampled_array(patient_path):

    start_time = time.time()

    patient_slices = load_scan(str(patient_path))

    patient_slices_hu = get_pixels_hu(patient_slices)

    print('HU loaded', time.time() - start_time)

    print(patient_slices_hu.shape)

    #lungmask_3d = np.apply_over_axes(seperate_lungs, patient_slices_hu, 0)

    idx = np.ndindex(patient_slices_hu.shape[0])

    patient_slices_hu_masked = np.zeros(patient_slices_hu.shape)

    for i in idx:

        patient_slices_hu_masked[i] = seperate_lungs(patient_slices_hu[i])

        #patient_slices_hu_masked[i, :, :] = np.where(lungmask, patient_slices_hu[i, :, :], -1000)

    #patient_slices_hu_masked = np.where(lungmask_3d, patient_slices_hu, -1000)

    print('mask generated', time.time() - start_time)

    resampled_array, spacing = torch_resample(patient_slices_hu_masked, patient_slices, [1,1,1])

    print('after resample', time.time() - start_time)

    return resampled_array, spacing

def get_features_from_3d_array(resampled_array, spacing):

    features = {}

    # volume of lungs

    cube_volume = spacing[0] * spacing[1] * spacing[2]

    total_lung_volume = (resampled_array[resampled_array > -900].shape[0] * cube_volume)

    lung_volume_in_liters = total_lung_volume / (1000*1000)

    features['lung_volume_in_liters'] = lung_volume_in_liters

    #HU unit binning

    bins_threshold = (resampled_array <= 300) & (resampled_array >= -900)

    total_hu_units_bin = resampled_array[bins_threshold].flatten().shape[0]

    bin_values, bins = np.histogram(resampled_array[bins_threshold].flatten(), bins=range(-900, 400, 100))

    features['total_hu_units_bin'] = total_hu_units_bin

    for i, _bin in enumerate(bins[:-1]):

        features[f'bin_{_bin}'] = bin_values[i] / total_hu_units_bin

    #mean, skew, kurtosis

    lung_threshold = (resampled_array <= -320) & (resampled_array >= -900)

    histogram_values, _ = np.histogram(resampled_array[lung_threshold].flatten(), bins=100)

    features['lung_mean_hu'] = np.mean(resampled_array[lung_threshold].flatten())

    features['lung_skew'] = skew(histogram_values)

    features['lung_kurtosis'] = kurtosis(histogram_values)

    #height_of_lung

    n_lung_pixels = lung_threshold.sum(axis=1).sum(axis=1)

    height_start = np.argwhere(n_lung_pixels > 1000).min()

    height_end = np.argwhere(n_lung_pixels > 1000).max()

    features['height_of_lung_cm'] = (height_end - height_start)/10

    return features
data_dir = Path('../input/osic-pulmonary-fibrosis-progression')
def seed_everything(seed=42):

    random.seed(seed)

    torch.manual_seed(seed)

    torch.cuda.manual_seed_all(seed)

    np.random.seed(seed)

    os.environ['PYTHONHASHSEED'] = str(seed)

    torch.backends.cudnn.deterministic = True

    torch.backends.cudnn.benchmark = False
seed_everything()
def process_submission(submission):

    submission['Weeks'] = submission['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))

    submission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])

    return submission
train = pd.read_csv(data_dir/'train.csv')

test = pd.read_csv(data_dir/'test.csv')

# dicom_meta = pd.read_pickle(data_dir/'train_dicom_df')/

submission = process_submission(pd.read_csv(data_dir/'sample_submission.csv'))

image_feature_df = pd.read_csv('../input/lung-image-features/patient_feature2_df.csv')
image_feature_df =image_feature_df.drop(['Unnamed: 0'], axis=1)
image_feature_df.head()
import torch

from torch.utils.data import Dataset, DataLoader
from sklearn.model_selection import StratifiedKFold, train_test_split, KFold

from sklearn.preprocessing import LabelEncoder

def frequency_binning(x, nbin):

    nlen = len(x)

    return np.unique(np.interp(np.linspace(0, nlen, nbin + 1),

                     np.arange(nlen),

                     np.sort(x)))

def replace_with_frequency_bins(arr, nbins=5):

    bins = frequency_binning(arr, nbins)

    out = pd.cut(arr, bins)

    return [inter.left if (type(inter) is not float) else inter for inter in out]

train['expected_fvc'] = np.round((100 * train['FVC']) / train['Percent'], 2)

test['expected_fvc'] = np.round((100 * test['FVC']) / test['Percent'], 2)

patient_df = train[['Patient', 'Weeks', 'Age', 'Sex', 'expected_fvc', 'SmokingStatus']]

patient_df['num_weeks'] = train[['Patient', 'Weeks']].groupby('Patient').transform('count')

patient_df['min_week'] = train[['Patient', 'Weeks']].groupby('Patient').transform('min')

_le = LabelEncoder()

patient_df['SmokingStatus'] = _le.fit_transform(train['SmokingStatus'])

patient_df['Sex'] = (patient_df['Sex'] == 'Male').astype(int)

patient_df['Age'] = replace_with_frequency_bins(patient_df['Age'], 5)

patient_df['num_weeks'] = replace_with_frequency_bins(patient_df['num_weeks'], 5)

patient_df['min_week'] = replace_with_frequency_bins(patient_df['min_week'], 5)

patient_df['expected_fvc'] = pd.Series(replace_with_frequency_bins(patient_df['expected_fvc'], 6)).astype(str)

patient_df = patient_df.drop('Weeks',axis=1).drop_duplicates()

patient_df.index = list(range(len(patient_df)))
patient_df = patient_df.merge(image_feature_df, left_on='Patient', right_on='patient_id').drop('patient_id', axis=1)
class LungDataset(Dataset):

    def __init__(self, mode, data, image_features, preprocessing_params=None, expand=True):

        data = data.drop_duplicates(subset=['Patient', 'Weeks'])

        self.inference_data = None

        self.ohe = False

        self.preprocessing_params = preprocessing_params if preprocessing_params is not None else {

            'min_week': -12,

            'max_week': 133

        }

        self.expand = expand

        self.raw_data = self.process_data(data.copy(), image_features, mode, self.preprocessing_params)

        self.data = self.raw_data[self.features]

        self.target = self.raw_data['FVC'] if mode != 'test' else None

        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'

        self.inference = False

        self.pass_y = True if mode != 'test' else False

    def expand_train_data(self, train):

        train_expanded = pd.DataFrame()

        for patient, patient_df in train.groupby('Patient'):

            patient_expanded = pd.DataFrame()

            expected_fvc = np.mean((100 * patient_df['FVC']) / patient_df['Percent'])

            for week in patient_df['Weeks'].sort_values():

                week_df = patient_df.copy()

                week_df['weeks_from_first_visit'] = week_df['Weeks'] - week

                week_df['first_test_fvc'] = patient_df.loc[patient_df['Weeks'] == week, 'FVC'].mean()

                week_df['first_test_week'] = week

                week_df['predict_week'] = patient_df['Weeks']

                week_df.drop(['Percent', 'Weeks'], axis=1, inplace=True)

                patient_expanded = pd.concat([patient_expanded, week_df], ignore_index=True)

            patient_expanded['expected_fvc'] = expected_fvc

            train_expanded = pd.concat([train_expanded, patient_expanded], ignore_index=True)

            train_expanded = train_expanded[(train_expanded['weeks_from_first_visit'] >= -12) & (train_expanded['weeks_from_first_visit'] <= 133)]

        return train_expanded

    def process_data(self, data, img_data, mode, params):

        if mode == 'train' and self.expand:

            data = self.expand_train_data(data)

        else:

            # expected_FVC

            data['expected_fvc'] = (100 * data['FVC']) / data['Percent']

            data['first_test_week'] = data[['Patient', 'Weeks']].groupby('Patient').transform('min')

            data['weeks_from_first_visit'] = data['Weeks'] - data['first_test_week']

            data['predict_week'] = data['Weeks']

            min_fvc_df = data.sort_values('first_test_week').groupby('Patient').head(1)[['Patient', 'FVC']]

            min_fvc_df.columns = ['Patient', 'first_test_fvc']

            data = data.merge(min_fvc_df, on='Patient', how='left')

        data['percent'] = data['first_test_fvc'] / data['expected_fvc']

        # age

        min_age = params.get('min_age', data['Age'].min())

        max_age = params.get('max_age', data['Age'].max())

        data['age'] = (data['Age'] - data['Age'].min()) / (data['Age'].max() - data['Age'].min())

        # OHE

        if self.ohe:

            sex_df = pd.get_dummies(data['Sex']).reset_index(drop=True)

            sex_columns = ['Male', 'Female']

            for col in sex_columns:

                if col not in sex_df:

                    sex_df[col] = 0

            smoke_df = pd.get_dummies(data['SmokingStatus']).reset_index(drop=True)

            smoke_columns = ['Currently smokes','Ex-smoker','Never smoked']

            for col in smoke_columns:

                if col not in smoke_df.columns:

                    smoke_df[col] = 0

        else:

            sex_map = {'Male': 0, 'Female': 1}

            smoke_map = {'Ex-smoker': 0, 'Never smoked': 1, 'Currently smokes': 2}

            data['sex'] = data['Sex'].map(sex_map)

            data['smoke'] = data['SmokingStatus'].map(smoke_map)

        # base week

        min_week = params.get('min_week')

        max_week = params.get('max_week')

        data['first_test_week'] = (data['first_test_week'] - min_week) / (max_week - min_week)

        data['weeks_from_first_visit'] = (data['weeks_from_first_visit'] - min_week) / (max_week - min_week)

        data['predict_week'] = (data['predict_week'] - min_week) / (max_week - min_week)

        # Opting min_FVC value

        min_FVC = params.get('min_fvc', min(data['expected_fvc'].min(), data['FVC'].min()))

        max_FVC = params.get('max_fvc', max(data['expected_fvc'].max(), data['FVC'].max()))

        data['expected_fvc'] = (data['expected_fvc'] - min_FVC) / (max_FVC - min_FVC)

        data['first_test_fvc'] = (data['first_test_fvc'] - min_FVC) /(max_FVC - min_FVC)

        data['FVC'] = (data['FVC'] - min_FVC) /(max_FVC - min_FVC)

        #print(data.shape, sex_df.shape, smoke_df.shape)

        data.reset_index(drop=True, inplace=True)

        if self.ohe:

            data = pd.concat([data, sex_df, smoke_df], axis=1)

        #print(data.shape)

        # update params

        self.preprocessing_params = {

            'min_age': min_age,

            'max_age': max_age,

            'min_week': min_week,

            'max_week': max_week,

            'min_fvc': min_FVC,

            'max_fvc': max_FVC

        }

        self.features = ['first_test_fvc',

                         'age', 'predict_week', 'percent', 'expected_fvc', 'weeks_from_first_visit'] #'first_test_week']

        #Image features

        if img_data is not None:

            # lung_volume

            missing_img_data = img_data[img_data['missing'] == 1].copy()

            img_data = img_data[img_data['missing'] == 0].copy()

            min_lung_vol = params.get('min_lung_vol', img_data['lung_volume_in_liters'].min())

            max_lung_vol = params.get('max_lung_vol', img_data['lung_volume_in_liters'].max())

            img_data['lung_volume'] = (img_data['lung_volume_in_liters'] - min_lung_vol) / (max_lung_vol - min_lung_vol)

            # lung height

            min_lung_height = params.get('min_lung_height', img_data['height_of_lung_cm'].min())

            max_lung_height = params.get('max_lung_height', img_data['height_of_lung_cm'].max())

            img_data['lung_height'] = (img_data['height_of_lung_cm'] - min_lung_height) / (max_lung_height - min_lung_height)

            #mean HU value

            min_hu_mean = params.get('min_hu_mean', img_data['lung_mean_hu'].min())

            max_hu_mean = params.get('max_hu_mean', img_data['lung_mean_hu'].max())

            img_data['hu_mean'] = (img_data['lung_mean_hu'] - min_hu_mean) / (max_hu_mean - min_hu_mean)

            #skew

            min_hu_skew = params.get('min_hu_skew', img_data['lung_skew'].min())

            max_hu_skew = params.get('max_hu_skew', img_data['lung_skew'].max())

            img_data['hu_skew'] = (img_data['lung_skew'] - min_hu_skew) / (max_hu_skew - min_hu_skew)

            #kurtosis

            min_hu_kurtosis = params.get('min_hu_kurtosis', img_data['lung_kurtosis'].min())

            max_hu_kurtosis = params.get('max_hu_kurtosis', img_data['lung_kurtosis'].max())

            img_data['hu_kurtosis'] = (img_data['lung_kurtosis'] - min_hu_kurtosis) / (max_hu_kurtosis - min_hu_kurtosis)

            img_features = ['lung_volume', 'lung_height', 'hu_mean', 'hu_skew', 'hu_kurtosis', 'missing']

            self.features += img_features

            self.preprocessing_params.update({

                'min_lung_vol': min_lung_vol,

                'max_lung_vol': max_lung_vol,

                'min_lung_height': min_lung_height,

                'max_lung_height': max_lung_height,

                'min_hu_mean': min_hu_mean,

                'max_hu_mean': max_hu_mean,

                'min_hu_skew': min_hu_skew,

                'max_hu_skew': max_hu_skew,

                'min_hu_kurtosis': min_hu_kurtosis,

                'max_hu_kurtosis': max_hu_kurtosis,

            })

            bin_features = [x for x in img_data.columns if 'bin_' in x]

            self.features += bin_features

            img_feature_df = img_data[[*img_features, *bin_features, 'patient_id']]

            for col in img_feature_df.columns:

                if col not in missing_img_data:

                    missing_img_data[col] = 0

            img_feature_df = pd.concat([img_feature_df, missing_img_data], ignore_index=True)

            img_feature_df = img_feature_df.fillna(0)

            data = data.merge(img_feature_df, left_on='Patient', right_on='patient_id')

        if self.ohe:

            self.features += [*sex_df.columns, *smoke_df.columns]

        else:

            self.features += ['sex', 'smoke']

        data.index = list(range(len(data)))

        # construct inference data

        if mode != 'train':

            inference_df = pd.DataFrame(

                itertools.product(data['Patient'].unique(), range(-12, 134)), columns=['Patient', 'Weeks']

            )

            min_week_df = data[data['weeks_from_first_visit'] == 12 / (133 + 12)].drop('Weeks', axis=1)

            inference_df = inference_df.merge(min_week_df, on='Patient', how='left')

            first_test_week = (inference_df['first_test_week'] * (max_week - min_week)) + min_week

            inference_df['weeks_from_first_visit'] = inference_df['Weeks'] - first_test_week

            inference_df['predict_week'] = inference_df['Weeks']

            inference_df['weeks_from_first_visit'] = (inference_df['weeks_from_first_visit'] - min_week) / (max_week - min_week)

            inference_df['predict_week'] = (inference_df['predict_week'] - min_week) / (max_week - min_week)

            self.inference_data = inference_df.reset_index(drop=True)

        return data

    def get_params(self):

        return self.preprocessing_params

    def get_inference_df(self, last_3_visits=False):

        return self.inference_data

    def set_inference(self, value):

        self.inference = value

    def get_y(self, value):

        self.pass_y = value

    def __len__(self):

        if not self.inference:

            return len(self.data)

        return len(self.inference_data)

    def __getitem__(self, idx):

        if not self.inference:

            data = self.data

        else:

            data = self.inference_data[self.features]

        if (self.target is None) or self.inference or (not self.pass_y):

            return torch.tensor(data.loc[idx].values, dtype=torch.float32).to(self.device)

        return torch.tensor(data.loc[idx].values, dtype=torch.float32).to(self.device), torch.tensor(self.target[idx], dtype=torch.float32).to(self.device)
train_dataset = LungDataset('train', train, image_feature_df, expand=False)
train_dataset.get_params()
def get_splits(stratify_df, train_df, image_df, kfolds, stratify_col=None, expand=True):

    splits = kfolds.split(stratify_df) if not stratify_col else kfolds.split(stratify_df, stratify_df[stratify_col])

    for train_index, valid_index in splits:

        train_patients = patient_df.loc[train_index, 'Patient']

        valid_patients = patient_df.loc[valid_index, 'Patient']

        batch_train_df = train_df[train_df['Patient'].isin(train_patients)]

        batch_valid_df = train_df[train_df['Patient'].isin(valid_patients)]

        batch_train_image_df = image_df[image_df['patient_id'].isin(train_patients)]

        batch_valid_image_df = image_df[image_df['patient_id'].isin(valid_patients)]

        train_dataset = LungDataset('train', batch_train_df, batch_train_image_df, expand=expand)

        params = train_dataset.get_params()

        valid_dataset = LungDataset('valid', batch_valid_df, batch_valid_image_df, params)

        yield train_dataset, valid_dataset
from pytorch_lightning.callbacks import Callback

class CVResultTracker(Callback):

    def __init__(self, result_file, cv_result_file):

        super().__init__()

        self.result_file = result_file

        self_cv_result_file = cv_result_file

    def write_file(self, path, df):

        if path.is_file():

            existing_df = pd.read_csv(path)

            df = pd.concat([existing_df, df], ignore_index=True)

        df.to_csv(path, index=False)

        return df

    def teardown(self, trainer, pl_module, stage):

        logger = None

        for logr in trainer.logger:

            logger = logr if 'CSVLogger' in str(logr.__class__) else None

        if logger:

            metrics_df = pd.read_csv(f'./{logger.save_dir}/{logger.name}/{logger.version}/metrics.csv')

            val_df = metrics_df.dropna(subset=['val_loss'])

            val_df['name'] = logger.name + '_' + logger.version

            val_df['model_name'] = logger.name.split('fold')[0]

            val_df_last = val_df[val_df['epoch'] == val_df['epoch'].max()]

            val_df_best = val_df[val_df['val_nll'] == val_df['val_nll'].min()]

            val_df_last = val_df_last.dropna(axis=1, how='all')

            val_df_best = val_df_best.dropna(axis=1, how='all')

            fold_results_best_path = Path(f'./{logger.save_dir}/fold_results_best.csv')

            fold_results_last_path = Path(f'./{logger.save_dir}/fold_results_last.csv')

            model_results_best_path = Path(f'./{logger.save_dir}/model_results_best.csv')

            model_results_last_path = Path(f'./{logger.save_dir}/model_results_last.csv')

            fold_results_best_df = self.write_file(fold_results_best_path, val_df_best)

            fold_results_last_df = self.write_file(fold_results_last_path, val_df_last)

            model_best_df = fold_results_best_df.drop('name', axis=1)

            model_last_df = fold_results_last_df.drop('name', axis=1)

            model_best_df = model_best_df.groupby('model_name', as_index=False).mean()

            model_last_df = model_last_df.groupby('model_name', as_index=False).mean()

            model_best_df.to_csv(model_results_best_path, index=False)

            model_last_df.to_csv(model_results_last_path, index=False)

import torch.nn as nn

import torch.optim as optim

import torch.nn.functional as F

import pytorch_lightning as pl

from pytorch_lightning.metrics.metric import TensorMetric

from pytorch_lightning.callbacks.early_stopping import EarlyStopping

from pytorch_lightning.callbacks import ModelCheckpoint

from pytorch_lightning.callbacks import LearningRateLogger

from pytorch_lightning import loggers

class NLL(TensorMetric):

    name = 'NLL'

    def __init__(self, min_fvc, max_fvc):

        super().__init__(name='nll')

        self.min_fvc = min_fvc

        self.max_fvc = max_fvc

    def forward(self, preds, target):

        preds = (preds * (self.max_fvc - self.min_fvc)) + self.min_fvc

        target = (target * (self.max_fvc - self.min_fvc)) + self.min_fvc

        sigma = preds[:, 2] - preds[:, 0]

        fvc_pred = preds[:, 1]

        #sigma_clip = sigma + C1

        sigma_clip = torch.max(sigma, torch.tensor(70.).to(self.device))

        delta = torch.abs(target - fvc_pred)

        delta = torch.min(delta, torch.tensor(1000.).to(self.device))

        sq2 = torch.sqrt(torch.tensor(2.).to(device))

        metric = (delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)

        # print(torch.mean(metric))

        return torch.mean(metric)

class MAE(TensorMetric):

    name = 'MAE'

    def __init__(self, min_fvc, max_fvc):

        super().__init__(name='MAE')

        self.min_fvc = min_fvc

        self.max_fvc = max_fvc

    def forward(self, preds, target):

        preds = (preds * (self.max_fvc - self.min_fvc)) + self.min_fvc

        target = (target * (self.max_fvc - self.min_fvc)) + self.min_fvc

        return torch.mean(torch.abs(target - preds[:, 1]))

class LQuantModel(pl.LightningModule):

    def __init__(self, learning_rate, total_steps, train_params, in_tabular_features=8,

                 out_quantiles=3):

        super(LQuantModel, self).__init__()

        self.learning_rate = learning_rate

        self.total_steps = total_steps

        self.fc1 = nn.Linear(in_tabular_features, 512)

        self.fc2 = nn.Linear(512, 256)

        self.fc3 = nn.Linear(256, out_quantiles)

        self.fc4 = nn.Linear(256, out_quantiles)

        self.train_params = train_params

        self.save_hyperparameters()

        self.metrics = [

            NLL(

                train_params.get('min_fvc'),

                train_params.get('max_fvc'),

            ),

            MAE(

                train_params.get('min_fvc'),

                train_params.get('max_fvc'),

            )

        ]

        self.eval_tracker = {}

    def forward(self, x):

        x = F.relu(self.fc1(x))

        x = F.relu(self.fc2(x))

        x1 = self.fc3(x)

        x2 = F.relu(self.fc4(x))

        preds = x1 + torch.cumsum(x2, axis=1)

        return preds

    def configure_optimizers(self):

        optimizer = optim.SGD(self.parameters(), lr=(self.learning_rate), weight_decay=0.01)

        scheduler = optim.lr_scheduler.OneCycleLR(

            optimizer,

            max_lr=self.learning_rate,

            total_steps=self.total_steps

        )

        return [optimizer], [{

            'scheduler': scheduler,

            'interval': 'step',

            'frequency': 1

        }]

    def set_loss_fn(self, loss_fn):

        self.loss_fn = loss_fn

    def training_step(self, batch, batch_idx):

        x, y = batch

        y_hat = self(x)

        loss = self.loss_fn(y_hat, y)

        result = pl.TrainResult(minimize=loss)

        result.log('train_loss', loss, prog_bar=True)

        for metric in self.metrics:

            metric.to(self.device)

            metric_value = metric(y_hat, y)

            result.log(f'train_{metric.name}', metric_value, on_epoch=True, on_step=False)

        return result

    def validation_step(self, batch, batch_idx):

        x, y = batch

        y_hat = self(x)

        loss = self.loss_fn(y_hat, y)

        result = pl.EvalResult(checkpoint_on=loss)

        result.log('val_loss', loss, on_epoch=True, on_step=False)

        for metric in self.metrics:

            metric.to(self.device)

            metric_value = metric(y_hat, y)

            result.log(f'val_{metric.name}', metric_value, on_epoch=True, on_step=False)

        return result

#     def validation_epoch_end(self, validation_results):

#         self.eval_tracker = {k: v[-1] for k, v in validation_results.items() if k[0:4] == 'val_'}

#         return validation_results

    def test_step(self, batch, batch_idx):

        y_hat = self(batch)

        result = pl.EvalResult()

        result.predictions = y_hat

        return result

class LTabnet(pl.LightningModule):

    def __init__(self, learning_rate, total_steps, train_params, in_tabular_features=8,

                 out_quantiles=3):

        super(LTabnet, self).__init__()

        self.learning_rate = learning_rate

        self.total_steps = total_steps

        self.tabnet = TabNet(

            input_dim=26,

            output_dim=3,

            n_a=64,

            n_d=64,

            cat_idxs=[24,25],

            cat_dims=[2,3],

            cat_emb_dim=[4,4],

        ).to(self.device)

        self.train_params = train_params

        self.save_hyperparameters()

        self.metrics = [

            NLL(

                train_params.get('min_fvc'),

                train_params.get('max_fvc'),

            ),

            MAE(

                train_params.get('min_fvc'),

                train_params.get('max_fvc'),

            ),

        ]

    def forward(self, x):

        preds, _ = self.tabnet(x)

        return preds

    def configure_optimizers(self):

        optimizer = optim.SGD(self.parameters(), lr=(self.learning_rate), weight_decay=0.01)

        scheduler = optim.lr_scheduler.OneCycleLR(

            optimizer,

            max_lr=self.learning_rate,

            total_steps=self.total_steps

        )

        return [optimizer], [{

            'scheduler': scheduler,

            'interval': 'step',

            'frequency': 1

        }]

    def set_loss_fn(self, loss_fn):

        self.loss_fn = loss_fn

    def training_step(self, batch, batch_idx):

        x, y = batch

        y_hat = self(x)

        loss = self.loss_fn(y_hat, y)

        result = pl.TrainResult(minimize=loss)

        result.log('train_loss', loss, prog_bar=True)

        for metric in self.metrics:

            metric.to(self.device)

            metric_value = metric(y_hat, y)

            result.log(f'train_{metric.name}', metric_value, on_epoch=True, on_step=False)

        return result

    def validation_step(self, batch, batch_idx):

        x, y = batch

        y_hat = self(x)

        loss = self.loss_fn(y_hat, y)

        result = pl.EvalResult(checkpoint_on=loss)

        result.log('val_loss', loss, on_epoch=True, on_step=False)

        for metric in self.metrics:

            metric.to(self.device)

            metric_value = metric(y_hat, y)

            result.log(f'val_{metric.name}', metric_value, on_epoch=True, on_step=False)

        return result

    def test_step(self, batch, batch_idx):

        y_hat = self(batch)

        result = pl.EvalResult()

        result.predictions = y_hat

        return result
class LungDataModule(pl.LightningDataModule):

    def __init__(self, train_dataset, valid_dataset, test_dataset, batch_size=128):

        super().__init__()

        self.train_dataset = train_dataset

        self.valid_dataset = valid_dataset

        self.test_dataset = test_dataset

        self.batch_size = batch_size

    def train_dataloader(self):

        return DataLoader(train_dataset, shuffle=True, batch_size=self.batch_size)

    def val_dataloader(self):

        return DataLoader(valid_dataset, shuffle=False, batch_size=self.batch_size)

    def test_dataloader(self):

        return DataLoader(test_dataset, shuffle=False, batch_size=self.batch_size)

def predict_dataloader(model, dataloader):

    predictions = []

    model.eval()

    for batch in dataloader:

        predictions.append(model(batch).detach().cpu().numpy())

    return np.vstack(predictions)
def quantile_loss(preds, target, quantiles, device):

    #assert not target.requires_grad

    #assert preds.size(0) == target.size(0)

    losses = []

    q = torch.tensor(quantiles).to(device).repeat(preds.shape[0], 1)

    errors = target.unsqueeze(1).repeat(1, 3) - preds

    losses = torch.max((q - 1) * errors, q * errors)

    return torch.mean(losses).unsqueeze(0)

def metric_score(preds, target, max_fvc, min_fvc, device):

    preds = (preds * (max_fvc - min_fvc)) + min_fvc

    target = (target * (max_fvc - min_fvc)) + min_fvc

    sigma = preds[:, 2] - preds[:, 0]

    fvc_pred = preds[:, 1]

    #sigma_clip = sigma + C1

    sigma_clip = torch.max(sigma, torch.tensor(70.).to(device))

    delta = torch.abs(target - fvc_pred)

    delta = torch.min(delta, torch.tensor(1000.).to(device))

    sq2 = torch.sqrt(torch.tensor(2.).to(device))

    metric = (delta / sigma_clip)*sq2 + torch.log(sigma_clip* sq2)

    # print(torch.mean(metric))

    return torch.mean(metric)

def metric_loss(preds, target, _lambda, quantiles, max_fvc, min_fvc, device):

    return ((_lambda * quantile_loss(preds, target, quantiles, device)) +

            ((1 - _lambda) * metric_score(preds, target, max_fvc, min_fvc, device)))

seed_everything(2020)

device = 'cuda' if torch.cuda.is_available() else 'cpu'

_skf = StratifiedKFold(n_splits=5)

_kf = KFold()

lr_monitor_callback = LearningRateLogger(logging_interval='step')
seed_everything(2020)

name = 'tabnet_img_featuresall_skf_mloss1_lr5e-3_faster_missing'

trained_models = []

cv_predictions = []

epochs = 350

batch_size = 128

for i, (train_dataset, valid_dataset) in enumerate(get_splits(patient_df, train, image_feature_df, _skf, 'expected_fvc',expand=True)):

    total_steps = epochs * math.ceil((len(train_dataset) / batch_size))

    train_params = train_dataset.get_params()

    loss_fn = partial(

        metric_loss,

        _lambda=1,

        quantiles=(0.3, 0.5, 0.7),

        max_fvc=train_params['max_fvc'],

        min_fvc=train_params['min_fvc'],

        device=device

    )

    pathdir = Path(f'./models/{name}/fold_{i}/')

    pathdir.mkdir(parents=True, exist_ok=True)

    checkpoint_callback = ModelCheckpoint(

        filepath=pathdir,

        save_top_k=1,

        save_last=True,

        verbose=False,

        monitor='val_nll',

        mode='min',

        prefix=''

    )

    cv_result_tracker = CVResultTracker('./fold_results.csv', './model_results.csv')

    model = LTabnet(5e-3, total_steps, train_params, in_tabular_features=29).to(device)

    model.set_loss_fn(loss_fn)

    test_dataset = LungDataset('test', test, None, train_params)

    test_dataset.set_inference(True)

    data_module = LungDataModule(train_dataset, valid_dataset, test_dataset, batch_size=batch_size)

    tb_logger = loggers.TensorBoardLogger('lightning_logs/', name=f'{name}', version=f'fold_{i}')

    csv_logger = loggers.CSVLogger('results', name=name, version=f'fold_{i}')

    trainer = pl.Trainer(

        max_epochs=epochs,

        check_val_every_n_epoch=5,

        logger=[tb_logger, csv_logger],

        checkpoint_callback=checkpoint_callback,

        gpus=1,

        callbacks=[lr_monitor_callback, cv_result_tracker]

    )

    trainer.fit(model, data_module)

    trained_models.append(model)


import pandas as pd
import numpy as np
import random
from sklearn import svm
from PIL import Image

# download present points of species
present = pd.read_csv('data/present_points.csv')
present = pd.DataFrame(present)
present_array = np.array([present['latitude'],present['longitude']])

y_array = np.array(present['latitude'].to_list())
x_array = np.array(present['longitude'].to_list())

# download worldclim data
worldclim = np.load('data/worldclim_new.npy')

def get_values_from_matrix(worldclim, pa, min_x, min_y):
    # get worldclim data to selected points
    new_matrix = []
    min_x_new = 21600 + round(min_x / 180 * 21600)
    min_y_new = 10800 - round(min_y / 90 * 10800)
    for i in range(len(pa[0])):
        x_new = (21600 + round(pa[1, i] / 180 * 21600)) - min_x_new
        y_new = (10800 - round(pa[0, i] / 90 * 10800)) - min_y_new
        new_matrix.append(worldclim[:, y_new, x_new])
    return new_matrix

def get_values(worldclim, pa, min_x, min_y):
    # get worldclim data to selected point
    new_matrix = []
    min_x_new = 21600 + round(min_x / 180 * 21600)
    min_y_new = 10800 - round(min_y / 90 * 10800)
    x_new = (21600 + round(pa[0] / 180 * 21600)) - min_x_new
    y_new = (10800 - round(pa[1] / 90 * 10800)) - min_y_new
    new_matrix.append(worldclim[:, y_new, x_new])
    return new_matrix

# get wordclim data for all presented points
x_pres = get_values_from_matrix(worldclim, present_array, 55, 47)

# define and train OCSVM on present points and worldclim data
clf = svm.OneClassSVM(nu=0.3, kernel="rbf", gamma=0.05)
clf.fit(x_pres)

# create a map of OCSVM prediction
wordclim_new = worldclim.T.reshape((3960000, 19))
prediction = clf.predict(wordclim_new)
prediction = np.array([1 if x == 1 else 0.5 for x in prediction]).reshape((3000, 1320)).T

new_matrix = (prediction * 255).astype(np.uint8)
img = Image.fromarray(new_matrix, mode='L')
img.save('pseudo_absence_rsep.jpg')

def find_random(x1, x2, y1, y2):  # RSEP pseudo_absence selecting strategy using OCSVM prediction
    random_x = random.uniform(x1, x2)
    random_y = random.uniform(y1, y2)
    random_array = np.array([random_x, random_y])
    random_data = get_values(worldclim, random_array, 55, 47)
    detector = clf.predict(random_data)
    if detector != -1:
        while detector != -1:
            random_x = random.uniform(x1, x2)
            random_y = random.uniform(y1, y2)
            random_array = np.array([random_x, random_y])
            random_data = get_values(worldclim, random_array, 55, 47)
            detector = clf.predict(random_data)
    else:
        random_x, random_y = random_x, random_y
    return random_x, random_y

def random_float_number(x1, x2, y1, y2, array_x, array_y):  # RSEB pseudo_absence selecting strategy
    random_x = random.uniform(x1, x2)
    random_y = random.uniform(y1, y2)
    if min(abs(array_x-random_x)) <= 0.0833 and min(abs(array_y-random_y)) <= 0.0833:  # set 10 km buffer zone
        while min(abs(array_x-random_x)) <= 0.0833 and min(abs(array_y-random_y)) <= 0.0833:
            random_x = random.uniform(x1, x2)
            random_y = random.uniform(y1, y2)
    else:
        random_x = random_x
        random_y = random_y
    return random_x, random_y

def random_coordinates(extent):  # RS pseudo_absence selecting strategy
    random_x = random.uniform(extent[0], extent[1])
    random_y = random.uniform(extent[2], extent[3])
    return random_x, random_y

extent = [65, 74, 43, 37]
x_pseudo_rs=[]
y_pseudo_rs=[]
x_pseudo_rseb=[]
y_pseudo_rseb=[]
x_pseudo_rsep=[]
y_pseudo_rsep=[]

# RSEP pseudo_absence sampling
for i in range(len(x_pres)*10):
    x_new, y_new = find_random(65.000, 74.000, 37.000, 43.000)
    x_pseudo_rsep.append(x_new)
    y_pseudo_rsep.append(y_new)

# save RSEP sampling in .csv file
species = ['pseudo_absence' for i in range(len(x_pseudo_rsep))]
dict_rsep = {
    'species': species,
    'longitude': x_pseudo_rsep,
    'latitude': y_pseudo_rsep
}
df_rsep = pd.DataFrame(dict_rsep)
df_rsep.to_csv('unused/pseudo_absence_rsep.csv', index=False)

# RSEB pseudo_absence sampling
for i in range(len(x_pres)*10):
    x_new, y_new = random_float_number(65.000, 74.000, 37.000, 43.000, x_array, y_array)
    x_pseudo_rseb.append(x_new)
    y_pseudo_rseb.append(y_new)

# save RSEB sampling in .csv file
dict_rseb = {
    'species': species,
    'longitude': x_pseudo_rseb,
    'latitude': y_pseudo_rseb
}
df_rseb = pd.DataFrame(dict_rseb)
df_rseb.to_csv('unused/pseudo_absence_rseb.csv', index=False)

# RS pseudo_absence sampling
for i in range(len(x_pres)*10):
    x_new, y_new = random_coordinates(extent)
    x_pseudo_rs.append(x_new)
    y_pseudo_rs.append(y_new)

# save RS sampling in .csv file
dict_rs = {
    'species': species,
    'longitude': x_pseudo_rs,
    'latitude': y_pseudo_rs
}
df_rs = pd.DataFrame(dict_rs)
df_rs.to_csv('unused/pseudo_absence_rs.csv', index=False)
print('saved')


"""
--- Day 7: Recursive Circus ---

Wandering further through the circuits of the computer, you come upon a tower of programs that have gotten themselves into a bit of trouble. A recursive algorithm has gotten out of hand, and now they're balanced precariously in a large tower.

One program at the bottom supports the entire tower. It's holding a large disc, and on the disc are balanced several more sub-towers. At the bottom of these sub-towers, standing on the bottom disc, are other programs, each holding their own disc, and so on. At the very tops of these sub-sub-sub-...-towers, many programs stand simply keeping the disc below them balanced but with no disc of their own.

You offer to help, but first you need to understand the structure of these towers. You ask each program to yell out their name, their weight, and (if they're holding a disc) the names of the programs immediately above them balancing on that disc. You write this information down (your puzzle input). Unfortunately, in their panic, they don't do this in an orderly fashion; by the time you're done, you're not sure which program gave which information.

For example, if your list is the following:

pbga (66)
xhth (57)
ebii (61)
havc (66)
ktlj (57)
fwft (72) -> ktlj, cntj, xhth
qoyq (66)
padx (45) -> pbga, havc, qoyq
tknk (41) -> ugml, padx, fwft
jptl (61)
ugml (68) -> gyxo, ebii, jptl
gyxo (61)
cntj (57)

...then you would be able to recreate the structure of the towers that looks like this:

                gyxo
              /
         ugml - ebii
       /      \
      |         jptl
      |
      |         pbga
     /        /
tknk --- padx - havc
     \        \
      |         qoyq
      |
      |         ktlj
       \      /
         fwft - cntj
              \
                xhth

In this example, tknk is at the bottom of the tower (the bottom program), and is holding up ugml, padx, and fwft. Those programs are, in turn, holding up other programs; in this example, none of those programs are holding up any other programs, and are all the tops of their own towers. (The actual tower balancing in front of you is much larger.)

Before you're ready to help them, you need to make sure your information is correct. What is the name of the bottom program?

--- Part Two ---

The programs explain the situation: they can't get down. Rather, they could get down, if they weren't expending all of their energy trying to keep the tower balanced. Apparently, one program has the wrong weight, and until it's fixed, they're stuck here.

For any program holding a disc, each program standing on that disc forms a sub-tower. Each of those sub-towers are supposed to be the same weight, or the disc itself isn't balanced. The weight of a tower is the sum of the weights of the programs in that tower.

In the example above, this means that for ugml's disc to be balanced, gyxo, ebii, and jptl must all have the same weight, and they do: 61.

However, for tknk to be balanced, each of the programs standing on its disc and all programs above it must each match. This means that the following sums must all be the same:

    ugml + (gyxo + ebii + jptl) = 68 + (61 + 61 + 61) = 251
    padx + (pbga + havc + qoyq) = 45 + (66 + 66 + 66) = 243
    fwft + (ktlj + cntj + xhth) = 72 + (57 + 57 + 57) = 243

As you can see, tknk's disc is unbalanced: ugml's stack is heavier than the other two. Even though the nodes above ugml are balanced, ugml itself is too heavy: it needs to be 8 units lighter for its stack to weigh 243 and keep the towers balanced. If this change were made, its weight would be 60.

Given that exactly one program is the wrong weight, what would its weight need to be to balance the entire tower?

"""

def parse(info):
    if '->' in info:
        prog_info, subprogr_info = info.split(' -> ')
        subprograms = subprogr_info.split(', ')
    else:
        prog_info = info
        subprograms = []
    prog_name, value = prog_info.split()
    value = int(value.replace('(', '').replace(')', ''))
    return prog_name, value, subprograms

def read():
    with open('inputs/day7.txt') as fd:
        programs = []
        for line in fd:
            programs.append(parse(line.strip()))
    return programs

def find_bottom(programs):
    possible_bottom = []
    seen_progs = set()
    for prog, _, subprograms in programs:
        if subprograms:
            if prog not in seen_progs:  # it can be the bottom prog
                possible_bottom.append(prog)
            for subprog in subprograms:
                seen_progs.add(subprog)
        else:  # leaf
            seen_progs.add(prog)
    # Final filter for programs
    return [prog for prog in possible_bottom if prog not in seen_progs][0]  # should only be one program

def test1():
    prog_list = """pbga (66)
xhth (57)
ebii (61)
havc (66)
ktlj (57)
fwft (72) -> ktlj, cntj, xhth
qoyq (66)
padx (45) -> pbga, havc, qoyq
tknk (41) -> ugml, padx, fwft
jptl (61)
ugml (68) -> gyxo, ebii, jptl
gyxo (61)
cntj (57)"""
    progs = [parse(l) for l in prog_list.split('\n')]
    assert 'tknk' == find_bottom(progs)

def part1():
    print(find_bottom(read()))

class Unbalanced(Exception):
    pass

class Program:

    def __init__(self, name, value, subprogs):
        self.subprograms = subprogs
        self.name = name
        self.value = value

    def get_weight(self, graph):
        """Recursive search for the weights. It will raise an
        exception when a node is not balanced.
        As it is recursive and a node cannot get its weights
        until the nodes above have gotten theirs,
        the exception will be raised for the deeper node"""
        if not self.subprograms:
            return self.value
        else:
            weight = None
            for i, p in enumerate(self.subprograms):
                subprog = graph.get(p)
                w = subprog.get_weight(graph)
                if weight is None:
                    weight = w
                else:
                    if weight != w:
                        raise Unbalanced()
            else:
                return weight * len(self.subprograms) + self.value

class Structure:

    def __init__(self, programs):
        self.programs = [Program(*prog) for prog in programs]

    def get(self, name):
        for prog in self.programs:
            if prog.name == name:
                return prog

    def find_unbalanced(self):
        for prog in self.programs:
            try:
                prog.get_weight(self)
            except Unbalanced:
                return prog.name

def expected_weight(programas):
    structure = Structure(programas)
    unbalanced = structure.find_unbalanced()
    prog = structure.get(unbalanced)
    weights = []
    for subprog in prog.subprograms:
        weights.append(structure.get(subprog).get_weight(structure))
    correct_weight = max(set(weights), key=weights.count)
    wrong_weight = min(set(weights), key=weights.count)
    wrong_prog = structure.get(prog.subprograms[weights.index(wrong_weight)])
    return correct_weight - wrong_weight + wrong_prog.value

def test2():
    prog_list = """pbga (66)
    xhth (57)
    ebii (61)
    havc (66)
    ktlj (57)
    fwft (72) -> ktlj, cntj, xhth
    qoyq (66)
    padx (45) -> pbga, havc, qoyq
    tknk (41) -> ugml, padx, fwft
    jptl (61)
    ugml (68) -> gyxo, ebii, jptl
    gyxo (61)
    cntj (57)"""
    assert 60 == expected_weight([parse(l) for l in prog_list.split('\n')])

def part2():
    print(expected_weight(read()))

if __name__ == '__main__':
    # test1()
    # part1()
    # test2()
    part2()


from datetime import datetime
from sqlalchemy.orm import backref
from models.db import db

class Order(db.Model):
    __tablename__ = 'orders'
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=False)
    created_at = db.Column(
        db.DateTime, default=datetime.utcnow, nullable=False)
    updated_at = db.Column(db.DateTime, default=datetime.utcnow(
    ), nullable=False, onupdate=datetime.utcnow)

    user = db.relationship("User", backref=db.backref('users', lazy=True))
    orderitems = db.relationship(
        "OrderItem", backref=db.backref('order_items', lazy=True))

    def __init__(self, user_id):
        self.user_id = user_id

    def json(self):
        return {
            "id": self.id,
            "user_id": self.user_id,
            "created_at": str(self.created_at),
            "updated_at": str(self.updated_at)}

    def create(self):
        db.session.add(self)
        db.session.commit()
        return self

    @classmethod
    def find_all(cls):
        allOrders = Order.query.all()
        return [order.json() for order in allOrders]

    @classmethod
    def find_by_id(cls, order_id):
        return Order.query.filter_by(id=order_id).first()

    @classmethod
    def delete(cls, self):
        db.session.delete(self)
        db.session.commit()


# -*- coding: utf-8 -*-
"""
Created on Thu Dec  6 02:42:08 2018

@author: AYSE
"""

#1.Kutuphaneler
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.metrics import r2_score

# Veri yukleme
veriler= pd.read_csv("maaslar.csv")

#eğitimseviyes=x, maaslar=y olarak bölüyoruz
#dataframe slice
x=veriler.iloc[:,1:2]
y=veriler.iloc[:,2:]
#numpy array dönüşümü
X=x.values
Y=y.values
#linear regression
#doğrusal model oluşturma
from sklearn.linear_model import LinearRegression
lin_reg=LinearRegression()
lin_reg.fit(X,Y)

#polynomial regression
#doğrusal olmayan(nonlinear) model oluşturma-2.derecen polinom
from sklearn.preprocessing import PolynomialFeatures

poly_reg=PolynomialFeatures(degree=2)
x_poly=poly_reg.fit_transform(X)
lin_reg2=LinearRegression()
lin_reg2.fit(x_poly,y)

#4.dereceden polinom
poly_reg3=PolynomialFeatures(degree=4)
x_poly3=poly_reg3.fit_transform(X)
lin_reg3=LinearRegression()
lin_reg3.fit(x_poly3,y)

#Görselleştirme
plt.scatter(X,Y,color='red')
#linear regression gibi kullanıyoruz fakat predict ederken herhangi bir değer vermeden polinomal features a dönüştürmeliyiz
plt.plot(x,lin_reg2.predict(poly_reg.fit_transform(X)),color='blue')
plt.show()

#2 boyutlu uzayda x,y i dağıtma
plt.scatter(X,Y,color='red')
#x'in karşılığı olan lin_reg içinden predict etme
plt.plot(x,lin_reg.predict(X),color='blue')
plt.show()

plt.scatter(X,Y,color='red')
plt.plot(x,lin_reg3.predict(poly_reg3.fit_transform(X)),color='blue')
plt.show()

#predicts

print(lin_reg.predict(11))
print(lin_reg.predict(6.6))

print(lin_reg2.predict(poly_reg.fit_transform(11)))
print(lin_reg2.predict(poly_reg.fit_transform(6.6)))

#verilerin ölceklenmesi
from sklearn.preprocessing import StandardScaler
sc1=StandardScaler()
x_scaler=sc1.fit_transform(X)# x_train kumesini al buna buna standardscaler'ı uygula ve yeni uygulanmis halini X_train kumesi olarak yap
sc2=StandardScaler()
y_scaler=sc2.fit_transform(Y)

#SVR Regresyon
from sklearn.svm import SVR
svr_reg=SVR(kernel='rbf')
svr_reg.fit(x_scaler,y_scaler)

plt.scatter(x_scaler,y_scaler,color='red')
plt.plot(x_scaler,svr_reg.predict(x_scaler),color='blue')
plt.show()
print(svr_reg.predict(11))
print(svr_reg.predict(6.6))

#Decision Tree Regresyon
from sklearn.tree import DecisionTreeRegressor
r_dt=DecisionTreeRegressor(random_state=0)
r_dt.fit(X,Y)#X'ten Y'yi öğren
Z=X+0.5
K=X-0.4
W=X+0.1
plt.scatter(X,Y,color='red')
plt.plot(x,r_dt.predict(X),color='blue')
plt.plot(x,r_dt.predict(Z),color='green')
plt.plot(x,r_dt.predict(K),color='yellow')
plt.plot(x,r_dt.predict(W),color='black')
plt.show()
print(r_dt.predict(11))
print(r_dt.predict(6.6))

#Random Forest Regresyonu
from sklearn.ensemble import RandomForestRegressor

rf_reg=RandomForestRegressor( n_estimators=10,random_state=0)
rf_reg.fit(X,Y)
print(rf_reg.predict(6.5))

plt.scatter(X,Y,color='red')
plt.plot(x,rf_reg.predict(X),color='blue')
plt.plot(x,rf_reg.predict(Z),color='yellow')
plt.plot(x,rf_reg.predict(K),color='green')
plt.show()

print("Random Forest R2 Değeri")
print(r2_score(Y,rf_reg.predict(X))) #gerçek Y(nin alacağı değerler) değerler actual değerler
print(r2_score(Y,rf_reg.predict(K)))
print(r2_score(Y,rf_reg.predict(Z)))

#summary R2 değerleri
print("*************")
print("Linear Regression R2 Değeri")
print(r2_score(Y,lin_reg.predict(X)))

print("Polynomial Regression R2 Değeri")
print(r2_score(Y,lin_reg2.predict(poly_reg.fit_transform(X))))

print("SVR R2 Değeri")
print(r2_score(y_scaler,svr_reg.predict(x_scaler)))

print("Decision Tree R2 Değeri")
print(r2_score(Y,r_dt.predict(X)))

print("Random Forest R2 Değeri")
print(r2_score(Y,rf_reg.predict(X)))


import csv

from io import StringIO
from typing import Dict, Iterator, List, Tuple, Union

DEFAULT_HASH_MAP_SIZE: int = 64

class HashMap:
    _size: int = None

    _map: List[List[Tuple[str, str]]] = None
    _length: int = None

    @property
    def size(self) -> int:
        return self._size

    def __init__(self, size: int):
        if size < 1:
            raise ValueError("Size must be greater than 0")

        self._size = size
        self._initialize_map()

    def _initialize_map(self) -> None:
        self._map = [None] * self._size
        self._length = 0

    def get(self, key: str, default: str = None) -> Union[str, None]:
        try:
            return self[key]

        except KeyError:
            return default

    def pop(self, key: str) -> str:
        value = self[key]

        del self[key]

        return value

    def keys(self) -> Iterator[str]:
        for item in self.items():
            yield item[0]

    def items(self) -> Iterator[Tuple[str, str]]:
        for bucket in self._map:
            if bucket is not None:
                for item in bucket:
                    yield item

    def values(self) -> Iterator[str]:
        for item in self.items():
            yield item[1]

    def from_csv(self, csv_string: str) -> None:
        """
        Loads the key-value pairs from a CSV format string into the hash map.

        Example:
            >>> hash_map = HashMap(16)
            >>> hash_map.from_csv('a,1\\nb,2\\nc,3\\n')
            >>> hash_map
            HashMap(16): {'a': '1', 'b': '2', 'c': '3'}
            >>> hash_map.from_csv('d,4\\nb,5\\n')
            >>> hash_map
            HashMap(16): {'a': '1', 'b': '5', 'c': '3', 'd': '4'}
        """

        str_io: StringIO = StringIO(csv_string)

        reader = csv.reader(str_io)
        for key, value in reader:
            self[key] = value

    def to_csv(self) -> str:
        """
        Returns the key-value pairs from the hash map into a CSV format string.

        Example:
            >>> hash_map = HashMap(16)
            >>> hash_map['a'] = '1'
            >>> hash_map['b'] = '2'
            >>> hash_map['c'] = '3'
            >>> hash_map
            HashMap(16): {'a': '1', 'b': '2', 'c': '3'}
            >>> hash_map.to_csv()
            'a,1\\nb,2\\nc,3\\n'
        """

        str_io: StringIO = StringIO()

        writer = csv.writer(str_io)
        writer.writerows(self.items())

        return str_io.getvalue()

    def to_dict(self) -> Dict[str, str]:
        return dict(self.items())

    def clear(self) -> None:
        self._initialize_map()

    def __getitem__(self, key: str) -> str:
        hashed_key: int = hash(key) % self._size
        bucket: List[Tuple[str, str]] = self._map[hashed_key]

        if bucket is None:
            raise KeyError(key)

        for item in bucket:
            if item[0] == key:
                return item[1]

        raise KeyError(key)

    def __setitem__(self, key: str, value: str) -> None:
        hashed_key: int = hash(key) % self._size
        bucket: List[Tuple[str, str]] = self._map[hashed_key]

        if bucket is None:
            self._map[hashed_key] = [(key, value)]
            self._length += 1

            return

        for index, item in enumerate(bucket):
            if item[0] == key:
                bucket[index] = (key, value)

                return

        bucket.append((key, value))
        self._length += 1

    def __delitem__(self, key: str) -> None:
        hashed_key: int = hash(key) % self._size
        bucket: List[Tuple[str, str]] = self._map[hashed_key]

        if bucket is None:
            raise KeyError(key)

        for index, item in enumerate(bucket):
            if item[0] == key:
                del bucket[index]
                self._length -= 1

                if len(bucket) == 0:
                    self._map[hashed_key] = None

                return

        raise KeyError(key)

    def __bool__(self) -> bool:
        return bool(self._length)

    def __len__(self) -> int:
        return self._length

    def __iter__(self) -> Iterator[str]:
        return self.values()

    def __contains__(self, key: str) -> bool:
        try:
            self[key]

        except KeyError:
            return False

        return True

    def __str__(self) -> str:
        values: List[str] = []

        for key, value in self.items():
            values.append(f"{repr(key)}: {repr(value)}")

        return f"{{{', '.join(values)}}}"

    def __repr__(self) -> str:
        return f"HashMap({self._size}): {self}"


import bblfsh

from bblfsh import filter as filter_uast

def rule_chk(uast):
        findings = []
        is_left_literal = False
        is_right_literal = False

        left_node_pos = None
        right_node_pos = None

        #query = "//WhileStatement//InfixExpression"
        #query = "//*[@roleWhile and @roleBinary and @roleCondition and  @roleExpression]"
        #query = "//*[@roleWhile and @roleStatement and not(@roleBody)]"
        query = "//*[@roleWhile and @roleStatement and not(@roleBody)]//*[@roleRelational and @roleExpression and @roleBinary and @roleOperator] "
        print(query)
        node = filter_uast(uast,query)
        i =0
        for n in node:
            i = i + 1
            print('Node :{0}'.format(i) )
            is_left_literal = False
            is_right_literal = False

            left_node_pos = None
            right_node_pos = None

            j = 0
            for child  in n.children:
                j= j + 1
                print('Iteration {0} for node {1} '.format(j,i))
                #print(child)

                if ( bblfsh.role_id("NUMBER") in child.roles ) &   ( bblfsh.role_id("LEFT") in child.roles  )  :
                        is_left_literal = True
                        left_node_pos = child.start_position.line

                if ( bblfsh.role_id("NUMBER") in child.roles ) &   ( bblfsh.role_id("RIGHT") in child.roles  )  :
                        is_right_literal = True
                        right_node_pos = child.start_position.line

            if is_left_literal & is_right_literal :
                findings.append({"msg": "Number literals found in while condition",
                            "left literal at line": left_node_pos,
                            "right literal at line":right_node_pos})

        return findings


##### quick and dirty #####
data = open('day3.txt').read().splitlines()
alpha = 'abcdefghijklmnopqrstuvwkyz'
priorities = alpha+alpha.upper()

## part1
total = 0
for rucksack in data:
    half = len(rucksack)//2
    first_half = rucksack[0:half]
    second_half = rucksack[half:]
    in_both = set(first_half) & set(second_half)
    p = priorities.index(in_both.pop()) + 1
    total += p
print(total)

## part 2
total = 0
groups = [data[i:i+3] for i in range(0,len(data),3)]
shared = [set.intersection(*map(set,g)) for g in groups]
for s in shared:
    p = priorities.index(s.pop()) + 1
    total += p
print(total)

##### refactored #####
data = open('day3.txt').read().splitlines()
alpha = 'abcdefghijklmnopqrstuvwkyz'
priorities = alpha+alpha.upper()

def partition(n,xs):
    return [xs[i:i+n] for i in range(0,len(xs),n)]

def halve(xs):
    return partition(len(xs)//2,xs)

## part 1
intersections = [set.intersection(*map(set,halve(rucksack))) for rucksack in data]
score = sum(priorities.index(x.pop()) + 1 for x in intersections)
print(score)

## part 2
intersections = [set.intersection(*map(set,p)) for p in partition(3,data)]
score = sum(priorities.index(x.pop()) + 1 for x in intersections)
print(score)


import torch
import argparse
from torch.utils.data import DataLoader
from data_utils import ChoiceDataset
from models import TasteNetChoice
from train import train
from evaluation import summarize, printSummary, predictTastes, RMSE, ABSE, RE, RMSE_vector, ABSE_vector, RE_vector
from simulate import error_of_vot, error_of_vowt, dic_z, dic_z_z01, dic_z_zall, inc
import pickle
import os
import copy
import numpy as np
from regress_coef import regress
import itertools
from joblib import Parallel, delayed, parallel_backend

def one_run(data_train, data_dev, args, params):
    """
    A single model training run with one set of hyper-parameters and random initialization seed
    """
    # set up parameters
    # torch.manual_seed(params['seed'])
    input_size = 3  # z has 3 dim
    output_size = 1 if params['separate'] else 2
    args.layer_sizes = [input_size, params['hidden_size'], output_size]
    args.activation = params['activation']
    args.transform = params['transform']
    args.weight_decay = params['weight_decay']
    args.separate = params['separate']
    print(args)

    # train
    success = False
    while not success:
        # initialize model
        model = TasteNetChoice(args)
        _, _, model, success = train(model, data_train, data_dev, args, save=False)

    # evaluate
    result = one_run_summary(model, ds_train, ds_dev, ds_test)

    return result, model

def one_run_summary(model, ds_train, ds_dev, ds_test):
    # Metrics
    result = {}
    # 1. NLL and ACC for train/dev/test
    summary = summarize(model, ds_train, ds_dev, ds_test)
    result.update(summary)
    x = printSummary(summary, precision=3)
    print(x)

    # 2. Tastes (vots, vowts): predicted vs true (dollar per minute)
    allds = {"train": ds_train, "dev": ds_dev, "test": ds_test}

    pred_vots = [predictTastes(model, allds[name].z)['vots'] for name in ["train", "dev", "test"]]
    true_vots = [allds[name].vots for name in ["train", "dev", "test"]]

    result['vot_rmse'] = RMSE(pred_vots, true_vots).item()
    result['vot_mabse'] = ABSE(pred_vots, true_vots).item()
    result['vot_re'] = RE(pred_vots, true_vots).item()

    pred_vowts = [predictTastes(model, allds[name].z)['vowts'] for name in ["train", "dev", "test"]]
    true_vowts = [allds[name].vowts for name in ["train", "dev", "test"]]

    result['vowt_rmse'] = RMSE(pred_vowts, true_vowts).item()
    result['vowt_mabse'] = ABSE(pred_vowts, true_vowts).item()
    result['vowt_re'] = RE(pred_vowts, true_vowts).item()

    # 3. Tastes (vots, vowts) on simulated z: unit is dollar per hour!
    input_z = copy.deepcopy(dic_z)
    sim_pred_vots, sim_true_vots, rmse, mabse, re = error_of_vot(model, dic_z, input_z, ds_train.params)
    result['sim_vot_rmse'] = rmse
    result['sim_vot_mabse'] = mabse
    result['sim_vot_re'] = re

    sim_pred_vowts, sim_true_vowts, rmse, mabse, re = error_of_vowt(model, dic_z, input_z, ds_train.params)
    result['sim_vowt_rmse'] = rmse
    result['sim_vowt_mabse'] = mabse
    result['sim_vowt_re'] = re

    # 4. Coefficients from regression
    coefs_asc1 = model.getParams()[-1][0].item()
    coefs_time, coefs_wait = regress(model, dic_z, dic_z_zall)
    coefs_pred = [coefs_asc1] + coefs_time + coefs_wait

    coefs_asc1_true = ds_train.asc1
    coefs_time_true = ds_train.coefs_time
    coefs_wait_true = ds_train.coefs_wait
    coefs_true = [coefs_asc1_true] + coefs_time_true + coefs_wait_true

    coefs_pred = torch.Tensor(coefs_pred)
    coefs_true = torch.Tensor(coefs_true)

    rmse = RMSE_vector(coefs_pred, coefs_true)
    mabse = ABSE_vector(coefs_pred, coefs_true)
    re = RE_vector(coefs_pred, coefs_true)

    result['coef_rmse'] = rmse
    result['coef_mabse'] = mabse
    result['coef_re'] = re
    result['coef'] = coefs_pred
    result['coef_true'] = coefs_true
    return result

parser = argparse.ArgumentParser(description='TasteNet-MNL (A Toy Example)')
parser.add_argument("--data_dir", type=str, default="../toy_data")
parser.add_argument("--data_file", type=str, default="data_10k.pkl", help="data dictionary in pickle format")
parser.add_argument("--output_dir", type=str, default="../result")
parser.add_argument("--scenario", type=str)

parser.add_argument('--cuda', action='store_true', default=False,
                    help='enables CUDA training')

# Training params
parser.add_argument('--batch_size', type=int, default=64,
                    help='input batch size for training (default: 64)')
parser.add_argument('--num_epochs', type=int, default=100,
                    help='maximum number of epochs to train (default: 100)')
parser.add_argument("--lr", type=float, default=0.001, help="learning rate")
parser.add_argument("--nll_tol", type=float, default=0.0001, help="tolerance for nll convergence")

args = parser.parse_args()

cuda = args.cuda and torch.cuda.is_available()
device = torch.device("cuda" if cuda else "cpu")

# load data
data = pickle.load(open(os.path.join(args.data_dir, args.data_file), "rb"))

ds_train = ChoiceDataset(data['train'])
ds_dev = ChoiceDataset(data['dev'])
ds_test = ChoiceDataset(data['test'])

data_train = DataLoader(ds_train, batch_size=args.batch_size, shuffle=True, num_workers=5)
data_dev = DataLoader(ds_dev, batch_size=args.batch_size, shuffle=False, num_workers=5)
data_test = DataLoader(ds_test, batch_size=args.batch_size, shuffle=False, num_workers=5)

# Set hyperparam_candidates
hyperparam_candidates = dict(
        hidden_size=[200],
        weight_decay=[0.0001, 0.001],
        activation=["relu"],
        transform=["relu"],
        separate=[False],
        no=range(100) # number of runs with different random initialization
    )

hyperparam_sets = [dict(zip(hyperparam_candidates.keys(), values))
                       for values
                       in itertools.product(*hyperparam_candidates.values())]
print(f"{len(hyperparam_sets)} set of params")
print(hyperparam_sets)

result_all = []
model_all = []
for params in hyperparam_sets:
    result, model = one_run(data_train, data_dev, args, params)
    result_all.append(result)
    model_all.append(model)

evals_name = f"{args.scenario}_evals.pkl"
models_name = f"{args.scenario}_models.pkl"
args_name = f"{args.scenario}_args.pkl"
hyper_name = f"{args.scenario}_hyper.pkl"

pickle.dump(result_all, open(os.path.join(args.output_dir, evals_name), "wb"))
pickle.dump(model_all, open(os.path.join(args.output_dir, models_name), "wb"))
pickle.dump(hyperparam_sets, open(os.path.join(args.output_dir, hyper_name), "wb"))
pickle.dump(args, open(os.path.join(args.output_dir, args_name), "wb"))


from item import Item
from tools import *
import databasetools
import servicetools
import plugintools

def menu(item):
    itemlist = list()
    if not item.sectionId:
        item.sectionId = 0
    folders = databasetools.executeSQL(
        'SELECT * FROM astro_folders WHERE subFolderId == {}'.format(item.sectionId))
    actions = databasetools.executeSQL(
        'SELECT * FROM astro_actions WHERE keepFolderId == {}'.format(item.sectionId))
    if get_setting('sort_items_alphabetically'):
        folders = sorted(folders, key=lambda item: item[2])
        actions = sorted(actions, key=lambda act: act[1])
    if get_setting('dev_mode') and bool(item.mainmenu):
        itemlist.append(item.clone(
                        title='[COLOR orange]Astro[/COLOR] - %s' % (
                            getLocalizedString(32001)),
                        thumbnail=ICON,
                        fanart=FANART,
                        action='open_developer_modal'
                        ))
    for id, sectionId, name, thumbnail, subFolderId, onClick, backFolderId in folders:
        item.mainmenu = False
        itemlist.append(item.clone(
                        id=id,
                        title=name,
                        thumbnail=prepareThumbnail(thumbnail),
                        fanart=FANART,
                        sectionId=sectionId,
                        subFolderId=subFolderId,
                        action='menu'
                        ))
    for action in actions:
        id, label, folder, path, filename, thumbnail, \
            icon, fanart, window, isplayable, isfolder, \
            file, isstream, description, hasVideo, picture, onClick, keepFolderId = action
        item.mainmenu = False
        itemlist.append(item.clone(
                        id=id,
                        title=label,
                        thumbnail=prepareThumbnail(thumbnail),
                        fanart=FANART,
                        path=path,
                        url=path,
                        action='runPlugin'
                        ))

    return itemlist

def open_developer_modal(item):
    from modals.developermodal import DeveloperModal
    window = DeveloperModal(TITLE)
    window.doModal()
    del window

def run_customize_modal(item):
    from modals.customizemodal import CustomizeModal
    window = CustomizeModal(TITLE)
    window.doModal()
    del window

def set_admin_password(item):
    return plugintools.set_admin_password()

def runPlugin(item):
    xbmc.executebuiltin('Dialog.Close(all,true)')
    xbmc.executebuiltin(
        'ActivateWindow(10025, "%s")' % (item.path))

def moveElement(item):
    return plugintools.moveElement(item)

def renameSQL(item):
    return plugintools.renameSQL(item)

def deleteSQL(item):
    return plugintools.deleteSQL(item)

def changeIcon(item):
    return plugintools.changeIcon(item)

def run(item):
    itemlist = list()
    if not item.action:
        logger('Item sin acci\xc3\xb3n')
        return
    itemlist = eval(item.action)(item)
    if itemlist is not None:
        if not len(itemlist):
            itemlist.append(Item(
                title='[COLOR orange]Sin resultados[/COLOR]', thumbnail=ICON, fanart=FANART))

    if itemlist:
        render_items(itemlist, item)

if __name__ == '__main__':
    from updater import Updater
    try:
        Updater().checkAvailableVersion(silent=False)
    except Exception as e:
        logger("Ha ocurrido un error leyendo la última versión de Astro")
        logger(f"ERROR: {e}")
    if not databasetools.checkFileExists():
        databasetools.initialiseDatabaseFile()

    try:
        servicetools.updateAstroRemoteConfig()
    except Exception as e:
        logger("Ha ocurrido un error leyendo la última versión de los datos")
        logger(f"ERROR: {e}")
    if sys.argv[2]:
        item = Item().fromurl(sys.argv[2])
    else:
        item = Item(action='menu', mainmenu=True)
    run(item)


# coding: utf-8

"""
Perform inference with a CD UNet/UNet++ model for for Sentinel-2 datasets

@Author: Tony Di Pilato

Created on Wed Feb 19, 2020
"""

import os
import numpy as np
from tensorflow import keras as K
import cdUtils
import cdModels
from osgeo import gdal
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('--size', type=int, default=128)
parser.add_argument('-cpt', type=int, default=500) # Number of crops per tiff
parser.add_argument('--channels', '-ch', type=int, default=13)
parser.add_argument('--loss', '-l', type=str, default='bce', help='bce, wbced or dice')
parser.add_argument('--model', type=str, default='EF', help='EF, Siam or SiamDiff')
parser.add_argument('--city', type=str, default='rome', help='rome or sot')
parser.add_argument('--year', '-y', type=int, default='2015', help='from 2015 to 2019, take the pair thisyear-nextyear')
parser.add_argument('--histmatched', '-hm', type=bool, default='False', help='select True to use histmatched post images')

args = parser.parse_args()

img_size = args.size
channels = args.channels
cpt = args.cpt
mod = args.model
loss = args.loss
city = args.city
year = str(args.year)+'-'+str(args.year + 1)
hm = args.histmatched
classes = 1
model_dir = '../models/' + mod + '/'

model_name = mod+'_'+str(img_size)+'_cpt-'+str(cpt)+'-'+loss+'_'+str(channels)+'channels'
model_dir = model_dir + model_name + '/'
model_name = model_name + '-final'

# Load the model
if(loss=='wbced'):
    model = K.models.load_model(model_dir + model_name + '.h5', custom_objects={'weighted_bce_dice_loss': cdModels.weighted_bce_dice_loss})
else:
    model = K.models.load_model(model_dir + model_name + '.h5')
model.summary()
print("Model loaded!")

if(city=='rome'):
    dataset_dir = '../../CD_Sentinel2/tiff/Rome/'
    infres_dir = '../results/rome/'
    cm_name = 'Rome-' + model_name
else:
    if(hm):
        dataset_dir = '../Stoke_On_Trent/histmatched/' + year + '/'
        cm_name = 'SoThm-' + model_name + '_' + year
    else:
        dataset_dir = '../Stoke_On_Trent/' + year + '/'
        cm_name = 'SoT-' + model_name + '_' + year
    infres_dir = '../results/SoT/'

os.makedirs(infres_dir, exist_ok=True)

for image in os.listdir(dataset_dir):
    if((image.startswith('2') or image.startswith('p')) and ('pre' in image)):
        img_pre = image
    elif((image.startswith('2') or image.startswith('p')) and ('post' in image)):
        img_post = image

print(img_pre)
print(img_post)

tiffData_1 = gdal.Open(dataset_dir + img_pre)
raster1 = cdUtils.build_raster_fromMultispectral(tiffData_1, channels)
tiffData_2 = gdal.Open(dataset_dir + img_post)
raster2 = cdUtils.build_raster_fromMultispectral(tiffData_2, channels)
raster = np.concatenate((raster1,raster2), axis=2)
padded_raster = cdUtils.pad(raster, img_size)
test_image = cdUtils.crop(padded_raster, img_size, img_size)

# Create inputs for the Neural Network
inputs = np.asarray(test_image, dtype='float32')
inputs_1 = inputs[:,:,:,:channels]
inputs_2 = inputs[:,:,:,channels:]
inputs = [inputs_1, inputs_2]

# Perform inference
results = model.predict(inputs)

# Build the complete change map
shape = (padded_raster.shape[0], padded_raster.shape[1], classes)
padded_cm = cdUtils.uncrop(shape, results, img_size, img_size)
cm = cdUtils.unpad(raster.shape, padded_cm)

cm = np.squeeze(cm)
cm = np.rint(cm) # we are only interested in change/unchange
cm = cm.astype(np.uint8)

# Now create the georeferenced change map
cdUtils.createGeoCM(infres_dir + cm_name + '.tif', tiffData_1, cm)

print('Georeferenced change map created at %s' %infres_dir)


rows = int(input())

matrix = []

for _ in range(rows):
    inner_list = [int(x) for x in input().split(", ")]
    matrix.append(inner_list)

primary = []
secondary = []

for r in range(rows):
    primary.append(matrix[r][r])
    secondary.append(matrix[r][rows - 1 - r])

print(primary)
print(secondary)

# unpacked

print(", ".join(str(x) for x in primary))
print(", ".join(str(x) for x in secondary))


#!/usr/bin/env python
from __future__ import print_function
from setuptools import setup, find_packages
from setuptools.command.test import test as TestCommand
import io
import os

here = os.path.abspath(os.path.dirname(__file__))

def read_version(version_filepath):
    import re

    verstrline = open(version_filepath, "rt").read()
    VSRE = r"^__version__ = ['\"]([^'\"]*)['\"]"
    mo = re.search(VSRE, verstrline, re.M)
    return mo.group(1)

#
def read(*filenames, **kwargs):
    encoding = kwargs.get("encoding", "utf-8")
    sep = kwargs.get("sep", "\n")
    buf = []
    for filename in filenames:
        with io.open(filename, encoding=encoding) as f:
            buf.append(f.read())
    return sep.join(buf)

long_description = read("README.md")

setup(
    name="Instarecon",
    version=read_version("src/_version.py"),
    url="http://github.com/vergl4s/instarecon/",
    license="MIT License",
    author="Luis Teixeira",
    install_requires=[
        "dnspython>=1.14.0",
        "ipaddr>=2.1.11",
        "ipaddress>=1.0.16",
        "ipwhois>=0.13.0",
        "pythonwhois>=2.4.3",
        "requests>=2.10.0",
        "shodan>=1.5.3",
    ],
    scripts=["scripts/instarecon.py"],
    author_email="luis@teix.co",
    description="Automated basic digital reconnaissance",
    packages=find_packages("."),
    include_package_data=True,
    classifiers=[
        "Development Status :: 4 - Beta",
        "Natural Language :: English",
        "Programming Language :: Python :: 2.7",
        "Intended Audience :: Developers, Hackers",
        "License :: OSI Approved :: MIT License",
    ],
    test_suite="tests",
)


#!/usr/bin/env python
import time
import serial
import random
import binascii
import pickle

tty="/dev/ttyAMA0"
baud=57600

config = []

config.append("sys reset")

#change these to values given
config.append("mac set devaddr 260114C8")
config.append("mac set deveui 006117063BA98093")
config.append("mac set appeui 70B3D57EF0005B5B")
config.append("mac set appskey XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
config.append("mac set nwkskey XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")

config.append("mac join abp")

config.append("sys set pindig GPIO0 1")

def send(data, p):
	p.write(data+"\x0d\x0a")
	data.rstrip()
	print(data)
	time.sleep(0.1)
	rdata=p.readline()
	rdata=rdata[:-1]
	print (rdata)

p = serial.Serial(tty , baud)

time.sleep(1)

for c in config:
    send(c, p)

#replace this with the sensor data.
#order is light, red, green, blue, x,y,z, heading, temp, pressure

user_data =[898,166,147,135,-0.0025634765625,0.02557373046875,1.0029296875,137.28,28.2777963716,100794.557256]

data = binascii.hexlify(pickle.dumps(user_data)).upper()
print(data)

send("mac tx cnf 1 {}".format(data), p)

time.sleep(1)
rdata=p.readline()
rdata=rdata[:-1]
print (rdata)
time.sleep(5)
rdata=p.readline()
rdata=rdata[:-1]
print (rdata)


from collections import deque
cardeck = deque()

n = int(input())
for i in range(1,n+1):
    cardeck.append(i)
while True:
    if len(cardeck) == 1:
        break
    cardeck.popleft()
    k = cardeck.popleft()
    cardeck.append(k)
print(cardeck[0])


from manimlib import *
import numpy as np
import random

class homoto(Scene):
	def construct(self):
		text = Text("Homotopy")
		self.play(FadeIn(text))
		#self.wait(3)
		self.play(FadeOut(text))

		title = Text("Homotopy").shift(UP*3.5).scale(0.6)
		self.play(FadeIn(title))

		obj1 = Text("This is text").shift(LEFT*4)
		obj2 = Square(fill_opacity=1).set_color(YELLOW).shift(LEFT*3 + DOWN*3)
		obj3 = Circle(fill_opacity=1).set_color(BLUE)
		obj4 = Rectangle(fill_opacity=1).set_color(GREEN).shift(DOWN*2)

		#self.add(obj2)

		test_homotopy = lambda x, y, z, t: (
			x + interpolate(-3, 3, 2*t if t<=0.5 else 1), # First 5 Seconds
			y + interpolate(0, 3, 2*(t-0.5) if t>0.5 else 0), # Next 5 Seconds
			z)

		test_homotopy2 = lambda x, y, z, t: (
			x + interpolate(-3, 3, 4*t if t<=0.25 else 1)
				+ interpolate(3, -3, 4*(t-0.5) if (t>=0.5 and t<=0.75) else (0 if t<=0.5 else 1)),
			y + interpolate(-3, 3, 4*(t-0.25) if (t>0.25 and t<0.5) else (0 if t<=0.25 else 1))
				+ interpolate(3, -3, 4*(t-0.75) if (t>0.75) else (0 if t<=0.75 else 1)),
			z)

		self.play(Homotopy(test_homotopy, obj3, run_time=10, rate_func=linear))
		#ComplexHomotopy


from builtins import range
import geocal_swig
from math import *
import numpy as np
import os.path
import re
from .shape_file import *

def _new_from_init(cls, version, *args):
    '''Handle older versions'''
    if(cls.pickle_format_version() < version):
        raise RuntimeException("Class is expecting a pickled object with version number %d, but we found %d" % (cls.pickle_format_version(), version))
    inst = cls.__new__(cls)
    # Added Fit_height_offset in version 2. For version 1, just set this to
    # False
    if(version ==1):
        inst.__init__(args[0], args[1], args[2], args[3], None, None, False)
    else:
        inst.__init__(*args)
    return inst

class GdalImageGroundConnection(geocal_swig.RpcImageGroundConnection):
    '''This is a convenience class that both reads a GDAL image, and
    creates a RPC based ground connection. Note that you can edit
    the RPC if desired, just change the values of the attribute
    rpc'''
    def __init__(self, fname, dem, rpc = None, title=None, image_mask=None,
                 ground_mask=None, fit_height_offset=False):
        self.fname = fname
        img = geocal_swig.GdalMultiBand(fname)
        if(not rpc):
            rpc = img.gdal_raster_image(0).rpc
        if(img.number_band == 1):
            img = img.gdal_raster_image(0)
        if(not title):
            title = os.path.basename(fname)
        geocal_swig.RpcImageGroundConnection.__init__(self, rpc, dem, img, title,
                                                 image_mask, ground_mask,
                                                 fit_height_offset)

    @classmethod
    def pickle_format_version(cls):
        return 2

    def __reduce__(self):
        '''Base class RpcImageGroundConnection can be pickled, but the
        init function for this class is different. Give a simple reduce
        to handle this.'''
        return _new_from_init, (self.__class__,
                                self.__class__.pickle_format_version(),
                                self.fname,
                                self.dem, self.rpc, self.title,
                                self.image_mask, self.ground_mask,
                                self.fit_height_offset)

class VicarImageGroundConnection(geocal_swig.RpcImageGroundConnection):
    '''This is a convenience class that both reads a VICAR image, and
    creates a RPC based ground connection. Note that you can edit
    the RPC if desired, just change the values of the attribute
    rpc'''
    def __init__(self, fname, dem, rpc = None, title=None, image_mask=None,
                 ground_mask=None, fit_height_offset=False):
        self.fname = fname
        # Check for multiple bands
        m = re.match(r'(.*)(\d+):(\d+)(.*)', fname)
        if(m):
            base = m.group(1)
            baseend = m.group(4)
            img = RasterImageMultiBandVariable()
            for i in range(int(m.group(2)), int(m.group(3)) + 1):
                img.add_raster_image(geocal_swig.VicarLiteRasterImage("%s%d%s" % (base, i, baseend)))
            if(not rpc):
                rpc = img.raster_image(0).rpc
        else:
            img = geocal_swig.VicarLiteRasterImage(fname)
            if(not rpc):
                rpc = img.rpc
        if(not title):
            title = os.path.basename(fname)
        geocal_swig.RpcImageGroundConnection.__init__(self, rpc, dem, img, title,
                                                 image_mask, ground_mask,
                                                 fit_height_offset)

    @classmethod
    def is_vicar_file(cls, fname):
        '''Return true if file is a vicar file'''
        m = re.match(r'(.*)(\d+):(\d+)(.*)', fname)
        if(m):
            base = m.group(1)
            baseend = m.group(4)
            i = int(m.group(2))
            return geocal_swig.VicarLiteFile.is_vicar_file("%s%d%s" % (base, i, baseend))
        else:
            return geocal_swig.VicarLiteFile.is_vicar_file(fname)

    @classmethod
    def pickle_format_version(cls):
        return 2

    def __reduce__(self):
        '''Base class RpcImageGroundConnection can be pickled, but the
        init function for this class is different. Give a simple reduce
        to handle this.'''
        return _new_from_init, (self.__class__,
                                self.__class__.pickle_format_version(),
                                self.fname,
                                self.dem, self.rpc, self.title,
                                self.image_mask, self.ground_mask,
                                self.fit_height_offset)

def _view_angle(self, image_coordinate = None, delta_h = 100):
    '''This return the zenith and azimuth view angles for a particular
    image coordinate as seen on the surface.

    This uses the equations and definitions as given in "Level 1
    Georectification and Registration Algorithm Theoretical Basis" JPL
    D-11532, Rev. B section 6.3.1.

    We determine the view angle by intersecting with the surface and a
    second surface delta_h above the dem. We then take the difference of
    the two to give the look angle. You can optionally supply the delta_h
    to use. For the look vector to be a useful concept it needs to be
    close to a straight line, meaning that in most cases the size of delta_h
    shouldn't change the results much.

    The center of the image is used if you don't supply image coordinates.

    The zenith and azimuth angles are return in that order, in degrees.
    '''
    if(image_coordinate is None):
        image_coordinate = geocal_swig.ImageCoordinate(self.number_line / 2.0,
                                                  self.number_sample /  2.0)
    gc1 = self.ground_coordinate(image_coordinate)
    h = gc1.height_reference_surface
    d = geocal_swig.SimpleDem(h + delta_h)
    gc2 = self.ground_coordinate_dem(image_coordinate, d)
    e1 = gc1.convert_to_cf()
    e2 = gc2.convert_to_cf()
    slat = sin(radians(gc1.latitude))
    clat = cos(radians(gc1.latitude))
    slon = sin(radians(gc1.longitude))
    clon = cos(radians(gc1.longitude))
    # Matrix used to change to local coordinates, with local up as z
    # and local north as x
    to_lc = np.array([[-clon * slat, -slon * slat,  clat],
                      [-slon,         clon,         0],
                      [-clon * clat,  -slon * clat, -slat]])
    lv = np.array([e2.position[0] - e1.position[0],
                   e2.position[1] - e1.position[1],
                   e2.position[2] - e1.position[2]])
    lv = lv / sqrt(np.dot(lv, lv))
    lc_dir = to_lc.dot(lv)
    az = degrees(atan2(lc_dir[1], lc_dir[0]))
    if(az < 0) : az += 360
    zen = 180 - degrees(acos(lc_dir[2] / sqrt(np.dot(lc_dir, lc_dir))))
    return zen, az

geocal_swig.ImageGroundConnection.view_angle = _view_angle

def _footprint_geometry(self, cconver = geocal_swig.GeodeticConverter()):
    '''Return a ogr Geometry object describing the footprint of the
    ImageGroundConnection. This includes the 4 corners of the image
    projected to the surface.

    You can supply the coordinates to use, the default it Geodetic
    longitude and latitude.

    This can then be used to write information to a ShapeFile.'''
    corners = []
    for pt in [geocal_swig.ImageCoordinate(0,0),
               geocal_swig.ImageCoordinate(self.image.number_line, 0),
               geocal_swig.ImageCoordinate(self.image.number_line,
                                      self.image.number_sample),
               geocal_swig.ImageCoordinate(0, self.image.number_sample)]:
        x, y, z = cconver.convert_to_coordinate(self.ground_coordinate(pt))
        corners.append((x, y))
    return ShapeLayer.polygon_2d(corners)

if(have_shape_file):
    geocal_swig.ImageGroundConnection.footprint_geometry = _footprint_geometry

__all__ = ["GdalImageGroundConnection", "VicarImageGroundConnection"]


def sum_digit_string(my_str):
    """
    -------------------------------------------------------
    Sums all the digits in my_str, ignores non-digit characters
    Use: total = sum_digit_string (my_str)
    -------------------------------------------------------
    Parameters:
         my_str: string that has single-digit numbers (str)
    returns total:
         sum of all the single digit number (integer >= 0)
    -------------------------------------------------------
    """
    total = 0
    if my_str == "":
        total = None
    else:
        for ch in my_str:
            if ch.isdigit() == True:
                total += int(ch)
    return total

def find_frequent(my_str):
    """
    -------------------------------------------------------
    Find the character(s) that occur the most in a string
    Use: value = find_frequent(my_str)
    -------------------------------------------------------
    Parameters:
         my_str: string (str)
    returns:
         value - a list of all the ch that occured the most in the string (list)
    -------------------------------------------------------
    """
    maxi = 0
    if my_str == "":
        value = None
    else:
        value = []
        for ch in my_str:
            if ch != " ":
                if my_str.count(ch) > maxi:
                    maxi = my_str.count(ch)
        for ch in my_str:
            if ch != " ":
                if my_str.count(ch) == maxi:
                    value.append(ch)
        value = list(dict.fromkeys(value))
    return value

def string_capitalizer(my_str):
    """
    -------------------------------------------------------
    capitalizes all characthers that should be capatilzed in a phrase
    Use: new_str = string_capitalizer(my_str)
    -------------------------------------------------------
    Parameters:
         my_str: string (str)
    returns:
         new_str: capaitalised string
    -------------------------------------------------------
    """
    new_str = my_str
    count = 0
    if my_str == "":
        new_str = None
    else:
        for ch in my_str:
            if count == 0:
                new_str = ch.upper() + new_str[count + 1:]
            elif (ch == "." or ch == "?") and count < len(my_str) - 1:
                count2 = 0
                new_str2 = new_str[count + 1:]
                for ch2 in new_str2:
                    if ch2 != " ":
                        new_str = new_str[0:(count + 1) + (count2)] + new_str2[count2].upper() + new_str2[count2 + 1:]
                        break
                    count2 += 1
            count += 1
    return new_str

def is_word_chain (my_list):
    """
    -------------------------------------------------------
    Checks if a list of words is a word chain
    Use: value = is_word_chain (my_list)
    -------------------------------------------------------
    Parameters:
         my_list: a list of strings (list)
    returns value:
         true: if the list is a word chain
         false: if the list is not a word chain
    -------------------------------------------------------
    """
    count = 0
    countF = 0
    value = False
    for i in my_list:
        new_i = i.replace(" ", "")
        if count < len(my_list) - 1:
            if new_i[len(new_i) - 1] == my_list[count + 1][0]:
                value = True
            else:
                countF += 1
        count += 1
    if countF > 0:
        value = False
    return value