# Humber flood embankment analysis

# A suite of shapefile data for understanding spatial distribution of available LIDAR and Geophysical
# survey-derived geotechnical data for engineered flood defence embankments on the Humber estuary, England.

import os
import geopandas as gpd
import matplotlib.pyplot as plt
from cartopy.feature import ShapelyFeature
import cartopy.crs as ccrs
import matplotlib.patches as mpatches
import matplotlib.lines as mlines
# these are modules that require importing

# generates a legend to be populated by the datesets later;
# for more info see https://matplotlib.org/stable/tutorials/intermediate/legend_guide.html
def generate_handles(labels, colors, edge='k', alpha=1):
    lc = len(colors)
    handles = []
    for i in range(len(labels)):
        handles.append(mpatches.Rectangle((0, 0), 1, 1, facecolor=colors[i % lc], edgecolor=edge, alpha=alpha))
    return handles

# set up the scale bar in preparation for finalised map, based on scale of Humber estuary, 20km fits best
# for for info see https://stackoverflow.com/q/32333870 and also https://stackoverflow.com/a/35705477
def scale_bar(ax, location=(0.92, 0.95)):
    x0, x1, y0, y1 = ax.get_extent()
    sbx = x0 + (x1 - x0) * location[0]
    sby = y0 + (y1 - y0) * location[1]

    ax.plot([sbx, sbx - 20000], [sby, sby], color='k', linewidth=9, transform=ax.projection)
    ax.plot([sbx, sbx - 10000], [sby, sby], color='k', linewidth=6, transform=ax.projection)
    ax.plot([sbx-10000, sbx - 20000], [sby, sby], color='w', linewidth=6, transform=ax.projection)

    ax.text(sbx, sby-4500, '20 km', transform=ax.projection, fontsize=10)
    ax.text(sbx-12500, sby-4500, '10 km', transform=ax.projection, fontsize=10)
    ax.text(sbx-24500, sby-4500, '0 km', transform=ax.projection, fontsize=10)

    # open Flood Areas polygon shapefile as this will be the basemap for subsequent data;
    # note that this and subsequent shapefiles are loaded using Geopandas, which associates each with a GeoDataFrame;
    # GeoDataFrame - tabular data structure, adds 'geometry' column to the shaoefile attribute table.
    areas = gpd.read_file('HSCR_Flood_Areas/HSCR_FloodAreas_2080406.shp')

    # see a subset of the first 5 lines, the key column is FloodArea_ and basemap will be rendered on this basis;
    # note the additional 'geometry' column, as this is a GeoDataFrame.
    areas.head(5)

    # open flood embankment network polylines shapefile; contains all the earth flood defence embankments around the Humber
    defences = gpd.read_file('Flood_embankment_network/Embankment_alignment_Humber_primary.shp')

    # open low sections polyline shapefile; contains LIDAR-derived data of where flood embankments are lower in height
    # than adjacent sections and could present a flood risk from overtopping during storm events when water levels are high
    Low_sections = gpd.read_file('LIDAR_derived_data/Embankment_low_sections_defects.shp')

    # open steep slopes polygon shapefile; contains LIDAR-derived data of where flood embankments slopes, either front (facing the water) or rear (facing the land),
    # are steeper than the standard angle of repose and could theoretically affect the long-term stability of the embankments
    Steep_slopes = gpd.read_file('LIDAR_derived_data/Embankment_steep_slope_anomalies.shp')

    # open palaeochannels point data shapefile; contains LIDAR-derived data of where infilled former channels, either natural
    # or engineered are situated directly below earth embankments and may theoretically affect the structural stability
    Palaeochannels = gpd.read_file('LIDAR_derived_data/Palaeochannel_anomalies_intersect_embank.shp')

    # open geophysical survey polyline shapefile; contains interpreted data from DEMP (Dipole Electromagnetic Profiling)
    # as to the material composition of surveyed flood embankments, which provide geotechnical context to assessments
    Geophysical_survey_bank_materials = gpd.read_file(
        'Geophysical_survey_data_material_composition/Geophysical_survey_embankment_materials.shp')

    # now the key datasets are opened, comparative analysis can be started; first on the defence, see subset of columns, rows
    # in the GeoDataFrame.
    defences.head(5)

    # to establish the linear length of earth embankment flood defences on the Humber estuary and main tidal tributary
    # rivers, the 'Shape_Leng' column is totalled up in metres and converted to kilometres (divide by 1000).
    defences['Shape_Leng'].sum() / 1000

    # the next dataset to be compared is the low sections; see subset of the data and note again a 'Shape_Leng' column
    Low_sections.head(5)

    # sum and convert to km
    Low_sections['Shape_Leng'].sum() / 1000

    # find low sections linear length as a percentage of defences to understand the scale of this type of this issue.
    # output should be 1.5% rounded up, so an issue which only affects a tiny proportion of the embankments.
    Low_sections['Shape_Leng'].sum() / defences['Shape_Leng'].sum() * 100

    # Steep slopes is a polygonal dataset; in this instance Shape_Leng refers to the perimeter of each polygon in the dataset,
    # rather than the linear length of embankment which is affected; Shape_Area does not relate to linear length either.
    Steep_slopes.head(5)

    # Alternative analysis can be carried out on how many of these occurrences are present.
    Steep_slopes['Face'].count()

    # There are over 4000 instances, more context can be derived from understanding which face (slope) of the embankments
    # are affected, the subset of the GeoDataframe above just indicated 'front' but a check can be carried out to
    # establish whether rear slopes are also listed.
    type_Steep_slopes = list(Steep_slopes.Face.unique())
    print('Name of unique features: {}'.format(type_Steep_slopes))

    # Calculate how many of the 4029 are affecting the front slope/face
    Steep_slopes[Steep_slopes['Face'] == 'Front'].count()

    # Calculate how many are affecting the rear slope/face
    Steep_slopes[Steep_slopes['Face'] == 'Rear'].count()

    # Calculate the instances of rear slope/face as a percentage of the total instance and by deduction,
    # the remaining percentage of front slope: 74.3 % rear (to 1 d.p) and 25.7 % front.
    2995 / 4029 * 100

    # See a subset of the Palaeochannels point data; the key column for further analysis is 'Class'.
    Palaeochannels.head(5)

    # Given the data are points, linear length calculation is not applicable; instead calculate instances.
    Palaeochannels['Class'].count()

    # 53 palaeochannels are identified in the data; define how many descriptive types there are.
    num_Palaeochannels = len(Palaeochannels.channel.unique())
    print('Number of unique features: {}'.format(num_Palaeochannels))

    # List the names of these 2 unique features.
    type_Palaeochannels = list(Palaeochannels.channel.unique())
    print('Name of unique features: {}'.format(type_Palaeochannels))

    # Calculate how many of the overall 53 are classed as 'depression'
    # 'depression' means a palaeochannel is lower in elevation than the surrounding land, as identified in LIDAR analysis.
    Palaeochannels.loc[Palaeochannels['channel'] == 'Depression'].count()

    # Calculate 49 depressions as a % of total 53 palaeochannels; answer is 92.5% (1 d.p), therefore 'extrusions'
    # where a palaeochannel is higher in elevation than the surrounding land constitute 7.5% of the total.
    49 / 53 * 100

    # Final data to analyse are the geophysical survey results; note there are a significant number of columns (27).
    Geophysical_survey_bank_materials.head(5)

    # List the column headings to aid appreciation of what data are included.
    Geophysical_survey_bank_materials.columns

    # Now establish the number of features i.e. the rows.
    rows, cols = Geophysical_survey_bank_materials.shape
    print('Number of features: {}'.format(rows))

    # Calculate the linear distance of survey conducted, using 'Tot_dist_m' which accurately reflects the actual length of
    # survey undertaken, rather than 'Shape_leng', which is a measure of the polyline lengths. Convert to km.
    Geophysical_survey_bank_materials['Tot_dist_m'].sum() / 1000

    # As these data are linear, a percentage calculation can be made in respect to the overall lemgth of earth embankments;
    # 37.2 % (to 1 d.p) has been surveyed, leaving 62.3 % available for future survey.
    Geophysical_survey_bank_materials['Tot_dist_m'].sum() / defences['Shape_Leng'].sum() * 100

    # The geophysical results have been interpreted in respect of the types of material within the embankments;
    # Create a list of all these material types;
    # Material types are based on four standard material types - clay, silt, sand, gravel. Where any of these materials
    # are considered the dominant constiuents, the name is shown in upper case, with lesser constituents in lower case.
    type_materials = list(Geophysical_survey_bank_materials.Material.unique())
    print('Name of unique features: {}'.format(type_materials))

    # Flood embankments should optimally be constructed from finer-grained, cohesive materials - clays and silts;
    # Where coarser-grained, less cohesive (and therefore more permeable) materials predominate there is theoretically
    # a greater risk of seepage of water through an embankment body and possibly also increased settlement of the crest
    # height as there are moe micro voids in coarser-grained materials;
    # This analysis, by linear length of material types will give a valuable overview.
    Geophysical_survey_bank_materials.groupby(['Material'])['Tot_dist_m'].sum() / 1000

    # The inherent saturation levels of the flood embankments is also a key factor in understanding embankment effectivness;
    # 'High' saturation indicates waterlogging,which is sub-optimal, as is 'Low' , which indicates excessive permeability;
    # By definition 'Normal' is optimal and from this analysis is the dominant Saturation type - 85 of 125 km.
    Geophysical_survey_bank_materials.groupby(['Saturation'])['Tot_dist_m'].sum() / 1000

    # To understand the spatial coverage of geophysical survey, data can be grouped by Hydraulic Units, into which the
    # Humber estuary interest area is sub-divided based on it's hydraulic functioning in times of flood.
    # Calculate and convert to km.
    Geophysical_survey_bank_materials.groupby(['HS_Hyd_Uni'])['Tot_dist_m'].sum() / 1000

    # Calculate the number of Hydraulic Units in which geophysical survey was carried out.
    num_Units_surveyed = len(Geophysical_survey_bank_materials.HS_Hyd_Uni.unique())
    print('Number of unique features: {}'.format(num_Units_surveyed))

    # Create the full list of Hydraulic Units on the Humber, based on the Areas GeoDataFrame.
    full_list_Units = list(areas.UNIT.unique())
    print('Name of unique features: {}'.format(full_list_Units))

    # Count the total number of Hydraulic Units
    # There are 15, therefore geophysical survey has been carried out in 10 of 15.
    num_Units = len(areas.UNIT.unique())
    print('Number of unique features: {}'.format(num_Units))

    # Create map displaying above data

    myFig = plt.figure(figsize=(16, 16))  # creates figure with definied dimensions in inches

    myCRS = ccrs.OSGB()  # Set the CartoPy Coordinate reference system to the Ordnance Survey Great Britain to ensure
    # compatibility with all the datasets.

    ax = plt.axes(projection=myCRS)  # create axes using the OSGB projection.

    # Add the Areas basemap, edged in light grey and with a white face colour.
    areas_feature = ShapelyFeature(areas['geometry'], myCRS, edgecolor='lightgray', facecolor='w')
    xmin, ymin, xmax, ymax = areas.total_bounds
    # Add this feature to the map
    ax.add_feature(areas_feature)

    # set the zoom extent to match the basemap shapefile extent with a 10km buffer on the x and y axes
    ax.set_extent([xmin - 10000, xmax + 10000, ymin - 10000, ymax + 10000], crs=myCRS)

    myFig  # draw the updated map

    # To colour in the Areas basemap, the number of colours required will be the total Flood Area features.
    num_areas = len(areas.FloodArea_.unique())
    print('Number of unique features: {}'.format(num_areas))

    # choose 35 colours from here https://matplotlib.org/stable/gallery/color/named_colors.html
    areas_colors = ['sandybrown', 'bisque', 'tan', 'moccasin', 'floralwhite', 'gold', 'darkkhaki',
                    'lightgoldenrodyellow', 'olivedrab', 'chartreuse', 'palegreen', 'mediumspringgreen',
                    'lightseagreen', 'paleturquoise', 'darkturquoise', 'deepskyblue', 'mediumpurple', 'darkorchid',
                    'plum', 'm', 'palevioletred', 'lightgray', 'lightcoral', 'mistyrose', 'peachpuff', 'navajowhite',
                    'orange', 'lemonchiffon', 'yellowgreen', 'c', 'skyblue', 'violet', 'fuchsia', 'indianred', 'salmon',
                    'y']

    # list and sort the areas
    area_numbers = list(areas.FloodArea_.unique())
    area_numbers.sort()

    # set more parameters as arguments, including the geometry (from the GeoDataFrame), edge colour, line width
    # and transparency (alpha - on a scale 0 to 1); face colour is set to the listed colours above.
    for ii, name in enumerate(area_numbers):
        feat = ShapelyFeature(areas.loc[areas['FloodArea_'] == name, 'geometry'],
                              myCRS,
                              edgecolor='w',
                              facecolor=areas_colors[ii],
                              linewidth=0.1,
                              alpha=0.25)
        ax.add_feature(feat)  # add features to map

    myFig  # draw updated map

    # Add defences in the same way
    defences = ShapelyFeature(defences['geometry'],
                              myCRS,
                              edgecolor='b',  # blue
                              linewidth=2.0,
                              alpha=0.25
                              )
    ax.add_feature(defences)  # add features to the map

    myFig  # draw updated map

    # Add geophysical survey data
    Geophysical_survey_bank_materials = ShapelyFeature(Geophysical_survey_bank_materials['geometry'],
                                                       myCRS,
                                                       edgecolor='lime',
                                                       linewidth=1.0,
                                                       )
    ax.add_feature(Geophysical_survey_bank_materials)

    myFig

    # Add low sections data
    Low_sections = ShapelyFeature(Low_sections['geometry'],
                                  myCRS,
                                  edgecolor='darkorange',
                                  linewidth=1.0)
    ax.add_feature(Low_sections)

    myFig

    # Add Steep slopes, note as these data are polygons, the facecolor needs to be set
    Steep_slopes = ShapelyFeature(Steep_slopes['geometry'],
                                  myCRS,
                                  edgecolor='k',  # black
                                  facecolor='k',  # due to scale of map, a separate colour will not be discernible
                                  linewidth=0.1)
    ax.add_feature(Steep_slopes)

    myFig

    # Palaeochannels are point data so do not need the ShapelyFeature class from cartopy.feature, ax.plot can be used directly.

    Palaeochannels_handle = ax.plot(Palaeochannels.geometry.x, Palaeochannels.geometry.y, 'o', color='0.7', ms=3,
                                    transform=myCRS)
    # 'o' is circle, courtesy of list here https://stackoverflow.com/questions/8409095/set-markers-for-individual-points-on-a-line
    # colour is String representation of float value in closed interval [0, 1] for grayscale values, os 0.7 is light grey
    # see https://matplotlib.org/stable/tutorials/colors/colors.html

    myFig

    # Use generate-handles function to add labels and colours. Note - Areas is omitted for visual clarity, given there
    # are 35 areas.

    defences_handle = generate_handles(['Flood embankments'], ['b'], alpha=0.25)

    Low_sections_handle = generate_handles(['Low sections of embankments'], ['darkorange'])

    Steep_slopes_handle = generate_handles(['Steep embankment slopes'], ['k'])

    Palaeochannels_handle = generate_handles(['Palaeochannel intersections'], ['chocolate'])

    Geophysical_survey_bank_materials_handle = generate_handles(['Geophysical survey extents'], ['lime'])

    # generate legend content and parameters
    # for more info see https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.legend.html

    handles = defences_handle + Low_sections_handle + Steep_slopes_handle + Palaeochannels_handle + Geophysical_survey_bank_materials_handle  # use '+' to concatenate (combine) lists
    labels = ['Flood embankments', 'Low sections of embankments', 'Steep embankment slopes',
              'Palaeochannel intersections', 'Geophysical survey extents']

    leg = ax.legend(handles, labels, title='Legend', title_fontsize=14,
                    fontsize=12, loc='upper left', frameon=True, framealpha=1)

    myFig

    # now add scale bar which was previously specified
    scale_bar(ax)

    myFig

    # save the map to the current folder in png format; also specify no border (bbox) and resolution in dpi (dots per inch).
    myFig.savefig('Humber_flood_embankment_data.png', bbox_inches='tight', dpi=800)


# coding: utf-8
"""
An example of training two sentences classification model with
QNLI dataset.

@author: Weijie Liu
"""
import os
import torch
import sys
fastbert_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
sys.path.append(fastbert_dir)
from fastbert import FastBERT_S2

train_dataset_path = "../../datasets/lcqmc/train.tsv"
dev_dataset_path = "../../datasets/lcqmc/dev.tsv"
model_saving_path = "/tmp/fastbert_lcqmc.bin"

def loading_dataset(dataset_path):
    sents_a, sents_b, labels = [], [], []
    with open(dataset_path, 'r', encoding='utf-8') as infile:
        for i, line in enumerate(infile):
            if i == 0:
                continue
            line = line.strip().split('\t')
            sents_a.append(line[0])
            sents_b.append(line[1])
            labels.append(line[2])
    return sents_a, sents_b, labels

def main():

    sents_a_train, sents_b_train, labels_train = loading_dataset(train_dataset_path)
    sents_a_dev, sents_b_dev, labels_dev = loading_dataset(dev_dataset_path)
    labels = list(set(labels_train))
    print("Labels: ", labels)  # [0, 1]

    # FastBERT
    model = FastBERT_S2(
        kernel_name="uer_bert_tiny_zh",
        labels=labels,
        device="cuda:0" if torch.cuda.is_available() else "cpu"
    )

    # # FastGPT
    # model = FastGPT_S2(
    #     kernel_name="uer_gpt_zh",
    #     labels=labels,
    #     device="cuda:0" if torch.cuda.is_available() else "cpu"
    # )

    model.fit(
        sents_a_train=sents_a_train,
        sents_b_train=sents_b_train,
        labels_train=labels_train,
        sents_a_dev=sents_a_dev,
        sents_b_dev=sents_b_dev,
        labels_dev=labels_dev,
        batch_size=32,
        finetuning_epochs_num=5,
        distilling_epochs_num=10,
        learning_rate=2e-5,
        report_steps=500,
        model_saving_path=model_saving_path,
        verbose=True,
    )

if __name__ == "__main__":
    main()


import pandas as pd
import numpy as np

from scipy import stats

import gc

def calculate_prob_dist_for_data_augmentation(X_list,granularity):
    X = pd.concat(X_list)
    X_treatment = X[X['cp_type']==0]
    prob_dist = []
    dist = np.arange(0,1,1/granularity)
    for col in X.columns[4:]:
        x = X_treatment[col].values
        kernel = stats.gaussian_kde(x)
        prob = kernel.pdf(dist)
        prob_dist.append(prob)
    prob_dist = np.array(prob_dist)
    return prob_dist

def generate_data_augmentation(X,prob_dist,granularity,max_dev=0.1,normal_std_dev=0.1,additional=10):
    x = X.values[:,4:].copy().astype(np.float16)

    max_dev_steps = int(max_dev*granularity)

    #Calculate normal matrix
    normal_p = np.arange(-max_dev*granularity,max_dev*granularity+1,1)
    normal_p = normal_p/granularity
    normal_p = 1/(normal_std_dev)*np.exp(-(normal_p*normal_p)/normal_std_dev**2)
    normal_p = normal_p.astype(np.float16)
    normal_p = np.repeat(normal_p[np.newaxis,:], x.shape[1], axis=0)
    normal_p = np.repeat(normal_p[np.newaxis,:,:], x.shape[0], axis=0)

    #Transform x so that it rounds to the desired granularity
    x_rounded = (np.round(x*granularity)).astype(int)

    #For each and every value a in x, we want to calculate a vector of probability of size 2n+1 such as
    #The probability value at index 0 is the probability that we remove max_dev to a
    i_steps = np.arange(-max_dev_steps,max_dev_steps+1,1) #initialization vector for the steps
    i_initial = np.tile(np.array([[i_steps]]),(x.shape[0],x.shape[1],1))
    x_rounded_repeated = np.repeat(x_rounded[:, :, np.newaxis], i_steps.shape[0], axis=2)
    idx = i_initial + x_rounded_repeated
    idx = idx.copy()
    idx = np.clip(idx,0,granularity-1) #For each

    del i_initial, x_rounded_repeated
    gc.collect()

    #prob_candidates = prob_dist[0,0,idx].copy()
    prob_candidates = np.zeros(idx.shape)
    for j in range(idx.shape[1]):
        prob_candidates[:,j,:] = np.take(prob_dist[j,:],idx[:,j,:])

    del idx
    gc.collect()

    prob_candidates = prob_candidates*normal_p
    prob_candidates = prob_candidates.copy()

    del normal_p
    gc.collect()

    prob_candidates = prob_candidates/prob_candidates.sum(axis=2)[:,:,np.newaxis]

    var = np.zeros([x.shape[0],x.shape[1],additional])
    i_steps_norm = i_steps/max_dev_steps*max_dev
    print('calculating probas')
    for k in range(x.shape[1]):
        for i in range(x.shape[0]):
            var[i,k,:] = np.random.choice(i_steps_norm,size=additional,p=prob_candidates[i,k,:])

    var_list = []
    for i in range(additional):
        var_list.append(var[:,:,i].copy())
    return var_list


# -*- coding: utf-8 -*-
"""
Flexible implementation of the hard-coded test_features_ct.py script.

Created on Thu Dec 10 11:41:03 2015

@author: Sadi
"""

from __future__ import print_function

# Basic Packages
import numpy as np
import dill
from time import time
from joblib import Parallel, delayed
import joblib
import sys
import matplotlib.pyplot as plt
import os

# Sci
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.image import extract_patches_2d
from sklearn.feature_extraction.image import reconstruct_from_patches_2d
from skimage.feature import hog

# Helpers
from helper_io import get_filenames, find_biggest_slice, get_nifti_slice
from helper_eigen import extract_roi_patches_w, get_randoms_w
from helper_gabor import generate_kernels, compute_feats

# CONSTANTS
path = "ct/" # path to the CTs and the associated labels
im_name = "CT.nii.gz" # string used to select the images
lb_name = "CT_CTV.nii.gz" # string used to select the labels
psize = 12 # patch "radius", so the patch dimensions are psize x psize
# how many "principal component" patches to generate per slice per class
# for example, 10 will result in 10 PC patches for masked, 10 PC patches
# for non-masked regions.
pc_max = 500
ct_cap_min = -1000 # minimum CT brightness (set ct_cap_max for no clipping)
ct_cap_max = 2000 # maximum CT brightness (set to 0 for no clipping)
ct_monte = 1 # 1 will use random patches; 0 will use PCA patches
pickle = "slice_infos_12.pkl" # path and filename to save the SliceInfos
recons = "recons_12_3.pkl" # path and filename to store the patches of reconstruction
generate = 0 # toggle generation of new SliceInfos
classify = 0 # test classification engine
no_trees = 10 # number of trees to use for classifier
fullspec_i = 3
crop = (128, 128)
np.seterr(all='ignore')

# BIGTEST CONSTANTS
psizes = [12]

"""
Class to contain all the training data from a slice extracted from a 3D MRI.
Consists of the filename of the slice, the "number" of the slice (i.e. the
index of the slice in the 3D MRI), the original slice image, the associated
label image, their orientations, masked/unmasked: patches, gabor features,
HOG features, and "percentage values"; the percentage of label in that
particular patch.
"""
class SliceInfo():
    def __init__(self, filename, slice_no,
                 slice_im, slice_im_or,
                 slice_lb, slice_lb_or,
                 patches_m_pc, patches_n_pc,
                 feats_m, feats_n,
                 vals_m=None, vals_n=None,
                 hogs_m=None, hogs_n=None):
        self.filename = filename
        self.slice_no = slice_no

        self.slice_im = slice_im
        self.slice_im_or = slice_im_or
        self.slice_lb = slice_lb
        self.slice_lb_or = slice_lb_or

        self.patches_m_pc = patches_m_pc
        self.patches_n_pc = patches_n_pc
        self.feats_m = feats_m
        self.feats_n = feats_n

        self.hogs_m = hogs_m
        self.hogs_n = hogs_n

        self.vals_m = vals_m
        self.vals_n = vals_n

    def get_info(self):
        print("Filename: ", self.filename)
        print("Slice No.: ", self.slice_no)
        print("Image: ", np.array(self.slice_im).shape)
        print("Orientation: ", np.array(self.slice_im_or).shape)
        print("Label: ", np.array(self.slice_lb).shape)
        print("Label Orientation: ", np.array(self.slice_lb_or).shape)
        print("Masked Patches: ", np.array(self.patches_m_pc).shape)
        print("Unmasked Patches: ", np.array(self.patches_n_pc).shape)
        print("Masked Features: ", np.array(self.feats_m).shape)
        print("Unmasked Features: ", np.array(self.feats_n).shape)
        print("Masked Values Shape: ", np.array(self.vals_m).shape)
        print("Unmasked Values Shape: ", np.array(self.vals_n).shape)
        if self.hogs_m:
            print("Masked HOGs: ", np.array(self.hogs_m).shape)
        if self.hogs_n:
            print("Unmasked HOGs: ", np.array(self.hogs_n).shape)
        if self.vals_m:
            print("Masked Values: ", self.vals_m)
        if self.vals_n:
            print("Unmasked Values: ", self.vals_n)

"""
Deprecated.
"""
#def create_pc_patches(slice_im, slice_lb):
#    # for CT, cap the image
#    if ct_cap_max:
#        slice_im = np.clip(slice_im, ct_cap_min, ct_cap_max)
#
#    # extract all of the patches
#    # _m means "masked" patches, i.e. patches that are in the ROI
#    # _n means "non-masked" patches, i.e. patches that aren't in the ROI
#    patches_m, patches_n = extract_roi_patches(slice_im, slice_lb, psize)
#
#    # use random patches to represent an image
#    if ct_monte:
#        # _m_pc means "masked" "principal components"
#        # _n_pc means "non-masked" "principal components"
#        patches_m_pc = get_randoms(patches_m, pc_max)
#        patches_n_pc = get_randoms(patches_n, pc_max)
#
#    # generate eigenpatches using PCA
#    else:
#        patches_m_pc = get_eigenpatches(patches_m, psize, pc_max)
#        patches_n_pc = get_eigenpatches(patches_n, psize, pc_max)
#
#    return patches_m_pc, patches_n_pc

"""
Deprecated.
"""
#def create_sliceinfo(images_fn, labels_fn, kernels, i):
#    # figure out the biggest slice
#    slice_no = find_biggest_slice(path + labels_fn[i])
#
#    # get the slice, label, and associated orientations
#    slice_im, slice_im_or = get_nifti_slice(path + images_fn[i], slice_no)
#    slice_lb, slice_lb_or = get_nifti_slice(path + labels_fn[i], slice_no)
#
#    # if crop, we crop the image down
#    if crop:
#        crop_x, crop_y = crop
#        start_x = slice_im.shape[0] / 2 - crop_x / 2
#        end_x = start_x + crop_x
#        start_y = slice_im.shape[1] / 2 - crop_y / 2
#        end_y = start_y + crop_y
#
#        slice_im = slice_im[start_x:end_x, start_y:end_y]
#        slice_lb = slice_lb[start_x:end_x, start_y:end_y]
#
#    # figure out the principal patches
#    pc_payload = (slice_im, slice_lb)
#    patches_m_pc, patches_n_pc = create_pc_patches(*pc_payload)
#
#    # compute gabor features for the patches
#    feats_m = []
#    feats_n = []
#    for patch in patches_m_pc:
#        feats_m.append(compute_feats(patch, kernels))
#    for patch in patches_n_pc:
#        feats_n.append(compute_feats(patch, kernels))
#
#    # package it into a SliceInfo object
#    si_payload = (images_fn[i], slice_no,
#                  slice_im, slice_im_or,
#                  slice_lb, slice_lb_or,
#                  patches_m_pc, patches_n_pc,
#                  feats_m, feats_n)
#
#    return SliceInfo(*si_payload)

"""
Deprecated.
"""
#def create_sliceinfos(images_fn, labels_fn):
#    kernels = generate_kernels() # create gabor kernels
#    slice_infos = []
#
#    if len(sys.argv) < 2:
#        for i in range(len(images_fn)):
#            slice_infos.append(create_sliceinfo(images_fn, labels_fn, kernels, i))
#
#    else:
#        slice_infos = Parallel(n_jobs=int(sys.argv[1]))(delayed(create_sliceinfo)(images_fn, labels_fn, kernels, i) for i in range(len(images_fn)))
#
#    return slice_infos

"""
Deprecated.
"""
#def classify_patch(RF, kernels, patches, i):
#    # compute the feats of the patch
#    patch = patches[i]
#    feat = compute_feats(patch, kernels)
#    feat = feat.flatten().reshape(1, -1)
#    prediction = RF.predict(feat)
#    print("Classifying patch {}/{}: {}".format(i, len(patches), prediction))
#    if prediction == 'M':
#        return np.ones(patch.shape)
#    elif prediction == 'N':
#        return np.zeros(patch.shape)
#    else:
#        return np.full(patch.shape, prediction)

"""
Given an image (most likely a small patch), generates HOG features for that
image. Returns a list of numbers.

In this release, the HOG features have fixed parameters. In the future, they
will be modifiable through global constants.
"""
def compute_hogs(patch):
    fd, hog_im = hog(patch, orientations=8, pixels_per_cell=(4, 4),
                     cells_per_block=(1, 1), visualise=True)
    return fd

"""
Given an image and associated label, generate the "principal component patches"
of the image separated into two separate groups: masked and unmasked PCPs.

A tuple of four sets is returned: masked PCPs, unmasked PCPs, masked values,
and unmasked values. The explanations for "PCP" and "value" are described
below.

The image is (optionally) preprocessed if ct_cap_max is set to clip the
brightness values of the pixels to the range [ct_cap_min, ct_cap_max]. This
is generally done to filter out extremely bright seeds that appear in
prostate CT scans.

The masked and unmasked patches are created from a function call (see
"extract_roi_patches_w"). Masked patches are described as patches of the
image which have at least one pixel that is "pixel of interest". The pixel
of interest is defined as a pixel whose associated pixel in the label image is
high/bright/1. In addition, each patch also has a "value", which describes
the percentage of pixels of interest in that patch as a decimal (i.e. between
0 and 1).

All the masked and unmasked patches are reduced into a "principal component"
set, which is a random selection of the existing masked and unmasked patches.
The number of random patches selected is defined by the constant pc_max. The
number of random patches is guaranteed and thus the number of unmasked and
masked PCPs will always be equal. If ct_max exceeds the number of patches
available in the selection pool of all masked/unmasked patches, patches will be
duplicated to fill out the required ct_max PCPs. So if ct_max = 500 and you
only have 30 masked patches available, the 30 will be approximately repeated
across the 500 PCPs produced.

By extension, the PCPs may have duplicates even if the available patches exceed
ct_max. This is because the RNG doesn't check if an existing patch has already
been added to the PCP record, so the RNG may produce the same patch multiple
times.
"""
def create_pc_patches_w(slice_im, slice_lb):
    # for CT, cap the image
    if ct_cap_max:
        slice_im = np.clip(slice_im, ct_cap_min, ct_cap_max)

    # extract all of the patches
    # _m means "masked" patches, i.e. patches that are in the ROI
    # _n means "non-masked" patches, i.e. patches that aren't in the ROI
    patches_m, patches_n, vals_m, vals_n = extract_roi_patches_w(slice_im,
                                                               slice_lb,
                                                               psize)

    # use random patches to represent an image
    if ct_monte:
        # _m_pc means "masked" "principal components"
        # _n_pc means "non-masked" "principal components"
        patches_m_pc, vals_m_pc = get_randoms_w(patches_m, vals_m, pc_max)
        patches_n_pc, vals_n_pc = get_randoms_w(patches_n, vals_n, pc_max)

    # PCA is deprecated in this implementation
#    else:
#        patches_m_pc = get_eigenpatches(patches_m, psize, pc_max)
#        patches_n_pc = get_eigenpatches(patches_n, psize, pc_max)

    return patches_m_pc, patches_n_pc, vals_m_pc, vals_n_pc

"""
Given a list of image filenames, a list of label filenames, a set of Gabor
kernels, and the index of the image in the list of images, produces a
"SliceInfo" object from the data. These filenames reference existing 3DMRI
Nifti volumes that are in a folder defined by the "path" constant at the top.
Images are defined with the extension stored in "im_name" and the label
extension stored in "lb_name", both constants defined at the top.

This is done by first finding the largest label in the 3D volume. The largest
label is the label image whose contains the most amount of bright pixels. This
is referred to as the "slice no." for a particular SliceInfo object.

The slice_no slice of each 3DMRI and associated 3D Label is extracted and
stored as raw image data (see "get_nifti_slice" in helper_io).

The slice is optionally cropped down to a fixed size around the center as
defined by the crop constant defined at the top. If the crop constant is None
or 0, the slice isn't cropped. This cropping applies to both the image and
the label; they need to be associated with each other.

The principal patches of the slice are computed with "create_pc_patches_w" (see
above). Each principal patch is described in terms of features (see
"compute_feats" and "compute_hogs"). The combination of the slice information,
the patches and the features are stored in a SliceInfo object which is
finally returned.
"""
def create_sliceinfo_w(images_fn, labels_fn, kernels, i):
    # figure out the biggest slice
    print("Creating Weighted Slice Info... {}/{}: {}".format(i+1, len(images_fn), labels_fn[i]))
    slice_no = find_biggest_slice(path + labels_fn[i])

    # get the slice, label, and associated orientations
    slice_im, slice_im_or = get_nifti_slice(path + images_fn[i], slice_no)
    slice_lb, slice_lb_or = get_nifti_slice(path + labels_fn[i], slice_no)

    # if crop, we crop the image down
    if crop:
        crop_x, crop_y = crop
        start_x = slice_im.shape[0] / 2 - crop_x / 2
        end_x = start_x + crop_x
        start_y = slice_im.shape[1] / 2 - crop_y / 2
        end_y = start_y + crop_y

        slice_im = slice_im[start_x:end_x, start_y:end_y]
        slice_lb = slice_lb[start_x:end_x, start_y:end_y]

    # figure out the principal patches
    pc_payload = (slice_im, slice_lb)
    patches_m_pc, patches_n_pc, vals_m, vals_n = create_pc_patches_w(*pc_payload)

    # compute gabor features for the patches
    feats_m = []
    feats_n = []
    hogs_m = []
    hogs_n = []
    for patch in patches_m_pc:
        feats_m.append(compute_feats(patch, kernels))
        hogs_m.append(compute_hogs(patch))
    for patch in patches_n_pc:
        feats_n.append(compute_feats(patch, kernels))
        hogs_n.append(compute_hogs(patch))

    # package it into a SliceInfo object
    si_payload = (images_fn[i], slice_no,
                  slice_im, slice_im_or,
                  slice_lb, slice_lb_or,
                  patches_m_pc, patches_n_pc,
                  feats_m, feats_n,
                  vals_m, vals_n,
                  hogs_m, hogs_n)

    return SliceInfo(*si_payload)

"""
Given a list of filenames of 3DMRI Nifti images and a list of filenames of
of their associated labels, creates a list of SliceInfo objects, each of which
containing the slice information, patches and features associated to a
particular Nifti filename in the input arguments.

Each SliceInfo object is created on a separate process, with the maximum
number of processes defined as the first commandline argument. If there is no
commandline argument or if the number of processes is 1, the SliceInfos are
generated serially.

Returns the list of the SliceInfo objects.
"""
def create_sliceinfos_w(images_fn, labels_fn):
    print("Creating weighted Slice Infos...")
    kernels = generate_kernels() # create gabor kernels
    slice_infos = []

    if len(sys.argv) < 2:
        for i in range(len(images_fn)):
            slice_infos.append(create_sliceinfo_w(images_fn, labels_fn, kernels, i))

    else:
        slice_infos = Parallel(n_jobs=int(sys.argv[1]))(delayed(create_sliceinfo_w)(images_fn, labels_fn, kernels, i) for i in range(len(images_fn)))

    return slice_infos

"""
Given access to an entire set of SliceInfo objects and given an index which
will be the index of the "test slice", creates a set of "features" and
associated "labels" which will be used to train the random forest classifier.
The sets are reshaped signficantly so that the classifier can fit to them.

Returns a tuple containing the set of samples and set of labels for an RF
to fit directly against them. The sample/label set could also be used for
any other machine learning application which supports inputs of this shape
(2D features; 1D labels). The training features leave-n-out, where n = 1 and
the left out data corresponds to the ith SliceInfo object.

There is some deprecated code in here. Originally, patches and features were
labelled in a binary manner. That is, a patch was described as being a
"masked" patch or an "unmasked" patch, where patches containing at least one
pixel of interest were labelled as "masked". Now, the SliceInfos generally
use patch "values" instead, which describe what percentage of the patch or
associated feature describing the patch contains bright pixels in the
associated label patch.
"""
def generate_training_feats(slice_infos, i):
    train_m = []
    train_n = []

    labels_m = []
    labels_n = []

    hogs_m = []
    hogs_n = []

    # go through each slice
    for j in range(len(slice_infos)):
        # if it's the ith slice, don't include it in the training
        if j == i:
            continue
        train_m.append(slice_infos[j].feats_m)
        train_n.append(slice_infos[j].feats_n)
        if slice_infos[j].vals_m != None:
            labels_m.extend(slice_infos[j].vals_m)
            labels_n.extend(slice_infos[j].vals_n)
        if slice_infos[j].hogs_m != None:
            hogs_m.append(slice_infos[j].hogs_m)
            hogs_n.append(slice_infos[j].hogs_n)

    train_m = np.array(train_m)
    train_n = np.array(train_n)
    hogs_m = np.array(hogs_m)
    hogs_n = np.array(hogs_n)

    tms = train_m.shape
    tns = train_n.shape
    hms = hogs_m.shape
    hns = hogs_n.shape

    print(tms, tns)
    print(hms, hns)

    train_m = train_m.reshape(tms[0] * tms[1], tms[2] * tms[3])
    train_n = train_n.reshape(tns[0] * tns[1], tns[2] * tns[3])
    hogs_m = hogs_m.reshape(hms[0] * hms[1], hms[2])
    hogs_n = hogs_n.reshape(hns[0] * hns[1], hns[2])

    train_m = np.concatenate((train_m, hogs_m), axis=1)
    train_n = np.concatenate((train_n, hogs_n), axis=1)

    print(train_m.shape, train_n.shape)
    #print(hogs_m.shape, hogs_n.shape)

    #raise Exception

    samples = np.concatenate((train_m, train_n))
    if slice_infos[0].vals_m == None:
        labels = ['M' for k in train_m] + ['N' for p in train_n]
    else:
        labels = labels_m + labels_n

    return samples, labels

"""
Given a set of features, a set of associated labels, and the number of trees
that the classifier should use, trains a random forest classifier on the data
and returns a reference to the classifier for use in predicting the classes
of patches.
"""
def train_rf_classifier(features, labels, no_trees):
    rf = RandomForestClassifier(n_estimators=no_trees, n_jobs=-1)
    rf.fit(features, labels)
    return rf

"""
Given access to the full set of existing SliceInfo objects, and an index which
represents a "test slice", generates training data using all the SliceInfo
objects and their features (except for the ith SliceInfo object) and trains
a random forest classifier on that data (see "generate_training_feats" and
"train_rf_classifier"). Then, asks the trained classifier to predict the
classes of the patches stored inside the test slice (the ith SliceInfo object).
Records whether the random forest correctly classified each slice, and then
displays the results. The predictions are only on the existing features stored
for training in the SliceInfo object, so it doesn't check every single patch.
Essentially, this means that it only tests on the PCPs of the ith SliceInfo.

**This function is not updated for the HOG featureset that is now added by
default to the SliceInfo class. Therefore, you'll get an error and the
program will crash if you try to call this function right now.
"""
def test_rf_feats(slice_infos, i):
    print("Testing Case {}/{}: ".format(i+1, len(slice_infos)), end="")

    # We need to train on masked and unmasked features from every
    # slice except for this one
    feats, labels = generate_training_feats(slice_infos, i)

    # Train the classifier
    RF = train_rf_classifier(feats, labels, no_trees)

    # Test the RF on each of the feats from the test slice
    test_sl = slice_infos[i]
    successes = 0
    trials = 0
    total = len(test_sl.feats_m) + len(test_sl.feats_n)
    for feat_m in test_sl.feats_m:
        # flatten the feature so it can be tested
        feat_m = feat_m.flatten().reshape(1, -1)
        if RF.predict(feat_m) == 'M':
            successes += 1
        trials += 1
        if(len(sys.argv) < 2):
            print("\rTesting Case {}/{}: {}/{}/{} successes.".format(
            i+1, len(slice_infos), successes, trials, total), end="")
    for feat_n in test_sl.feats_n:
        # flatten the feature so it can be tested
        feat_n = feat_n.flatten().reshape(1, -1)
        if RF.predict(feat_n) == 'N':
            successes += 1
        trials += 1
        if(len(sys.argv) < 2):
            print("\rTesting Case {}/{}: {}/{}/{} successes.".format(
            i+1, len(slice_infos), successes, trials, total), end="")

    print("")
    print(np.asarray(slice_infos[i].feats_m).shape)

    return successes, trials

"""
Works like test_rf_feats, but instead of testing the classifier on the PCPs
of each SliceInfo object, tests the classifier on *every* patch that is
generated from the slice image.

**This function is not updated for the HOG featureset that is now added by
default to the SliceInfo class. Therefore, you'll get an error and the
program will crash if you try to call this function right now.
"""
def test_rf_feats_fullspec(slice_infos, i):
    feats, labels = generate_training_feats(slice_infos, i)
    RF = train_rf_classifier(feats, labels, no_trees)

    test_sl = slice_infos[i]
    image = test_sl.slice_im
    label = test_sl.slice_lb
    patches_m, patches_n = extract_roi_patches(image, label, psize)
    patches_n = patches_n
    plabels_m = ['M' for m in patches_m]
    plabels_n = ['N' for n in patches_n]
    kernels = generate_kernels()

    tot = len(patches_m) + len(patches_n)

    res1 = []
    res2 = []

    t0 = time()

    if len(sys.argv) >= 2:
        res1 = Parallel(n_jobs=int(sys.argv[1]))(delayed(check_classify)(RF, kernels, p, plabels_m[i], i, tot) for i, p in enumerate(patches_m))
        res2 = Parallel(n_jobs=int(sys.argv[1]))(delayed(check_classify)(RF, kernels, p, plabels_n[i], i, tot) for i, p in enumerate(patches_n))
    else:
        for i,p in enumerate(patches_m): # go through each patch, and classify it!
            res1.append(check_classify(RF, kernels, p, plabels_m[i], i, tot))
        pass

    dt = time() - t0

    print(res1.count(True) + res2.count(True))
    print("Finished in {:.2f} seconds.".format(dt))

"""
Helper function to check the prediction for a single patch, used in
test_rf_feats_fullspec.

**This function is not updated for the HOG featureset that is now added by
default to the SliceInfo class. Therefore, you'll get an error and the
program will crash if you try to call this function right now.
"""
def check_classify(RF, kernels, patch, patch_label, i, tot):
    # kernel the patch
    print("Checking Classifying patch {}/{}".format(i, tot))
    feat = compute_feats(patch, kernels)
    feat = feat.flatten().reshape(1, -1)
    prediction = RF.predict(feat)
    if prediction == patch_label:
        return True
    else:
        return False

def classify_patch_w(fn, kernels, patches, i):
    RF = joblib.load(fn)
    patch = patches[i]
    feat = compute_feats(patch, kernels)
    feat = feat.flatten().reshape(1, -1)
    hogs = compute_hogs(patch)
    hogs = hogs.flatten().reshape(1, -1)
    feat = np.concatenate((feat, hogs), axis=1)

    prediction = RF.predict(feat)
    print("Classifying patch {}/{}: {}".format(i, len(patches), prediction))
    if prediction == 'M':
        return np.ones(patch.shape)
    elif prediction == 'N':
        return np.zeros(patch.shape)
    else:
        return np.full(patch.shape, prediction)

"""
Generate labels using Random Forest on a particular
"""
def rf_reconstruct(slice_infos, i):
    feats, labels = generate_training_feats(slice_infos, i)
    labels = np.array(labels)
    RF = train_rf_classifier(feats, labels, no_trees)

    test_sl = slice_infos[i]
    image = test_sl.slice_im

    kernels = generate_kernels()

    # break the image into patches; all of these will be classified
    patch_size = (psize, psize)
    # _a stands for "all"
    patches_a = extract_patches_2d(image, patch_size)
    # _p stands for "predict"

    # dump the RF
    fn_rf = 'rf.joblib'
    joblib.dump(RF, fn_rf)

    # check each patch
    if len(sys.argv) >= 2:
        #patches_p = Parallel(n_jobs=int(sys.argv[1]))(delayed(classify_patch)(RF, kernels, patches_a, i) for i in range(len(patches_a)))
        patches_p = Parallel(n_jobs=int(sys.argv[1]))(delayed(classify_patch_w)(fn_rf, kernels, patches_a, i) for i in range(len(patches_a)))

    else:
        patches_p = []
        for i in range(len(patches_a)):
            patches_p.append(classify_patch(RF, kernels, patches_a, i))

    # reconstruct based on the patch
    recons_im = reconstruct_from_patches_2d(np.asarray(patches_p), image.shape)
    print(recons_im.shape)

    # save patches_p to the drive, because it took so much work to make!
    with open(recons, 'wb') as f:
        dill.dump(recons_im, f)

"""
Threshold a reconstructed image by pushing numbers below value down to 0 and
numbers equal to or above value up to 1.
"""
def threshold(image, value, round=None):
    threshold = np.array(image) # make a copy
    threshold[threshold < value] = 0
    if round:
        threshold[threshold >= value] = 1
    return threshold

"""
Uses the label to mask out the image.
"""
def mask_out(image, label):
    mask = np.where(label == 0, -1000, image)
    return mask

def compare_im(recons_im, real_lb, display=False):
    if display:
        plt.imshow(recons_im)
        plt.show()

        plt.imshow(real_lb)
        plt.show()

    recons_dice = recons_im.astype(int)
    label_dice = real_lb.astype(int)
    dice = float(2*np.count_nonzero(recons_dice & label_dice))/(np.count_nonzero(recons_dice)
    + np.count_nonzero(label_dice))

    return dice

def run():
    if generate:
        t0 = time()
        # get lists of files to process
        images_fn, labels_fn = get_filenames(path, im_name, lb_name)

        # generate sliceinfos for all those images
        if generate == 1:
            slice_infos = create_sliceinfos(images_fn, labels_fn)
        elif generate == 2:
            slice_infos = create_sliceinfos_w(images_fn, labels_fn)
            print(slice_infos[0].vals_m[:30])

        dt = time() - t0
        print("Finished in %.2f seconds." % dt)

        # save the slice info
        with open(pickle, 'wb') as f:
            dill.dump(slice_infos, f)

    else:
        with open(pickle, 'rb') as f:
            slice_infos = dill.load(f)

    total_successes = 0
    total_trials = 0

    if classify == 1: # Go through each slice and classify PCPs
    # PCP = principal component patches

        t0 = time()

        if len(sys.argv) >= 2:
            res = Parallel(n_jobs=int(sys.argv[1]))(delayed(test_rf_feats)(slice_infos, i) for i in range(len(slice_infos)))
            for successes, trials in res:
                total_successes += successes
                total_trials += trials

        else:
            for i in range(len(slice_infos)):
                successes, trials = test_rf_feats(slice_infos, i)
                total_successes += successes
                total_trials += trials

        print("Rate: {}/{} = {:.2f}".format(total_successes, total_trials,
              float(total_successes)/float(total_trials)*100))

        dt = time() - t0
        print("Took %.2f seconds." % dt)

    elif classify == 2: # check classification of all patches for one slice
        test_rf_feats_fullspec(slice_infos, fullspec_i)
        pass

    elif classify == 3: # fully reconstruct for one slice
        rf_reconstruct(slice_infos, fullspec_i)

    elif classify == 4: # show the reconstruct for one slice
        with open(recons, 'rb') as f:
            recons_im = dill.load(f)

        real_lb = slice_infos[fullspec_i].slice_lb

#        for threshval in range(0, 101):
#            threshval = threshval / 100.0
#            recons_th = threshold(recons_im, threshval, True)
#            print("%.2f" % threshval, "%.2f" % compare_im(recons_th, real_lb))

        recons_th = threshold(recons_im, 0.45, True)
        compare_im(recons_th, real_lb, True)

def plot_save_comparisons():
    with open(pickle, 'rb') as f:
        slice_infos = dill.load(f)

    threshv = 0.50

    # go through each case
    for i in range(0, 35):
        with open("recons_ctv/recons_12_" + str(i) + ".pkl", 'rb') as f:
            recons_im = dill.load(f)
            recons_th = threshold(recons_im, threshv, True)
            recons_tv = threshold(recons_im, threshv)
        real_lb = slice_infos[i].slice_lb
        if ct_cap_max:
            slice_im = np.clip(slice_infos[i].slice_im, ct_cap_min, ct_cap_max)
        # plot each
        plt.figure(figsize=(8,12))
        #plt.subplot(4, 1, 1)
        plt.suptitle("Prostate Reconstructions with HOG at Threshold {}, Case {}".format(threshv,i))
        plt.axis('off')
        plt.subplot(3, 2, 1)
        plt.axis('off')
        plt.imshow(slice_im, cmap=plt.cm.gray)
        plt.subplot(3, 2, 2)
        plt.axis('off')

        plt.imshow(slice_im, cmap=plt.cm.gray)
        plt.subplot(3, 2, 3)
        plt.axis('off')

        plt.imshow(recons_tv)
        plt.subplot(3, 2, 4)
        plt.axis('off')

        plt.imshow(real_lb)
        plt.subplot(3, 2, 5)
        plt.axis('off')

        plt.imshow(mask_out(slice_im, recons_th), cmap=plt.cm.gray)
        plt.subplot(3, 2, 6)
        plt.axis('off')
        plt.imshow(mask_out(slice_im, real_lb), cmap=plt.cm.gray)
        plt.savefig('compares_ctv/case_' + str(i) + '.png')

def bigtest():
    # really long test to generate SliceInfos at all radiuses

    images_fn, labels_fn = get_filenames(path, im_name, lb_name)

    # go through all candidate psizes
    for p in psizes:
        global psize
        psize = p
        t0 = time()
        slice_infos = create_sliceinfos_w(images_fn, labels_fn)
        dt = time() - t0
        print("Finished in %.2f seconds." % dt)

        # save the slice info
        with open("slice_infos_" + str(psize) + ".pkl", 'wb') as f:
            dill.dump(slice_infos, f)

def bigtest2():

    for radius in [12]: # go through each SliceInfo
        global fullspec_i
        global recons
        with open("slice_infos_" + str(radius) + ".pkl", 'rb') as f:
            slice_infos = dill.load(f)
        for x in range(0, 35):
            fullspec_i = x
            recons = "recons_" + str(radius) + "_" + str(fullspec_i) + ".pkl"
            rf_reconstruct(slice_infos, fullspec_i)

def get_all_similarities(repath):
    similarities = {}
    for fn in sorted(os.listdir(repath)):
        sep = fn.split('.')[0].split('_')
        radius = int(sep[1])
        if similarities.get(radius) == None:
            similarities[radius] = []
        with open(repath + fn, 'rb') as f:
            similarities[radius].append(dill.load(f))

    #plt.imshow(similarities[10][0])

    rad_sims = []

    # generate similarity index for each image
    for radius in sorted(similarities.keys()):

        print("Radius: ", radius)

        with open("slice_infos_" + str(radius) + ".pkl", 'rb') as f:
            slice_infos = dill.load(f)

        slice_sims = []

        for index, recons_im in enumerate(similarities[radius]):
            # load the real label
            real_lb = slice_infos[index].slice_lb

            print("Slice: ", index)

            thresh_sims = []
            # go through all possible thresholds
            for threshval in range(0, 101):
                threshval = threshval / 100.0

                # mask the generated recons_im to a specific threshval
                recons_th = threshold(recons_im, threshval, True)

                # Compare them
                compare_val = compare_im(recons_th, real_lb)
                thresh_sims.append(compare_val)

            slice_sims.append(thresh_sims)

        rad_sims.append(slice_sims)

    # pull the means
    emp = []
    for i in range(len(rad_sims)):
        emp.append(np.array(rad_sims)[i].mean(axis=0))

    with open("similarities.pkl", 'wb') as f:
        dill.dump(rad_sims, f)

    np.savetxt("emp.csv", np.array(emp), delimiter=",")

def bulk_rename(repath):
    for fn in sorted(os.listdir(repath)):
        if len(fn) == 15:
            os.rename(repath + "/" + fn, repath + "/" + fn[:10] + '0' + fn[10:])

if __name__ == "__main__": # only run if it's the main module
    #get_all_similarities("recons_bladder/")
    #bigtest2()
    #run()
    #plot_save_comparisons()
    #bigtest()
    pass


# -*- coding: utf-8 -*-

import datetime
import json
import logging

from django.conf import settings
from django.db import models
from django.db.transaction import atomic
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _
from kubernetes import client as k8s_client

from bkmonitor.utils.consul import BKConsul
from metadata import config
from metadata.utils import consul_tools

from .cluster import BCSClusterInfo
from .replace import ReplaceConfig
from .utils import ensure_data_id_resource, is_equal_config

logger = logging.getLogger("metadata")

class BCSResource(models.Model):
    """kubernetes资源描述"""

    # 资源信息配置
    # 资源组
    GROUP: str = "monitoring.coreos.com"
    # 版本信息
    VERSION: str = "v1"
    # 资源复数名
    PLURAL: str = None
    PLURALS: str = None
    # 使用用途
    USAGE: str = None

    # 资源集群信息
    # 集群ID，此处和cluster.ClusterInfo.
    cluster_id = models.CharField("集群ID", max_length=128)
    namespace = models.CharField("归属命名空间名", max_length=512)
    name = models.CharField("资源名", max_length=128)

    # 数据元信息
    # 此处的元信息记录的是data_id，因为实际的数据上报后可以继续分为多个结果表(result_table)，
    # 而此处记录为data_id话就可以预留后续拆分结果表的能力
    bk_data_id = models.BigIntegerField("数据源ID", db_index=True)
    is_custom_resource = models.BooleanField("是否自定义资源", default=True)
    # 用于渲染配置时使用
    is_common_data_id = models.BooleanField("是否使用公共data_id", default=True)

    record_create_time = models.DateTimeField("资源记录创建时间")
    resource_create_time = models.DateTimeField("资源在集群中创建时间")

    class Meta:
        abstract = True
        # 对于同一个资源，只可以分配一次资源
        unique_together = ("cluster_id", "namespace", "name")

    @staticmethod
    def make_bcs_client(cluster_id: int) -> k8s_client.ApiClient:
        """
        返回一个bcs操作句柄
        :param cluster_id: 集群ID
        :return:
        """
        try:
            cluster_info = BCSClusterInfo.objects.get(cluster_id=cluster_id)
        except BCSClusterInfo.DoesNotExist as err:
            logger.exception("failed to get bcs client for cluster->[%s] as cluster info is not exists.", cluster_id)
            raise ValueError(_("集群[%s]不存在") % cluster_id) from err

        return cluster_info.api_client

    def should_refresh_own_dataid(self):
        """
        判断该resource是否应该向k8s刷新只属于自己的dataid resource
        """
        # 该resource使用的非公共dataid，则需要刷新
        if not self.is_common_data_id:
            return True
        # 该resource存在配置，则需要刷新
        if ReplaceConfig.objects.filter(
            is_common=False,
            custom_level=ReplaceConfig.CUSTOM_LEVELS_RESOURCE,
            cluster_id=self.cluster_id,
            resource_name=self.name,
            resource_namespace=self.namespace,
            resource_type=self.PLURAL,
        ).exists():
            return True
        return False

    @classmethod
    def refresh_custom_resource(cls, cluster_id):
        """
        刷新自定义资源信息，追加部署的资源，更新未同步的资源
        :param cluster_id: 集群ID
        :return: True | False
        """
        api_client = cls.make_bcs_client(cluster_id=cluster_id)
        custom_client = k8s_client.CustomObjectsApi(api_client)

        # 1. 获取所有命名空间下的本资源信息
        resource_list = custom_client.list_cluster_custom_object(
            group=config.BCS_RESOURCE_GROUP_NAME,
            version=config.BCS_RESOURCE_VERSION,
            plural=config.BCS_RESOURCE_DATA_ID_RESOURCE_PLURAL,
        )
        logger.info("cluster->[%s] got resource->[%s] total->[%s]", cluster_id, cls.PLURAL, len(resource_list["items"]))

        resource_items = {}
        # 遍历所有的资源信息生成字典
        for resource in resource_list["items"]:
            name = resource["metadata"]["name"]
            resource_items[name] = resource

        # 2.遍历所有的resource,对符合条件的数据进行更新操作,只操作已经更新为非公共dataid的resource
        for item in cls.objects.filter(cluster_id=cluster_id):
            # 判断是否需要刷新独立的dataid resource
            if not item.should_refresh_own_dataid():
                continue
            # 检查k8s集群里是否已经存在对应resource
            if item.config_name not in resource_items.keys():
                # 如果k8s_resource不存在，则增加
                ensure_data_id_resource(api_client=api_client, resource_name=item.config_name, config_data=item.config)
                logger.info("cluster->[%s] add new resource->[%s]", cluster_id, item.config)
            else:
                # 否则检查信息是否一致，不一致则更新
                if not is_equal_config(item.config, resource_items[item.config_name]):
                    ensure_data_id_resource(
                        api_client=api_client, resource_name=item.config_name, config_data=item.config
                    )
                    logger.info("cluster->[%s] update resource->[%s]", cluster_id, item.config)

    @classmethod
    @atomic(config.DATABASE_CONNECTION_NAME)
    def refresh_resource(cls, cluster_id: int, common_data_id: int) -> bool:
        """
        刷新集群资源信息，追加未发现的资源,删除已不存在的资源
        :param cluster_id: 集群ID
        :param common_data_id: 公共data_id，所有的资源都会注册到该data_id下
        :return: True | False
        """
        api_client = cls.make_bcs_client(cluster_id=cluster_id)
        custom_client = k8s_client.CustomObjectsApi(api_client)

        # 1. 获取所有命名空间下的本资源信息
        resource_list = custom_client.list_cluster_custom_object(
            group=cls.GROUP, version=cls.VERSION, plural=cls.PLURALS
        )
        logger.info("cluster->[%s] got resource->[%s] total->[%s]", cluster_id, cls.PLURAL, len(resource_list["items"]))

        resource_name_list = []
        # 遍历所有的资源信息
        bulk_create_data = []
        for resource in resource_list["items"]:
            namespace = resource["metadata"]["namespace"]
            name = resource["metadata"]["name"]
            resource_name_list.append("{}_{}".format(namespace, name))

            # 2. 判断是否已经注册
            if cls.objects.filter(
                cluster_id=cluster_id,
                namespace=namespace,
                name=name,
            ).exists():
                # 如果已经存在，继续下一个
                logger.info(
                    "cluster->[%s] resource->[%s] under namespace->[%s] is already exists, nothing will do.",
                    cluster_id,
                    cls.PLURAL,
                    resource["metadata"]["namespace"],
                )
                continue

            # 组装资源记录关系信息，用于批量创建
            bulk_create_data.append(
                cls(
                    cluster_id=cluster_id,
                    namespace=namespace,
                    name=name,
                    bk_data_id=common_data_id,
                    # 由于是公共data_id批量创建，都是公共的data_id
                    is_common_data_id=True,
                    # 由于是拉取集群获得的信息，所以都是自定义资源
                    is_custom_resource=True,
                    record_create_time=datetime.datetime.now(),
                    resource_create_time=datetime.datetime.now(),
                )
            )
            logger.info(
                "cluster->[%s] now create resource->[%s] name->[%s] under namespace->[%s] with data_id->[%s]"
                " success.",
                cluster_id,
                cls.PLURAL,
                name,
                namespace,
                common_data_id,
            )

        # 批量创建
        cls.objects.bulk_create(bulk_create_data)

        # 删除已经不存在的resource映射
        resource_name_set = set(resource_name_list)
        for item in cls.objects.filter(cluster_id=cluster_id):
            key = f"{item.namespace}_{item.name}"
            if key not in resource_name_set:
                item.delete()
                logger.info(
                    "cluster->[%s] delete monitor info->[%s] name->[%s] namespace->[%s] update success.",
                    cluster_id,
                    cls.PLURAL,
                    item.name,
                    item.namespace,
                )

        logger.info("cluster->[%s] all resource->[%s] update success.", cluster_id, cls.PLURAL)
        return True

    @cached_property
    def bcs_client(self) -> k8s_client.ApiClient:
        return self.make_bcs_client(self.cluster_id)

    @cached_property
    def bk_env_label(self) -> str:
        try:
            cluster_info = BCSClusterInfo.objects.get(cluster_id=self.cluster_id)
        except BCSClusterInfo.DoesNotExist as err:
            logger.exception("cluster->[%s] is not exists, err: %s", self.cluster_id, err)
            raise ValueError(_("集群[{}]不存在").format(self.cluster_id))
        return cluster_info.bk_env or settings.BCS_CLUSTER_BK_ENV_LABEL

    @cached_property
    def config_name(self) -> str:

        prefix = "common" if self.is_common_data_id else "custom"
        end = "custom" if self.is_custom_resource else "system"

        if self.bk_env_label:
            return f"{self.bk_env_label}-{prefix}-{self.USAGE}-{self.name}-{end}"
        return f"{prefix}-{self.USAGE}-{self.name}-{end}"

    @cached_property
    def config(self) -> dict:
        # 获取全局replace配置
        replace_config = ReplaceConfig.get_common_replace_config()
        # 获取集群层级replace配置
        cluster_replace_config = ReplaceConfig.get_cluster_replace_config(cluster_id=self.cluster_id)
        # 获取resource层级replace配置
        custom_replace_config = ReplaceConfig.get_resource_replace_config(
            cluster_id=self.cluster_id,
            resource_name=self.name,
            resource_namespace=self.namespace,
            resource_type=self.PLURAL,
        )

        # 将replace配置逐层覆盖
        replace_config[ReplaceConfig.REPLACE_TYPES_METRIC].update(
            cluster_replace_config[ReplaceConfig.REPLACE_TYPES_METRIC]
        )
        replace_config[ReplaceConfig.REPLACE_TYPES_DIMENSION].update(
            cluster_replace_config[ReplaceConfig.REPLACE_TYPES_DIMENSION]
        )
        replace_config[ReplaceConfig.REPLACE_TYPES_METRIC].update(
            custom_replace_config[ReplaceConfig.REPLACE_TYPES_METRIC]
        )
        replace_config[ReplaceConfig.REPLACE_TYPES_DIMENSION].update(
            custom_replace_config[ReplaceConfig.REPLACE_TYPES_DIMENSION]
        )

        cluster = BCSClusterInfo.objects.get(cluster_id=self.cluster_id)
        labels = {"usage": self.USAGE, "isCommon": "false", "isSystem": "false"}

        result = {
            "apiVersion": f"{config.BCS_RESOURCE_GROUP_NAME}/{config.BCS_RESOURCE_VERSION}",
            "kind": f"{config.BCS_RESOURCE_DATA_ID_RESOURCE_KIND}",
            "metadata": {
                "name": cluster.compose_dataid_resource_name(self.config_name),
                "labels": cluster.compose_dataid_resource_label(labels),
            },
            "spec": {
                "dataID": self.bk_data_id,
                "labels": {"bcs_cluster_id": self.cluster_id, "bk_biz_id": str(cluster.bk_biz_id)},
                "metricReplace": replace_config[ReplaceConfig.REPLACE_TYPES_METRIC],
                "dimensionReplace": replace_config[ReplaceConfig.REPLACE_TYPES_DIMENSION],
            },
        }

        # 追加一个指定的资源对应描述
        result["spec"]["monitorResource"] = {"namespace": self.namespace, "kind": self.PLURAL, "name": self.name}

        return result

    def change_data_id(self, data_id: int):
        """
        将resource使用的data_id改为新的data_id
        :param data_id: 新的data_id
        :param is_common_data_id: 更改的新data_id是否公共data_id，默认修改后，就不是公共data_id
        :return:
        """

        # 0.检查cluster的公共dataid信息，判断是否与传入的dataid发生重复了,阻挡重复的情况
        cluster = BCSClusterInfo.objects.get(cluster_id=self.cluster_id)
        for usage, register_info in cluster.DATASOURCE_REGISTER_INFO.items():
            common_data_id = getattr(cluster, register_info["datasource_name"])
            if common_data_id == data_id:
                logger.error(
                    "input change dataid->[%s] conflict with common dataid->[%s] in cluster->[%s]",
                    data_id,
                    usage,
                    self.cluster_id,
                )
                raise ValueError(_("不允许传入公共dataid"))

        # 1. 修改当前资源的data_id
        self.bk_data_id = data_id

        # dataid发生修改，说明一定是改为非公共dataid，则对应调整标志位
        self.is_common_data_id = False

        self.save()
        logger.info(
            "cluster->[%s] namespace->[%s] resource->[%s] name->[%s] change to data_id->[%s] is_common->[%s]",
            self.cluster_id,
            self.namespace,
            self.PLURAL,
            self.name,
            self.bk_data_id,
            self.is_common_data_id,
        )

        # 2. 更新data_id资源
        ensure_data_id_resource(api_client=self.bcs_client, resource_name=self.config_name, config_data=self.config)

        return

    def make_data_id_resource_config(self):
        """
        渲染生成data_id配置文件
        :return:
        """

    def to_json(self):
        return {
            "cluster_id": self.cluster_id,
            "namespace": self.namespace,
            "name": self.name,
            "bk_data_id": self.bk_data_id,
            "is_custom_resource": self.is_custom_resource,
            "is_common_data_id": self.is_common_data_id,
            "resource_type": self.PLURAL,
            "resource_usage": self.USAGE,
        }

    def refresh_info_to_consul(self):
        """
        刷新当前资源dataid到consul
        仅添加当前dataid到对应路径，不会对其他dataid做修改，或者删除
        :return:
        """
        try:
            if not BCSClusterInfo.objects.filter(cluster_id=self.cluster_id).exists():
                logger.error("filter data_id:{}, cluster_id:{} failed".format(self.bk_data_id, self.cluster_id))
                return
            cluster_infos = BCSClusterInfo.objects.filter(cluster_id=self.cluster_id)
            path = "{}/project_id/{}/cluster_id/{}".format(
                config.CONSUL_PATH, cluster_infos[0].project_id, self.cluster_id
            )
            hash_consul = consul_tools.HashConsul()
            # 试图获取当前project_id, cluster_id下的data_id_list
            _, val = hash_consul.get(path)
            vals = []
            if val is not None:
                vals = json.loads(val.get("Value", "[]"))
            # 若当前data_id_list中没有这个data_id，则添加，否则直接返回
            append_data_id_list = [
                data_id for data_id in [cluster_infos[0].K8sMetricDataID, self.bk_data_id] if data_id not in vals
            ]
            if len(append_data_id_list) == 0:
                return
            vals.extend(append_data_id_list)
            hash_consul.put(path, vals)

        except Exception as e:
            logger.error("loads key:{}, value:{} failed:{}".format(path, val, e))

    @classmethod
    def refresh_all_to_consul(cls):
        """
        刷新资源与k8s集群对应的所有时序data_id到consul
        :return:
        """
        # 所有的时序dataid = k8scluster的默认K8sMetricDataID + resource(PodMonitorInfo,ServiceMonitorInfo) 的data_id_list
        path_template = config.CONSUL_PATH + "/project_id/{}/cluster_id/{}"
        info_dict = {}
        cluster_dict = {}
        # 获取所有的k8s_cluster, 构造一个字典cluster_dict {"cluster_id": "project_id"}
        # 同时初始化 构造出一个字典info_dict {"config.CONSUL_PATH/project_id/{}/cluster_id/{}": "[data_id]"}
        bcs_clusters = BCSClusterInfo.objects.all()
        for cluster in bcs_clusters:
            info_key = path_template.format(cluster.project_id, cluster.cluster_id)
            info_dict[info_key] = {cluster.K8sMetricDataID}
            cluster_dict[cluster.cluster_id] = cluster.project_id
        # 获取所有resource
        pod_monitor_infos = PodMonitorInfo.objects.all()
        service_monitor_infos = ServiceMonitorInfo.objects.all()
        try:
            # 遍历并构造出一个字典{"{}/project_id/{}/cluster_id/{}": "[data_id]"}
            for resources in [pod_monitor_infos, service_monitor_infos]:
                for resource in resources:
                    dict_key = path_template.format(cluster_dict[resource.cluster_id], resource.cluster_id)
                    info_dict.get(dict_key, set()).add(resource.bk_data_id)

            # 构造完成之后覆盖consul中的内容
            hash_consul = consul_tools.HashConsul()
            for path, vals in info_dict.items():
                hash_consul.put(path, vals)
        except Exception as e:
            logger.error("refresh all info into consul failed:{}".format(e))

    @classmethod
    def clean_all_bcs_info(cls):
        """
        清除consul上所有资源对应的所有data_id信息
        :return:
        """
        # 直接删除所有的 {}/project_id 下的所有kv
        hash_consul = consul_tools.HasrhConsul()
        root_path = config.CONSUL_PATH + "/project_id"
        bk_consul = BKConsul(
            host=hash_consul.host, port=hash_consul.port, scheme=hash_consul.scheme, verify=hash_consul.verify
        )
        bk_consul.kv.delete(root_path, recurse=True)

class PodMonitorInfo(BCSResource):
    """kubernetes中的pod Monitor信息"""

    PLURAL = "PodMonitor"
    PLURALS = "podmonitors"
    USAGE = "metric"

class ServiceMonitorInfo(BCSResource):
    """kubernetes中的service Monitor信息"""

    PLURAL = "ServiceMonitor"
    PLURALS = "servicemonitors"
    USAGE = "metric"

class LogCollectorInfo(BCSResource):
    """kubernetes中的log Collector信息"""

    GROUP = "bk.tencent.com"
    VERSION = "v1alpha1"
    PLURAL = "BkLogConfig"
    PLURALS = "bklogconfigs"
    USAGE = "log"

    @classmethod
    @atomic(config.DATABASE_CONNECTION_NAME)
    def refresh_resource(cls, cluster_id: int, common_data_id: int) -> bool:
        """
        刷新集群资源信息，追加未发现的资源,删除已不存在的资源
        :param cluster_id: 集群ID
        :param common_data_id: 公共data_id，兼容参数，日志采集场景下不允许公共的dataid
        :return: True | False
        """
        api_client = cls.make_bcs_client(cluster_id=cluster_id)
        custom_client = k8s_client.CustomObjectsApi(api_client)

        # 1. 获取所有命名空间下的本资源信息
        resource_list = custom_client.list_cluster_custom_object(
            group=cls.GROUP, version=cls.VERSION, plural=cls.PLURALS
        )
        logger.info("cluster->[%s] got resource->[%s] total->[%s]", cluster_id, cls.PLURAL, len(resource_list["items"]))

        resource_name = []
        # 遍历所有的资源信息
        for resource in resource_list["items"]:
            namespace = resource["metadata"]["namespace"]
            name = resource["metadata"]["name"]
            resource_name.append("{}_{}".format(namespace, name))
            data_id = resource["spec"]["data_id"]

            # 2. 判断是否已经注册
            if cls.objects.filter(
                cluster_id=cluster_id,
                namespace=namespace,
                name=name,
            ).exists():
                # 如果已经存在，继续下一个
                logger.info(
                    "cluster->[%s] resource->[%s] under namespace->[%s] is already exists, nothing will do.",
                    cluster_id,
                    cls.PLURAL,
                    resource["metadata"]["namespace"],
                )
                continue

            # 3. 创建新的资源记录关系信息
            cls.objects.create(
                cluster_id=cluster_id,
                namespace=namespace,
                name=name,
                bk_data_id=data_id,
                # 由于是公共data_id批量创建，都是公共的data_id
                is_common_data_id=False,
                # 由于是拉取集群获得的信息，所以都是自定义资源
                is_custom_resource=True,
                record_create_time=datetime.datetime.now(),
                resource_create_time=datetime.datetime.now(),
            )
            logger.info(
                "cluster->[%s] now create resource->[%s] name->[%s] under namespace->[%s] with data_id->[%s]"
                " success.",
                cluster_id,
                cls.PLURAL,
                name,
                namespace,
                data_id,
            )

        # 删除已经不存在的resource映射
        for item in cls.objects.filter(cluster_id=cluster_id):
            key = "{}_{}".format(item.namespace, item.name)
            if key not in resource_name:
                item.delete()
                logger.info(
                    "cluster->[%s] delete log collector info->[%s] name->[%s] namespace->[%s] update success.",
                    cluster_id,
                    cls.PLURAL,
                    item.name,
                    item.namespace,
                )

        logger.info("cluster->[%s] log resource->[%s] update success.", cluster_id, cls.PLURAL)
        return True


import json
from django.core.exceptions import ObjectDoesNotExist
from django.core.signing import Signer
from django.views.decorators.csrf import csrf_exempt
from django.views.generic.list import ListView
from django.views.generic.detail import DetailView
from django.views.generic.base import TemplateView

from workshop.forms import CustomerForm
from .models import Service, Basket, BasketItem, CouponCode, Customer, Order
from django.http import JsonResponse, HttpResponseRedirect, HttpResponse
from django.shortcuts import redirect, render, get_list_or_404
# from .forms import CustomerForm
from django.urls import reverse
from django.db.models import Q
from django.template.loader import render_to_string

class ServiceListView(ListView):
    template_name = 'workshop/serviceList.html'
    paginate_by = 32

    def get_queryset(self):
        slug = self.kwargs.get("category")
        if slug is not None:
            queryset = get_list_or_404(Service.objects.filter(category__slug=slug))
        else:
            queryset = Service.objects.all()
        return queryset

class ServiceDetailView(DetailView):
    template_name = 'workshop/serviceDetail.html'
    model = Service

def basket_adding(request):
    return_dict = dict()
    session_key = request.session.session_key
    data = request.POST
    service = Service.objects.get(id=data.get("product_id"))
    new_basket, created = Basket.objects.get_or_create(session_key=session_key)
    basket_item, created = BasketItem.objects.get_or_create(service=service)
    new_basket.services.add(basket_item)
    new_basket.count_total_price()
    product_total = new_basket.services.count()
    return_dict["service_total"] = product_total
    return JsonResponse(return_dict)

def remove_from_basket(request):
    return_dict = dict()
    session_key = request.session.session_key
    data = request.POST
    if data.get("is_delete") == 'true':
        basket = Basket.objects.get(session_key=session_key)
        basket.products.get(id=data.get("service_id")).delete()
        return_dict["is_deleted"] = True
        return_dict['service_total'] = basket.products.count()
        if basket.products.count() < 1 or basket.total <= 3.50:
            basket.delete()
            return_dict['empty_basket'] = render_to_string('workshop/emptyBasket.html')
        else:
            basket.count_total_price()
        return_dict['basket_total'] = basket.total
    return JsonResponse(return_dict)

class CartPageView(TemplateView):
    template_name = 'workshop/cart.html'
    model = Service

def discount(request):
    session_key = request.session.session_key
    if request.method == 'POST':
        code = request.POST.get('cupone_code')
        basket = Basket.objects.get(session_key=session_key)
        try:
            coupon_code = CouponCode.objects.get(code=code)
            if basket.coupon_code:
                return JsonResponse({'status': 'already'})
            elif coupon_code.code == 'HEARTNECKLACE':
                basket.coupon_code = coupon_code
                product = Service.objects.get(name__iexact='Heart necklace')
                basket_item = BasketItem.objects.create(material=None, se=product, size=None, amount=1)
                basket.products.add(basket_item)
                basket.save()
                return JsonResponse({'status': 'successfully', 'discount': basket.coupon_code.discount})
            else:
                basket.coupon_code = coupon_code
                basket.discount()
                return JsonResponse({'status': 'successfully', 'basket_total': basket.total,
                                     'discount': render_to_string('shop/sale.html', {
                                         'discount': basket.coupon_code.discount})})
        except:
            return JsonResponse({'status': 'invalid'})
    return HttpResponse('/')

def checkout_step_one(request, basket_url):
    try:
        session_key = Signer().unsign(basket_url)
        basket = Basket.objects.get(session_key=session_key)
    except:
        return HttpResponseRedirect(reverse('workshop:service'))
    template_name = 'workshop/checkout.html'
    initial_customer = Customer.objects.filter(session_key=session_key).first()
    if request.method == 'POST':
        form = CustomerForm(request.POST or None)
        if form.is_valid():
            customer, created = Customer.objects.get_or_create(session_key=session_key, defaults={**form.cleaned_data})
            if created:
                customer.session_key = session_key
                customer.save()
            order, created = Order.objects.get_or_create(basket=basket)
            order.customer = customer
            order.save()
            return redirect(reverse('shop:checkout_step_two', kwargs={'basket_url': basket_url}))
    else:
        form = CustomerForm(initial={'email': initial_customer.email if initial_customer else '',
                                     'first_name': initial_customer.first_name if initial_customer else '',
                                     'last_name': initial_customer.last_name if initial_customer else '',
                                     'street': initial_customer.street if initial_customer else '',
                                     'country': initial_customer.country if initial_customer else '',
                                     'city': initial_customer.city if initial_customer else '',
                                     'post_code': initial_customer.post_code if initial_customer else ''})
    return render(request, template_name, {'form': form})


#!/usr/bin/python
import argparse

from norec4dna.OnlineEncoder import OnlineEncoder
from norec4dna.rules.FastDNARules import FastDNARules
from norec4dna.ErrorCorrection import nocode, get_error_correction_encode
from norec4dna.distributions.OnlineDistribution import OnlineDistribution

class demo_online_encode:
    @staticmethod
    # def encode(file, error_correction=nocode, asdna=True):
    def encode(file, asdna=True,  error_correction=nocode, insert_header=False, save_number_of_chunks_in_packet=False,
               save_as_fasta=True, save_as_zip=True, overhead=0.40, epsilon=0.068, quality=7, upper_bound=1.0,
               checksum_len_str=None):
        dist = OnlineDistribution(epsilon)
        number_of_chunks = dist.get_size()
        dna_rules = FastDNARules()
        if asdna:
            rules = dna_rules
        else:
            rules = None
        encoder = OnlineEncoder(
            file, number_of_chunks, dist, epsilon, quality, error_correction=error_correction, quality_len_format="B",
            insert_header=insert_header, check_block_number_len_format="H", number_of_chunks_len_format="H", rules=rules,
            save_number_of_chunks_in_packet=save_number_of_chunks_in_packet, drop_upper_bound=upper_bound,
            checksum_len_str=checksum_len_str)  # , pseudo_decoder=pseudo)
        encoder.set_overhead_limit(overhead)
        #encoder.encode_file(split_to_multiple_files=True, save_as_dna=asdna)
        encoder.encode_to_packets()
        if save_as_fasta:
            encoder.save_packets_fasta(file_ending="_Online", seed_is_filename=True)
        elif save_as_zip:
            encoder.save_packets_zip(save_as_dna=True, file_ending="_Online", seed_is_filename=True)
        else:
            encoder.save_packets(True, save_as_dna=True, seed_is_filename=True, clear_output=True)

        encoder.save_packets(split_to_multiple_files=True, save_as_dna=True)
        print("Number of Chunks=%s" % encoder.number_of_chunks)
        return encoder

if __name__ == "__main__":
    # try:
    parser = argparse.ArgumentParser()
    #parser.add_argument("--chunk_size", metavar="chunk_size", required=False, type=int, default=DEFAULT_CHUNK_SIZE,
    #                    help="size of chunks to split the file into")
    parser.add_argument("filename", metavar="file", type=str, help="the file to Encode")
    parser.add_argument("--error_correction", metavar="error_correction", required=False, type=str, default="nocode",
                        help="Error Correction Method to use; possible values: \
                        nocode, crc, reedsolomon, dna_reedsolomon (default=nocode)")
    parser.add_argument("--repair_symbols", metavar="repair_symbols", required=False, type=int, default=2,
                        help="number of repair symbols for ReedSolomon (default=2)")
    parser.add_argument("--insert_header", action="store_true", required=False, default=False)
    parser.add_argument("--save_number_of_chunks", metavar="save_number_of_chunks", required=False, type=bool,
                        default=False)
    parser.add_argument("--save_as_fasta", action="store_true", required=False)
    parser.add_argument("--save_as_zip", action="store_true", required=False)
    parser.add_argument("--header_crc_str", metavar="header_crc_str", required=False, type=str, default="")
    parser.add_argument("--drop_upper_bound", metavar="drop_upper_bound", required=False, type=float, default=0.5,
                        help="upper bound for calculated error probability of packet before dropping")
    parser.add_argument("--overhead", metavar="overhead", required=False, type=float, default=0.40,
                        help="desired overhead of packets")
    parser.add_argument("--epsilon", metavar="epsilon", required=False, type=float, default=0.068,
                        help="epsilon value of the coding")
    parser.add_argument("--quality", metavar="quality", required=False, type=int, default=7,
                        help="quality value of the coding")
    args = parser.parse_args()
    _file = args.filename
    #_chunk_size = args.chunk_size
    _no_repair_symbols = args.repair_symbols
    _insert_header = args.insert_header
    _save_number_of_chunks = args.save_number_of_chunks
    _upper_bound = args.drop_upper_bound
    _error_correction = get_error_correction_encode(args.error_correction, _no_repair_symbols)
    _save_as_fasta = args.save_as_fasta
    _save_as_zip = args.save_as_zip
    _header_crc_str = args.header_crc_str
    _overhead = args.overhead
    _epsilon = args.epsilon
    _quality = args.quality
    # for _file in input_files:
    print("File to encode: " + str(_file))
    demo = demo_online_encode()
    encoder_instance = demo.encode(_file, error_correction=_error_correction,
                                   insert_header=_insert_header, save_number_of_chunks_in_packet=_save_number_of_chunks,
                                   save_as_fasta=_save_as_fasta, save_as_zip=_save_as_zip, overhead=_overhead,
                                   epsilon=_epsilon, quality=_quality, upper_bound=_upper_bound,
                                   checksum_len_str=_header_crc_str)
    conf = {'error_correction': args.error_correction, 'repair_symbols': _no_repair_symbols, 'asdna': True,
            'number_of_splits': 0, 'quality': encoder_instance.quality, 'epsilon': encoder_instance.epsilon,
            'find_minimum_mode': False, 'seq_seed': False}
    config_filename = encoder_instance.save_config_file(conf, section_name="Online_" + _file)
    print("Saved config file: %s" % config_filename)
    # input("Press Enter to continue ...")


from django.shortcuts import render
from .models import Productos, Categorias
# Create your views here.

def busqueda(request,id):
    if request.method == 'GET':
        identificador=str(id)
        query1="SELECT * FROM testing.mainapp_productos WHERE idCategoria="+identificador
        productos = Productos.objects.raw(query1)
    categorias = Categorias.objects.all()
    contexto = {
        'productos' : productos,
        'categorias' : categorias
    }

    return render(request,'index.html', contexto)

def home(request):

    if request.method == 'GET':
        query1="SELECT * FROM testing.mainapp_productos"
        productos = Productos.objects.raw(query1)
    else :
        print('que mierda hace aqui')
        busqueda = request.POST['busqueda']
        query1="SELECT * FROM testing.mainapp_productos WHERE nombreProducto LIKE '%%"+busqueda+"%%'"
        productos = Productos.objects.raw(query1)
    categorias = Categorias.objects.all()
    contexto = {
        'productos' : productos,
        'categorias' : categorias
    }
    return render(request,'index.html', contexto)


from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.exceptions import AirflowException

from datetime import datetime, timedelta
import logging
from plugins.redshift import get_redshift_connection

def exec_sql(**context):
    schema = context['params']['schema']
    table = context['params']['table']
    select_sql = context['params']['sql']

    logging.info(schema)
    logging.info(table)
    logging.info(select_sql)

    cur = get_redshift_connection()

    # temp table 생성
    sql = f"""DROP TABLE IF EXISTS {schema}.temp_{table};CREATE TABLE {schema}.temp_{table} AS """
    sql += select_sql
    cur.execute(sql)

    cur.execute(f"""SELECT COUNT(1) FROM {schema}.temp_{table}""")
    count = cur.fetchone()[0]
    if count == 0:
        raise ValueError(f"{schema}.{table} didn't have any record")
    else:
        pass

    # 기존 테이블 삭제 및 temp table을 table로 바꿈
    try:
        sql = f"""DROP TABLE IF EXISTS {schema}.{table};ALTER TABLE {schema}.temp_{table} RENAME to {table};"""
        sql += "COMMIT;"
        logging.info(sql)
        cur.execute(sql)

    except Exception as e:
        cur.execute("ROLLBACK")
        logging.error('Failed to sql. Completed ROLLBACK!')
        raise AirflowException("")

dag = DAG(
    dag_id="build-gender-summary",
    start_date=datetime(2021, 12, 10),
    schedule='@once',
    catchup=False
)

# 집계 쿼리
execsql = PythonOperator(
    task_id='exec_summary_sql',
    python_callable=exec_sql,
    params={
        'schema': 'star1996416',
        'table': 'gender_summary',
        'sql': """
        SELECT gender, count(*) as cnt
        FROM star1996416.name_gender
        GROUP BY gender
        ORDER BY cnt;
        """
    },
    dag=dag
)


#!/usr/bin/env python3

'''
This script takes two <.rtsc> along with the base <.fasta> to calculate the reactivity
scores of each nucelotide of each transcript. If used without a <.scale> file, the script will generate
a <.scale> to be used with this script when calculating any additinonal reactvities from other samples that
are to be compared with the original.
Transcripts which produce a 0 average on the normalization scale will not have their final reactivity calculated,
and these transcripts may be logged to a file (optional). If given a <.scale> file, it will apply it as the normalization scale.
Transcripts not included in the <.scale> will not be calculated and can be logged to a file (optional).
Normalization (2-8%) may be turned off for the reactivity calculation, as can taking the natural log of the reactivity values,
via options.
'''

#Imports
import math
import argparse
import os
import sf3libs.sf3io as sfio

#Functions
def calculate_raw_reactivity(reagent_minus,reagent_plus,nlog_off=False):
    '''Calculates raw reactivity, with or without the natural log'''
    key_set,data_out = set(reagent_plus.keys()).intersection(set(reagent_minus.keys())),{}
    for key in key_set:
        plus_vector = [math.log(value+1,math.e) for value in reagent_plus[key]] if nlog_off == False else reagent_plus[key]
        minus_vector = [math.log(value+1,math.e) for value in reagent_minus[key]] if nlog_off == False else reagent_minus[key]
        sum_plus,sum_minus,length = sum(plus_vector),sum(minus_vector),len(plus_vector)
        if sum_plus != 0 and sum_minus != 0:
            nrm_plus_vector = [float(y)/float(sum_plus)*length for y in plus_vector]
            nrm_minus_vector = [float(z)/float(sum_minus)*length for z in minus_vector]
            plus_minus = [max(0,a-b) for a, b in zip(nrm_plus_vector,nrm_minus_vector)]
            data_out[key] = plus_minus
    return data_out

def generate_normalization_scale(derived_reactivities,transcript_seqs,specificity):
    '''Generates the 2-8% scale to normalize against'''
    data = {}
    for transcript, reactivities in derived_reactivities.items():
        sequence = transcript_seqs[transcript]
        accepted = sorted([reactivities[k] for k in range(1,len(reactivities)) if sequence[k-1] in specificity],reverse=True)
        top = accepted[int(len(accepted)*0.02):int(len(accepted)*0.1)]
        top_average = sum(top)/len(top) if len(top) > 0 else 0
        if top_average > 0:
            data[transcript] = top_average
    return data

def write_norm_scale(scale_dictionary,outfile):
    '''Writes out a normalization scale file'''
    with open(outfile,'w') as g:
        g.write(','.join(['transcript','value'])+'\n')
        for transcript, value in sorted(scale_dictionary.items()):
            g.write(','.join([transcript,str(value)])+'\n')

def read_norm_scale(normalization_file):
    '''Reads in a normalization scale file'''
    info = {}
    with open(normalization_file, 'r') as f:
        for line in f:
            if line.startswith('transcript'):
                continue
            else:
                transcript,value = line.strip().split(',')
                info[transcript] = float(value)
    return info

def calculate_final_reactivity(derived_reactivities,sequences,specificity,threshold,nrm_scale,norm_off=False):
    '''Calculates the final reactivity'''
    data_out,missing_transcripts = {},{}
    for transcript, reactivities in derived_reactivities.items():
        if transcript in nrm_scale:
            normalizer = nrm_scale[transcript] if norm_off == False else 1
            seq = sequences[transcript]
            normalized_values =[str(float('%.3f'%min((reactivities[x]/normalizer), threshold))) if seq[x-1] in specificity else 'NA' for x in range(1,len(reactivities))]+['NA']
            data_out[transcript] = normalized_values
        else:
            missing_transcripts[transcript] = None
    return data_out,missing_transcripts

def main():
    parser = argparse.ArgumentParser(description='Generates a <.react> file from two <.rtsc> files')
    in_files = parser.add_argument_group('Input')
    in_files.add_argument('control',type=str,help='Control <.rtsc> file')
    in_files.add_argument('treatment',type=str,help='Reagent <.rtsc> file')
    in_files.add_argument('fasta',type=str,help='Transcript <.fasta> file')
    settings = parser.add_argument_group('Settings')
    settings.add_argument('-restrict',default = None,metavar='<.txt>',help='Limit analysis to these specific transcripts')
    settings.add_argument('-scale',type=str,default= None,metavar='<.scale>',help='Provide a normalizaiton file for calculation')
    settings.add_argument('-threshold',type=float,default=7.0,help='[default = 7.0] Maximum Reactivity Cap')
    settings.add_argument('-ln_off',action='store_true', help='Do not take the natural log of the stop counts')
    settings.add_argument('-nrm_off',action='store_true',help='Do not perform final 2-8'+u"\uFF05"+' reactivity normalization')
    settings.add_argument('-bases',type=str,default = 'AC',metavar='ACGT',help='[default = AC] Reaction Specificity')
    out_files = parser.add_argument_group('Output')
    out_files.add_argument('-save_fails',action='store_true',help='Log transcripts with zero or missing scales')
    out_files.add_argument('-name',type=str,default = None, help='Specify output file name')
    args = parser.parse_args()

    #Create output name
    base_tag = [x.split(os.sep)[-1].replace('.rtsc','') for x in [args.control,args.treatment]]
    log_tag = ['ln'] if args.ln_off == False else []
    nrm_tag = ['nrm'] if args.ln_off == False else []
    base_name = '_'.join(base_tag+log_tag+nrm_tag)
    out_name = base_name+'.react' if args.name == None else sfio.check_extension(args.name,'.react')

    #Read in data
    control_data,treatment_data = map(sfio.read_rtsc,[args.control,args.treatment])

    #Apply filter if enabled
    if args.restrict:
        covered = sfio.read_restrict(args.restrict)
        control_data = {n:s for n,s in control_data.items() if n in covered}
        treatment_data = {n:s for n,s in treatment_data.items() if n in covered}

    #Calculate Derived Reactivity
    data = calculate_raw_reactivity(control_data,treatment_data,args.ln_off)

    #Read in transcript sequences
    seqs = sfio.read_fasta(args.fasta)

    #Generate and write scale, or read a <.scale> file in
    normalizaiton_scale = generate_normalization_scale(data,seqs,args.bases) if args.scale == None else read_norm_scale(args.scale)
    if args.scale == None:
        write_norm_scale(normalizaiton_scale,out_name.replace('.react','.scale'))

    #Calculate Final Reactivity
    out_reactivity,out_missing = calculate_final_reactivity(data,seqs,args.bases,args.threshold,normalizaiton_scale,args.nrm_off)

    #Write Out
    sfio.write_react(out_reactivity,out_name,sort_flag=True)

    #Write Out Fails
    if args.save_fails:
        sfio.write_keys(out_missing,out_name.replace('.react','_unresolvable_transcripts.txt'))

if __name__ == '__main__':
    main()