#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 23 12:04:30 2020

@author: aggarwal
"""
#%%
""""import essential packages"""

import pandas as pd
import numpy as np
from osgeo import gdal
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import  roc_curve, auc
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from rotation_forest import RotationForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import ExtraTreesClassifier
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
import pickle
#import graphviz
from sklearn import tree
from sklearn.feature_selection import SelectFromModel
from sklearn.utils import shuffle

#%%

"""function to return pixel values"""
def value(ds):
    rows = ds.RasterYSize
    cols = ds.RasterXSize
    band1 = ds.GetRasterBand(1).ReadAsArray()
    data = np.array(band1)
    values = []
    for row in range(0,rows):
        for col in range(0,cols):
            values.append(data[row,col])
    return values

"""function to return latitude and longitude"""
def latlong(ds):
    rows = ds.RasterYSize
    cols = ds.RasterXSize
    xoff,a,b,yoff,d,e = ds.GetGeoTransform()
    lat =[]
    long = []
    def pixel2coord(x,y):
        xp = a*y+b*x+a*0.5+b*0.5+xoff
        yp = d*y+e*x+d*0.5+e*0.5+yoff
        lat.append(xp)
        long.append(yp)
    for row in range(0,rows):
        for col in range(0,cols):
            pixel2coord(row,col)
    return lat, long

#%%

"""load images and extract values for features"""
ASPECT = value(gdal.Open("../extract/aspect/ASPECT.tif"))
BUFFER_FAULT2 = value(gdal.Open("../extract/buffer/distance_2/fault/buffer_fault2.tif"))
BUFFER_RIVER2 = value(gdal.Open("../extract/buffer/distance_2/river/buffer_river2.tif"))
BUFFER_ROAD2 = value(gdal.Open("../extract/buffer/distance_2/road/buffer_road2.tif"))
DEM = value(gdal.Open("../extract/dem/DEM.tif"))
LITHO = value(gdal.Open("../extract/litho/LITHO.tif"))
PLAN_CURVETURE = value(gdal.Open("../extract/plan_curveture/PLAN_CURVETURE.tif"))
PROFILE_CURVETURE = value(gdal.Open("../extract/profile_curveture/PROFILE_CURVETURE.tif"))
SLOPE_LENGTH = value(gdal.Open("../extract/slope_length/SLOPE_LENGTH.tif"))
SLOPEW = value(gdal.Open("../extract/slope/SLOPE.tif"))
SPI= value(gdal.Open("../extract/spi/SPI.tif"))
STI = value(gdal.Open("../extract/sti/STI.tif"))
TWI = value(gdal.Open("../extract/twi/twi_12.tif"))
LAND_COVER= value(gdal.Open("../extract/land_cover/LAND_COVER1.tif"))
LANDSLIDE = value(gdal.Open("../extract/landslide/landslide_2.tif"))

"""calculate latitude, longitude and round to integer"""

lat, long = latlong(gdal.Open("../extract/land_cover/LAND_COVER1.tif"))
lat = [round(x) for x in lat]
long = [round(x) for x in long]

#%%
"""check if total no of data points are equal in all"""
features_len = []
features_len.append(len(ASPECT))
features_len.append(len(BUFFER_FAULT2))
features_len.append(len(BUFFER_RIVER2))
features_len.append(len(BUFFER_ROAD2))
features_len.append(len(DEM))
features_len.append(len(LITHO))
features_len.append(len(PLAN_CURVETURE))
features_len.append(len(PROFILE_CURVETURE))
features_len.append(len(SLOPE_LENGTH))
features_len.append(len(SLOPEW))
features_len.append(len(SPI))
features_len.append(len(STI))
features_len.append(len(TWI))
features_len.append(len(LAND_COVER))
features_len.append(len(LANDSLIDE))
features_len.append(len(lat))
features_len.append(len(long))

print(features_len)

"""if length of features are not same, recheck the data"""

#%%
"""make dataframe from features and create csv"""

d = {'Latitude' :lat, 'Longitude' :long, 'Fault' :BUFFER_FAULT2,
     'River' :BUFFER_RIVER2, 'Road' :BUFFER_ROAD2, 'Lithology' :LITHO, 'Land_Cover' :LAND_COVER,
     'Dem' :DEM, 'Aspect' :ASPECT, 'Plan_Curvature' :PLAN_CURVETURE, 'Profile_Curvature' :PROFILE_CURVETURE,
     'Slope_Length' :SLOPE_LENGTH, 'Slope' :SLOPEW, 'SPI' :SPI, 'STI' :STI, 'TWI' :TWI,
      'Landslide' :LANDSLIDE}

df = pd.DataFrame(d, columns =['Latitude', 'Longitude', 'Fault' , 'River' , 'Road' ,'Lithology', 'Land_Cover',
                               'Dem', 'Aspect', 'Plan_Curvature','Profile_Curvature',
                               'Slope_Length', 'Slope', 'SPI', 'STI', 'TWI','Landslide'] )

df.to_csv('../csv/complete_data.csv',index=False)

#%%
df = pd.read_csv('../csv/complete_data.csv')
"""
Now we need to calculate the number of values outside study region( no data values).
They should be equal for all the features
We can either save them in 2D list for each feature or print them out in the
console

value_counts() function gives the count of each unique entry in a series in
descending order
"""

#printing value_counts in the console
print(df['Aspect'].value_counts())
print(df['Fault'].value_counts())
print(df['River'].value_counts())
print(df['Road'].value_counts())
print(df['Dem'].value_counts())
print(df['Lithology'].value_counts())
print(df['Plan_Curvature'].value_counts())
print(df['Profile_Curvature'].value_counts())
print(df['Slope_Length'].value_counts())
print(df['Slope'].value_counts())
print(df['SPI'].value_counts())
print(df['STI'].value_counts())
print(df['TWI'].value_counts())
print(df['Land_Cover'].value_counts())
print(df['Landslide'].value_counts() )

#saving them in a list
mode_values = []

for x in df:
    mode_values.append(df[x].value_counts())

"""
analyse value_count for all the features,
Since number of no data values is more, first value of
value_counts() function is same for all features, if not check the original
images"""

"""
assign a variable drop_value as one of the no data value,
here value of buffer fault is being used"""

drop_value = 65536

"""save the index of all the pixels with value as drop_value in a list"""

indexNames = df[df['Fault'] == drop_value].index

"""from the original dataframe, drop all the rows of index in indexNames"""
df.drop(indexNames , inplace = True)

#write the dataframe into a csv file
df.to_csv('../csv/row_truncated_data.csv', index = False)

#%%
df = pd.read_csv('../csv/row_truncated_data.csv')
""""multi collinearity check"""

"""vif values between 1-5 have low collinearity
                5-10 , moderate
                >10, high collinearity
                """
df_new = df.iloc[:,2:16]
vif = pd.DataFrame()
vif['vif factor'] = [variance_inflation_factor(df_new.values, i) for i in range(df_new.shape[1])]
vif['features'] = df_new.columns

corr = pd.DataFrame()
corr = df_new.iloc[:,5:]
corr_matrix=corr.corr()
#%%
"""preprocessing - basic layout for on  one hot encoding the data

X = df.iloc[:,2:16]
Y = df.iloc[:,16]

landc = LabelBinarizer().fit_transform(X.LandCover)
litho = LabelBinarizer().fit_transform(X.Lithology)

X = X.drop(['LandCover','Lithology'], axis = 1)
landc = pd.DataFrame(landc)
litho = pd.DataFrame(litho)

X = pd.concat([X,landc,litho],axis=1)
"""
#%%
"""preprocessing 0 - handling profile and plan curvature aspect, all buffer variables"""

i = df['Plan_Curvature']
Plan_Curvature = []
for x in i:
    if(x<0):
        Plan_Curvature.append(2)
    elif(x>0):
        Plan_Curvature.append(1)
    else:
        Plan_Curvature.append(0)

i = df['Profile_Curvature']
Profile_Curvature = []
for x in i:
    if(x<0):
        Profile_Curvature.append(2)
    elif(x>0):
        Profile_Curvature.append(1)
    else:
        Profile_Curvature.append(0)

df = df.drop(['Plan_Curvature','Profile_Curvature'], axis = 1)
Plan_Curvature = pd.Series(Plan_Curvature)
Profile_Curvature = pd.Series(Profile_Curvature)
df = pd.concat([df,Plan_Curvature,Profile_Curvature], axis=1)
df = df.rename(columns={0:'Plan_Curvature',1:'Profile_Curvature'})

i = df['Aspect']
j = []
for x in i:
    if(x<0):
        j.append(0)
    elif(x>=0 and x<22.5):
        j.append(1)
    elif(x>=22.5 and x<67.5):
        j.append(2)
    elif(x>=67.5 and x<112.5):
        j.append(3)
    elif(x>=112.5 and x<157.5):
        j.append(4)
    elif(x>=157.5 and x<202.5):
        j.append(5)
    elif(x>=202.5 and x<247.5):
        j.append(6)
    elif(x>=247.5 and x<292.5):
        j.append(7)
    elif(x>=292.5 and x<337.5):
        j.append(8)
    else:
        j.append(1)
df = df.drop(['Aspect'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Aspect'})

i = df['Fault']
Fault = []
for x in i:
    if(x==50):
        Fault.append(0)
    elif(x==100):
        Fault.append(1)
    elif(x==150):
        Fault.append(2)
    elif(x==200):
        Fault.append(3)
    elif(x==250):
        Fault.append(4)
    elif(x==300):
        Fault.append(5)
df = df.drop(['Fault'],axis=1)
Fault = pd.Series(Fault)
df = pd.concat([df,Fault], axis=1)
df = df.rename(columns={0:'Fault'})

i = df['River']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['River'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'River'})

i = df['Road']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['Road'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Road'})

#moving landslide in the last
landslide = df['Landslide']
df = df.drop(['Landslide'],axis=1)
df = pd.concat([df,landslide], axis=1)

#saving csv file
df.to_csv('../csv/processed0_truncated_data.csv')

#%%
"""preprocessing 1 - handling DEM derivatives, all buffer variables"""
"""Class (as per Jenks natural break)"""

i = df['Plan_Curvature']
Plan_Curvature = []
for x in i:
    if(x<=-2.543456388):
        Plan_Curvature.append(0)
    elif(x>=-2.543456387 and x<=-0.957240183):
        Plan_Curvature.append(1)
    elif(x>=-0.957240182 and x<=-0.092031344):
        Plan_Curvature.append(2)
    elif(x>=-0.092031343 and x<=0.773177495):
        Plan_Curvature.append(3)
    elif(x>=0.773177496 and x<=2.215192226):
        Plan_Curvature.append(4)
    else:
        Plan_Curvature.append(5)

i = df['Profile_Curvature']
Profile_Curvature = []
for x in i:
    if(x<=-3.46856041):
        Profile_Curvature.append(5)
    elif(x>=-3.46856040 and x<=-1.346149886):
        Profile_Curvature.append(4)
    elif(x>=-3.46856040 and x<=-1.346149886):
        Profile_Curvature.append(3)
    elif(x>=-0.133343871 and x<=1.079462141):
        Profile_Curvature.append(2)
    elif(x>=1.079462142 and x<=3.050271913):
        Profile_Curvature.append(1)
    else:
        Profile_Curvature.append(0)

df = df.drop(['Plan_Curvature','Profile_Curvature'], axis = 1)
Plan_Curvature = pd.Series(Plan_Curvature)
Profile_Curvature = pd.Series(Profile_Curvature)
df = pd.concat([df,Plan_Curvature,Profile_Curvature], axis=1)
df = df.rename(columns={0:'Plan_Curvature',1:'Profile_Curvature'})

i = df['Aspect']
j = []
for x in i:
    if(x<0):
        j.append(0)
    elif(x>=0 and x<22.5):
        j.append(1)
    elif(x>=22.5 and x<67.5):
        j.append(2)
    elif(x>=67.5 and x<112.5):
        j.append(3)
    elif(x>=112.5 and x<157.5):
        j.append(4)
    elif(x>=157.5 and x<202.5):
        j.append(5)
    elif(x>=202.5 and x<247.5):
        j.append(6)
    elif(x>=247.5 and x<292.5):
        j.append(7)
    elif(x>=292.5 and x<337.5):
        j.append(8)
    else:
        j.append(1)
df = df.drop(['Aspect'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Aspect'})

i = df['Fault']
Fault = []
for x in i:
    if(x==50):
        Fault.append(0)
    elif(x==100):
        Fault.append(1)
    elif(x==150):
        Fault.append(2)
    elif(x==200):
        Fault.append(3)
    elif(x==250):
        Fault.append(4)
    elif(x==300):
        Fault.append(5)
df = df.drop(['Fault'],axis=1)
Fault = pd.Series(Fault)
df = pd.concat([df,Fault], axis=1)
df = df.rename(columns={0:'Fault'})

i = df['River']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['River'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'River'})

i = df['Road']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['Road'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Road'})

i = df['Dem']
j = []
for x in i:
    if(x<=887):
        j.append(0)
    elif(x>=888 and x<=1437):
        j.append(1)
    elif(x>=1438 and x<=2057):
        j.append(2)
    elif(x>=2058 and x<=2748):
        j.append(3)
    elif(x>=2749 and x<=3457):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['Dem'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Dem'})

i = df['Slope']
Slope = []
for x in i:
    if(x<=10.87785184):
        Slope.append(0)
    elif(x>=10.87785185 and x<=21.16771168):
        Slope.append(1)
    elif(x>=21.16771169 and x<=29.69359556):
        Slope.append(2)
    elif(x>=29.69359557 and x<=37.92548343):
        Slope.append(3)
    elif(x>=37.92548344 and x<=47.33335529):
        Slope.append(4)
    else:
        Slope.append(5)
df = df.drop(['Slope'],axis=1)
Slope = pd.Series(Slope)
df = pd.concat([df,Slope], axis=1)
df = df.rename(columns={0:'Slope'})

i = df['TWI']
j = []
for x in i:
    if(x<=3.375663230):
        j.append(0)
    elif(x>=3.375663231 and x<=5.585415369):
        j.append(1)
    elif(x>=5.585415370 and x<=8.679068363):
        j.append(2)
    elif(x>=8.679068364 and x<=11.44125854):
        j.append(3)
    elif(x>=11.44125855 and x<=13.65101068):
        j.append(4)
    else:
        j.append(5)

df = df.drop(['TWI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'TWI'})

i = df['STI']
j = []
for x in i:
    if(x<=1.129179711):
        j.append(0)
    elif(x>=1.129179712 and x<=4.516718846):
        j.append(1)
    elif(x>=4.516718847 and x<=10.72720726):
        j.append(2)
    elif(x>=10.72720727 and x<=20.88982466):
        j.append(3)
    elif(x>=20.88982467 and x<=40.65046961):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['STI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'STI'})

i = df['SPI']
j = []
for x in i:
    if(x<=7.843282782):
        j.append(0)
    elif(x>=7.843282783 and x<=39.21641391):
        j.append(1)
    elif(x>=39.21641392 and x<=113.7276003):
        j.append(2)
    elif(x>=113.7276004 and x<=282.3581801):
        j.append(3)
    elif(x>=282.3581802 and x<=615.6976984):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['SPI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'SPI'})

i = df['Slope_Length']
j = []
for x in i:
    if(x<=108.0233130):
        j.append(0)
    elif(x>=108.0233131 and x<=324.0699391):
        j.append(1)
    elif(x>=324.0699392 and x<=669.7445408):
        j.append(2)
    elif(x>=669.7445409 and x<=1188.256443):
        j.append(3)
    elif(x>=1188.256444 and x<=2052.442948):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['Slope_Length'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Slope_Length'})

#moving landslide in the last
landslide = df['Landslide']
df = df.drop(['Landslide'],axis=1)
df = pd.concat([df,landslide], axis=1)

#saving csv file
df.to_csv('../csv/processed1_truncated_data.csv')

#%%
"""preprocessing 2 - handling DEM derivatives, all buffer variables"""
"""Class (equal interval)"""

i = df['Plan_Curvature']
Plan_Curvature = []
for x in i:
    if(x<=-13.86327203):
        Plan_Curvature.append(0)
    elif(x>=-13.86327202 and x<=-7.734709422):
        Plan_Curvature.append(1)
    elif(x>=-7.734709421 and x<=-1.606146812):
        Plan_Curvature.append(2)
    elif(x>=-1.606146811 and x<=4.522415797):
        Plan_Curvature.append(3)
    elif(x>=4.522415798 and x<=10.65097841):
        Plan_Curvature.append(4)
    else:
        Plan_Curvature.append(5)

i = df['Profile_Curvature']
Profile_Curvature = []
for x in i:
    if(x<=-12.51407193):
        Profile_Curvature.append(5)
    elif(x>=-12.51407192 and x<=-6.096306772):
        Profile_Curvature.append(4)
    elif(x>=-6.096306771 and x<= 0.321458383):
        Profile_Curvature.append(3)
    elif(x>=0.321458383 and x<=6.739223538):
        Profile_Curvature.append(2)
    elif(x>=6.739223539 and x<=13.15698869):
        Profile_Curvature.append(1)
    else:
        Profile_Curvature.append(0)

df = df.drop(['Plan_Curvature','Profile_Curvature'], axis = 1)
Plan_Curvature = pd.Series(Plan_Curvature)
Profile_Curvature = pd.Series(Profile_Curvature)
df = pd.concat([df,Plan_Curvature,Profile_Curvature], axis=1)
df = df.rename(columns={0:'Plan_Curvature',1:'Profile_Curvature'})

i = df['Aspect']
j = []
for x in i:
    if(x<0):
        j.append(0)
    elif(x>=0 and x<22.5):
        j.append(1)
    elif(x>=22.5 and x<67.5):
        j.append(2)
    elif(x>=67.5 and x<112.5):
        j.append(3)
    elif(x>=112.5 and x<157.5):
        j.append(4)
    elif(x>=157.5 and x<202.5):
        j.append(5)
    elif(x>=202.5 and x<247.5):
        j.append(6)
    elif(x>=247.5 and x<292.5):
        j.append(7)
    elif(x>=292.5 and x<337.5):
        j.append(8)
    else:
        j.append(1)
df = df.drop(['Aspect'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Aspect'})

i = df['Fault']
Fault = []
for x in i:
    if(x==50):
        Fault.append(0)
    elif(x==100):
        Fault.append(1)
    elif(x==150):
        Fault.append(2)
    elif(x==200):
        Fault.append(3)
    elif(x==250):
        Fault.append(4)
    elif(x==300):
        Fault.append(5)
df = df.drop(['Fault'],axis=1)
Fault = pd.Series(Fault)
df = pd.concat([df,Fault], axis=1)
df = df.rename(columns={0:'Fault'})

i = df['River']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['River'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'River'})

i = df['Road']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['Road'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Road'})

i = df['Dem']
j = []
for x in i:
    if(x<=1073):
        j.append(0)
    elif(x>=1074 and x<=1827):
        j.append(1)
    elif(x>=1828 and x<=2580):
        j.append(2)
    elif(x>=2581 and x<=3333):
        j.append(3)
    elif(x>=3334 and x<=4087):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['Dem'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Dem'})

i = df['Slope']
Slope = []
for x in i:
    if(x<=12.34783182):
        Slope.append(0)
    elif(x>=12.34783183 and x<=24.98965963):
        Slope.append(1)
    elif(x>=24.98965964 and x<=37.63148744):
        Slope.append(2)
    elif(x>=37.63148745 and x<=49.97931925):
        Slope.append(3)
    elif(x>=49.97931926 and x<=62.62114707):
        Slope.append(4)
    else:
        Slope.append(5)
df = df.drop(['Slope'],axis=1)
Slope = pd.Series(Slope)
df = pd.concat([df,Slope], axis=1)
df = df.rename(columns={0:'Slope'})

i = df['TWI']
j = []
for x in i:
    if(x<=-3.087861776):
        j.append(0)
    elif(x>=-3.087861775 and x<=1.607861519):
        j.append(1)
    elif(x>=1.60786152 and x<=6.303584814):
        j.append(2)
    elif(x>=6.303584815 and x<=10.99930811):
        j.append(3)
    elif(x>=10.99930812 and x<=15.6950314):
        j.append(4)
    else:
        j.append(5)

df = df.drop(['TWI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'TWI'})

i = df['STI']
j = []
for x in i:
    if(x<=23.99506887):
        j.append(0)
    elif(x>=23.99506888 and x<=47.99013774):
        j.append(1)
    elif(x>=47.99013775 and x<=71.9852066):
        j.append(2)
    elif(x>=71.98520661 and x<=95.98027547):
        j.append(3)
    elif(x>=95.98027548 and x<=119.9753443):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['STI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'STI'})

i = df['SPI']
j = []
for x in i:
    if(x<=166.66975910):
        j.append(0)
    elif(x>=166.6697592 and x<=333.3395182):
        j.append(1)
    elif(x>=333.3395183 and x<=500.0092773):
        j.append(2)
    elif(x>=500.0092774 and x<=666.6790365):
        j.append(3)
    elif(x>=666.6790366 and x<=833.3487956):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['SPI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'SPI'})

i = df['Slope_Length']
j = []
for x in i:
    if(x<=918.1981608):
        j.append(0)
    elif(x>=918.1981609 and x<=1836.396322):
        j.append(1)
    elif(x>=1836.396323 and x<=2754.594482):
        j.append(2)
    elif(x>=2754.594483 and x<=3672.792643):
        j.append(3)
    elif(x>=3672.792644 and x<=4590.990804):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['Slope_Length'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Slope_Length'})

#moving landslide in the last
landslide = df['Landslide']
df = df.drop(['Landslide'],axis=1)
df = pd.concat([df,landslide], axis=1)

#saving csv file
df.to_csv('../csv/processed2_truncated_data.csv')

#%%
"""preprocessing 3 - handling DEM derivatives, all buffer variables"""
"""Class (quantile)"""

i = df['Plan_Curvature']
Plan_Curvature = []
for x in i:
    if(x>=-19.99183464 and x<=-2.543456383):
        Plan_Curvature.append(0)
    elif(x>=-2.543456382 and x<=-0.957240191):
        Plan_Curvature.append(1)
    elif(x>=-0.957240190 and x<=-0.092031344):
        Plan_Curvature.append(2)
    elif(x>=-0.092031343 and x<=0.340573075):
        Plan_Curvature.append(3)
    elif(x>=0.340573076 and x<=0.917378968):
        Plan_Curvature.append(4)
    elif(x>=0.917378969 and x<=16.777954102):
        Plan_Curvature.append(5)

i = df['Profile_Curvature']
Profile_Curvature = []
for x in i:
    if(x>=-18.93183708 and x<=-1.194549134):
        Profile_Curvature.append(5)
    elif(x>=-1.194549133 and x<=-0.436545376):
        Profile_Curvature.append(4)
    elif(x>=-0.436545375 and x<=0.018256879):
        Profile_Curvature.append(3)
    elif(x>=0.018256880 and x<=0.473059134):
        Profile_Curvature.append(2)
    elif(x>=0.473059135 and x<=1.382663645):
        Profile_Curvature.append(1)
    elif(x>=1.382663646 and x<=19.57475385):
        Profile_Curvature.append(0)

df = df.drop(['Plan_Curvature','Profile_Curvature'], axis = 1)
Plan_Curvature = pd.Series(Plan_Curvature)
Profile_Curvature = pd.Series(Profile_Curvature)
df = pd.concat([df,Plan_Curvature,Profile_Curvature], axis=1)
df = df.rename(columns={0:'Plan_Curvature',1:'Profile_Curvature'})

i = df['Aspect']
j = []
for x in i:
    if(x<0):
        j.append(0)
    elif(x>=0 and x<22.5):
        j.append(1)
    elif(x>=22.5 and x<67.5):
        j.append(2)
    elif(x>=67.5 and x<112.5):
        j.append(3)
    elif(x>=112.5 and x<157.5):
        j.append(4)
    elif(x>=157.5 and x<202.5):
        j.append(5)
    elif(x>=202.5 and x<247.5):
        j.append(6)
    elif(x>=247.5 and x<292.5):
        j.append(7)
    elif(x>=292.5 and x<337.5):
        j.append(8)
    else:
        j.append(1)
df = df.drop(['Aspect'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Aspect'})

i = df['Fault']
Fault = []
for x in i:
    if(x==50):
        Fault.append(0)
    elif(x==100):
        Fault.append(1)
    elif(x==150):
        Fault.append(2)
    elif(x==200):
        Fault.append(3)
    elif(x==250):
        Fault.append(4)
    elif(x==300):
        Fault.append(5)
df = df.drop(['Fault'],axis=1)
Fault = pd.Series(Fault)
df = pd.concat([df,Fault], axis=1)
df = df.rename(columns={0:'Fault'})

i = df['River']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['River'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'River'})

i = df['Road']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['Road'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Road'})

i = df['Dem']
j = []
for x in i:
    if(x>=320 and x<=958):
        j.append(0)
    elif(x>=959 and x<=1259):
        j.append(1)
    elif(x>=1260 and x<=1525):
        j.append(2)
    elif(x>=1526 and x<=1968):
        j.append(3)
    elif(x>=1969 and x<=2890):
        j.append(4)
    elif(x>=2891 and x<=4840):
        j.append(5)
df = df.drop(['Dem'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Dem'})

i = df['Slope']
Slope = []
for x in i:
    if(x>=0 and x<=14.4058079):
        Slope.append(0)
    elif(x>=14.4058080 and x<=22.34369567):
        Slope.append(1)
    elif(x>=22.34369568 and x<=28.51761157):
        Slope.append(2)
    elif(x>=28.51761158 and x<=34.39753149):
        Slope.append(3)
    elif(x>=34.39753150 and x<=41.45343538):
        Slope.append(4)
    elif(x>=41.45343539 and x<=74.96897888):
        Slope.append(5)
df = df.drop(['Slope'],axis=1)
Slope = pd.Series(Slope)
df = pd.concat([df,Slope], axis=1)
df = df.rename(columns={0:'Slope'})

i = df['TWI']
j = []
for x in i:
    if(x>=-7.783585072 and x<=4.038588872):
        j.append(0)
    elif(x>=4.038588873 and x<=5.806390583):
        j.append(1)
    elif(x>=5.806390584 and x<=10.22589486):
        j.append(2)
    elif(x>=10.22589487 and x<=11.66223375):
        j.append(3)
    elif(x>=11.66223376 and x<=12.76710982):
        j.append(4)
    elif(x>=12.76710983 and x<=20.3907547):
        j.append(5)

df = df.drop(['TWI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'TWI'})

i = df['STI']
j = []
new = []
for x in i:
    if(x<0.000001):
        new.append(0)
new = shuffle(new)
l1 = new[:203092]
l2 = new[203092:]
j = l1.copy()
for x in l2:
    j.append(1)

for x in i:
    if(x>0 and x<=0.564589856):
        j.append(1)
    elif(x>=0.564589857 and x<=1.129179711):
        j.append(2)
    elif(x>=1.129179712 and x<=2.258359423):
        j.append(3)
    elif(x>=2.258359424 and x<=4.516718846):
        j.append(4)
    elif(x>=4.516718847 and x<=143.9704132):
        j.append(5)
df = df.drop(['STI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'STI'})

i = df['SPI']
j = []
new = []
for x in i:
    if(x<0.000001):
        new.append(0)
new = shuffle(new)
l1 = new[:203092]
l2 = new[203092:]
j = l1.copy()
for x in l2:
    j.append(1)

for x in i:
    if(x>0 and x<=3.921641391):
        j.append(1)
    elif(x>=3.921641392 and x<=7.843282782):
        j.append(2)
    elif(x>=7.843282783 and x<=15.68656556):
        j.append(3)
    elif(x>=15.68656557 and x<=631.37313113):
        j.append(4)
    elif(x>=31.37313114 and x<=1000.018555):
        j.append(5)
df = df.drop(['SPI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'SPI'})

i = df['Slope_Length']
j = []
new = []
for x in i:
    if(x<0.000001):
        new.append(0)
new = shuffle(new)
l1 = new[:203092]
l2 = new[203092:]
j = l1.copy()
for x in l2:
    j.append(1)

for x in i:
    if(x>0 and x<=21.60466261):
        j.append(1)
    elif(x>=21.60466262 and x<=86.41865043):
        j.append(2)
    elif(x>=86.41865044 and x<=172.8373009):
        j.append(3)
    elif(x>=172.8373010 and x<=345.6746017):
        j.append(4)
    elif(x>=345.6746018 and x<=5509.188965):
        j.append(5)
df = df.drop(['Slope_Length'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Slope_Length'})

#moving landslide in the last
landslide = df['Landslide']
df = df.drop(['Landslide'],axis=1)
df = pd.concat([df,landslide], axis=1)

#saving csv file
df.to_csv('../csv/processed3_truncated_data.csv')
#%%
import pandas as pd
import numpy as np
from osgeo import gdal
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import  roc_curve, auc
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from rotation_forest import RotationForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import ExtraTreesClassifier
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
import pickle
#import graphviz
from sklearn import tree
from sklearn.feature_selection import SelectFromModel

df = pd.read_csv("../csv/processed3_truncated_data.csv")
df=df.drop(df.columns[0],axis=1) # one extra column of index created in processed and hence removed.

""""since data is unbalanced, stratify the data without replacement"""

df1 = df.Landslide[df.Landslide.eq(2)].index
df2 = df.Landslide[df.Landslide.eq(1)].sample(len(df1),replace=False).index

df_new = df1.union(df2)
df_final= df.loc[df_new]
df_final.to_csv('../csv/stratified_data.csv', index = False)

""""creating train test split and saving them for future use after reading startified data"""

#df_final = pd.read_csv("../csv/stratified_data0.csv")
X = df_final.iloc[:,:16]
Y = df_final.iloc[:,16]

X_train, X_test, y_train, y_test = train_test_split(X,Y, random_state=1,test_size = 0.3)

#saving latitude and longtitude along with independent variable
y_train = pd.concat([y_train,X_train['Latitude'],X_train['Longitude']],axis=1)
y_test = pd.concat([y_test,X_test['Latitude'],X_test['Longitude']],axis=1)

X_train = X_train.iloc[:,2:]  # remove 0 and 1 index of lat long
X_test = X_test.iloc[:,2:]    # remove 0 and 1 index of lat long

#precautionary check if there are any nan values in the data
print(X_train.isnull().values.any())
print(y_train.isnull().values.any())
print(X_test.isnull().values.any())
print(y_test.isnull().values.any())

#writing train test splits to the system for future use
X_train.to_csv('../csv/X_train.csv', index = False)
X_test.to_csv('../csv/X_test.csv', index = False)
y_train.to_csv('../csv/Y_train.csv', index = False)
y_test.to_csv('../csv/Y_test.csv', index = False)

#obtain labels without lat long
Y_train = y_train.iloc[:,0]
Y_test = y_test.iloc[:,0]

#%%
import pandas as pd
import numpy as np
from osgeo import gdal
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import  roc_curve, auc
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from rotation_forest import RotationForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import ExtraTreesClassifier
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
import pickle
#import graphviz
from sklearn import tree
from sklearn.feature_selection import SelectFromModel
"""to import already saved train test splits"""
X_train = pd.read_csv('../csv/0/X_train.csv')
y_train = pd.read_csv('../csv/0/Y_train.csv')
X_test = pd.read_csv('../csv/0/X_test.csv')
y_test = pd.read_csv('../csv/0/Y_test.csv')

Y_train = y_train.iloc[:,0]
Y_test = y_test.iloc[:,0]

#%%
"""DECISION TREE OPTIMISATION"""

params = {
        'criterion': ['gini','entropy'],
        'max_depth': [None,2,3, 4, 5],
        'min_samples_leaf': [1,2,3, 4, 5,6],
        'min_samples_split': [2,3, 4, 5,6]
        }

decision_model = DecisionTreeClassifier(random_state = 1)
grid_model = GridSearchCV(decision_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
grid_model.fit(X_train,Y_train)

#print("dt auc")
#print(grid_model.best_params_)
#Print out scores on validation set
#print(random_model.score(X_test,Y_test))
#print(grid_model.score(X_test,Y_test))
#print out scores on training data
#print(random_model.best_score_)
#print("train_score_auc:"+str(grid_model.best_score_))

""""OPTIMISED DECISION TREE"""

dt_model = DecisionTreeClassifier(random_state=1,min_samples_leaf=6,max_depth=5)
dt_model.fit(X_train, Y_train)
dt_score = dt_model.score(X_test, Y_test)
dt_score_train = dt_model.score(X_train, Y_train)

cm_dt = confusion_matrix(Y_test, dt_model.predict(X_test))

dt_predict  = dt_model.predict(X_test)
dt_probs = dt_model.predict_proba(X_test)
dtfpr = dict()
dttpr = dict()
dtroc_auc = dict()
dtfpr, dttpr, _ = roc_curve(Y_test, dt_probs[:,1], drop_intermediate = False,pos_label=2)
dtroc_auc = auc(dtfpr,dttpr)

print(dt_model.feature_importances_)
#%%
"""save dt model"""

pickle.dump(dt_model, open('decison_model.sav', 'wb'))

"""load saved model"""

load_dt_model = pickle.load(open('decison_model.sav', 'rb'))

#%% """Render decision tree"""

import os
os.environ['PATH'] += os.pathsep + 'C:\graphviz\bin'
features = []
for i in X_train:
    features.append(i)
output = ['0','1']
dot_data = tree.export_graphviz(dt_model, out_file = None, filled = True,
                                rounded = True, special_characters = True,
                                class_names= output, feature_names=features)
graph = graphviz.Source(dot_data)
graph.render("landslide_dt")

"""will save the graph in pwd"""

#%%
"""RANDOM FOREST OPTIMISATION"""

params = {
        'max_depth': [None,2,3, 4, 5],
        'n_estimators': [100,250,500],
        'min_samples_leaf': [1,2,3, 4, 5,6],
        'min_samples_split': [2,3, 4, 5,6]
        }

random_model = RandomForestClassifier(random_state = 1)
grid_model = GridSearchCV(random_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
grid_model.fit(X_train,Y_train)

#%%
#Print out best parameters
print("Random auc")
print(grid_model.best_params_)
#Print out scores on validation set

print(grid_model.score(X_test,Y_test))
#print out scores on training data

print(grid_model.best_score_)

#%%
"""OPTIMISED RANDOM FOREST"""

rf_model = RandomForestClassifier(random_state=1,n_estimators=500, max_depth=5)
rf_model.fit(X_train, Y_train)
rf_score = rf_model.score(X_test, Y_test)
rf_score_train = rf_model.score(X_train, Y_train)

cm_rf = confusion_matrix(Y_test, rf_model.predict(X_test))
rf_predict  = rf_model.predict(X_test)
rf_probs = rf_model.predict_proba(X_test)
rffpr = dict()
rftpr = dict()
rfroc_auc = dict()
rffpr, rftpr, _ = roc_curve(Y_test, rf_probs[:,1], drop_intermediate = False,pos_label=2)
rfroc_auc = auc(rffpr,rftpr)

#%%
"""save rf model"""

pickle.dump(rf_model, open('random_model.sav', 'wb'))

"""load saved model"""

load_rf_model = pickle.load(open('random_model.sav', 'rb'))

#%%
"""ROTATION FOREST OPTIMISATION"""

params = {
        'max_depth': [None,2,3, 4, 5],
        'n_estimators': [100,250,500],
        'criterion': ['gini','entropy'],
        'min_samples_leaf': [1,2,3, 4, 5],
        'min_samples_split': [2,3, 4, 5]
        }

rotation_model = RotationForestClassifier(random_state = 1)
grid_model = GridSearchCV(rotation_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
grid_model.fit(X_train,Y_train)

#%%
#Print out best parameters
print("rotation auc")
print(grid_model.best_params_)
#Print out scores on validation set

print(grid_model.score(X_test,Y_test))
#print out scores on training data

print(grid_model.best_score_)
#%%
"""OPTIMISED ROTATION FOREST"""

rt_model = RotationForestClassifier(random_state=1, max_depth = 5)
rt_model.fit(X_train,Y_train)
rt_score = rt_model.score(X_test, Y_test)
rt_score_train = rt_model.score(X_train, Y_train)

cm_rt = confusion_matrix(Y_test, rt_model.predict(X_test))
rt_predict  = rt_model.predict(X_test)
rt_probs = rt_model.predict_proba(X_test)
rtfpr = dict()
rttpr = dict()
rtroc_auc = dict()
rtfpr, rttpr, _ = roc_curve(Y_test, rt_probs[:,1], drop_intermediate = False,pos_label=2)
rtroc_auc = auc(rtfpr,rttpr)

#%%
"""save rt model"""

pickle.dump(rt_model, open('rotation_model.sav', 'wb'))

"""load saved model"""

load_rt_model = pickle.load(open('rotation_model.sav', 'rb'))

#%%
"""ADABOOST OPTIMISATION """

learning_rate = list(range(100,300,10))
rate = []
for i in learning_rate:
    rate.append(float(i/100))
params = {
        'n_estimators': [50,100,250,500,800,1000],
        'learning_rate': rate,
        'algorithm':['SAMME','SAMME.R']
        }

ada_model = AdaBoostClassifier(random_state = 1)
adagrid_model = GridSearchCV(ada_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
adagrid_model.fit(X_train,Y_train)

#%%
#Print out best parameters
print("ada roc")
print(adagrid_model.best_params_)
#Print out scores on validation set

print(adagrid_model.score(X_test,Y_test))
#print out scores on training data

print(adagrid_model.best_score_)
#%%
"""OPTIMISED AdaBoost Classifier"""

adb_model = AdaBoostClassifier(random_state=1, learning_rate=1.6,n_estimators=1000,algorithm='SAMME')
adb_model.fit(X_train, Y_train)
adb_score = adb_model.score(X_test,Y_test)
adb_score_train = adb_model.score(X_train, Y_train)

cm_ad = confusion_matrix(Y_test, adb_model.predict(X_test))
adb_predict  = adb_model.predict(X_test)
ad_probs = adb_model.predict_proba(X_test)
adfpr = dict()
adtpr = dict()
adroc_auc = dict()
adfpr, adtpr, _ = roc_curve(Y_test, ad_probs[:,1], drop_intermediate = False,pos_label=2)
adroc_auc = auc(adfpr,adtpr)

#%%
"""save adb model"""

pickle.dump(adb_model, open('adaboost_model.sav', 'wb'))

"""load saved model"""

load_adb_model = pickle.load(open('adaboost_model.sav', 'rb'))

#%%
"""EXTRA TREE OPTIMISATION"""

learning_rate = list(range(100,300,10))
rate = []
for i in learning_rate:
    rate.append(float(i/100))
params = {
        'n_estimators': [100,200,500],
        'max_depth': [None,2,3,4,5],
        'criterion': ['gini','entropy'],
        'min_samples_leaf': [1,2,3, 4, 5,6],
        'min_samples_split': [2,3, 4, 5,6]
        }

extra_model = ExtraTreesClassifier(random_state = 1)
extragrid_model = GridSearchCV(extra_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
extragrid_model.fit(X_train,Y_train)

#%%
#Print out best parameters
print("extra auc")
print(extragrid_model.best_params_)
#Print out scores on validation set

print(extragrid_model.score(X_test,Y_test))
#print out scores on training data

print(extragrid_model.best_score_)

#%%
"""OPTIMISED Extra Tree Classifier"""

ex_model = ExtraTreesClassifier(random_state=1,max_depth=5)
ex_model.fit(X_train,Y_train)
ex_score = ex_model.score(X_test, Y_test)
ex_score_train = ex_model.score(X_train, Y_train)

cm_ex = confusion_matrix(Y_test, ex_model.predict(X_test))
ex_predict  = ex_model.predict(X_test)
ex_probs = ex_model.predict_proba(X_test)
exfpr = dict()
extpr = dict()
exroc_auc = dict()
exfpr, extpr, _ = roc_curve(Y_test, ex_probs[:,1], drop_intermediate = False,pos_label=2)
exroc_auc = auc(exfpr,extpr)

#%%
"""save extra model"""

pickle.dump(ex_model, open('extra_model.sav', 'wb'))

"""load saved model"""

load_ex_model = pickle.load(open('extra_model.sav', 'rb'))

#%%
"""OPTIMISING XGBOOST"""

learning_rate = list(range(100,260,10))
rate = []
for i in learning_rate:
    rate.append(float(i/100))
params = {
        'learning_rate': rate,
        'max_depth': [3,4,5,6],
        'gamma':[0.5,1,1.5,2,5],
        'colsample_bytree':[0.8,0.8,1.0],
        'min_child_weight':[1,5,10],
        'subsample': [0.6,0.8,1.0]
        }

xg_model = XGBClassifier(random_state = 1)
xggrid_model = GridSearchCV(xg_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
xggrid_model.fit(X_train,Y_train)

#%%
#Print out best parameters
print("xg auc")
print(xggrid_model.best_params_)
#Print out scores on validation set

print(xggrid_model.score(X_test,Y_test))
#print out scores on training data

print(xggrid_model.best_score_)

#%%
"""OPTIMISED implement XGBoost"""

xg_model = XGBClassifier(random_state=1, colsample_bytree=0.8, gamma = 5,
                         max_depth=6)
xg_model.fit(X_train, Y_train)
xg_score = xg_model.score(X_test, Y_test)
xg_score_train = xg_model.score(X_train, Y_train)

cm_xg = confusion_matrix(Y_test, xg_model.predict(X_test))
xg_predict  = xg_model.predict(X_test)
xg_probs = xg_model.predict_proba(X_test)
xgfpr = dict()
xgtpr = dict()
xgroc_auc = dict()
xgfpr, xgtpr, _ = roc_curve(Y_test, xg_probs[:,1], drop_intermediate = False,pos_label=2)
xgroc_auc = auc(xgfpr,xgtpr)

    #%%
"""save xgboost model"""

pickle.dump(xg_model, open('xgboost_model.sav', 'wb'))

"""load saved model"""

load_xg_model = pickle.load(open('xgboost_model.sav', 'rb'))

#%%
"""optimizing logistic regression
penalty =['l2','none']
test_accuracy = []
train_accuracy = []
for i in penalty:
    model = LogisticRegression(random_state=1, penalty=i)
    model.fit(X_train,Y_train)
    test_accuracy.append(model.score(X_test,Y_test))
    train_accuracy.append(model.score(X_train, Y_train))

solver = ['newton-cg','lbfgs','liblinear','sag','saga']
test_accuracy = []
train_accuracy = []
for i in solver:
    model = LogisticRegression(random_state=1, solver=i)
    model.fit(X_train,Y_train)
    test_accuracy.append(model.score(X_test,Y_test))
    train_accuracy.append(model.score(X_train, Y_train))
#same in all

test_accuracy = []
train_accuracy = []
for i in range(1,101):
    model = LogisticRegression(random_state=1, C=i,max_iter=1000)
    model.fit(X_train,Y_train)
    test_accuracy.append(model.score(X_test,Y_test))
    train_accuracy.append(model.score(X_train, Y_train))

"""
#%%
"""implementing logistic regression"""

lt_model = LogisticRegression(random_state=1)
lt_model.fit(X_train,Y_train)
lt_score = lt_model.score(X_test,Y_test)
lt_score_train = lt_model.score(X_train,Y_train)

cm_lt = confusion_matrix(Y_test, lt_model.predict(X_test))
lt_predict  = lt_model.predict(X_test)
lt_probs = lt_model.predict_proba(X_test)
ltfpr = dict()
lttpr = dict()
ltroc_auc = dict()
ltfpr, lttpr, _ = roc_curve(Y_test, lt_probs[:,1], drop_intermediate = False,pos_label=2)
ltroc_auc = auc(ltfpr,lttpr)

#%%
"""save logistic model"""

pickle.dump(lt_model, open('logistic_model.sav', 'wb'))

"""load saved model"""

load_lt_model = pickle.load(open('logistic_model.sav', 'rb'))

#%%
"""plotting individual roc curves for all the algos

plt.figure()
plt.plot(dtfpr,dttpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.title("Decision_Tree")
plt.text(0.8,0.2,"auc = "+str(round(dtroc_auc,4)))
plt.show()

plt.figure()
plt.plot(rffpr,rftpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.text(0.8,0.2,"auc = "+str(round(rfroc_auc,4)))
plt.title("Random_Forest")
plt.show()

plt.figure()
plt.plot(rtfpr,rttpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.text(0.8,0.2,"auc = "+str(round(rtroc_auc,4)))
plt.title("Rotation_Forest")
plt.show()

plt.figure()
plt.plot(adfpr,adtpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.text(0.8,0.2,"auc = "+str(round(adroc_auc,4)))
plt.title("AdaBoost")
plt.show()

plt.figure()
plt.plot(exfpr,extpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.text(0.8,0.2,"auc = "+str(round(exroc_auc,4)))
plt.title("Extra_Tree")
plt.show()

plt.figure()
plt.plot(xgfpr,xgtpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.title("XGBoost")
plt.text(0.8,0.2,"auc = "+str(round(xgroc_auc,4)))
plt.show()

plt.figure()
plt.plot(ltfpr,lttpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.title("Logistic")
plt.text(0.8,0.2,"auc = "+str(round(ltroc_auc,4)))
plt.show()
"""
#%%
"""graph for all roc scores combined"""

data = [['red','Decision', round((100*dtroc_auc),2)],
        ['blue','Rotation', round((100*rtroc_auc),2)],
        ['black','AdaBoost', round((100*adroc_auc),2)],
        ['darkorange','XGBoost', round((100*xgroc_auc),2)],
        ['green','Random', round((100*rfroc_auc),2)],
        ['magenta','Extra Tree', round((100*exroc_auc),2)]]

def sort_data(data):
    data.sort(key = lambda x:x[2])
    return data

data = sort_data(data)
colors=[]

for x in range(6):
    colors.append(data[x].pop(0))

plt.figure()
plt.plot(dtfpr,dttpr, color = 'red')
plt.plot(rffpr,rftpr, color = 'green')
plt.plot(rtfpr,rttpr, color = 'blue')
plt.plot(adfpr,adtpr, color = 'black')
plt.plot(exfpr,extpr, color = 'magenta')
plt.plot(xgfpr,xgtpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.table(cellText=data,cellLoc='left',
          rowColours=colors, colWidths=[0.2,0.15],
          loc=4).auto_set_font_size(False)
plt.text(0.65,0.38,"algorithm")
plt.text(0.9,0.38,"auc")
plt.title("Area Under Curve")
plt.show()

#%%
"""taking majority vote"""
labels = pd.DataFrame()
labels['dt'] = dt_predict
labels['rf'] = rf_predict
labels['rt'] = rt_predict
labels['adb'] = adb_predict
labels['ex'] = ex_predict
labels['xg'] = xg_predict
labels['lt'] = lt_predict

majority_vote = labels.mode(axis = 1)

cm_majority_vote = confusion_matrix(Y_test,majority_vote)

#%%
"""calculating weighted output"""
score_list = list([dt_score,rt_score,rf_score,adb_score,ex_score,xg_score,lt_score])
total = sum(score_list)

weights = []
for i in score_list:
    weights.append(i/total)

weights = pd.Series(weights)
weighted_output = np.dot(labels,weights)

weighted_output = np.round(weighted_output)
cm_weights = confusion_matrix(Y_test,weighted_output)

#%%
data = pd.DataFrame(X_train)
data['Landslide'] = Y_train
data2 = pd.DataFrame(X_test)
data2['Landslide'] = Y_test

data.to_csv("../csv_files/train.csv", index= False)
data2.to_csv("../csv_files/test.csv", index= False)


from collections import deque

def bfs():
    q = deque([n])
    while q:
        node = q.popleft()
        if node == k:
            print(time[node])
            return
        for i in (node-1, node+1, node*2):
            if 0 <= i <= MAX_N:
                if not time[i]:
                    q.append(i)
                    time[i] = time[node] + 1

n, k = map(int, input().split())
MAX_N = 100000
time = [0] * (MAX_N + 1)

bfs()


#!/usr/bin/env python
"""
 Select transcript based on their size (i.e size of mature/spliced transcript).
"""
import argparse
import os
import sys

from pygtftk import arg_formatter
from pygtftk.cmd_object import CmdObject
from pygtftk.gtf_interface import GTF
from pygtftk.utils import message

__updated__ = "2018-01-20"

def make_parser():
    """The program parser."""
    parser = argparse.ArgumentParser(add_help=True)

    parser.add_argument('-i', '--inputfile',
                        help="Path to the GTF file. Default to STDIN",
                        default=sys.stdin,
                        metavar="GTF",
                        type=arg_formatter.FormattedFile(mode='r', file_ext=('gtf', 'gtf.gz')))

    parser.add_argument('-o', '--outputfile',
                        help="Output file.",
                        default=sys.stdout,
                        metavar="GTF",
                        type=arg_formatter.FormattedFile(mode='w', file_ext=('gtf')))

    parser.add_argument('-m', '--min-size',
                        help="Minimum size.",
                        default=0,
                        type=int)

    parser.add_argument('-M', '--max-size',
                        help="Maximum size.",
                        default=1000000000,
                        type=int)

    return parser

def select_by_tx_size(inputfile=None,
                      outputfile=None,
                      min_size=None,
                      max_size=None):
    """
    Select features by size.
    """

    msg = "Selecting 'mature/spliced transcript by size (range: [{m},{M}])."
    msg = msg.format(m=str(min_size),
                     M=str(max_size))
    message(msg)

    GTF(inputfile
        ).select_by_transcript_size(min_size,
                                    max_size
                                    ).write(outputfile,
                                            gc_off=True)

def main():
    myparser = make_parser()
    args = myparser.parse_args()
    args = dict(args.__dict__)
    select_by_tx_size(**args)

if __name__ == '__main__':
    main()

else:

    test = """
    #select_by_tx_size
    @test "select_by_tx_size_1" {
     result=`gtftk get_example | gtftk select_by_tx_size -m 14 | wc -l`
      [ "$result" -eq 6 ]
    }
    #select_by_tx_size
    @test "select_by_tx_size_2" {
     result=`gtftk get_example | gtftk select_by_tx_size -m 15 | wc -l`
      [ "$result" -eq 0 ]
    }
    #select_by_tx_size
    @test "select_by_tx_size_3" {
     result=`gtftk get_example | gtftk select_by_tx_size -m 0 | wc -l`
      [ "$result" -eq 60 ]
    }
    #select_by_tx_size
    @test "select_by_tx_size_4" {
     result=`gtftk get_example | gtftk feature_size -t mature_rna | gtftk select_by_tx_size -m 8 -M 8|wc -l`
      [ "$result" -eq 16 ]
    }
    """

    CmdObject(name="select_by_tx_size",
              message="Select transcript based on their size (i.e size of mature/spliced transcript).",
              parser=make_parser(),
              fun=os.path.abspath(__file__),
              group="selection",
              desc=__doc__,
              updated=__updated__,
              test=test)


import os
import queue
import re
import shutil
from typing import List, Optional, Iterable, Tuple

from sflkit.instrumentation import Instrumentation
from sflkit.instrumentation.file_instrumentation import FileInstrumentation
from sflkit.language.visitor import ASTVisitor
from sflkit.logger import LOGGER

class DirInstrumentation(Instrumentation):
    def __init__(self, visitor: ASTVisitor):
        super().__init__(visitor)
        self.file_instrumentation = FileInstrumentation(visitor)

    @staticmethod
    def check_included(element: str, includes: Optional[Iterable[str]]):
        return not includes or any(re.match(include, element) for include in includes)

    def handle_element(
        self,
        element: str,
        file_queue: queue.Queue[Tuple[str, bool]],
        src: os.PathLike,
        dst: os.PathLike,
        suffixes: List[str],
        check: bool,
    ):
        if os.path.isdir(os.path.join(src, element)):
            LOGGER.debug(f"I found a subdir at {element}.")
            os.makedirs(os.path.join(dst, element), exist_ok=True)
            for f in os.listdir(os.path.join(src, element)):
                file_queue.put((os.path.join(element, f), check))
        elif (
            not check
            and any(element.endswith(f".{suffix}") for suffix in suffixes)
            and not os.path.islink(os.path.join(src, element))
        ):
            LOGGER.debug(f"I found a file I can instrument at {element}.")
            self.file_instrumentation.instrument(
                os.path.join(src, element),
                os.path.join(dst, element),
                file=element,
            )
        else:
            LOGGER.debug(f"I found a file I will not instrument at {element}.")
            shutil.copy(
                os.path.join(src, element),
                os.path.join(dst, element),
                follow_symlinks=False,
            )

    def instrument(
        self,
        src: os.PathLike,
        dst: os.PathLike,
        suffixes: List[str] = None,
        file: str = "",
        includes: Optional[Iterable[str]] = None,
        excludes: Optional[Iterable[str]] = None,
    ):
        if suffixes is None:
            raise ValueError("DirInstrumentation requires suffixes")
        if excludes is None:
            excludes = list()
        if not os.path.exists(src):
            raise ValueError(f"Path {src} does not exist")
        if os.path.exists(dst):
            if os.path.isdir(dst):
                shutil.rmtree(dst)
            else:
                os.remove(dst)
        if os.path.isfile(src):
            LOGGER.debug(f"I found a file I can instrument at {src}.")
            self.file_instrumentation.instrument(
                src, dst, suffixes=suffixes, file=os.path.split(src)[-1]
            )
        else:
            os.makedirs(dst, exist_ok=True)
            file_queue = queue.Queue()
            file_queue.put(("", True))
            while not file_queue.empty():
                element, check = file_queue.get()
                if check and self.check_included(element, includes):
                    self.handle_element(element, file_queue, src, dst, suffixes, False)
                elif element != "" and any(
                    re.match(exclude, element) for exclude in excludes
                ):
                    if os.path.isdir(os.path.join(src, element)):
                        shutil.copytree(
                            os.path.join(src, element),
                            os.path.join(dst, element),
                            symlinks=True,
                        )
                    else:
                        shutil.copy(
                            os.path.join(src, element),
                            os.path.join(dst, element),
                            follow_symlinks=False,
                        )
                    continue
                else:
                    self.handle_element(element, file_queue, src, dst, suffixes, check)
            self.events = self.file_instrumentation.events
        LOGGER.info(f"I found {len(self.events)} events in {src}.")


import gzip
import codecs
import os
import json
from subprocess import Popen, PIPE
from tqdm import tqdm
import networkx as nx
from collections import defaultdict
from spacy.tokens import Doc
import multiprocess as mp
import pickle
import re
import spacy
import multiprocessing
from pathlib import Path
import pickle
environ = os.environ.copy()
environ['JAVA_HOME'] = '/usr/local/jdk-11.0.2'
environ['PATH'] += ':/usr/local/jdk-11.0.2/bin'

dir_path = os.path.dirname(os.path.realpath(__file__))
home_path = str(Path.home())
EXTRACTOR_JAR = os.path.join(dir_path, 'features-javac/extractor/target/'
                                       'features-javac-extractor-1.0.0-SNAPSHOT-jar-with-dependencies.jar')
DOT_JAR = os.path.join(dir_path, 'features-javac/dot/target/features-javac-dot-1.0.0-SNAPSHOT-jar-with-dependencies.jar')
RAW_FILE = os.path.join(home_path, 'data/code_search_data/CodeSearchNet/resources/data/java/final/jsonl')
JAVA_BASE = os.path.join(home_path, 'data/code_search_data/CodeSearchNet/resources/data/java_dedupe_definitions_v2.pkl')
JAVA_BASE_DIR = os.path.join(home_path, 'data/code_search_data/CodeSearchNet/resources/data/java_base')
BASE_GRAPH_FILE = os.path.join(home_path, 'data/code_search_data/CodeSearchNet/resources/data/java_base_gz')

class WhitespaceTokenizer(object):
    def __init__(self, vocab):
        self.vocab = vocab

    def __call__(self, text):
        words = text.split(' ')
        # All tokens 'own' a subsequent space character in this tokenizer
        spaces = [True] * len(words)
        return Doc(self.vocab, words=words, spaces=spaces)
NLP = spacy.load('en')
NLP.tokenizer = WhitespaceTokenizer(NLP.vocab)

def load_jsonl_gz(file_path):
    reader = codecs.getreader('utf-8')
    data = []
    with gzip.open(file_path) as f:
        json_list = list(reader(f))
    data.extend([json.loads(jline) for jline in json_list])
    return data

def check_existed(sample, java_func_dir):
    couple = sample['url'].split('/')[-1].split('#')
    class_name = couple[0].split('.java')[0]
    start = couple[1].split('-')[0].replace('L', '')
    end = couple[1].split('-')[1].replace('L', '')
    if 'repo' in sample.keys():
        project = sample['repo'].replace('/', '-')
    else:
        project = sample['nwo'].replace('/', '-')
    file_name = os.path.join(java_func_dir, project + '_' + class_name + '_' + str(start) + '_' + str(end))
    if os.path.exists(file_name):
        return True
    else:
        return False

def write_sample_to_java_file(sample, java_func_dir):
    couple = sample['url'].split('/')[-1].split('#')
    class_name = couple[0].split('.java')[0]
    start = couple[1].split('-')[0].replace('L', '')
    end = couple[1].split('-')[1].replace('L', '')
    if 'repo' in sample.keys():
        project = sample['repo'].replace('/', '-')
    else:
        project = sample['nwo'].replace('/', '-')
    file_name = os.path.join(java_func_dir, project + '_' + class_name + '_' + str(start) + '_' + str(end))
    if not os.path.exists(file_name):
        os.makedirs(file_name)
    if 'code' in sample.keys():
        function_text = sample['code']
    else:
        function_text = sample['function']
    function_text = wrap_function_dummy_class(function_text, class_name)
    wrapped_file_path = write_wrapped_function(function_text, class_name, file_name)
    write_json(sample, os.path.join(file_name, class_name + '.json'))
    return wrapped_file_path, file_name

def write_wrapped_function(function_text, class_name, file_name):
    wrapped_file_path = os.path.join(file_name, class_name + '.java')
    with open(wrapped_file_path, 'w', encoding='utf-8') as writer:
        writer.write(function_text)
    return wrapped_file_path

def wrap_function_dummy_class(function_text, class_name):
    function_text = 'public class ' + class_name + ' { \n' + function_text + '\n }'
    return function_text

def write_json(data, file_path):
    with open(file_path, 'w', encoding='utf-8') as f:
        json.dump(data, f, indent=4)

def build_java_graphs():
    types = os.listdir(RAW_FILE)
    for type in types:
        samples = []
        type_dir = os.path.join(RAW_FILE, type)
        for file in os.listdir(type_dir):
            if file.endswith('.gz') and file.startswith('java'):
                samples.extend(load_jsonl_gz(os.path.join(type_dir, file)))
        java_func_dir = os.path.join(type_dir, 'java_funcs')
        if not os.path.exists(java_func_dir):
            os.makedirs(java_func_dir)
        results = parallel_process(samples, build_single_graph, args=(java_func_dir, ))
        succeed = 0
        for result in results:
            if result:
                succeed += 1
        print('%s built %d graphs totally' % (type, succeed))

def build_java_base_graphs():
    definitions = pickle.load(open(JAVA_BASE, 'rb'))
    if not os.path.exists(JAVA_BASE_DIR):
        os.makedirs(JAVA_BASE_DIR)
    results = parallel_process(definitions, build_single_graph, args=(JAVA_BASE_DIR, ))
    succeed = 0
    for result in results:
        if result:
            succeed += 1
    print('%s built %d graphs totally' % (type, succeed))

def build_single_graph(sample, java_func_dir):
    try:
        if 'function_tokens' in sample.keys():
            if len(sample['function_tokens']) > 200:
                return False
        if check_existed(sample, java_func_dir):
            return False
        wrapped_file_path, file_name = write_sample_to_java_file(sample, java_func_dir)
        generate_proto_file(wrapped_file_path)
        if os.path.exists(wrapped_file_path + '.proto'):
            generate_dot_json_file(wrapped_file_path)
            return True
        return False
    except:
        return False

def parallel_process(array, function, args=(), n_cores=None):
    if n_cores is 1:
        return [function(x, *args) for x in tqdm(array)]
    with tqdm(total=len(array)) as pbar:
        def update(*args):
            pbar.update()
        if n_cores is None:
            n_cores = multiprocessing.cpu_count()
        with multiprocessing.Pool(processes=n_cores) as pool:
            jobs = [
                pool.apply_async(function, (x, *args), callback=update) for x in array
            ]
            results = [job.get() for job in jobs]
        return results

def generate_proto_file(wrapped_file_path):
    command = '/usr/local/jdk-11.0.2/bin/javac -cp %s -Xplugin:FeaturePlugin %s' % (EXTRACTOR_JAR, wrapped_file_path)
    p1 = Popen(command, cwd='/', shell=True, env=environ, stdout=PIPE, stderr=PIPE)
    p1.communicate()

def generate_dot_json_file(wrapped_file_path):
    command = '/usr/local/jdk-11.0.2/bin/java -jar %s -i %s -o %s -j %s' % (DOT_JAR,
                                                                            os.path.join(wrapped_file_path + '.proto'),
                                                                            os.path.join(wrapped_file_path + '.dot'),
                                                                            os.path.join(wrapped_file_path + '.json'))
    p1 = Popen(command, cwd='/', shell=True, env=environ, stdout=PIPE, stderr=PIPE)
    p1.communicate()

def normalize_graph(graph, func_nodes):
    method_nodes = []
    method_edges = []
    sorted_nodes = sorted(graph['node'], key=lambda i: int(i['id']))
    mapping = {}
    index = 0
    nodes_to_subtokenize = {}
    token_sequential_list = []
    seq_token_in_node = []
    try:
        for sorted_node in sorted_nodes:
            if sorted_node['id'] in func_nodes:
                if 'TOKEN' in sorted_node['type']:
                    subtokens = subtokenizer(sorted_node['contents'])
                    if len(subtokens) > 1:
                        for subtoken in subtokens:
                            dummy_node = sorted_node.copy()
                            dummy_node['contents'] = subtoken
                            dummy_node['id_sorted'] = index
                            dummy_node['subtoken'] = True
                            dummy_node['ori_token'] = sorted_node['contents']
                            token_sequential_list.append(subtoken)
                            seq_token_in_node.append(index)
                            method_nodes.append(dummy_node)
                            if sorted_node['id'] not in mapping.keys():
                                mapping[sorted_node['id']] = index
                            if sorted_node['id'] not in nodes_to_subtokenize.keys():
                                nodes_to_subtokenize[sorted_node['id']] = [index]
                            else:
                                nodes_to_subtokenize[sorted_node['id']].append(index)
                            index += 1
                    else:
                        sorted_node['id_sorted'] = index
                        sorted_node['subtoken'] = False
                        method_nodes.append(sorted_node)
                        mapping[sorted_node['id']] = index
                        seq_token_in_node.append(index)
                        token_sequential_list.append(sorted_node['contents'])
                        index += 1
                else:
                    sorted_node['id_sorted'] = index
                    method_nodes.append(sorted_node)
                    mapping[sorted_node['id']] = index
                    index += 1
        for edge in graph['edge']:
            if edge['sourceId'] in func_nodes and edge['destinationId'] in func_nodes and edge['type'] in \
                    ['AST_CHILD', 'NEXT_TOKEN', 'COMPUTED_FROM', 'LAST_USE', 'LAST_WRITE']:
                edge['sourceId'] = mapping[edge['sourceId']]
                edge['destinationId'] = mapping[edge['destinationId']]
                method_edges.append(edge)
        for key in nodes_to_subtokenize.keys():
            for index in range(1, len(nodes_to_subtokenize[key])):
                edge = {}
                edge['sourceId'] = nodes_to_subtokenize[key][index-1]
                edge['destinationId'] = nodes_to_subtokenize[key][index]
                edge['type'] = 'SUB_TOKEN'
                method_edges.append(edge)
        if not len(seq_token_in_node) == len(token_sequential_list):
            code_graph = {}
        else:
            code_graph = {'nodes': method_nodes, 'edges': method_edges, 'seq_token_in_node': seq_token_in_node,
                          'tokens': token_sequential_list}
    except:
        code_graph = {}
    return code_graph

def subtokenizer(identifier):
    if identifier == 'MONKEYS_AT':
        return [identifier]
    splitter_regex = re.compile('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)')
    identifiers = re.split('[._\-]', identifier)
    subtoken_list = []
    for identifier in identifiers:
        matches = splitter_regex.finditer(identifier)
        for subtoken in [m.group(0) for m in matches]:
            subtoken_list.append(subtoken)
    return subtoken_list

def single_post_process(key, chunk):
    file_graph_name = os.path.join(os.path.join(RAW_FILE, key), 'graph_' + key + '_gnn.jsonl.gz')
    count = 0
    with gzip.GzipFile(file_graph_name, 'wb') as gnn_file:
        for file in tqdm(chunk):
            class_name = file.split('_')[1]
            if os.path.exists(os.path.join(RAW_FILE, key, 'java_funcs', file, class_name + '.java.json')):
                try:
                    sub_graph_nodes = clip_dot_graph(os.path.join(RAW_FILE, key,
                                                                  'java_funcs', file, class_name + '.java.dot'))
                    with open(os.path.join(RAW_FILE, key, 'java_funcs', file, class_name + '.java.json')) as reader:
                        graph = json.load(reader)
                    with open(os.path.join(RAW_FILE, key, 'java_funcs', file, class_name + '.json')) as reader:
                        raw_sample = json.load(reader)
                    raw_sample['code_graph'] = normalize_graph(graph, sub_graph_nodes)
                    if raw_sample['docstring_tokens']:
                        doc_summary = ' '.join(re.sub(r'[^A-Za-z0-9 ]+', ' ',
                                                      ' '.join(raw_sample['docstring_tokens'])).split())
                        if doc_summary:
                            doc_graph = build_desc_graph(doc_summary)
                            if doc_graph:
                                raw_sample['doc_graph'] = normalize_des_graph(doc_graph)
                            else:
                                continue
                    else:
                        raw_sample['doc_graph'] = {}
                    if raw_sample['code_graph'] and raw_sample['doc_graph']:
                        save_sample_to_jsonl_gz(raw_sample, gnn_file)
                        count += 1
                    del raw_sample
                except:
                    continue
        print('there are %d samples have code graph and doc graph in %s' % (count, key))
        gnn_file.close()

def post_process():
    files = os.listdir(RAW_FILE)
    chunk_files = dict()
    process_list = []
    for file in files:
        raw_funcs_dir = os.path.join(RAW_FILE, file, 'java_funcs')
        chunk_files[file] = os.listdir(raw_funcs_dir)
    for key in chunk_files.keys():
        process = mp.Process(target=single_post_process,
                             args=(key, chunk_files[key]))
        process_list.append(process)
        process.start()
    for process in process_list:
        process.join()
    # single_post_process('test', chunk_files['test'])

def build_desc_graph(desc, file=None):
    try:
        if str(desc).endswith('.'):
            desc = desc[0: len(desc)-1]
        desc = ' '.join(desc.split())
        doc = NLP(desc)
        g_features = []
        dep_tree = defaultdict(list)
        boundary_nodes = []
        for sent in doc.sents:
            boundary_nodes.append(sent[-1].i)
            for each in sent:
                g_features.append(each.text)
                if each.i != each.head.i:                   # Not a root
                    dep_tree[each.head.i].append({'node': each.i, 'edge': each.dep_})

        for i in range(len(boundary_nodes) - 1):
            # Add connection between neighboring dependency trees
            dep_tree[boundary_nodes[i]].append({'node': boundary_nodes[i] + 1, 'edge': 'neigh'})
            dep_tree[boundary_nodes[i] + 1].append({'node': boundary_nodes[i], 'edge': 'neigh'})
        edges = []
        for key, values in dep_tree.items():
            for value in values:
                edges.append((value['edge'], key, value['node']))
        if edges:
            des_graph = {'backbone_sequence': g_features, 'edges': edges}
        else:
            des_graph = {}
    except:
        des_graph = {}
    return des_graph

def normalize_des_graph(des_graph):
    new_tokens = []
    new_edges = []
    nodes_to_subtokenize = {}
    mapping = {}
    count = 0
    for index, token in enumerate(des_graph['backbone_sequence']):
        subtokens = subtokenizer(token)
        if len(subtokens) > 1:
            for subtoken in subtokens:
                if index not in nodes_to_subtokenize.keys():
                    nodes_to_subtokenize[index] = [count]
                    mapping[index] = count
                else:
                    nodes_to_subtokenize[index].append(count)
                new_tokens.append(subtoken)
                count += 1
        else:
            mapping[index] = count
            new_tokens.extend(subtokens)
            count += 1
    for edge in des_graph['edges']:
        new_edges.append((edge[0].upper(), mapping[edge[1]], mapping[edge[2]]))
    for key in nodes_to_subtokenize.keys():
        for index in range(1, len(nodes_to_subtokenize[key])):
            new_edges.append(('SUB_TOKEN', nodes_to_subtokenize[key][index - 1], nodes_to_subtokenize[key][index]))
    for index in range(len(new_tokens) - 1):
        new_edges.append(('NEXT_TOKEN', index, index + 1))
    return {'backbone_sequence': new_tokens, 'edges': new_edges}

def clip_dot_graph(dot_file_path):
    nx_g = nx.drawing.nx_agraph.read_dot(dot_file_path)
    method_node_id = 0
    for node_id in nx_g.nodes():
        label = nx_g.nodes[node_id]['label']
        if label == 'MEMBERS':
            method_node_id = node_id
            break
        else:
            continue
    sub_graph_nodes = nx.algorithms.dag.descendants(nx_g, method_node_id)
    sub_graph_nodes = sorted(sub_graph_nodes, key=lambda i: int(i))
    sub_graph_nodes = sub_graph_nodes[:-1]
    return sub_graph_nodes

def save_sample_to_jsonl_gz(function, out_file):
    writer = codecs.getwriter('utf-8')
    writer(out_file).write(json.dumps(function))
    writer(out_file).write('\n')

def chunks(l, n):
    for i in range(0, len(l), n):
        yield l[i:i + n]

def post_base_process():
    files = os.listdir(JAVA_BASE_DIR)
    chunk_files = list(chunks(files, 100000))
    print('there are %d chunks' % len(chunk_files))
    process_list = []
    for index in range(len(chunk_files)):
        process = mp.Process(target=single_post_base_process,
                             args=(chunk_files[index], index))
        process_list.append(process)
        process.start()
    for process in process_list:
        process.join()
    # single_post_base_process(chunk_files[0], 0)

def single_post_base_process(files, index):
    count = 0
    if not os.path.exists(BASE_GRAPH_FILE):
        os.makedirs(BASE_GRAPH_FILE)
    with gzip.GzipFile(os.path.join(BASE_GRAPH_FILE, 'base_graph_gnn_' + str(index) + '.jsonl.gz'), 'wb') as gnn_file:
        for file in tqdm(files):
            class_name = file.split('_')[1]
            try:
                sub_graph_nodes = clip_dot_graph(
                    os.path.join(JAVA_BASE_DIR, file, class_name + '.java.dot'))
                with open(os.path.join(JAVA_BASE_DIR, file, class_name + '.java.json')) as reader:
                    graph = json.load(reader)
                with open(os.path.join(JAVA_BASE_DIR, file, class_name + '.json')) as reader:
                    raw_sample = json.load(reader)
                raw_sample['code_graph'] = normalize_graph(graph, sub_graph_nodes)
                if raw_sample['code_graph']:
                    save_sample_to_jsonl_gz(raw_sample, gnn_file)
                    count += 1
                del raw_sample
            except:
                continue
        print('there are %d samples in chunk %d' % (count, index))
        gnn_file.close()

if __name__ == '__main__':
    build_java_graphs()
    build_java_base_graphs()
    post_process()
    post_base_process()


from email.policy import default
from django.db import models
import uuid

from users.models import Profile
# Create your models here.
# For database tables

class Project(models.Model):
    owner = models.ForeignKey(
        Profile, null=True, blank=True, on_delete=models.CASCADE)

    title = models.CharField(max_length=200)
    description = models.TextField(null=True, blank=True)
    # For the project image
    featured_image = models.ImageField(
        null=True, blank=True, default="default.jpg")
    demo_link = models.CharField(max_length=2000, null=True, blank=True)
    source_link = models.CharField(max_length=2000, null=True, blank=True)
    created = models.DateTimeField(auto_now_add=True)
    id = models.UUIDField(default=uuid.uuid4, unique=True,
                          primary_key=True, editable=False)
    # Tags can be connected to multiple models # Added Model Tag inside
    tags = models.ManyToManyField('Tag', blank=True)

    vote_total = models.IntegerField(default=0, null=True, blank=True)
    vote_ratio = models.IntegerField(default=0, null=True, blank=True)

    def __str__(self):
        return self.title

    class Meta:
        ordering = ['-vote_ratio', '-vote_total','title']

    @property
    def imageURL(self):
        try:
            url = self.featured_image.url
        except:
            url = 'https://devsociety-bucket.s3.ap-south-1.amazonaws.com/default.jpg'
        return url

    @property
    def reviewers(self):
        queryset = self.review_set.all().values_list('owner__id', flat=True)
        # This is gonna give us the ids of people that have reviewed
        return queryset

    @property
    def getVoteCount(self):
        reviews = self.review_set.all()
        upVotes = reviews.filter(value="up").count()
        totalVotes = reviews.count()  # Its gonna count the number of reviews in the query set
        ratio = (upVotes/totalVotes)*100

        self.vote_total = totalVotes
        self.vote_ratio = ratio

        self.save()

class Review(models.Model):
    VOTE_TYPE = (('up', 'Up Vote'), ('down', 'Down Vote')
                 )
    # CASCADE will delete all the reviews if project is deleted
    owner = models.ForeignKey(Profile, on_delete=models.CASCADE, null=True)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    body = models.TextField(null=True, blank=True)
    value = models.CharField(max_length=200, choices=VOTE_TYPE)
    created = models.DateTimeField(auto_now_add=True)
    id = models.UUIDField(default=uuid.uuid4, unique=True,
                          primary_key=True, editable=False)

    class Meta:
        unique_together = [['owner', 'project']]

    def __str__(self):
        return self.value

class Tag(models.Model):
    name = models.CharField(max_length=200)
    created = models.DateTimeField(auto_now_add=True)
    id = models.UUIDField(default=uuid.uuid4, unique=True,
                          primary_key=True, editable=False)

    def __str__(self):
        return self.name


import json
from typing import Any

from django.utils.translation import get_language
from rest_framework import serializers
from rest_framework.utils.serializer_helpers import ReturnDict

from my_apps.shop.models import Banner, BasketItem, Category, Faq, Order, Product, Review

class CategorySerializer(serializers.ModelSerializer):
    sub = serializers.SerializerMethodField()
    icon = serializers.ImageField(source="img_small")
    url = serializers.CharField(source="slug")

    class Meta:
        model = Category
        fields = [
            "id",
            "url",
            "name",
            "icon",
            "img",
            "sub",
        ]

    def get_sub(self, obj) -> ReturnDict[Any, Any]:
        serializer = CategorySerializer(obj.get_sub_categories(), context=self.context, many=True)
        return serializer.data

class ProductCatalogSerializer(serializers.ModelSerializer):
    category = serializers.CharField(source="get_category_name")

    class Meta:
        model = Product
        fields = [
            "id",
            "name",
            "quantity",
            "img",
            "category",
            "price",
            "discount",
            "global_rating",
        ]

class ProductCardSerializer(serializers.ModelSerializer):
    category = serializers.CharField(source="get_category_name")
    code = serializers.CharField(source="slug")
    img = serializers.SerializerMethodField()
    # faq = serializers.CharField(source="get_faq_list")

    class Meta:
        model = Product
        fields = [
            "id",
            "name",
            "category",
            "price",
            "discount",
            "img",
            "quantity",
            "code",
            "global_rating",
            "description",
            # "faq",
            "rate_by_criteria",
            "rate_by_stars",
        ]

    def get_reviews(self, obj):
        serializer = ReviewSerializer(obj.get_rewievs(), context=self.context, many=True)
        return serializer.data

    #
    # def get_faq(self, obj) -> str:
    #     list_faq: list = obj.get_list_faq()
    #     return json.dumps(list_faq)

    def get_img(self, obj):
        request = self.context.get("request")
        domain = request.build_absolute_uri("/")

        list_foto = obj.images()
        list_link = []
        for image in list_foto:
            list_link.append(domain + image)
        return list_link

class OrderSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Order
        fields = ["id", "customer", "manager", "status", "order_date", "total_amount"]

    status = serializers.CharField(source="get_status_display")

class CreateReviewSerializer(serializers.ModelSerializer):
    class Meta:
        model = Review
        fields = ["text", "date"]

    def save(self, owner, product):
        text = self.validated_data["text"]
        rewiev = Review(text=text, author=owner, product=product)
        rewiev.save()
        return rewiev

class ReviewSerializer(serializers.ModelSerializer):
    author = serializers.CharField(source="get_user_name")

    class Meta:
        model = Review
        fields = ["id", "author", "text", "rate_by_stars", "rate_by_criteria"]

class BannerSerializer(serializers.ModelSerializer):
    class Meta:
        model = Banner
        fields = ["id", "img", "mobileImg", "link"]

class BasketSerializer(serializers.Serializer):
    product_id = serializers.UUIDField()
    amount = serializers.IntegerField()

class BasketItemSerializer(serializers.ModelSerializer):
    class Meta:
        model = BasketItem
        fields = ["product", "quantity"]

class OrderIdSerializer(serializers.Serializer):
    order_id = serializers.UUIDField()


import sys
from itertools import combinations

N = int(sys.stdin.readline())

area = [i for i in range(1,N+1)]
human_num = dict()
num = list(map(int,sys.stdin.readline().rstrip().split()))
for i in range(1,N+1) :
    human_num[i] = num[i-1]

node = dict()
visited = [0] * (N+1)
for i in range(1,N+1) :
    line = list(map(int,sys.stdin.readline().rstrip().split()))
    temp = []
    for j in range(line[0]) :
        temp.append(line[j+1])
    node[i] = temp
minVal = float('inf')

def dfs(x,arr) :
    global flag
    visited[x] = 1
    # print(x,arr,visited)
    if visited.count(1) == len(arr) :
        # print(x, arr, visited)
        flag = True
        return
    else :
        for n in node[x] :
            if n in arr and visited[n] == 0:
                visited[n] = 1
                dfs(n,arr)

for n in range(1,N+1) :
    com = list(combinations(area,n))
    for c in com :
        one, two = [], []
        for i in c :
            one.append(i)
        for a in area :
            if a not in one :
                two.append(a)
        visited = [0] * (N+1)
        flag = False
        dfs(one[0], one)
        if flag == True :
            visited = [0] * (N + 1)
            flag = False
            if len(two) > 0 :
                dfs(two[0], two)

        if flag == True :
            cnt1, cnt2 = 0, 0
            for o in one:
                cnt1 += human_num[o]
            for t in two:
                cnt2 += human_num[t]
            minVal = min(minVal, (abs(cnt1 - cnt2)))
if minVal == float('inf') :
    print(-1)
else :
    print(minVal)


# -*- coding: utf-8 -*-

"""
This module provides functions for several types of feedback

mode 0: Kitaev phase estimation. arxiv: quant-ph/9511026
mode 1: Adaptive repeated phase estimation PRA 63, 053804 (2001)
mode 2: Feedback off, Returns 0.
mode 3: returns a random number depending on m, x

More information can be found in arxiv: 1506.05033 (our paper) or my
M.Sc. thesis

Algorithm for Arpe:
    Following PRA 63, 053804 (2001) (readable summeries in our paper or my
    thesis), we optimize the Holevo variance of P(theta|n_m) where theta is
    the eigenvalue estimate of the measurement given the bit-string
    of results n_m.

    The probability distribution given result n_m is computed in P(m, x, mode)
    The function _S_optimize(phi, m, x, mode) computes the Holevo variance of
    and then averages over P(m+1, x, mode) and P(m+1, x x + 2^(m+1), mode),
    (in this example x < 2^m).
    Then, the optimal feedback is found in feedback_function(m, x, mode=1)
    using scipy.optimize.
    The function feedback(m, x, mode) is a wrapper to save time, as the
    feedback depends only on previous results, but x is the current result
    (thus holds e.g. feedback(1,0,mode)==feedback(1,1,mode) and so on)

    The wave function in u, v basis after the protocol can be obtained
    analytically and is implemented in psiv_noc(v, m, x, mode).
    The optimal estimate for the eigenvalue can be obtained by an integral over
    P(m, x, mode), this is done in estimate(m, x, mode)
    Using this estimate, the center of the wave function is shifted to v=0 in
    psiv(v, m, x, mode).

    With the corrected wave function, the probability to obtain a good codeword
    is obtained in P_no_error(m, x, mode), which integrates over psiv() in the
    interval v \in (-\sqrt{\pi}/6,\sqrt{\pi}/6).

    If the module is called as script, P_no_error() is evaluated for all 0<m<9,
    all x and all modes, the results are shown as a histogram.

Conventions:
    n_m: List of the binary measurement results. Entry m is the result of the
            m-th round.
    m: All functions take a round m as argument. This truncates the bit-string
        n_m, to the first m entries.
        e.g. P(m=1,x=1,mode) = P(m=1, x=7, mode)
    x: Some functions are memoized. For efficient memoizing, a function cannot
        take lists as input. Thus, the list n_m is converted to the integer x
"""

if __name__ == '__main__':
    import matplotlib
    matplotlib.use('Qt4Agg')

from scipy.special import binom
import numpy as np
import scipy.integrate
import scipy.optimize as opt
import time
import matplotlib.pylab as plt
import sys
import itertools

import unittest

from misc import Memoize
from qutip import *

# import test_data

M = 10
theta = np.linspace(0, 2*np.pi, 2**M+1)

# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# TEST

class TestFeedback(unittest.TestCase):
    def test_feedback(self):
        for m in np.arange(1, 5):
            for x in np.arange(2**(m-1)):
                self.assertEqual(feedback(m, 2*x, 3), feedback(m, 2*x+1, 3))

    def test_feedback_function_rpe(self):
        self.assertEqual(feedback_function(2, 1, 0), 0)
        self.assertEqual(feedback_function(3, 1, 0), np.pi/2)
        for m in np.arange(1, 5):
            for x in np.arange(2**m):
                self.assertEqual(feedback_function(m, x, 0),
                                 np.mod(m*np.pi/2, np.pi))

    def test_feedback_function_arpe(self):
        self.assertEqual(feedback_function(0, 0, 1), 0)
        self.assertAlmostEqual(
            feedback_function(1, 1, 1),
            feedback_function(1, 0, 1),
            places=5
            )

        self.assertAlmostEqual(
            feedback_function(1, 0, 1),
            4.712388950181709,
            places=5
            )
        for m in np.arange(1, 5):
            for x in np.arange(2**m):
                self.assertAlmostEqual(
                    feedback_function(m, x, 1),
                    test_data.feedback_data[m-1][x],
                    places=5
                    )

    def test_feedback_function_off(self):
        for m in np.arange(1, 5):
            for x in np.arange(2**m):
                self.assertEqual(feedback(m, x, 2), 0)

    def test_feedback_function_random(self):
        for m in np.arange(5):
            for x in np.arange(2**m):
                self.assertEqual(feedback_function(m, x, 3), random_list[m, x])

    def test__distrib(self):
        for x in np.arange(2**4):
            for j in np.arange(5):
                self.assertAlmostEqual(
                    np.abs(_distrib(4, x, j, 1)),
                    np.abs(test_data.distrib_data[x, j]),
                    places=5)

# class TestStringMethods(unittest.TestCase):

# def test_upper(self):
    # self.assertEqual('foo'.upper(), 'FOO')

# def test_isupper(self):
    # self.assertTrue('FOO'.isupper())
    # self.assertFalse('Foo'.isupper())

# def test_split(self):
    # s = 'hello world'
    # self.assertEqual(s.split(), ['hello', 'world'])
    # check that s.split fails when the separator is not a string
    # with self.assertRaises(TypeError):
        # s.split(2)

# if __name__ == '__main__':
    # unittest.main()

# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# FUNCTIONS

def shorten(m, n_m):
    return np.take(n_m, np.arange(m))

# Checked, OK
@Memoize
def P(m, n_m, mode):
    """
    Compute the probability P(theta|x) of initial phase theta given results x.

    Args:
        m: Integer, current round of the protocol
        x: Integer, measurement results of the whole protocol
        mode: Integer, mode for feedback

    Returns:
        Array P(theta|x)
    """
    if m == 0:
        return 1
    n_m = shorten(m, n_m)
    p = P(m-1, n_m, mode) *\
        np.cos(0.5*(np.pi*n_m[m-1] + feedback(m, n_m, mode)+theta))**2
    return p

# phase estimate of result x at round m
# Checked
@Memoize
def estimate(m, n_m, mode):
    """
    Compute the estimate of the protocol given results x.

    Uses the algorithm of Berry, Wiseman, Breslin, PRA 63, 053804 (2001).
    More information can be found in arxiv: 1506.05033 (our paper) or my
    M.Sc. thesis

    Args:
        m: Integer, current round of the protocol
        x: Integer, measurement results of the whole protocol
        mode: Integer, mode for feedback

    Returns:
        Float, best estimate given the results
    """

    # Note that the numpy doc on np.angle is wrong,
    # the angle is in the interval (-pi,pi).
    n_m = shorten(m, n_m)
    p = P(m, n_m, mode)*np.exp(1j*theta)
    return np.angle(scipy.integrate.romb(p, 2*np.pi/(2**M + 1)))

# -----------------------------------------------------------------------------
# FEEDBACK

# Functions needed to obtain the feedback in various protocols are defined here
# An explanation of the protocols is found in the doc of feedback_function

def feedback(m, n_m, mode):
    """
    wrapper for feedback_function, saves some time (f(1,0) = f(1,1) etc.)

    the feedback does not depend on the last measurement result, as it is
    applied before the last measurement. This wrapper helps with memoization.

    Args:
        m: Integer, current round of the protocol
        x: Integer, measurement results of the whole protocol
        mode: Integer, mode for feedback

    Returns:
        Float, feedback
    """
    n_m = shorten(m-1, n_m)
    return feedback_function(m-1, n_m, mode)

# Checked, seems good
@Memoize
def feedback_function(m, n_m, mode):
    """
    Compute feedback depending on mode (rpe, arpe, off). wrapped by feedback()

    mode 0: Kitaev phase estimation. arxiv: quant-ph/9511026
    mode 1: Adaptive repeated phase estimation PRA 63, 053804 (2001)
    mode 2: Feedback off, Returns 0.
    mode 3: returns a random number depending on m, x

    More information can be found in arxiv: 1506.05033 (our paper) or my
    M.Sc. thesis

    Args:
        m: Integer, current round of the protocol
        x: Integer, measurement results of the whole protocol
        mode: Integer, mode for feedback

    Returns:
        Float, feedback
    """
    if mode in (0, 'rpe'):
        return np.mod(m*np.pi/2, np.pi)

    elif mode in (1, 'arpe'):
        if m == 0:
            # if no measurement has been made, any feedback is ok. Choose 0.
            return 0
        return opt.fminbound(_S_optimize, 0, 2*np.pi, args=(m, n_m, mode))

    elif mode in (2, 'off'):
        return 0

    elif mode in (3, 'random'):
        # Memoization ensures that multiple calls with same args yield same res
        return np.random.random_sample()

# TODO uses global var M
def _S_optimize(phi, m, n_m, mode):
    """
    Form Average of the Holevo variance over results u=0,1

    Helper function for adaptive repeated phase estimation.
    More information on the algorithm can be found in arxiv/1506.05033 (our
    paper), or Berry, Wiseman, Breslin, PRA 63, 053804 (2001).

    Args:
        phi: float, 'true' phase that is measured
        m: Integer, current round of the protocol
        x: Integer, measurement results of the whole protocol
        mode: Integer, mode for feedback

    Returns:
        Float, feedback
    """
    S_sum = np.float(0)
    for u in np.arange(2):
        S = np.exp(1j*theta)*P(m, n_m, mode)*np.cos(0.5*(np.pi*u+phi+theta))**2
        S_sum += np.abs(scipy.integrate.romb(S, 2*np.pi/(2**M + 1)))
    return -S_sum

# -----------------------------------------------------------------------------
# STATE

def psiv(v, m, n_m, mode):
    """
    Return the corrected state after the idealized protocol u, v basis.

    Return the state in u,v basis after applying a correcting shift obtained
    by estimate().
    The function assumes a delta-peak as initial state (u=0). This can be done
    as the errors in u and v commute and u is computed with other means.
    This assumption is tested and holds.

    Args:
        v: Linspace in interval (-np.sqrt(np.pi)/6, np.sqrt(np.pi)/6),
            shift error v.
        m: Integer, current round of the protocol
        x: Integer, measurement results of the whole protocol
        mode: Integer, mode for feedback

    Returns:
        Array, Abs**2 of the wavefunction for the values of v
    """

    # The estimate is added, as we measure -v, where v is the shift (see paper)
    n_m = shorten(m, n_m)
    vc = v + estimate(m, n_m, mode)/(2*np.sqrt(np.pi))
    return psiv_noc(vc, m, n_m, mode)

def psiv_noc(v, m, n_m, mode):
    """
    Return the uncorrected state after the idealized protocol u, v basis.

    Return the state in u,v basis.
    The function assumes a delta-peak as initial state (u=0). This can be done
    as the errors in u and v commute and u is computed with other means.
    This assumption is tested and holds.

    Args:
        v: Linspace in interval (-np.sqrt(np.pi)/6, np.sqrt(np.pi)/6),
            shift error v.
        m: Integer, current round of the protocol
        x: Integer, measurement results of the whole protocol
        mode: Integer, mode for feedback

    Returns:
        Array, Abs**2 of the wavefunction for the values of v
    """
    n_m = shorten(m, n_m)
    psi = np.zeros(len(v), dtype=np.complex_)
    for s in np.arange(m+1):
        psi += np.exp(1j*np.sqrt(np.pi)*2*s*v)*_distrib(m, n_m, s, mode)
    return psi/_norm(m, n_m, mode)

# Checked, seems good
# checked second time, in depth. algorithm is OK.
@Memoize
def _distrib(m, n_m, s, mode):
    """
    Obtain distribution of squeezed states forming a code state defined by x

    More information can be found in arxiv: 1506.05033 (our paper) or my
    M.Sc. thesis

    Args:
        m: Integer, current round of the protocol
        x: Integer, measurement results of the whole protocol
        s: Integer, running index of the squeezed state
        mode: Mode (rpe, arpe, off) for feedback integer 0,1 or 2

    Returns:
        Complex, Weight of the squeezed state s in a code state
    """
    if s == 0:
        return 1
    n_comb = np.int(binom(m, s))
    phases = np.zeros(m, dtype=np.complex_)
    for i in np.arange(m):
        phases[i] = feedback(i+1, n_m, mode) + n_m[i]*np.pi
    t = itertools.chain.from_iterable(itertools.combinations(phases, s))
    t = np.fromiter(t, dtype=np.complex_, count=n_comb*s)
    t.resize((n_comb, s))
    t = np.exp(1j*t.sum(axis=1))
    return t.sum()

def _norm(m, n_m, mode):
    """
    Approximate the norm of a code state after m measurements with result x

    The function assumes that the squeezed states are approximately orthogonal.
    If a codeword is even slightly useful, this assumption is good to 10^-9.

    Args:
        m: Integer, current round of the protocol
        x: Integer, measurement results of the whole protocol
        mode: Mode (rpe, arpe, off) for feedback integer 0,1 or 2

    Returns:
        Float, norm of the state
    """
    nrm = 0
    for s in np.arange(m+1):
        nrm += np.abs(_distrib(m, n_m, s, mode))**2
    return np.sqrt(nrm*np.sqrt(np.pi))

# -----------------------------------------------------------------------------
# ANALYSIS

@Memoize
def P_no_error(m, n_m, mode):
    """
    Compute the probability to obtain a good code state given m, x and mode

    Args:
        m: Integer, current round of the protocol
        x: Integer, measurement results of the whole protocol
        mode: Mode (rpe, arpe, off) for feedback integer 0,1 or 2

    Returns:
        Float, probability
    """
    n_m = shorten(m, n_m)
    v = np.linspace(-np.sqrt(np.pi)/6, np.sqrt(np.pi)/6, 2**M + 1)
    psi = np.abs(psiv(v, m, n_m, mode))**2
    return scipy.integrate.romb(psi, np.sqrt(np.pi)/(3*(2**M + 1)))

# -----------------------------------------------------------------------------
# MISC

def _parallel_helper(n_m, m, mode):
    # feedback_function(m, x, mode)
    # estimate(m, x, mode)
    return 1 - P_no_error(m, n_m, mode)

def result_range(Min, Max, m):
    """Return range min-max as bit strings of length m"""
    return np.arange(Min, Max)[:, np.newaxis] >> np.arange(m)[::-1] & 1

# -----------------------------------------------------------------------------
# -----------------------------------------------------------------------------
# COMPUTE HISTOGRAM

if __name__ == '__main__':
    # unittest.main(verbosity=2)

    print("Simulating feedback without weighing on ideal code")
    Mmax = 9

    numBins = 500
    plotcut = 4
    plotList = [4, 5, 6, 7, 8]

    # initialize
    plt.close()

    delta_theta = np.zeros((Mmax, 2**Mmax))
    hist_array = np.zeros((4, Mmax, numBins))
    bin_array = np.zeros((4, Mmax, numBins+1))

    # compute
    for mode in np.arange(4):
        mode_name = ['rpe', 'arpe', 'off', 'random'][mode]
        P_array = [np.zeros(2**m) for m in np.arange(1, Mmax+1)]
        for m in np.arange(1, Mmax+1):
            # print(m)
            result_list = result_range(0, 2**m, m)
            P_array[m-1] = serial_map(_parallel_helper, result_list,
                                      task_args=(m, mode))
            hist, bins = np.histogram(np.clip(P_array[m-1], 0, 0.2),
                                      numBins, range=(0, 1), density=True)
            hist_array[mode, m-1] = hist/numBins
            bin_array[mode, m-1] = bins
        print(mode_name+' done')
    print('computing done')
    hist_array = hist_array.transpose(1, 0, 2)
    bin_array = bin_array.transpose(1, 0, 2)

    # initialize plotting
    fig, ax_array = plt.subplots(len(plotList), 4, sharex='col', sharey='row')
    fig.set_size_inches(8.75*2, 11)
    plt.rc('font', size=14, **{'family': 'sans-serif',
                               'sans-serif': ['Helvetica']})
    plt.rc('text', usetex=True)
    ylabels = [r"${0}$".format(0.1*x) for x in range(11)]
    xlabels = [r"${0}$".format(0.05*x) for x in range(5)]
    xlabels[-1] = r'$> 0.2$'

    # plot
    for mode in np.arange(4):
        for i, m in enumerate(plotList):
            bins = bin_array[m-1][mode]
            hist = hist_array[m-1][mode]
            widths = np.diff(bins)
            widths[100] = 0.03
            ax = ax_array[i][mode]
            ax.bar(bins[:-1], hist, widths, color="None")
            ax.set_xlim([0, 0.21])
            plt.xticks(0.05 * np.arange(5))
            plt.yticks(0.1 * np.arange(11))
            ax.set_xticklabels(xlabels)
            ax.set_yticklabels(ylabels)
            ax.set_ylim([0, 0.6])
            if mode == 0:
                ax.set_ylabel(r'$P$', rotation='horizontal')
                ax.yaxis.labelpad = 20
            if mode == 3:
                ax.yaxis.set_label_position("right")
                ax.set_ylabel(str(m), fontsize=16, rotation='horizontal')
                ax.yaxis.labelpad = 31
            if m == max(plotList):
                ax.set_xlabel(r'$P_{\mathrm{error,p}}^{\sqrt{\pi}/6}$')
    ax_array[0][0].set_title(r'RPE', fontsize=20)
    ax_array[0][1].set_title(r'ARPE', fontsize=20)
    ax_array[0][2].set_title(r'OFF', fontsize=20)
    ax_array[0][3].set_title(r'RAND', fontsize=20)
    plt.suptitle(r'M', fontsize=20, x=0.95, y=0.92)
    # fig.savefig('histograms2.eps')
    plt.show()