import os
import sys
import traceback
import pdb
import re
import numpy as np
import pandas as pd
from math import log
from collections import Counter
from functools import reduce
from operator import add
from multiprocessing import Pool

# directory name: {dataset}_{search-l}_{type}_{# of the shard}
# usage "python anss_jh_breakdown-parser.py [log directory] [# of the thread] [query depth] [# of the query]"

OUTLIER_THRESHOLD = 50000000
# OUTLIER_THRESHOLD = 20000000

DIRECTORY = sys.argv[1]
if DIRECTORY[-1] == '/':
    DIRECTORY = DIRECTORY[:-1]
THREAD_NUM = int(sys.argv[2])
QUERY_DEPTH = int(sys.argv[3])
QUERY_NUM = int(sys.argv[4])
# NDP_NUM = int(sys.argv[4])
if len(sys.argv) == 7:
    TYPE = sys.argv[5]
else:
    TYPE = ""

FILE = 'debug.log'
TERMINAL_FILE = 'system.terminal'
POSTFIX = "-BREAKDOWN"

TYPE_LIST = [
    'disk',
    'distributed',
    'base',
    'ndp',
    'cache',
    'nearest',
    'infinite',
]

NDP_LIST = [
    'ndp',
    'cache',
    'nearest',
]

COLUMNS = [
    "update",
    "calculation",
    "graph",
    "query vector",
    "sq entry set",
    "doorbell",
    "sq dma",
    "cq dma",
    "polling",
    "accumulation",
    "merge",
    "network",
    "total",
    "submission",
]

RDMA_PATTERN = r'([\d \+]* \d*)(?= B)'
RDMA_REGEX = re.compile(RDMA_PATTERN)

ACCURACY_PATTERN = r'([0-9]*[.]?[0-9]+) ([+-]?([0-9]*[.])?[0-9]+) [+-]?([0-9]*[.])?[0-9]+ [+-]?([0-9]*[.])?[0-9]+'
ACCURACY_REGEX = re.compile(ACCURACY_PATTERN)

DATASET_PATTERN = r'(?<=datasetDir: ")[\s\S]*\/(\w*)(?=")'
DATASET_REGEX = re.compile(DATASET_PATTERN)

def getInitDict():
    _tmp_dict = {}
    for column in COLUMNS:
        _tmp_dict[column] = 0
    return _tmp_dict

OUT_COLUMNS = ["dataset", "search-L", "type"]
OUT_COLUMNS.extend(COLUMNS)
OUT_COLUMNS.extend(["end-to-end", "accuracy", "num query", "num ndp"])

MAGIC_WORDS = ['P', 'Q', 'm5_print']

INITIAL_VALUE = -1

### point ###
# host
SET_START = 'set_start'
QUERY_SET = 'query_set'
DISTANCE_START = 'distance_start'
SEND_DOORBELL_DONE = 'send_doorbell_done'
POLLING_END_DONE = 'polling_end_done'
DISTANCE_END = 'distance_end'
UPDATE_END = 'update_end'
TRAVERSE_END = 'traverse_end'
SET_END = 'set_end'
# device
SEND_DOORBELL = 'send_doorbell'
RECV_DOORBELL = 'recv_doorbell'
RECV_SQ = 'recv_sq'
GET_QUERY_VECTOR_START = 'get_query_vector_start'
COMPUTATION_START = 'computation_start'
ALLOCATE_UNIT = 'allocate_unit'
DEALLOCATE_UNIT = 'deallocate_unit'
COMPUTATION_END = 'computation_end'
CQ_DMA_END = 'cq_dma_end'
POLLING_END = 'polling_end'
# distributed
DISTRIBUTED_MASTER_SEND = 'distributed_master_send'
DISTRIBUTED_SLAVE_RECV = 'distributed_slave_recv'
DISTRIBUTED_SLAVE_SEND = 'distributed_slave_send'
DISTRIBUTED_MASTER_RECV = 'distributed_master_recv'
DISTRIBUTED_POSTPROCESSING = 'distributed_postprocessing'
DISTRIBUTED_MERGE_END = 'distributed_end'

DISTRIBUTED_POINT_LIST = [
    DISTRIBUTED_MASTER_SEND,
    DISTRIBUTED_SLAVE_RECV,
    DISTRIBUTED_SLAVE_SEND,
    DISTRIBUTED_MASTER_RECV,
    DISTRIBUTED_POSTPROCESSING,
    DISTRIBUTED_MERGE_END,
]

class OneQuery:
    def __init__(self, type, thread_id, query_index, ndp_num=0, interface="dmaone"):
        self.type = type
        self.thread_id = thread_id
        self.query_index = query_index
        self.ndp_num = ndp_num
        self.interface = interface

        self.doorbell_counter = 0
        self.first_doorbell = 0
        self.cq_dma_counter = 0
        self.polling_counter = 0
        self.timestamps = {
            SET_START: 0,
            QUERY_SET: 0,
            DISTANCE_START: 0,
            SEND_DOORBELL: 0,
            SEND_DOORBELL_DONE: 0,
            RECV_DOORBELL: 0,
            RECV_SQ: 0,
            GET_QUERY_VECTOR_START: 0,
            COMPUTATION_START: 0,
            COMPUTATION_END: 0,
            CQ_DMA_END: 0,
            POLLING_END: 0,
            POLLING_END_DONE: 0,
            DISTANCE_END: 0,
            UPDATE_END: 0,
            TRAVERSE_END: 0,
            SET_END: 0,
        }
        self.ticks = getInitDict()

    def one_hop_reset(self):
        self.doorbell_counter = 0
        self.first_doorbell = 0
        self.cq_dma_counter = 0
        self.polling_counter = 0
        self.timestamps = {
            SET_START: 0,
            QUERY_SET: 0,
            DISTANCE_START: 0,
            SEND_DOORBELL: 0,
            SEND_DOORBELL_DONE: 0,
            RECV_DOORBELL: 0,
            RECV_SQ: 0,
            GET_QUERY_VECTOR_START: 0,
            COMPUTATION_START: 0,
            COMPUTATION_END: 0,
            CQ_DMA_END: 0,
            POLLING_END: 0,
            POLLING_END_DONE: 0,
            DISTANCE_END: 0,
            UPDATE_END: 0,
            TRAVERSE_END: 0,
            SET_END: 0,
        }

    def query_reset(self):
        self.one_hop_reset()
        self.ticks = getInitDict()

    def update_ticks(self, timestamp):
        if self.type in NDP_LIST:
            if self.timestamps[SET_START] != 0:
                assert self.timestamps[QUERY_SET] != 0
                if self.interface == 'dmaone' or self.interface == 'dma':
                    assert self.timestamps[GET_QUERY_VECTOR_START] != 0
                    self.ticks["query vector"] += self.timestamps[COMPUTATION_START] - self.timestamps[GET_QUERY_VECTOR_START]
                self.ticks["query vector"] += self.timestamps[QUERY_SET] - self.timestamps[SET_START]

                self.ticks["graph"] += self.timestamps[DISTANCE_START] - self.timestamps[QUERY_SET]
                self.ticks["update"] += timestamp - self.timestamps[DISTANCE_END]
            elif self.timestamps[TRAVERSE_END] == 0:
                self.ticks["graph"] += self.timestamps[DISTANCE_START] - self.timestamps[UPDATE_END]
                self.ticks["update"] += timestamp - self.timestamps[DISTANCE_END]
                # self.ticks["submission"] += self.timestamps[SEND_DOORBELL_DONE] - self.timestamps[DISTANCE_START]
            else:
                self.ticks["graph"] += self.timestamps[TRAVERSE_END] - self.timestamps[UPDATE_END]
                self.ticks["update"] += timestamp - self.timestamps[TRAVERSE_END]

            if self.interface == 'dmaone':
                if  self.timestamps[DISTANCE_START] < self.first_doorbell:
                    self.ticks["sq entry set"] += self.first_doorbell - self.timestamps[DISTANCE_START]
                    self.ticks["doorbell"] += self.timestamps[RECV_DOORBELL] - self.first_doorbell
            else:
                self.ticks["sq entry set"] += self.timestamps[SEND_DOORBELL] - self.timestamps[DISTANCE_START]
                self.ticks["doorbell"] += self.timestamps[RECV_DOORBELL] - self.timestamps[SEND_DOORBELL]

            if self.interface != 'mmio':
                self.ticks["sq dma"] += self.timestamps[RECV_SQ] - self.timestamps[RECV_DOORBELL]

            self.ticks["calculation"] += self.timestamps[COMPUTATION_END] - self.timestamps[COMPUTATION_START]

            if self.cq_dma_counter != self.ndp_num:
                self.ticks["cq dma"] += self.timestamps[POLLING_END] - self.timestamps[COMPUTATION_END]
            else:
                self.ticks["cq dma"] += min(self.timestamps[CQ_DMA_END], self.timestamps[POLLING_END]) - self.timestamps[COMPUTATION_END]
                if (self.timestamps[POLLING_END] - min(self.timestamps[CQ_DMA_END], self.timestamps[POLLING_END]) < OUTLIER_THRESHOLD):
                    self.ticks["polling"] += self.timestamps[POLLING_END] - min(self.timestamps[CQ_DMA_END], self.timestamps[POLLING_END])
                if OUTLIER_THRESHOLD < self.timestamps[POLLING_END] - min(self.timestamps[CQ_DMA_END], self.timestamps[POLLING_END]):
                    print(f'{timestamp}: {self.timestamps[POLLING_END] - self.timestamps[CQ_DMA_END]}')

            if self.timestamps[DISTANCE_START] < self.first_doorbell:
                self.ticks["accumulation"] += self.timestamps[DISTANCE_END] - self.timestamps[POLLING_END]
        else:
            if self.timestamps[SET_START] != 0:
                self.ticks["graph"] += self.timestamps[DISTANCE_START] - self.timestamps[SET_START]
            else:
                self.ticks["graph"] += self.timestamps[DISTANCE_START] - self.timestamps[UPDATE_END]
            self.ticks["calculation"] += self.timestamps[DISTANCE_END] - self.timestamps[DISTANCE_START]
            self.ticks["update"] += timestamp - self.timestamps[DISTANCE_END]

        self.one_hop_reset()

    def error(self, log_line):
        print(log_line)
        traceback.print_tb()
        exit(1)

    def insert_timestamp(self, point, timestamp, log_line, row_list):
        if point == UPDATE_END or point == SET_END:
            self.update_ticks(timestamp)

        # increment doorbell counter
        if point == SEND_DOORBELL:
            if self.doorbell_counter == 0:
                self.first_doorbell = timestamp
            self.doorbell_counter += 1

        # increment CQ DMA counter
        if point == CQ_DMA_END and \
           self.timestamps[COMPUTATION_END] != 0 :
            self.cq_dma_counter += 1

        if point == POLLING_END:
            self.polling_counter += 1
        if point == TRAVERSE_END:
            self.polling_counter += self.ndp_num

        # insert timestamp
        self.timestamps[point] = timestamp

        if point == SET_END:
            self.ticks['total'] = sum(self.ticks.values())
            row_list.append(self.ticks)
            self.query_reset()

def update_distributed_tick(timestamp, point, distributed, network_memcpy, network_update, network_merge):
    if point == DISTRIBUTED_POSTPROCESSING:
        network_memcpy += distributed[DISTRIBUTED_SLAVE_RECV] - distributed[DISTRIBUTED_MASTER_SEND]
        network_memcpy += distributed[DISTRIBUTED_MASTER_RECV] - distributed[DISTRIBUTED_SLAVE_SEND]
        network_update += timestamp - distributed[DISTRIBUTED_MASTER_RECV]
    elif point == DISTRIBUTED_MERGE_END:
        network_merge += timestamp - distributed[DISTRIBUTED_POSTPROCESSING]
    else:
        print("Wrong point")
        exit(1)

    return (network_memcpy, network_update, network_merge)

def parsing(file_full_name):
    path = file_full_name[0]
    file = file_full_name[1]

    setting = path.split('/')[-1]
    if TYPE:
        type = TYPE
    else:
        type = setting.split('_')[-2]

    if type not in TYPE_LIST:
        print(f"wrong type: {type}")
        exit(1)
    elif type == 'nearest':
        query_depth = QUERY_DEPTH
    else:
        query_depth = 1

    if type in NDP_LIST:
        ndp_num = int(setting.split('_')[-1])
    elif type == "distributed":
        ndp_num = int(setting.split('_')[-1])
    else:
        ndp_num = 0

    # get end-to-end latency
    start_tick = 0
    end_tick = 0
    end_to_end = 0
    is_first = True

    dataset = ''
    accuracy = 0
    search_l = 0

    queries = []
    for i in range(THREAD_NUM):
        queries.append([])
        for j in range(query_depth):
            queries[i].append(OneQuery(type, i, j, ndp_num))

    # distributed
    distributed = {
        DISTRIBUTED_MASTER_SEND: 0,
        DISTRIBUTED_SLAVE_RECV: 0,
        DISTRIBUTED_SLAVE_SEND: 0,
        DISTRIBUTED_MASTER_RECV: 0,
        DISTRIBUTED_POSTPROCESSING: 0,
        DISTRIBUTED_MERGE_END: 0,
    }
    distributed_merge = 0
    distributed_update = 0
    network_memcpy = 0
    network_link = 0

    row_list = []
    output = setting + POSTFIX

    print(f'{path}/{file}')
    f = open(f'{path}/{file}')
    lines = f.read().splitlines()
    for line in lines:
        row = line.split(': ')
        magic_word = row[2]
        if magic_word not in MAGIC_WORDS:
            continue

        # distributed
        if magic_word == 'm5_print':
            rdma_matching = RDMA_REGEX.search(line)
            # 3.7 us per 512 bytes
            network_link += (eval(rdma_matching.group()) / 512 * 3700000)
            continue

        tick = int(row[0])
        point = row[-1]
        # dev_index = int(row[-2])
        query_index = int(row[-3])
        thread_id = int(row[-4])

        # print(line)

        # distributed
        if point in DISTRIBUTED_POINT_LIST:
            if point == DISTRIBUTED_POSTPROCESSING or point == DISTRIBUTED_MERGE_END:
                (network_memcpy, distributed_update, distributed_merge) = update_distributed_tick(tick, point, distributed, network_memcpy, distributed_update, distributed_merge)
            distributed[point] = tick
            continue

        queries[thread_id][query_index].insert_timestamp(point, tick, line, row_list)

        if point == SET_START and is_first:
            start_tick = tick
            is_first = False

        if point == SET_END:
            end_tick = tick

    end_to_end = end_tick - start_tick

    # get accuracy
    # get search-l
    terminal_file = open(os.path.join(path, TERMINAL_FILE))
    lines = terminal_file.read().splitlines()
    for line in lines:
        dataset_match = DATASET_REGEX.search(line)
        if dataset_match:
            dataset = dataset_match.group(1)
        accuracy_match = ACCURACY_REGEX.search(line)
        if accuracy_match:
            accuracy = float(accuracy_match.group(2))
            search_l = int(accuracy_match.group(1))

    df = pd.DataFrame(row_list, columns=COLUMNS).fillna(0).astype(int)

    # distributed
    if type == 'distributed':
        end_to_end += distributed_update + network_memcpy
        end_to_end //= ndp_num
        end_to_end += network_link + distributed_merge
        distributed_merge //= QUERY_NUM
        distributed_update //= (ndp_num * QUERY_NUM)
        network_memcpy //= (ndp_num * QUERY_NUM)
        network_link //= QUERY_NUM
        df['total'] += distributed_update + network_memcpy + network_link
        df['merge'] = distributed_merge + distributed_update
        df['network'] = network_memcpy + network_link

    # generate mean
    mean = df.mean().values.tolist()
    mean = [dataset, search_l, type] + mean
    mean.extend([end_to_end, accuracy, QUERY_NUM, ndp_num])

    df.to_csv(f'{DIRECTORY}/{output}.csv', index=False)
    f.close()

    return mean

if __name__ == '__main__':
    # pool input
    file_full_names = []
    for (path, dir, files) in os.walk(DIRECTORY):
        for file in files:
            if file != FILE:
                continue
            file_full_names.append((path, file))

    # multiprocessing
    with Pool(1) as p:
        means = p.map(parsing, file_full_names)

    output = DIRECTORY.split('/')[-1] + POSTFIX
    df = pd.DataFrame(means, columns=OUT_COLUMNS).fillna(0)
    df['search-L'] = pd.to_numeric(df['search-L'])
    df.sort_values(["type", "search-L"],
        axis=0,
        ascending=[True, True],
        inplace=True)
    print(df)

    df.to_csv(f'{DIRECTORY}/{output}.csv', index=False)
    # df.to_excel(f'{DIRECTORY}/{output}.xlsx', index=False)


import math

first_primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101]

def is_prime(num):
    if num < 2:
        return False
    if num in first_primes:
        return True
    elif num < 101:
        return False

    for p in first_primes:
        if num%p == 0:
            return False

    for i in range(101, int(math.sqrt(num))+1):
        if num%i == 0:
            return False

    return True


from django.shortcuts import render
from django.conf import settings
from django.contrib.auth.decorators import login_required

# import models
from apps.core.models.message_block import MessageBlock
from apps.core.models.todo_action_status import TodoActionStatus
from apps.core.models.todo import Todo
from apps.core.models.summary_log_end_users import SummaryLogEndUsers

# import views
from apps.core.views.vendor_common.login_user_info import *

@login_required
def index(request):
    user_obj = get_login_user_objects(request)
    vendor_branch_id = user_obj["vendor_branch"].id
    vendor_id = user_obj["vendor_branch"].vendor.id

    """ Analytics Index """
    BG_COLOR_PRESET_LIST = ["#1FB6E9", "#F15424", "#29C4A9", "#F6FB4E", "#242D3C", "#E2DADB", "#C52F72"]
    BG_PROGRESS_BAR_TYPE_LIST = ["bg-danger", "bg-info", "bg-success", "bg-primary", "bg-warning"]

    user_obj = get_login_user_objects(request)

    # Total Conversation Rate
    try:
        sql_total_conversion_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/total_conversation_rate.sql"
        sql_total_conversion_base = open(sql_total_conversion_file, 'r').read()
        sql_total_conversion = sql_total_conversion_base.format(vendor_id=str(vendor_id))
        print("sql_total_conversion ====================")
        print(sql_total_conversion)

        summary_log_end_user = SummaryLogEndUsers.objects.raw(sql_total_conversion)[0]
        total_conversion_rate = round(summary_log_end_user.conversion_rate, 2)
    except Exception as e:
        total_conversion_rate = 0

    # Total Visitors
    try:
        sql_total_visitor_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/total_visitors.sql"
        sql_total_visitor_base = open(sql_total_visitor_file, 'r').read()
        sql_total_visitor = sql_total_visitor_base.format(vendor_id=str(vendor_id))
        print("sql_total_visitor ====================")
        print(sql_total_visitor)
        summary_log_end_user = SummaryLogEndUsers.objects.raw(sql_total_visitor)[0]
        total_visitors = summary_log_end_user.visitor_cnt
    except Exception as e:
        total_visitors = 0
    print("★")
    print(total_visitors)

    # get Todos numbers
    todo_action_status = TodoActionStatus.objects.filter(is_delete=False).exclude(name='DONE')
    todo_action_status_list = []
    for status in todo_action_status:
        todo_action_status_list.append(status.id)

    todos = Todo.objects.filter(vendor_branch_id=user_obj["vendor_branch"].id, todo_action_status_id__in=todo_action_status_list, is_delete=False)
    todocount = todos.count()

    # Weekly Conversion
    sql_weekly_conversion_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/weekly_conversion_rate.sql"
    sql_weekly_conversion_base = open(sql_weekly_conversion_file, 'r').read()
    sql_weekly_conversion = sql_weekly_conversion_base.format(vendor_id=str(vendor_id))
    print("sql_weekly_conversion ====================")
    print(sql_weekly_conversion)
    summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_weekly_conversion)

    label_list = []
    data_list = []
    bg_color_list = []
    days_list = ["SUN", "MON", "TUE", "WEB", "THU", "FRI", "SAT"]

    # Should be 7 elements
    match_flg = 0
    for i in range(0, 7):
        label_list.append("\"" + days_list[i] + "\"")
        bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[i] + "\"")

        for cv in summary_log_end_users:
            match_flg = 0
            if i == int(cv.log_day):
                if cv.conversion_rate is not None:
                    data_list.append(cv.conversion_rate)
                else:
                    data_list.append(0)
                match_flg = 1
                break

        if match_flg == 0:
            data_list.append(0)

        print("data_list --------")
        print(data_list)

    w_cv_label_list_string = ','.join(map(str, label_list))
    w_cv_data_list_string = ','.join(map(str, data_list))
    w_cv_bg_color_list_string = ','.join(map(str, bg_color_list))

    w_cv_rate_chart = {
        "labels": w_cv_label_list_string,
        "data": w_cv_data_list_string,
        "bg_color": w_cv_bg_color_list_string,
    }

    # Weekly Visitors
    sql_weekly_visitors_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/weekly_visitors.sql"
    sql_weekly_visitors_base = open(sql_weekly_visitors_file, 'r').read()
    sql_weekly_visitors = sql_weekly_visitors_base.format(vendor_id=str(vendor_id))
    print("sql_weekly_visitors ====================")
    print(sql_weekly_visitors)
    summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_weekly_visitors)

    label_list = []
    data_list = []
    bg_color_list = []
    days_list = ["SUN", "MON", "TUE", "WEB", "THU", "FRI", "SAT"]

    # Should be 7 elements
    match_flg = 0
    for i in range(0,7):
        label_list.append("\"" + days_list[i] + "\"")
        bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[i] + "\"")

        for visitor in summary_log_end_users:
            match_flg = 0
            if i == int(visitor.log_day):
                data_list.append(visitor.weekly_visitor_cnt)
                match_flg = 1
                break

        if match_flg == 0:
            data_list.append(0)

    w_visitor_label_list_string = ','.join(map(str, label_list))
    w_visitor_data_list_string = ','.join(map(str, data_list))
    w_visitor_bg_color_list_string = ','.join(map(str, bg_color_list))

    w_visitor_rate_chart = {
        "labels": w_visitor_label_list_string,
        "data": w_visitor_data_list_string,
        "bg_color": w_visitor_bg_color_list_string,
    }

    # Device Rate > 1: mobile 2: tablet  3: pc  4: other
    sql_device_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/device_rate.sql"
    sql_device_rate_base = open(sql_device_file, 'r').read()
    sql_device_rate = sql_device_rate_base.format(vendor_id=str(vendor_id))
    print("sql_device_rate ====================")
    print(sql_device_rate)
    summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_device_rate)
    label_list = []
    data_list = []
    bg_color_list = []
    for device in summary_log_end_users:
        if device.device_type == 1:
            label_list.append("\"" + "Mobile" + "\"")
            bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[0] + "\"")

        elif device.device_type == 2:
            label_list.append("\"" + "Tablet" + "\"")
            bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[1] + "\"")

        elif device.device_type == 3:
            label_list.append("\"" + "PC" + "\"")
            bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[2] + "\"")

        else:
            label_list.append("\"" + "Other" + "\"")
            bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[3] + "\"")

        data_list.append(device.os_family_cnt)

    # Convert List to String
    device_label_list_string = ','.join(map(str, label_list))
    device_bg_color_list_string = ','.join(map(str, bg_color_list))
    device_data_list_string = ','.join(map(str, data_list))

    device_rate_chart = {
        "labels": device_label_list_string,
        "data": device_data_list_string,
        "bg_color": device_bg_color_list_string,
    }
    print("device_rate_chart ==== ")
    print(device_rate_chart)

    # OS Rate
    sql_os_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/os_rate.sql"
    sql_os_rate_base = open(sql_os_file, 'r').read()
    sql_os_rate = sql_os_rate_base.format(vendor_id=str(vendor_id))
    print("sql_os_rate_base ====================")
    print(sql_os_rate)
    summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_os_rate)
    label_list = []
    data_list = []
    bg_color_list = []

    for i, os in enumerate(summary_log_end_users):
        label_list.append("\"" + os.os_family + "\"")
        data_list.append(os.os_family_cnt)
        bg_color_index = i % 6
        bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[bg_color_index] + "\"")

    # Convert List to String
    os_label_list_string = ','.join(map(str, label_list))
    os_data_list_string = ','.join(map(str, data_list))
    os_bg_color_list_string = ','.join(map(str, bg_color_list))

    os_rate_chart = {
        "labels": os_label_list_string,
        "data": os_data_list_string,
        "bg_color": os_bg_color_list_string,
    }

    print("os_rate_chart ==== ")
    print(os_rate_chart)

    # Browser Rate
    sql_browser_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/browser_rate.sql"
    sql_browser_rate_base = open(sql_browser_file, 'r').read()
    sql_browser_rate = sql_browser_rate_base.format(vendor_id=str(vendor_id))
    print("sql_browser_rate ====================")
    print(sql_browser_rate)
    summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_browser_rate)
    label_list = []
    data_list = []
    bg_color_list = []

    for i, browser in enumerate(summary_log_end_users):
        label_list.append("\"" + browser.browser_family + "\"")
        data_list.append(browser.browser_family_cnt)
        bg_color_index = i % 6
        bg_color_list.append("\"" + BG_COLOR_PRESET_LIST[bg_color_index] + "\"")

    # Convert List to String
    browser_label_list_string = ','.join(map(str, label_list))
    browser_data_list_string = ','.join(map(str, data_list))
    browser_bg_color_list_string = ','.join(map(str, bg_color_list))

    browser_rate_chart = {
        "labels": browser_label_list_string,
        "data": browser_data_list_string,
        "bg_color": browser_bg_color_list_string,
    }

    print("browser_rate_chart ==== ")
    print(browser_rate_chart)

    # Conversation_Percent
    sql_conversation_rate_file = settings.BASE_DIR + "/apps/core/views/vendor_contactchat/statistics_sql/conversation_rate.sql"
    sql_conversation_rate_base = open(sql_conversation_rate_file, 'r').read()

    conversation_bar_list = []
    message_blocks = MessageBlock.objects.filter(vendor_branch_id=vendor_branch_id, is_delete=False).all()
    for message_block in message_blocks:
        tmp_bar_dict = {
            "message_block_name": message_block.admin_text,
            "progress_numerator_list": [],
            "progress_denominator_list": []
        }

        sql_conversation_rate = sql_conversation_rate_base.format(vendor_id=str(vendor_id), message_block_id=str(message_block.id))
        print("debug: sql_conversation_rate")
        print(sql_conversation_rate)
        summary_log_end_users = SummaryLogEndUsers.objects.raw(sql_conversation_rate)

        for cv in summary_log_end_users:
            tmp_bar_dict["progress_numerator_list"].append(int(cv.progress_numerator))
            tmp_bar_dict["progress_denominator_list"].append(int(cv.progress_denominator))

        conversation_bar_list.append(tmp_bar_dict)

    print("conversation_bar_list =========")
    print(conversation_bar_list)

    conversation_rate_list = []
    for i, cv in enumerate(conversation_bar_list):
        if len(cv["progress_denominator_list"]) == 0:
            percentage = 0
        else:
            percentage = (sum(cv["progress_numerator_list"]) / sum(cv["progress_denominator_list"])) * 100

        tmp_dict = {
            "message_block_name": cv["message_block_name"],
            "bg_type": BG_PROGRESS_BAR_TYPE_LIST[i % len(BG_PROGRESS_BAR_TYPE_LIST)],
            "percentage": str(int(percentage))
        }

        conversation_rate_list.append(tmp_dict)

    print("conversation_rate_list")
    print(conversation_rate_list)

    context = {
        "title": "Analytics",
        "total_conversion_rate": total_conversion_rate,
        "total_visitors": total_visitors,
        "todocount" : todocount,
        "todos": todos,
        "w_cv_rate_chart": w_cv_rate_chart,
        "w_visitor_rate_chart": w_visitor_rate_chart,
        "device_rate_chart": device_rate_chart,
        "os_rate_chart": os_rate_chart,
        "browser_rate_chart": browser_rate_chart,
        "conversation_rate_list": conversation_rate_list,
        "namespace": user_obj["service_namespace"]
    }

    return render(request, "vendor_contactchat/statistics.html", context)


from functools import partial
from kivy.uix.anchorlayout import AnchorLayout
from kivy.clock import Clock
from constants import green_color, red_color, maroon_color
from application.application_utils import create_grid_layout, create_label
from application.screens.games.base_game import BaseGame
from application.widgets.image_button import ImageButton

class NumberGapFillingGameScreen(BaseGame):

    def __init__(self, screen_name, sign_visual_dict, parent_function, rng=None, **kwargs):
        super().__init__(screen_name, sign_visual_dict, parent_function, rng, 'NumberGapFilling', **kwargs)
        self.current_index = 0
        self.__create_question()
        self.__build_screen()

    def __create_question(self):
        self.first_operand = self.rng.integers(0, 10)
        self.second_operand = self.rng.integers(1, 10)
        self.result = str(self.first_operand * self.second_operand)
        self.hidden_word = ' '.join(['_'] * len(self.result))

    def __build_screen(self):
        self.grid_layout = create_grid_layout(rows=2, spacing='5dp')
        self.layout.add_widget(self.grid_layout)
        self.__build_instructions()
        self.__build_buttons()

    def __build_instructions(self):
        self.grid_layout_text = create_grid_layout(rows=2, size_hint=(1, 0.1))
        self.grid_layout.add_widget(self.grid_layout_text)

        instructions = ('Select the signs in the correct order to complete the ' +
            f'result of: {self.first_operand} x {self.second_operand}')
        instructions_label = create_label(text=instructions, size_hint=(1, 0.4))
        self.grid_layout_text.add_widget(instructions_label)

        self.word_label = create_label(text=self.hidden_word, size_hint=(1, 0.6))
        self.grid_layout_text.add_widget(self.word_label)

    def __build_buttons(self):
        anchor_layout = AnchorLayout()
        self.grid_layout.add_widget(anchor_layout)

        self.grid_layout_signs = create_grid_layout(rows=2, spacing='7dp',
            cols=len(self.sign_visual_dict) // 2, size_hint=(1, 0.9))
        anchor_layout.add_widget(self.grid_layout_signs)

        options = list(self.sign_visual_dict.keys())
        self.rng.shuffle(options)
        for sign in options:
            self.__add_button(sign)

    def __add_button(self, sign):
        visual_path = self.sign_visual_dict[sign]
        button = ImageButton(on_press=self.button_callback, image_source=visual_path, size_hint_y=0.4)
        button.id = sign
        self.grid_layout_signs.add_widget(button)

    def button_callback(self, instance):
        if self.__correct_answer(instance.id):
            self.__trigger_color_change(instance, green_color)
            self.__update_hidden_word()
            if self.__is_game_finished():
                self.finish_game()
        else:
            self.num_mistakes += 1
            self.__trigger_color_change(instance, red_color)

    def __correct_answer(self, selected_sign):
        return selected_sign == self.result[self.current_index]

    def __update_hidden_word(self):
        index = 2 * self.current_index
        self.hidden_word = self.hidden_word[:index] + self.result[self.current_index] + self.hidden_word[index+1:]
        self.word_label.text = self.hidden_word
        self.current_index += 1

    def set_match_color_to_default(self, button_pressed, *args):
        button_pressed.border_color = maroon_color

    def __trigger_color_change(self, button_pressed, color):
        button_pressed.border_color = color
        partial_func = partial(self.set_match_color_to_default, button_pressed)
        Clock.schedule_once(partial_func, 0.2)

    def __is_game_finished(self):
        return self.current_index >= len(self.result)


# This code is meant to solve a problem on a CTF. The problem was called funsum.
# Technically, the code should work to find the the number 12345 in a list of sequential number.
# The problem is that I am basically brute forcing the function that finds all of the possible combinations of a list of numbers.
# After a few digists into the list, the calculations become to expensive for the CPU.
# Please help make this better.
# The commented out generator below is meant to provide the code with starting point
# num_list = [num for num in range(13, 20)]
num_list = [13, 14, 15, 16, 17, 18, 19]

def base_args(lst: list):
    """Returns a list of single digit nums in string format"""
    temp_lst = [[num for num in str(element)] for element in lst]
    num_list = sum(temp_lst, [])
    num_list = [num_list]
    return num_list

def perm(ref_lst: list, main_list: list,  reference: int):
    """Creates a list of lists of some of the possible numbers
    Also, this kinda brute forces the list so it will get expensive quick"""
    for num, _ in enumerate(ref_lst):
        index = num
        temp_list = ref_lst.copy()
        temp_list[index:index + reference-1] = [''.join(temp_list[index:index+reference-1])]
        if temp_list not in main_list and '' not in temp_list:
            main_list.append(temp_list)
            temp_list = []
        else:
            temp_list = []

def create(nums: list):
    """Returns the full list of lists of all possible number combinations"""
    round = 0
    ref = len(nums[0])
    try:
        while True:
            for ref_num in range(3, ref+1):
                perm(nums[round], nums, ref_num)
            round += 1
    except IndexError:
        print('Done')
    return nums

def funsum(str_list: list):
    """This simply adds all elements of a list together"""
    num_list = [int(num) for num in str_list]
    return sum(num_list)

def main():
    count = 2
    results = {}
    for _ in range(len(num_list)):
        start_list = base_args(num_list[:count])
        fun_list = create(start_list)
        ser_list = [funsum(item) for item in fun_list]
        results.update({num_list[count-1]: ser_list})
        if 12345 in results[num_list[count-1]]:
            print(results)
            break
        else:
            count += 1
            results = {}
            continue

if __name__ == '__main__':
    main()


from mininet.node import Switch
from mininet.util import quietRun

class LinuxBridge(Switch):
    """The Linux Bridge for Mininet.  Much simpler than the
    OVSKernelSwitch and all its complex machinary."""
    def __init__(self, name, **kwargs):
        Switch.__init__(self, name, **kwargs)

    @staticmethod
    def setup():
        pass # for now

    def start(self, controllers):
        # Controller support isnt there for the Linux Bridge
        brctl = '/sbin/brctl'
        ifconfig = '/sbin/ifconfig'

        cmds = [
            "%s delbr %s",
            "%s addbr %s"
        ]

        map(lambda s: quietRun(s % (brctl, self.name)), cmds)

        for intf in self.intfs.values():
            #don't do this, it will remove the qdiscs :(
            #quietRun("%s %s down" % (ifconfig, intf))
            quietRun("%s addif %s %s" % (brctl, self.name, intf))

        quietRun("%s %s up" % (ifconfig, self.name))

        for intf in self.intfs.values():
            quietRun("%s %s up" % (ifconfig, intf))

    def stop(self):
        ifconfig = '/sbin/ifconfig'
        quietRun("%s %s down" % (ifconfig, self.name))
        self.deleteIntfs()


"""
129. Sum Root to Leaf Numbers
You are given the root of a binary tree containing digits from 0 to 9 only.
Each root-to-leaf path in the tree represents a number.
- For example, the root-to-leaf path 1->2->3 represents the number 123.

Return the total sum of all root-to-leaf numbers. Test cases are generated so that the answer will fit in a 32-bit integer.

A leaf node is a node with no children.

Example1:
        1
      /   \
    2     3
Input: root = [1,2,3]
Output: 25
Explanation:
The root-to-leaf path 1->2 represents the number 12.
The root-to-leaf path 1->3 represents the number 13.
Therefore, sum = 12 + 13 = 25.

Example2:
               4
            /      \
          9         0
        /   \
      5      1
Input: root = [4,9,0,5,1]
Output: 1026
Explanation:
The root-to-leaf path 4->9->5 represents the number 495.
The root-to-leaf path 4->9->1 represents the number 491.
The root-to-leaf path 4->0 represents the number 40.
Therefore, sum = 495 + 491 + 40 = 1026.
"""

"""
Note:
1. Recursive DFS: O(n) time | O(h) space
2. Iterative DFS: O(n) time | O(h) space
3. Iterative BFS: O(n) time | O(n) space
"""

from typing import List
from collections import deque
import unittest
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def sumNumbers(self, root: TreeNode) -> int:
        result = [0]
        self.dfs(root, 0, result)
        return result[0]

    def dfs(self, root: TreeNode, temp: int, result: List[int]):
        if not root.left and not root.right:
            result[0] += temp * 10 + root.val
            return
        if root.left:
            self.dfs(root.left, temp * 10 + root.val, result)
        if root.right:
            self.dfs(root.right, temp * 10 + root.val, result)
        return

    def sumNumbers2(self, root: TreeNode) -> int:
        total = 0
        if not root:
            return 0
        stack = [(root, 0)]
        while stack:
            node, currentSum = stack.pop()
            currentSum += node.val
            if not node.left and not node.right:
                total += currentSum
            else:
                currentSum *= 10
                if node.right:
                    stack.append((node.right, currentSum))
                if node.left:
                    stack.append((node.left, currentSum))
        return total

    def sumNumbers3(self, root: TreeNode) -> int:
        queue, result = deque([]), 0
        queue.append(root)
        while queue:
            node = queue.popleft()
            if not node.left and not node.right:
                result += node.val
            if node.left:
                node.left.val += node.val * 10
                queue.append(node.left)

            if node.right:
                node.right.val += node.val * 10
                queue.append(node.right)
        return result

# Unit Tests
funcs = [Solution().sumNumbers, Solution().sumNumbers2, Solution().sumNumbers3]

class TestSumNumbers(unittest.TestCase):
    def testSumNumbers1(self):
        for func in funcs:
            root = TreeNode(1, TreeNode(2), TreeNode(3))
            self.assertEqual(func(root=root), 25)

    def testSumNumbers2(self):
        for func in funcs:
            root = TreeNode(4, TreeNode(
                9, TreeNode(5), TreeNode(1)), TreeNode(0))
            self.assertEqual(func(root=root), 1026)

if __name__ == "__main__":
    unittest.main()


import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import warnings
warnings.filterwarnings('ignore')
import argparse
from typing import List
from src.models.named_entity_recognition_rnn import RNNModel, DataPreprocessor
from src.utils.config_loader import NamedEntityRecognitionConfigReader

def evaluate_model(questions_list: List[str], config: NamedEntityRecognitionConfigReader) -> None:
    """
    Wrapper function that calls the model deserializer and returns prediction
    Args:
        model_file_url: relative path for the model file
        questions_list: list of strings to perform inference on
    Return:
        list of probabilies for positive class for each input word
    """
    pre_processor = None
    trained_model = None
    trained_model, preprocessor_file = RNNModel.load_model(config.h5_model_url, config.class_file_url,
                                                           config.preprocessor_file_url, collect_from_gdrive=False)
    if trained_model is None:
        raise ValueError("there is no corresponding model file")
    pre_processor = DataPreprocessor.load_preprocessor(preprocessor_file)
    questions_list_encoded, questions_list_tokenized, n_tokens =\
        DataPreprocessor.preprocess_data(questions_list, pre_processor)
    labels_list = trained_model.predict(questions_list_encoded, pre_processor["labels_to_idx"], n_tokens)
    display_result = trained_model.visualize(questions_list_tokenized, labels_list)
    print(display_result)

def parse_arguments():
    """
    Parse file arguments
    """
    parser = argparse.ArgumentParser(description="Predict NER for a given input text")
    parser.add_argument('text', help="texts separated by space, each text must be surrounded with quotes", nargs='+')
    return parser.parse_args()

def main():
    """main function"""
    args = parse_arguments()
    qlist = args.text
    root_dir = os.environ.get("MARABOU_HOME")
    if root_dir is None:
        raise ValueError("please make sure to setup the environment variable MARABOU_ROOT to\
                         point for the root of the project")
    config_file_path = os.path.join(root_dir, "marabou/train/config/config_named_entity_recognition.json")
    config_file = NamedEntityRecognitionConfigReader(config_file_path)
    evaluate_model(qlist, config_file)

if __name__ == '__main__':
    main()


#! /usr/bin/env python
# -*- coding: utf-8 -*-

"""Merge two existed dataset."""

import argparse
import logging
from pathlib import Path
from os.path import dirname as parent

from fairseq.molecule_utils.external.fairseq_dataset_build_utils import merge_and_build
from fairseq.molecule_utils.basic import smiles_utils as smu

def main():
    parser = argparse.ArgumentParser(description='Merge two existed dataset.')
    parser.add_argument('-i1', '--input-dir-1', dest='input_dir_1', type=Path, required=True, help='Input data directory 1')
    parser.add_argument('--in1', '--input-name-1', dest='input_name_1', type=str, required=True, help='Input name 1')
    parser.add_argument('-i2', '--input-dir-2', dest='input_dir_2', type=Path, required=True, help='Input data directory 2')
    parser.add_argument('--in2', '--input-name-2', dest='input_name_2', type=str, required=True, help='Input name 2')
    parser.add_argument('-o', '--output-dir', type=Path, required=True, help='Output data directory')
    parser.add_argument('--on', '--output-name', dest='output_name', type=str, default=None,
                        help='Output name, default to input name')
    parser.add_argument('--keep-duplicates', action='store_true')
    parser.add_argument('-v', '--verbose', action='store_true')

    args = parser.parse_args()

    logging.basicConfig(
        format='%(levelname)s:%(filename)s:%(message)s', level=logging.DEBUG if args.verbose else logging.INFO)
    smu.disable_rdkit_log()

    merge_and_build(
        input_dir_1=args.input_dir_1,
        input_name_1=args.input_name_1,
        input_dir_2=args.input_dir_2,
        input_name_2=args.input_name_2,
        output_dir=args.output_dir,
        output_name=args.output_name,
        fairseq_root=Path(parent(parent(parent(__file__)))),
        pre_dicts_root=Path(parent(parent(parent(__file__)))) / 'dict',
        keep_duplicates=args.keep_duplicates,
    )

if __name__ == '__main__':
    main()


from __future__ import annotations

from abc import ABC, abstractmethod

class AppsignalCLICommand(ABC):
    @abstractmethod
    def run(self) -> int:
        raise NotImplementedError

    _push_api_key: str | None
    _name: str | None

    def _input_push_api_key(self) -> None:
        key = input("Please enter your Push API key: ")
        self._push_api_key = key
        if self._push_api_key == "":
            self._input_push_api_key()

    def _input_name(self) -> None:
        self._name = input("Please enter the name of your application: ")
        if self._name == "":
            self._input_name()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 23 12:04:30 2020

@author: aggarwal
"""
#%%
""""import essential packages"""

import pandas as pd
import numpy as np
from osgeo import gdal
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import  roc_curve, auc
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from rotation_forest import RotationForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import ExtraTreesClassifier
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
import pickle
#import graphviz
from sklearn import tree
from sklearn.feature_selection import SelectFromModel
from sklearn.utils import shuffle

#%%

"""function to return pixel values"""
def value(ds):
    rows = ds.RasterYSize
    cols = ds.RasterXSize
    band1 = ds.GetRasterBand(1).ReadAsArray()
    data = np.array(band1)
    values = []
    for row in range(0,rows):
        for col in range(0,cols):
            values.append(data[row,col])
    return values

"""function to return latitude and longitude"""
def latlong(ds):
    rows = ds.RasterYSize
    cols = ds.RasterXSize
    xoff,a,b,yoff,d,e = ds.GetGeoTransform()
    lat =[]
    long = []
    def pixel2coord(x,y):
        xp = a*y+b*x+a*0.5+b*0.5+xoff
        yp = d*y+e*x+d*0.5+e*0.5+yoff
        lat.append(xp)
        long.append(yp)
    for row in range(0,rows):
        for col in range(0,cols):
            pixel2coord(row,col)
    return lat, long

#%%

"""load images and extract values for features"""
ASPECT = value(gdal.Open("../extract/aspect/ASPECT.tif"))
BUFFER_FAULT2 = value(gdal.Open("../extract/buffer/distance_2/fault/buffer_fault2.tif"))
BUFFER_RIVER2 = value(gdal.Open("../extract/buffer/distance_2/river/buffer_river2.tif"))
BUFFER_ROAD2 = value(gdal.Open("../extract/buffer/distance_2/road/buffer_road2.tif"))
DEM = value(gdal.Open("../extract/dem/DEM.tif"))
LITHO = value(gdal.Open("../extract/litho/LITHO.tif"))
PLAN_CURVETURE = value(gdal.Open("../extract/plan_curveture/PLAN_CURVETURE.tif"))
PROFILE_CURVETURE = value(gdal.Open("../extract/profile_curveture/PROFILE_CURVETURE.tif"))
SLOPE_LENGTH = value(gdal.Open("../extract/slope_length/SLOPE_LENGTH.tif"))
SLOPEW = value(gdal.Open("../extract/slope/SLOPE.tif"))
SPI= value(gdal.Open("../extract/spi/SPI.tif"))
STI = value(gdal.Open("../extract/sti/STI.tif"))
TWI = value(gdal.Open("../extract/twi/twi_12.tif"))
LAND_COVER= value(gdal.Open("../extract/land_cover/LAND_COVER1.tif"))
LANDSLIDE = value(gdal.Open("../extract/landslide/landslide_2.tif"))

"""calculate latitude, longitude and round to integer"""

lat, long = latlong(gdal.Open("../extract/land_cover/LAND_COVER1.tif"))
lat = [round(x) for x in lat]
long = [round(x) for x in long]

#%%
"""check if total no of data points are equal in all"""
features_len = []
features_len.append(len(ASPECT))
features_len.append(len(BUFFER_FAULT2))
features_len.append(len(BUFFER_RIVER2))
features_len.append(len(BUFFER_ROAD2))
features_len.append(len(DEM))
features_len.append(len(LITHO))
features_len.append(len(PLAN_CURVETURE))
features_len.append(len(PROFILE_CURVETURE))
features_len.append(len(SLOPE_LENGTH))
features_len.append(len(SLOPEW))
features_len.append(len(SPI))
features_len.append(len(STI))
features_len.append(len(TWI))
features_len.append(len(LAND_COVER))
features_len.append(len(LANDSLIDE))
features_len.append(len(lat))
features_len.append(len(long))

print(features_len)

"""if length of features are not same, recheck the data"""

#%%
"""make dataframe from features and create csv"""

d = {'Latitude' :lat, 'Longitude' :long, 'Fault' :BUFFER_FAULT2,
     'River' :BUFFER_RIVER2, 'Road' :BUFFER_ROAD2, 'Lithology' :LITHO, 'Land_Cover' :LAND_COVER,
     'Dem' :DEM, 'Aspect' :ASPECT, 'Plan_Curvature' :PLAN_CURVETURE, 'Profile_Curvature' :PROFILE_CURVETURE,
     'Slope_Length' :SLOPE_LENGTH, 'Slope' :SLOPEW, 'SPI' :SPI, 'STI' :STI, 'TWI' :TWI,
      'Landslide' :LANDSLIDE}

df = pd.DataFrame(d, columns =['Latitude', 'Longitude', 'Fault' , 'River' , 'Road' ,'Lithology', 'Land_Cover',
                               'Dem', 'Aspect', 'Plan_Curvature','Profile_Curvature',
                               'Slope_Length', 'Slope', 'SPI', 'STI', 'TWI','Landslide'] )

df.to_csv('../csv/complete_data.csv',index=False)

#%%
df = pd.read_csv('../csv/complete_data.csv')
"""
Now we need to calculate the number of values outside study region( no data values).
They should be equal for all the features
We can either save them in 2D list for each feature or print them out in the
console

value_counts() function gives the count of each unique entry in a series in
descending order
"""

#printing value_counts in the console
print(df['Aspect'].value_counts())
print(df['Fault'].value_counts())
print(df['River'].value_counts())
print(df['Road'].value_counts())
print(df['Dem'].value_counts())
print(df['Lithology'].value_counts())
print(df['Plan_Curvature'].value_counts())
print(df['Profile_Curvature'].value_counts())
print(df['Slope_Length'].value_counts())
print(df['Slope'].value_counts())
print(df['SPI'].value_counts())
print(df['STI'].value_counts())
print(df['TWI'].value_counts())
print(df['Land_Cover'].value_counts())
print(df['Landslide'].value_counts() )

#saving them in a list
mode_values = []

for x in df:
    mode_values.append(df[x].value_counts())

"""
analyse value_count for all the features,
Since number of no data values is more, first value of
value_counts() function is same for all features, if not check the original
images"""

"""
assign a variable drop_value as one of the no data value,
here value of buffer fault is being used"""

drop_value = 65536

"""save the index of all the pixels with value as drop_value in a list"""

indexNames = df[df['Fault'] == drop_value].index

"""from the original dataframe, drop all the rows of index in indexNames"""
df.drop(indexNames , inplace = True)

#write the dataframe into a csv file
df.to_csv('../csv/row_truncated_data.csv', index = False)

#%%
df = pd.read_csv('../csv/row_truncated_data.csv')
""""multi collinearity check"""

"""vif values between 1-5 have low collinearity
                5-10 , moderate
                >10, high collinearity
                """
df_new = df.iloc[:,2:16]
vif = pd.DataFrame()
vif['vif factor'] = [variance_inflation_factor(df_new.values, i) for i in range(df_new.shape[1])]
vif['features'] = df_new.columns

corr = pd.DataFrame()
corr = df_new.iloc[:,5:]
corr_matrix=corr.corr()
#%%
"""preprocessing - basic layout for on  one hot encoding the data

X = df.iloc[:,2:16]
Y = df.iloc[:,16]

landc = LabelBinarizer().fit_transform(X.LandCover)
litho = LabelBinarizer().fit_transform(X.Lithology)

X = X.drop(['LandCover','Lithology'], axis = 1)
landc = pd.DataFrame(landc)
litho = pd.DataFrame(litho)

X = pd.concat([X,landc,litho],axis=1)
"""
#%%
"""preprocessing 0 - handling profile and plan curvature aspect, all buffer variables"""

i = df['Plan_Curvature']
Plan_Curvature = []
for x in i:
    if(x<0):
        Plan_Curvature.append(2)
    elif(x>0):
        Plan_Curvature.append(1)
    else:
        Plan_Curvature.append(0)

i = df['Profile_Curvature']
Profile_Curvature = []
for x in i:
    if(x<0):
        Profile_Curvature.append(2)
    elif(x>0):
        Profile_Curvature.append(1)
    else:
        Profile_Curvature.append(0)

df = df.drop(['Plan_Curvature','Profile_Curvature'], axis = 1)
Plan_Curvature = pd.Series(Plan_Curvature)
Profile_Curvature = pd.Series(Profile_Curvature)
df = pd.concat([df,Plan_Curvature,Profile_Curvature], axis=1)
df = df.rename(columns={0:'Plan_Curvature',1:'Profile_Curvature'})

i = df['Aspect']
j = []
for x in i:
    if(x<0):
        j.append(0)
    elif(x>=0 and x<22.5):
        j.append(1)
    elif(x>=22.5 and x<67.5):
        j.append(2)
    elif(x>=67.5 and x<112.5):
        j.append(3)
    elif(x>=112.5 and x<157.5):
        j.append(4)
    elif(x>=157.5 and x<202.5):
        j.append(5)
    elif(x>=202.5 and x<247.5):
        j.append(6)
    elif(x>=247.5 and x<292.5):
        j.append(7)
    elif(x>=292.5 and x<337.5):
        j.append(8)
    else:
        j.append(1)
df = df.drop(['Aspect'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Aspect'})

i = df['Fault']
Fault = []
for x in i:
    if(x==50):
        Fault.append(0)
    elif(x==100):
        Fault.append(1)
    elif(x==150):
        Fault.append(2)
    elif(x==200):
        Fault.append(3)
    elif(x==250):
        Fault.append(4)
    elif(x==300):
        Fault.append(5)
df = df.drop(['Fault'],axis=1)
Fault = pd.Series(Fault)
df = pd.concat([df,Fault], axis=1)
df = df.rename(columns={0:'Fault'})

i = df['River']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['River'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'River'})

i = df['Road']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['Road'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Road'})

#moving landslide in the last
landslide = df['Landslide']
df = df.drop(['Landslide'],axis=1)
df = pd.concat([df,landslide], axis=1)

#saving csv file
df.to_csv('../csv/processed0_truncated_data.csv')

#%%
"""preprocessing 1 - handling DEM derivatives, all buffer variables"""
"""Class (as per Jenks natural break)"""

i = df['Plan_Curvature']
Plan_Curvature = []
for x in i:
    if(x<=-2.543456388):
        Plan_Curvature.append(0)
    elif(x>=-2.543456387 and x<=-0.957240183):
        Plan_Curvature.append(1)
    elif(x>=-0.957240182 and x<=-0.092031344):
        Plan_Curvature.append(2)
    elif(x>=-0.092031343 and x<=0.773177495):
        Plan_Curvature.append(3)
    elif(x>=0.773177496 and x<=2.215192226):
        Plan_Curvature.append(4)
    else:
        Plan_Curvature.append(5)

i = df['Profile_Curvature']
Profile_Curvature = []
for x in i:
    if(x<=-3.46856041):
        Profile_Curvature.append(5)
    elif(x>=-3.46856040 and x<=-1.346149886):
        Profile_Curvature.append(4)
    elif(x>=-3.46856040 and x<=-1.346149886):
        Profile_Curvature.append(3)
    elif(x>=-0.133343871 and x<=1.079462141):
        Profile_Curvature.append(2)
    elif(x>=1.079462142 and x<=3.050271913):
        Profile_Curvature.append(1)
    else:
        Profile_Curvature.append(0)

df = df.drop(['Plan_Curvature','Profile_Curvature'], axis = 1)
Plan_Curvature = pd.Series(Plan_Curvature)
Profile_Curvature = pd.Series(Profile_Curvature)
df = pd.concat([df,Plan_Curvature,Profile_Curvature], axis=1)
df = df.rename(columns={0:'Plan_Curvature',1:'Profile_Curvature'})

i = df['Aspect']
j = []
for x in i:
    if(x<0):
        j.append(0)
    elif(x>=0 and x<22.5):
        j.append(1)
    elif(x>=22.5 and x<67.5):
        j.append(2)
    elif(x>=67.5 and x<112.5):
        j.append(3)
    elif(x>=112.5 and x<157.5):
        j.append(4)
    elif(x>=157.5 and x<202.5):
        j.append(5)
    elif(x>=202.5 and x<247.5):
        j.append(6)
    elif(x>=247.5 and x<292.5):
        j.append(7)
    elif(x>=292.5 and x<337.5):
        j.append(8)
    else:
        j.append(1)
df = df.drop(['Aspect'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Aspect'})

i = df['Fault']
Fault = []
for x in i:
    if(x==50):
        Fault.append(0)
    elif(x==100):
        Fault.append(1)
    elif(x==150):
        Fault.append(2)
    elif(x==200):
        Fault.append(3)
    elif(x==250):
        Fault.append(4)
    elif(x==300):
        Fault.append(5)
df = df.drop(['Fault'],axis=1)
Fault = pd.Series(Fault)
df = pd.concat([df,Fault], axis=1)
df = df.rename(columns={0:'Fault'})

i = df['River']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['River'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'River'})

i = df['Road']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['Road'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Road'})

i = df['Dem']
j = []
for x in i:
    if(x<=887):
        j.append(0)
    elif(x>=888 and x<=1437):
        j.append(1)
    elif(x>=1438 and x<=2057):
        j.append(2)
    elif(x>=2058 and x<=2748):
        j.append(3)
    elif(x>=2749 and x<=3457):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['Dem'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Dem'})

i = df['Slope']
Slope = []
for x in i:
    if(x<=10.87785184):
        Slope.append(0)
    elif(x>=10.87785185 and x<=21.16771168):
        Slope.append(1)
    elif(x>=21.16771169 and x<=29.69359556):
        Slope.append(2)
    elif(x>=29.69359557 and x<=37.92548343):
        Slope.append(3)
    elif(x>=37.92548344 and x<=47.33335529):
        Slope.append(4)
    else:
        Slope.append(5)
df = df.drop(['Slope'],axis=1)
Slope = pd.Series(Slope)
df = pd.concat([df,Slope], axis=1)
df = df.rename(columns={0:'Slope'})

i = df['TWI']
j = []
for x in i:
    if(x<=3.375663230):
        j.append(0)
    elif(x>=3.375663231 and x<=5.585415369):
        j.append(1)
    elif(x>=5.585415370 and x<=8.679068363):
        j.append(2)
    elif(x>=8.679068364 and x<=11.44125854):
        j.append(3)
    elif(x>=11.44125855 and x<=13.65101068):
        j.append(4)
    else:
        j.append(5)

df = df.drop(['TWI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'TWI'})

i = df['STI']
j = []
for x in i:
    if(x<=1.129179711):
        j.append(0)
    elif(x>=1.129179712 and x<=4.516718846):
        j.append(1)
    elif(x>=4.516718847 and x<=10.72720726):
        j.append(2)
    elif(x>=10.72720727 and x<=20.88982466):
        j.append(3)
    elif(x>=20.88982467 and x<=40.65046961):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['STI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'STI'})

i = df['SPI']
j = []
for x in i:
    if(x<=7.843282782):
        j.append(0)
    elif(x>=7.843282783 and x<=39.21641391):
        j.append(1)
    elif(x>=39.21641392 and x<=113.7276003):
        j.append(2)
    elif(x>=113.7276004 and x<=282.3581801):
        j.append(3)
    elif(x>=282.3581802 and x<=615.6976984):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['SPI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'SPI'})

i = df['Slope_Length']
j = []
for x in i:
    if(x<=108.0233130):
        j.append(0)
    elif(x>=108.0233131 and x<=324.0699391):
        j.append(1)
    elif(x>=324.0699392 and x<=669.7445408):
        j.append(2)
    elif(x>=669.7445409 and x<=1188.256443):
        j.append(3)
    elif(x>=1188.256444 and x<=2052.442948):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['Slope_Length'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Slope_Length'})

#moving landslide in the last
landslide = df['Landslide']
df = df.drop(['Landslide'],axis=1)
df = pd.concat([df,landslide], axis=1)

#saving csv file
df.to_csv('../csv/processed1_truncated_data.csv')

#%%
"""preprocessing 2 - handling DEM derivatives, all buffer variables"""
"""Class (equal interval)"""

i = df['Plan_Curvature']
Plan_Curvature = []
for x in i:
    if(x<=-13.86327203):
        Plan_Curvature.append(0)
    elif(x>=-13.86327202 and x<=-7.734709422):
        Plan_Curvature.append(1)
    elif(x>=-7.734709421 and x<=-1.606146812):
        Plan_Curvature.append(2)
    elif(x>=-1.606146811 and x<=4.522415797):
        Plan_Curvature.append(3)
    elif(x>=4.522415798 and x<=10.65097841):
        Plan_Curvature.append(4)
    else:
        Plan_Curvature.append(5)

i = df['Profile_Curvature']
Profile_Curvature = []
for x in i:
    if(x<=-12.51407193):
        Profile_Curvature.append(5)
    elif(x>=-12.51407192 and x<=-6.096306772):
        Profile_Curvature.append(4)
    elif(x>=-6.096306771 and x<= 0.321458383):
        Profile_Curvature.append(3)
    elif(x>=0.321458383 and x<=6.739223538):
        Profile_Curvature.append(2)
    elif(x>=6.739223539 and x<=13.15698869):
        Profile_Curvature.append(1)
    else:
        Profile_Curvature.append(0)

df = df.drop(['Plan_Curvature','Profile_Curvature'], axis = 1)
Plan_Curvature = pd.Series(Plan_Curvature)
Profile_Curvature = pd.Series(Profile_Curvature)
df = pd.concat([df,Plan_Curvature,Profile_Curvature], axis=1)
df = df.rename(columns={0:'Plan_Curvature',1:'Profile_Curvature'})

i = df['Aspect']
j = []
for x in i:
    if(x<0):
        j.append(0)
    elif(x>=0 and x<22.5):
        j.append(1)
    elif(x>=22.5 and x<67.5):
        j.append(2)
    elif(x>=67.5 and x<112.5):
        j.append(3)
    elif(x>=112.5 and x<157.5):
        j.append(4)
    elif(x>=157.5 and x<202.5):
        j.append(5)
    elif(x>=202.5 and x<247.5):
        j.append(6)
    elif(x>=247.5 and x<292.5):
        j.append(7)
    elif(x>=292.5 and x<337.5):
        j.append(8)
    else:
        j.append(1)
df = df.drop(['Aspect'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Aspect'})

i = df['Fault']
Fault = []
for x in i:
    if(x==50):
        Fault.append(0)
    elif(x==100):
        Fault.append(1)
    elif(x==150):
        Fault.append(2)
    elif(x==200):
        Fault.append(3)
    elif(x==250):
        Fault.append(4)
    elif(x==300):
        Fault.append(5)
df = df.drop(['Fault'],axis=1)
Fault = pd.Series(Fault)
df = pd.concat([df,Fault], axis=1)
df = df.rename(columns={0:'Fault'})

i = df['River']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['River'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'River'})

i = df['Road']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['Road'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Road'})

i = df['Dem']
j = []
for x in i:
    if(x<=1073):
        j.append(0)
    elif(x>=1074 and x<=1827):
        j.append(1)
    elif(x>=1828 and x<=2580):
        j.append(2)
    elif(x>=2581 and x<=3333):
        j.append(3)
    elif(x>=3334 and x<=4087):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['Dem'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Dem'})

i = df['Slope']
Slope = []
for x in i:
    if(x<=12.34783182):
        Slope.append(0)
    elif(x>=12.34783183 and x<=24.98965963):
        Slope.append(1)
    elif(x>=24.98965964 and x<=37.63148744):
        Slope.append(2)
    elif(x>=37.63148745 and x<=49.97931925):
        Slope.append(3)
    elif(x>=49.97931926 and x<=62.62114707):
        Slope.append(4)
    else:
        Slope.append(5)
df = df.drop(['Slope'],axis=1)
Slope = pd.Series(Slope)
df = pd.concat([df,Slope], axis=1)
df = df.rename(columns={0:'Slope'})

i = df['TWI']
j = []
for x in i:
    if(x<=-3.087861776):
        j.append(0)
    elif(x>=-3.087861775 and x<=1.607861519):
        j.append(1)
    elif(x>=1.60786152 and x<=6.303584814):
        j.append(2)
    elif(x>=6.303584815 and x<=10.99930811):
        j.append(3)
    elif(x>=10.99930812 and x<=15.6950314):
        j.append(4)
    else:
        j.append(5)

df = df.drop(['TWI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'TWI'})

i = df['STI']
j = []
for x in i:
    if(x<=23.99506887):
        j.append(0)
    elif(x>=23.99506888 and x<=47.99013774):
        j.append(1)
    elif(x>=47.99013775 and x<=71.9852066):
        j.append(2)
    elif(x>=71.98520661 and x<=95.98027547):
        j.append(3)
    elif(x>=95.98027548 and x<=119.9753443):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['STI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'STI'})

i = df['SPI']
j = []
for x in i:
    if(x<=166.66975910):
        j.append(0)
    elif(x>=166.6697592 and x<=333.3395182):
        j.append(1)
    elif(x>=333.3395183 and x<=500.0092773):
        j.append(2)
    elif(x>=500.0092774 and x<=666.6790365):
        j.append(3)
    elif(x>=666.6790366 and x<=833.3487956):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['SPI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'SPI'})

i = df['Slope_Length']
j = []
for x in i:
    if(x<=918.1981608):
        j.append(0)
    elif(x>=918.1981609 and x<=1836.396322):
        j.append(1)
    elif(x>=1836.396323 and x<=2754.594482):
        j.append(2)
    elif(x>=2754.594483 and x<=3672.792643):
        j.append(3)
    elif(x>=3672.792644 and x<=4590.990804):
        j.append(4)
    else:
        j.append(5)
df = df.drop(['Slope_Length'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Slope_Length'})

#moving landslide in the last
landslide = df['Landslide']
df = df.drop(['Landslide'],axis=1)
df = pd.concat([df,landslide], axis=1)

#saving csv file
df.to_csv('../csv/processed2_truncated_data.csv')

#%%
"""preprocessing 3 - handling DEM derivatives, all buffer variables"""
"""Class (quantile)"""

i = df['Plan_Curvature']
Plan_Curvature = []
for x in i:
    if(x>=-19.99183464 and x<=-2.543456383):
        Plan_Curvature.append(0)
    elif(x>=-2.543456382 and x<=-0.957240191):
        Plan_Curvature.append(1)
    elif(x>=-0.957240190 and x<=-0.092031344):
        Plan_Curvature.append(2)
    elif(x>=-0.092031343 and x<=0.340573075):
        Plan_Curvature.append(3)
    elif(x>=0.340573076 and x<=0.917378968):
        Plan_Curvature.append(4)
    elif(x>=0.917378969 and x<=16.777954102):
        Plan_Curvature.append(5)

i = df['Profile_Curvature']
Profile_Curvature = []
for x in i:
    if(x>=-18.93183708 and x<=-1.194549134):
        Profile_Curvature.append(5)
    elif(x>=-1.194549133 and x<=-0.436545376):
        Profile_Curvature.append(4)
    elif(x>=-0.436545375 and x<=0.018256879):
        Profile_Curvature.append(3)
    elif(x>=0.018256880 and x<=0.473059134):
        Profile_Curvature.append(2)
    elif(x>=0.473059135 and x<=1.382663645):
        Profile_Curvature.append(1)
    elif(x>=1.382663646 and x<=19.57475385):
        Profile_Curvature.append(0)

df = df.drop(['Plan_Curvature','Profile_Curvature'], axis = 1)
Plan_Curvature = pd.Series(Plan_Curvature)
Profile_Curvature = pd.Series(Profile_Curvature)
df = pd.concat([df,Plan_Curvature,Profile_Curvature], axis=1)
df = df.rename(columns={0:'Plan_Curvature',1:'Profile_Curvature'})

i = df['Aspect']
j = []
for x in i:
    if(x<0):
        j.append(0)
    elif(x>=0 and x<22.5):
        j.append(1)
    elif(x>=22.5 and x<67.5):
        j.append(2)
    elif(x>=67.5 and x<112.5):
        j.append(3)
    elif(x>=112.5 and x<157.5):
        j.append(4)
    elif(x>=157.5 and x<202.5):
        j.append(5)
    elif(x>=202.5 and x<247.5):
        j.append(6)
    elif(x>=247.5 and x<292.5):
        j.append(7)
    elif(x>=292.5 and x<337.5):
        j.append(8)
    else:
        j.append(1)
df = df.drop(['Aspect'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Aspect'})

i = df['Fault']
Fault = []
for x in i:
    if(x==50):
        Fault.append(0)
    elif(x==100):
        Fault.append(1)
    elif(x==150):
        Fault.append(2)
    elif(x==200):
        Fault.append(3)
    elif(x==250):
        Fault.append(4)
    elif(x==300):
        Fault.append(5)
df = df.drop(['Fault'],axis=1)
Fault = pd.Series(Fault)
df = pd.concat([df,Fault], axis=1)
df = df.rename(columns={0:'Fault'})

i = df['River']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['River'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'River'})

i = df['Road']
j = []
for x in i:
    if(x==40):
        j.append(0)
    elif(x==80):
        j.append(1)
    elif(x==120):
        j.append(2)
    elif(x==160):
        j.append(3)
    elif(x==200):
        j.append(4)
    elif(x==240):
        j.append(5)
df = df.drop(['Road'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Road'})

i = df['Dem']
j = []
for x in i:
    if(x>=320 and x<=958):
        j.append(0)
    elif(x>=959 and x<=1259):
        j.append(1)
    elif(x>=1260 and x<=1525):
        j.append(2)
    elif(x>=1526 and x<=1968):
        j.append(3)
    elif(x>=1969 and x<=2890):
        j.append(4)
    elif(x>=2891 and x<=4840):
        j.append(5)
df = df.drop(['Dem'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Dem'})

i = df['Slope']
Slope = []
for x in i:
    if(x>=0 and x<=14.4058079):
        Slope.append(0)
    elif(x>=14.4058080 and x<=22.34369567):
        Slope.append(1)
    elif(x>=22.34369568 and x<=28.51761157):
        Slope.append(2)
    elif(x>=28.51761158 and x<=34.39753149):
        Slope.append(3)
    elif(x>=34.39753150 and x<=41.45343538):
        Slope.append(4)
    elif(x>=41.45343539 and x<=74.96897888):
        Slope.append(5)
df = df.drop(['Slope'],axis=1)
Slope = pd.Series(Slope)
df = pd.concat([df,Slope], axis=1)
df = df.rename(columns={0:'Slope'})

i = df['TWI']
j = []
for x in i:
    if(x>=-7.783585072 and x<=4.038588872):
        j.append(0)
    elif(x>=4.038588873 and x<=5.806390583):
        j.append(1)
    elif(x>=5.806390584 and x<=10.22589486):
        j.append(2)
    elif(x>=10.22589487 and x<=11.66223375):
        j.append(3)
    elif(x>=11.66223376 and x<=12.76710982):
        j.append(4)
    elif(x>=12.76710983 and x<=20.3907547):
        j.append(5)

df = df.drop(['TWI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'TWI'})

i = df['STI']
j = []
new = []
for x in i:
    if(x<0.000001):
        new.append(0)
new = shuffle(new)
l1 = new[:203092]
l2 = new[203092:]
j = l1.copy()
for x in l2:
    j.append(1)

for x in i:
    if(x>0 and x<=0.564589856):
        j.append(1)
    elif(x>=0.564589857 and x<=1.129179711):
        j.append(2)
    elif(x>=1.129179712 and x<=2.258359423):
        j.append(3)
    elif(x>=2.258359424 and x<=4.516718846):
        j.append(4)
    elif(x>=4.516718847 and x<=143.9704132):
        j.append(5)
df = df.drop(['STI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'STI'})

i = df['SPI']
j = []
new = []
for x in i:
    if(x<0.000001):
        new.append(0)
new = shuffle(new)
l1 = new[:203092]
l2 = new[203092:]
j = l1.copy()
for x in l2:
    j.append(1)

for x in i:
    if(x>0 and x<=3.921641391):
        j.append(1)
    elif(x>=3.921641392 and x<=7.843282782):
        j.append(2)
    elif(x>=7.843282783 and x<=15.68656556):
        j.append(3)
    elif(x>=15.68656557 and x<=631.37313113):
        j.append(4)
    elif(x>=31.37313114 and x<=1000.018555):
        j.append(5)
df = df.drop(['SPI'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'SPI'})

i = df['Slope_Length']
j = []
new = []
for x in i:
    if(x<0.000001):
        new.append(0)
new = shuffle(new)
l1 = new[:203092]
l2 = new[203092:]
j = l1.copy()
for x in l2:
    j.append(1)

for x in i:
    if(x>0 and x<=21.60466261):
        j.append(1)
    elif(x>=21.60466262 and x<=86.41865043):
        j.append(2)
    elif(x>=86.41865044 and x<=172.8373009):
        j.append(3)
    elif(x>=172.8373010 and x<=345.6746017):
        j.append(4)
    elif(x>=345.6746018 and x<=5509.188965):
        j.append(5)
df = df.drop(['Slope_Length'],axis=1)
j = pd.Series(j)
df = pd.concat([df,j], axis=1)
df = df.rename(columns={0:'Slope_Length'})

#moving landslide in the last
landslide = df['Landslide']
df = df.drop(['Landslide'],axis=1)
df = pd.concat([df,landslide], axis=1)

#saving csv file
df.to_csv('../csv/processed3_truncated_data.csv')
#%%
import pandas as pd
import numpy as np
from osgeo import gdal
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import  roc_curve, auc
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from rotation_forest import RotationForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import ExtraTreesClassifier
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
import pickle
#import graphviz
from sklearn import tree
from sklearn.feature_selection import SelectFromModel

df = pd.read_csv("../csv/processed3_truncated_data.csv")
df=df.drop(df.columns[0],axis=1) # one extra column of index created in processed and hence removed.

""""since data is unbalanced, stratify the data without replacement"""

df1 = df.Landslide[df.Landslide.eq(2)].index
df2 = df.Landslide[df.Landslide.eq(1)].sample(len(df1),replace=False).index

df_new = df1.union(df2)
df_final= df.loc[df_new]
df_final.to_csv('../csv/stratified_data.csv', index = False)

""""creating train test split and saving them for future use after reading startified data"""

#df_final = pd.read_csv("../csv/stratified_data0.csv")
X = df_final.iloc[:,:16]
Y = df_final.iloc[:,16]

X_train, X_test, y_train, y_test = train_test_split(X,Y, random_state=1,test_size = 0.3)

#saving latitude and longtitude along with independent variable
y_train = pd.concat([y_train,X_train['Latitude'],X_train['Longitude']],axis=1)
y_test = pd.concat([y_test,X_test['Latitude'],X_test['Longitude']],axis=1)

X_train = X_train.iloc[:,2:]  # remove 0 and 1 index of lat long
X_test = X_test.iloc[:,2:]    # remove 0 and 1 index of lat long

#precautionary check if there are any nan values in the data
print(X_train.isnull().values.any())
print(y_train.isnull().values.any())
print(X_test.isnull().values.any())
print(y_test.isnull().values.any())

#writing train test splits to the system for future use
X_train.to_csv('../csv/X_train.csv', index = False)
X_test.to_csv('../csv/X_test.csv', index = False)
y_train.to_csv('../csv/Y_train.csv', index = False)
y_test.to_csv('../csv/Y_test.csv', index = False)

#obtain labels without lat long
Y_train = y_train.iloc[:,0]
Y_test = y_test.iloc[:,0]

#%%
import pandas as pd
import numpy as np
from osgeo import gdal
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import  roc_curve, auc
import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from rotation_forest import RotationForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import ExtraTreesClassifier
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
import pickle
#import graphviz
from sklearn import tree
from sklearn.feature_selection import SelectFromModel
"""to import already saved train test splits"""
X_train = pd.read_csv('../csv/0/X_train.csv')
y_train = pd.read_csv('../csv/0/Y_train.csv')
X_test = pd.read_csv('../csv/0/X_test.csv')
y_test = pd.read_csv('../csv/0/Y_test.csv')

Y_train = y_train.iloc[:,0]
Y_test = y_test.iloc[:,0]

#%%
"""DECISION TREE OPTIMISATION"""

params = {
        'criterion': ['gini','entropy'],
        'max_depth': [None,2,3, 4, 5],
        'min_samples_leaf': [1,2,3, 4, 5,6],
        'min_samples_split': [2,3, 4, 5,6]
        }

decision_model = DecisionTreeClassifier(random_state = 1)
grid_model = GridSearchCV(decision_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
grid_model.fit(X_train,Y_train)

#print("dt auc")
#print(grid_model.best_params_)
#Print out scores on validation set
#print(random_model.score(X_test,Y_test))
#print(grid_model.score(X_test,Y_test))
#print out scores on training data
#print(random_model.best_score_)
#print("train_score_auc:"+str(grid_model.best_score_))

""""OPTIMISED DECISION TREE"""

dt_model = DecisionTreeClassifier(random_state=1,min_samples_leaf=6,max_depth=5)
dt_model.fit(X_train, Y_train)
dt_score = dt_model.score(X_test, Y_test)
dt_score_train = dt_model.score(X_train, Y_train)

cm_dt = confusion_matrix(Y_test, dt_model.predict(X_test))

dt_predict  = dt_model.predict(X_test)
dt_probs = dt_model.predict_proba(X_test)
dtfpr = dict()
dttpr = dict()
dtroc_auc = dict()
dtfpr, dttpr, _ = roc_curve(Y_test, dt_probs[:,1], drop_intermediate = False,pos_label=2)
dtroc_auc = auc(dtfpr,dttpr)

print(dt_model.feature_importances_)
#%%
"""save dt model"""

pickle.dump(dt_model, open('decison_model.sav', 'wb'))

"""load saved model"""

load_dt_model = pickle.load(open('decison_model.sav', 'rb'))

#%% """Render decision tree"""

import os
os.environ['PATH'] += os.pathsep + 'C:\graphviz\bin'
features = []
for i in X_train:
    features.append(i)
output = ['0','1']
dot_data = tree.export_graphviz(dt_model, out_file = None, filled = True,
                                rounded = True, special_characters = True,
                                class_names= output, feature_names=features)
graph = graphviz.Source(dot_data)
graph.render("landslide_dt")

"""will save the graph in pwd"""

#%%
"""RANDOM FOREST OPTIMISATION"""

params = {
        'max_depth': [None,2,3, 4, 5],
        'n_estimators': [100,250,500],
        'min_samples_leaf': [1,2,3, 4, 5,6],
        'min_samples_split': [2,3, 4, 5,6]
        }

random_model = RandomForestClassifier(random_state = 1)
grid_model = GridSearchCV(random_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
grid_model.fit(X_train,Y_train)

#%%
#Print out best parameters
print("Random auc")
print(grid_model.best_params_)
#Print out scores on validation set

print(grid_model.score(X_test,Y_test))
#print out scores on training data

print(grid_model.best_score_)

#%%
"""OPTIMISED RANDOM FOREST"""

rf_model = RandomForestClassifier(random_state=1,n_estimators=500, max_depth=5)
rf_model.fit(X_train, Y_train)
rf_score = rf_model.score(X_test, Y_test)
rf_score_train = rf_model.score(X_train, Y_train)

cm_rf = confusion_matrix(Y_test, rf_model.predict(X_test))
rf_predict  = rf_model.predict(X_test)
rf_probs = rf_model.predict_proba(X_test)
rffpr = dict()
rftpr = dict()
rfroc_auc = dict()
rffpr, rftpr, _ = roc_curve(Y_test, rf_probs[:,1], drop_intermediate = False,pos_label=2)
rfroc_auc = auc(rffpr,rftpr)

#%%
"""save rf model"""

pickle.dump(rf_model, open('random_model.sav', 'wb'))

"""load saved model"""

load_rf_model = pickle.load(open('random_model.sav', 'rb'))

#%%
"""ROTATION FOREST OPTIMISATION"""

params = {
        'max_depth': [None,2,3, 4, 5],
        'n_estimators': [100,250,500],
        'criterion': ['gini','entropy'],
        'min_samples_leaf': [1,2,3, 4, 5],
        'min_samples_split': [2,3, 4, 5]
        }

rotation_model = RotationForestClassifier(random_state = 1)
grid_model = GridSearchCV(rotation_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
grid_model.fit(X_train,Y_train)

#%%
#Print out best parameters
print("rotation auc")
print(grid_model.best_params_)
#Print out scores on validation set

print(grid_model.score(X_test,Y_test))
#print out scores on training data

print(grid_model.best_score_)
#%%
"""OPTIMISED ROTATION FOREST"""

rt_model = RotationForestClassifier(random_state=1, max_depth = 5)
rt_model.fit(X_train,Y_train)
rt_score = rt_model.score(X_test, Y_test)
rt_score_train = rt_model.score(X_train, Y_train)

cm_rt = confusion_matrix(Y_test, rt_model.predict(X_test))
rt_predict  = rt_model.predict(X_test)
rt_probs = rt_model.predict_proba(X_test)
rtfpr = dict()
rttpr = dict()
rtroc_auc = dict()
rtfpr, rttpr, _ = roc_curve(Y_test, rt_probs[:,1], drop_intermediate = False,pos_label=2)
rtroc_auc = auc(rtfpr,rttpr)

#%%
"""save rt model"""

pickle.dump(rt_model, open('rotation_model.sav', 'wb'))

"""load saved model"""

load_rt_model = pickle.load(open('rotation_model.sav', 'rb'))

#%%
"""ADABOOST OPTIMISATION """

learning_rate = list(range(100,300,10))
rate = []
for i in learning_rate:
    rate.append(float(i/100))
params = {
        'n_estimators': [50,100,250,500,800,1000],
        'learning_rate': rate,
        'algorithm':['SAMME','SAMME.R']
        }

ada_model = AdaBoostClassifier(random_state = 1)
adagrid_model = GridSearchCV(ada_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
adagrid_model.fit(X_train,Y_train)

#%%
#Print out best parameters
print("ada roc")
print(adagrid_model.best_params_)
#Print out scores on validation set

print(adagrid_model.score(X_test,Y_test))
#print out scores on training data

print(adagrid_model.best_score_)
#%%
"""OPTIMISED AdaBoost Classifier"""

adb_model = AdaBoostClassifier(random_state=1, learning_rate=1.6,n_estimators=1000,algorithm='SAMME')
adb_model.fit(X_train, Y_train)
adb_score = adb_model.score(X_test,Y_test)
adb_score_train = adb_model.score(X_train, Y_train)

cm_ad = confusion_matrix(Y_test, adb_model.predict(X_test))
adb_predict  = adb_model.predict(X_test)
ad_probs = adb_model.predict_proba(X_test)
adfpr = dict()
adtpr = dict()
adroc_auc = dict()
adfpr, adtpr, _ = roc_curve(Y_test, ad_probs[:,1], drop_intermediate = False,pos_label=2)
adroc_auc = auc(adfpr,adtpr)

#%%
"""save adb model"""

pickle.dump(adb_model, open('adaboost_model.sav', 'wb'))

"""load saved model"""

load_adb_model = pickle.load(open('adaboost_model.sav', 'rb'))

#%%
"""EXTRA TREE OPTIMISATION"""

learning_rate = list(range(100,300,10))
rate = []
for i in learning_rate:
    rate.append(float(i/100))
params = {
        'n_estimators': [100,200,500],
        'max_depth': [None,2,3,4,5],
        'criterion': ['gini','entropy'],
        'min_samples_leaf': [1,2,3, 4, 5,6],
        'min_samples_split': [2,3, 4, 5,6]
        }

extra_model = ExtraTreesClassifier(random_state = 1)
extragrid_model = GridSearchCV(extra_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
extragrid_model.fit(X_train,Y_train)

#%%
#Print out best parameters
print("extra auc")
print(extragrid_model.best_params_)
#Print out scores on validation set

print(extragrid_model.score(X_test,Y_test))
#print out scores on training data

print(extragrid_model.best_score_)

#%%
"""OPTIMISED Extra Tree Classifier"""

ex_model = ExtraTreesClassifier(random_state=1,max_depth=5)
ex_model.fit(X_train,Y_train)
ex_score = ex_model.score(X_test, Y_test)
ex_score_train = ex_model.score(X_train, Y_train)

cm_ex = confusion_matrix(Y_test, ex_model.predict(X_test))
ex_predict  = ex_model.predict(X_test)
ex_probs = ex_model.predict_proba(X_test)
exfpr = dict()
extpr = dict()
exroc_auc = dict()
exfpr, extpr, _ = roc_curve(Y_test, ex_probs[:,1], drop_intermediate = False,pos_label=2)
exroc_auc = auc(exfpr,extpr)

#%%
"""save extra model"""

pickle.dump(ex_model, open('extra_model.sav', 'wb'))

"""load saved model"""

load_ex_model = pickle.load(open('extra_model.sav', 'rb'))

#%%
"""OPTIMISING XGBOOST"""

learning_rate = list(range(100,260,10))
rate = []
for i in learning_rate:
    rate.append(float(i/100))
params = {
        'learning_rate': rate,
        'max_depth': [3,4,5,6],
        'gamma':[0.5,1,1.5,2,5],
        'colsample_bytree':[0.8,0.8,1.0],
        'min_child_weight':[1,5,10],
        'subsample': [0.6,0.8,1.0]
        }

xg_model = XGBClassifier(random_state = 1)
xggrid_model = GridSearchCV(xg_model,params,scoring='roc_auc',verbose=1,n_jobs=-1)
xggrid_model.fit(X_train,Y_train)

#%%
#Print out best parameters
print("xg auc")
print(xggrid_model.best_params_)
#Print out scores on validation set

print(xggrid_model.score(X_test,Y_test))
#print out scores on training data

print(xggrid_model.best_score_)

#%%
"""OPTIMISED implement XGBoost"""

xg_model = XGBClassifier(random_state=1, colsample_bytree=0.8, gamma = 5,
                         max_depth=6)
xg_model.fit(X_train, Y_train)
xg_score = xg_model.score(X_test, Y_test)
xg_score_train = xg_model.score(X_train, Y_train)

cm_xg = confusion_matrix(Y_test, xg_model.predict(X_test))
xg_predict  = xg_model.predict(X_test)
xg_probs = xg_model.predict_proba(X_test)
xgfpr = dict()
xgtpr = dict()
xgroc_auc = dict()
xgfpr, xgtpr, _ = roc_curve(Y_test, xg_probs[:,1], drop_intermediate = False,pos_label=2)
xgroc_auc = auc(xgfpr,xgtpr)

    #%%
"""save xgboost model"""

pickle.dump(xg_model, open('xgboost_model.sav', 'wb'))

"""load saved model"""

load_xg_model = pickle.load(open('xgboost_model.sav', 'rb'))

#%%
"""optimizing logistic regression
penalty =['l2','none']
test_accuracy = []
train_accuracy = []
for i in penalty:
    model = LogisticRegression(random_state=1, penalty=i)
    model.fit(X_train,Y_train)
    test_accuracy.append(model.score(X_test,Y_test))
    train_accuracy.append(model.score(X_train, Y_train))

solver = ['newton-cg','lbfgs','liblinear','sag','saga']
test_accuracy = []
train_accuracy = []
for i in solver:
    model = LogisticRegression(random_state=1, solver=i)
    model.fit(X_train,Y_train)
    test_accuracy.append(model.score(X_test,Y_test))
    train_accuracy.append(model.score(X_train, Y_train))
#same in all

test_accuracy = []
train_accuracy = []
for i in range(1,101):
    model = LogisticRegression(random_state=1, C=i,max_iter=1000)
    model.fit(X_train,Y_train)
    test_accuracy.append(model.score(X_test,Y_test))
    train_accuracy.append(model.score(X_train, Y_train))

"""
#%%
"""implementing logistic regression"""

lt_model = LogisticRegression(random_state=1)
lt_model.fit(X_train,Y_train)
lt_score = lt_model.score(X_test,Y_test)
lt_score_train = lt_model.score(X_train,Y_train)

cm_lt = confusion_matrix(Y_test, lt_model.predict(X_test))
lt_predict  = lt_model.predict(X_test)
lt_probs = lt_model.predict_proba(X_test)
ltfpr = dict()
lttpr = dict()
ltroc_auc = dict()
ltfpr, lttpr, _ = roc_curve(Y_test, lt_probs[:,1], drop_intermediate = False,pos_label=2)
ltroc_auc = auc(ltfpr,lttpr)

#%%
"""save logistic model"""

pickle.dump(lt_model, open('logistic_model.sav', 'wb'))

"""load saved model"""

load_lt_model = pickle.load(open('logistic_model.sav', 'rb'))

#%%
"""plotting individual roc curves for all the algos

plt.figure()
plt.plot(dtfpr,dttpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.title("Decision_Tree")
plt.text(0.8,0.2,"auc = "+str(round(dtroc_auc,4)))
plt.show()

plt.figure()
plt.plot(rffpr,rftpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.text(0.8,0.2,"auc = "+str(round(rfroc_auc,4)))
plt.title("Random_Forest")
plt.show()

plt.figure()
plt.plot(rtfpr,rttpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.text(0.8,0.2,"auc = "+str(round(rtroc_auc,4)))
plt.title("Rotation_Forest")
plt.show()

plt.figure()
plt.plot(adfpr,adtpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.text(0.8,0.2,"auc = "+str(round(adroc_auc,4)))
plt.title("AdaBoost")
plt.show()

plt.figure()
plt.plot(exfpr,extpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.text(0.8,0.2,"auc = "+str(round(exroc_auc,4)))
plt.title("Extra_Tree")
plt.show()

plt.figure()
plt.plot(xgfpr,xgtpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.title("XGBoost")
plt.text(0.8,0.2,"auc = "+str(round(xgroc_auc,4)))
plt.show()

plt.figure()
plt.plot(ltfpr,lttpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.title("Logistic")
plt.text(0.8,0.2,"auc = "+str(round(ltroc_auc,4)))
plt.show()
"""
#%%
"""graph for all roc scores combined"""

data = [['red','Decision', round((100*dtroc_auc),2)],
        ['blue','Rotation', round((100*rtroc_auc),2)],
        ['black','AdaBoost', round((100*adroc_auc),2)],
        ['darkorange','XGBoost', round((100*xgroc_auc),2)],
        ['green','Random', round((100*rfroc_auc),2)],
        ['magenta','Extra Tree', round((100*exroc_auc),2)]]

def sort_data(data):
    data.sort(key = lambda x:x[2])
    return data

data = sort_data(data)
colors=[]

for x in range(6):
    colors.append(data[x].pop(0))

plt.figure()
plt.plot(dtfpr,dttpr, color = 'red')
plt.plot(rffpr,rftpr, color = 'green')
plt.plot(rtfpr,rttpr, color = 'blue')
plt.plot(adfpr,adtpr, color = 'black')
plt.plot(exfpr,extpr, color = 'magenta')
plt.plot(xgfpr,xgtpr, color = 'darkorange')
plt.plot([0,1],[0,1], color = 'navy',linestyle = '--')
plt.xlabel("False positive rate")
plt.ylabel("True positive rate")
plt.table(cellText=data,cellLoc='left',
          rowColours=colors, colWidths=[0.2,0.15],
          loc=4).auto_set_font_size(False)
plt.text(0.65,0.38,"algorithm")
plt.text(0.9,0.38,"auc")
plt.title("Area Under Curve")
plt.show()

#%%
"""taking majority vote"""
labels = pd.DataFrame()
labels['dt'] = dt_predict
labels['rf'] = rf_predict
labels['rt'] = rt_predict
labels['adb'] = adb_predict
labels['ex'] = ex_predict
labels['xg'] = xg_predict
labels['lt'] = lt_predict

majority_vote = labels.mode(axis = 1)

cm_majority_vote = confusion_matrix(Y_test,majority_vote)

#%%
"""calculating weighted output"""
score_list = list([dt_score,rt_score,rf_score,adb_score,ex_score,xg_score,lt_score])
total = sum(score_list)

weights = []
for i in score_list:
    weights.append(i/total)

weights = pd.Series(weights)
weighted_output = np.dot(labels,weights)

weighted_output = np.round(weighted_output)
cm_weights = confusion_matrix(Y_test,weighted_output)

#%%
data = pd.DataFrame(X_train)
data['Landslide'] = Y_train
data2 = pd.DataFrame(X_test)
data2['Landslide'] = Y_test

data.to_csv("../csv_files/train.csv", index= False)
data2.to_csv("../csv_files/test.csv", index= False)


#!/usr/bin/env python
import sys

topics=["music","piano","guitar","violin","orchestra","sing","song","play","jazz","blues"]

for line in sys.stdin:
    #--- split the line into words ---
    words = line.strip().split()

    for i in range(len(words)-1):
        if words[i] in topics:
            print('%s|%s\t%s' % (words[i],words[i+1], "1"))