import os
import numpy as np
from tqdm import tqdm

import torch
from torch.utils import data
import torchvision.transforms as transform
from torch.nn.parallel.scatter_gather import gather

import encoding.utils as utils
from encoding.datasets import get_segmentation_dataset, test_batchify_fn
from encoding.models import get_model, get_segmentation_model, MultiEvalModule

from option import Options
from utils import *

ids = ['2008_000050', '2008_000052', '2008_000054', '2008_000090', '2008_000254', '2008_000257', '2008_000424', '2008_000533',
       '2008_000659', '2008_000804', '2008_000848', '2008_001041', '2008_001062', '2008_001380', '2008_001531', '2008_001544',
       '2008_002207', '2008_002283', '2008_003336', '2008_004907', '2008_005764', '2008_006646', '2008_007247', '2008_008437',
       '2009_000919', '2009_001370', '2009_001413', '2009_001673', '2009_002078', '2008_008011', '2009_000335', '2008_006151',
       '2008_007190', '2008_005257', '2008_005808', '2008_004910', '2008_005139', '2008_004592', '2008_004852', '2008_003826',
       '2008_004221', '2008_002589', '2008_002965', '2008_001669', '2008_002234', '2008_000942', '2008_001445', '2008_000418',
       '2008_000783', '2008_000149', '2008_000195', '2008_000064', '2010_000524', '2008_004910', '2009_003973', '2009_003247']

four2two = False

def test(args):
    # output folder
    outdir = '%s/test/'%(args.out_dir)
    if not os.path.exists(outdir):
        os.makedirs(outdir)
    # data transforms
    input_transform = transform.Compose([
        transform.ToTensor(),
        transform.Normalize([.485, .456, .406], [.229, .224, .225])])
    # dataset
    if args.eval:
        testset = get_segmentation_dataset(args.dataset, split='val', mode='testval',
                                           transform=input_transform, root=args.data_folder)
    else:#set split='test' for test set
        testset = get_segmentation_dataset(args.dataset, split='val', mode='vis',
                                           transform=input_transform, root=args.data_folder)
    # dataloader
    loader_kwargs = {'num_workers': args.workers, 'pin_memory': True} \
        if args.cuda else {}
    test_data = data.DataLoader(testset, batch_size=args.test_batch_size,
                                drop_last=False, shuffle=False,
                                collate_fn=test_batchify_fn, **loader_kwargs)
    names = []
    if args.dataset == 'pcontext_detail':
        for name in test_data.dataset.ids:
            names.append(name['file_name'].split('/')[-1].split('.')[0])
    elif args.dataset == 'skinny' or args.dataset == 'dark':
        for name in test_data.dataset.images:
            names.append(name.split('/')[-1].split('.')[0])
    elif args.dataset == 'coco' or args.dataset == 'cityscapes' or args.dataset == 'ade20k':
        names = test_data.dataset.images

    if args.model_zoo is not None:
        model = get_model(args.model_zoo, pretrained=True)
    else:
        model = get_segmentation_model(args.model, dataset=args.dataset,
                                       backbone=args.backbone, aux=args.aux,
                                       se_loss=args.se_loss, norm_layer=torch.nn.BatchNorm2d,
                                       base_size=args.base_size, crop_size=args.crop_size,
                                       multi_grid=args.multi_grid, multi_dilation=args.multi_dilation,
                                       root=args.pretrained_home+'/pretrain_models',
                                       skin=args.skin, ft=args.ft,
                                       eval=True)
        # resuming checkpoint
        if args.resume is None or not os.path.isfile(args.resume):
            raise RuntimeError("=> no checkpoint found at '{}'" .format(args.resume))
        print(args.resume)
        checkpoint = torch.load(args.resume)
        # strict=False, so that it is compatible with old pytorch saved models
        model.load_state_dict(checkpoint['state_dict'], strict=False)

    # print(model)
    num_class = testset.num_class
    # for 4 class dark to 2 class skinny
    if four2two:
        evaluator = MultiEvalModule(model, 4, multi_scales=args.multi_scales).cuda()
    else:
        evaluator = MultiEvalModule(model, testset.num_class, multi_scales=args.multi_scales).cuda()
    evaluator.eval()

    tbar = tqdm(test_data)
    def eval_batch(image, dst, evaluator, eval_mode, name):
        if eval_mode:
            # evaluation mode on validation set
            targets = dst
            outputs = evaluator.parallel_forward(image)

            # for 4 class dark to 2 class skinny
            if four2two:
                num_class = 2
            else:
                num_class = testset.num_class

            batch_inter, batch_union, batch_correct, batch_label = 0, 0, 0, 0
            if num_class == 2:
                total_tp, total_pos, total_real = 0, 0, 0
            for output, target in zip(outputs, targets):
                correct, labeled = utils.batch_pix_accuracy(output.data.cpu(), target, four2two)
                inter, union = utils.batch_intersection_union(
                    output.data.cpu(), target, num_class, four2two)
                batch_correct += correct
                batch_label += labeled
                batch_inter += inter
                batch_union += union

                if num_class == 2:
                    tp, pos, real = utils.batch_f1_score(output, target, num_class)
                    total_tp += tp
                    total_pos += pos
                    total_real += real

            if num_class == 2:
                return batch_correct, batch_label, batch_inter, batch_union, total_tp, total_pos, total_real
            return batch_correct, batch_label, batch_inter, batch_union

    total_inter, total_union, total_correct, total_label = \
        np.int64(0), np.int64(0), np.int64(0), np.int64(0)
    if num_class == 2:
        total_tp, total_pos, total_real = np.int64(0), np.int64(0), np.int64(0)
    for i, (image, dst) in enumerate(tbar):
        image, dst = image[0].unsqueeze_(0).cuda(), dst[0].unsqueeze_(0).cuda()
        if torch_ver == "0.3":
            image = Variable(image, volatile=True)
            correct, labeled, inter, union = eval_batch(image, dst, evaluator, args.eval, names[i])
        else:
            with torch.no_grad():
                if num_class == 2:
                    correct, labeled, inter, union, tp, pos, real = eval_batch(image, dst, evaluator, args.eval, names[i])
                else:
                    correct, labeled, inter, union = eval_batch(image, dst, evaluator, args.eval, names[i])
        pixAcc, mIoU, IoU = 0, 0, 0
        if args.eval:
            total_correct += correct.astype('int64')
            total_label += labeled.astype('int64')
            total_inter += inter.astype('int64')
            total_union += union.astype('int64')
            pixAcc = np.float64(1.0) * total_correct / (np.spacing(1, dtype=np.float64) + total_label)
            IoU = np.float64(1.0) * total_inter / (np.spacing(1, dtype=np.float64) + total_union)
            mIoU = IoU.mean()

            if num_class == 2:
                total_tp += tp
                total_pos += pos
                total_real += real
                prec = 1.0 * total_tp / (total_pos + np.spacing(1))
                recall = 1.0 * total_tp / (total_real + np.spacing(1))

                f1 = 2 * prec * recall / (prec + recall + np.spacing(1))

                tbar.set_description(
                    'F1: %.4f, IoU: %.4f' % (f1, IoU[-1]))
            else:
                tbar.set_description(
                    'pixAcc: %.4f, mIoU: %.4f' % (pixAcc, mIoU))
    return pixAcc, mIoU, IoU, num_class

def eval_multi_models(args):
    if args.resume_dir is None or not os.path.isdir(args.resume_dir):
        raise RuntimeError("=> no checkpoint found at '{}'".format(args.resume_dir))
    for resume_file in os.listdir(args.resume_dir):
        if os.path.splitext(resume_file)[1] == '.tar':
            args.resume = os.path.join(args.resume_dir, resume_file)
            assert os.path.exists(args.resume)
            if not args.eval:
                test(args)
                continue
            pixAcc, mIoU, IoU, num_class = test(args)

            txtfile = args.resume
            txtfile = txtfile.replace('pth.tar', 'txt')
            if not args.multi_scales:
                txtfile = txtfile.replace('.txt', 'result_mIoU_%.4f.txt'%mIoU)
            else:
                txtfile = txtfile.replace('.txt', 'multi_scale_result_mIoU_%.4f.txt'%mIoU)
            fh = open(txtfile, 'w')
            print("================ Summary IOU ================\n")
            for i in range(0,num_class):
                print("%3d: %.4f\n" %(i,IoU[i]))
                fh.write("%3d: %.4f\n" %(i,IoU[i]))
            print("Mean IoU over %d classes: %.4f\n" % (num_class, mIoU))
            print("Pixel-wise Accuracy: %2.2f%%\n" % (pixAcc * 100))
            fh.write("Mean IoU over %d classes: %.4f\n" % (num_class, mIoU))
            fh.write("Pixel-wise Accuracy: %2.2f%%\n" % (pixAcc * 100))
            fh.close()
    print('Evaluation is finished!!!')

if __name__ == "__main__":
    args = Options().parse()
    torch.manual_seed(args.seed)
    args.test_batch_size = torch.cuda.device_count()
    eval_multi_models(args)


def fun_test(*args, **kwargs):
    count = 0
    try:
        for item in args:
            int(item)
            print(item)
            count +=1
        for key,value in kwargs.items():
            print(key, value)
            count += 1
        return count
    except ValueError as err:
        return err


#!/usr/bin/env python
import os
import sys
import numpy as np
import pandas as pd
from pathlib import Path
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import tensorflow as tf
from keras.layers import Dense, Dropout, BatchNormalization, LeakyReLU, Conv1D, MaxPooling1D, Flatten, Input
from keras.optimizers import Adam
from keras.losses import Huber
from keras.callbacks import EarlyStopping
from keras.regularizers import l2
from keras.models import Sequential, Model
import joblib

params = {
    'results_path': 'results',
    'data_path': 'data',
    'nfp': 2,
    'n_data_subset': 10000,
    'model': 'nn', # 'cnn' or 'nn'
    'optimizer': Adam,
    'learning_rate': 0.005,
    'epochs': 100,
    'batch_size': 64,
    'early_stopping_patience': 20,
    'test_size': 0.2,
    'random_state': 42,
    'reg_strength': 1e-7,
    'dropout_rate': 1e-3,
    'validation_split': 0.2,
    'decay_steps': 1000,
    'decay_rate': 0.9,
}

def build_neural_network(input_shape, output_shape, reg_strength=1e-7, dropout_rate=3e-3):
    model = Sequential([
        Dense(256, activation='relu', input_shape=(input_shape,), kernel_regularizer=l2(reg_strength)),
        Dropout(dropout_rate),
        Dense(128, activation='relu', kernel_regularizer=l2(reg_strength)),
        Dropout(dropout_rate),
        Dense(64, activation='relu', kernel_regularizer=l2(reg_strength)),
        Dropout(dropout_rate),
        Dense(32, activation='relu', kernel_regularizer=l2(reg_strength)),
        Dropout(dropout_rate),
        Dense(output_shape)
    ])
    return model

def build_cnn(input_shape, output_shape, reg_strength=1e-7, dropout_rate=3e-3):
    input_layer = Input(shape=input_shape)

    x = Conv1D(64, kernel_size=3, activation='relu', padding='same', kernel_regularizer=l2(reg_strength))(input_layer)
    x = BatchNormalization()(x)
    x = Dropout(dropout_rate)(x)

    x = Conv1D(128, kernel_size=3, activation='relu', padding='same', kernel_regularizer=l2(reg_strength))(x)
    x = BatchNormalization()(x)
    x = Dropout(dropout_rate)(x)

    x = Flatten()(x)

    x = Dense(128, activation='relu', kernel_regularizer=l2(reg_strength))(x)
    x = Dropout(dropout_rate)(x)

    output_layer = Dense(output_shape, activation='linear')(x)

    return Model(input_layer, output_layer)

def preprocess_data(X_train, X_test, Y_train, Y_test, params):
    scaler_x = StandardScaler().fit(X_train)
    X_train = scaler_x.transform(X_train)
    X_test = scaler_x.transform(X_test)
    scaler_y = StandardScaler().fit(Y_train)
    Y_train = scaler_y.transform(Y_train)
    Y_test = scaler_y.transform(Y_test)

    if params['model'] == 'cnn':
        X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], 1)
        X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], 1)
        input_shape = (X_train.shape[1], 1)
    else:
        input_shape = X_train.shape[1]

    output_shape = Y_train.shape[1]

    return X_train, X_test, input_shape, output_shape, scaler_x, Y_train, Y_test, scaler_y

def create_dataset(X_train, Y_train, params):
    train_dataset = tf.data.Dataset.from_tensor_slices((X_train, Y_train))
    train_dataset = train_dataset.shuffle(buffer_size=1024).batch(params['batch_size']).prefetch(tf.data.experimental.AUTOTUNE)

    return train_dataset

# Set the number of field periods and model type
print('Usage: python use_nn_qsc.py [nfp] [model], where nfp=2, 3 or 4 and model=nn or cnn. Defaults to nfp=2 and model=nn.')
if len(sys.argv) > 1:
    if sys.argv[1] in ['2','3','4']:
        params['nfp'] = int(sys.argv[1])
    else:
        raise ValueError('NFP must be either 2, 3 or 4')
    if len(sys.argv) > 2:
        if sys.argv[2] in ['nn', 'cnn']:
            params['model'] = sys.argv[2]
        else:
            raise ValueError('Model must be either "nn" or "cnn"')

this_path = str(Path(__file__).parent.resolve())
general_results_path = os.path.join(this_path, params['results_path'])
results_path = os.path.join(general_results_path, f'nfp{params["nfp"]}')
os.makedirs(results_path, exist_ok=True)

filename = os.path.join(this_path, params['data_path'], f'qsc_out.random_scan_nfp{params["nfp"]}.csv')
df = pd.read_csv(filename)

# Only use a subset of parameters
for column in df.columns:
    if df[column].dtype.byteorder == '>':
        df[column] = df[column].values.byteswap().newbyteorder()
df['ysum'] = df.loc[:, df.columns.str.startswith('y')].sum(axis=1)
df = df.sort_values(by='ysum', ascending=True).head(params['n_data_subset'])
df = df.drop(columns='ysum')

x_columns = [col for col in df.columns if col.startswith('x')]
y_columns = [col for col in df.columns if col.startswith('y')]

## ACTUALLY SOLVING THE INVERSE PROBLEM
Y = df[x_columns].values
X = df[y_columns].values

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=params['test_size'], random_state=params['random_state'])

X_train, X_test, input_shape, output_shape, scaler_x, Y_train, Y_test, scaler_y = preprocess_data(X_train, X_test, Y_train, Y_test, params)

if params['model'] == 'cnn':
    model = build_cnn(input_shape, output_shape, reg_strength=params['reg_strength'], dropout_rate=params['dropout_rate'])
else:
    model = build_neural_network(input_shape, output_shape, reg_strength=params['reg_strength'], dropout_rate=params['dropout_rate'])

learning_rate = tf.keras.optimizers.schedules.ExponentialDecay(
    initial_learning_rate=params['learning_rate'], decay_steps=params['decay_steps'], decay_rate=params['decay_rate']
)

model.compile(optimizer=params['optimizer'](learning_rate=learning_rate), loss=Huber(), metrics=['mae'])

early_stopping = EarlyStopping(monitor='val_loss', patience=params['early_stopping_patience'], restore_best_weights=True)

train_dataset = create_dataset(X_train, Y_train, params)
validation_dataset = create_dataset(X_test, Y_test, params)

model.fit(train_dataset, epochs=params['epochs'], validation_data=validation_dataset, validation_split=params['validation_split'], verbose=1, callbacks=[early_stopping])

loss, mae = model.evaluate(X_test, Y_test, verbose=0)
print(f"Test MAE: {mae}")
print(f"Test Loss: {loss}")

predictions = model(X_test)

# Save the model and scaler
model.save(os.path.join(results_path, f"nn_qsc_nfp{params['nfp']}_model{params['model']}.h5"))
joblib.dump(scaler_x, os.path.join(results_path, f"nn_qsc_nfp{params['nfp']}_scaler_x.pkl"))
joblib.dump(scaler_y, os.path.join(results_path, f"nn_qsc_nfp{params['nfp']}_scaler_y.pkl"))
print(f"Model and scaler saved in {results_path}")

plt.figure(figsize=(8, 8))
plt.scatter(Y_test, predictions, label='Predicted values', alpha=0.5)
min_x = np.min([np.min(Y_test), np.min(predictions)])
max_x = np.max([np.max(Y_test), np.max(predictions)])
plt.plot([min_x, max_x], [min_x, max_x], 'r', label='Perfect predictions')
plt.xlabel('True values')
plt.ylabel('Predicted values')
plt.title(f'nfp={params["nfp"]}, model={params["model"]}, metric={mae:.3f}, loss={loss:.3f}, epochs={params["epochs"]}, batch_size={params["batch_size"]}')
plt.legend()
plt.grid(True)
plt.savefig(os.path.join(results_path, f"nn_qsc_nfp{params['nfp']}_model{params['model']}.png"))
plt.show()


def max_len(arr):
    res = [0]
    dfs(0, set(), arr, res)
    return res[0]

def dfs(index, path, arr, res):
    res[0] = max(res[0], len(path))

    if index == len(arr):
        return

    for i in range(index, len(arr)):
        if len(set(arr[i])) != len(arr[i]):
            continue
        if not set(arr[i]) & path:
            dfs(i+1, path | set(arr[i]), arr, res)

def max_len_dp(arr):
    res = [set()]
    for a in arr:
        if len(a) != len(set(a)):
            continue
        for c in res[:]:
            if not c & set(a):
                res.append(c | set(a))

    return max(len(c) for c in res)

def max_len_recursion(arr):
    return dfs_re(set(), 0, arr)

def dfs_re(used: set, index: int, arr):
    """
    Given letters already used and the available words
    in the arr (arr[index:]). Return the max len
    """
    if index == len(arr):
        return len(used)

    max_l = dfs_re(used, index+1, arr)

    w = arr[index]
    if len(w) == len(set(w)) and not (used & set(w)):
        max_l = max(max_l, dfs_re(used | set(w), index+1, arr))

    return max_l


# -*- coding: utf-8 -*-
"""
Created on Wed Dec  6 11:04:29 2017

@author: Louie.Wong
"""

import tweepy
import pandas as pd
import numpy as np
from textblob import TextBlob
from twitter_credentials import *

# API's setup:
def twitter_setup():
    """
    Utility function to setup the Twitter's API
    with our access keys provided.
    """
    # Authentication and access using keys:
    auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)

    # Return API with authentication:
    api = tweepy.API(auth)
    return api

# Create an extractor object:
extractor = twitter_setup()

# Create a tweet list as follows:
#cnn
#realDonaldTrump
#HillaryClinton
#notderb
user = 'realDonaldTrump'
trumptweets = tweepy.Cursor(extractor.user_timeline, screen_name='@'+ user, tweet_mode = 'extended').items()
tweets = []
for tweet in trumptweets:
    tweets.append(tweet)
#Add twitter data into Tweets DataFrame

tweet_df = pd.DataFrame(data = [tweet.full_text for tweet in tweets], columns = ['Tweets'])
tweet_df['id'] = np.array([tweet.id for tweet in tweets])
tweet_df['date'] = np.array([tweet.created_at for tweet in tweets])
tweet_df['likes'] = np.array([tweet.favorite_count for tweet in tweets])
tweet_df['RTs'] = np.array([tweet.retweet_count for tweet in tweets])
tweet_df['source'] = np.array([tweet.source for tweet in tweets])

# Add Sentiment Polarity data into Tweets DataFrame
SA = []
for tweet in tweets:
    print(tweet.full_text)
    print(TextBlob(tweet.full_text).sentiment.polarity)
    if TextBlob(tweet.full_text).sentiment.polarity > 0:
        SA.append('Positive')
    elif TextBlob(tweet.full_text).sentiment.polarity == 0:
        SA.append('Neutual')
    else:
        SA.append('Negative')

tweet_df['SA'] = SA

#Add Hashtags data into Hashtags DataFrame
Hashtags = [[0],[1]]
for i in range(len(tweets)):
    for j in range(len(tweets[i].entities.get('hashtags'))):
        Hashtags[0].append(tweets[i].id)
        Hashtags[1].append(tweets[i].entities.get('hashtags')[j]['text'])
Hashtags_df = pd.DataFrame(data = np.array(Hashtags).T[1:, :], columns = ['tweet_id', 'Hashtags'])

#Add Mentions data into Mentions DataFrame
Mentions = [[0],[1], [2], [3]]
for i in range(len(tweets)):
    for j in range(len(tweets[i].entities.get('user_mentions'))):
        Mentions[0].append(tweets[i].id)
        Mentions[1].append(tweets[i].entities.get('user_mentions')[j]['id'])
        Mentions[2].append(tweets[i].entities.get('user_mentions')[j]['name'])
        Mentions[3].append(tweets[i].entities.get('user_mentions')[j]['screen_name'])
Mentions_df = pd.DataFrame(data = np.array(Mentions).T[1:, :], columns = ['tweet_id', 'User_id', 'name', 'screen_name'])
#---------------------
tweet_Pos = tweet_df[tweet_df.SA == 'Positive']
tweet_Neg = tweet_df[tweet_df.SA == 'Negative']
tweet_Neu = tweet_df[tweet_df.SA == 'Neutual']

writer = pd.ExcelWriter('TwitterData.xlsx', engine='xlsxwriter')
tweet_df.to_excel(writer, sheet_name = 'tweets')
Hashtags_df.to_excel(writer, sheet_name = 'hashtags')
Mentions_df.to_excel(writer, sheet_name = 'mentions')


import os
import moviepy.editor as mp

input_folder = 'Python batch video converter bot\\input'
output_folder = 'Python batch video converter bot\\converted'

target_codec = 'libx264'
target_format = 'mp4'

resolution = 480 # here you can change quality of the converted videos

try:
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    for file in os.listdir(input_folder):
        if file.endswith('.mp4'):
            input_file = os.path.join(input_folder, file)
            output_file = os.path.join(output_folder, file)

            clip = mp.VideoFileClip(input_file)
            clip = clip.fx(mp.vfx.resize, height=resolution)
            clip.write_videofile(output_file, codec=target_codec, audio_codec='aac', preset='slow')

            print('Converted:', file)
except Exception as ex:
    print(f'Error {ex}')


import streamlit as st
import database as db
from pandas import DataFrame

st.set_page_config(page_title="Travel Register", page_icon="ðŸšŒ", layout="centered")

# Hiding menu and footer (Production use only)
hide_menu_style = """
    <style>
    #MainMenu {visibility: hidden; }
    footer {visibility: hidden; }
    </style>
"""
# st.markdown(hide_menu_style, unsafe_allow_html=True)

st.title("ðŸšŒ Travel Register!")

form = st.form(key="annotation")

name = st.text_input(label="Enter your name: ")
mode = st.selectbox("Mode of transport: ", ["Car", "Bus"])
if mode == "Bus":
    routes = db.read_routes().keys()
    route = st.selectbox("Select route: ", routes)

    stops = db.read_routes()[route]
    stop = st.selectbox("Select stop: ", stops)
else:
    route = "None"
    stop = "None"

submit = st.button("Submit")

if submit:
    if not name:
        st.error("Please enter your name..!")
    elif name in db.fetch_users().keys():
        st.error("You have already submitted your data. Please contact the admin to reset your data..!")
    else:
        db.add_user(name, mode, route, stop)
        st.success("Successfully submitted your data..!")

st.write("### Admin Only: ")
key = st.text_input("Enter admin key: ")
if st.button("Show data"):
    if key.lower() == "admin":
        with st.spinner("Loading data..."):
            users = db.fetch_users()
            raw = []
            for user in users.keys():
                raw.append((user, users[user]["mode"], users[user]["route"], users[user]["stop"]))
            df = DataFrame(raw, columns=("Name", "Mode", "Route", "Stop"))
            df.index += 1
            st.dataframe(df)
    else:
        st.error("Verification failed...!")


import redis
'''
使用redis连接池
'''
redisPool=redis.ConnectionPool(host='127.0.0.1',port=6379)
r = redis.StrictRedis(connection_pool=redisPool)

def saveData(key,value):
    r.set(key,value)
    return True

def getValue(key):
    result=r.get(key)
    if result!=None:
        # r.expire(key,1800)
        result=result.decode('utf-8')
    return result

def delData(key):
    r.delete(key)
    return True

def getFlagValue(key):
    result = r.get(key)
    if result != None:
        result = result.decode('utf-8')
    return result

def getFlagValueInt(key):
    result = r.get(key)
    if result != None:
        result = result.decode('utf-8')
    else:
        result='0'
    return int(result)


# -*-coding: utf-8 -*-

import os
import coroutine

def consumer():
    r = ''
    while True:
        # get message and return with r
        n = yield r
        if not n:
            print('not %d' % n)
            return
        print('[Consumer]Consume %d...' % n)
        r = '200 OK'

def producer(c):
    # start generator
    c.send(None)

    n = 1;
    while n < 5:
        print('[Producer]produce %d' % n)
        # swith to consumer to run
        r = c.send(n)
        n += 1
        print('[Producer]consume handle return %s' % r)

    c.close()

def list_dir(path, target):
    for dirpath, dirnames, filenames in os.walk(path):
        for filename in filenames:
            print('list send', filename)
            target.send(filename)

def coroutine(func):
    def wrapper(*args, **kwargs):
        gen = func(*args, **kwargs)
        print(func.__name__, 'next begin')
        next(gen)
        print(func.__name__, 'next end')
        return gen

    wrapper.__name__ = func.__name__
    wrapper.__dict__ = func.__dict__
    wrapper.__doc__ = func.__doc__

    return wrapper

@coroutine
def filter_str(pattern, target):
    while True:
        print('filter begin')
        filename = (yield)
        if pattern in filename:
            print('filter send %s' % filename)
            target.send(filename)
            print('filter send done')

@coroutine
def print_match():
    while True:
        print('print begin')
        result = (yield)
        print(result)
        print('print end', result)

if __name__ == '__main__':
    c = consumer()
    producer(c)
    list_dir('.', filter_str('.txt', print_match()))

# output
# [Producer]produce 1
# [Consumer]Consume 1...
# [Producer]consume handle return 200 OK
# [Producer]produce 2
# [Consumer]Consume 2...
# [Producer]consume handle return 200 OK


import cv2

def digital_zoom(frame, zoom_factor, zoom_center):
    h, w = frame.shape[:2]
    x, y = zoom_center

    # Calculate the new width and height based on the zoom factor
    new_w = int(w / zoom_factor)
    new_h = int(h / zoom_factor)

    # Define the region of interest (ROI) for zooming
    x1 = max(0, x - new_w // 2)
    y1 = max(0, y - new_h // 2)
    x2 = min(w, x + new_w // 2)
    y2 = min(h, y + new_h // 2)

    # Zoom in on the ROI
    zoomed_frame = frame[y1:y2, x1:x2]

    # Resize the zoomed frame to the original frame size
    zoomed_frame = cv2.resize(zoomed_frame, (w, h))

    return zoomed_frame

def main():
    capture = cv2.VideoCapture(0)

    while True:
        ret, frame = capture.read()
        if not ret:
            break

        # Get the center coordinates of the frame
        center_x, center_y = frame.shape[1] // 2, frame.shape[0] // 2

        # Zoom factor (increase this value to zoom in)
        zoom_factor = 2

        # Apply digital zoom to the frame
        zoomed_frame = digital_zoom(frame, zoom_factor, (center_x, center_y))

        # Display the zoomed frame
        cv2.imshow('Digital Zoom', zoomed_frame)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    capture.release()
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main()


import sys

from PyQt5.QtCore import pyqtSlot
from PyQt5.QtWidgets import QDialog

from E5Gui import E5MessageBox
from E5Gui.E5ProcessDialog import E5ProcessDialog
from E5Gui.E5Application import e5App

from .MicroPythonDevices import MicroPythonDevice
from .MicroPythonWidget import HAS_QTCHART

import Preferences

class EspDevice(MicroPythonDevice):
    """
    Class implementing the device for ESP32 and ESP8266 based boards.
    """
    def __init__(self, microPythonWidget, parent=None):
        """
        Constructor

        @param microPythonWidget reference to the main MicroPython widget
        @type MicroPythonWidget
        @param parent reference to the parent object
        @type QObject
        """
        super(EspDevice, self).__init__(microPythonWidget, parent)

    def setButtons(self):
        """
        Public method to enable the supported action buttons.
        """
        super(EspDevice, self).setButtons()
        self.microPython.setActionButtons(
            run=True, repl=True, files=True, chart=HAS_QTCHART)

    def forceInterrupt(self):
        """
        Public method to determine the need for an interrupt when opening the
        serial connection.

        @return flag indicating an interrupt is needed
        @rtype bool
        """
        return True

    def deviceName(self):
        """
        Public method to get the name of the device.

        @return name of the device
        @rtype str
        """
        return self.tr("ESP8266, ESP32")

    def canStartRepl(self):
        """
        Public method to determine, if a REPL can be started.

        @return tuple containing a flag indicating it is safe to start a REPL
            and a reason why it cannot.
        @rtype tuple of (bool, str)
        """
        return True, ""

    def canStartPlotter(self):
        """
        Public method to determine, if a Plotter can be started.

        @return tuple containing a flag indicating it is safe to start a
            Plotter and a reason why it cannot.
        @rtype tuple of (bool, str)
        """
        return True, ""

    def canRunScript(self):
        """
        Public method to determine, if a script can be executed.

        @return tuple containing a flag indicating it is safe to start a
            Plotter and a reason why it cannot.
        @rtype tuple of (bool, str)
        """
        return self.canStartRepl()

    def runScript(self, script):
        """
        Public method to run the given Python script.

        @param script script to be executed
        @type str
        """
        pythonScript = script.split("\n")
        self.sendCommands(pythonScript)

    def canStartFileManager(self):
        """
        Public method to determine, if a File Manager can be started.

        @return tuple containing a flag indicating it is safe to start a
            File Manager and a reason why it cannot.
        @rtype tuple of (bool, str)
        """
        return True, ""

    def addDeviceMenuEntries(self, menu):
        """
        Public method to add device specific entries to the given menu.

        @param menu reference to the context menu
        @type QMenu
        """
        connected = self.microPython.isConnected()

        act = menu.addAction(self.tr("Erase Flash"),
                             self.__eraseFlash)
        act.setEnabled(not connected)
        act = menu.addAction(self.tr("Flash MicroPython Firmware"),
                             self.__flashMicroPython)
        act.setEnabled(not connected)
        menu.addSeparator()
        act = menu.addAction(self.tr("Flash Additional Firmware"),
                             self.__flashAddons)
        act.setEnabled(not connected)
        menu.addSeparator()
        act = menu.addAction(self.tr("Reset Device"), self.__resetDevice)
        menu.addSeparator()
        menu.addAction(self.tr("Install 'esptool.py'"), self.__installEspTool)

    @pyqtSlot()
    def __eraseFlash(self):
        """
        Private slot to erase the device flash memory.
        """
        ok = E5MessageBox.yesNo(
            self.microPython,
            self.tr("Erase Flash"),
            self.tr("""Shall the flash of the selected device really be"""
                    """ erased?"""))
        if ok:
            flashArgs = [
                "-u",
                "-m", "esptool",
                "--port", self.microPython.getCurrentPort(),
                "erase_flash",
            ]
            dlg = E5ProcessDialog(self.tr("'esptool erase_flash' Output"),
                                  self.tr("Erase Flash"))
            res = dlg.startProcess(sys.executable, flashArgs)
            if res:
                dlg.exec_()

    @pyqtSlot()
    def __flashMicroPython(self):
        """
        Private slot to flash a MicroPython firmware to the device.

        @exception ValueError raised to indicate an unsupported chip type
        """
        from .EspFirmwareSelectionDialog import EspFirmwareSelectionDialog
        dlg = EspFirmwareSelectionDialog()
        if dlg.exec_() == QDialog.Accepted:
            chip, firmware, _ = dlg.getData()
            if chip == "esp8266":
                flashAddress = "0x0000"
            elif chip == "esp32":
                flashAddress = "0x1000"
            else:
                raise ValueError(self.tr("Unsupported chip type '{0}'.")
                                 .format(chip))
            flashArgs = [
                "-u",
                "-m", "esptool",
                "--chip", chip,
                "--port", self.microPython.getCurrentPort(),
                "write_flash",
                flashAddress,
                firmware,
            ]
            dlg = E5ProcessDialog(self.tr("'esptool write_flash' Output"),
                                  self.tr("Flash MicroPython Firmware"))
            res = dlg.startProcess(sys.executable, flashArgs)
            if res:
                dlg.exec_()

    @pyqtSlot()
    def __flashAddons(self):
        """
        Private slot to flash some additional firmware images.
        """
        from .EspFirmwareSelectionDialog import EspFirmwareSelectionDialog
        dlg = EspFirmwareSelectionDialog(addon=True)
        if dlg.exec_() == QDialog.Accepted:
            chip, firmware, flashAddress = dlg.getData()
            flashArgs = [
                "-u",
                "-m", "esptool",
                "--chip", chip,
                "--port", self.microPython.getCurrentPort(),
                "write_flash",
                flashAddress.lower(),
                firmware,
            ]
            dlg = E5ProcessDialog(self.tr("'esptool write_flash' Output"),
                                  self.tr("Flash Additional Firmware"))
            res = dlg.startProcess(sys.executable, flashArgs)
            if res:
                dlg.exec_()

    @pyqtSlot()
    def __resetDevice(self):
        """
        Private slot to reset the connected device.
        """
        self.microPython.commandsInterface().execute([
            "import machine",
            "machine.reset()",
        ])

    @pyqtSlot()
    def __installEspTool(self):
        """
        Private slot to install the esptool package via pip.
        """
        pip = e5App().getObject("Pip")
        pip.installPackages(["esptool"], interpreter=sys.executable)

    def getDocumentationUrl(self):
        """
        Public method to get the device documentation URL.

        @return documentation URL of the device
        @rtype str
        """
        return Preferences.getMicroPython("MicroPythonDocuUrl")


import streamlit as st
from streamlit_option_menu import option_menu
import machine_learning as ml
import feature_extraction as fe
from bs4 import BeautifulSoup
import requests as re
import matplotlib.pyplot as plt
import validators
import pandas as pd
from sklearn import datasets
from sklearn.ensemble import RandomForestClassifier
import shutil
from st_aggrid import AgGrid
import streamlit.components.v1 as components
import base64

# 2. horizontal menu
# selected2 = option_menu(None, ["Home", "Upload", "Tasks", 'Settings'],
#     icons=['house', 'cloud-upload', "list-task", 'gear'],
#     menu_icon="cast", default_index=0, orientation="horizontal")
# selected2

with st.sidebar:
    selected = option_menu("Navigation Menu", ['UltraSec Phishing Detection','Phishing Prediction','Phishing Prediction By Upload CSV File','Project Details','Example',
    'Model Test Result','Data Set','Extension Download','Evil Twin Detector Download'],
        icons=['house','caret-right','book','list-task','bar-chart-line','caret-right','caret-right','download','download'], menu_icon="cast", default_index=0)
if selected == "UltraSec Phishing Detection":
    st.title('Phishing Detection')
    # st.write('This ML-based app is developed for educational purposes. Objective of the app is detecting phishing websites only using content data. Not URL!'
    #          ' You can see the details of approach, data set, and feature set if you click on _"See The Details"_. ')
    choice = st.selectbox("Please select your machine learning model",
                    [
                        'Logistic Regression','Gaussian Naive Bayes', 'Support Vector Machine', 'Decision Tree', 'Random Forest',
                        'AdaBoost', 'Neural Network', 'K-Neighbours'
                    ]
                    )
    model = ml.nb_model
    if choice == 'Logistic Regression':
        model = ml.lr_model
        st.write('LR model is selected!')
    elif choice == 'Gaussian Naive Bayes':
        model = ml.nb_model
        st.write('GNB model is selected!')
    elif choice == 'Support Vector Machine':
        model = ml.svm_model
        st.write('SVM model is selected!')
    elif choice == 'Decision Tree':
        model = ml.dt_model
        st.write('DT model is selected!')
    elif choice == 'Random Forest':
        model = ml.rf_model
        st.write('RF model is selected!')
    elif choice == 'AdaBoost':
        model = ml.ab_model
        st.write('AB model is selected!')
    elif choice == 'Neural Network':
        model = ml.nn_model
        st.write('NN model is selected!')
    else:
        model = ml.kn_model
        st.write('KN model is selected!')
    url = st.text_input('Enter the URL')
    # check the url is valid or not
    if st.button('Check!'):
        my_bar = st.progress(0)
        with st.spinner('Wait for it...'):
            try:
                valid=validators.url(url)
                if valid==True:
                    st.success("Url Format is valid")
                else:
                    st.warning("Invalid url")
                response = re.get(url, verify=False, timeout=4)
                if response.status_code == 404:
                    st.warning("404 Client Error. HTTP connection was not successful for the URL: "+url)
                elif response.status_code != 200:
                    print(". HTTP connection was not successful for the URL: ", url)
                    st.warning("Attention! This web page is a potential PHISHING! HTTP connection was not successful for the URL: "+url)
                else:
                    soup = BeautifulSoup(response.content, "html.parser")
                    vector = [fe.create_vector(soup)]  # it should be 2d array, so I added []
                    result = model.predict(vector)
                    if result[0] == 0:
                        st.success("This web page seems a legitimate!")
                        st.balloons()
                    else:
                        st.warning("Attention! This web page is a potential PHISHING!")
                        st.snow()
            except re.exceptions.RequestException as e:
                print("--> ", e)
                st.error(e,icon=None)
        my_bar = st.progress(100)
        st.success('Done!')
        # except re.exceptions.HTTPError as err:
        #     raise SystemExit(err)
if selected == "Phishing Prediction":
    st.title('Phishing Detection')
    # check the url is valid or not
    def evaluate(index,model,url):
        with st.spinner('Wait for it...'):
            try:
                valid=validators.url(url)
                if valid==True:
                    # st.success("Url Format is valid")
                    url_valid = "Url Format is valid"
                else:
                    # st.warning("Invalid url")
                    url_valid = "Invalid url"

                response = re.get(url, verify=False, timeout=4)
                if response.status_code == 404:
                    # st.warning("404 Client Error. HTTP connection was not successful for the URL: "+url)
                    url_response = "404 Client Error. HTTP connection was not successful for the URL: "
                elif response.status_code != 200:
                    print(". HTTP connection was not successful for the URL: ", url)
                    # st.warning("Attention! This web page is a potential PHISHING! HTTP connection was not successful for the URL: "+url)
                    url_response = "Attention! This web page is a potential PHISHING! HTTP connection was not successful for the URL: "
                else:
                    soup = BeautifulSoup(response.content, "html.parser")
                    vector = [fe.create_vector(soup)]  # it should be 2d array, so I added []
                    result = model.predict(vector)
                    if result[0] == 0:
                        # st.success("This web page seems a legitimate!")
                        url_response = "This web page seems a legitimate!"
                        # st.balloons()
                    else:
                        # st.warning("Attention! This web page is a potential PHISHING!")
                        url_response ="Attention! This web page is a potential PHISHING!"
                        # st.snow()
                data_return = {}
                data_return = {
                    index : [
                        url_valid,
                        url_response
                    ]
                }
                # data_return = {
                #     index : index,
                #     index+ ' Format':url_valid,
                #     index+ ' Result':url_response,
                # }

            except re.exceptions.RequestException as e:
                print("--> ", e)
                st.error(e,icon=None)
        return data_return
    url = st.text_input('Enter the URL')
    if st.button('Check!'):
        my_bar = st.progress(0)
        complete_progress = 0

        # st.write('Logistic Regression')
        model = ml.lr_model
        index = 'LR'
        lr_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('Gaussian Naive Bayes')
        model = ml.nb_model
        index = 'NB'
        gnb_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('Support Vector Machine')
        model = ml.svm_model
        index = 'SVM'
        svm_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('Decision Tree')
        model = ml.dt_model
        index = 'DT'
        dt_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('Random Forest')
        model = ml.rf_model
        index = 'RF'
        rf_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('AdaBoost')
        model = ml.ab_model
        index = 'AB'
        ab_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('Neural Network')
        model = ml.nn_model
        index = 'NN'
        nn_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('K-Neighbours')
        model = ml.kn_model
        index = 'KN'
        kn_result = evaluate(index,model,url)
        complete_progress+=9
        my_bar.progress(complete_progress)

        # def Merge(dict1, dict2,dict3,dict4,dict5,dict6,dict7,dict8):
        #     res = {**dict1, **dict2,**dict3,**dict4,**dict5,**dict6,**dict7,**dict8}
        #     return res

        column_index = ['Logistic Regression','Gaussian Naive Bayes', 'Support Vector Machine', 'Decision Tree', 'Random Forest',
        'AdaBoost', 'Neural Network', 'K-Neighbours']

        # Driver code
        dict_result = {k:v for d in (lr_result,gnb_result,svm_result,dt_result,rf_result,ab_result,nn_result,kn_result) for k,v in d.items()}
        # dict_result= Merge(lr_result,gnb_result,svm_result,dt_result,rf_result,ab_result,nn_result,kn_result)
        # st.write(dict_result)
        # dict_result = {k:v for d in (lr_result,gnb_result) for k,v in d.items()}

        # lr = pd.DataFrame(lr_result)
        # nb = pd.DataFrame(gnb_result)

        # lr.reset_index(drop=True, inplace=True)
        # nb.reset_index(drop=True, inplace=True)

        # concated = pd.concat([lr,nb],axis=0)
        # st.write(dict_result)

        # result_df = pd.concat([pd.DataFrame(lr_result),pd.DataFrame(gnb_result),
        # pd.DataFrame(svm_result),pd.DataFrame(dt_result),pd.DataFrame(rf_result),
        # pd.DataFrame(ab_result),pd.DataFrame(nn_result,kn_result)],axis=0,index=indexes)
        # # st.write(result_df)
        row_index = ['URL Format Validity','Result']
        result_df = pd.DataFrame(data=dict_result,index=row_index)
        result_df_T = result_df.transpose()
        result_concat = pd.concat([ml.df_results,result_df_T],axis=1)
        result_concat.index = column_index
        st.success('Done!')
        st.dataframe(result_concat)
        # AgGrid(result_concat)
if selected == "Phishing Prediction By Upload CSV File":
    st.header('Phishing Prediction By Upload CSV File')
    def evaluate_csv(url):
        try:
            valid=validators.url(url)
            if valid==True:
                # st.success("Url Format is valid")
                dataframe['URL Validity'] = 'Valid'
            else:
                # st.warning("Invalid url")
                dataframe['URL Format Validity'] = 'Invalid'
            response = re.get(url, verify=False, timeout=4)
            if response.status_code == 404:
                # st.warning("404 Client Error. HTTP connection was not successful for the URL: "+url)
                dataframe['Response'] = '404 Error'
            elif response.status_code != 200:
                # print(". HTTP connection was not successful for the URL: ", url)
                # st.warning("Attention! This web page is a potential PHISHING! HTTP connection was not successful for the URL: "+url)
                dataframe['Response'] = 'HTTP Connection was not successful'
            else:
                soup = BeautifulSoup(response.content, "html.parser")
                vector = [fe.create_vector(soup)]  # it should be 2d array, so I added []
                result = model.predict(vector)
                if result[0] == 0:
                    # st.success("This web page seems a legitimate!")
                    # st.balloons()
                    dataframe['Response'] = 'Legitimate'
                else:
                    # st.warning("Attention! This web page is a potential PHISHING!")
                    # st.snow()
                    dataframe['Response'] = 'Phishing'
            dataframe['Error'] = ''
        except re.exceptions.RequestException as e:
            print("--> ", e)
            # st.error(e,icon=None)
            dataframe['Error'] = e

    # if "points_1" not in st.session_state:
    #     # points for guesses
    #     st.session_state.points_1 = 0

    @st.cache
    def convert_df(df):
        # IMPORTANT: Cache the conversion to prevent computation on every rerun
        return df.to_csv().encode('utf-8')

    uploaded_file = st.file_uploader("Choose a file")
    if uploaded_file is not None:
        # Can be used wherever a "file-like" object is accepted:
        chunk = pd.read_csv(uploaded_file,chunksize=1000)
        dataframe = pd.concat(chunk)
        st.write(dataframe)

        choice = st.selectbox("Please select your machine learning model",
                [
                    'Logistic Regression','Gaussian Naive Bayes', 'Support Vector Machine', 'Decision Tree', 'Random Forest',
                    'AdaBoost', 'Neural Network', 'K-Neighbours'
                ]
                )
        model = ml.nb_model
        if choice == 'Logistic Regression':
            model = ml.lr_model
            st.write('LR model is selected!')
        elif choice == 'Gaussian Naive Bayes':
            model = ml.nb_model
            st.write('GNB model is selected!')
        elif choice == 'Support Vector Machine':
            model = ml.svm_model
            st.write('SVM model is selected!')
        elif choice == 'Decision Tree':
            model = ml.dt_model
            st.write('DT model is selected!')
        elif choice == 'Random Forest':
            model = ml.rf_model
            st.write('RF model is selected!')
        elif choice == 'AdaBoost':
            model = ml.ab_model
            st.write('AB model is selected!')
        elif choice == 'Neural Network':
            model = ml.nn_model
            st.write('NN model is selected!')
        else:
            model = ml.kn_model
            st.write('KN model is selected!')

        if len(dataframe) <51 :
            # check the url is valid or not
            if st.button('Check!'):
                if uploaded_file is not None:
                    my_bar = st.progress(0)
                    with st.spinner('Wait for it...'):
                        dataframe.apply(lambda row : evaluate_csv(row['url']),axis=1)
                        AgGrid(dataframe)
                    my_bar.progress(100)
                    st.success('Done!')
                    csv = convert_df(dataframe)
                    st.download_button(
                        label="Download Prediction (CSV)",
                        data=csv,
                        file_name='Download Result(CSV).csv',
                        mime='text/csv',
                        help='Download Prediction Result in CSV format '
                    )
        else:
            st.write('Maximum of 50 rows of Urls are accepted. PLEASE ReUpload')
if selected == "Project Details":
    # with st.expander("PROJECT DETAILS"):
    st.subheader('Approach')
    st.write('I used _supervised learning_ to classify phishing and legitimate websites. '
            'I benefit from content-based approach and focus on html of the websites. '
            'Also, I used scikit-learn for the ML models.'
            )
    st.write('For this educational project, '
            'I created my own data set and defined features, some from the literature and some based on manual analysis. '
            'I used requests library to collect data, BeautifulSoup module to parse and extract features. ')
    st.write('The source code and data sets are available in the below Github link:')
    st.write('_https://github.com/emre-kocyigit/phishing-website-detection-content-based_')
    st.subheader('Data set')
    st.write('I used _"phishtank.org"_ & _"tranco-list.eu"_ as data sources.')
    st.write('Totally 26584 websites ==> **_16060_ legitimate** websites | **_10524_ phishing** websites')
    st.write('Data set was created in October 2022.')
    # ----- FOR THE PIE CHART ----- #
    labels = 'phishing', 'legitimate'
    phishing_rate = int(ml.phishing_df.shape[0] / (ml.phishing_df.shape[0] + ml.legitimate_df.shape[0]) * 100)
    legitimate_rate = 100 - phishing_rate
    sizes = [phishing_rate, legitimate_rate]
    explode = (0.1, 0)
    fig, ax = plt.subplots()
    ax.pie(sizes, explode=explode, labels=labels, shadow=True, startangle=90, autopct='%1.1f%%')
    ax.axis('equal')
    st.pyplot(fig)
    # ----- !!!!! ----- #
    st.write('Features + URL + Label ==> Dataframe')
    st.markdown('label is 1 for phishing, 0 for legitimate')
    number = st.slider("Select row number to display", 0, 100)
    st.dataframe(ml.legitimate_df.head(number))
    @st.cache
    def convert_df(df):
        # IMPORTANT: Cache the conversion to prevent computation on every rerun
        return df.to_csv().encode('utf-8')
    csv = convert_df(ml.df)
    st.download_button(
        label="Download data as CSV",
        data=csv,
        file_name='phishing_legitimate_structured_data.csv',
        mime='text/csv',
    )
if selected == "Example":
    # with st.expander('EXAMPLE PHISHING URLs:'):
    st.title('EXAMPLE PHISHING URLs:')
    st.write('_https://rtyu38.godaddysites.com/_')
    st.write('_https://karafuru.invite-mint.com/_')
    st.write('_https://defi-ned.top/h5/#/_')
    st.write('_https://beeflash.net/_')
    st.caption('REMEMBER, PHISHING WEB PAGES HAVE SHORT LIFECYCLE! SO, THE EXAMPLES SHOULD BE UPDATED!')
if selected == "Model Test Result":
    st.subheader('Features')
    st.write('I used only content-based features. I didn\'t use url-based faetures like length of url etc.'
            'Most of the features extracted using find_all() method of BeautifulSoup module after parsing html.')
    st.subheader('Results')
    st.write('I used 8 different ML classifiers of scikit-learn and tested them implementing k-fold cross validation.'
            'Firstly obtained their confusion matrices, then calculated their accuracy, precision and recall scores.'
            'Comparison table is below:')
    st.table(ml.df_results)
    st.write('LR --> Logistic Regression')
    st.write('NB --> Gaussian Naive Bayes')
    st.write('SVM --> Support Vector Machine')
    st.write('DT --> Decision Tree')
    st.write('RF --> Random Forest')
    st.write('AB --> AdaBoost')
    st.write('NN --> Neural Network')
    st.write('KN --> K-Neighbours')
if selected == "Data Set":
    @st.cache
    def load_data(nrows):
        data = pd.read_csv('malicious_phish.csv', nrows=nrows)
        return data
    # st.text_input(label, value="", max_chars=None, key=None, type="default", help=None, autocomplete=None, on_change=None, args=None, kwargs=None, *, placeholder=None, disabled=False, label_visibility="visible")
    phish_data = load_data(1000)
    # st.write(weekly_data)
    # st.subheader('Weekly Demand Data')
    # AgGrid(phish_data)

    number = st.slider("Select row number to display", 50, 500)
    st.dataframe(phish_data.head(number))
    # #Bar Chart
    # st.bar_chart(weekly_data)
if selected == "Extension Download":
    st.title('Download Page')
    st.subheader('Download the latest version Ultrasec Phishing Detector')
    shutil.make_archive('phishing-detection-plugin-main', 'zip', 'phishing-detection-plugin-main')
    with open("phishing-detection-plugin-main.zip", "rb") as fp:
        btn1 = st.download_button(
            label="Download UltraSec Phishing Detector 1.0.1",
            data=fp,
            file_name='UltraSec-Phishing-Detector.zip',
            mime='application/zip',
            key="1",
        )
    st.markdown("Looking for way to install extension? [**Watch Video Below**](#video-to-install-ultrasec-phishing-detector-chrome-extension)")
    st.subheader('Instructions')
    st.markdown("[1. Step 1 - download a zip file with the extension](#step-1-download-a-zip-file-with-the-extension)")
    st.markdown("[2. Step 2 - extract the contents of the zip file](#step-2-extract-the-contents-of-the-zip-file)")
    st.markdown("[3. Step 3 - open the extension page in google chrome](#step-3-open-the-extension-page-in-google-chrome)")
    st.markdown("[4. Step 4 - activate developer mode](#step-4-activate-developer-mode)")
    st.markdown("[5. Step 5 - load unpacked extension](#step-5-load-unpacked-extension)")
    st.header('Step 1 - download a zip file with the extension')
    with open("phishing-detection-plugin-main.zip", "rb") as fp:
        btn2 = st.download_button(
            label="Download UltraSec Phishing Detector 1.0.1",
            data=fp,
            file_name='UltraSec-Phishing-Detector.zip',
            mime='application/zip',
            key="2",
        )
    st.header('Step 2 - extract the contents of the zip file')
    st.write('Right click on the downloaded zip file, then click "Extract Here".')
    st.header('Step 3 - open the extension page in google chrome')
    st.write('''There are several ways todo that.

    Option 1: type chrome://extensions in the url bar and press enter.
    Option 2: click on the tree dots in the top right of the browser, then click "More tools" then click "Extensions".
    ''')
    st.header('Step 4 - activate developer mode')
    st.write('Turn on the switch on the top right of the page that says "Developer mode"')
    st.header('Step 5 - load unpacked extension')
    st.write('''Click on the button on the top left of the page that says "Load unpacked".

    Then select a folder that contains the manifest.json file.
    ''')
    st.header('Video to install UltraSec Phishing Detector Chrome Extension.')
    video_file = open('install_extension.mp4', 'rb')
    video_bytes = video_file.read()
    st.video(video_bytes)
if selected == 'Evil Twin Detector Download':
    # embed streamlit docs in a streamlit app
    # components.iframe("https://docs.streamlit.io/en/latest")
    # components.html("""

    # """)

    # def show_pdf(file_path):
    #     with open(file_path,"rb") as f:
    #         base64_pdf = base64.b64encode(f.read()).decode('utf-8')
    #     pdf_display = f'<iframe src="data:application/pdf;base64,{base64_pdf}" width="800" height="800" type="application/pdf"></iframe>'
    #     st.markdown(pdf_display, unsafe_allow_html=True)

    # show_pdf('training.pdf')

    st.header('Evil Twin Detector Download')
    st.caption('An evil twin attack takes place when an attacker sets up a fake Wi-Fi access point hoping that users will connect to it instead of a legitimate one. When users connect to this access point, all the data they share with the network passes through a server controlled by the attacker. An attacker can create an evil twin with a smartphone or other internet-capable device and some readily available software. Evil twin attacks are more common on public Wi-Fi networks which are unsecured and leave your personal data vulnerable.')
    st.markdown('[**Download Evin Twin Detector**](https://github.com/ryanlee13288/EvilTwinDetectFYP)')


#!/usr/bin/env python
import rospy
import sys
import tensorflow as tf
from tensorflow.contrib.layers import flatten
from std_msgs.msg import UInt8
from std_msgs.msg import String
from sensor_msgs.msg import Image
import numpy as np
import pickle
import cv2
import csv
from cv_bridge import CvBridge, CvBridgeError
import glob
import time
from sklearn.svm import LinearSVC
from sklearn.utils import shuffle
from sklearn.externals import joblib
# for scikit-learn <= 0.17 use:
from sklearn.cross_validation import train_test_split
# otherwise:
# from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from skimage.feature import hog
from scipy.ndimage.measurements import label

bridge = CvBridge()

color_space = 'YCrCb'
orient = 11
pix_per_cell = 8
cell_per_block = 2
hog_channel = 'ALL'
spatial_size = (16, 16)
hist_bins = 32
spatial_feat = True
hist_feat = True
hog_feat = True

EPOCHS = 50
BATCH_SIZE = 128

svc = joblib.load("Saved Training Model/train_model_2.m")
X_scaler = joblib.load("Saved Training Model/scaler_2.save")

label_names = []
with open('./signnames.csv', 'rt') as f:
	reader = csv.reader(f)
	label_names = list(reader)

# read the traffic sign name file
label_name = []
for i in range(43):
	label_name.append(label_names[i+1][1])

def normalized(data):
	return (data/255.0 - 0.5)

def LeNet(x):
	# Arguments used for tf.truncated_normal, randomly defines variables for the weights and biases for each layer
	mu = 0
	sigma = 0.1

	# SOLUTION: Layer 1: Convolutional. Input = 32x32x3. Output = 30x30x32.
	conv1_W = tf.Variable(tf.truncated_normal(shape=(3, 3, 3, 32), mean = mu, stddev = sigma))
	conv1_b = tf.Variable(tf.zeros(32))
	conv1   = tf.nn.conv2d(x, conv1_W, strides=[1, 1, 1, 1], padding='VALID') + conv1_b

	# SOLUTION: Activation.
	conv1 = tf.nn.relu(conv1)

	# SOLUTION: Pooling. Input = 30x30x32. Output = 15x15x32.
	conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')

	# SOLUTION: Layer 2: Convolutional. Input = 15x15x32. Output = 13x13x64.
	conv2_W = tf.Variable(tf.truncated_normal(shape=(3, 3, 32, 64), mean = mu, stddev = sigma))
	conv2_b = tf.Variable(tf.zeros(64))
	conv2   = tf.nn.conv2d(conv1, conv2_W, strides=[1, 1, 1, 1], padding='VALID') + conv2_b

	# SOLUTION: Activation.
	conv2 = tf.nn.relu(conv2)

	# SOLUTION: Pooling. Input = 13x13x64. Output = 6x6x64.
	conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')

	# SOLUTION: Layer 3: Convolutional. Input = 6x6x64. Output = 4x4x128.
	conv3_W = tf.Variable(tf.truncated_normal(shape=(3, 3, 64, 128), mean = mu, stddev = sigma))
	conv3_b = tf.Variable(tf.zeros(128))
	conv3   = tf.nn.conv2d(conv2, conv3_W, strides=[1, 1, 1, 1], padding='VALID') + conv3_b

	# SOLUTION: Activation.
	conv3 = tf.nn.relu(conv3)

	# SOLUTION: Pooling. Input = 4x4x128. Output = 2x2x128.
	conv3 = tf.nn.max_pool(conv3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')

	# SOLUTION: Flatten. Input = 2x2x128. Output = 512.
	fc0   = flatten(conv3)

	# SOLUTION: Layer 4: Fully Connected. Input = 512. Output = 120.
	fc1_W = tf.Variable(tf.truncated_normal(shape=(512, 120), mean = mu, stddev = sigma))
	fc1_b = tf.Variable(tf.zeros(120))
	fc1   = tf.matmul(fc0, fc1_W) + fc1_b

	# SOLUTION: Activation.
	fc1    = tf.nn.relu(fc1)

	# SOLUTION: Layer 5: Fully Connected. Input = 120. Output = 84.
	fc2_W  = tf.Variable(tf.truncated_normal(shape=(120, 84), mean = mu, stddev = sigma))
	fc2_b  = tf.Variable(tf.zeros(84))
	fc2    = tf.matmul(fc1, fc2_W) + fc2_b

	# SOLUTION: Activation.
	fc2    = tf.nn.relu(fc2)

	# SOLUTION: Layer 6: Fully Connected. Input = 84. Output = 43.
	fc3_W  = tf.Variable(tf.truncated_normal(shape=(84, 43), mean = mu, stddev = sigma))
	fc3_b  = tf.Variable(tf.zeros(43))
	logits = tf.matmul(fc2, fc3_W) + fc3_b

	return logits

x = tf.placeholder(tf.float32, (None, 32, 32, 3))
logits = LeNet(x)
saver = tf.train.Saver()
init_g = tf.global_variables_initializer()

# prototype functions for Histogram of Oriented Gradients(HOG):

def color_hist(img, nbins=32, bins_range=(0, 256)):
	# Compute the histogram of the RGB channels separately
	channel1_hist = np.histogram(img[:,:,0], bins=nbins, range=bins_range)
	channel2_hist = np.histogram(img[:,:,1], bins=nbins, range=bins_range)
	channel3_hist = np.histogram(img[:,:,2], bins=nbins, range=bins_range)
	# Concatenate the histograms into a single feature vector
	hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))
	# Return the color histograms
	return hist_features

def bin_spatial(img, size=(16, 16)):
	# Use cv2.resize().ravel() to create the feature vector
	features = cv2.resize(img, size).ravel() # Remove this line!
	# Return the feature vector
	return features

def get_hog_features(img, orient, pix_per_cell, cell_per_block,
						vis=False, feature_vec=True):
	# Call with two outputs if vis==True
	if vis == True:
		hog_features, hog_image = hog(img, orientations=orient,
								  pixels_per_cell=(pix_per_cell, pix_per_cell),
								  cells_per_block=(cell_per_block, cell_per_block),
								  transform_sqrt=False, block_norm = 'L2-Hys',
								  visualise=vis, feature_vector=feature_vec)
		return hog_features, hog_image
	# Otherwise call with one output
	else:
		hog_features = hog(img, orientations=orient,
					   pixels_per_cell=(pix_per_cell, pix_per_cell),
					   cells_per_block=(cell_per_block, cell_per_block),
					   transform_sqrt=False, block_norm = 'L2-Hys',
					   visualise=vis, feature_vector=feature_vec)
		return hog_features

# prototype functions for Sliding Window Search

# Define a function to draw bounding boxes
def draw_boxes(img, bboxes, color=(0, 0, 255), thick=6):
	# Make a copy of the image
	draw_img = np.copy(img)
	# Iterate through the bounding boxes
	for bbox in bboxes:
		# Draw a rectangle given bbox coordinates
		cv2.rectangle(draw_img, bbox[0], bbox[1], color, thick)
	# Return the image copy with boxes drawn
	return draw_img

def find_signs(img, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell,
			  cell_per_block, spatial_size, hist_bins, hog_channel):

	draw_img = np.copy(img)
	img = img.astype(np.float32)

	img_tosearch = img[ystart:ystop,:,:]
	ctrans_tosearch = convert_color(img_tosearch, conv='RGB2YCrCb')
	if scale != 1:
		imshape = ctrans_tosearch.shape
		ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))

	# select colorspace channel for HOG
	if hog_channel == 'ALL':
		ch1 = ctrans_tosearch[:,:,0]
		ch2 = ctrans_tosearch[:,:,1]
		ch3 = ctrans_tosearch[:,:,2]
	else:
		ch1 = ctrans_tosearch[:,:,hog_channel]

	# Define blocks and steps as above
	nxblocks = (ch1.shape[1] // pix_per_cell) - cell_per_block + 1
	nyblocks = (ch1.shape[0] // pix_per_cell) - cell_per_block + 1
	nfeat_per_block = orient*cell_per_block**2

	# 32 was the orginal sampling rate, with 4 cells and 8 pix per cell
	window = 32
	nblocks_per_window = (window // pix_per_cell) - cell_per_block + 1
	cells_per_step = 1  # Instead of overlap, define how many cells to step
	nxsteps = (nxblocks - nblocks_per_window) // cells_per_step + 1
	nysteps = (nyblocks - nblocks_per_window) // cells_per_step + 1

	# Compute individual channel HOG features for the entire image
	hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
	if hog_channel == 'ALL':
		hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
		hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)

	# Initialize postive windows
	on_windows = []

	for xb in range(nxsteps):
		for yb in range(nysteps):
			ypos = yb*cells_per_step
			xpos = xb*cells_per_step
			# Extract HOG for this patch
			hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
			if hog_channel == 'ALL':
				hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
				hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
				hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3))
			else:
				hog_features = hog_feat1

			xleft = xpos*pix_per_cell
			ytop = ypos*pix_per_cell

			# Extract the image patch
			subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window, xleft:xleft+window], (32,32))

			# Get color features
			spatial_features = bin_spatial(subimg, size=spatial_size)
			hist_features = color_hist(subimg, nbins=hist_bins)

			# Scale features and make a prediction
			test_features = X_scaler.transform(np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1))
			#test_features = X_scaler.transform(np.hstack((shape_feat, hist_feat)).reshape(1, -1))
			test_prediction = svc.predict(test_features)

			if test_prediction == 1:
				xbox_left = np.int(xleft*scale)
				ytop_draw = np.int(ytop*scale)
				win_draw = np.int(window*scale)
				cv2.rectangle(draw_img,(xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart),(0,0,255),6)
				on_windows.append(((xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart)))

	return draw_img, on_windows

def add_heat(heatmap, bbox_list):
	# Iterate through list of bboxes
	for box in bbox_list:
		# Add += 1 for all pixels inside each bbox
		# Assuming each "box" takes the form ((x1, y1), (x2, y2))
		heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1

	# Return updated heatmap
	return heatmap# Iterate through list of bboxes

def apply_threshold(heatmap, threshold):
	# Zero out pixels below the threshold
	heatmap[heatmap <= threshold] = 0
	# Return thresholded map
	return heatmap

def draw_labeled_bboxes(img, labels):
	sign_label = []
	sign_position = []
	# Iterate through all detected signs
	for car_number in range(1, labels[1]+1):
		# Find pixels with each car_number label value
		nonzero = (labels[0] == car_number).nonzero()
		# Identify x and y values of those pixels
		nonzeroy = np.array(nonzero[0])
		nonzerox = np.array(nonzero[1])
		### set the threshold for the size of box, to make sure it only consider the sign near the robot
		if (np.max(nonzerox) - np.min(nonzerox) >= 60 and np.max(nonzeroy) - np.min(nonzeroy) >= 60):
			# Define a bounding box based on min/max x and y
			bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
			# Draw the box on the image
			cv2.rectangle(img, bbox[0], bbox[1], (0,1,0), 3)
			sign_label.append(car_number)
			position = [np.min(nonzerox), np.min(nonzeroy), np.max(nonzerox), np.max(nonzeroy)]
			sign_position.append(position)
	# Return the image
	return img, sign_label, sign_position

def convert_color(img, conv='RGB2YCrCb'):
	if conv == 'RGB2YCrCb':
		return cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)
	if conv == 'BGR2YCrCb':
		return cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
	if conv == 'RGB2LUV':
		return cv2.cvtColor(img, cv2.COLOR_RGB2LUV)

def detector(image):

	image_backup = np.copy(image)

	windows = []

	search_config = [(100, 180, 1.5), (96, 192, 2), (80, 200, 2,5), (72, 232, 3), (64, 232, 3.5),
					 (64, 232, 4), (64, 232, 5), (40, 300, 6), (40, 300, 8)]
	# search_config = [(190, 280, 1), (200, 300, 1.5), (150, 480, 2.5)]

	for config in search_config:
		out_img, out_windows = find_signs(image, config[0], config[1], config[2], svc, X_scaler,
				orient, pix_per_cell, cell_per_block, spatial_size, hist_bins, hog_channel)
		windows.append(out_windows)

	windows = [item for sublist in windows for item in sublist]

	# Initialize the heatmap
	heat = np.zeros_like(image[:,:,0].astype(np.float))

	# Add heat to each windows in windows list
	heat = add_heat(heat, windows)

	# Apply threshold to help remove false positives
	heat = apply_threshold(heat, 6)

	# Visualize the heatmap when displaying
	heatmap = np.clip(heat, 0, 255)

	# Find final windows from heatmap using lable function
	labels = label(heatmap)
	num_signs = labels[1]
	draw_img, sign_label, sign_position = draw_labeled_bboxes(np.copy(image), labels)

	# judge the types of traffic sign
	sign_types= []
	for i in range(len(sign_label)):
		sign = image_backup[sign_position[i][1]:sign_position[i][3], sign_position[i][0]:sign_position[i][2]]
		sign = np.uint8(sign*255)
		sign = cv2.resize(sign, (32, 32))
		sign = cv2.cvtColor(sign, cv2.COLOR_RGB2BGR)
		sign = np.expand_dims(sign, axis=0)
		sign_normalized = normalized(sign)
		sign_type = sess.run(tf.argmax(logits, 1), feed_dict={x: sign_normalized})
		sign_types.append(sign_type)

		# anotate the traffic sign name in image
		cv2.putText(draw_img, '%s' %label_name[int('%d' %sign_type)], (sign_position[i][0],
					sign_position[i][1]-10), cv2.FONT_HERSHEY_COMPLEX, 0.8, (0,1,0), 2)
#         cv2.putText(draw_img, '%d: %s' %(sign_type, label_name[sign_type]), (sign_position[i][1]-20,
#                     sign_position[i][0]), cv2.FONT_HERSHEY_COMPLEX, 6, (0,0,255), 25)

	return np.uint8(draw_img*255), sign_types

def publish_image(image, args):
	draw_pub = args[0]
	sign_pub = args[1]

	cv2_img = bridge.imgmsg_to_cv2(image, "rgb8")

	rospy.loginfo("image received")

	# convert to mpimg.imread form
	mpimg = cv2_img/(255.0)
	draw, sign_types = detector(mpimg)
	drawmsg = bridge.cv2_to_imgmsg(draw, "rgb8")
	draw_pub.publish(drawmsg)
	for sign_type in sign_types:
		sign_pub.publish(sign_type)

	return

def get_image():
	rospy.init_node('get_image')

	image_topic = "/color_filter"

	bridge = CvBridge()
	draw_pub = rospy.Publisher("draw_topic", Image, queue_size=1)
	sign_pub = rospy.Publisher("sign_types", UInt8, queue_size=1)
	# heat_pub = rospy.Publisher("heat_topic", Image, queue_size=1)
	rospy.Subscriber(image_topic, Image, publish_image, callback_args=(draw_pub, sign_pub), queue_size=1, buff_size=2**24)

	rospy.spin()

if __name__ == '__main__':
	with tf.Session() as sess:
		sess.run(init_g)
		saver.restore(sess, tf.train.latest_checkpoint('Saved Training Model/'))
		sess = tf.get_default_session()
		get_image()


# coding=utf-8
from settings import *

# NAMES
LIKE = 'Like'
LIKED_BY = 'Liked By'
RATE = 'Rate'
RATED_BY = 'Rated By'
FAVORITE = 'Favorite'
FAVORITE_OF = 'Favorite Of'
DENOUNCE = 'Denounce'
DENOUNCED_BY = 'Denounced By'
AUTHOR = 'Author'
AUTHORED_BY = 'Authored By'
TARGET = 'Target'
TARGETED_BY = 'Targeted By'

# 'READ AS' STRINGS
LIKE_STR = 'likes'
LIKED_BY_STR = 'is liked by'
RATE_STR = 'rated'
RATED_BY_STR = 'is rated by'
FAVORITE_STR = 'has as favorite'
FAVORITE_OF_STR = 'is favorite of'
DENOUNCE_STR = 'denounces'
DENOUNCED_BY_STR = 'is denounced by'
AUTHOR_STR = 'authored'
AUTHORED_BY_STR = 'was authored by'
TARGET_STR = 'targets'
TARGETED_BY_STR = 'is targeted by'


import matplotlib.pyplot as plt
import xarray as xr
import tempfile
import dask
from dask.diagnostics import ProgressBar
import os
import numpy as np
import pandas as pd
import matplotlib.gridspec as gridspec
from cartopy import crs as ccrs
import lib_added_value as lav
import argparse
import gc

def addedValueStatMSE(X_dict):

    ress = ['LR', 'HRd']
    av = av_norm = 0
    for res in ress:
        av       += lav.AVmse(X_dict['obs{}'.format(res)], X_dict['gdd{}'.format(res)], X_dict['rcm{}'.format(res)])
        av_norm  += lav.AVmse_norm(X_dict['obs{}'.format(res)], X_dict['gdd{}'.format(res)], X_dict['rcm{}'.format(res)])
    avs = (av/av_norm)
    return avs

def addedValueStatCorr(X_dict):

    ress = ['LR', 'HRd']
    av = av_norm = 0
    for res in ress:
        av       += lav.AVcorr(X_dict['obs{}'.format(res)], X_dict['gdd{}'.format(res)], X_dict['rcm{}'.format(res)])
        av_norm  += lav.AVcorr_norm(X_dict['obs{}'.format(res)], X_dict['gdd{}'.format(res)], X_dict['rcm{}'.format(res)])
    avs = (av/av_norm)
    return avs

def addedValueStatBias(X_dict):

    ress = ['LR', 'HRd']
    av = av_norm = 0
    for res in ress:
        av       += lav.AVbias(X_dict['obs{}'.format(res)], X_dict['gdd{}'.format(res)], X_dict['rcm{}'.format(res)])
        av_norm  += lav.AVbias_norm(X_dict['obs{}'.format(res)], X_dict['gdd{}'.format(res)], X_dict['rcm{}'.format(res)])
    avs = (av/av_norm)
    return avs

def addedValueStatStd(X_dict):

    ress = ['LR', 'HRd']
    av = av_norm = 0
    for res in ress:
        av       += lav.AVstd(X_dict['obs{}'.format(res)], X_dict['gdd{}'.format(res)], X_dict['rcm{}'.format(res)])
        av_norm  += lav.AVstd_norm(X_dict['obs{}'.format(res)], X_dict['gdd{}'.format(res)], X_dict['rcm{}'.format(res)])
    avs = (av/av_norm)
    return avs

def corrCommonBias(X_dict):

    ress = ['LR'] #, 'HRd']
    Xgdd = Xrcm = Xobs = 0
    for res in ress:
        Xgdd += X_dict['gdd{}'.format(res)]
        Xrcm += X_dict['rcm{}'.format(res)]
        Xobs += X_dict['obs{}'.format(res)]

    r = lav.correlation(Xgdd-Xobs, Xrcm-Xobs, dim=['lat','lon'])

    return r

def boxPlotHelper(arrList, nameList, title='', ylabel='', ofile=None):

    fig = plt.figure(figsize=(5, 3.7))
    leg = []
    xx = 0
    yy = []

    lss = []
    for i, dummy in enumerate(arrList):
        x, y  = zip(*sorted(dummy.items()))
        ls = []
        if len(y) > 1: # Only plot if there is more than one entry for this (e.g. don't plot gdds if there is only one gdd)
            for p in range(len(y)):
                if np.isnan(y[p]): #< Skip if correlation is nan
                    continue
                l, = plt.plot(xx+(0.1*(p+1)/len(y)), y[p], marker='o', label=x[p])
                ls.append(l)
            lss.append(ls)
            xx += 1
            yy.append(nameList[i])

    plt.xticks(range(xx), yy, rotation=20, weight='bold')  # Set text labels and properties.
    plt.ylim([-0.5, 1])
    plt.yticks([-0.5, 0, 0.5, 1], weight='bold')
    plt.gca().axhline(c='grey', lw=2, linestyle='--')

    #< legend
    xx = 0
    locs, labels = plt.yticks()            # Get locations and labels
    dloc = locs[1]-locs[0]
    for i, dummy in enumerate(arrList):
        x, y  = zip(*sorted(dummy.items()))
        if len(y) > 1:
            leg.append(plt.gca().legend(lss[xx], x, loc='upper center', bbox_to_anchor=(xx-0.25,locs[0]-0.75*dloc, 0.5, 0.1), bbox_transform=plt.gca().transData))
            xx += 1
    ax = plt.gca()
    [ax.add_artist(l) for l in leg]
    ax.set_title(title, size='large', weight='bold')
    ax.set_ylabel(ylabel, weight='bold')

    plt.savefig(ofile, bbox_inches='tight')

# Parse input arguments
def parse_args(parser):
    parser.add_argument("--ipath", dest='ipath', type=str, help="Path to input data.")
    parser.add_argument("-o", "--outpath", default='', type=str, help="Comma-separated list of input files. (The first file will be the reference)")
    parser.add_argument("--op", default='mse', type=str, help="What statistic should be used to summarise the added value (mse, corr, bias, var, commbias)")
    parser.add_argument("--gdds", nargs='?', help="Comma-separated list of input driving models (e.g. ACCESS1-0)")
    parser.add_argument("--varnames", nargs='?', help="Comma-separated list of input variable names (in the order of gdd, rcm, obs)")
    parser.add_argument("--yrStart", type=int, help="Year to start processing.")
    parser.add_argument("--yrEnd", type=int, help="Year to end processing.")
    parser.add_argument("--quantiles", dest='quantiles', type=str, nargs='?', help="Quantiles to calculate added value for.")
    parser.add_argument("--seasons", nargs='?', default='annual', help="Comma-separated list of seasons (annual, DJF, etc.)")
    parser.add_argument("--interp_method", dest='interp_method', type=str, default="nearest", help="Interpolation method used to interpolate to high-res")
    parser.add_argument("--resolution", dest='resolution', type=str, default='', help="Resolution (i.e. 0p05deg or 0p11deg)")
    parser.add_argument("--prefix", dest='prefix', type=str, default='', help="Prefix for output files")
    parser.add_argument("--project", dest='project', type=str, default='', help="Project (e.g. barpa, narclim) used for filename")
    parser.add_argument("--regions", dest='regions', type=str, default='', help="Regions input file.")
    parser.add_argument("--debug", dest='debug', type=lav.str2bool, const=True, nargs='?', default=False, help="Debugging plots")
    return parser.parse_args()

def main():

        #< Increase garbage threshold
        g0, g1, g2 = gc.get_threshold()
        gc.set_threshold(g0 * 3, g1 * 3, g2 * 3)

        # User argument input
        parser    = argparse.ArgumentParser(description='Plot some overview plots of the added value analysis.')
        args      = parse_args(parser)
        ipath     = args.ipath
        gdds      = list(filter(None, args.gdds.split(",")))
        varnames  = list(filter(None, args.varnames.split(",")))
        outpath   = args.outpath
        op        = args.op
        yrStart   = args.yrStart
        yrEnd     = args.yrEnd
        quantiles = [i for i in list(filter(None, args.quantiles.split(",")))]
        seasons   = list(filter(None,args.seasons.split(",")))
        interp_method = args.interp_method
        res         = args.resolution
        prefix    = args.prefix
        project   = args.project
        ifile_regions = args.regions
        l_debug_first = args.debug

        ifiles_dummy = ipath + '/' + 'X.{}_{}.{}.0p11deg.q{}.{}.{}-{}.interp_{}.nc'
        oplot  = outpath + '/' + prefix + '.boxplot.added_value.{}.{}.q{}.EASTAUS.{}.{}.png'

        #< Read the regions files
        print('Read in the added value regions definition...')
        print(ifile_regions)
        regions = xr.load_dataset(ifile_regions)['regions']
        regions['lat'] = regions['lat'].astype('float32')
        regions['lon'] = regions['lon'].astype('float32')

        qDummy  = {}
        for gdd in gdds:
            varDummy  = {}
            for q in quantiles:
                if lav.isfloat(q):
                    q = float(q)

                if isinstance(q, float):
                    print(f'{q*100} quantile...')
                    qstr = '{}'.format(q).replace('.','p')
                elif isinstance(q, str):
                    print(f'{q}...')
                    qstr = q
                else:
                    print(f'Type {type(q)} of q not supported')

                seasonDummy  = {}
                for var in varnames:

                    #< read data
                    ifiles      = ifiles_dummy.format(project, gdd, var, qstr, '{}', yrStart, yrEnd, interp_method)
                    print(ifiles)
                    X           = {}
                    regionDummy = {}
                    for season in seasons:
                        avs     = {}
                        for region in ['coast', 'topo', 'flat']:
                            if region == 'coast':
                                iregion = 1
                            elif region == 'topo':
                                iregion = 2
                            elif region == 'flat':
                                iregion = 3

                            for model in ['obs', 'gdd', 'rcm']:
                                for res in ['LR', 'HRd']:
                                    ifile = ifiles.format(season)
                                    X[model, res] = xr.open_dataset(ifile)['{}{}'.format(model,res)]

                            # #< Just for testing the script!!!!
                            # ##################################
                            # # RCM = GDD -> AV = 0
                            # # X['rcm', 'LR']  = X['gdd', 'LR']
                            # # X['rcm', 'HRd'] = X['gdd', 'HRd']
                            # # RCM = OBS -> AV = 1
                            # # X['rcm', 'LR']  = X['obs', 'LR']
                            # # X['rcm', 'HRd'] = X['obs', 'HRd']
                            # # RCM = (GCM + OBS) / 2 -> AV = 0.6
                            # X['rcm', 'LR']  = (X['obs', 'LR'] + X['gdd', 'LR']) / 2.
                            # X['rcm', 'HRd'] = (X['obs', 'HRd'] + X['gdd', 'HRd']) / 2.
                            # ##################################

                            #< Cut overlap
                            regions, X['obs', 'LR'], X['obs', 'HRd'], X['gdd', 'LR'], X['gdd', 'HRd'], X['rcm', 'LR'], X['rcm', 'HRd'] = lav.cut_overlap(regions, X['obs', 'LR'], X['obs', 'HRd'], X['gdd', 'LR'], X['gdd', 'HRd'], X['rcm', 'LR'], X['rcm', 'HRd'], dims=['lat', 'lon'])

                            #< Group by region
                            lav.check_coordinates_match(regions, X['obs', 'LR'], X['obs', 'HRd'], X['gdd', 'LR'], X['gdd', 'HRd'], X['rcm', 'LR'], X['rcm', 'HRd'], dims=['lat','lon'])
                            X_dict = {'obsLR': X['obs', 'LR'].where(regions==iregion),   'gddLR': X['gdd', 'LR'].where(regions==iregion),   'rcmLR': X['rcm', 'LR'].where(regions==iregion),
                                      'obsHRd': X['obs', 'HRd'].where(regions==iregion), 'gddHRd': X['gdd', 'HRd'].where(regions==iregion), 'rcmHRd': X['rcm', 'HRd'].where(regions==iregion)}

                            if l_debug_first:
                                plt.figure()
                                X_dict['obsLR'].plot.pcolormesh()
                                plt.figure()
                                X_dict['obsHRd'].plot.pcolormesh()
                                plt.figure()
                                X_dict['gddLR'].plot.pcolormesh()
                                plt.figure()
                                X_dict['gddHRd'].plot.pcolormesh()
                                plt.figure()
                                X_dict['rcmLR'].plot.pcolormesh()
                                plt.figure()
                                X_dict['rcmHRd'].plot.pcolormesh()
                                plt.show()

                            #< Select the operation in which added value is measured
                            if op == 'mse':
                                avs[region] = addedValueStatMSE(X_dict)
                            elif op == 'corr':
                                avs[region] = addedValueStatCorr(X_dict)
                            elif op == 'bias':
                                avs[region] = addedValueStatBias(X_dict)
                            elif op == 'var':
                                avs[region] = addedValueStatStd(X_dict)
                            elif op == 'commbias':
                                avs[region] = corrCommonBias(X_dict)

                            print(q, var, season, region)
                            print(avs[region].values)
                        l_debug_first = False

            #< Combine the added values in dictionaries and convert to xarray in the end
                        regionDummy[season] = xr.concat([avs[key] for key in avs], dim=pd.Index(avs.keys(), name='region'), coords='minimal')
                    seasonDummy[var] = xr.concat([regionDummy[key] for key in regionDummy], dim=pd.Index(regionDummy.keys(), name='season'), coords='minimal')
                varDummy[q] = xr.concat([seasonDummy[key] for key in seasonDummy], dim=pd.Index(seasonDummy.keys(), name='var'), coords='minimal')
            qDummy[gdd] = xr.concat([varDummy[key] for key in varDummy], dim=pd.Index([str(k) for k in varDummy.keys()], name='q'), coords='minimal')
        avs = xr.concat([qDummy[key] for key in qDummy], dim=pd.Index(qDummy.keys(), name='gdd'), coords='minimal')

        #< Calculate mean over every other dimension
        avsregion = avs.mean(['var', 'q', 'season', 'gdd']).to_dict()
        avsseason = avs.mean(['var', 'q', 'region', 'gdd']).to_dict()
        avsvar    = avs.mean(['region', 'q', 'season', 'gdd']).to_dict()
        avsq      = avs.mean(['var', 'region', 'season', 'gdd']).to_dict()
        avsgdd    = avs.mean(['var', 'region', 'season', 'q']).to_dict()

        #< Combine in dict
        regionDummy = dict(zip(avsregion['coords']['region']['data'], avsregion['data']))
        seasonDummy = dict(zip(avsseason['coords']['season']['data'], avsseason['data']))
        varDummy    = dict(zip(avsvar['coords']['var']['data'], avsvar['data']))
        qDummy      = dict(zip(avsq['coords']['q']['data'], avsq['data']))
        gddDummy    = dict(zip(avsgdd['coords']['gdd']['data'], avsgdd['data']))

        #< Select the operation in which added value is measured
        if op == 'mse':
            title  = 'Added value (MSE) summary'
            ofile  = oplot.format('av.mse.summary', '{}', '{}', '{}', '{}').replace('.{}','').replace('.q{}','')
            ylabel = 'AV'
        elif op == 'corr':
            title  = 'Added value (Corr) summary'
            ofile  = oplot.format('av.corr.summary', '{}', '{}', '{}', '{}').replace('.{}','').replace('.q{}','')
            ylabel = 'AV'
        elif op == 'bias':
            title  = 'Added value (Bias) summary'
            ofile  = oplot.format('av.bias.summary', '{}', '{}', '{}', '{}').replace('.{}','').replace('.q{}','')
            ylabel = 'AV'
        elif op == 'var':
            title  = 'Added value (Standard deviation) summary'
            ofile  = oplot.format('av.variance.summary', '{}', '{}', '{}', '{}').replace('.{}','').replace('.q{}','')
            ylabel = 'AV'
        elif op == 'commbias':
            title  = 'Common large-scale Bias\nR(GDD-Obs,RCM-Obs) summary'
            ofile  = oplot.format('av.bias-corr.summary', '{}', '{}', '{}', '{}').replace('.{}','').replace('.q{}','')
            ylabel = 'Corr'

        boxPlotHelper(arrList=[regionDummy, seasonDummy, varDummy, gddDummy, qDummy],
                      nameList=['region', 'season', 'var', 'gdd', 'q'], title=title,
                      ylabel=ylabel, ofile=ofile)

################################################################################

if __name__ == '__main__':
    import dask.distributed
    import sys

    #< Print the script name
    print(sys.argv[0])

    # Get the number of CPUS in the job and start a dask.distributed cluster
    threads      = int(os.environ.get('THREADS_PER_WORKER','1'))
    n_workers    = 2 if os.environ["HOSTNAME"].startswith("gadi-login") else max(int(os.environ["PBS_NCPUS"]) // threads,1)
    memory_limit = '1000mb' if os.environ["HOSTNAME"].startswith("gadi-login") else int(os.environ["PBS_VMEM"]) / n_workers
    client       = dask.distributed.Client(n_workers=n_workers, threads_per_worker=threads, memory_limit=memory_limit, local_dir=tempfile.mkdtemp())

    #< Print client summary
    print('### Client summary')
    print(client)
    print('\n\n')

    #< Call the main function
    main()

    #< Close the client
    client.shutdown()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Apr  4 15:42:34 2020

@author: Anwin Mathai
"""
#!/usr/bin/env python
# coding: utf-8

# In[1]:

#importing seaborn for visualisation and setting its style.
import seaborn as sns
sns.set()
sns.set(style="darkgrid")

# In[2]:

#Loading data
iris = sns.load_dataset("iris")
print(iris.head())

# In[6]:

#Knowing the dimension of the data
iris.shape

# In[7]:

#Barplot of different variables.
sns.barplot(x="species", y="sepal_length", data=iris[:150])

# In[8]:

sns.barplot(x="species", y="sepal_width", data=iris[:150])

# In[9]:

sns.barplot(x="species", y="petal_length", data=iris[:150])

# In[10]:

sns.barplot(x="species", y="petal_width", data=iris[:150])

# In[12]:

# scatter plot sepals
sns.relplot(x="sepal_length", y="sepal_width", data=iris[:150], kind="scatter");

# In[13]:

# scatter plot sepals
sns.relplot(x="petal_length", y="petal_width", data=iris[:150], kind="scatter");

# Hue semantic

# In[15]:

#To know the difference between the sepal length and sepal widths in each species
sns.relplot(x="sepal_length", y="sepal_width", hue="species",data=iris[:150]);

# In[16]:

#To know the difference between the petal length and petal widths in each species
sns.relplot(x="petal_length", y="petal_width", hue="species",data=iris[:150]);

# In[18]:

#catplot on the sepal length and width
sns.catplot(x="sepal_length", y="sepal_width",kind="point",data=iris)

# In[19]:

#catplot on the petal length and width
sns.catplot(x="petal_length", y="petal_width",kind="point",data=iris)

# In[20]:
#Relations between all four parameters of the data with species of iris

sns.pairplot(iris, hue='species', height=2.5);

# In[ ]:


# NLTK Chapter 1

# Download all the books used in the NLTK book. Only need to do the following command once:
import nltk
# nltk.download()

#To get floating-point division by default (without it, ⅓ is zero!)
from __future__ import division

# Reading all the books in NLTKs collection to memory (there are nine plus some sentences).
from nltk.book import *

# If you only want one of them, say text1, do the obvious:
from nltk.book import text1

# text1, text2,...,text9 are nine different text object. More specifically, each object is of class nltk.text.Text
text1[1]    # returns the second (note that python indexes start with 0) word of the first book in the collection, Moby Dick.
                # text1[1] is a string object
text1[1][2] # is the third letter of the second word in Moby Dick.

# You can of course define your own text (as a Python list object):
myText = ["This", "is", "my","text","and","there","is","nothing","you","can","do","about","it","!"]
myText[4]

## Searching Text ##
text1.concordance("monstrous")  # Searches for occurences of words and displays part of the lines where it appear
                                # (i.e. it also gives the context in which the word appears)

#text1.similar("whatever") # Lists the words that are similar to some word (similar in which sense?)

#text2.common_contexts(["monstrous","very"])

# Dispersion plot - To find out WHERE a word appears (you need the module pylab which ships with the Spyder IDE)
text4.dispersion_plot(["citizens", "democracy", "freedom", "duties", "America"])

# How long is the book (words and other characters (tokens) like punctuation)?
len(text1)

# How long is the second word?
len(text1[1])

# Give me the set of distinct tokens!
set(text1)

# How many tokes among the first 100 words?
len(set(text1[0:99]))

mobyTokens = set(text1)
type(mobyTokens)    # set() returns a SET object, which cannot be indexed.
                        # This gives an error mobyTokens[1]

# But set objects can be sorted:
sorted(mobyTokens)

# Actually, any Python list object can be sorted:
sorted(myText)

# Python is nice because return arguments can be directly subsetted:
sorted(mobyTokens)[900:910]

# Lexical richness (#words/#tokens) [Don’t forget from __future__ import division]
len(text1)/len(set(text1))

# How many times does a word appear in the text?
text1.count("do")

# Percentage of the text occupied by a word, see E28 below for a better function.
from nltk.book import text5 # Chat conversations
100*text5.count("mildly")/len(text5)

# Define a function that computes lexical diversity
def lexical_diversity(text):
        return len(text)/len(set(text))

#Note that our new function can be used on any text, even your own:
lexical_diversity(myText)

# You can combine two lists with text (the addition operator concatenates strings and lists):
myText1 = ["This", "is", "my","text","and"]

myText2 = ["there","is","nothing","you","can","do","about","it","!"]

myText1 + myText2

# Adding a word to a list (appending a word)
myText.append("LOL")

# We can find the FIRST position of given word:
myText.index('about')

# Frequency distribution
from nltk import FreqDist

fdist1 = FreqDist(text1)

vocabulary = fdist1.keys()

frequencies = fdist1.values()

fdist1['whale']

fdist1.plot(20)

fdist1.plot(20, cumulative = True)

# Finding the really long words (using Pythons list comprehension):
V = set(text1)

[w for w in V if len(w)>15]

# Note that the variable w in the list comprehension is just a dummy. The following gives the same result
[whatever for whatever in V if len(whatever)>15]

# Finding the long words (more than seven letters) that appear more than seven times
[w for w in V if len(w)>7 and fdist1[w]>7]

# Counting the number of characters in each word in a text
[len(w) for w in text1]

# Bigram function returns a list of bigrams
from nltk import bigrams, trigrams

bigrams(myText2)

trigrams(myText2)

bigramsText1 = bigrams(text1) # bigramsText1[0] is the tuple containing the first bigram

# Collocations are frequent bigrams from words that are not so common as unigrams.
# This function returns nothing, just prints the collocations to screen
text1.collocations()

# Computing the frequency distribution of word lengths. Returns a dictionary.
fdistWordLength = FreqDist([len(w) for w in text1])

fdistWordLength.keys() # The different word lengths
fdistWordLength.values() # The frequency of each word length
fdistWordLength.items() # Shows both keys and values at the same time

fdist1['the']
fdist1.freq('the') # Frequency of the word ‘the’
fdist1.max()

# String methods

s = "MatTias"

s.lower()

s.upper()

s.startswith("ma")

"T" in s

# Find all the words in Moby Dick that ends with -ableness. Sort then alphabetically.
from nltk.book import text2, text3, text5, text7
sorted([w for w in set(text1) if w.endswith('ableness')])

sorted([w for w in set(text7) if '-' in w and 'index' in w])

sorted([wd for wd in set(text3) if wd.istitle() and len(wd) > 10])

sorted([w for w in set(text7) if not w.islower()])

sorted([t for t in set(text2) if 'cie' in t or 'cei' in t])

import nltk
nltk.chat.chatbots()

# EXERCISES Chapter 1

# E1
# E2
# E3
3*sent1
# E4
len(text2)
len([w.lower() for w in set(text2) if w.isalpha()])
# E5
# E6
text2.dispersion_plot(["Elinor","Marianne","Edward","Willoughby"])
# E7
text5.collocations()
# E8
# E9 a))
myString = "I can write whatever I feel like here"
print (myString)
# E9 b)
otherString = ", I can"
myString + otherString
myString + " " + otherString
" ".join([myString,otherString]) # Pythonista Alternative

# E10
mySent = ["This","is","my","sentence"]
sentence = " ".join(mySent)
sentence.split()
# E11
phrase1 = ["This","is","my","sentence"]
phrase2 = ["And","there","is","nothing","you","can","so","about","it"]
" ".join(phrase1 + phrase2)
len(phrase1 + phrase2) - (len(phrase1) + len(phrase2))
# E12
# E13
sent1[2][2]
# E14
[sent3[i] == "the" for i in range(len(sent3))]
[sent3[i] == "the" for i,w in enumerate(sent3)] # Alternative
# E15
sorted([w for w in text5 if w.startswith("b")])
# E16
# E17
text9.index('sunset')
text9[621:644]
# E18
sorted(set(sent1 + sent2 + sent3 + sent4 + sent5 + sent6 + sent7 + sent8 + sent9))
# E19
# E20
# not w.islower() is True for punctuation, whereas w.isupper() is False
# E21
text2[-2:]
# E22
fourLetterWords = [w for w in text5 if len(w) == 4]
FreqDist(fourLetterWords).items()
FreqDist(fourLetterWords).plot()
#E23
for i in range(len(text6)):
        if text6[i].isupper():
                print (text6[i])
#E24
[w for w in set(text6) if w.endswith("ize") and w.isalpha()]
[w for w in set(text6) if "z" in w  and w.isalpha()]
[w for w in set(text6) if "pt" in w  and w.isalpha()]
[w for w in set(text6) if w.isalpha() and w[1:].islower() and w[0].isupper()]
#E25a
sent = ["she","sells","sea","shells","by","the","sea","shore"]
for w in sent:
        if w.startswith("sh"):
                print (w)
# E25b
for w in sent:
        if len(w)>4:
                print (w)

# E26
averageWordLength = sum([len(w) for w in text1])/len(text1)

# E27. Not sure about this one since not all elements from set are words.
def vocabSize(text):
        ''' Computes vocabulary size of text. Words are defined liberally, including
        punctuation and other non-letter symbols '''
        return len(set(text))
vocabSize(text1)

# E28
def percent(word, text):
        ''' Computes the percentage that word appears in text '''
        count = text.count(word.lower()) + text.count(word.upper()) + text.count(word[0].upper()
                                                                                + word[1:].lower())
        return 100*count/len(text)
percent('Lol',text5)

# E29
# set1 < set2 is True if every element of set1 is an element of set2
# Can be used to check if a sentence is a part of another sentence.