# NLTK Chapter 1

# Download all the books used in the NLTK book. Only need to do the following command once:
import nltk
# nltk.download()

#To get floating-point division by default (without it, ⅓ is zero!)
from __future__ import division

# Reading all the books in NLTKs collection to memory (there are nine plus some sentences).
from nltk.book import *

# If you only want one of them, say text1, do the obvious:
from nltk.book import text1

# text1, text2,...,text9 are nine different text object. More specifically, each object is of class nltk.text.Text
text1[1]    # returns the second (note that python indexes start with 0) word of the first book in the collection, Moby Dick.
                # text1[1] is a string object
text1[1][2] # is the third letter of the second word in Moby Dick.

# You can of course define your own text (as a Python list object):
myText = ["This", "is", "my","text","and","there","is","nothing","you","can","do","about","it","!"]
myText[4]

## Searching Text ##
text1.concordance("monstrous")  # Searches for occurences of words and displays part of the lines where it appear
                                # (i.e. it also gives the context in which the word appears)

#text1.similar("whatever") # Lists the words that are similar to some word (similar in which sense?)

#text2.common_contexts(["monstrous","very"])

# Dispersion plot - To find out WHERE a word appears (you need the module pylab which ships with the Spyder IDE)
text4.dispersion_plot(["citizens", "democracy", "freedom", "duties", "America"])

# How long is the book (words and other characters (tokens) like punctuation)?
len(text1)

# How long is the second word?
len(text1[1])

# Give me the set of distinct tokens!
set(text1)

# How many tokes among the first 100 words?
len(set(text1[0:99]))

mobyTokens = set(text1)
type(mobyTokens)    # set() returns a SET object, which cannot be indexed.
                        # This gives an error mobyTokens[1]

# But set objects can be sorted:
sorted(mobyTokens)

# Actually, any Python list object can be sorted:
sorted(myText)

# Python is nice because return arguments can be directly subsetted:
sorted(mobyTokens)[900:910]

# Lexical richness (#words/#tokens) [Don’t forget from __future__ import division]
len(text1)/len(set(text1))

# How many times does a word appear in the text?
text1.count("do")

# Percentage of the text occupied by a word, see E28 below for a better function.
from nltk.book import text5 # Chat conversations
100*text5.count("mildly")/len(text5)

# Define a function that computes lexical diversity
def lexical_diversity(text):
        return len(text)/len(set(text))

#Note that our new function can be used on any text, even your own:
lexical_diversity(myText)

# You can combine two lists with text (the addition operator concatenates strings and lists):
myText1 = ["This", "is", "my","text","and"]

myText2 = ["there","is","nothing","you","can","do","about","it","!"]

myText1 + myText2

# Adding a word to a list (appending a word)
myText.append("LOL")

# We can find the FIRST position of given word:
myText.index('about')

# Frequency distribution
from nltk import FreqDist

fdist1 = FreqDist(text1)

vocabulary = fdist1.keys()

frequencies = fdist1.values()

fdist1['whale']

fdist1.plot(20)

fdist1.plot(20, cumulative = True)

# Finding the really long words (using Pythons list comprehension):
V = set(text1)

[w for w in V if len(w)>15]

# Note that the variable w in the list comprehension is just a dummy. The following gives the same result
[whatever for whatever in V if len(whatever)>15]

# Finding the long words (more than seven letters) that appear more than seven times
[w for w in V if len(w)>7 and fdist1[w]>7]

# Counting the number of characters in each word in a text
[len(w) for w in text1]

# Bigram function returns a list of bigrams
from nltk import bigrams, trigrams

bigrams(myText2)

trigrams(myText2)

bigramsText1 = bigrams(text1) # bigramsText1[0] is the tuple containing the first bigram

# Collocations are frequent bigrams from words that are not so common as unigrams.
# This function returns nothing, just prints the collocations to screen
text1.collocations()

# Computing the frequency distribution of word lengths. Returns a dictionary.
fdistWordLength = FreqDist([len(w) for w in text1])

fdistWordLength.keys() # The different word lengths
fdistWordLength.values() # The frequency of each word length
fdistWordLength.items() # Shows both keys and values at the same time

fdist1['the']
fdist1.freq('the') # Frequency of the word ‘the’
fdist1.max()

# String methods

s = "MatTias"

s.lower()

s.upper()

s.startswith("ma")

"T" in s

# Find all the words in Moby Dick that ends with -ableness. Sort then alphabetically.
from nltk.book import text2, text3, text5, text7
sorted([w for w in set(text1) if w.endswith('ableness')])

sorted([w for w in set(text7) if '-' in w and 'index' in w])

sorted([wd for wd in set(text3) if wd.istitle() and len(wd) > 10])

sorted([w for w in set(text7) if not w.islower()])

sorted([t for t in set(text2) if 'cie' in t or 'cei' in t])

import nltk
nltk.chat.chatbots()

# EXERCISES Chapter 1

# E1
# E2
# E3
3*sent1
# E4
len(text2)
len([w.lower() for w in set(text2) if w.isalpha()])
# E5
# E6
text2.dispersion_plot(["Elinor","Marianne","Edward","Willoughby"])
# E7
text5.collocations()
# E8
# E9 a))
myString = "I can write whatever I feel like here"
print (myString)
# E9 b)
otherString = ", I can"
myString + otherString
myString + " " + otherString
" ".join([myString,otherString]) # Pythonista Alternative

# E10
mySent = ["This","is","my","sentence"]
sentence = " ".join(mySent)
sentence.split()
# E11
phrase1 = ["This","is","my","sentence"]
phrase2 = ["And","there","is","nothing","you","can","so","about","it"]
" ".join(phrase1 + phrase2)
len(phrase1 + phrase2) - (len(phrase1) + len(phrase2))
# E12
# E13
sent1[2][2]
# E14
[sent3[i] == "the" for i in range(len(sent3))]
[sent3[i] == "the" for i,w in enumerate(sent3)] # Alternative
# E15
sorted([w for w in text5 if w.startswith("b")])
# E16
# E17
text9.index('sunset')
text9[621:644]
# E18
sorted(set(sent1 + sent2 + sent3 + sent4 + sent5 + sent6 + sent7 + sent8 + sent9))
# E19
# E20
# not w.islower() is True for punctuation, whereas w.isupper() is False
# E21
text2[-2:]
# E22
fourLetterWords = [w for w in text5 if len(w) == 4]
FreqDist(fourLetterWords).items()
FreqDist(fourLetterWords).plot()
#E23
for i in range(len(text6)):
        if text6[i].isupper():
                print (text6[i])
#E24
[w for w in set(text6) if w.endswith("ize") and w.isalpha()]
[w for w in set(text6) if "z" in w  and w.isalpha()]
[w for w in set(text6) if "pt" in w  and w.isalpha()]
[w for w in set(text6) if w.isalpha() and w[1:].islower() and w[0].isupper()]
#E25a
sent = ["she","sells","sea","shells","by","the","sea","shore"]
for w in sent:
        if w.startswith("sh"):
                print (w)
# E25b
for w in sent:
        if len(w)>4:
                print (w)

# E26
averageWordLength = sum([len(w) for w in text1])/len(text1)

# E27. Not sure about this one since not all elements from set are words.
def vocabSize(text):
        ''' Computes vocabulary size of text. Words are defined liberally, including
        punctuation and other non-letter symbols '''
        return len(set(text))
vocabSize(text1)

# E28
def percent(word, text):
        ''' Computes the percentage that word appears in text '''
        count = text.count(word.lower()) + text.count(word.upper()) + text.count(word[0].upper()
                                                                                + word[1:].lower())
        return 100*count/len(text)
percent('Lol',text5)

# E29
# set1 < set2 is True if every element of set1 is an element of set2
# Can be used to check if a sentence is a part of another sentence.


import base64

supported_encoding_types = ['b16', 'b32', 'b64']

class EncodingTypeError(Exception):
    """Encoding type exception

    @ivar encoding_type: The encoding type that was specified
    @ivar supported_types: A list of the supported encoding types
    """
    def __init__(self, encoding_type, supported_types):
        message = "Invalid encoding type specified: %s.  Supported encoding" \
                  " types are: %s" % (encoding_type, supported_encoding_types)
        # Call the base class constructor with the parameters it needs
        Exception.__init__(self, message)

def encode_data(data, encoding="b64", url_safe=False):
    """Encode data using the base64 library.

    @param data: Data to be encoded
    @param encoding: Encoding type.  Supported types are 'b16', 'b32', and 'b64'
        The default is 'b64'.
    @param url_safe: Only valid if encoding is 'b64'.  If True, this causes the
        encoding to be performed by the urlsafe_b64encode function. Default is
        False
    @param data: Data to be encoded
    @raise EncodingTypeError: This exception occurs if an invalid encoding type
        is specified
    """
    encoded_data = ""
    if encoding in supported_encoding_types:
        if encoding == "b16":
            encoded_data = base64.b16encode(data)
        elif encoding == "b32":
            encoded_data = base64.b32encode(data)
        else:  # We assume that the encoding type is the default (b64)
            if url_safe:
                encoded_data = base64.urlsafe_b64encode(data)
            else:
                encoded_data = base64.b64encode(data)
        return encoded_data
    else:
        raise EncodingTypeError(encoding, supported_encoding_types)

def decode_data(encoded_data, encoding="b64", url_safe=False):
    """Decode previously encoded data using the base64 library or base64
    compatible library.  For security purposes, some optional parameters are
    not allowed in some of the supported encoding types.  See
    U{http://docs.python.org/library/base64.html}

    @param encoded_data: Encoded data that will be decoded
    @param encoding: Encoding type.  Supported types are 'b16', 'b32', and 'b64'
        The default is 'b64'.
    @param url_safe: Only valid if encoding is 'b64'.  If True, this causes the
        decoding to be performed by the urlsafe_b64decode function. Default is
        False
    @raise EncodingTypeError: This exception occurs if an invalid encoding type
        is specified
    @raise TypeError: This exception occurs if data is incorrectly padded or if
        there are non-alphabet characters present in the string.
    """
    decoded_data = ""
    if encoding in supported_encoding_types:
        if encoding == "b16":
            decoded_data = base64.b16decode(encoded_data)
        elif encoding == "b32":
            decoded_data = base64.b32decode(encoded_data)
        else:  # We assume that the encoding type is the default (b64)
            if url_safe:
                decoded_data = base64.urlsafe_b64decode(encoded_data)
            else:
                decoded_data = base64.b64decode(encoded_data)
        return decoded_data
    else:
        raise EncodingTypeError(encoding, supported_encoding_types)


import itertools
import numpy
import tensorflow

def double_lu(alpha=-2):
    return lambda x: tensorflow.where(tensorflow.greater(x, 0), x / alpha, x * alpha)

class InvertibleAutoencoder():
    def __init__(self, input_layer):
        self.outputs = [input_layer]

    def add_dense(self,
                  kernel,
                  filters,
                  strides,
                  activation):
        input_shape = numpy.prod(kernel)
        output_shape = numpy.prod(filters)
        w_shape = input_shape * (input_shape + 1) / 2
        with tensorflow.variable_scope('layer_{0}'.format(len(self.outputs))):
            W = tensorflow.get_variable('weights', (w_shape,),
                                initializer=tensorflow.constant_initializer(1.0))
            s = 2 * W / (W ** 2 + 1)
            c = (1 - W ** 2) / (1 + W ** 2)
            j = 1
            i = 0
            final_shape = max(input_shape, output_shape)
            final_rotation = tensorflow.eye(final_shape)
            for c0, s0 in zip(tensorflow.unstack(c), tensorflow.unstack(s)):
                if i == j or i >= output_shape:
                    j += 1
                    i = 0
                if j >= output_shape:
                    break
                givens_matrix = tensorflow.SparseTensor(indices=[[i, i], [j, j], [i, j], [j, i]],
                                values=tensorflow.stack([c0, c0, -s0, s0]),
                                                        dense_shape=[final_shape, final_shape])
                final_rotation = tensorflow.sparse_tensor_dense_matmul(givens_matrix,
                                                                       final_rotation)
                i += 1
            final_rotation = tensorflow.slice(final_rotation, [0,] * 2, [output_shape, input_shape])
            bias = tensorflow.get_variable('bias_{0}'.format(len(self.outputs)),
                                output_shape,
                                initializer=tensorflow.constant_initializer(0.0))

            # convolution
            padded = tensorflow.pad(self.outputs[-1], [[k // 2, k // 2] for k in kernel])
            output_shape = [dim // stride for dim, stride in zip(self.outputs[-1].shape, strides)] + filters
            output = tensorflow.Variable(tensorflow.zeros(shape=output_shape, dtype=tensorflow.float32))
            for i in itertools.product(*[range(0, dim, stride) for dim, stride in zip(self.outputs[-1].shape, strides)]):
                k = [i0 // s for i0, s in zip(i, strides)]
                f = tensorflow.matmul(final_rotation,
                                      tensorflow.reshape(tensorflow.slice(padded,
                                                                          i,
                                                                          kernel),
                                                         [input_shape, 1]))
                tensorflow.scatter_nd_update(output, k, tensorflow.squeeze(f))
            self.outputs.append(activation(output + bias))

if __name__ == '__main__':
    a = InvertibleAutoencoder(tensorflow.placeholder(tensorflow.float32, shape=(128, 128)))
    a.add_dense([2, 2], [4], [2, 2], double_lu(2))


import torch
from util import pad
from metrics import compute_metrics

def compute_validation_outputs(model, val_iter, field, optional_names=[]):
    loss, predictions, answers = [], [], []
    outputs = [[] for _ in range(len(optional_names))]
    for batch_idx, batch in enumerate(val_iter):
        l, p = model(batch)
        loss.append(l)
        predictions.append(pad(p, 150, dim=-1, val=field.vocab.stoi['<pad>']))
        a = None
        if hasattr(batch, 'wikisql_id'):
            a = batch.wikisql_id.data.cpu()
        elif hasattr(batch, 'squad_id'):
            a = batch.squad_id.data.cpu()
        elif hasattr(batch, 'woz_id'):
            a = batch.woz_id.data.cpu()
        else:
            a =  pad(batch.answer.data.cpu(), 150, dim=-1, val=field.vocab.stoi['<pad>'])
        answers.append(a)
        for opt_idx, optional_name in enumerate(optional_names):
            outputs[opt_idx].append(getattr(batch, optional_name).data.cpu())
    loss = torch.cat(loss, 0) if loss[0] is not None else None
    predictions = torch.cat(predictions, 0)
    answers = torch.cat(answers, 0)
    return loss, predictions, answers, [torch.cat([pad(x, 150, dim=-1, val=field.vocab.stoi['<pad>']) for x in output], 0) for output in outputs]

def get_clip(val_iter):
    return -val_iter.extra if val_iter.extra > 0 else None

def all_reverse(tensor, world_size, field, clip, dim=0):
    if world_size > 1:
        tensor = tensor.float() # tensors must be on cpu and float for all_gather
        all_tensors = [torch.zeros_like(tensor) for _ in range(world_size)]
        torch.distributed.barrier() # all_gather is experimental for gloo, found that these barriers were necessary
        torch.distributed.all_gather(all_tensors, tensor)
        torch.distributed.barrier()
        tensor = torch.cat(all_tensors, 0).long() # tensors must be long for reverse
    # for distributed training, dev sets are padded with extra examples so that the
    # tensors are all of a predictable size for all_gather. This line removes those extra examples
    return field.reverse(tensor)[:clip]

def gather_results(model, val_iter, field, world_size, optional_names=[]):
    loss, predictions, answers, outputs = compute_validation_outputs(model, val_iter, field, optional_names=optional_names)
    clip = get_clip(val_iter)
    if not hasattr(val_iter.dataset.examples[0], 'squad_id') and not hasattr(val_iter.dataset.examples[0], 'wikisql_id') and not hasattr(val_iter.dataset.examples[0], 'woz_id'):
        answers = all_reverse(answers, world_size, field, clip)
    return loss, all_reverse(predictions, world_size, field, clip), answers, [all_reverse(x, world_size, field, clip) for x in outputs],

def print_results(keys, values, rank=None, num_print=1):
    print()
    start = rank * num_print if rank is not None else 0
    end = start + num_print
    values = [val[start:end] for val in values]
    for ex_idx in range(len(values[0])):
        for key_idx, key in enumerate(keys):
            value = values[key_idx][ex_idx]
            v = value[0] if isinstance(value, list) else value
            print(f'{key}: {repr(v)}')
        print()

def validate(task, val_iter, model, logger, field, world_size, rank, num_print=10, args=None):
    with torch.no_grad():
        model.eval()
        required_names = ['greedy', 'answer']
        optional_names = ['context', 'question']
        loss, predictions, answers, results = gather_results(model, val_iter, field, world_size, optional_names=optional_names)
        predictions = [p.replace('UNK', 'OOV') for p in predictions]
        names = required_names + optional_names
        if hasattr(val_iter.dataset.examples[0], 'wikisql_id') or hasattr(val_iter.dataset.examples[0], 'squad_id') or hasattr(val_iter.dataset.examples[0], 'woz_id'):
            answers = [val_iter.dataset.all_answers[sid] for sid in answers.tolist()]
        metrics, answers = compute_metrics(predictions, answers, bleu='iwslt' in task or 'multi30k' in task, dialogue='woz' in task,
            rouge='cnn' in task, logical_form='sql' in task, corpus_f1='zre' in task, args=args)
        results = [predictions, answers] + results
        print_results(names, results, rank=rank, num_print=num_print)

        return loss, metrics


"""
2021.4
idea:
1. recursive with cache
    - boundary: res[0]=1, res[x<0]=0
    - recursive: res[x] = sum([res[x-num] for num in nums:])
2. DP (from top to bottom)
    - set dp list
    - dp states equation: dp[x] += dp[x-num] for num in nums
"""

from typing import List
from functools import wraps

class Solution:
    def __init__(self):
        self.cache = {}
    def combinationSum4(self, nums: List[int], target: int) -> int:
        if target < 0:
            return 0
        if target == 0:
            return 1
        res = 0
        for num in nums:
            x = target - num
            if x not in self.cache:
                self.cache[x] = self.combinationSum4(nums, x)
            res += self.cache[x]
        return res

    def combinnationSum4_dp(self, nums, target):
        dp = [0] * (target+1)
        dp[0] = 1
        res = 0
        for i in range(target+1):
            for num in nums:
                if i >= num:
                    dp[i] += dp[i-num]
        return dp[target]

if __name__ == '__main__':
    nums = [1,2,3]
    target = 4
    S = Solution()
    print(S.combinationSum4_recursive(nums, target))


import numpy as np
import cv2
import math

from utils import find_best_line, preprocess_image

def detect_lines(image):
    hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)

    lower_blue = np.array([100, 50, 50])
    upper_blue = np.array([130, 255, 255])

    mask_blue = cv2.inRange(hsv, lower_blue, upper_blue)
    res_blue = cv2.bitwise_and(image, image, mask=mask_blue)
    canny_blue = cv2.Canny(res_blue, 50, 200, None, 3)

    lines_blue = cv2.HoughLinesP(canny_blue, 1, np.pi / 180, 50, None, 50, 20)

    detected_line_points = [[(0,0), (0,0)], [(0,0), (0,0)]]
    if lines_blue is not None:
        print("[INFO]: Detected {} blue lines in the given frame.".format(len(lines_blue)))

        if len(lines_blue) < 2:
            print("[ERROR]: Detected less then 2 blue lines in the given frame.")
            exit(1)

        detected_line_points[0][0], detected_line_points[0][1] = find_best_line(lines_blue)
        cv2.line(image, (detected_line_points[0][0][0], detected_line_points[0][0][1]), (detected_line_points[0][1][0], detected_line_points[0][1][1]),(0, 0, 255), 1, cv2.LINE_AA)

    lower_green = np.array([60, 100, 100])
    upper_green = np.array([70, 255, 255])

    mask_green = cv2.inRange(hsv, lower_green, upper_green)
    res_green = cv2.bitwise_and(image, image, mask=mask_green)
    canny_green = cv2.Canny(res_green, 50, 200, None, 3)

    lines_green = cv2.HoughLinesP(canny_green, 1, np.pi / 180, 50, None, 50, 20)

    if lines_green is not None:
        print("[INFO]: Detected {} green lines in the given frame.".format(len(lines_green)))

        if len(lines_green) < 2:
            print("[ERROR]: Detected less then 2 green lines in the given frame.")
            exit(1)

        detected_line_points[0][0], detected_line_points[0][1] = find_best_line(lines_blue)
        cv2.line(image, (detected_line_points[1][0][0], detected_line_points[1][0][1]),
                 (detected_line_points[1][1][0], detected_line_points[1][1][1]), (0, 0, 255), 1, cv2.LINE_AA)

    return detected_line_points

def resize_region(region):
    return cv2.resize(region, (28, 28), interpolation=cv2.INTER_NEAREST)

def select_roi(image_orig, image_bin):
    img, contours, hierarchy = cv2.findContours(image_bin.copy(), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)

    regions_array = []
    for contour in contours:
        x, y, w, h = cv2.boundingRect(contour)  # koordinate i velicina granicnog pravougaonika
        if (h > 12 and w >= 12) or (h > 12 and w <= 4 and w >= 2):
            region = image_bin[y:y + h + 1, x:x + w + 1]
            regions_array.append([resize_region(region), (x, y, w, h)])

    regions_array = sorted(regions_array, key=lambda x: x[1][0])

    sorted_regions = []
    sorted_regions_coords = []
    for region in regions_array:
        sorted_regions.append(region[0])
        sorted_regions_coords.append(region[1])

    return sorted_regions,sorted_regions_coords,

def search_for_detection(detections, detection):
    indices = []
    i = 0
    for d in detections:
        (X1, Y1) = detection.center
        (X2, Y2) = d.center

        distance = math.sqrt(math.pow((X1 - X2),2) + math.pow((Y1 - Y2),2))

        if distance < 20:
            indices.append(i)

        i += 1

    return indices


import matplotlib
matplotlib.use('TkAgg')
from pylab import *
import networkx as nx
import random as rd
import numpy as np

#-------------------------
# WATTS-STROGATZ MODEL
# grid structure
#-------------------------

# num_nodes = 30 # number of nodes

# def initialize(n = num_nodes):
#     global g
#     g = nx.Graph()
#     #determining how many nodes to plot and the grid
#     #dimensions for a nice rectangular grid
#     width = np.ceil(np.sqrt(n))
#     height = np.floor(np.sqrt(n))
#     n = width * height

#     for i in range(int(n)):
#         if not (i < width):
#             g.add_edge(i, i - width)    #adds vertical connections

#         if i % width != 0:
#             g.add_edge(i, i - 1)        #adds horizontal connections

#     g.pos = nx.spring_layout(g)
#     g.count = 0

# def observe():
#     global g
#     cla()
#     nx.draw(g, pos = g.pos)
#     plt.show()

# def update():
#     global g
#     g.count += 1
#     if g.count % 20 == 0: # rewiring once in every 20 steps
#         nds = list(g.nodes)
#         i = rd.choice(nds)
#         if g.degree[i] > 0:
#             g.remove_edge(i, rd.choice(list(g.neighbors(i))))
#             nds.remove(i)
#             for j in g.neighbors(i):
#                 nds.remove(j)
#             g.add_edge(i, rd.choice(nds))

#     # simulation of node movement
#     g.pos = nx.spring_layout(g, pos = g.pos, iterations = 5)

# import pycxsimulator
# pycxsimulator.GUI().start(func=[initialize, observe, update])

#-------------------------
# BARABASI-ALBERT MODEL
#
#-------------------------

m0 = 5 # number of nodes in initial condition
m = 5 # number of edges per new node

def initialize():
    global g
    g = nx.complete_graph(m0)
    g.pos = nx.spring_layout(g)
    g.count = 0

def observe():
    global g
    cla()
    nx.draw(g, pos = g.pos)

def pref_select(nds):
    global g
    r = uniform(0, sum(g.degree(i) for i in nds))
    x=0
    for i in nds:
        x += g.degree[i]
        if r <= x:
            return i

def update():
    global g
    g.count += 1
    if g.count % 20 == 0: # network growth once in every 20 steps
        nds = list(g.nodes)
        newcomer = max(nds) + 1
        for i in range(m):
            j = pref_select(nds)
            g.add_edge(newcomer, j)
            nds.remove(j)
        g.pos[newcomer] = (0, 0)

    # simulation of node movement
    g.pos = nx.spring_layout(g, pos = g.pos, iterations = 5)

import pycxsimulator
pycxsimulator.GUI().start(func=[initialize, observe, update])


"""*********************************************************

NAME:     ex7

AUTHOR:   Paul Haddon Sanders IV, Ph.D

VERSION:  1. K-means Clustering

*********************************************************"""

import numpy as np
from scipy import io
from PIL import Image as im
from pdb import set_trace as pb

### Get data ###############################################

mat1  = io.loadmat('ex7data2.mat')
mat2  = io.loadmat('bird_small.mat')
mat3  = im.open('image.jpg')
X     = mat1['X']
A     = mat2['A']
B     = np.asarray(mat3)

Aproc     = np.resize(A,(A.shape[0]**2,A.shape[2]))

np.savetxt('kmeans.txt',Aproc)

Bproc     = np.resize(B,(B.shape[0]**2,B.shape[2]))

### 1.1 Implementing K-means ###############################

class kmeans:

    def __init__(self,k=3,n_iters=10):
        self.k       = k
        self.n_iters = n_iters

    def find_centroid(self,X):
        """Find centroid using K-means algorithm

        Parameters
        ----------
        X : {array-like}, shape = [n_samples,n_features]
          Training vectors, where n_samples is the number of samples and n_features is the number of features

        Returns
        -------
        self : object

        """

        final_distance = np.zeros(5)
        new_centroid   = np.zeros((5,self.k,X.shape[1]))

        """ Run the algorithm randomly several times """

        for iter1 in range(5):

            """ Randomly select centroids from (unique) X
                values """

            add_choice = X[np.random.choice(X.shape[0],self.k,replace=False)]

            choice = np.unique(add_choice,axis=0)

            while len(choice) < self.k:

                add_choice = X[np.random.choice(X.shape[0],self.k,replace=False)]

                choice = np.append(choice,add_choice,axis=0)
                choice = np.unique(choice,axis=0)

            new_centroid[iter1] = choice[:self.k]

            """ Proceed to find the new centroid """

            for iter2 in range(self.n_iters):

                """ Calculate distances """

                distances = np.array([((X - val)**2.0).sum(1) for val in new_centroid[iter1]])

                """ Find the nearest centroid position """

                centroid_number = distances.T.argmin(1)

                """ Get the new centroids by averaging """

                new_centroid[iter1] = np.array([X[np.where(centroid_number == val)].sum(0) / np.where(centroid_number == val)[0].shape[0] for val in range(self.k)])

            """ Check the error in the new centroid """

            distances = np.array([((X - val)**2.0).sum(1) for val in new_centroid[iter1]])
            centroid_number = distances.T.argmin(1)
            final_distance[iter1] = np.array([((X[np.where(centroid_number == i)] - val)**2.0).sum() for i,val in enumerate(new_centroid[iter1])]).sum()

        """ Select the centroid with the lowest error """

        self.centroid = new_centroid[final_distance.argmin()]

        return self

    def compress_data(self,X):

        distances = np.array([((X - val)**2.0).sum(1) for val in self.centroid])

        Xcopy = self.centroid[distances.T.argmin(1)]

        return Xcopy

    def change_pixel(self,pixel):

        distances = np.array([((pixel - val)**2.0).sum() for val in self.centroid])

        self.centroid[distances.T.argmin()] = pixel

        return self

### 1.4 Image compression with K-means #####################

km1 = kmeans(k=16)
km1.find_centroid(Aproc)

Acomp = km1.compress_data(Aproc)
Acomp = Acomp.astype('uint8')
Acomp = np.resize(Acomp,A.shape)

### 1.5 Use your own image #################################

km2  = kmeans(k=10)
km2.find_centroid(Aproc)
km2.change_pixel([255,255,255])

Bcomp     = km2.compress_data(Bproc)
Bcomp     = np.resize(Bcomp,B.shape)
Bcomp     = Bcomp.astype('uint8')

### output #################################################

image1 = im.new('RGB',(A.shape[0]*2,A.shape[0]))
image1_orig = im.fromarray(A)
image1_comp = im.fromarray(Acomp)

image1.paste(image1_orig,(0,0))
image1.paste(image1_comp,(Acomp.shape[0],0))

image1.show()

image2 = im.new('RGB',(B.shape[0]*2,B.shape[0]))
image2_orig = im.fromarray(B)
image2_comp = im.fromarray(Bcomp)

image2.paste(image2_orig,(0,0))
image2.paste(image2_comp,(Bcomp.shape[0],0))

image2.show()


#!/usr/bin/env python
import rospy
import sys
import tensorflow as tf
from tensorflow.contrib.layers import flatten
from std_msgs.msg import UInt8
from std_msgs.msg import String
from sensor_msgs.msg import Image
import numpy as np
import pickle
import cv2
import csv
from cv_bridge import CvBridge, CvBridgeError
import glob
import time
from sklearn.svm import LinearSVC
from sklearn.utils import shuffle
from sklearn.externals import joblib
# for scikit-learn <= 0.17 use:
from sklearn.cross_validation import train_test_split
# otherwise:
# from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from skimage.feature import hog
from scipy.ndimage.measurements import label

bridge = CvBridge()

color_space = 'YCrCb'
orient = 11
pix_per_cell = 8
cell_per_block = 2
hog_channel = 'ALL'
spatial_size = (16, 16)
hist_bins = 32
spatial_feat = True
hist_feat = True
hog_feat = True

EPOCHS = 50
BATCH_SIZE = 128

svc = joblib.load("Saved Training Model/train_model_2.m")
X_scaler = joblib.load("Saved Training Model/scaler_2.save")

label_names = []
with open('./signnames.csv', 'rt') as f:
	reader = csv.reader(f)
	label_names = list(reader)

# read the traffic sign name file
label_name = []
for i in range(43):
	label_name.append(label_names[i+1][1])

def normalized(data):
	return (data/255.0 - 0.5)

def LeNet(x):
	# Arguments used for tf.truncated_normal, randomly defines variables for the weights and biases for each layer
	mu = 0
	sigma = 0.1

	# SOLUTION: Layer 1: Convolutional. Input = 32x32x3. Output = 30x30x32.
	conv1_W = tf.Variable(tf.truncated_normal(shape=(3, 3, 3, 32), mean = mu, stddev = sigma))
	conv1_b = tf.Variable(tf.zeros(32))
	conv1   = tf.nn.conv2d(x, conv1_W, strides=[1, 1, 1, 1], padding='VALID') + conv1_b

	# SOLUTION: Activation.
	conv1 = tf.nn.relu(conv1)

	# SOLUTION: Pooling. Input = 30x30x32. Output = 15x15x32.
	conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')

	# SOLUTION: Layer 2: Convolutional. Input = 15x15x32. Output = 13x13x64.
	conv2_W = tf.Variable(tf.truncated_normal(shape=(3, 3, 32, 64), mean = mu, stddev = sigma))
	conv2_b = tf.Variable(tf.zeros(64))
	conv2   = tf.nn.conv2d(conv1, conv2_W, strides=[1, 1, 1, 1], padding='VALID') + conv2_b

	# SOLUTION: Activation.
	conv2 = tf.nn.relu(conv2)

	# SOLUTION: Pooling. Input = 13x13x64. Output = 6x6x64.
	conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')

	# SOLUTION: Layer 3: Convolutional. Input = 6x6x64. Output = 4x4x128.
	conv3_W = tf.Variable(tf.truncated_normal(shape=(3, 3, 64, 128), mean = mu, stddev = sigma))
	conv3_b = tf.Variable(tf.zeros(128))
	conv3   = tf.nn.conv2d(conv2, conv3_W, strides=[1, 1, 1, 1], padding='VALID') + conv3_b

	# SOLUTION: Activation.
	conv3 = tf.nn.relu(conv3)

	# SOLUTION: Pooling. Input = 4x4x128. Output = 2x2x128.
	conv3 = tf.nn.max_pool(conv3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')

	# SOLUTION: Flatten. Input = 2x2x128. Output = 512.
	fc0   = flatten(conv3)

	# SOLUTION: Layer 4: Fully Connected. Input = 512. Output = 120.
	fc1_W = tf.Variable(tf.truncated_normal(shape=(512, 120), mean = mu, stddev = sigma))
	fc1_b = tf.Variable(tf.zeros(120))
	fc1   = tf.matmul(fc0, fc1_W) + fc1_b

	# SOLUTION: Activation.
	fc1    = tf.nn.relu(fc1)

	# SOLUTION: Layer 5: Fully Connected. Input = 120. Output = 84.
	fc2_W  = tf.Variable(tf.truncated_normal(shape=(120, 84), mean = mu, stddev = sigma))
	fc2_b  = tf.Variable(tf.zeros(84))
	fc2    = tf.matmul(fc1, fc2_W) + fc2_b

	# SOLUTION: Activation.
	fc2    = tf.nn.relu(fc2)

	# SOLUTION: Layer 6: Fully Connected. Input = 84. Output = 43.
	fc3_W  = tf.Variable(tf.truncated_normal(shape=(84, 43), mean = mu, stddev = sigma))
	fc3_b  = tf.Variable(tf.zeros(43))
	logits = tf.matmul(fc2, fc3_W) + fc3_b

	return logits

x = tf.placeholder(tf.float32, (None, 32, 32, 3))
logits = LeNet(x)
saver = tf.train.Saver()
init_g = tf.global_variables_initializer()

# prototype functions for Histogram of Oriented Gradients(HOG):

def color_hist(img, nbins=32, bins_range=(0, 256)):
	# Compute the histogram of the RGB channels separately
	channel1_hist = np.histogram(img[:,:,0], bins=nbins, range=bins_range)
	channel2_hist = np.histogram(img[:,:,1], bins=nbins, range=bins_range)
	channel3_hist = np.histogram(img[:,:,2], bins=nbins, range=bins_range)
	# Concatenate the histograms into a single feature vector
	hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))
	# Return the color histograms
	return hist_features

def bin_spatial(img, size=(16, 16)):
	# Use cv2.resize().ravel() to create the feature vector
	features = cv2.resize(img, size).ravel() # Remove this line!
	# Return the feature vector
	return features

def get_hog_features(img, orient, pix_per_cell, cell_per_block,
						vis=False, feature_vec=True):
	# Call with two outputs if vis==True
	if vis == True:
		hog_features, hog_image = hog(img, orientations=orient,
								  pixels_per_cell=(pix_per_cell, pix_per_cell),
								  cells_per_block=(cell_per_block, cell_per_block),
								  transform_sqrt=False, block_norm = 'L2-Hys',
								  visualise=vis, feature_vector=feature_vec)
		return hog_features, hog_image
	# Otherwise call with one output
	else:
		hog_features = hog(img, orientations=orient,
					   pixels_per_cell=(pix_per_cell, pix_per_cell),
					   cells_per_block=(cell_per_block, cell_per_block),
					   transform_sqrt=False, block_norm = 'L2-Hys',
					   visualise=vis, feature_vector=feature_vec)
		return hog_features

# prototype functions for Sliding Window Search

# Define a function to draw bounding boxes
def draw_boxes(img, bboxes, color=(0, 0, 255), thick=6):
	# Make a copy of the image
	draw_img = np.copy(img)
	# Iterate through the bounding boxes
	for bbox in bboxes:
		# Draw a rectangle given bbox coordinates
		cv2.rectangle(draw_img, bbox[0], bbox[1], color, thick)
	# Return the image copy with boxes drawn
	return draw_img

def find_signs(img, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell,
			  cell_per_block, spatial_size, hist_bins, hog_channel):

	draw_img = np.copy(img)
	img = img.astype(np.float32)

	img_tosearch = img[ystart:ystop,:,:]
	ctrans_tosearch = convert_color(img_tosearch, conv='RGB2YCrCb')
	if scale != 1:
		imshape = ctrans_tosearch.shape
		ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))

	# select colorspace channel for HOG
	if hog_channel == 'ALL':
		ch1 = ctrans_tosearch[:,:,0]
		ch2 = ctrans_tosearch[:,:,1]
		ch3 = ctrans_tosearch[:,:,2]
	else:
		ch1 = ctrans_tosearch[:,:,hog_channel]

	# Define blocks and steps as above
	nxblocks = (ch1.shape[1] // pix_per_cell) - cell_per_block + 1
	nyblocks = (ch1.shape[0] // pix_per_cell) - cell_per_block + 1
	nfeat_per_block = orient*cell_per_block**2

	# 32 was the orginal sampling rate, with 4 cells and 8 pix per cell
	window = 32
	nblocks_per_window = (window // pix_per_cell) - cell_per_block + 1
	cells_per_step = 1  # Instead of overlap, define how many cells to step
	nxsteps = (nxblocks - nblocks_per_window) // cells_per_step + 1
	nysteps = (nyblocks - nblocks_per_window) // cells_per_step + 1

	# Compute individual channel HOG features for the entire image
	hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
	if hog_channel == 'ALL':
		hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
		hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)

	# Initialize postive windows
	on_windows = []

	for xb in range(nxsteps):
		for yb in range(nysteps):
			ypos = yb*cells_per_step
			xpos = xb*cells_per_step
			# Extract HOG for this patch
			hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
			if hog_channel == 'ALL':
				hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
				hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel()
				hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3))
			else:
				hog_features = hog_feat1

			xleft = xpos*pix_per_cell
			ytop = ypos*pix_per_cell

			# Extract the image patch
			subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window, xleft:xleft+window], (32,32))

			# Get color features
			spatial_features = bin_spatial(subimg, size=spatial_size)
			hist_features = color_hist(subimg, nbins=hist_bins)

			# Scale features and make a prediction
			test_features = X_scaler.transform(np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1))
			#test_features = X_scaler.transform(np.hstack((shape_feat, hist_feat)).reshape(1, -1))
			test_prediction = svc.predict(test_features)

			if test_prediction == 1:
				xbox_left = np.int(xleft*scale)
				ytop_draw = np.int(ytop*scale)
				win_draw = np.int(window*scale)
				cv2.rectangle(draw_img,(xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart),(0,0,255),6)
				on_windows.append(((xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart)))

	return draw_img, on_windows

def add_heat(heatmap, bbox_list):
	# Iterate through list of bboxes
	for box in bbox_list:
		# Add += 1 for all pixels inside each bbox
		# Assuming each "box" takes the form ((x1, y1), (x2, y2))
		heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1

	# Return updated heatmap
	return heatmap# Iterate through list of bboxes

def apply_threshold(heatmap, threshold):
	# Zero out pixels below the threshold
	heatmap[heatmap <= threshold] = 0
	# Return thresholded map
	return heatmap

def draw_labeled_bboxes(img, labels):
	sign_label = []
	sign_position = []
	# Iterate through all detected signs
	for car_number in range(1, labels[1]+1):
		# Find pixels with each car_number label value
		nonzero = (labels[0] == car_number).nonzero()
		# Identify x and y values of those pixels
		nonzeroy = np.array(nonzero[0])
		nonzerox = np.array(nonzero[1])
		### set the threshold for the size of box, to make sure it only consider the sign near the robot
		if (np.max(nonzerox) - np.min(nonzerox) >= 60 and np.max(nonzeroy) - np.min(nonzeroy) >= 60):
			# Define a bounding box based on min/max x and y
			bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
			# Draw the box on the image
			cv2.rectangle(img, bbox[0], bbox[1], (0,1,0), 3)
			sign_label.append(car_number)
			position = [np.min(nonzerox), np.min(nonzeroy), np.max(nonzerox), np.max(nonzeroy)]
			sign_position.append(position)
	# Return the image
	return img, sign_label, sign_position

def convert_color(img, conv='RGB2YCrCb'):
	if conv == 'RGB2YCrCb':
		return cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)
	if conv == 'BGR2YCrCb':
		return cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
	if conv == 'RGB2LUV':
		return cv2.cvtColor(img, cv2.COLOR_RGB2LUV)

def detector(image):

	image_backup = np.copy(image)

	windows = []

	search_config = [(100, 180, 1.5), (96, 192, 2), (80, 200, 2,5), (72, 232, 3), (64, 232, 3.5),
					 (64, 232, 4), (64, 232, 5), (40, 300, 6), (40, 300, 8)]
	# search_config = [(190, 280, 1), (200, 300, 1.5), (150, 480, 2.5)]

	for config in search_config:
		out_img, out_windows = find_signs(image, config[0], config[1], config[2], svc, X_scaler,
				orient, pix_per_cell, cell_per_block, spatial_size, hist_bins, hog_channel)
		windows.append(out_windows)

	windows = [item for sublist in windows for item in sublist]

	# Initialize the heatmap
	heat = np.zeros_like(image[:,:,0].astype(np.float))

	# Add heat to each windows in windows list
	heat = add_heat(heat, windows)

	# Apply threshold to help remove false positives
	heat = apply_threshold(heat, 6)

	# Visualize the heatmap when displaying
	heatmap = np.clip(heat, 0, 255)

	# Find final windows from heatmap using lable function
	labels = label(heatmap)
	num_signs = labels[1]
	draw_img, sign_label, sign_position = draw_labeled_bboxes(np.copy(image), labels)

	# judge the types of traffic sign
	sign_types= []
	for i in range(len(sign_label)):
		sign = image_backup[sign_position[i][1]:sign_position[i][3], sign_position[i][0]:sign_position[i][2]]
		sign = np.uint8(sign*255)
		sign = cv2.resize(sign, (32, 32))
		sign = cv2.cvtColor(sign, cv2.COLOR_RGB2BGR)
		sign = np.expand_dims(sign, axis=0)
		sign_normalized = normalized(sign)
		sign_type = sess.run(tf.argmax(logits, 1), feed_dict={x: sign_normalized})
		sign_types.append(sign_type)

		# anotate the traffic sign name in image
		cv2.putText(draw_img, '%s' %label_name[int('%d' %sign_type)], (sign_position[i][0],
					sign_position[i][1]-10), cv2.FONT_HERSHEY_COMPLEX, 0.8, (0,1,0), 2)
#         cv2.putText(draw_img, '%d: %s' %(sign_type, label_name[sign_type]), (sign_position[i][1]-20,
#                     sign_position[i][0]), cv2.FONT_HERSHEY_COMPLEX, 6, (0,0,255), 25)

	return np.uint8(draw_img*255), sign_types

def publish_image(image, args):
	draw_pub = args[0]
	sign_pub = args[1]

	cv2_img = bridge.imgmsg_to_cv2(image, "rgb8")

	rospy.loginfo("image received")

	# convert to mpimg.imread form
	mpimg = cv2_img/(255.0)
	draw, sign_types = detector(mpimg)
	drawmsg = bridge.cv2_to_imgmsg(draw, "rgb8")
	draw_pub.publish(drawmsg)
	for sign_type in sign_types:
		sign_pub.publish(sign_type)

	return

def get_image():
	rospy.init_node('get_image')

	image_topic = "/color_filter"

	bridge = CvBridge()
	draw_pub = rospy.Publisher("draw_topic", Image, queue_size=1)
	sign_pub = rospy.Publisher("sign_types", UInt8, queue_size=1)
	# heat_pub = rospy.Publisher("heat_topic", Image, queue_size=1)
	rospy.Subscriber(image_topic, Image, publish_image, callback_args=(draw_pub, sign_pub), queue_size=1, buff_size=2**24)

	rospy.spin()

if __name__ == '__main__':
	with tf.Session() as sess:
		sess.run(init_g)
		saver.restore(sess, tf.train.latest_checkpoint('Saved Training Model/'))
		sess = tf.get_default_session()
		get_image()


from flask import Flask, request, jsonify
from model import *

app = Flask("Test server")

width = 30
height = 30
num_agents = 0
num_boxes = 0
storageModel = None
currentStep = 0

def sortAgents():
    organizingAgents = []
    boxAgents = []
    for a, x, y in storageModel.grid.coord_iter():
        for agent in a:
            if isinstance(agent, OrganizingAgent):
                organizingAgents.append(agent)
            elif isinstance(agent, BoxAgent):
                boxAgents.append(agent)

    organizingAgents.sort(key=lambda x: x.unique_id)
    boxAgents.sort(key=lambda x: x.unique_id)

    return organizingAgents, boxAgents

@app.route("/")
def default():
    print("Request Recieved")
    return "Hello there!"

@app.route("/config", methods=['POST'])
def configure():
    global num_agents, num_boxes, width, height, storageModel, currentStep
    num_agents = int(request.form.get("numAgents"))
    num_boxes = int(request.form.get("numBoxes"))
    width = int(request.form.get("width"))
    height = int(request.form.get("height"))
    currentStep = 0
    print(
        f"Recieved num_agents = {num_agents}, num_boxes = {num_boxes}, width = {width}, height = {height}")
    storageModel = StorageModel(num_agents, num_boxes, width, height)

    return jsonify({"OK": num_agents})

@app.route("/getOrganizing", methods=['GET'])
def getAgents():
    global storageModel
    organizingAgents, boxAgents = sortAgents()
    agentPositions = [{"x": a.pos[0], "y": 0.5, "z": a.pos[1]}
                      for (a) in organizingAgents]

    return jsonify({'positions': agentPositions})

@app.route("/getBoxes", methods=['GET'])
def getBoxes():
    global storageModel
    organizingAgents, boxAgents = sortAgents()
    agentPositions = [{"x": a.pos[0], "y": 0.5, "z": a.pos[1]}
                      for (a) in boxAgents]

    return jsonify({'positions': agentPositions})

@app.route('/update', methods=['GET'])
def updateModel():
    global currentStep, storageModel
    storageModel.step()
    currentStep += 1
    return jsonify({'message': f'Model updated to step {currentStep}.', 'currentStep': currentStep})

app.run()


import csv
import numpy as np
import sqlite3
import json

def Round(a):
    return float(np.round(float(a), 4))

def convert_bool(a):
    if a == "True":
        return True
    return False

# Make a set of refs which are referred to by any source in either hm
lmxb = set()
hmxb = set()
with open("data/summaryA.csv") as file1:
    csv_reader = csv.DictReader(file1)
    for row in csv_reader:
        if row["class"] == "lmxb":

            for i in row["r_Opt"].split(','):
                if i == "":
                    continue
                lmxb.add(i.strip())
            for i in row["r_Fx"].split(','):
                if i == "":
                    continue
                lmxb.add(i.strip())
            for i in row["r_Vmag"].split(','):
                if i == "":
                    continue
                lmxb.add(i.strip())
            for i in row["r_Ppulse"].split(','):
                if i == "":
                    continue
                lmxb.add(i.strip())

        elif row["class"] == "hmxb":

            for i in row["r_Opt"].split(','):
                if i == "":
                    continue
                hmxb.add(i.strip())
            for i in row["r_Fx"].split(','):
                if i == "":
                    continue
                hmxb.add(i.strip())
            for i in row["r_Vmag"].split(','):
                if i == "":
                    continue
                hmxb.add(i.strip())
            for i in row["r_Ppulse"].split(','):
                if i == "":
                    continue
                hmxb.add(i.strip())
        else:
            print("UNKNOWN SOURCE")

hmxb_full = {}
lmxb_full = {}
with open("data/hmxbrefs.csv") as hmxbref_file:
    # use numpy to select only those rows  which have
    dr = csv.DictReader(hmxbref_file)
    for row in dr:
        hmxb_full[row["id"]] = (row["id"], row["bib"], row["Name"], row["desc"])

with open("data/lmxbrefs.csv") as lmxbref_file:
    # use numpy to select only those rows  which have
    dr = csv.DictReader(lmxbref_file)
    for row in dr:
        lmxb_full[row["id"]] = (row["id"], row["bib"], row["Name"], row["desc"])

hmxb_to_add = []
for i in hmxb:
    if i not in hmxb_full:
        continue
    t = hmxb_full[i]
    hmxb_to_add.append((t[0] + "_hmxb", t[1], t[2], t[3]))

lmxb_to_add = []
for i in lmxb:
    if i not in lmxb_full:
        continue
    t = lmxb_full[i]
    lmxb_to_add.append((t[0] + "_lmxb", t[1], t[2], t[3]))

hmxb_to_add = hmxb_to_add + lmxb_to_add

with open('data/publications.csv', 'r') as fin:
    dr = csv.DictReader(fin)
    to_db_pub = [(i['Idx'], i['Title'], i['Keywords'], i['Abstract'], i['Bib'], i['Authors']) for i in dr]

# sourceA -paper mapping tuple
src_pub = []
with open('data/summaryA.csv') as f:
    dr = csv.DictReader(f)
    for row in dr:
        if row['Papers'] == "":
            continue
        for ppr_id in row['Papers'].split(','):
            src_pub.append((int(row['Id']), int(ppr_id.strip())))

# To fill db with catalog B sources
with open('data/summaryB.csv', 'r') as fin:
    dr = csv.DictReader(fin)
    len_cat_B = 0
    to_db_src_B = [(i['idx'], str(i['object']), str(i['obsid']), Round(i['ra']), Round(i['dec']), str(i['ins']),
                    str(i['date_time']), str(i['proposal_id']), str(i['target_id']), i['abstract'], i['PI'])
                   for i in dr]

# To fill db with catalog A sources
with open('data/summaryA.csv', 'r') as fA:
    dr = csv.DictReader(fA)
    count_A = 0
    for i in dr:
        count_A += 1

print("in A ", count_A)

with open('data/summaryA.csv', 'r') as fA:
    l1 = []

    dr = csv.DictReader(fA)
    l1 = [(i['Id'], str(i['Name']), Round(i['RA']), Round(i['DE']), i['B-V'], i['Cat'], i['class'], i['E(B-V)'], i['Fx'],
           i['GLAT'], i['GLON'], i['Opt'], i['Porb'], i['Porb2'], i['Ppulse'], i['Range'], i['SpType'], i['Type'], i['U-B'], i['Vmag'],
           i['r_Fx'], i['r_Opt'], i['r_Ppulse'], i['r_Vmag']) for i in dr]

with open('data/observed_A.csv', 'r') as f_obs:
    l2 = []
    dr = csv.DictReader(f_obs)
    l2 = [(i["observed_uvit"], i["observed_sxt"], i["observed_laxpc"], i["observed_czti"], i['ID_A']) for i in dr]

to_db_src_A = []

for i in zip(l1, l2):
    a = i[0]
    b = i[1]
    to_db_src_A.append((a[0], a[1], a[2], a[3], convert_bool(b[0]), convert_bool(b[1]), convert_bool(b[2]), convert_bool(b[3]), a[4], a[5],
                        a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15], a[16], a[17], a[18], a[19], a[20], a[21], a[22], a[23]))

    # id,Name,RA,Dec,uvit,sxt,laxpc,czti,category

with open('data/observed_A_full.txt') as json_file:
    data = json.load(json_file)
    # dictionaries
    uvit = data['uvit']
    laxpc = data['laxpc']
    czti = data['czti']
    sxt = data['sxt']

obs_lax = []
obs_sx = []
obs_cz = []
obs_uv = []

for val in l2:
    uv = val[0]
    sx = val[1]
    lax = val[2]
    cz = val[3]
    id_src = val[4]
    if (uv == 'True'):
        c = uvit[str(id_src)]
        for src_B in c:
            obs_uv.append((id_src, src_B))
    if (cz == 'True'):
        c = czti[str(id_src)]
        for src_B in c:
            obs_cz.append((id_src, src_B))
    if (lax == 'True'):
        c = laxpc[str(id_src)]
        for src_B in c:
            obs_lax.append((id_src, src_B))
    if (sx == 'True'):
        c = sxt[str(id_src)]
        for src_B in c:
            obs_sx.append((id_src, src_B))

con = sqlite3.connect('db.sqlite3')
cursor = con.cursor()

# insert publication data to db
try:
    cursor.executemany("INSERT INTO source_publication(identifier,Name,Keywords,Abstract,Bib,Authors) VALUES(?,?,?,?,?,?)", to_db_pub)
    con.commit()
except Exception as error:
    print(error)

# insert catalog A data to db
try:
    cursor.executemany("INSERT INTO source_sourcea(id, Name,RA,Dec,isObserved_uvit,isObserved_sxt,isObserved_laxpc,isObserved_czti,B_V,Cat,Class,E_BV,Fx,GLAT,GLON,Opt,Porb,Porb2,Ppulse,Range,SpType,Type,U_B,Vmag,r_Fx,r_Opt,r_Ppulse,r_Vmag) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", to_db_src_A)
    con.commit()
except Exception as error:
    print(error)

# insert catalog B data to db
try:
    cursor.executemany(
        "INSERT INTO source_sourceb(id,Object,obsid,RA,Dec,instrument,date_time,proposal_id,target_id,observer,abstract) VALUES(?,?,?,?,?,?,?,?,?,?,?)", to_db_src_B)
    con.commit()
except Exception as error:
    print(error)

# insert common uvit data to db
try:
    cursor.executemany("INSERT INTO source_sourcea_uvit(sourcea_id,sourceb_id) VALUES(?,?)", obs_uv)
    con.commit()
except Exception as error:
    print(error)

# insert common sxt data to db
try:
    cursor.executemany("INSERT INTO source_sourcea_sxt(sourcea_id,sourceb_id) VALUES(?,?)", obs_sx)
    con.commit()
except Exception as error:
    print(error)

# insert common laxpc data to db
try:
    cursor.executemany("INSERT INTO source_sourcea_laxpc(sourcea_id,sourceb_id) VALUES(?,?)", obs_lax)
    con.commit()
except Exception as error:
    print(error)

# insert common czti data to db
try:
    cursor.executemany("INSERT INTO source_sourcea_czti(sourcea_id,sourceb_id) VALUES(?,?)", obs_cz)
    con.commit()
except Exception as error:
    print(error)

# insert hmxbrefs and lmxbrefs data  to db
try:
    cursor.executemany("INSERT INTO source_refs(id,bib,Name,desc) VALUES(?,?,?,?)", hmxb_to_add)
    con.commit()
except Exception as error:
    print(error)

# insert catalog B source and publication data to db
for entry in src_pub:
    try:
        cursor.execute("INSERT INTO source_sourcea_Publications(sourcea_id,publication_id) VALUES(?,?)", entry)
        con.commit()
    except Exception as error:
        print(error)

con.commit()

con.close()


#
#  The Model to be loaded into the training code
#
#
import torch.nn as nn

class FashionCNN(nn.Module):
    """FashionCNN Model

    A Pytorch model to run classification on the Fashion-MNist Dataset

    """

    def __init__(self):
        """Pytorch Model Initialisation

        This consists of a simple network with 2 Convolution Layers to encode the images
        followed by three Linear layers to produce the final classification

        """
        super(FashionCNN, self).__init__()

        self.layer1 = nn.Sequential(
            nn.Conv2d(in_channels=1, out_channels=32, kernel_size=3, padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )

        self.layer2 = nn.Sequential(
            nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(2)
        )

        self.fc1 = nn.Linear(in_features=64*6*6, out_features=600)
        self.drop = nn.Dropout2d(0.25)
        self.fc2 = nn.Linear(in_features=600, out_features=120)
        self.fc3 = nn.Linear(in_features=120, out_features=10)

    def forward(self, x):
        """Pytorch Model Forward function

        Takes a sample batch of images and feeds it through the network
        Pytorch uses N x C x H x W

        Arguments:
             x (Torch.tensor): a valid batch of images

        """

        out = self.layer1(x)
        out = self.layer2(out)
        out = out.view(out.size(0), -1)
        out = self.fc1(out)
        out = self.drop(out)
        out = self.fc2(out)
        out = self.fc3(out)

        return out


#tests / conftest.py
import logging
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from app.database import Base
import sys
sys.path.append(r"C:/Users/61435/OneDrive - PHM Technology/Python/BasicApp")

SQLALCHEMY_TEST_DATABASE_URL = "sqlite:///./test.db"
engine = create_engine(SQLALCHEMY_TEST_DATABASE_URL)
TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

@pytest.fixture(scope="session")
def db():
    Base.metadata.create_all(bind=engine)
    session = TestingSessionLocal()
    yield session
    session.close()
    Base.metadata.drop_all(bind=engine)

@pytest.fixture(scope="function")
def clear_data():
    for table in reversed(Base.metadata.sorted_tables):
        engine.execute(table.delete())
    yield


from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage
from django.shortcuts import get_list_or_404, get_object_or_404
from rest_framework.response import Response
from rest_framework.views import APIView

from api.serializers import ProductSerializer
from shop.models import Product

class ProductView(APIView):
    def get(self, request, *args, **kwargs):
        if kwargs.get('product_pk'):
            objects = get_list_or_404(Product, shop_id=kwargs['shop_pk'], category_id=kwargs['categories_pk'], pk=kwargs['product_pk'])
        elif len(kwargs) == 0:
            objects = Product.objects.all()
        elif kwargs.get('categories_pk'):
            objects = get_list_or_404(Product, shop_id=kwargs['shop_pk'], category_id=kwargs['categories_pk'])
            paginator = Paginator(objects, 5)
            page = request.GET.get('page')
            if page:
                try:
                    objects = paginator.page(page)
                except PageNotAnInteger:
                    objects = paginator.page(1)
                except EmptyPage:
                    return Response("Такой страницы нет")
        else:
            objects = get_list_or_404(Product, shop_id=kwargs['shop_pk'])
        serializer = ProductSerializer(objects, many=True)
        return Response(serializer.data)

    def post(self, request, *args, **kwargs):
        serializer = ProductSerializer(data=request.data)
        if serializer.is_valid():
            product = serializer.save()
            return Response(serializer.data)
        else:
            return Response(serializer.errors, status=400)

    def put(self, request, *args, **kwargs):
        objects = get_object_or_404(Product, shop_id=kwargs['shop_pk'], category_id=kwargs['categories_pk'], pk=kwargs['product_pk'])
        serializer = ProductSerializer(objects, request.data)
        if serializer.is_valid():
            product = serializer.save()
            objects.update_counter += 1
            objects.save()
            return Response(serializer.data)
        else:
            return Response(serializer.errors, status=400)


import streamlit as st
from streamlit_option_menu import option_menu
import machine_learning as ml
import feature_extraction as fe
from bs4 import BeautifulSoup
import requests as re
import matplotlib.pyplot as plt
import validators
import pandas as pd
from sklearn import datasets
from sklearn.ensemble import RandomForestClassifier
import shutil
from st_aggrid import AgGrid
import streamlit.components.v1 as components
import base64

# 2. horizontal menu
# selected2 = option_menu(None, ["Home", "Upload", "Tasks", 'Settings'],
#     icons=['house', 'cloud-upload', "list-task", 'gear'],
#     menu_icon="cast", default_index=0, orientation="horizontal")
# selected2

with st.sidebar:
    selected = option_menu("Navigation Menu", ['UltraSec Phishing Detection','Phishing Prediction','Phishing Prediction By Upload CSV File','Project Details','Example',
    'Model Test Result','Data Set','Extension Download','Evil Twin Detector Download'],
        icons=['house','caret-right','book','list-task','bar-chart-line','caret-right','caret-right','download','download'], menu_icon="cast", default_index=0)
if selected == "UltraSec Phishing Detection":
    st.title('Phishing Detection')
    # st.write('This ML-based app is developed for educational purposes. Objective of the app is detecting phishing websites only using content data. Not URL!'
    #          ' You can see the details of approach, data set, and feature set if you click on _"See The Details"_. ')
    choice = st.selectbox("Please select your machine learning model",
                    [
                        'Logistic Regression','Gaussian Naive Bayes', 'Support Vector Machine', 'Decision Tree', 'Random Forest',
                        'AdaBoost', 'Neural Network', 'K-Neighbours'
                    ]
                    )
    model = ml.nb_model
    if choice == 'Logistic Regression':
        model = ml.lr_model
        st.write('LR model is selected!')
    elif choice == 'Gaussian Naive Bayes':
        model = ml.nb_model
        st.write('GNB model is selected!')
    elif choice == 'Support Vector Machine':
        model = ml.svm_model
        st.write('SVM model is selected!')
    elif choice == 'Decision Tree':
        model = ml.dt_model
        st.write('DT model is selected!')
    elif choice == 'Random Forest':
        model = ml.rf_model
        st.write('RF model is selected!')
    elif choice == 'AdaBoost':
        model = ml.ab_model
        st.write('AB model is selected!')
    elif choice == 'Neural Network':
        model = ml.nn_model
        st.write('NN model is selected!')
    else:
        model = ml.kn_model
        st.write('KN model is selected!')
    url = st.text_input('Enter the URL')
    # check the url is valid or not
    if st.button('Check!'):
        my_bar = st.progress(0)
        with st.spinner('Wait for it...'):
            try:
                valid=validators.url(url)
                if valid==True:
                    st.success("Url Format is valid")
                else:
                    st.warning("Invalid url")
                response = re.get(url, verify=False, timeout=4)
                if response.status_code == 404:
                    st.warning("404 Client Error. HTTP connection was not successful for the URL: "+url)
                elif response.status_code != 200:
                    print(". HTTP connection was not successful for the URL: ", url)
                    st.warning("Attention! This web page is a potential PHISHING! HTTP connection was not successful for the URL: "+url)
                else:
                    soup = BeautifulSoup(response.content, "html.parser")
                    vector = [fe.create_vector(soup)]  # it should be 2d array, so I added []
                    result = model.predict(vector)
                    if result[0] == 0:
                        st.success("This web page seems a legitimate!")
                        st.balloons()
                    else:
                        st.warning("Attention! This web page is a potential PHISHING!")
                        st.snow()
            except re.exceptions.RequestException as e:
                print("--> ", e)
                st.error(e,icon=None)
        my_bar = st.progress(100)
        st.success('Done!')
        # except re.exceptions.HTTPError as err:
        #     raise SystemExit(err)
if selected == "Phishing Prediction":
    st.title('Phishing Detection')
    # check the url is valid or not
    def evaluate(index,model,url):
        with st.spinner('Wait for it...'):
            try:
                valid=validators.url(url)
                if valid==True:
                    # st.success("Url Format is valid")
                    url_valid = "Url Format is valid"
                else:
                    # st.warning("Invalid url")
                    url_valid = "Invalid url"

                response = re.get(url, verify=False, timeout=4)
                if response.status_code == 404:
                    # st.warning("404 Client Error. HTTP connection was not successful for the URL: "+url)
                    url_response = "404 Client Error. HTTP connection was not successful for the URL: "
                elif response.status_code != 200:
                    print(". HTTP connection was not successful for the URL: ", url)
                    # st.warning("Attention! This web page is a potential PHISHING! HTTP connection was not successful for the URL: "+url)
                    url_response = "Attention! This web page is a potential PHISHING! HTTP connection was not successful for the URL: "
                else:
                    soup = BeautifulSoup(response.content, "html.parser")
                    vector = [fe.create_vector(soup)]  # it should be 2d array, so I added []
                    result = model.predict(vector)
                    if result[0] == 0:
                        # st.success("This web page seems a legitimate!")
                        url_response = "This web page seems a legitimate!"
                        # st.balloons()
                    else:
                        # st.warning("Attention! This web page is a potential PHISHING!")
                        url_response ="Attention! This web page is a potential PHISHING!"
                        # st.snow()
                data_return = {}
                data_return = {
                    index : [
                        url_valid,
                        url_response
                    ]
                }
                # data_return = {
                #     index : index,
                #     index+ ' Format':url_valid,
                #     index+ ' Result':url_response,
                # }

            except re.exceptions.RequestException as e:
                print("--> ", e)
                st.error(e,icon=None)
        return data_return
    url = st.text_input('Enter the URL')
    if st.button('Check!'):
        my_bar = st.progress(0)
        complete_progress = 0

        # st.write('Logistic Regression')
        model = ml.lr_model
        index = 'LR'
        lr_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('Gaussian Naive Bayes')
        model = ml.nb_model
        index = 'NB'
        gnb_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('Support Vector Machine')
        model = ml.svm_model
        index = 'SVM'
        svm_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('Decision Tree')
        model = ml.dt_model
        index = 'DT'
        dt_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('Random Forest')
        model = ml.rf_model
        index = 'RF'
        rf_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('AdaBoost')
        model = ml.ab_model
        index = 'AB'
        ab_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('Neural Network')
        model = ml.nn_model
        index = 'NN'
        nn_result = evaluate(index,model,url)
        complete_progress+=13
        my_bar.progress(complete_progress)

        # st.write('K-Neighbours')
        model = ml.kn_model
        index = 'KN'
        kn_result = evaluate(index,model,url)
        complete_progress+=9
        my_bar.progress(complete_progress)

        # def Merge(dict1, dict2,dict3,dict4,dict5,dict6,dict7,dict8):
        #     res = {**dict1, **dict2,**dict3,**dict4,**dict5,**dict6,**dict7,**dict8}
        #     return res

        column_index = ['Logistic Regression','Gaussian Naive Bayes', 'Support Vector Machine', 'Decision Tree', 'Random Forest',
        'AdaBoost', 'Neural Network', 'K-Neighbours']

        # Driver code
        dict_result = {k:v for d in (lr_result,gnb_result,svm_result,dt_result,rf_result,ab_result,nn_result,kn_result) for k,v in d.items()}
        # dict_result= Merge(lr_result,gnb_result,svm_result,dt_result,rf_result,ab_result,nn_result,kn_result)
        # st.write(dict_result)
        # dict_result = {k:v for d in (lr_result,gnb_result) for k,v in d.items()}

        # lr = pd.DataFrame(lr_result)
        # nb = pd.DataFrame(gnb_result)

        # lr.reset_index(drop=True, inplace=True)
        # nb.reset_index(drop=True, inplace=True)

        # concated = pd.concat([lr,nb],axis=0)
        # st.write(dict_result)

        # result_df = pd.concat([pd.DataFrame(lr_result),pd.DataFrame(gnb_result),
        # pd.DataFrame(svm_result),pd.DataFrame(dt_result),pd.DataFrame(rf_result),
        # pd.DataFrame(ab_result),pd.DataFrame(nn_result,kn_result)],axis=0,index=indexes)
        # # st.write(result_df)
        row_index = ['URL Format Validity','Result']
        result_df = pd.DataFrame(data=dict_result,index=row_index)
        result_df_T = result_df.transpose()
        result_concat = pd.concat([ml.df_results,result_df_T],axis=1)
        result_concat.index = column_index
        st.success('Done!')
        st.dataframe(result_concat)
        # AgGrid(result_concat)
if selected == "Phishing Prediction By Upload CSV File":
    st.header('Phishing Prediction By Upload CSV File')
    def evaluate_csv(url):
        try:
            valid=validators.url(url)
            if valid==True:
                # st.success("Url Format is valid")
                dataframe['URL Validity'] = 'Valid'
            else:
                # st.warning("Invalid url")
                dataframe['URL Format Validity'] = 'Invalid'
            response = re.get(url, verify=False, timeout=4)
            if response.status_code == 404:
                # st.warning("404 Client Error. HTTP connection was not successful for the URL: "+url)
                dataframe['Response'] = '404 Error'
            elif response.status_code != 200:
                # print(". HTTP connection was not successful for the URL: ", url)
                # st.warning("Attention! This web page is a potential PHISHING! HTTP connection was not successful for the URL: "+url)
                dataframe['Response'] = 'HTTP Connection was not successful'
            else:
                soup = BeautifulSoup(response.content, "html.parser")
                vector = [fe.create_vector(soup)]  # it should be 2d array, so I added []
                result = model.predict(vector)
                if result[0] == 0:
                    # st.success("This web page seems a legitimate!")
                    # st.balloons()
                    dataframe['Response'] = 'Legitimate'
                else:
                    # st.warning("Attention! This web page is a potential PHISHING!")
                    # st.snow()
                    dataframe['Response'] = 'Phishing'
            dataframe['Error'] = ''
        except re.exceptions.RequestException as e:
            print("--> ", e)
            # st.error(e,icon=None)
            dataframe['Error'] = e

    # if "points_1" not in st.session_state:
    #     # points for guesses
    #     st.session_state.points_1 = 0

    @st.cache
    def convert_df(df):
        # IMPORTANT: Cache the conversion to prevent computation on every rerun
        return df.to_csv().encode('utf-8')

    uploaded_file = st.file_uploader("Choose a file")
    if uploaded_file is not None:
        # Can be used wherever a "file-like" object is accepted:
        chunk = pd.read_csv(uploaded_file,chunksize=1000)
        dataframe = pd.concat(chunk)
        st.write(dataframe)

        choice = st.selectbox("Please select your machine learning model",
                [
                    'Logistic Regression','Gaussian Naive Bayes', 'Support Vector Machine', 'Decision Tree', 'Random Forest',
                    'AdaBoost', 'Neural Network', 'K-Neighbours'
                ]
                )
        model = ml.nb_model
        if choice == 'Logistic Regression':
            model = ml.lr_model
            st.write('LR model is selected!')
        elif choice == 'Gaussian Naive Bayes':
            model = ml.nb_model
            st.write('GNB model is selected!')
        elif choice == 'Support Vector Machine':
            model = ml.svm_model
            st.write('SVM model is selected!')
        elif choice == 'Decision Tree':
            model = ml.dt_model
            st.write('DT model is selected!')
        elif choice == 'Random Forest':
            model = ml.rf_model
            st.write('RF model is selected!')
        elif choice == 'AdaBoost':
            model = ml.ab_model
            st.write('AB model is selected!')
        elif choice == 'Neural Network':
            model = ml.nn_model
            st.write('NN model is selected!')
        else:
            model = ml.kn_model
            st.write('KN model is selected!')

        if len(dataframe) <51 :
            # check the url is valid or not
            if st.button('Check!'):
                if uploaded_file is not None:
                    my_bar = st.progress(0)
                    with st.spinner('Wait for it...'):
                        dataframe.apply(lambda row : evaluate_csv(row['url']),axis=1)
                        AgGrid(dataframe)
                    my_bar.progress(100)
                    st.success('Done!')
                    csv = convert_df(dataframe)
                    st.download_button(
                        label="Download Prediction (CSV)",
                        data=csv,
                        file_name='Download Result(CSV).csv',
                        mime='text/csv',
                        help='Download Prediction Result in CSV format '
                    )
        else:
            st.write('Maximum of 50 rows of Urls are accepted. PLEASE ReUpload')
if selected == "Project Details":
    # with st.expander("PROJECT DETAILS"):
    st.subheader('Approach')
    st.write('I used _supervised learning_ to classify phishing and legitimate websites. '
            'I benefit from content-based approach and focus on html of the websites. '
            'Also, I used scikit-learn for the ML models.'
            )
    st.write('For this educational project, '
            'I created my own data set and defined features, some from the literature and some based on manual analysis. '
            'I used requests library to collect data, BeautifulSoup module to parse and extract features. ')
    st.write('The source code and data sets are available in the below Github link:')
    st.write('_https://github.com/emre-kocyigit/phishing-website-detection-content-based_')
    st.subheader('Data set')
    st.write('I used _"phishtank.org"_ & _"tranco-list.eu"_ as data sources.')
    st.write('Totally 26584 websites ==> **_16060_ legitimate** websites | **_10524_ phishing** websites')
    st.write('Data set was created in October 2022.')
    # ----- FOR THE PIE CHART ----- #
    labels = 'phishing', 'legitimate'
    phishing_rate = int(ml.phishing_df.shape[0] / (ml.phishing_df.shape[0] + ml.legitimate_df.shape[0]) * 100)
    legitimate_rate = 100 - phishing_rate
    sizes = [phishing_rate, legitimate_rate]
    explode = (0.1, 0)
    fig, ax = plt.subplots()
    ax.pie(sizes, explode=explode, labels=labels, shadow=True, startangle=90, autopct='%1.1f%%')
    ax.axis('equal')
    st.pyplot(fig)
    # ----- !!!!! ----- #
    st.write('Features + URL + Label ==> Dataframe')
    st.markdown('label is 1 for phishing, 0 for legitimate')
    number = st.slider("Select row number to display", 0, 100)
    st.dataframe(ml.legitimate_df.head(number))
    @st.cache
    def convert_df(df):
        # IMPORTANT: Cache the conversion to prevent computation on every rerun
        return df.to_csv().encode('utf-8')
    csv = convert_df(ml.df)
    st.download_button(
        label="Download data as CSV",
        data=csv,
        file_name='phishing_legitimate_structured_data.csv',
        mime='text/csv',
    )
if selected == "Example":
    # with st.expander('EXAMPLE PHISHING URLs:'):
    st.title('EXAMPLE PHISHING URLs:')
    st.write('_https://rtyu38.godaddysites.com/_')
    st.write('_https://karafuru.invite-mint.com/_')
    st.write('_https://defi-ned.top/h5/#/_')
    st.write('_https://beeflash.net/_')
    st.caption('REMEMBER, PHISHING WEB PAGES HAVE SHORT LIFECYCLE! SO, THE EXAMPLES SHOULD BE UPDATED!')
if selected == "Model Test Result":
    st.subheader('Features')
    st.write('I used only content-based features. I didn\'t use url-based faetures like length of url etc.'
            'Most of the features extracted using find_all() method of BeautifulSoup module after parsing html.')
    st.subheader('Results')
    st.write('I used 8 different ML classifiers of scikit-learn and tested them implementing k-fold cross validation.'
            'Firstly obtained their confusion matrices, then calculated their accuracy, precision and recall scores.'
            'Comparison table is below:')
    st.table(ml.df_results)
    st.write('LR --> Logistic Regression')
    st.write('NB --> Gaussian Naive Bayes')
    st.write('SVM --> Support Vector Machine')
    st.write('DT --> Decision Tree')
    st.write('RF --> Random Forest')
    st.write('AB --> AdaBoost')
    st.write('NN --> Neural Network')
    st.write('KN --> K-Neighbours')
if selected == "Data Set":
    @st.cache
    def load_data(nrows):
        data = pd.read_csv('malicious_phish.csv', nrows=nrows)
        return data
    # st.text_input(label, value="", max_chars=None, key=None, type="default", help=None, autocomplete=None, on_change=None, args=None, kwargs=None, *, placeholder=None, disabled=False, label_visibility="visible")
    phish_data = load_data(1000)
    # st.write(weekly_data)
    # st.subheader('Weekly Demand Data')
    # AgGrid(phish_data)

    number = st.slider("Select row number to display", 50, 500)
    st.dataframe(phish_data.head(number))
    # #Bar Chart
    # st.bar_chart(weekly_data)
if selected == "Extension Download":
    st.title('Download Page')
    st.subheader('Download the latest version Ultrasec Phishing Detector')
    shutil.make_archive('phishing-detection-plugin-main', 'zip', 'phishing-detection-plugin-main')
    with open("phishing-detection-plugin-main.zip", "rb") as fp:
        btn1 = st.download_button(
            label="Download UltraSec Phishing Detector 1.0.1",
            data=fp,
            file_name='UltraSec-Phishing-Detector.zip',
            mime='application/zip',
            key="1",
        )
    st.markdown("Looking for way to install extension? [**Watch Video Below**](#video-to-install-ultrasec-phishing-detector-chrome-extension)")
    st.subheader('Instructions')
    st.markdown("[1. Step 1 - download a zip file with the extension](#step-1-download-a-zip-file-with-the-extension)")
    st.markdown("[2. Step 2 - extract the contents of the zip file](#step-2-extract-the-contents-of-the-zip-file)")
    st.markdown("[3. Step 3 - open the extension page in google chrome](#step-3-open-the-extension-page-in-google-chrome)")
    st.markdown("[4. Step 4 - activate developer mode](#step-4-activate-developer-mode)")
    st.markdown("[5. Step 5 - load unpacked extension](#step-5-load-unpacked-extension)")
    st.header('Step 1 - download a zip file with the extension')
    with open("phishing-detection-plugin-main.zip", "rb") as fp:
        btn2 = st.download_button(
            label="Download UltraSec Phishing Detector 1.0.1",
            data=fp,
            file_name='UltraSec-Phishing-Detector.zip',
            mime='application/zip',
            key="2",
        )
    st.header('Step 2 - extract the contents of the zip file')
    st.write('Right click on the downloaded zip file, then click "Extract Here".')
    st.header('Step 3 - open the extension page in google chrome')
    st.write('''There are several ways todo that.

    Option 1: type chrome://extensions in the url bar and press enter.
    Option 2: click on the tree dots in the top right of the browser, then click "More tools" then click "Extensions".
    ''')
    st.header('Step 4 - activate developer mode')
    st.write('Turn on the switch on the top right of the page that says "Developer mode"')
    st.header('Step 5 - load unpacked extension')
    st.write('''Click on the button on the top left of the page that says "Load unpacked".

    Then select a folder that contains the manifest.json file.
    ''')
    st.header('Video to install UltraSec Phishing Detector Chrome Extension.')
    video_file = open('install_extension.mp4', 'rb')
    video_bytes = video_file.read()
    st.video(video_bytes)
if selected == 'Evil Twin Detector Download':
    # embed streamlit docs in a streamlit app
    # components.iframe("https://docs.streamlit.io/en/latest")
    # components.html("""

    # """)

    # def show_pdf(file_path):
    #     with open(file_path,"rb") as f:
    #         base64_pdf = base64.b64encode(f.read()).decode('utf-8')
    #     pdf_display = f'<iframe src="data:application/pdf;base64,{base64_pdf}" width="800" height="800" type="application/pdf"></iframe>'
    #     st.markdown(pdf_display, unsafe_allow_html=True)

    # show_pdf('training.pdf')

    st.header('Evil Twin Detector Download')
    st.caption('An evil twin attack takes place when an attacker sets up a fake Wi-Fi access point hoping that users will connect to it instead of a legitimate one. When users connect to this access point, all the data they share with the network passes through a server controlled by the attacker. An attacker can create an evil twin with a smartphone or other internet-capable device and some readily available software. Evil twin attacks are more common on public Wi-Fi networks which are unsecured and leave your personal data vulnerable.')
    st.markdown('[**Download Evin Twin Detector**](https://github.com/ryanlee13288/EvilTwinDetectFYP)')


# Note: This is year-wise unlike the other webcrawlers!
import requests
from bs4 import BeautifulSoup
from pprint import pprint
import json

ses = requests.session()

def article_catch(url_list, j):
    articles = []
    for i, url in enumerate(url_list):
        Soup = BeautifulSoup(ses.get(url).text, "html.parser")
        # title = Soup.find('h1')
        article = Soup.find('article')
        title = article.find('h1', {'class': 'entry-title'})
        time_part = article.find('span', {'class': 'entry-meta-date updated'})
        fig = article.find('figure')
        content_div = article.find('div', {'class': 'entry-content clearfix'})
        content = ""
        for par in content_div.find_all('p'):
            if content != "":
                content = content + '\n' + par.text
            else:
                content = par.text
        try:
            image = fig.find('img').get('src')
        except AttributeError or TypeError:
            image = None
        dit = {
            'title': title.text,
            'time': time_part.find('a').text,
            'image': image,
            'content': content
        }
        articles.append(dit)
        with open('DailyBonnet{0}.json'.format(j), 'w+') as f:
            f.write(json.dumps(articles))

url_base = 'https://www.dailybonnet.com/2018/page'
for i in range(90):
    url2scrape = url_base + '/' + str(i+1)
    Soup = BeautifulSoup(ses.get(url2scrape).text, "html.parser")
    titl = Soup.find('title').text
    if "Daily" not in titl:
        print('page', i+1)
        break
    url_list = []
    for article in Soup.find_all('article'):
        head = article.find('h3', {'class': 'entry-title mh-posts-list-title'})
        url_list.append(head.find('a').get('href'))

    pprint(url_list)
    article_catch(url_list, i)
    print(i, "Done")


import uvicorn

from typing import Optional
from multiprocessing import cpu_count
from fastapi.responses import ORJSONResponse
from fastapi import FastAPI, Body, Depends, BackgroundTasks

from functions import process, get_all_users, filter_and_delete_messages, run_background
from dependencies import authenticate, allow_one_process

app = FastAPI(default_response_class=ORJSONResponse)

@app.get('/status', dependencies=[Depends(authenticate)])
def status():
    """
    Статус API
    """

    return {
        'status': 'Blocked' if process.is_blocked else 'Available',
        **process.last_state,
        **process.current_state
        }

@app.get('/delete_messages', dependencies=[Depends(authenticate), Depends(allow_one_process)])
async def delete_messages(background: BackgroundTasks,
                          filter: str = Body(..., embed=True),
                          count_only: Optional[bool] = Body(False, embed=True)):
    """
    Удалить письмо из почты всех пользователей
    """

    # Обозначить данное действие
    process.set_current_state(filter, action='DELETE' if not count_only else 'COUNT')

    try:

        # Заблокировать процесс
        process.add_worker(2)

        # Получить список всех пользователей
        mails: list[str] = get_all_users()

        # Выполнить задачу в фоновом режиме
        background.add_task(filter_and_delete_messages, mails, filter, count_only)

    except:
        raise

    finally:

        # Разблокировать процесс
        process.remove_worker_and_swap()

    return {'result': 'Accepted'}

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=8754)


import datetime

from mock import patch

import unittest

from cubicweb.devtools.testlib import CubicWebTC
from cubicweb.devtools import PostgresApptestConfiguration

from cubicweb_francearchives.testutils import EADImportMixin

from cubicweb_frarchives_edition.entities.kibana.sqlutils import create_kibana_authorities_sql

from pgfixtures import setup_module, teardown_module  # noqa

class KibanaIndexerImporterTC(EADImportMixin, CubicWebTC):
    configcls = PostgresApptestConfiguration

    def setup_database(self):
        super(KibanaIndexerImporterTC, self).setup_database()
        with self.admin_access.cnx() as cnx:
            cnx.create_entity(
                "Service",
                category="?",
                name="Les Archives Nationales",
                short_name="Les AN",
                code="FRAN",
            )
            scheme = cnx.create_entity("ConceptScheme", title="Title")
            concept0 = cnx.create_entity("Concept", cwuri="https://example.com", in_scheme=scheme)
            concept1 = cnx.create_entity("Concept", cwuri="https://foo.com", in_scheme=scheme)
            cnx.create_entity(
                "Label",
                label="example",
                language_code="en",
                kind="preferred",
                label_of=concept0,
            )
            cnx.create_entity(
                "Label",
                label="exemple",
                language_code="fr-fr",
                kind="preferred",
                label_of=concept0,
            )
            cnx.create_entity(
                "Label",
                label="foo",
                language_code="fr-fr",
                kind="preferred",
                label_of=concept1,
            )
            cnx.create_entity("SubjectAuthority", label="Exemple", same_as=(concept0, concept1))
            cnx.commit()

    def publish_ir(self, cnx):
        # this is done by frarchives_edition.tasks.import_ead.launch_task
        rset = cnx.execute(
            """Any S WHERE S is State, S state_of WF, X default_workflow WF,
            X name "FindingAid", WF initial_state S"""
        )
        cnx.system_sql(
            "INSERT INTO in_state_relation (eid_from, eid_to) "
            "SELECT cw_eid, %(eid_to)s FROM cw_findingaid WHERE "
            "NOT EXISTS (SELECT 1 FROM in_state_relation i "
            "WHERE i.eid_from = cw_eid)",
            {"eid_to": rset[0][0]},
        )
        for fa in cnx.find("FindingAid").entities():
            adapted = fa.cw_adapt_to("IWorkflowable")
            adapted.fire_transition_if_possible("wft_cmsobject_publish")
        cnx.commit()

    def create_authority_record(self, cnx):
        kind_eid = cnx.find("AgentKind", name="person")[0][0]
        name = "Jean Cocotte"
        return cnx.create_entity(
            "AuthorityRecord",
            record_id="FRAN_NP_006883",
            agent_kind=kind_eid,
            reverse_name_entry_for=cnx.create_entity(
                "NameEntry", parts=name, form_variant="authorized"
            ),
            xml_support="foo",
            start_date=datetime.datetime(1940, 1, 1),
            reverse_occupation_agent=cnx.create_entity("Occupation", term="éleveur de poules"),
            reverse_history_agent=cnx.create_entity("History", text="
Il aimait les poules
"),
        )

    def test_kibana_service_es(self):
        """Test for IKibanaIndexSerializable index"""
        with self.admin_access.cnx() as cnx:
            service = cnx.find("Service", code="FRAN").one()
            bc = cnx.create_entity(
                "BaseContent",
                title="program",
                content="31 juin",
                basecontent_service=service,
                reverse_children=cnx.create_entity(
                    "Section", title="Publication", name="publication"
                ),
            )
            cnx.commit()
            doc = service.cw_adapt_to("IKibanaIndexSerializable").serialize()
            self.assertEqual(0, doc["documents_count"])
            self.import_filepath(cnx, "FRAN_IR_050263.xml")
            service = cnx.find("Service", code="FRAN").one()
            s_ift = service.cw_adapt_to("IKibanaIndexSerializable")
            self.assertEqual(s_ift.es_id, service.eid)
            doc = s_ift.serialize()
            self.assertEqual(0, doc["documents_count"])
            self.publish_ir(cnx)
            service = cnx.find("Service", code="FRAN").one()
            s_ift = service.cw_adapt_to("IKibanaIndexSerializable")
            self.assertEqual(s_ift.es_id, service.eid)
            doc = s_ift.serialize()
            self.assertEqual(523, doc["documents_count"])
            self.assertEqual(523, doc["archives"])
            self.assertEqual(0, doc["siteres"])
            self.assertEqual(0, doc["nomina"])
            # publish BaseContent
            bc = cnx.entity_from_eid(bc.eid)
            bc.cw_adapt_to("IWorkflowable").fire_transition("wft_cmsobject_publish")
            cnx.commit()
            doc = service.cw_adapt_to("IKibanaIndexSerializable").serialize()
            self.assertEqual(524, doc["documents_count"])
            self.assertEqual(523, doc["archives"])
            self.assertEqual(1, doc["siteres"])

    def test_kibana_authority_es(self):
        """Test for IKibanaIndexSerializable index"""
        with self.admin_access.cnx() as cnx:
            create_kibana_authorities_sql(cnx)
            self.import_filepath(cnx, "FRAN_IR_050263.xml")
            create_kibana_authorities_sql(cnx)
            agent = cnx.execute(
                "Any X WHERE X is AgentAuthority, X label %(l)s", {"l": "Hugo, Victor"}
            ).one()
            record = self.create_authority_record(cnx)
            agent.cw_set(same_as=record)
            esdoc = agent.cw_adapt_to("IKibanaIndexSerializable").serialize()
            self.assertEqual(False, esdoc["quality"])
            agent.cw_set(quality=True)
            cnx.commit()
            self.assertEqual(len(agent.reverse_authority[0].index), 1)
            s_ift = agent.cw_adapt_to("IKibanaIndexSerializable")
            self.assertEqual(s_ift.es_id, agent.eid)
            doc = s_ift.serialize()
            self.assertEqual(0, cnx.system_sql("Select * from published.cw_FindingAid").rowcount)
            self.assertEqual(0, doc["documents_count"])  # AuthorityRecord is not taken account
            self.assertEqual(0, doc["services_count"])
            self.assertEqual([], doc["services"])
            initial_doc = agent.cw_adapt_to("IKibanaInitiaLAuthorityIndexSerializable").serialize()
            self.assertCountEqual(doc, initial_doc)
            # publish FindingAid
            self.publish_ir(cnx)
            create_kibana_authorities_sql(cnx)
            service = cnx.find("Service", code="FRAN").one()
            fa = cnx.find("FindingAid").one()
            self.assertEqual(fa.related_service.eid, service.eid)
            agent.cw_clear_all_caches()
            esdoc = agent.cw_adapt_to("IKibanaIndexSerializable").serialize()
            expected = {
                "creation_date": agent.creation_date,
                "cw_etype": "AgentAuthority",
                "documents_count": 1,
                "eid": agent.eid,
                "grouped_with": [],
                "grouped_with_count": 0,
                "is_grouped": False,
                "label": "Hugo, Victor",
                "location": [],
                "quality": True,
                "reindex_date": datetime.date.today().strftime("%Y-%m-%d"),
                "same_as": [
                    {"label": "FRAN_NP_006883", "source": "EAC-CPF", "uri": "FRAN_NP_006883"}
                ],
                "same_as_count": 1,
                "services": [
                    {
                        "code": "FRAN",
                        "eid": service.eid,
                        "level": "None",
                        "title": "Les Archives Nationales",
                    }
                ],
                "services_count": 1,
                "types": ["persname"],
                "urlpath": f"http://testing.fr/cubicweb/agent/{agent.eid}",
            }
            self.assertDictEqual(expected, esdoc)
            self.assertEqual(service.eid, esdoc["services"][0]["eid"])
            ini_esdoc = agent.cw_adapt_to("IKibanaInitiaLAuthorityIndexSerializable").serialize()
            self.assertCountEqual(doc, ini_esdoc)
            self.assertCountEqual(esdoc, ini_esdoc)

    @patch("elasticsearch.client.indices.IndicesClient.exists")
    @patch("elasticsearch.client.Elasticsearch.index")
    def test_location_authority(self, index, exists):
        with self.admin_access.cnx() as cnx:
            ce = cnx.create_entity
            loc1 = ce(
                "LocationAuthority",
                label="Paris (Paris, France)",
                longitude=48.85341,
                latitude=2.3488,
            )
            cnx.commit()
            esdoc = loc1.cw_adapt_to("IKibanaIndexSerializable").serialize()
            expected = {
                "creation_date": loc1.creation_date,
                "cw_etype": "LocationAuthority",
                "documents_count": 0,
                "eid": loc1.eid,
                "grouped_with": [],
                "grouped_with_count": 0,
                "is_grouped": False,
                "label": "Paris (Paris, France)",
                "location": [48.85341, 2.3488],
                "quality": False,
                "reindex_date": datetime.date.today().strftime("%Y-%m-%d"),
                "same_as": [],
                "same_as_count": 0,
                "services": [],
                "services_count": 0,
                "types": [],
                "urlpath": f"http://testing.fr/cubicweb/location/{loc1.eid}",
            }
            self.assertDictEqual(expected, esdoc)

    def test_subject_authority_same_as(self):
        """Test SubjectAuthority same_as."""
        with self.admin_access.cnx() as cnx:
            authority = cnx.find("SubjectAuthority").one()
            actual = authority.cw_adapt_to("IKibanaIndexSerializable").serialize()
            self.assertEqual(actual["same_as_count"], 2)
            self.assertCountEqual(
                actual["same_as"],
                [
                    {"label": "foo", "uri": "https://foo.com", "source": "Title"},
                    {"label": "exemple", "uri": "https://example.com", "source": "Title"},
                ],
            )

    def test_agent_authority_same_as(self):
        """Test SubjectAuthority same_as."""
        with self.admin_access.cnx() as cnx:
            kind_eid = cnx.find("AgentKind", name="person")[0][0]
            name = "La Toto compagnie"
            auth_rec = cnx.create_entity(
                "AuthorityRecord",
                record_id="FRAN_NP_006883",
                agent_kind=kind_eid,
                reverse_name_entry_for=cnx.create_entity(
                    "NameEntry", parts=name, form_variant="authorized"
                ),
                xml_support="foo",
            )
            authority = cnx.create_entity("AgentAuthority", label="person", same_as=auth_rec)
            cnx.commit()
            json = authority.cw_adapt_to("IKibanaIndexSerializable").serialize()
            self.assertCountEqual(
                json["same_as"],
                [{"label": "FRAN_NP_006883", "uri": "FRAN_NP_006883", "source": "EAC-CPF"}],
            )

if __name__ == "__main__":
    unittest.main


"""
Route Between Nodes: Given a directed graph, design an algorithm to find out whether there is a route between two nodes.

Performing BFS using queue to find the route between two nodes.S
"""
from collections import  deque
class Graph:
    def __init__(self, edges, N):
        self.adjacent = [[] for i in range(N)]
        for (source,dest) in edges:
             self.adjacent[source].append(dest)

def isConnected(graph,source,dest, visited):
    q  = deque()
    visited[source] = True    #source vertex is visited
    q.append(source)
    while q:
        v = q.popleft()
        if v == dest:    #if destination found
            return True
        for u in graph.adjacent[v]:    #for every adjacent node(v -> u)
            if not visited[u]:
                visited[u] = True       #marking as visited
                q.append(u)             #append to the queue
    return False
if __name__ == '__main__':
    edges = [(0,3),(1,0),(1,2),(1,4),(2,7),(3,4),(3,5),(4,3),(4,6),(5,6),(6,7) ]   #edges of the graph
    N = 8   #number of nodes in the graph
    graph = Graph(edges,N)
    visited =  [False] * N
    (source, dest) = (0,1)   #sourcea and destination vertex
    if isConnected(graph,source,dest,visited):
        print("Route exists between ", source, " to ", dest)
    else:
        print("No Route exists between ", source, " to ", dest)


from django.urls import path
from . import views

app_name = 'items'

urlpatterns = [
    path('', views.list_items, name='list_items'),
    path('adicionar/', views.add_item, name='add_item'),
    path('editar/<int:id_item>/', views.edit_item, name='edit_item'),
    path('excluir/<int:id_item>/', views.delete_item, name='delete_item'),
]