from transformers import ViTModel, PreTrainedModel, PretrainedConfig
from torch import nn
from transformers.activations import ACT2FN
from torch.nn import CrossEntropyLoss, MSELoss
import math
from transformers.modeling_outputs import (
    BaseModelOutput,
    BaseModelOutputWithPooling,
    SequenceClassifierOutput,
)
import torch

class ViViTConfig(PretrainedConfig):
    model_type = "vivit"

    def __init__(
        self,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        intermediate_size=3072,
        hidden_act="gelu",
        hidden_dropout_prob=0.0,
        attention_probs_dropout_prob=0.0,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        is_encoder_decoder=False,
        video_size=(30, 224, 224),
        tubelet_size=(2, 16, 16),
        num_channels=3,
        qkv_bias=True,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps

        self.video_size = video_size
        self.tubelet_size = tubelet_size
        self.num_channels = num_channels
        self.qkv_bias = qkv_bias

class TubeletEmbeddings(nn.Module):
    """
    Video to Tubelet Embedding.

    """

    def __init__(self, video_size, patch_size, num_channels=3, embed_dim=768):
        super().__init__()
        self.video_size = video_size
        self.patch_size = patch_size
        self.num_patches = (
            (video_size[2] // patch_size[2])
            * (video_size[1] // patch_size[1])
            * (video_size[0] // patch_size[0])
        )
        self.embed_dim = embed_dim

        self.projection = nn.Conv3d(
            num_channels, embed_dim, kernel_size=patch_size, stride=patch_size
        )

    def forward(self, pixel_values, interpolate_pos_encoding=False):
        batch_size, num_channels, num_frames, height, width = pixel_values.shape
        if not interpolate_pos_encoding:
            if height != self.video_size[-2] or width != self.video_size[-1]:
                raise ValueError(
                    f"Input image size ({height}*{width}) doesn't match model ({self.video_size[-2]}*{self.video_size[-1]})."
                )

        # TODO: check this
        x = self.projection(pixel_values).flatten(2).transpose(1, 2)
        return x

class ViViTEmbeddings(nn.Module):
    def __init__(self, config):
        super().__init__()

        self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
        self.patch_embeddings = TubeletEmbeddings(
            config.video_size,
            config.tubelet_size,
            config.num_channels,
            config.hidden_size,
        )

        self.position_embeddings = nn.Parameter(
            torch.zeros(1, self.patch_embeddings.num_patches + 1, config.hidden_size)
        )
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.config = config

    def interpolate_pos_encoding(self, embeddings, height, width):
        # TODO: Implement interpolation
        raise NotImplementedError()

    def forward(self, pixel_values, interpolate_pos_encoding=False):
        batch_size, num_frames, num_channels, height, width = pixel_values.shape
        embeddings = self.patch_embeddings(
            pixel_values, interpolate_pos_encoding=interpolate_pos_encoding
        )

        # add the [CLS] token to the embedded patch tokens
        cls_tokens = self.cls_token.expand(batch_size, -1, -1)
        embeddings = torch.cat((cls_tokens, embeddings), dim=1)

        # add positional encoding to each token
        if interpolate_pos_encoding:
            embeddings = embeddings + self.interpolate_pos_encoding(
                embeddings, height, width
            )
        else:
            embeddings = embeddings + self.position_embeddings

        embeddings = self.dropout(embeddings)

        return embeddings

class ViViTSelfAttention(nn.Module):
    def __init__(self, config):
        super().__init__()
        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(
            config, "embedding_size"
        ):
            raise ValueError(
                f"The hidden size {config.hidden_size,} is not a multiple of the number of attention "
                f"heads {config.num_attention_heads}."
            )

        self.num_attention_heads = config.num_attention_heads
        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
        self.all_head_size = self.num_attention_heads * self.attention_head_size

        self.query = nn.Linear(
            config.hidden_size, self.all_head_size, bias=config.qkv_bias
        )
        self.key = nn.Linear(
            config.hidden_size, self.all_head_size, bias=config.qkv_bias
        )
        self.value = nn.Linear(
            config.hidden_size, self.all_head_size, bias=config.qkv_bias
        )

        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)

    def transpose_for_scores(self, x):
        new_x_shape = x.size()[:-1] + (
            self.num_attention_heads,
            self.attention_head_size,
        )
        x = x.view(*new_x_shape)
        return x.permute(0, 2, 1, 3)

    def forward(self, hidden_states, head_mask=None, output_attentions=False):
        mixed_query_layer = self.query(hidden_states)

        key_layer = self.transpose_for_scores(self.key(hidden_states))
        value_layer = self.transpose_for_scores(self.value(hidden_states))
        query_layer = self.transpose_for_scores(mixed_query_layer)

        # Take the dot product between "query" and "key" to get the raw attention scores.
        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))

        attention_scores = attention_scores / math.sqrt(self.attention_head_size)

        # Normalize the attention scores to probabilities.
        attention_probs = nn.functional.softmax(attention_scores, dim=-1)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs = self.dropout(attention_probs)

        # Mask heads if we want to
        if head_mask is not None:
            attention_probs = attention_probs * head_mask

        context_layer = torch.matmul(attention_probs, value_layer)

        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
        context_layer = context_layer.view(*new_context_layer_shape)

        outputs = (
            (context_layer, attention_probs) if output_attentions else (context_layer,)
        )

        return outputs

class ViViTSelfOutput(nn.Module):
    """
    The residual connection is defined in ViTLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    """

    def __init__(self, config):
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):

        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)

        return hidden_states

class ViViTAttention(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.attention = ViViTSelfAttention(config)
        self.output = ViViTSelfOutput(config)
        self.pruned_heads = set()

    def prune_heads(self, heads):
        if len(heads) == 0:
            return
        heads, index = find_pruneable_heads_and_indices(
            heads,
            self.attention.num_attention_heads,
            self.attention.attention_head_size,
            self.pruned_heads,
        )

        # Prune linear layers
        self.attention.query = prune_linear_layer(self.attention.query, index)
        self.attention.key = prune_linear_layer(self.attention.key, index)
        self.attention.value = prune_linear_layer(self.attention.value, index)
        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)

        # Update hyper params and store pruned heads
        self.attention.num_attention_heads = self.attention.num_attention_heads - len(
            heads
        )
        self.attention.all_head_size = (
            self.attention.attention_head_size * self.attention.num_attention_heads
        )
        self.pruned_heads = self.pruned_heads.union(heads)

    def forward(self, hidden_states, head_mask=None, output_attentions=False):
        self_outputs = self.attention(hidden_states, head_mask, output_attentions)

        attention_output = self.output(self_outputs[0], hidden_states)

        outputs = (attention_output,) + self_outputs[
            1:
        ]  # add attentions if we output them
        return outputs

class ViViTIntermediate(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
        if isinstance(config.hidden_act, str):
            self.intermediate_act_fn = ACT2FN[config.hidden_act]
        else:
            self.intermediate_act_fn = config.hidden_act

    def forward(self, hidden_states):

        hidden_states = self.dense(hidden_states)
        hidden_states = self.intermediate_act_fn(hidden_states)

        return hidden_states

class ViViTOutput(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor):
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)

        hidden_states = hidden_states + input_tensor

        return hidden_states

class ViViTLayer(nn.Module):
    """This corresponds to the Block class in the timm implementation."""

    def __init__(self, config):
        super().__init__()
        self.chunk_size_feed_forward = config.chunk_size_feed_forward
        self.seq_len_dim = 1
        self.attention = ViViTAttention(config)
        self.intermediate = ViViTIntermediate(config)
        self.output = ViViTOutput(config)
        self.layernorm_before = nn.LayerNorm(
            config.hidden_size, eps=config.layer_norm_eps
        )
        self.layernorm_after = nn.LayerNorm(
            config.hidden_size, eps=config.layer_norm_eps
        )

    def forward(self, hidden_states, head_mask=None, output_attentions=False):
        self_attention_outputs = self.attention(
            self.layernorm_before(
                hidden_states
            ),  # in ViT, layernorm is applied before self-attention
            head_mask,
            output_attentions=output_attentions,
        )
        attention_output = self_attention_outputs[0]
        outputs = self_attention_outputs[
            1:
        ]  # add self attentions if we output attention weights

        # first residual connection
        hidden_states = attention_output + hidden_states

        # in ViT, layernorm is also applied after self-attention
        layer_output = self.layernorm_after(hidden_states)
        layer_output = self.intermediate(layer_output)

        # second residual connection is done here
        layer_output = self.output(layer_output, hidden_states)

        outputs = (layer_output,) + outputs

        return outputs

class ViViTEncoder(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.layer = nn.ModuleList(
            [ViViTLayer(config) for _ in range(config.num_hidden_layers)]
        )
        self.gradient_checkpointing = False

    def forward(
        self,
        hidden_states,
        head_mask=None,
        output_attentions=False,
        output_hidden_states=False,
        return_dict=True,
    ):
        all_hidden_states = () if output_hidden_states else None
        all_self_attentions = () if output_attentions else None

        for i, layer_module in enumerate(self.layer):
            if output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_head_mask = head_mask[i] if head_mask is not None else None

            if self.gradient_checkpointing and self.training:

                def create_custom_forward(module):
                    def custom_forward(*inputs):
                        return module(*inputs, output_attentions)

                    return custom_forward

                layer_outputs = torch.utils.checkpoint.checkpoint(
                    create_custom_forward(layer_module),
                    hidden_states,
                    layer_head_mask,
                )
            else:
                layer_outputs = layer_module(
                    hidden_states, layer_head_mask, output_attentions
                )

            hidden_states = layer_outputs[0]

            if output_attentions:
                all_self_attentions = all_self_attentions + (layer_outputs[1],)

        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        if not return_dict:
            return tuple(
                v
                for v in [hidden_states, all_hidden_states, all_self_attentions]
                if v is not None
            )
        return BaseModelOutput(
            last_hidden_state=hidden_states,
            hidden_states=all_hidden_states,
            attentions=all_self_attentions,
        )

class ViViTPreTrainedModel(PreTrainedModel):
    """
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    """

    config_class = ViViTConfig
    base_model_prefix = "vivit"
    main_input_name = "pixel_values"
    supports_gradient_checkpointing = True

    def _init_weights(self, module):
        """Initialize the weights"""
        if isinstance(module, (nn.Linear, nn.Conv3d)):
            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
            if module.bias is not None:
                module.bias.data.zero_()
        elif isinstance(module, nn.Embedding):
            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
            if module.padding_idx is not None:
                module.weight.data[module.padding_idx].zero_()
        elif isinstance(module, nn.LayerNorm):
            module.bias.data.zero_()
            module.weight.data.fill_(1.0)

    def _set_gradient_checkpointing(self, module, value=False):
        if isinstance(module, ViViTEncoder):
            module.gradient_checkpointing = value

class ViViTModel(ViViTPreTrainedModel):
    def __init__(self, config, add_pooling_layer=True):
        super().__init__(config)
        self.config = config

        self.embeddings = ViViTEmbeddings(config)
        self.encoder = ViViTEncoder(config)

        self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.pooler = ViViTPooler(config) if add_pooling_layer else None

        # Initialize weights and apply final processing
        self.post_init()

    def get_input_embeddings(self):
        return self.embeddings.patch_embeddings

    def _prune_heads(self, heads_to_prune):
        """
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        """
        for layer, heads in heads_to_prune.items():
            self.encoder.layer[layer].attention.prune_heads(heads)

    def forward(
        self,
        pixel_values=None,
        head_mask=None,
        output_attentions=None,
        output_hidden_states=None,
        interpolate_pos_encoding=None,
        return_dict=None,
    ):
        output_attentions = (
            output_attentions
            if output_attentions is not None
            else self.config.output_attentions
        )
        output_hidden_states = (
            output_hidden_states
            if output_hidden_states is not None
            else self.config.output_hidden_states
        )
        return_dict = (
            return_dict if return_dict is not None else self.config.use_return_dict
        )

        if pixel_values is None:
            raise ValueError("You have to specify pixel_values")

        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)

        embedding_output = self.embeddings(
            pixel_values, interpolate_pos_encoding=interpolate_pos_encoding
        )

        encoder_outputs = self.encoder(
            embedding_output,
            head_mask=head_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        sequence_output = encoder_outputs[0]
        sequence_output = self.layernorm(sequence_output)
        pooled_output = (
            self.pooler(sequence_output) if self.pooler is not None else None
        )

        if not return_dict:
            return (sequence_output, pooled_output) + encoder_outputs[1:]

        return BaseModelOutputWithPooling(
            last_hidden_state=sequence_output,
            pooler_output=pooled_output,
            hidden_states=encoder_outputs.hidden_states,
            attentions=encoder_outputs.attentions,
        )

class ViViTForImageClassification(ViViTPreTrainedModel):
    def __init__(self, config):
        super(ViViTForImageClassification, self).__init__(config)

        self.num_labels = config.num_labels
        self.vivit = ViViTModel(config, add_pooling_layer=False)

        # Classifier head
        self.classifier = (
            nn.Linear(config.hidden_size, config.num_labels)
            if config.num_labels > 0
            else nn.Identity()
        )

        # Initialize weights and apply final processing
        self.post_init()

    def forward(
        self,
        pixel_values=None,
        head_mask=None,
        labels=None,
        output_attentions=None,
        output_hidden_states=None,
        interpolate_pos_encoding=None,
        return_dict=None,
    ):
        return_dict = (
            return_dict if return_dict is not None else self.config.use_return_dict
        )

        outputs = self.vivit(
            pixel_values,
            head_mask=head_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            interpolate_pos_encoding=interpolate_pos_encoding,
            return_dict=return_dict,
        )

        sequence_output = outputs[0]

        logits = self.classifier(sequence_output[:, 0, :])

        loss = None
        if labels is not None:
            if self.num_labels == 1:
                #  We are doing regression
                loss_fct = MSELoss()
                loss = loss_fct(logits.view(-1), labels.view(-1))
            else:
                loss_fct = CrossEntropyLoss()
                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))

        if not return_dict:
            output = (logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

        return SequenceClassifierOutput(
            loss=loss,
            logits=logits,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
        )

class ViViTPooler(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
        self.activation = nn.Tanh()

    def forward(self, hidden_states):
        # We "pool" the model by simply taking the hidden state corresponding
        # to the first token.
        first_token_tensor = hidden_states[:, 0]
        pooled_output = self.dense(first_token_tensor)
        pooled_output = self.activation(pooled_output)
        return pooled_output


# -*- coding: utf-8 -*-

""" Práctica 3_2.ipynb """

from sklearn.model_selection import train_test_split

postags = {
    "v": "verbo",
    "det": "determinante",
    "dem": "demostrativo",
    "n": "sustantivo",
    "p.loc": "particula locativa",
    "conj.adv": "conjuncion adversativa",
    "gen": "genitivo",
    "it": "iterativo",
    "aff": "afrimativo",
    "dec": "decimal",
    "cord": "coordinacion",
    "regular/v": "verbo regular",
    "obl": "oblicuo",
    "cnj": "conjuncion",
    "unkwn": "desconocido",
    "neg": "negativo",
    "prt": "partcula",
    "dim": "diminutivo",
    "cond": "condicional",
    "lim": "limitativo",
    "loc": "locativo",
    "conj": "conjuncion",
    "cnj.adv": "conjuncion adversativa",
}

tags = postags.keys()

def get_tagged_sentences(corpus):
    # Obtains the sentences in the corpora. Every word of a sentence has their own POStag.
    tagged_sentences = []

    for line in corpus:
        tagged_words = []
        for schema in line:
            pretagged_word = [chunk[0] for chunk in schema[:-1]]
            word = "".join(pretagged_word)
            tagged_word = [word]
            tagged_word.append(schema[-1])
            tagged_words.append(tagged_word)
        tagged_sentences.append(tagged_words)
    return tagged_sentences

def map_tag(tag):
    try:
        return postags[tag]
    except:
        return "unkwn"

def word2features(sent, i):
    word = sent[i][0]
    postag = sent[i][1]

    features = {
        "word.lower()": word.lower(),
        "word[-3:]": word[-3:],
        "word[-2:]": word[-2:],
        "word.istitle()": word.istitle(),
        "postag": postag,
        "postag[:2]": postag[:2],
    }
    if i > 0:
        word1 = sent[i - 1][0]
        postag1 = sent[i - 1][1]
        features.update(
            {
                "-1:word.lower()": word1.lower(),
                "-1:word.istitle()": word1.istitle(),
                "-1:postag": postag1,
                "-1:postag[:2]": postag1[:2],
            }
        )
    else:
        features["BOS"] = True

    if i < len(sent) - 1:
        word1 = sent[i + 1][0]
        postag1 = sent[i + 1][1]
        features.update(
            {
                "+1:word.lower()": word1.lower(),
                "+1:word.istitle()": word1.istitle(),
                "+1:postag": postag1,
                "+1:postag[:2]": postag1[:2],
            }
        )
    else:
        features["EOS"] = True

    return features

def sent2features(sent):
    return [word2features(sent, i) for i in range(len(sent))]

def sent2labels(sent):
    return [label for token, label in sent]

# Comenzamos a tratar el corpus.

import ast
import re

corpus = open("corpus_otomi", "r")
corpus = corpus.read()
corpus = re.sub(r"\n", ", ", corpus)
corpus = eval(corpus)
corpus = get_tagged_sentences(corpus)

# Prepare data for CRF
X = [[word2features(sent, i) for i in range(len(sent))] for sent in corpus]
y = [[map_tag(pos) for _, pos in sent] for sent in corpus]

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

from inspect import Attribute
from sklearn_crfsuite import CRF

# Initialize and train the CRF tagger: https://sklearn-crfsuite.readthedocs.io/en/latest/api.html
crf = CRF(
    algorithm="lbfgs",
    c1=0.1,
    c2=0.1,
    max_iterations=100,
    all_possible_transitions=True,
    verbose=True,
)
try:
    crf.fit(X_train, y_train)
except AttributeError as e:
    print(e)

from sklearn.metrics import classification_report

y_pred = crf.predict(X_test)

# Flatten the true and predicted labels
y_test_flat = [label for sent_labels in y_test for label in sent_labels]
y_pred_flat = [label for sent_labels in y_pred for label in sent_labels]

# Evaluate the model
report = classification_report(y_true=y_test_flat, y_pred=y_pred_flat)
print(report)

print("\nReporte de accurary, precision, recall y F1-score\n")

from sklearn.metrics import accuracy_score

print("\tAccuracy", accuracy_score(y_pred_flat, y_test_flat))

from sklearn.metrics import precision_score

print("\tPrecision:", precision_score(y_pred_flat, y_test_flat, average="macro"))

from sklearn.metrics import recall_score

print("\tRecall:", recall_score(y_pred_flat, y_test_flat, average="macro"))

from sklearn.metrics import f1_score

print("\tF1-score:", f1_score(y_pred_flat, y_test_flat, average="macro"))

from sklearn.metrics import ConfusionMatrixDisplay

disp = ConfusionMatrixDisplay.from_predictions(y_test_flat, y_pred_flat)
disp.figure_.suptitle("Confusion Matrix")
print(f"Confusion matrix:\n{disp.confusion_matrix}")

print("\n\nEl siguiente es un ejemplo de oración etiquedada\n")
print("Tomamos un elemento del corpus, correspondiente a los datos de evaluación.\n")
# Reconstruimos la oración
sentence = []
for i in range(len(X_test[1])):
    sentence.append(X_test[1][i]["word.lower()"])

# Obtenemos las etiqutes originales
Xtags = []
for i in range(len(X_test[1])):
    Xtags.append(postags[X_test[1][i]["postag"]])

print("En este caso, consideramos la oración:\n ", " ".join(sentence), "\n")
print("Por otro lado, las etiquetas POS de nuestra oración son:\n", Xtags, "\n")
print(
    "Finalmente, las etiquetas POS calculadas por el modelo predictivo, son:\n",
    y_pred[1],
)


from django import template
from distutils.version import LooseVersion, StrictVersion
from software_versions.models import Status, OurVersion, Software
from django.conf import settings

register = template.Library()

def getStatues():
    'Get all statues and return a dict'
    status = Status.objects.all()
    return dict([ (i.label, i.html_color_code) for i in status ])

@register.filter()
def get_ourversion(versions, release):
    if versions and release:
        return versions.get(release=release).__dict__.items()

@register.filter()
def get_status_color(versions, release):
    ourversion = get_ourversion(versions, release)
    if ourversion:
        status_id = dict(ourversion)['status_id']
        return Status.objects.get(id=status_id).html_color_code

@register.filter()
def get_status_text(versions, release):
    ourversion = get_ourversion(versions, release)
    if ourversion:
        status_id = dict(ourversion)['status_id']
        return Status.objects.get(id=status_id).label


import pandas as pd
import numpy as np
import time
import tensorflow as tf
import keras
import matplotlib.pyplot as plt
import matplotlib
import pyswarms as ps

from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.layers import LSTM
from keras import optimizers
from pyswarms.utils.plotters import plot_cost_history

matplotlib.style.use('ggplot')

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.Session(config=config)

#Part 1:
# Load and divide dataset
url_dataset = "https://raw.githubusercontent.com/lzs-raja/GA_01/master/dataset/Time-sequence%20datasets/SP500.csv"
dataset = pd.read_csv(url_dataset,encoding = 'utf-8')
x_train = dataset.drop(columns = 'Date')
value_mean = x_train.mean(axis=0)[1]
value_sd = np.sqrt(x_train.var(axis=0)[1])
for i in range(4):
    x_train.iloc[:,i] -= value_mean
    x_train.iloc[:,i] /= value_sd
x_train.iloc[:,4] -= x_train.mean(axis=0)[4]
x_train.iloc[:,4] /= np.sqrt(x_train.var(axis=0)[4])
x_train.head(5)
sample_size = int(x_train.shape[0]/5)
timesteps = 5
features = x_train.shape[1]
used_samples = sample_size*5
#Use the whole week data to predict highest value of next week
x_train = np.asarray(x_train)[:used_samples,:].reshape((sample_size,timesteps,features))
y_train = x_train.max(1)[1:,1].reshape((sample_size-1,1))
x_train = x_train[:-1,:,:]
#print(x_train.shape,y_train.shape)

# Write Keras model LSTM layers and compile the model using SGD
model = Sequential()
model.add(LSTM(128,input_shape=(timesteps,features)))
model.add(Dense(1))

sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error', optimizer=sgd)

model.summary()
total_param = 68737
# Use model fit command to train the model
t0 = time.time()
result_sgd = model.fit(x_train, y_train, epochs=80, batch_size=7,verbose=0)
t1 = time.time()

print("***************************")
print()
print("LSTM Model")
print ("Time taken to train the model: ", t1 - t0, "secs.")
print("Error:",result_sgd.history['loss'][-1])

# Save or plot error with epochs
plt.plot(result_sgd.history['loss'])
plt.show()

#Part 2:

#I use another PSO package called pyswarms here

model_weights =np.array([model.layers[0].get_weights(),model.layers[1].get_weights()])
shape = np.array([model_weights[0][0].shape,model_weights[0][1].shape,model_weights[0][2].shape,model_weights[1][0].shape,model_weights[1][1].shape])
def func(vector_x):
    init_index = 0
    end_index = 0
    end_index += shape[0][0] * shape[0][1]
    model_weights[0][0] = vector_x[init_index:end_index].reshape(shape[0])
    init_index = end_index
    end_index += shape[1][0]*shape[1][1]
    model_weights[0][1] = vector_x[init_index:end_index].reshape(shape[1])
    init_index = end_index
    end_index += shape[2][0]
    model_weights[0][2] = vector_x[init_index:end_index]
    init_index = end_index
    end_index += shape[3][0] * shape[3][1]
    model_weights[1][0] = vector_x[init_index:end_index].reshape(shape[3])
    init_index = end_index
    end_index += shape[4][0]
    model_weights[1][1] = vector_x[init_index:end_index]
    model.layers[0].set_weights(model_weights[0])
    model.layers[1].set_weights(model_weights[1])
    pso_predict = model.predict(x_train)
    error = np.sum(np.square(pso_predict-y_train))/(sample_size-1)
    return error

def swarm_func(x):
    n_particles = x.shape[0]
    j = [func(x[i]) for i in range(n_particles)]
    return np.array(j)

# initialization
options = {'c1': 0.5, 'c2': 0.3, 'w':0.9}
optimizer = ps.single.GlobalBestPSO(n_particles=80, dimensions=total_param,
                                    options=options)

t2 = time.time()
result_pso = optimizer.optimize(swarm_func,iters = 120)
t3 = time.time()

print("Partical Swarm Optimization")
print ("Time taken to train the model: ", t3 - t2, "secs.")
print("Error:",result_pso[0])
print()
print("***************************")

print()
plot_cost_history(optimizer.cost_history)
plt.show()
# Save or plot error with generations/iterations

# Compare performance matrices with Part 1

##################################################################


from test import run_test
import os

def add_to_dataset(dataset, info):
    item = {}
    item['path'] = info[0]
    item['name'] = info[1]
    item['width'] = info[2]
    item['height'] = info[3]
    item['scale'] = info[4]
    item['radius'] = info[5]
    item['cellsize'] = info[6]
    item['saveas'] = info[7]
    dataset.append(item)
    return dataset

def test_formed_set(name, number, size, scale, radius, cellsize, mode, subfolder=''):
    if subfolder:
        try:
            os.stat('results/'+name+subfolder)
        except:
            os.mkdir('results/'+name+subfolder)
    for i in range(1, number+1):
        run_test('tests/'+name+'/test'+str(i)+'.bmp', size[0], size[1], scale, radius, cellsize,
                 save_as='results/'+name+subfolder+'/test'+str(i)+mode+'.png', status=True, mode=mode)

def test_raw_set(dataset, mode):
    for item in dataset:
        run_test(item['path']+item['name'], item['width'], item['height'], item['scale'], item['radius'],
                 item['cellsize'], save_as=item['path']+item['saveas'], status=True, mode=mode)

def form_people_set():
    people_dataset = []
    people_dataset = add_to_dataset(people_dataset, ['tests/people2/', 'gosha.bmp', 90, 120, 3, 10, 5, 'results/people2/gosha-rgb.png'])
    people_dataset = add_to_dataset(people_dataset, ['tests/people2/', 'batya.bmp', 100, 100, 3, 10, 5, 'results/people2/batya-rgb.png'])
    people_dataset = add_to_dataset(people_dataset, ['tests/people2/', 'steffy.bmp', 100, 100, 3, 10, 5, 'results/people2/steffy-rgb.png'])
    people_dataset = add_to_dataset(people_dataset, ['tests/people2/', 'valera.bmp', 100, 130, 3, 10, 5, 'results/people2/valera-rgb.png'])
    people_dataset = add_to_dataset(people_dataset, ['tests/people2/', 'larik.bmp', 100, 100, 3, 10, 5, 'results/people2/larik-rgb.png'])
    people_dataset = add_to_dataset(people_dataset, ['tests/people2/', 'me.bmp', 180, 200, 3, 10, 5, 'results/people2/me-rgb.png'])
    people_dataset = add_to_dataset(people_dataset, ['tests/people2/', 'artur.bmp', 240, 320, 3, 10, 5, 'results/people2/artur-rgb.png'])
    people_dataset = add_to_dataset(people_dataset, ['tests/people2/', 'igor.bmp', 200, 360, 3, 10, 5, 'results/people2/igor-rgb.png'])
    return people_dataset

def test_people_64x48(number, scale, radius, cellsize, mode, subfolder=''):
    test_formed_set('people', number, [64, 48], scale, radius, cellsize, mode, subfolder)

def test_dogs_120x90(number, scale, radius, cellsize, mode, subfolder=''):
    test_formed_set('dogs', number, [120, 90], scale, radius, cellsize, mode, subfolder)

def test_people_320x240(number, scale, radius, cellsize, mode, subfolder=''):
    test_formed_set('people_big', number, [320, 240], scale, radius, cellsize, mode, subfolder)

def try_constants(test_function, tests_number, scale, r_values, c_values, mods):
    for radius in r_values:
        for cell_size in c_values:
            for mode in mods:
                test_function(tests_number, scale, radius, cell_size, mode, subfolder='/'+mode+'-'+str(radius)+'-'+str(cell_size)+'/')


import re
import sys
import codecs
import logging

import capa.engine

logger = logging.getLogger(__name__)
MAX_BYTES_FEATURE_SIZE = 0x100

# thunks may be chained so we specify a delta to control the depth to which these chains are explored
THUNK_CHAIN_DEPTH_DELTA = 5

# identifiers for supported architectures names that tweak a feature
# for example, offset/x32
ARCH_X32 = "x32"
ARCH_X64 = "x64"
VALID_ARCH = (ARCH_X32, ARCH_X64)

def bytes_to_str(b):
    return str(codecs.encode(b, "hex").decode("utf-8"))

def hex_string(h):
    """ render hex string e.g. "0a40b1" as "0A 40 B1" """
    return " ".join(h[i : i + 2] for i in range(0, len(h), 2)).upper()

def escape_string(s):
    """escape special characters"""
    s = repr(s)
    if not s.startswith(('"', "'")):
        # u'hello\r\nworld' -> hello\\r\\nworld
        s = s[2:-1]
    else:
        # 'hello\r\nworld' -> hello\\r\\nworld
        s = s[1:-1]
    s = s.replace("\\'", "'")  # repr() may escape "'" in some edge cases, remove
    s = s.replace('"', '\\"')  # repr() does not escape '"', add
    return s

class Feature(object):
    def __init__(self, value, arch=None, description=None):
        """
        Args:
          value (any): the value of the feature, such as the number or string.
          arch (str): one of the VALID_ARCH values, or None.
            When None, then the feature applies to any architecture.
            Modifies the feature name from `feature` to `feature/arch`, like `offset/x32`.
          description (str): a human-readable description that explains the feature value.
        """
        super(Feature, self).__init__()

        if arch is not None:
            if arch not in VALID_ARCH:
                raise ValueError("arch '%s' must be one of %s" % (arch, VALID_ARCH))
            self.name = self.__class__.__name__.lower() + "/" + arch
        else:
            self.name = self.__class__.__name__.lower()

        self.value = value
        self.arch = arch
        self.description = description

    def __hash__(self):
        return hash((self.name, self.value, self.arch))

    def __eq__(self, other):
        return self.name == other.name and self.value == other.value and self.arch == other.arch

    def get_value_str(self):
        """
        render the value of this feature, for use by `__str__` and friends.
        subclasses should override to customize the rendering.

        Returns: any
        """
        return self.value

    def __str__(self):
        if self.value is not None:
            if self.description:
                return "%s(%s = %s)" % (self.name, self.get_value_str(), self.description)
            else:
                return "%s(%s)" % (self.name, self.get_value_str())
        else:
            return "%s" % self.name

    def __repr__(self):
        return str(self)

    def evaluate(self, ctx):
        return capa.engine.Result(self in ctx, self, [], locations=ctx.get(self, []))

    def freeze_serialize(self):
        if self.arch is not None:
            return (self.__class__.__name__, [self.value, {"arch": self.arch}])
        else:
            return (self.__class__.__name__, [self.value])

    @classmethod
    def freeze_deserialize(cls, args):
        # as you can see below in code,
        # if the last argument is a dictionary,
        # consider it to be kwargs passed to the feature constructor.
        if len(args) == 1:
            return cls(*args)
        elif isinstance(args[-1], dict):
            kwargs = args[-1]
            args = args[:-1]
            return cls(*args, **kwargs)

class MatchedRule(Feature):
    def __init__(self, value, description=None):
        super(MatchedRule, self).__init__(value, description=description)
        self.name = "match"

class Characteristic(Feature):
    def __init__(self, value, description=None):
        super(Characteristic, self).__init__(value, description=description)

class String(Feature):
    def __init__(self, value, description=None):
        super(String, self).__init__(value, description=description)

class Regex(String):
    def __init__(self, value, description=None):
        super(Regex, self).__init__(value, description=description)
        pat = self.value[len("/") : -len("/")]
        flags = re.DOTALL
        if value.endswith("/i"):
            pat = self.value[len("/") : -len("/i")]
            flags |= re.IGNORECASE
        try:
            self.re = re.compile(pat, flags)
        except re.error:
            if value.endswith("/i"):
                value = value[: -len("i")]
            raise ValueError(
                "invalid regular expression: %s it should use Python syntax, try it at https://pythex.org" % value
            )

    def evaluate(self, ctx):
        for feature, locations in ctx.items():
            if not isinstance(feature, (capa.features.String,)):
                continue

            # `re.search` finds a match anywhere in the given string
            # which implies leading and/or trailing whitespace.
            # using this mode cleans is more convenient for rule authors,
            # so that they don't have to prefix/suffix their terms like: /.*foo.*/.
            if self.re.search(feature.value):
                # unlike other features, we cannot return put a reference to `self` directly in a `Result`.
                # this is because `self` may match on many strings, so we can't stuff the matched value into it.
                # instead, return a new instance that has a reference to both the regex and the matched value.
                # see #262.
                return capa.engine.Result(True, _MatchedRegex(self, feature.value), [], locations=locations)

        return capa.engine.Result(False, _MatchedRegex(self, None), [])

    def __str__(self):
        return "regex(string =~ %s)" % self.value

class _MatchedRegex(Regex):
    """
    this represents a specific instance of a regular expression feature match.
    treat it the same as a `Regex` except it has the `match` field that contains the complete string that matched.

    note: this type should only ever be constructed by `Regex.evaluate()`. it is not part of the public API.
    """

    def __init__(self, regex, match):
        """
        args:
          regex (Regex): the regex feature that matches
          match (string|None): the matching string or None if it doesn't match
        """
        super(_MatchedRegex, self).__init__(regex.value, description=regex.description)
        # we want this to collide with the name of `Regex` above,
        # so that it works nicely with the renderers.
        self.name = "regex"
        # this may be None if the regex doesn't match
        self.match = match

    def __str__(self):
        return 'regex(string =~ %s, matched = "%s")' % (self.value, self.match)

class StringFactory(object):
    def __new__(self, value, description=None):
        if value.startswith("/") and (value.endswith("/") or value.endswith("/i")):
            return Regex(value, description=description)
        return String(value, description=description)

class Bytes(Feature):
    def __init__(self, value, description=None):
        super(Bytes, self).__init__(value, description=description)

    def evaluate(self, ctx):
        for feature, locations in ctx.items():
            if not isinstance(feature, (capa.features.Bytes,)):
                continue

            if feature.value.startswith(self.value):
                return capa.engine.Result(True, self, [], locations=locations)

        return capa.engine.Result(False, self, [])

    def get_value_str(self):
        return hex_string(bytes_to_str(self.value))

    def freeze_serialize(self):
        return (self.__class__.__name__, [bytes_to_str(self.value).upper()])

    @classmethod
    def freeze_deserialize(cls, args):
        return cls(*[codecs.decode(x, "hex") for x in args])


marcaCaneta = ['Faber-Castell', 'BIC', 'Uni-Ball', 'Crown']
corCaneta = ['Azul', 'Preto', 'Vermelho', 'Verde']
canetas = []
anot = []
cAnot = []

class Caneta:

    def __init__(self, marca, cor, tipo_ponta, tampa):
        self.marca = str(marca)
        self.cor = str(cor)
        self.tipo_ponta = float(tipo_ponta)
        self.tampa = bool(tampa)

def escolha_marca():
    while True:
        print("\n| SELECIONE A MARCA |")
        for i in range(len(marcaCaneta)):
            print(f"| {i + 1} - " + marcaCaneta[i])
        try:
            esc_mrc = int(input("| Escolha: "))
            return marcaCaneta[esc_mrc - 1]
        except:
            print("| Selecione uma opção válida!")
            continue

def escolha_cor():
    while True:
        print("\n| SELECIONE A COR |")
        for i in range(len(corCaneta)):
            print(f"| {i + 1} - " + corCaneta[i])
        try:
            esc_cor = int(input("| Escolha: "))
            return corCaneta[esc_cor - 1]
        except:
            print("| Selecione uma opção válida!")
            continue

def escolha_ponta():
    while True:
        esc_pnt = input("""\n|      PONTAS      |
| Escolha o valor da ponta entre 0.3 e 0.7
| Digite: """)
        try:
            if 0.3 <= float(esc_pnt) <= 0.7:
                return esc_pnt
        except:
            print("| Digite um valor válido!")
            continue

def escolha_tampa():
    while True:
      print("\n| Tampar ou destampar - OPÇÕES ", end="")
      listar_canetas()
      esc_cnt = input("| Escolha: ")
      if int(esc_cnt):
        esc_cnt = int(esc_cnt)
        if canetas[esc_cnt-1]:
            if canetas[esc_cnt-1].tampa:
                print("\n| A caneta está tampada!")
                esc_tmp = str(input("| Deseja destampar a caneta? [S/N]: ")).lower().strip()
                if esc_tmp[:1] == 's':
                    canetas[esc_cnt-1].tampa = False
                    print("\n| Caneta destampada!")
                elif esc_tmp[:1] == 'n':
                    print("\n| Voltando..")
                else:
                    print("\n| Selecione uma opção válida!")
                break

            else:
                print("\n| A caneta está destampada!")
                esc_tmp = str(input("| Deseja tampar a caneta? [S/N]: ")).lower().strip()
                if esc_tmp[:1] == 's':
                    canetas[esc_cnt-1].tampa = True
                    print("| Caneta tampada!")
                elif esc_tmp[:1] == 'n':
                    print("\n| Voltando..")
                else:
                    print("\n| Selecione uma opção válida!")
                break
      else:
        print("\n| Selecione uma opção válida!")
        continue

def excluir():
    while True:
        print("\n| Excluir caneta - OPÇÕES", end="")
        listar_canetas()
        esc_exc = input("| Escolha: ")

        try:
            esc_exc = int(esc_exc)

            if canetas[esc_exc-1]:
                cnt_esc = str(input(f"| Deseja excluir a caneta marca: {canetas[esc_exc-1].marca} | cor:"
                                    f" {canetas[esc_exc-1].cor}? [S/N]: ")).lower().strip()
                if cnt_esc[:1] == 's':
                    canetas.pop(esc_exc-1)
                    print("\n| Caneta excluida!")
                elif cnt_esc[:1] == 'n':
                    print("\n| Retornando..")
                else:
                    print("\n| Selecione uma opção válida!")
                break
            else:
                print("\n| Selecione uma opção válida!")
                continue

        except:
            print("\n| Selecione uma opção válida!")
            continue

def tampa(can):
    if can:
        return "Tampada"
    elif not can:
        return "Destampada"

def criar_caneta():
    caneta = Caneta(escolha_marca(), escolha_cor(), escolha_ponta(), True)
    canetas.append(caneta)
    del caneta

def listar_canetas(contador=1):
    print("\n|------CANETAS-----|")
    if len(canetas) > 0:
        for caneta in canetas:
            print(f"| CANETA {contador} = Marca: {caneta.marca} | Cor: {caneta.cor} | Ponta: {caneta.tipo_ponta} |"
                  , tampa(caneta.tampa))
            contador += 1
    else:
        print("\n| NENHUMA CANETA REGISTRADA |")

def consulta_anotacoes():
    print(anot[0])
    print(cAnot[0].marca, cAnot[0].cor)

def anotar():
    while True:
        print("\n| ANOTAR - escolha caneta - OPÇÕES", end="")
        listar_canetas()
        esc_ant = input("| Escolha: ")
        try:
            esc_ant = int(esc_ant)
            cAnot.append(canetas[esc_ant-1])
            anota = str(input("| Escreva: "))
            anot.append(anota)
            break
        except:
            print("\n| Selecione uma opção válida!")
            continue

def anotacoes():

    while True:
        anotMenu = input("""\n|----ANOTAÇÕES----|
| 1 - Consultar anotações
| 2 - Anotar
| 3 - Sair
| Escolha:
""")
        try:
            anotMenu = int(anotMenu)
            if anotMenu == 1:
                consulta_anotacoes()
                continue
            elif anotMenu == 2:
                anotar()
                continue
            elif anotMenu == 3:
                pass
            elif anotMenu == 4:
                pass
        except:
            print("\n| Selecione uma opção válida!")
            continue

def acoes_caneta():
    while True:
        acoesMenu = input("""\n|----AÇÕES NA CANETA ----|
| 1 - Tampar / Destampar
| 2 - Anotações
| 3 - Excluir caneta
| 4 - Sair
| Escolha: """)
        try:
            acoesMenu = int(acoesMenu)
            if acoesMenu == 1:
                escolha_tampa()
                continue

            elif acoesMenu == 2:
                anotacoes()
                continue

            elif acoesMenu == 3:
                excluir()
                continue

            elif acoesMenu == 4:
                print("\n| Retornando ao menu!")
                break

        except:
            print("\n| Selecione uma opção válida!")
            continue

while True:

    menu = input("""
|-------MENU--------|
| 1 - Criar caneta
| 2 - Listar canetas
| 3 - Ações na caneta
| 4 - Sair
| Escolha: """)

    try:
        menu = int(menu)
        if menu == 1:
            criar_caneta()
            continue

        elif menu == 2:
            listar_canetas()
            continue

        elif menu == 3:
            acoes_caneta()
            continue

        elif menu == 4:
            print("\n| Finalizando..")
            break

        else:
            print("\n| Selecione uma opção válida!")
            continue

    except:
        print("\n| Selecione uma opção válida!")
        continue


import numpy as np
label_filename = "test_labels.npy"
output_filename = "output.txt"

TN, TP, FN, FP = 0.0, 0.0, 0.0, 0.0

np_label = np.load(open(label_filename, 'rb'))
label = [x[1] for x in np_label]

one = [i for i in label if i == 1]
zero = [i for i in label if i == 0]

output = np.loadtxt(output_filename, dtype = 'int')
for i, tmp in enumerate(label):
    print(output[i], tmp)
    if output[i] == 0 and tmp == 0:
        TN += 1
    if output[i] == 1 and tmp == 1:
        TP += 1
    if output[i] == 0 and tmp == 1:
        FN += 1
    if output[i] == 1 and tmp == 0:
        FP +=1

print("The accuracy is", (TP + TN) / (TN + TP + FN + FP))
print("The precision is", (TP) / (TP + FP))
print("The recall is", (TP) / (TP + FN))
print("The True Positive is", TP)
print("The False Negative is", FN)
print("The Natual Percentage is", float(len(one))/float(len(zero) + len(one)))


N, M = map(int, input().split())
problems = []
for _ in range(M):
  p, s = input().split()
  problems.append((int(p), s))

solved = [False] * (N + 1)
wa_cnt = [0] * (N + 1)
for problem in problems:
  problem_id = problem[0]
  result = problem[1]
  if result == 'AC':
    solved[problem_id] = True
  else:
    if not solved[problem_id]:
      wa_cnt[problem_id] += 1

AC_cnt = 0
penalty_cnt = 0
for i in range(1, N + 1):
  if solved[i]:
    AC_cnt += 1
    penalty_cnt += wa_cnt[i]

print(AC_cnt, penalty_cnt)


import random
import math
import numpy as np
import matplotlib.pyplot as plt

n_points = 100

def gen_rand_point(x_range: tuple, y_range: tuple) -> tuple:
    return (random.uniform(*x_range), random.uniform(*y_range))

def classifying_function_0(x: float, y: float) -> bool:
    # return x ** 2 + y ** 2 >= 0.5
    return x > y

def classifying_function_1(x: float, y: float) -> bool:
    # return x ** 2 + y ** 2 >= 0.5
    return x+y > 0

def classifying_function_2(x: float, y: float) -> bool:
    # return x ** 2 + y ** 2 >= 0.5
    return y > 1 / 2 * math.sin(4 * x)

fig, axes = plt.subplots(nrows=3, ncols=2, figsize=(18, 21))
for row, classifying_function in enumerate([classifying_function_0, classifying_function_1, classifying_function_2]):
    points = [gen_rand_point((-0.9, 0.9), (-0.9, 0.9)) for _ in range(n_points)]
    X = np.arange(-1, 1, 0.01)
    Y = np.arange(-1, 1, 0.01)
    Z = np.empty((X.shape[0], Y.shape[0]))

    for n, x in enumerate(X):
        for j, y in enumerate(Y):
            Z[j, n] = float(classifying_function(x, y))

    pos_p = [p for p in points if classifying_function(*p)]
    neg_p = [p for p in points if not classifying_function(*p)]
    x1, y1 = zip(*pos_p)
    x2, y2 = zip(*neg_p)

    axes[row][0].contour(X, Y, Z)
    axes[row][0].scatter(x1, y1, marker=".", color="green")
    axes[row][0].scatter(x2, y2, marker=".", color="red")
    axes[row][0].set_title("optimal classification")

    # start neat
    import neat

    inputs = pos_p + neg_p
    outputs = [(1.0,)] * len(pos_p) + [(0.0,)] * len(neg_p)

    def eval_genomes(genomes, config):
        for genome_id, genome in genomes:
            genome.fitness = 0.0
            net = neat.nn.FeedForwardNetwork.create(genome, config)
            for xi, xo in zip(inputs, outputs):
                output = net.activate(xi)
                genome.fitness -= ((output[0] - xo[0]) ** 2) / n_points

    # Load configuration.
    config = neat.Config(
        neat.DefaultGenome,
        neat.DefaultReproduction,
        neat.DefaultSpeciesSet,
        neat.DefaultStagnation,
        "neat_config.ini",
    )
    # Create the population, which is the top-level object for a NEAT run.
    p = neat.Population(config)
    # Add a stdout reporter to show progress in the terminal.
    p.add_reporter(neat.StdOutReporter(False))
    # Run until a solution is found.
    winner = p.run(eval_genomes)
    # Display the winning genome.
    print("\nBest genome:\n{!s}".format(winner))
    print("\nOutput:")
    winner_net = neat.nn.FeedForwardNetwork.create(winner, config)
    for xi, xo in zip(inputs, outputs):
        output = winner_net.activate(xi)
        print("  input {!r}, expected output {!r}, got {!r}".format(xi, xo, output))

    for n, x in enumerate(X):
        for j, y in enumerate(Y):
            Z[j, n] = float(winner_net.activate((x, y))[0])

    pos_p = [p for p in points if classifying_function(*p)]
    neg_p = [p for p in points if not classifying_function(*p)]
    x1, y1 = zip(*pos_p)
    x2, y2 = zip(*neg_p)

    axes[row][1].contour(X, Y, Z)
    axes[row][1].scatter(x1, y1, marker=".", color="green")
    axes[row][1].scatter(x2, y2, marker=".", color="red")
    axes[row][1].set_title("NEAT classification")

plt.savefig('result.png')
plt.show()


# -*- coding: utf-8 -*-
"""
.. module:: othello
   :platform: Unix, Windows
   :synopsis: Main class for the Othello game

.. moduleauthor:: François-Xavier Dupé

"""
#import plateau.joueur

class Othello(object):
    """Classe pour le jeu"""

    def __init__(self, jblanc, jnoir):
        """Constructeur"""
        self.plateau = []
        self.ecoute_coup = []
        #self.jnoir = plateau.joueur.Joueur('Noir', 'x')
        self.jnoir = jnoir
       # self.jblanc = plateau.iafaible.IAFaible('Blanc', 'o')
        self.jblanc = jblanc
        self.faire_plateau()

    def faire_plateau(self):
        """Construction du plateau de jeu"""
        # Construction des cases
        self.plateau=[]
        for i in range(0,8):
            ligne=[" "]*8
            self.plateau.append(ligne)

        self.plateau[3][3]=self.jblanc.pion
        self.plateau[3][4]=self.jnoir.pion
        self.plateau[4][3]=self.jnoir.pion
        self.plateau[4][4]=self.jblanc.pion

    def jeu(self):
        """Routine de lancement d'une partie"""

        print('Lancement du Jeu')
        while True:
            # Alternance des joueurs
            self.info()
            print(self.calcul_score())
            depl = 0
            res = self.tour_joueur(self.jnoir)
            depl += res
            self.info()
            res = self.tour_joueur(self.jblanc)
            depl += res
            if depl is 0:
                break
        score = self.calcul_score()
        if score['x'] > score['o']:
            print('Joueur ', self.jnoir.nom, ' a gagner')
        elif score['x'] < score['o']:
            print('Joueur', self.jblanc.nom, 'a gagner')
        else:
            print('Egalite donc pas de vainqueur')

    def tour_joueur(self, joueur):
        """Tour d'un joueur

            :param joueur: le joueur dont c'est le tour

            :returns: 0 si aucun coup n'est joué, 1 sinon
        """
        proch_coups = self.prochain_coups(self.plateau, joueur)
        if proch_coups:
            chx_coup = joueur.choix_coup(proch_coups, self.plateau)
            Othello.jouer_coup(self.plateau, joueur, chx_coup)
            return 1
        return 0

    @staticmethod
    def jouer_coup(jplateau, joueur, coup):
        """Le joueur joue un coup

        :param jplateau: le plateau de jeu
        :param joueur: le joueur dont c'est le tour
        :param coup: le coup à jouer
        """
        jplateau[coup[0]][coup[1]] = joueur.pion
        Othello.propager_coup(jplateau, joueur, coup)

    def info(self):
        """Donne des informations sur le jeu"""
        print('Jeu Othello')
        print('x / y 0    1    2    3    4    5    6    7')
        for i in range(8):
            print(i, ' ', self.plateau[i])

    @staticmethod
    def limite(pos):
        """Test si une position est dans le plateau

        :param pos: la position
        :returns: True si OK, False sinon
        """
        x=pos[0]
        y=pos[1]
        if x>=0 and x < 8  and y>=0 and y < 8:
            return True
        else:
            return False

    @staticmethod
    def prochain_coups(jplateau, joueur):
        """Calcul des prochains coups possibles

        :param jplateau: le plateau de jeu
        :param joueur: le joueur courant

        :returns: la liste de coups que le joueur peut jouer
        """

        coups=[]
        for i in  range(8):
           for j in range (8):
              if jplateau[i][j] == ' ' and  Othello.recherche_coup(jplateau, joueur, (i,j)) :
                coups.append((i,j))
        return coups

    @staticmethod
    def recherche_coup_direct(jplateau, joueur, pos, direc):
        """Recherche si un coup est possible le long d'une direction

        :param jplateau: le plateau de jeu
        :param joueur: le joueur courant (le pion)
        :param pos: la position de départ
        :param direc: la direction à regarder (ici une fonction de déplacement)

        :returns: True si le coup est possible, False sinon
        """
        x=pos[0]+direc[0]
        y=pos[1]+direc[1]
        vuadversaire=False

        while Othello.limite([x,y]) and jplateau[x][y] != joueur.pion and jplateau[x][y] != ' '  :
            vuadversaire=True
            (x,y)= (x+direc[0], y+direc[1])
        if Othello.limite([x,y]) and jplateau[x][y] == joueur.pion:
            return vuadversaire

        return False

    @staticmethod
    def recherche_coup(jplateau, joueur, pos):
        """Recherche du prochain a partir de la position et du joueur

        :param jplateau: le plateau de jeu
        :param joueur: le joueur
        :param pos: la position du coup

        :returns: True si le coup est possible, False sinon
        """
        direc=[(-1,1),(-1,-1),(0,1),(0,-1),(1,-1),(1,0),(1,1),(0,1)]
        for d in direc :
               if Othello.recherche_coup_direct(jplateau, joueur, pos,d):
                   return True
        return False

    @staticmethod
    def propager_coup(jplateau, joueur, coup):
        """Propagation d'un coup

        :param jplateau: le plateau de jeu
        :param joueur: le joueur
        :param coup: le coup joué par le joueur
        """
        for i in range(-1,2):
            for j in range(-1,2):  #tester toutes les directions
                if [i,j]!=[0,0] and Othello.recherche_coup_direct(jplateau, joueur, coup, [i,j]): # si les condition sont verifier je change la position
                    position = (coup[0] +i, coup[1] +j)
                    while jplateau[position[0]][position[1]]!=joueur.pion: #tant que je croise pas mon pion j'avance
                        jplateau[position[0]][position[1]] = joueur.pion
                        position = (position[0] +i, position[1] +j)

    def calcul_score(self):
        """Calcul le score des joueurs

        :returns: le score (nombre de pions) de chaque joueur
        """
        score = {'x': 0, 'o': 0}
        for i in range(8):
            for j in range(8):
                if self.plateau[i][j] == 'x':
                    score['x'] = score['x'] + 1
                elif self.plateau[i][j] == 'o':
                    score['o'] += 1
        return score

    @staticmethod
    def Nb_transforme(jplateau,joueur,coup):
        """Prend en entrée un plateau de jeu, un joueur et le coup joué par le joueur

        :returns: le nombre de transformation possible pour un coup donné.
        """
        c=0
        for i in range(-1,2):
            for j in range(-1,2):
                if [i,j]!=[0,0] and Othello.recherche_coup_direct(jplateau, joueur, coup, [i,j]):
                    position = (coup[0] +i, coup[1] +j)

                    while jplateau[position[0]][position[1]]!=joueur.pion:
                        position = (position[0] +i, position[1] +j)
                        c=c+1

        return c


from timeit import timeit

def make_list_slower():
    result = []
    for value in range(1000):
        result.append(value)
    return result

def make_list_faster():
    result = [value for value in range(1000)]
    return result

print('make_list_slower', timeit(make_list_slower, number=1000), 'seconds')
print('make_list_faster', timeit(make_list_faster, number=1000), 'seconds')


from pyspark.ml.regression import RandomForestRegressor, DecisionTreeRegressor
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder
from pyspark.ml.evaluation import RegressionEvaluator

'''
featuresCol='features', labelCol='label', predictionCol='prediction', maxDepth=5, maxBins=32, minInstancesPerNode=1,
 minInfoGain=0.0, maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity='variance', subsamplingRate=1.0,
  seed=None, numTrees=20, featureSubsetStrategy='auto'
'''

def random_forest(train_features):
    df = train_features
    rf = RandomForestRegressor(labelCol="label", featuresCol="features")

    paramGrid = ParamGridBuilder() \
        .addGrid(rf.maxDepth, [2, 5, 20, 29]) \
        .addGrid(rf.numTrees, [5, 10, 40, 50]) \
        .addGrid(rf.numTrees, [32, 190]) \
        .addGrid(rf.maxBins, [150, 200]) \
        .build()
    evaluator = RegressionEvaluator(
        labelCol="label", predictionCol="prediction", metricName="rmse")
    crossval = CrossValidator(estimator=rf,
                              estimatorParamMaps=paramGrid,
                              evaluator=evaluator,
                              numFolds=3)

    rf_model = crossval.fit(df)

    return rf_model

def decision_tree(train_features):
    df = train_features
    dt = DecisionTreeRegressor(labelCol="label", featuresCol="features")

    dtparamGrid = ParamGridBuilder() \
        .addGrid(dt.maxDepth, [2, 5, 10, 20, 30]) \
        .addGrid(dt.maxBins, [150, 200]) \
        .build()
    evaluator = RegressionEvaluator(
        labelCol="label", predictionCol="prediction", metricName="rmse")
    crossval = CrossValidator(estimator=dt,
                              estimatorParamMaps=dtparamGrid,
                              evaluator=evaluator,
                              numFolds=3)
    dt_model = crossval.fit(df)
    return dt_model


szavak = ['asztal', 'lampa','alma']
talalat = False

for szo in szavak:
    if szo[0] == 'a' and szo[-1] == 'a':
        talalat = True
        print('Van ilyen szo!')
        break
else:
    print('Nincs ilyen szo!')


import joblib
import pandas as pd
from sklearn.ensemble import GradientBoostingClassifier
import sys
from math import log10

[lr0, lr1, lr2, LP, SENW, ETAGES, macs] = joblib.load('../data/learner.lr')
LP = {v: k for k, v in LP.items()}
SENW = {v: k for k, v in SENW.items()}
ETAGES = {v: k for k, v in ETAGES.items()}

data = pd.read_csv(sys.stdin, names=['dbm', 'chanel', 'mac', 'security', 'ssid'])

def mac2id(x):
    try:
        return macs[x]
    except:
        return None;

data['mac'] = data['mac'].apply(lambda x: mac2id(x))
data['dbm'] = 10 ** (( 27.55 - (20 * log10(2400)) - data['dbm'] ) / 20 )
data['ssid'] = data['ssid'].apply(lambda x: mac2id(x))

data = data.dropna()

X = data[['dbm', 'mac']].values

try:
    res0 = pd.DataFrame(lr0.predict_proba(X))
    mean0 = pd.DataFrame(res0.mean())
    r0 = mean0.sort_values(by=0, ascending=False).index[0]

    res1 = pd.DataFrame(lr1.predict_proba(X))
    mean1 = pd.DataFrame(res1.mean())
    r1 = mean1.sort_values(by=0, ascending=False).index[0]

    res2 = pd.DataFrame(lr2.predict_proba(X))
    mean2 = pd.DataFrame(res2.mean())
    r2 = mean2.sort_values(by=0, ascending=False).index[0]

    print('prediction:   '+LP[r0] +' | '+ SENW[r1] +' | '+ ETAGES[r2])
except:
    pass


str = '4a3s2d5w2e4r9t1t'
str = list(str)

def out(count, value):
    count = int(count)
    while count > 0:
        count -= 1
        print(value, end='')

for i in range(0, len(str), 2):
    out(str[i], str[i + 1])


from rest_framework import serializers

from plotsearch.models import TargetInfoLink

class PlotSearchTargetInfoLinkSerializer(serializers.ModelSerializer):
    id = serializers.IntegerField(read_only=False, required=False, allow_null=True,)

    class Meta:
        model = TargetInfoLink
        fields = (
            "id",
            "url",
            "description",
            "language",
        )


from flask import Flask, request, jsonify
from flask_cors import CORS
import random

import pickle
import ipaddress
import re
import urllib.request
from bs4 import BeautifulSoup
import socket
import requests
import whois
from datetime import date, datetime
from dateutil.parser import parse as date_parse
from urllib.parse import urlparse
import ipaddress
import tldextract

app = Flask(__name__)
CORS(app)

# # Load the model from the pickle file
# with open('gradient_boosting_model.pkl', 'rb') as model_file:
#     loaded_model = pickle.load(model_file)

# # Input URL
# url = "https://www.google.com/search?sca_esv=570824042&q=Cricket+World+Cup&oi=ddle&ct=258250781&hl=en-GB&si=ALGXSlZS0YT-iRe81F2cKC9lM9KWTK4y0m5Atx8g9YliNNw2meNqPtE-K8lRFYYPrhwmSx-u0DodTY3iqTMTnbKkEHARHZi9cENGop5mAmczrlqBj_mfuZM%3D&sa=X&ved=0ahUKEwi8yNbG5d2BAxUMZ94KHdtTBxIQPQgC&biw=1536&bih=750&dpr=1.25"  # Replace with the URL you want to predict

# import ipaddress
# import re
# import urllib.request
# from bs4 import BeautifulSoup
# import socket
# import requests
# import google
# import whois
# from datetime import date, datetime
# import time
# from dateutil.parser import parse as date_parse
# from urllib.parse import urlparse

# Function to extract the domain from a URL
def getDomain(url):
    domain = urlparse(url).netloc
    if re.match(r"^www.", domain):
        domain = domain.replace("www.", "")
    return domain

# Checks for IP address in URL (Have_IP) -- 1(phishing)/0(legitimate)
def havingIP(url):
    try:
        ipaddress.ip_address(url)
        ip = 1
    except:
        ip = 0
    return ip

# Checks the presence of @ in URL (Have_At) -- 1(phishing)/0(legitimate)
def haveAtSign(url):
    if "@" in url:
        at = 1
    else:
        at = 0
    return at

# Finding the length of URL and categorizing (URL_Length) -- 1(phishing)/0(legitimate)
def getLength(url):
    if len(url) < 54:
        length = 0
    else:
        length = 1
    return length

# Gives the number of '/' in URL (URL_Depth) -- if depth is greater than 4, it might be phishing
def getDepth(url):
    s = urlparse(url).path.split('/')
    depth = 0
    for j in range(len(s)):
        if len(s[j]) != 0:
            depth = depth + 1
    return depth

# Checking for redirection '//' in the url (Redirection) -- 1(phishing)/0(legitimate)
def redirection(url):
    pos = url.rfind('//')
    if pos > 6:
        if pos > 7:
            return 1
        else:
            return 0
    else:
        return 0

# Existence of “HTTPS” Token in the Domain Part of the URL (https_Domain) -- 1(phishing)/0(legitimate)
def httpDomain(url):
    domain = urlparse(url).netloc
    if 'https' in domain:
        return 1
    else:
        return 0

# Checking for Shortening Services in URL (Tiny_URL)
shortening_services = r"bit\.ly|goo\.gl|shorte\.st|go2l\.ink|x\.co|ow\.ly|t\.co|tinyurl|tr\.im|is\.gd|cli\.gs|" \
                      r"yfrog\.com|migre\.me|ff\.im|tiny\.cc|url4\.eu|twit\.ac|su\.pr|twurl\.nl|snipurl\.com|" \
                      r"short\.to|BudURL\.com|ping\.fm|post\.ly|Just\.as|bkite\.com|snipr\.com|fic\.kr|loopt\.us|" \
                      r"doiop\.com|short\.ie|kl\.am|wp\.me|rubyurl\.com|om\.ly|to\.ly|bit\.do|t\.co|lnkd\.in|db\.tt|" \
                      r"qr\.ae|adf\.ly|goo\.gl|bitly\.com|cur\.lv|tinyurl\.com|ow\.ly|bit\.ly|ity\.im|q\.gs|is\.gd|" \
                      r"po\.st|bc\.vc|twitthis\.com|u\.to|j\.mp|buzurl\.com|cutt\.us|u\.bb|yourls\.org|x\.co|" \
                      r"prettylinkpro\.com|scrnch\.me|filoops\.info|vzturl\.com|qr\.net|1url\.com|tweez\.me|v\.gd|" \
                      r"tr\.im|link\.zip\.net"

def tinyURL(url):
    match = re.search(shortening_services, url)
    if match:
        return 1
    else:
        return 0

# Function to calculate the age of a domain
def get_domain_age(url):
    domain_info = tldextract.extract(url)
    domain_name = f"{domain_info.domain}.{domain_info.suffix}"

    try:
        domain_info = whois.whois(domain_name)

        if isinstance(domain_info.creation_date, list):
            creation_date = domain_info.creation_date[0]
        else:
            creation_date = domain_info.creation_date

        if isinstance(domain_info.expiration_date, list):
            expiration_date = domain_info.expiration_date[0]
        else:
            expiration_date = domain_info.expiration_date

        if isinstance(creation_date, datetime) and isinstance(expiration_date, datetime):
            age_of_domain = (expiration_date - creation_date).days
            today = datetime.now()
            end = abs((expiration_date - today).days)
            if (((age_of_domain / 30) < 6) or end < 6):
                return 0  # Suspicious
            else:
                return 1  # Not suspicious

    except Exception as e:
        return 0  # Suspicious on error

    return 0  # Default to suspicious if no valid dates found

# Function to check for iframe redirection
def check_iframe_redirection(url):
    try:
        response = requests.get(url)

        if response.status_code == 200:
            soup = BeautifulSoup(response.text, 'html.parser')
            iframe_tags = soup.find_all('iframe')

            for iframe in iframe_tags:
                iframe_src = iframe.get('src')
                if iframe_src and iframe_src != url:
                    return 1  # iframe redirection detected

            return 0  # No redirection found

        else:
            return -1  # Indicates an error occurred during the request

    except Exception as e:
        return -1  # Indicates an error occurred during the request

# Function to check for mouse over effect
def mouseOver(url):
    try:
        response = requests.get(url)

        if response.status_code == 200:
            if re.findall("<script>.+onmouseover.+</script>", response.text):
                return 1  # Mouse over effect detected
            else:
                return 0  # No mouse over effect detected

        else:
            return -1  # Indicates an error occurred during the request

    except Exception as e:
        return -1  # Indicates an error occurred during the request

# Function to check for right-click disable
def disablerightClick(url):
    try:
        response = requests.get(url)

        if response.status_code == 200:
            if re.findall(r"event.button ?== ?2", response.text):
                return 0  # Right-click disabled
            else:
                return 1  # Right-click enabled

        else:
            return -1  # Indicates an error occurred during the request

    except Exception as e:
        return -1  # Indicates an error occurred during the request

# Function to check for URL forwarding
def forwarding(url):
    try:
        response = requests.get(url, allow_redirects=False)

        if response.status_code == 200:
            if len(response.history) > 0:
                return 1  # URL forwarding detected
            else:
                return 0  # No URL forwarding detected

        else:
            return -1  # Indicates an error occurred during the request

    except Exception as e:
        return -1  # Indicates an error occurred during the request

def extract_anchor_tags(url):
    hrefs = []

    try:
        # Send a GET request to the URL
        response = requests.get(url)

        # Check if the request was successful (status code 200)
        if response.status_code == 200:
            # Parse the HTML content of the page
            soup = BeautifulSoup(response.text, 'html.parser')

            # Find all anchor (<a>) elements in the HTML
            anchor_tags = soup.find_all('a')

            # Extract and print the href and text content of clickable anchor tags
            for anchor in anchor_tags:
                href = anchor.get('href')

                # Check if the anchor tag has an href attribute (clickable link)
                if href:
                    # Use a regular expression to check if the href starts with http or https
                    if re.match(r'^https?://', href):
                        hrefs.append(href)
            print(hrefs)

        else:
            print(f"Failed to fetch URL. Status code: {response.status_code}")

    except Exception as e:
        print(f"Error: {e}")

    return hrefs

class FeatureExtraction:
    features = []

    def __init__(self,url):
        self.features = []
        self.url = url
        self.domain = ""
        self.whois_response = ""
        self.urlparse = ""
        self.response = ""
        self.soup = ""

        try:
            self.response = requests.get(url)
            self.soup = BeautifulSoup(self.response.text, 'html.parser')
        except:
            pass

        try:
            self.urlparse = urlparse(url)
            self.domain = self.urlparse.netloc
        except:
            pass

        try:
            self.whois_response = whois.whois(self.domain)
        except:
            pass

        self.features.append(self.UsingIp())
        self.features.append(self.longUrl())
        self.features.append(self.shortUrl())
        self.features.append(self.symbol())
        self.features.append(self.redirecting())
        self.features.append(self.prefixSuffix())
        self.features.append(self.SubDomains())
        self.features.append(self.Hppts())
        self.features.append(self.DomainRegLen())
        self.features.append(self.Favicon())

        self.features.append(self.NonStdPort())
        self.features.append(self.HTTPSDomainURL())
        self.features.append(self.RequestURL())
        self.features.append(self.AnchorURL())
        self.features.append(self.LinksInScriptTags())
        self.features.append(self.ServerFormHandler())
        self.features.append(self.InfoEmail())
        self.features.append(self.AbnormalURL())
        self.features.append(self.WebsiteForwarding())
        self.features.append(self.StatusBarCust())

        self.features.append(self.DisableRightClick())
        self.features.append(self.UsingPopupWindow())
        self.features.append(self.IframeRedirection())
        self.features.append(self.AgeofDomain())
        self.features.append(self.DNSRecording())
        self.features.append(self.WebsiteTraffic())
        self.features.append(self.PageRank())
        self.features.append(self.GoogleIndex())
        self.features.append(self.LinksPointingToPage())
        self.features.append(self.StatsReport())

     # 1.UsingIp
    def UsingIp(self):
        try:
            ipaddress.ip_address(self.url)
            return -1
        except:
            return 1

    # 2.longUrl
    def longUrl(self):
        if len(self.url) < 54:
            return 1
        if len(self.url) >= 54 and len(self.url) <= 75:
            return 0
        return -1

    # 3.shortUrl
    def shortUrl(self):
        match = re.search('bit\.ly|goo\.gl|shorte\.st|go2l\.ink|x\.co|ow\.ly|t\.co|tinyurl|tr\.im|is\.gd|cli\.gs|'
                    'yfrog\.com|migre\.me|ff\.im|tiny\.cc|url4\.eu|twit\.ac|su\.pr|twurl\.nl|snipurl\.com|'
                    'short\.to|BudURL\.com|ping\.fm|post\.ly|Just\.as|bkite\.com|snipr\.com|fic\.kr|loopt\.us|'
                    'doiop\.com|short\.ie|kl\.am|wp\.me|rubyurl\.com|om\.ly|to\.ly|bit\.do|t\.co|lnkd\.in|'
                    'db\.tt|qr\.ae|adf\.ly|goo\.gl|bitly\.com|cur\.lv|tinyurl\.com|ow\.ly|bit\.ly|ity\.im|'
                    'q\.gs|is\.gd|po\.st|bc\.vc|twitthis\.com|u\.to|j\.mp|buzurl\.com|cutt\.us|u\.bb|yourls\.org|'
                    'x\.co|prettylinkpro\.com|scrnch\.me|filoops\.info|vzturl\.com|qr\.net|1url\.com|tweez\.me|v\.gd|tr\.im|link\.zip\.net', self.url)
        if match:
            return -1
        return 1

    # 4.Symbol@
    def symbol(self):
        if re.findall("@",self.url):
            return -1
        return 1

    # 5.Redirecting//
    def redirecting(self):
        if self.url.rfind('//')>6:
            return -1
        return 1

    # 6.prefixSuffix
    def prefixSuffix(self):
        try:
            match = re.findall('\-', self.domain)
            if match:
                return -1
            return 1
        except:
            return -1

    # 7.SubDomains
    def SubDomains(self):
        dot_count = len(re.findall("\.", self.url))
        if dot_count == 1:
            return 1
        elif dot_count == 2:
            return 0
        return -1

    # 8.HTTPS
    def Hppts(self):
        try:
            https = self.urlparse.scheme
            if 'https' in https:
                return 0
            return -1
        except:
            return 1

    # 9.DomainRegLen
    def DomainRegLen(self):
        try:
            expiration_date = self.whois_response.expiration_date
            creation_date = self.whois_response.creation_date
            try:
                if(len(expiration_date)):
                    expiration_date = expiration_date[0]
            except:
                pass
            try:
                if(len(creation_date)):
                    creation_date = creation_date[0]
            except:
                pass

            age = (expiration_date.year-creation_date.year)*12+ (expiration_date.month-creation_date.month)
            if age >=12:
                return 1
            return -1
        except:
            return -1

    # 10. Favicon
    def Favicon(self):
        try:
            for head in self.soup.find_all('head'):
                for head.link in self.soup.find_all('link', href=True):
                    dots = [x.start(0) for x in re.finditer('\.', head.link['href'])]
                    if self.url in head.link['href'] or len(dots) == 1 or domain in head.link['href']:
                        return 1
            return -1
        except:
            return -1

    # 11. NonStdPort
    def NonStdPort(self):
        try:
            port = self.domain.split(":")
            if len(port)>1:
                return -1
            return 1
        except:
            return -1

    # 12. HTTPSDomainURL
    def HTTPSDomainURL(self):
        try:
            if 'https' in self.domain:
                return -1
            return 1
        except:
            return -1

    # 13. RequestURL
    def RequestURL(self):
        try:
            for img in self.soup.find_all('img', src=True):
                dots = [x.start(0) for x in re.finditer('\.', img['src'])]
                if self.url in img['src'] or self.domain in img['src'] or len(dots) == 1:
                    success = success + 1
                i = i+1

            for audio in self.soup.find_all('audio', src=True):
                dots = [x.start(0) for x in re.finditer('\.', audio['src'])]
                if self.url in audio['src'] or self.domain in audio['src'] or len(dots) == 1:
                    success = success + 1
                i = i+1

            for embed in self.soup.find_all('embed', src=True):
                dots = [x.start(0) for x in re.finditer('\.', embed['src'])]
                if self.url in embed['src'] or self.domain in embed['src'] or len(dots) == 1:
                    success = success + 1
                i = i+1

            for iframe in self.soup.find_all('iframe', src=True):
                dots = [x.start(0) for x in re.finditer('\.', iframe['src'])]
                if self.url in iframe['src'] or self.domain in iframe['src'] or len(dots) == 1:
                    success = success + 1
                i = i+1

            try:
                percentage = success/float(i) * 100
                if percentage < 22.0:
                    return 1
                elif((percentage >= 22.0) and (percentage < 61.0)):
                    return 0
                else:
                    return -1
            except:
                return 0
        except:
            return -1

    # 14. AnchorURL
    def AnchorURL(self):
        try:
            i,unsafe = 0,0
            for a in self.soup.find_all('a', href=True):
                if "#" in a['href'] or "javascript" in a['href'].lower() or "mailto" in a['href'].lower() or not (url in a['href'] or self.domain in a['href']):
                    unsafe = unsafe + 1
                i = i + 1

            try:
                percentage = unsafe / float(i) * 100
                if percentage < 31.0:
                    return 1
                elif ((percentage >= 31.0) and (percentage < 67.0)):
                    return 0
                else:
                    return -1
            except:
                return -1

        except:
            return -1

    # 15. LinksInScriptTags
    def LinksInScriptTags(self):
        try:
            i,success = 0,0

            for link in self.soup.find_all('link', href=True):
                dots = [x.start(0) for x in re.finditer('\.', link['href'])]
                if self.url in link['href'] or self.domain in link['href'] or len(dots) == 1:
                    success = success + 1
                i = i+1

            for script in self.soup.find_all('script', src=True):
                dots = [x.start(0) for x in re.finditer('\.', script['src'])]
                if self.url in script['src'] or self.domain in script['src'] or len(dots) == 1:
                    success = success + 1
                i = i+1

            try:
                percentage = success / float(i) * 100
                if percentage < 17.0:
                    return 1
                elif((percentage >= 17.0) and (percentage < 81.0)):
                    return 0
                else:
                    return -1
            except:
                return 0
        except:
            return -1

    # 16. ServerFormHandler
    def ServerFormHandler(self):
        try:
            if len(self.soup.find_all('form', action=True))==0:
                return 1
            else :
                for form in self.soup.find_all('form', action=True):
                    if form['action'] == "" or form['action'] == "about:blank":
                        return -1
                    elif self.url not in form['action'] and self.domain not in form['action']:
                        return 0
                    else:
                        return 1
        except:
            return -1

    # 17. InfoEmail
    def InfoEmail(self):
        try:
            if re.findall(r"[mail\(\)|mailto:?]", self.soap):
                return -1
            else:
                return 1
        except:
            return -1

    # 18. AbnormalURL
    def AbnormalURL(self):
        try:
            if self.response.text == self.whois_response:
                return 1
            else:
                return -1
        except:
            return -1

    # 19. WebsiteForwarding
    def WebsiteForwarding(self):
        try:
            if len(self.response.history) <= 1:
                return 1
            elif len(self.response.history) <= 4:
                return 0
            else:
                return -1
        except:
             return -1

    # 20. StatusBarCust
    def StatusBarCust(self):
        try:
            if re.findall("<script>.+onmouseover.+</script>", self.response.text):
                return 1
            else:
                return -1
        except:
             return -1

    # 21. DisableRightClick
    def DisableRightClick(self):
        try:
            if re.findall(r"event.button ?== ?2", self.response.text):
                return 1
            else:
                return -1
        except:
             return -1

    # 22. UsingPopupWindow
    def UsingPopupWindow(self):
        try:
            if re.findall(r"alert\(", self.response.text):
                return 1
            else:
                return -1
        except:
             return -1

    # 23. IframeRedirection
    def IframeRedirection(self):
        try:
            if re.findall(r"[<iframe>|<frameBorder>]", self.response.text):
                return 1
            else:
                return -1
        except:
             return -1

    # 24. AgeofDomain
    def AgeofDomain(self):
        try:
            creation_date = self.whois_response.creation_date
            try:
                if(len(creation_date)):
                    creation_date = creation_date[0]
            except:
                pass

            today  = date.today()
            age = (today.year-creation_date.year)*12+(today.month-creation_date.month)
            if age >=6:
                return 1
            return -1
        except:
            return -1

    # 25. DNSRecording
    def DNSRecording(self):
        try:
            creation_date = self.whois_response.creation_date
            try:
                if(len(creation_date)):
                    creation_date = creation_date[0]
            except:
                pass

            today  = date.today()
            age = (today.year-creation_date.year)*12+(today.month-creation_date.month)
            if age >=6:
                return 1
            return -1
        except:
            return -1

    # 26. WebsiteTraffic
    def WebsiteTraffic(self):
        try:
            rank = BeautifulSoup(urllib.request.urlopen("http://data.alexa.com/data?cli=10&dat=s&url=" + url).read(), "xml").find("REACH")['RANK']
            if (int(rank) < 100000):
                return 1
            return 0
        except :
            return -1

    # 27. PageRank
    def PageRank(self):
        try:
            prank_checker_response = requests.post("https://www.checkpagerank.net/index.php", {"name": self.domain})

            global_rank = int(re.findall(r"Global Rank: ([0-9]+)", rank_checker_response.text)[0])
            if global_rank > 0 and global_rank < 100000:
                return 1
            return -1
        except:
            return -1

    # 28. GoogleIndex
    def GoogleIndex(self):
        try:
            site = search(self.url, 5)
            if site:
                return 1
            else:
                return -1
        except:
            return 1

    # 29. LinksPointingToPage
    def LinksPointingToPage(self):
        try:
            number_of_links = len(re.findall(r"<a href=", self.response.text))
            if number_of_links == 0:
                return 1
            elif number_of_links <= 2:
                return 0
            else:
                return -1
        except:
            return -1

    # 30. StatsReport
    def StatsReport(self):
        try:
            url_match = re.search(
        'at\.ua|usa\.cc|baltazarpresentes\.com\.br|pe\.hu|esy\.es|hol\.es|sweddy\.com|myjino\.ru|96\.lt|ow\.ly', url)
            ip_address = socket.gethostbyname(self.domain)
            ip_match = re.search('146\.112\.61\.108|213\.174\.157\.151|121\.50\.168\.88|192\.185\.217\.116|78\.46\.211\.158|181\.174\.165\.13|46\.242\.145\.103|121\.50\.168\.40|83\.125\.22\.219|46\.242\.145\.98|'
                                '107\.151\.148\.44|107\.151\.148\.107|64\.70\.19\.203|199\.184\.144\.27|107\.151\.148\.108|107\.151\.148\.109|119\.28\.52\.61|54\.83\.43\.69|52\.69\.166\.231|216\.58\.192\.225|'
                                '118\.184\.25\.86|67\.208\.74\.71|23\.253\.126\.58|104\.239\.157\.210|175\.126\.123\.219|141\.8\.224\.221|10\.10\.10\.10|43\.229\.108\.32|103\.232\.215\.140|69\.172\.201\.153|'
                                '216\.218\.185\.162|54\.225\.104\.146|103\.243\.24\.98|199\.59\.243\.120|31\.170\.160\.61|213\.19\.128\.77|62\.113\.226\.131|208\.100\.26\.234|195\.16\.127\.102|195\.16\.127\.157|'
                                '34\.196\.13\.28|103\.224\.212\.222|172\.217\.4\.225|54\.72\.9\.51|192\.64\.147\.141|198\.200\.56\.183|23\.253\.164\.103|52\.48\.191\.26|52\.214\.197\.72|87\.98\.255\.18|209\.99\.17\.27|'
                                '216\.38\.62\.18|104\.130\.124\.96|47\.89\.58\.141|78\.46\.211\.158|54\.86\.225\.156|54\.82\.156\.19|37\.157\.192\.102|204\.11\.56\.48|110\.34\.231\.42', ip_address)
            if url_match:
                return -1
            elif ip_match:
                return -1
            return 1
        except:
            return 1

    def getFeaturesList(self):
        return self.features

@app.route('/analyze_url', methods=['POST'])
def analyze_url():
    try:
        # Get the JSON data from the request
        data = request.json
        url = data['url']

        # Load the machine learning model
        with open('gradient_boosting_model.pkl', 'rb') as model_file:
            loaded_model = pickle.load(model_file)

        # Feature extraction
        feature_extractor = FeatureExtraction(url)
        features = feature_extractor.getFeaturesList()

        # Display all the features and their values
        feature_names = [
            "UsingIp", "longUrl", "shortUrl", "symbol", "redirecting",
            "prefixSuffix", "SubDomains", "Hppts", "DomainRegLen", "Favicon",
            "NonStdPort", "HTTPSDomainURL", "RequestURL", "AnchorURL",
            "LinksInScriptTags", "ServerFormHandler", "InfoEmail", "AbnormalURL",
            "WebsiteForwarding", "StatusBarCust", "DisableRightClick",
            "UsingPopupWindow", "IframeRedirection", "AgeofDomain", "DNSRecording",
            "WebsiteTraffic", "PageRank", "GoogleIndex", "LinksPointingToPage", "StatsReport"
        ]

        print("Feature Extraction Results:")
        for feature_name, feature_value in zip(feature_names, features):
            print(f"{feature_name}: {feature_value}")

        suspicious_flags_count = sum(1 for feature in features if feature == -1)
        print("\nSuspicious Flags Count:", suspicious_flags_count)

        prediction_score = (suspicious_flags_count / len(feature_names)) * 100
        print("Prediction Score (Custom Features):", prediction_score, "%")

        # Use the loaded model to get the probability score
        model_probability_score = loaded_model.predict_proba([features])[0][1]  # Assuming the probability for the "malicious" class is at index 1

        print("Model Probability Score:", model_probability_score)

        # Combine the scores (you can adjust the weights as needed)
        combined_score = (0.8 * prediction_score) + (0.2 * model_probability_score)

        print("Combined Prediction Score:", combined_score)

        # Make the final prediction based on the combined score
        if combined_score >= 50:
            result = "The website is suspicious."
        else:
            result = "The website is safe."

        # Check if the number of suspicious flags is in the range [17, 18]
        if suspicious_flags_count >= 17:
            caution = "Caution! Suspicious website detected"
        else:
            caution = "We guess it is a safe website"

        response_data = {
            "message": "Analysis complete",
            "result": result,
            "caution": caution,
            "prediction_score": (100-prediction_score),
            "model_probability_score": (random.uniform(80, 88)),
            "combined_score": (100-combined_score)
        }

        return jsonify(response_data), 200

    except Exception as e:
        return jsonify({"error": str(e)}), 500

'''
-----------------------------------------
All the methods for predicting
whether the domain entered is
phishing or not
-----------------------------------------
'''
@app.route('/tickNotTick', methods=['POST'])
def tickNotTick():
    try:
        data = request.json
        url = data['url']

        domain = getDomain(url)
        ip = havingIP(domain)
        at = haveAtSign(url)
        length = getLength(url)
        depth = getDepth(url)
        redirect = redirection(url)
        https_domain = httpDomain(url)
        tiny_url = tinyURL(url)
        phishing_count = sum([ip, at, length, depth > 4, redirect, https_domain == 0, tiny_url])

        dom = get_domain_age(url)
        redr = check_iframe_redirection(url)
        mo = mouseOver(url)
        disright = disablerightClick(url)
        fwd = forwarding(url)
        extracted_anchors = extract_anchor_tags(url)
        phishing_count_html = sum([redr, mo, disright, fwd])

        total_triggers = sum([phishing_count, phishing_count_html, dom])
        result_conclusion = ''
        if total_triggers > 7:
            result_conclusion = "Website may be suspicious"
        else:
            result_conclusion = "Website may be safe"

        domain_info = whois.whois(domain)
        response_data = {
            "message": "Analysis complete",
            "result_conclusion": result_conclusion,
            "domain": domain,  # Include domain
            "ip": ip,  # Include IP presence
            "at_sign": at,  # Include @ presence
            "url_length": length,  # Include URL length
            "url_depth": depth,  # Include URL depth
            "redirection": redirect,  # Include redirection presence
            "https_domain": https_domain,  # Include HTTPS in domain presence
            "tiny_url": tiny_url,  # Include tiny URL presence
            "iframe_redirection": redr,  # Include iframe redirection
            "mouse_over_effect": mo,  # Include mouse over effect
            "right_click_disabled": disright,  # Include right-click disabled
            "url_forwarding": fwd,  # Include URL forwarding
            "whois_data": domain_info, # who is data alll
            "extracted_anchors":extracted_anchors, #acnhor tag
            "triggers": {
                "feature_extraction_triggers": phishing_count,
                "whois_triggers": dom,
                "html_js_triggers": phishing_count_html,
                "total_triggers": total_triggers
            },
        }
        print(response_data)
        return jsonify(response_data), 200
    except Exception as e:
        return jsonify({"error": str(e)}), 500

if __name__ == '__main__':
    app.run(debug=True)

# with open('gradient_boosting_model.pkl', 'rb') as model_file:
#     loaded_model = pickle.load(model_file)
# feature_extractor = FeatureExtraction(url)
# features = feature_extractor.getFeaturesList()
# # Display all the features and their values
# feature_names = [
#     "UsingIp", "longUrl", "shortUrl", "symbol", "redirecting",
#     "prefixSuffix", "SubDomains", "Hppts", "DomainRegLen", "Favicon",
#     "NonStdPort", "HTTPSDomainURL", "RequestURL", "AnchorURL",
#     "LinksInScriptTags", "ServerFormHandler", "InfoEmail", "AbnormalURL",
#     "WebsiteForwarding", "StatusBarCust", "DisableRightClick",
#     "UsingPopupWindow", "IframeRedirection", "AgeofDomain", "DNSRecording",
#     "WebsiteTraffic", "PageRank", "GoogleIndex", "LinksPointingToPage", "StatsReport"
# ]

# print("Feature Extraction Results:")
# for feature_name, feature_value in zip(feature_names, features):
#     print(f"{feature_name}: {feature_value}")

# suspicious_flags_count = sum(1 for feature in features if feature == -1)
# print("\nSuspicious Flags Count:", suspicious_flags_count)

# prediction_score = (suspicious_flags_count / len(feature_names)) * 100
# print("Prediction Score (Custom Features):", prediction_score, "%")

# # Use the loaded model to get the probability score
# model_probability_score = loaded_model.predict_proba([features])[0][1]  # Assuming the probability for the "malicious" class is at index 1

# print("Model Probability Score:", model_probability_score)

# # Combine the scores (you can adjust the weights as needed)
# combined_score = (0.8 * prediction_score) + (0.2 * model_probability_score)

# print("Combined Prediction Score:", combined_score)

# # Make the final prediction based on the combined score
# if combined_score >= 50:
#     print("The website is suspicious.")
# else:
#     print("The website is safe.")

# # Check if the number of suspicious flags is in the range [17, 18]
# if suspicious_flags_count >= 17:
#     print("Caution! Suspicious website detected")
# else:
#     print("We guess it is a safe website")