import logging
from typing import Dict, List, Optional

from gql import gql

from .gql_queries import *

LIST_EXPOSED_PORT_FIELD = ["basePathVariableName", "isRewriteUrl", "isAuthenticationRequired", "port", "name"]

class Apps:
    def __init__(self, saagie_api):
        self.saagie_api = saagie_api
        self.client = saagie_api.client

    def list_for_project(
        self, project_id: str, instances_limit: int = None, pprint_result: Optional[bool] = None
    ) -> Dict:
        """List apps of project.
        NB: You can only list apps if you have at least the viewer role on
        the project.

        Parameters
        ----------
        project_id : str
            UUID of your project (see README on how to find it)
        instances_limit: int
            limit the number of instances to return, default to no limit
        pprint_result : bool, optional
            Whether to pretty print the result of the query, default to
            saagie_api.pprint_global

        Returns
        -------
        dict
            Dict of app information
        """
        params = {"projectId": project_id, "instancesLimit": instances_limit}
        return self.saagie_api.client.execute(
            query=gql(GQL_LIST_APPS_FOR_PROJECT), variable_values={"id": project_id}, pprint_result=pprint_result
        )

    def get_info(self, app_id: str, instances_limit: int = None, pprint_result: Optional[bool] = None) -> Dict:
        """Get app with given UUID.

        Parameters
        ----------
        app_id : str
            UUID of your app
        pprint_result : bool, optional
            Whether to pretty print the result of the query, default to
            saagie_api.pprint_global
        instances_limit: int
            limit the number of instances to return, default to no limit

        Returns
        -------
        dict
            Dict of app information
        """
        params = {"id": app_id, "instancesLimit": instances_limit}
        return self.saagie_api.client.execute(
            query=gql(GQL_GET_APP_INFO), variable_values=params, pprint_result=pprint_result
        )

    def create_from_scratch(
        self,
        project_id: str,
        app_name: str,
        image: str,
        description: str = "",
        exposed_ports: List[Dict] = None,
        storage_paths: List = None,
        storage_size_in_mb: int = 128,
        release_note: str = "",
        docker_credentials_id: str = None,
        emails: List = None,
        status_list: List = None,
    ) -> Dict:
        """Create an app in a specific project
        Parameters
        ----------
        project_id : str
            ID of the project
        app_name: str
            Name of the app
        description: str
            Description of the app
        image: str
            tag of the Docker Image
            ex: hello-world:nanoserver-ltc2022
        docker_credentials_id: str
            Credentials's ID for the image if the image is not public
        exposed_ports: List[dict]
            List of dict of exposed ports
            Each dict should contains 'port' as key
            Ex: [{"basePathVariableName":"SAAGIE_BASE_PATH",
                "isRewriteUrl":True,
                "isAuthenticationRequired":True,
                "port":5000,
                "name":"Test Port"}]
        storage_paths: List[String], optional
            List of strings indicating the volume path to the persistent storage
        storage_size_in_mb: int, optional
            Storage size in mb for the volume path
        release_note: str,
            Release note for the app version
        emails: List[String], optional
            Emails to receive alerts for the app, each item should be a valid email
        status_list: List[String], optional
            Receive an email when the job status change to a specific status
            Each item of the list should be one of these following values: "REQUESTED", "QUEUED",
            "RUNNING", "FAILED", "KILLED", "KILLING", "SUCCEEDED", "UNKNOWN", "AWAITING", "SKIPPED"
        Returns
        -------
        dict
            Dict of app information
        """
        if storage_paths is None:
            storage_paths = []
        if exposed_ports is None:
            exposed_ports = []

        params = {
            "projectId": project_id,
            "name": app_name,
            "description": description,
            "releaseNote": release_note,
            "storageSizeInMB": storage_size_in_mb,
            "image": image,
            "exposedPorts": exposed_ports,
            "storagePaths": storage_paths,
        }

        check_format_exposed_port = self.check_exposed_ports(exposed_ports)
        if not check_format_exposed_port:
            raise ValueError(
                "❌ The parameter 'exposed_ports' should be a list of dict. Each dict should contains the key 'port'."
                f"All accept key of each dict is: '{LIST_EXPOSED_PORT_FIELD}'"
            )

        technology = "Docker image"
        technology_catalog = "Saagie"
        technologies_for_project = self.saagie_api.projects.get_apps_technologies(project_id)["appTechnologies"]
        technologies_for_project = [tech["id"] for tech in technologies_for_project]
        # For apps, docker technology is always available
        repositories = self.saagie_api.get_repositories_info()["repositories"]
        # Filter saagie repo
        saagie_repo = [repo for repo in repositories if repo["name"] == technology_catalog][0]["technologies"]
        # Get id of docker technology
        docker_id = [repo["id"] for repo in saagie_repo if repo["label"] == technology][0]
        # Add docker technology to the list of technologies for the project
        technologies_for_project.append(docker_id)
        params = self.saagie_api.check_technology(params, technology, technology_catalog, technologies_for_project)

        if docker_credentials_id:
            params["dockerCredentialsId"] = docker_credentials_id

        if emails:
            params = self.saagie_api.check_alerting(emails, params, status_list)
        result = self.saagie_api.client.execute(query=gql(GQL_CREATE_APP), variable_values=params)
        logging.info("✅ App [%s] successfully created", app_name)
        return result

    def create_from_catalog(
        self,
        project_id: str,
        app_name: str,
        technology: str,
        context: str,
        technology_catalog: str = "Saagie",
        description: str = "",
        storage_size_in_mb: int = 128,
        release_note: str = "",
        emails: List = None,
        status_list: List = None,
    ) -> Dict:
        """Create an app in a specific project
        Parameters
        ----------
        project_id : str
            ID of the project
        app_name: str
            Name of the app
        technology: str
            Label of the technology
        context: str
            The runtimes context to use for the chosen technology
        technology_catalog: str
            Name of the technology catalog
        description: str
            Description of the app
        storage_size_in_mb: int, optional
            Storage size in mb for the volume path
        release_note: str,
            Release note for the app version
        emails: List[String], optional
            Emails to receive alerts for the app, each item should be a valid email
        status_list: List[String], optional
            Receive an email when the job status change to a specific status
            Each item of the list should be one of these following values: "REQUESTED", "QUEUED",
            "RUNNING", "FAILED", "KILLED", "KILLING", "SUCCEEDED", "UNKNOWN", "AWAITING", "SKIPPED"
        Returns
        -------
        dict
            Dict of app information
        """
        # Initialize parameters to request GQL Graph
        params = {
            "projectId": project_id,
            "name": app_name,
            "description": description,
            "releaseNote": release_note,
            "storageSizeInMB": storage_size_in_mb,
        }

        # Get all id technologies in our project
        technologies_for_project = self.saagie_api.projects.get_apps_technologies(project_id)["appTechnologies"]
        technologies_for_project = [tech["id"] for tech in technologies_for_project]

        # Check if the technology exist
        params = self.saagie_api.check_technology(params, technology, technology_catalog, technologies_for_project)
        app_id = params["technologyId"]

        # Check if app_is is available in our project
        techno_app = self.saagie_api.projects.get_apps_technologies(project_id=project_id)
        app_is_available = [app["id"] for app in techno_app["appTechnologies"] if app["id"] == app_id]
        if not app_is_available:
            raise ValueError(
                f"❌ App '{technology}' is not available in the project: '{project_id}'. Check your project settings"
            )

        # Get different runtimes of app
        runtimes = self.saagie_api.get_runtimes(app_id)["technology"]["appContexts"]
        # Check if runtime is available
        available_runtimes = [app for app in runtimes if app["available"] is True]
        context_app = [app for app in available_runtimes if app["label"] == context]

        if not context_app:
            available_contexts = [app["label"] for app in available_runtimes]
            raise ValueError(
                f"❌ Runtime '{context}' of the app '{technology}' doesn't exist or is not available in the project: "
                f"'{project_id}'. Available runtimes are: '{available_contexts}'"
            )
        context_app_info = context_app[0]

        exposed_ports = context_app_info["ports"]
        # change key names in the list of dict exposed_ports
        exposed_ports = [
            {
                "basePathVariableName": port["basePath"],
                "isRewriteUrl": port["rewriteUrl"],
                "isAuthenticationRequired": True,
                "port": port["port"],
                "name": port["name"],
            }
            if port["scope"] == "PROJECT"
            else {
                "basePathVariableName": port["basePath"],
                "isRewriteUrl": port["rewriteUrl"],
                "isAuthenticationRequired": False,
                "port": port["port"],
                "name": port["name"],
            }
            for port in exposed_ports
        ]

        # Get image_name with concatenated image name with tag (version)
        image_app = f"{context_app_info['dockerInfo']['image']}:{context_app_info['dockerInfo']['version']}"

        check_format_exposed_port = self.check_exposed_ports(exposed_ports)
        if not check_format_exposed_port:
            raise ValueError(
                "❌ The parameter 'exposed_ports' should be a list of dict. Each dict should contains the key 'port'."
                f"All accept key of each dict is: '{LIST_EXPOSED_PORT_FIELD}'"
            )

        if emails:
            params = self.saagie_api.check_alerting(emails, params, status_list)

        # Add collected information (Docker image, ports exposed, path for the storage) to the list of parameters
        params.update({"image": image_app, "exposedPorts": exposed_ports, "storagePaths": context_app_info["volumes"]})

        result = self.saagie_api.client.execute(query=gql(GQL_CREATE_APP), variable_values=params)
        logging.info("✅ App [%s] successfully created", app_name)
        return result

    def edit(
        self,
        app_id: str,
        app_name: str = None,
        description: str = None,
        emails: List = None,
        status_list: List = None,
    ) -> Dict:
        """Edit an app
        Each optional parameter can be set to change the value of the corresponding field.
        Parameters
        ----------
        app_id : str
            UUID of your app
        app_name : str, optional
            App name
            If not filled, defaults to current value, else it will change the app's name
        description : str, optional
            Description of app
            if not filled, defaults to current value, else it will change the description of the app
        emails: List[String], optional
            Emails to receive alerts for the app, each item should be a valid email,
            If you want to remove alerting, please set emails to [] or list()
            if not filled, defaults to current value
        status_list: List[String], optional
            Receive an email when the app status change to a specific status.
            Each item of the list should be one of these following values: "REQUESTED", "QUEUED",
            "RUNNING", "FAILED", "KILLED", "KILLING", "SUCCEEDED", "UNKNOWN", "AWAITING", "SKIPPED"
            You cannot change only the status_lit, you must set both emails and status_list.
        Returns
        -------
        dict
            Dict of app information
        """
        params = {"id": app_id}
        previous_app_version = self.get_info(app_id, instances_limit=1, pprint_result=False)["labWebApp"]

        if app_name:
            params["name"] = app_name
        else:
            params["name"] = previous_app_version["name"]

        if description:
            params["description"] = description
        else:
            params["description"] = previous_app_version["description"]

        if emails:
            params = self.saagie_api.check_alerting(emails, params, status_list)
        elif isinstance(emails, List):
            params["alerting"] = None
        else:
            previous_alerting = previous_app_version["alerting"]
            if previous_alerting:
                params["alerting"] = {
                    "emails": previous_alerting["emails"],
                    "statusList": previous_alerting["statusList"],
                }

        result = self.saagie_api.client.execute(query=gql(GQL_EDIT_APP), variable_values=params)
        logging.info("✅ App [%s] successfully edited", app_name)
        return result

    def delete(self, app_id: str) -> Dict:
        """Delete a given app

        Parameters
        ----------
        app_id : str
            UUID of your app (see README on how to find it)

        Returns
        -------
        dict
            Dict of deleted app

        """
        result = self.saagie_api.client.execute(query=gql(GQL_DELETE_APP), variable_values={"appId": app_id})
        logging.info("✅ App [%s] successfully deleted", app_id)
        return result

    def run(self, app_id: str) -> Dict:
        """Run a given app

        Parameters
        ----------
        app_id : str
            UUID of your app (see README on how to find it)

        Returns
        -------
        dict
            Dict of the given app information
        """
        result = self.saagie_api.client.execute(query=gql(GQL_RUN_APP), variable_values={"appId": app_id})
        logging.info("✅ App [%s] successfully started", app_id)
        return result

    def stop(self, app_instance_id: str) -> Dict:
        """Stop a given job instance

        Parameters
        ----------
        app_instance_id : str
            UUID of your application instance (see README on how to find it)

        Returns
        -------
        dict
            app instance information
        """
        result = self.saagie_api.client.execute(
            query=gql(GQL_STOP_APP_INSTANCE), variable_values={"appInstanceId": app_instance_id}
        )
        logging.info("✅ App instance [%s] successfully stopped", app_instance_id)
        return result

    @staticmethod
    def check_exposed_ports(exposed_ports: List):
        """
        Check
        Parameters
        ----------
        exposed_ports : list
            List of exposed ports, each item of the list should be a dict
            and each dict should have 'port' as key
        Returns
        -------
        bool
            True if all exposed port is in the validate format
            Otherwise False
        """

        if not isinstance(exposed_ports, List):
            return False

        if len(exposed_ports):
            check_type = all(isinstance(ep, Dict) for ep in exposed_ports)
            if check_type:
                check_port = all("port" in ep.keys() for ep in exposed_ports)
                if check_port:
                    check_every_key = all(
                        all(elem in LIST_EXPOSED_PORT_FIELD for elem in ep.keys()) for ep in exposed_ports
                    )
                    if check_every_key:
                        return True
                    return False
                return False
            return False
        return True


import gensim
from words_match import Get_Text
def get_words(sub):
    temp = []
    words = []
    for i in range(len(sub)):
        for j in range(1, len(sub[i])):
            temp.append(sub[i][j])
        words.append(temp)
        temp = []
    return words

def takeSecond(elem):   #指定list中第二个元素进行排序
    return elem[1]

# def cosine(vector1,vector2):
#     cosV12 = np.dot(vector1, vector2) / (linalg.norm(vector1) * linalg.norm(vector2))
#     return cosV12

if __name__ == '__main__':
    les_word = "D:\Learn Chinese\self-adap\data\learning_item_words.txt"
    sce_word = "D:\Learn Chinese\self-adap\data\scenario_words.txt"
    sce_words = Get_Text(sce_word)
    les_words = Get_Text(les_word)
    #print(len(sce_words))
    # for i in range(len(sce_words)):
    #     print(sce_words[i])
    #print(len(les_words))
    lesson_words = get_words(les_words)
    scenario_words = get_words(sce_words)
    res = []
    temp=[]
    model = gensim.models.KeyedVectors.load_word2vec_format('vectors.txt', binary=False)
    for i in range(len(lesson_words)):
        for j in range(len(scenario_words)):
            try:
                score = model.n_similarity(lesson_words[i], scenario_words[j])
                '''
                利用词向量比较两个词语串的相似度
                '''
                res.append([j+1,score])
            except KeyError as e:
                res.append([j+1,e])
    # for i in range(len(lesson_words)):
    #     for j in range(len(scenario_words)):
    #         vec1 = doc_model.infer_vector(lesson_words[i],steps=50,alpha=0.025)
    #         vec2 = doc_model.infer_vector(scenario_words[j], steps=50, alpha=0.025)
    #         res.append([j,cosine(vec1,vec2)])
        res.sort(key=takeSecond, reverse=True)
        for i in range(10):
            temp.append(res[i])
        print(res)
        res=[]


#!/usr/bin/env python
# encoding: utf-8
#
# config.py
#
# Created by José Sánchez-Gallego on 18 Jun 2017.

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import pathlib
import yaml

def merge(user, default):
    """Merges a user configuration with the default one."""

    if not user:
        return default

    if isinstance(user, dict) and isinstance(default, dict):
        for kk, vv in default.items():
            if kk not in user:
                user[kk] = vv
            else:
                user[kk] = merge(user[kk], vv)

    return user

def get_config(user_path):
    """Returns a dictionary object with configuration options."""

    user_path = pathlib.Path(user_path).expanduser()
    user = user_path.exists() and yaml.load(open(str(user_path), 'r'), Loader=yaml.FullLoader)

    default_path = pathlib.Path(__file__).parents[0] / '../etc/lvmsurveysim_defaults.yaml'
    default = yaml.load(open(str(default_path), 'r'), Loader=yaml.FullLoader)

    return merge(user, default)


import requests
import mycommons.mqtt_wrap as mqtt
import mycommons.data_structure as ds
import json

credentials_file = '../credentials/ibm.json'
ibm_service = 'images'
max_images = 1

def init(client:mqtt.MQTTClient):
    with open(credentials_file, 'r') as fin:
        credentials = json.load(fin)[ibm_service]
    client.key = 'apikey:{}'.format(credentials['key'])
    client.instance = credentials['url']

def process_imgs(client:mqtt.MQTTClient, user:ds.User):
    if user.img_topics is not None:
        raise ds.UnableToProcessUser('already completed')

    # get a unique list of images' urls
    images = []
    for act in user.activities:
        images += act.images

    # for each image (until max_images): get info from API
    topics, count = [], 0
    for image in images:
        count += 1
        url = "https://{}@gateway.watsonplatform.net/visual-recognition/api/v3/classify?url={}&version=2018-03-19".format(client.key, image)
        r = requests.get(url)
        res = json.loads(r.content)
        for cl in res['images'][0]['classifiers'][0]['classes']:
            topics.append(cl['class'])
        if count >= max_images:
            break

    # if got no topics, set error, than send
    if len(topics) == 0:
        raise ds.UnableToProcessUser('did not found images')

    return ds.User(img_topics=topics), f"sucessfully called IBM API to compute photos for:{user.id}"

# test API
def main(process_type='ibm_vision', id='0'):
    mqtt.MQTTClient(process_type=process_type, funct=process_imgs, init=init, id=id, type=mqtt.CLASSIFIER)

if __name__ == '__main__':
    main()


#!/usr/bin/python3
"""
base.py module
"""
import turtle
import json
import csv

class Base:
    """
    Base class.
    """

    __nb_objects = 0

    def __init__(self, id=None):
        """
        Initialize a Base instance.

        Args:
            id (int, optional): Unique identifier for the instance. Defaults to None.
        """
        if id is not None:
            self.id = id
        else:
            Base.__nb_objects += 1
            self.id = Base.__nb_objects

    @classmethod
    def save_to_file_csv(cls, list_objs):
        """
        Write the CSV representation of list_objs to a file.

        Args:
            list_objs (list): List of instances to be written to the file.
        """
        csv_file = cls.__name__ + ".csv"
        fieldnames = ['id', 'width', 'height', 'x', 'y'] if cls.__name__ == "Rectangle" else ['id', 'size', 'x', 'y']

        with open(csv_file, "w") as file:
            writer = csv.DictWriter(file, fieldnames=fieldnames)
            writer.writeheader()
            writer.writerows(obj.to_dictionary() for obj in list_objs) if list_objs else []

    @classmethod
    def load_from_file_csv(cls):
        """
        Load a list of instances from a CSV file.

        Returns:
            list: List of instances.
        """
        csv_file = cls.__name__ + ".csv"
        fieldnames = ['id', 'width', 'height', 'x', 'y'] if cls.__name__ == "Rectangle" else ['id', 'size', 'x', 'y']

        try:
            with open(csv_file, "r") as file:
                reader = csv.reader(file, delimiter=',')
                return [cls(**{field: int(value) for field, value in zip(fieldnames, row) if value}) for i, row in enumerate(reader) if i > 0]
        except FileNotFoundError:
            return []

    @classmethod
    def load_from_file(cls):
        """
        Load a list of instances from a JSON file.

        Returns:
            list: List of instances.
        """
        try:
            with open(str(cls.__name__) + ".json", "r") as file:
                l = cls.from_json_string(file.read())

            num_ins = len(l)
            instances = []
            for y in range(num_ins):
                instances.append(cls.create(**l[y]))

            return instances
        except FileNotFoundError:
            return []

    @classmethod
    def create(cls, **dictionary):
        """
        Create an instance with all attributes already set.

        Args:
            **dictionary: Dictionary containing the instance's attributes.

        Returns:
            Base: New instance with the given attributes.
        """
        if cls.__name__ == "Rectangle":
            dummy = cls(1, 1)
        else:
            dummy = cls(1)
        dummy.update(**dictionary)
        return dummy

    @classmethod
    def save_to_file(cls, list_objs):
        """
        Write the JSON representation of list_objs to a file.

        Args:
            list_objs (list): List of instances to be written to the file.
        """
        if list_objs is None or list_objs == []:
            l = []
        else:
            l = [i.to_dictionary() for i in list_objs]
        with open(cls.__name__ + ".json", "w") as file:
            file.write(cls.to_json_string(l))

    @staticmethod
    def from_json_string(json_string):
        """
        Return the list from the JSON string representation json_string.

        Args:
            json_string (str): JSON string representation.

        Returns:
            list: List from the JSON string representation.
        """
        if json_string is None or len(json_string) == 0:
            return json.loads("[]")
        return json.loads(json_string)

    @staticmethod
    def to_json_string(list_dictionaries):
        """
        Return the JSON string representation of list_dictionaries.

        Args:
            list_dictionaries (list): List of dictionaries.

        Returns:
            str: JSON string representation of list_dictionaries.
        """
        if list_dictionaries is None or len(list_dictionaries) == 0:
            return json.dumps([])
        return json.dumps(list_dictionaries)

    @staticmethod
    def draw(list_rectangles, list_squares):
        """
        Draw all Rectangles and Squares using the Turtle graphics module.

        Args:
            list_rectangles (list): List of Rectangle instances.
            list_squares (list): List of Square instances.
        """
        #tha screen
        screen = turtle.Screen()
        screen.setup(width=800, height=600)

        #za object
        t = turtle.Turtle()
        #speed
        t.speed(1)

        #draw the rectangle
        for rect in list_rectangles:
            t.penup()
            t.goto(rect.x, rect.y)
            t.pendown()
            t.forward(rect.width)
            t.left(90)
            t.forward(rect.height)
            t.left(90)
            t.forward(rect.width)
            t.left(90)
            t.forward(rect.height)
            t.left(90)

        #draw the square
        for square in list_squares:
            t.penup()
            t.goto(square.x, square.y)
            t.pendown()
            t.forward(square.size)
            t.left(90)
            t.forward(square.size)
            t.left(90)
            t.forward(square.size)
            t.left(90)
            t.forward(square.size)
            t.left(90)

        screen.exitonclick()


#!/usr/bin/env python

"""
    Read all fasta files and build a sorted OTU contingency
    table.
"""

from __future__ import print_function

__author__ = "Frédéric Mahé <frederic.mahe@cirad.fr>, edited by Joe Vineis <jvineis@gmail.com>"
__date__ = "2022/03/15"
__version__ = "1.0"

import os
import re
import sys
import operator
import argparse

#*****************************************************************************#
#                                                                             #
#                                  Functions                                  #
#                                                                             #
#*****************************************************************************#

parser = argparse.ArgumentParser(description='''This script takes the output returned from swarmv3.1 - when run like this "swarm -d 1 -f -t 40 -z pooled-samples-derep.fa -i pooled-samples-derep-struct.txt -s pooled-samples-derep-stats.txt -w pooled-samples-node-representatives.fa -o pooled-samples-node-table.txt"''')
parser.add_argument('-repfa', help = "The fasta of represenataive sequences for each swarm (-w flag) and then sorted using vsearch")
parser.add_argument('-stats' , help = "The swarm stats file (-s flag)")
parser.add_argument('-swarms', help = "The swarms table output of the -o flag in swarm")
parser.add_argument('-l' , help = "A single column list of the names of the dereplicated fasta files-should look something like this ERR562370-primer-derep - and the corresponding fasta file should look like this ERR562370-primer-derep.fa")
args = parser.parse_args()

def representatives_parse():
    """
    Get seed sequences.
    """
    separator = ";size="
    representatives_file = args.repfa
    representatives = dict()
    with open(representatives_file, "r") as representatives_file:
        for line in representatives_file:
            if line.startswith(">"):
                amplicon = line.strip(">;\n").split(separator)[0]
            else:
                representatives[amplicon] = line.strip()

    return representatives

def stats_parse():
    """
    Map OTU seeds and stats.
    """
    separator = "\t"
    stats_file = args.stats
    stats = dict()
    seeds = dict()
    with open(stats_file, "r") as stats_file:
        for line in stats_file:
            cloud, mass, seed, seed_abundance = line.strip().split(separator)[0:4]
            stats[seed] = int(mass)
            seeds[seed] = (int(seed_abundance), int(cloud))
    # Sort OTUs by decreasing mass
    sorted_stats = sorted(stats.items(),
                          key=operator.itemgetter(1, 0))
    sorted_stats.reverse()

    return stats, sorted_stats, seeds

def swarms_parse():
    """
    Map OTUs.
    """
    separator = "_[0-9000000]+|;size=[0-9000000]+;?| "  # parsing of abundance annotations
    swarms_file = args.swarms
    swarms = dict()
    with open(swarms_file, "r") as swarms_file:
        for line in swarms_file:
            line = line.strip()
            amplicons = re.split(separator, line)[0::2]
            seed = amplicons[0]
            swarms[seed] = [amplicons]

    return swarms

def fasta_parse():
    """
    Map amplicon ids, abundances and samples. These must be dereplicated fastas.
    """
    separator = ";size="
    fasta_files = open(args.l, 'r')
    samples = dict()
    amplicons2samples = dict()
    for fasta_file in fasta_files:
        fa = fasta_file.strip()+".fa"
        sample = fasta_file.split("-")[0]
        samples[sample] = samples.get(sample, 0) + 1
        with open(fa, "r") as fasta_file:
            for line in fasta_file:
                if line.startswith(">"):
                    amplicon, abundance = line.strip(">;\n").split(separator)
                    abundance = int(abundance)
                    if amplicon not in amplicons2samples:
                        amplicons2samples[amplicon] = {sample: abundance}
                    else:
                        # deal with duplicated samples
                        amplicons2samples[amplicon][sample] = amplicons2samples[amplicon].get(sample, 0) + abundance
    # deal with duplicated samples
    duplicates = [sample for sample in samples if samples[sample] > 1]
    if duplicates:
        print("Warning: some samples are duplicated", file=sys.stderr)
        print("\n".join(duplicates), file=sys.stderr)
    samples = sorted(samples.keys())

    return amplicons2samples, samples

def print_table(representatives, stats, sorted_stats,
                swarms, amplicons2samples,
                samples, seeds):
    """
    Export results.
    """
    # Print table header
    print("OTU", "total", "cloud",
          "amplicon", "length", "abundance",
          "spread",
          "sequence",
          "\t".join(samples),
          sep="\t", file=sys.stdout)

    # Print table content
    i = 1
    for seed, abundance in sorted_stats:
        sequence = representatives[seed]
        occurrences = dict([(sample, 0) for sample in samples])
        for amplicons in swarms[seed]:
            for amplicon in amplicons:
                for sample in samples:

                    occurrences[sample] += amplicons2samples[amplicon].get(sample, 0)
        spread = len([occurrences[sample] for sample in samples if occurrences[sample] > 0])
        sequence_abundance, cloud = seeds[seed]

        # output
        print(i, abundance, cloud,
              seed, len(sequence), sequence_abundance,
              spread, sequence,
              "\t".join([str(occurrences[sample]) for sample in samples]),
              sep="\t", file=sys.stdout)
        i += 1

    return

def main():
    """
    Read all fasta files and build a sorted OTU contingency table.
    """
    # Parse the representative sequences for each swarm
    representatives = representatives_parse()

    # Parse stats
    stats, sorted_stats, seeds = stats_parse()

    # Parse swarms
    swarms = swarms_parse()

    # Parse fasta files
    amplicons2samples, samples = fasta_parse()

    # Print table header
    print_table(representatives, stats, sorted_stats, swarms,
                amplicons2samples, samples,
                seeds)

    return

#*****************************************************************************#
#                                                                             #
#                                     Body                                    #
#                                                                             #
#*****************************************************************************#

if __name__ == '__main__':

    main()

sys.exit(0)


#########################################
# AB Testing
#########################################

# -------------------------------------------------------------------------------------------------------------------

# Business Problem

""""
Company X has recently introduced a new type of bidding, average bidding,
as an alternative to the current type of bidding called maximum bidding.
One of our clients, bombabomba.com, decided to test this new feature and
wants to do an A/B test to see if average bidding converts more than maximum bidding.
"""

# Dataset Story:
""""
In this dataset, which includes the website information of bombabomba.com,
there is information such as the number of advertisements that users see and click,
as well as earnings information from here.

There are two separate data sets, the control and test groups.

The max binding strategy was presented to the control group, and the average binding strategy was presented to the test group.
"""

# Features:

# Total Features : 4
# Total Row : 40
# CSV File Size : 26 KB

""""
- Impression : Ad views count
- Click : The number of clicks on the displayed ad
- Purchase : The number of products purchased after the ads clicked
- Earning : Earnings after purchased products
"""

# -------------------------------------------------------------------------------------------------------------------

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import shapiro, levene, ttest_ind

pd.set_option('display.max_columns', None)
pd.set_option('display.float_format', lambda x: '%.4f' % x)

A_ = pd.read_excel("ab_testing.xlsx", sheet_name="Control Group")
B_ = pd.read_excel("ab_testing.xlsx", sheet_name="Test Group")

# Missing Value Analysis - Function
def missing_values_analysis(df):
    na_columns_ = [col for col in df.columns if df[col].isnull().sum() > 0]
    n_miss = df[na_columns_].isnull().sum().sort_values(ascending=True)
    ratio_ = (df[na_columns_].isnull().sum() / df.shape[0] * 100).sort_values(ascending=True)
    missing_df = pd.concat([n_miss, np.round(ratio_, 2)], axis=1, keys=['Total Missing Values', 'Ratio'])
    missing_df = pd.DataFrame(missing_df)
    return missing_df

# Let's define the auxiliary functions that we will use in the study
def check_df(df, head=5, box=False, column="Purchase"):
    print("--------------------- Shape ---------------------")
    print(df.shape)

    print("---------------------- Types --------------------")
    print(df.dtypes)

    print("--------------------- Head ---------------------")
    print(df.head(head))

    print("--------------------- Missing Value Analysis ---------------------")
    print(missing_values_analysis(df))

    print("--------------------- Quantiles ---------------------")
    print(df.quantile([0, 0.05, 0.50, 0.95, 0.99, 1]).T)

    print("--------------------- BOX PLOT ---------------------")
    if box == True:
        sns.boxplot(x=df[column])
        print(plt.show())

check_df(A_,box=True)

check_df(B_,box=True)

""""
The success criterion for Bombomba.com is the Number of Purchases.
For this reason, we will examine the Procurement variable.
There are no missing values in either dataset.
Also, there are no outliers in the Number of Purchases.

-----> Everything is going well...

"""

#--------------------------------------------------------------------

# Let's start A/B TEST
""""
It is used when it is desired to make a comparison between the mean of two groups.

1. Assumption Check¶
    1- Normality Assumption
    2- Variance Homogeneity

2. Implementation of the Hypothesis
    1- Independent two-sample t-test (parametric test) if assumptions are met.
    2- Mannwhitneyu test if assumptions are not met (non-parametric test).

If the normality is not provided, the mannwhitneyu test should be applied directly.
If the normality assumption is provided but the variance homogeneity is not provided,
the equal_var parameter can be set to False for the two-sample t-test.
"""

# Let's name them clearly, as we're going to merge and organize the datasets.

A_.columns = [i+"_A" for i in A_.columns]
A_.head()

B_.columns = [i+"_B" for i in B_.columns]
B_.head()

# Merge the dataset
df = pd.concat([A_, B_], axis=1)
df.head()

""""
   Impression_A   Click_A  Purchase_A  Earning_A  Impression_B   Click_B     Purchase_B  Earning_B
0    82529.4593 6090.0773    665.2113  2311.2771   120103.5038 3216.5480     702.1603    1939.6112
1    98050.4519 3382.8618    315.0849  1742.8069   134775.9434 3635.0824     834.0543    2929.4058
2    82696.0235 4167.9657    458.0837  1797.8274   107806.6208 3057.1436     422.9343    2526.2449
3   109914.4004 4910.8822    487.0908  1696.2292   116445.2755 4650.4739     429.0335    2281.4286
4   108457.7626 5987.6558    441.0340  1543.7202   145082.5168 5201.3877     749.8604    2781.6975
"""

# Hypotheses

""""
---- OUR HYPOTHESIS H0 AND H1 ----

H0: M1 = M2
--> There is no statistical difference between the average purchase earned,
    by the maximum binding strategy and the average purchase achieved by the average binding strategy.

H1: M1 != M2
--> There is a statistical difference between the average purchase earned,
    by the maximum binding strategy and the average purchases earned by the average binding strategy.
"""

print(" Mean of purchase of control group: %.4f" %A_['Purchase'].mean(), "\n",
      "Mean of purchase of test group: %.4f"  %B_['Purchase'].mean())

# OUT:  Mean of purchase of control group: 550.8941
# OUT:   Mean of purchase of test group: 582.1061

""""
There is a mathematical difference when looking at the purchasing rates for the two groups.
Group B, the average binding strategy, seems to be more successful.
But we do not know whether this difference is statistically significant.
To understand this, we must apply hypothesis testing.
"""

# Assumption Check

""""
1. Normality Assumption
    -H0: There is no statistically significant difference between sample distribution and theoretical normal distribution

    -H1: There is statistically significant difference between sample distribution and theoretical normal distribution

The test rejects the hypothesis of normality when the p-value is less than or equal to 0.05.
We do not want to reject the null hypothesis in the tests that might be considered for assumptions.

p-value < 0.05 (H0 rejected)
p-value > 0.05 (H0 not rejected)
"""

# Shapiro-Wilks Test for Control Group
test_st , p_value = shapiro(A_["Purchase_A"])
print('Test statistic = %.4f, P-Value = %.4f' % (test_st, p_value))

# OUT : Test statistic = 0.9773, P-Value = 0.5891

""""
*** P-VALUE = 0.5891 ***
--> Since the p value is not less than 0.05,
the h0 hypothesis cannot be rejected. That is, the assumption of normality is provided for the control group.
"""

# Shapiro-Wilks Test for Control Group
test_st , p_value = shapiro(B_["Purchase_B"])
print('Test statistic = %.4f, P-Value = %.4f' % (test_st, p_value))

# OUT: Test statistic = 0.9589, p-Value = 0.1541

""""
*** P-VALUE = 0.1541 ***
--> Since the p value is not less than 0.05,
the h0 hypothesis cannot be rejected. That is, the assumption of normality is provided for the control group.
"""

"""
1. Variance Assumption

    -H0: the compared groups have equal variance.
    -H1: the compared groups do not have equal variance.

We do not want to reject the null hypothesis in the tests that might be considered for assumptions.

p-value < 0.05 (H0 rejected)
p-value > 0.05 (H0 not rejected)
"""

# Levene Test
test_st, p_value = levene(df["Purchase_A"],df["Purchase_B"])
print('Test statistic = %.4f, p-Value = %.4f' %(test_st, p_value))

# OUT : Test statistic = 2.6393, p-Value = 0.1083

""""
*** P-VALUE = 0.1083 ***
--> Since the p value is not less than 0.05,
the h0 hypothesis cannot be rejected. That is, the assumption of variance is provided.
"""

# Assumptions provided. Thus, we can apply the independent two-sample t-test (parametric test).

test_st, p_value = ttest_ind(df["Purchase_A"], df["Purchase_B"], equal_var=True)
print('tvalue = %.4f, pvalue = %.4f' %(test_st, p_value))

# OUT : tvalue = -0.9416, pvalue = 0.3493

# Result :
"""
**** P-VALUE = 0.3493 Since the p value is not less than 0.05, the h0 hypothesis cannot be rejected.
---> So, There is no statistically significant difference between, the Control( “maximum bidding”)
     campaign and Test group(average bidding) campaign.
"""


from matplotlib import pyplot as plt
import matplotlib

font = {'family':'MicroSoft YaHei',
        'weight':'bold',
        'size':12}
matplotlib.rc("font",**font)
matplotlib.rc("font",family='MicroSoft YaHei',weight="bold",size=8)

y_1 = [1,0,1,1,2,4,3,2,3,4,4,5,6,5,4,3,3,1,1,1]
y_2 = [1,0,3,1,2,2,3,3,2,1,2,1,1,1,1,1,1,1,1,1]

x = range(11,31)

# 绘制时指定：color,linestyle,linewidth,alpha
# color='r';r红色；g绿色；b蓝色；w白色；c青色；y黄色；k黑色；
# linesytle='--'; 线条风格
# linewidth=5; 线条粗细
# alpha=0.5； 线条透明度

plt.figure(figsize=(15,7),dpi=80)

plt.plot(x,y_1,label="小红",color='k',linestyle='--',linewidth=6)
plt.plot(x,y_2,label="小兰",color='y')

_xtick_labels = ["{}岁".format(i) for i in x]
plt.xticks(x,_xtick_labels)
plt.yticks(range(0,8))

# 绘制网格,设置网格透明度（0-1）
plt.grid(alpha=0.5)

# 添加图例,以及图例的位置
plt.legend(loc="upper left")

plt.show()


import pdfplumber

def get_text(source_document):
    l = []
    txt_l = []
    invoice_lines = []
    txt =""
    with pdfplumber.open(source_document) as pdf:
        for page in pdf.pages:
            text = page.extract_text()
            # table = page.extract_table()
            # tables = page.extract_tables()
            # words = page.extract_words()
            # print(words)
            # for i in words:
            #     l.append(i["text"])
            # l.extend(words)
            txt_l.extend(text.splitlines())
            # print(text.)
            # print(table)
            # print(tables)
            # print(words)

        for text_line in txt_l:
            if text_line[0]!="(":
                invoice_lines.append(text_line)

        for lines in invoice_lines:
            txt=txt+lines+"\n"

    return txt


from typing import Any, List

import lightning.pytorch as pl
from lightning.pytorch.strategies import (DDPStrategy, DeepSpeedStrategy,
                                          SingleDeviceStrategy,
                                          SingleTPUStrategy, Strategy)
from lightning.pytorch.trainer import Trainer

process_independent_strategies: List[Strategy] = [
    DDPStrategy,
    DeepSpeedStrategy,
]
fully_dependent_strategies: List[Strategy] = [SingleDeviceStrategy]

tpu_strategies: List[Strategy] = [SingleTPUStrategy]

supported_strategies: List[Strategy] = (
    process_independent_strategies + fully_dependent_strategies
)

def get_global_batch_size_in_trainer(
    local_batch_size: int,
    trainer: Trainer,
) -> int:
    """Get global batch size used by a trainer based on the local batch size.

    Args:
        local_batch_size: The local batch size used by the trainer.
        trainer: The trainer used.

    Raises:
        AttributeError: The strategy is not supported.

    Returns:
        The global batch size.
    """

    strategy = get_trainer_strategy(trainer)
    devices = trainer.num_devices
    num_nodes = trainer.num_nodes
    if not any(
        [
            isinstance(strategy, supported_strategy)
            for supported_strategy in supported_strategies
        ]
    ):
        raise AttributeError(f"Strategy {strategy} not supported.")
    else:
        if any([isinstance(strategy, tpu_strategy) for tpu_strategy in tpu_strategies]):
            return local_batch_size * devices
        elif any(
            [
                isinstance(strategy, process_independent_strategy)
                for process_independent_strategy in process_independent_strategies
            ]
        ):
            return local_batch_size * devices * num_nodes
        elif any(
            [
                isinstance(strategy, fully_dependent_strategy)
                for fully_dependent_strategy in fully_dependent_strategies
            ]
        ):
            return local_batch_size
        else:
            raise AttributeError(f"Strategy {strategy} not supported.")

def get_local_batch_size_in_trainer(
    global_batch_size: int,
    trainer: Trainer,
) -> int:
    """Get local batch size used by a trainer based on the global batch size.

    Args:
        global_batch_size: The global batch size used by the trainer.
        strategy: The trainer used.

    Raises:
        AttributeError: The strategy is not supported.

    Returns:
        The local batch size.
    """
    strategy = get_trainer_strategy(trainer)
    devices = trainer.num_devices
    num_nodes = trainer.num_nodes
    if not any(
        [
            isinstance(strategy, supported_strategy)
            for supported_strategy in supported_strategies
        ]
    ):
        raise AttributeError(f"Strategy {strategy} not supported.")
    else:
        if any([isinstance(strategy, tpu_strategy) for tpu_strategy in tpu_strategies]):
            return global_batch_size // devices
        elif any(
            [
                isinstance(strategy, process_independent_strategy)
                for process_independent_strategy in process_independent_strategies
            ]
        ):
            return global_batch_size // devices // num_nodes
        elif any(
            [
                isinstance(strategy, fully_dependent_strategy)
                for fully_dependent_strategy in fully_dependent_strategies
            ]
        ):
            return global_batch_size
        else:
            raise AttributeError(f"Strategy {strategy} not supported.")

def get_num_devices_in_trainer(
    trainer: Trainer,
) -> int:
    """Get the number of devices used by the trainer.

    Args:
        trainer: The trainer.

    Raises:
        AttributeError: The strategy used by trainer is not supported

    Returns:
        The number of devices used by trainer.
    """
    strategy = get_trainer_strategy(trainer)
    if not any(
        [
            isinstance(strategy, supported_strategy)
            for supported_strategy in supported_strategies
        ]
    ):
        raise AttributeError(f"Strategy {strategy} not supported.")
    else:
        if any([isinstance(strategy, tpu_strategy) for tpu_strategy in tpu_strategies]):
            return trainer.num_devices
        elif any(
            [
                isinstance(strategy, process_independent_strategy)
                for process_independent_strategy in process_independent_strategies
            ]
        ):
            return trainer.num_devices * trainer.num_nodes
        elif any(
            [
                isinstance(strategy, fully_dependent_strategy)
                for fully_dependent_strategy in fully_dependent_strategies
            ]
        ):
            return 1
        else:
            raise AttributeError(f"Strategy {strategy} not supported.")

def get_trainer_strategy(trainer: Trainer) -> Any:
    """Retrieve the strategy from a trainer.

    Args:
        trainer: The trainer.

    Returns:
        The strategy.
    """
    if pl.__version__ < "1.6.0":
        return trainer.training_type_plugin
    else:
        return trainer.strategy

def is_strategy_ddp(strategy: Any) -> bool:
    """Test if strategy is ddp.

    Args:
        strategy: The strategy.

    Returns:
        ``True`` if strategy is ddp.
    """
    return any(
        [
            isinstance(strategy, process_strategy)
            for process_strategy in process_independent_strategies
        ]
    )

def is_strategy_tpu(strategy: Any) -> bool:
    """Test if strategy is tpu.

    Args:
        strategy: The strategy.

    Returns:
        ``True`` if strategy is tpu.
    """

    return any([isinstance(strategy, tpu_strategy) for tpu_strategy in tpu_strategies])


from PyQt5.QtWidgets import QApplication
from services.QueriesService import QueriesService
from windows.MainWindow import MainWindow

def exec_sql_initial():
    service = QueriesService()
    service.run_sql_initial()
    service.get_configuration_smartband()

exec_sql_initial()
app = QApplication([])
main_window = MainWindow()
main_window.show()
app.exec_()


import boto3,yaml
import botocore
from sys import argv,exit,stdout

DOWNLOADED_DATA = 0;

def getopts(argv):
    opts = {}  # Empty dictionary to store key-value pairs.
    while argv:  # While there are arguments left to parse...
        if argv[0][0] == '-':  # Found a "-name value" pair.
            opts[argv[0]] = argv[1]  # Add key and value to the dictionary.
        argv = argv[1:]  # Reduce the argument list by copying it starting from index 1.
    return opts

args = getopts(argv)

with open('config.yaml') as f:
      data = yaml.safe_load(f);

if('-awsKey' not in args and 'awsKey' not in data):
      print("aws access key required (-awsKey)");
      exit(1);

if('-awsSecret' not in args and 'awsSecret' not in data):
      print("aws access secret required (-awsSecret)");
      exit(1);

if('-bucket' not in args and 'bucket' not in data):
      print("bucket name required (-bucket)");
      exit(1);

if('-fileKey' not in args):
      print("download file key required (-fileKey)");
      exit(1);

awsKey = args['-awsKey'] if '-awskey' in args  else data['awsKey'];
awsSecret = args['-awsSecret'] if '-awsSecret' in args else data['awsSecret'];
awsBucket = args['-bucket'] if '-bucket' in args else data['bucket'];

AWS_ACCESS_KEY_ID = awsKey;
AWS_SECRET_ACCESS_KEY = awsSecret;
bucket_name = awsBucket;

BUCKET_NAME = awsBucket; # replace with your bucket name
KEY = args['-fileKey']; # replace with your object key

s3 = boto3.client(
    's3',
    aws_access_key_id=AWS_ACCESS_KEY_ID,
    aws_secret_access_key=AWS_SECRET_ACCESS_KEY
)

def callback(inp):
    global DOWNLOADED_DATA;
    DOWNLOADED_DATA = DOWNLOADED_DATA + inp;
    stdout.write("\rdownloaded " + str(DOWNLOADED_DATA/1024) + "kb");
    stdout.flush();
   # print(str(inp/1024) + "Kb");

try:
    s3.download_file(BUCKET_NAME, args['-fileKey'],args['-fileKey'],None,callback);
except botocore.exceptions.ClientError as e:
    if e.response['Error']['Code'] == "404":
        print("The object dose not exist.");
        exit(404);
    else:
        print(e.response);
        exit(405);


from pyspark.sql.functions import dayofweek, hour

weekend_zeros = df.filter((hour("Time (Local)") >= 17) | (hour("Time (Local)") < 3)).filter((dayofweek("Date") == 6) | (dayofweek("Date") == 7)).agg(sum("Zeroes").alias("weekend_zeros")).collect()[0]["weekend_zeros"]

total_zeros = df.agg(sum("Zeroes").alias("total_zeros")).collect()[0]["total_zeros"]

percent_weekend_zeros = weekend_zeros / total_zeros * 100

print("The percentage of zeros that occurred on weekends is:", percent_weekend_zeros, "%")

#The percentage of zeros that occurred on weekends is: 41.333414829040026 %


# -*- coding: utf-8 -*-
"""
Created on Mon Nov 30 13:33:52 2020

@author: atish
"""

from nearpy import Engine
from nearpy.hashes import RandomBinaryProjections
from nearpy.filters import NearestFilter, UniqueFilter
import numpy as np
import os
import pandas as pd
import time
import tracemalloc
import matplotlib.pyplot as plt
nodes = []
edges = []
timeused = []
memory = []

import os
os.chdir('C:/Users/atish/Downloads/Assignments/Assignments' +\
         '/social network analysis/project')

directory_list = os.listdir()

plot_data = []
for file in directory_list:

    matrix = pd.read_csv(directory_list[1])
    n = [round(len(matrix)/100), round(len(matrix)/50), round(len(matrix)/25),
         round(len(matrix)/5)]

    for number in n:
        matrix = pd.read_csv(directory_list[0], nrows=number)

        matrix.columns = ['node1', 'node2']
        matrix = matrix.astype(int)
        t1 = time.time()
        tracemalloc.start()
        #matrix = matrix.iloc[:number, :]

        val = matrix.drop_duplicates().node2.value_counts()
        val = val.sort_index()

        matrix['node2'] = pd.to_numeric(matrix['node2'], downcast='signed')
        matrix['node1'] = pd.to_numeric(matrix['node1'], downcast='signed')
        data = pd.crosstab(matrix['node2'], matrix['node1'])
        dimension = data.shape[1]
        rbp = RandomBinaryProjections('rbp', 2)
        engine = Engine(dimension, lshashes=[rbp])
        nodes.append(len(data))
        edges.append(len(matrix))

        for index in range(len(data)):
            v = np.array(data.iloc[index, :])
            engine.store_vector(v, 'data_%d' % index)

        current, peak = tracemalloc.get_traced_memory()
        print(f"Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB")
        tracemalloc.stop()
        memory.append(peak / 10**6)
        tracemalloc.start()

        query_data = data.index

        N = []
        for i in range(len(query_data)):
            query = data[data.index == query_data[i]].iloc[0, :]
            query = pd.to_numeric(query, downcast='integer')
            query = np.array(query)
            N.append(engine.neighbours(query))

        t2 = time.time()

        total_time = t2-t1

        current, peak = tracemalloc.get_traced_memory()
        print(f"Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB")
        print("Total time taken is " + str(round(total_time)) + " seconds")
        tracemalloc.stop()

        timeused.append(round(total_time))

nodet = plt.plot( timeused[4:], nodes[4:], label='Nodes explored')
edget = plt.plot( timeused[4:], edges[4:], label='Edges explored')
memt = plt.plot( timeused[4:], memory[4:], label='Memory Used')
plt.legend()

plt.ylabel('log scale for nodes edges and memory')
plt.yscale('log')

plt.xlabel('Time in seconds')
plt.title('Nearpy algorithm - nodes, edges and memory in graph Vs Total time taken')
plt.savefig('enron.png')

plt.plot(edges, timeused)
plt.ylabel('Number of edges in a graph')
plt.xlabel('Time in seconds')
plt.title('Nearpy algorithm - nodes, edges and memory in graph Vs Total time taken')
plt.savefig('usermovie edges time.png')

Dataset = matrix
a = matrix['node1'].unique()

Dataset['node2'] = Dataset['node2'].astype(int)
Dataset['node1'] = Dataset['node1'].str.replace('_', '')
from keras.preprocessing.text import one_hot
matrix_one_hot = []

for text in (Dataset['node1']):
    try:
        matrix_one_hot.append(one_hot(str(text),len(a))[0])
    except:
        matrix_one_hot.append(0)

matrix['node1'] = Dataset['node1']
matrix['node2'] = Dataset['node2']

matrix.to_csv('large_noweights.csv', index=False)

matrix.columns = matrix.columns.str.replace('^ +| +$', '_')
matrix.columns = matrix.columns.str.replace('^ +', '_')
matrix.columns = matrix.columns.str.replace(' +$', '_')
matrix['node2'] = matrix['node2'].astype(str)

matrix['node2'] = matrix['node2'].replace('28487, 11896, 27217', '28487')

Data = pd.read_csv('enron_gephi.csv')


# Author: Siddharth Palani Natarajan

# ME 468 - Traffic Sign Detection for Autonomous Vehicle

#########################################################

# =============================================================================
# PROJECT CHRONO - http:#projectchrono.org
#
# Copyright (c) 2014 projectchrono.org
# All right reserved.
#
# Use of this source code is governed by a BSD-style license that can be found
# in the LICENSE file at the top level of the distribution and at
# http:#projectchrono.org/license-chrono.txt.
#
# =============================================================================

import pychrono.core as chrono
import pychrono.vehicle as veh
import pychrono.sensor as sens
import math
import os
import numpy as np
import matplotlib.pyplot as plt
import cv2
import pickle

# =============================================================================

# The path to the Chrono data directory containing various assets (meshes, textures, data files)
# is automatically set, relative to the default location of this demo.
# If running from a different directory, you must change the path to the data directory with:
chrono.SetChronoDataPath(os.getenv("CHRONO_DATA_DIR"))
veh.SetDataPath(chrono.GetChronoDataPath() + 'vehicle/')

# ---------------------
# Simulation parameters
# ---------------------

# Initial vehicle location and orientation
initLoc = chrono.ChVectorD(0, 0, 0.4)
initRot = chrono.ChQuaternionD(1, 0, 0, 0)
finalLoc = chrono.ChVectorD(100, 0, 0.4)
# Visualization type for vehicle parts (PRIMITIVES, MESH, or NONE)
chassis_vis_type = veh.VisualizationType_MESH
suspension_vis_type = veh.VisualizationType_PRIMITIVES
steering_vis_type = veh.VisualizationType_PRIMITIVES
wheel_vis_type = veh.VisualizationType_NONE
tire_vis_type = veh.VisualizationType_MESH

# Simulation step sizes
step_size = 1e-3
tire_step_size = step_size

# Simulation end time
tend = 30

# view camera images
vis = True
save = False
noise = False

# =============================================================================

# =============================================================================
# =============================================================================
# Create signs

def AddFixedObstacles(system) :
    # Create contact material, of appropriate type. Use default properties
    matNSC = chrono.ChMaterialSurfaceNSC()
    matSMC = chrono.ChMaterialSurfaceSMC()
    material = matNSC
    material2 = matSMC

    #########asphalt
    aspha = chrono.ChBodyEasyBox(10000,2,1,0.1,True,False,material2)
    aspha.SetPos(chrono.ChVectorD(0,0,0.0001))
    aspha.SetBodyFixed(True)

    aspha_asset = aspha.GetAssets()[0]
    visual_asset = chrono.CastToChVisualization(aspha_asset)

    vis_mat = chrono.ChVisualMaterial()
    vis_mat.SetKdTexture("/srv/home/yrhee6/ME468/PJ/textures/floor.png")
    visual_asset.material_list.append(vis_mat)

    system.AddBody(aspha)

    #####speed signs 30
    speedsign = chrono.ChBodyEasyCylinder(1, 0.2, 0.1, True, True, material)
    speedsign.SetPos(chrono.ChVectorD(30,-5,5))
    speedsign.SetRot(chrono.Q_from_AngAxis(3.141592/2, chrono.VECT_Z))

    speedsign.SetBodyFixed(True)
    speedsign_asset = speedsign.GetAssets()[0]
    visual_asset = chrono.CastToChVisualization(speedsign_asset)

    vis_mat = chrono.ChVisualMaterial()

    vis_mat.SetKdTexture("/srv/home/yrhee6/ME468/PJ/textures/30.png")
    visual_asset.material_list.append(vis_mat)
    system.AddBody(speedsign)

    #####speed signs 60
    speedsign = chrono.ChBodyEasyCylinder(1, 0.2, 0.1, True, True, material)
    speedsign.SetPos(chrono.ChVectorD(60,-5,5))
    speedsign.SetRot(chrono.Q_from_AngAxis(3.141592/2, chrono.VECT_Z))

    speedsign.SetBodyFixed(True)
    speedsign_asset = speedsign.GetAssets()[0]
    visual_asset = chrono.CastToChVisualization(speedsign_asset)

    vis_mat = chrono.ChVisualMaterial()

    vis_mat.SetKdTexture("/srv/home/yrhee6/ME468/PJ/textures/60.png")
    visual_asset.material_list.append(vis_mat)
    system.AddBody(speedsign)

    #####speed signs 0
    speedsign = chrono.ChBodyEasyCylinder(1, 0.2, 0.1, True, True, material)
    speedsign.SetPos(chrono.ChVectorD(90,-5,5))
    speedsign.SetRot(chrono.Q_from_AngAxis(3.141592/2, chrono.VECT_Z))

    speedsign.SetBodyFixed(True)
    speedsign_asset = speedsign.GetAssets()[0]
    visual_asset = chrono.CastToChVisualization(speedsign_asset)

    vis_mat = chrono.ChVisualMaterial()

    vis_mat.SetKdTexture("/srv/home/yrhee6/ME468/PJ/textures/stop.png")
    visual_asset.material_list.append(vis_mat)
    system.AddBody(speedsign)

print( "Copyright (c) 2017 projectchrono.org\n")

# --------------
# Vehicle System
# --------------

# Create the vehicle, set parameters, and initialize
gator = veh.Gator()
gator.SetContactMethod(chrono.ChContactMethod_NSC)
gator.SetChassisFixed(False)
gator.SetInitPosition(chrono.ChCoordsysD(initLoc, initRot))
gator.SetBrakeType(veh.BrakeType_SHAFTS)
gator.SetTireType(veh.TireModelType_TMEASY)
gator.SetTireStepSize(tire_step_size)
gator.SetInitFwdVel(0.0)
gator.Initialize()

gator.SetChassisVisualizationType(chassis_vis_type)
gator.SetSuspensionVisualizationType(suspension_vis_type)
gator.SetSteeringVisualizationType(steering_vis_type)
gator.SetWheelVisualizationType(wheel_vis_type)
gator.SetTireVisualizationType(tire_vis_type)

# ------------------
# Terrain
# ------------------
terrain = veh.RigidTerrain(gator.GetSystem())
patch_mat = chrono.ChMaterialSurfaceNSC()
patch_mat.SetFriction(0.9)
patch_mat.SetRestitution(0.01)
patch = terrain.AddPatch(patch_mat,
                         chrono.ChVectorD(0, 0, 0), chrono.ChVectorD(0, 0, 1),
                         600, 600)
patch.SetColor(chrono.ChColor(0.8, 0.8, 1.0))
patch.SetTexture(veh.GetDataFile("terrain/textures/tile4.jpg"), 1200, 1200)
terrain.Initialize()

asset = patch.GetGroundBody().GetAssets()[0]
visual_asset = chrono.CastToChVisualization(asset)

vis_mat = chrono.ChVisualMaterial()
vis_mat.SetKdTexture(chrono.GetChronoDataFile("sensor/textures/mud.png"))
vis_mat.SetRoughness(0.99)

visual_asset.material_list.append(vis_mat)

# -----------------------
# Sensor Manager
# -----------------------
manager = sens.ChSensorManager(gator.GetSystem())
intensity = 1.0
manager.scene.AddPointLight(chrono.ChVectorF(0, 0, 100), chrono.ChVectorF(intensity, intensity, intensity), 500.0)
b = sens.Background()
b.mode = sens.BackgroundMode_ENVIRONMENT_MAP
b.env_tex = chrono.GetChronoDataFile("sensor/textures/quarry_01_4k.hdr")
manager.scene.SetBackground(b)

##AddObstacles
AddFixedObstacles(gator.GetSystem())

# ------------------------------------------------
# Camera Sensor
# ------------------------------------------------

offset_pose = chrono.ChFrameD(chrono.ChVectorD(.1, 0, 1.45), chrono.Q_from_AngAxis(.2, chrono.ChVectorD(0, 1, 0)))

# Camera Sensor Parameters
cam_update_rate = 10
image_width = 1280
image_height = 720
fov = 1.5707963268
cam_lag = 0
cam_collection_time = 1. / float(cam_update_rate)

out_dir = "SENSOR_OUTPUT/SENSORS_PY"

cam = sens.ChCameraSensor(
    gator.GetChassisBody(),
    cam_update_rate,
    offset_pose,
    image_width,
    image_height,
    fov
)

cam.SetName("Camera Sensor")
cam.SetLag(cam_lag)
cam.SetCollectionWindow(cam_collection_time)

cam.PushFilter(sens.ChFilterRGBA8Access())
cam.PushFilter(sens.ChFilterSave(out_dir + "/rgb/"))

manager.AddSensor(cam)

# ------------------------------------------------
# GPS Sensor
# ------------------------------------------------

# GPS Sensor Parameters
gps_update_rate = 10
gps_reference = chrono.ChVectorD(-89.400, 43.070, 260.0)
gps_lag = 0
gps_collection_time = 0
noise_none = sens.ChNoiseNone()

offset_pose = chrono.ChFrameD(chrono.ChVectorD(.1, 0, 1.45), chrono.Q_from_AngAxis(.2, chrono.ChVectorD(0, 1, 0)))

gps = sens.ChGPSSensor(
    gator.GetChassisBody(),
    gps_update_rate,
    offset_pose,
    gps_reference,
    noise_none
)

gps.SetName("GPS Sensor")
gps.SetLag(gps_lag)
gps.SetCollectionWindow(gps_collection_time)
gps.PushFilter(sens.ChFilterGPSAccess())

manager.AddSensor(gps)

# ------------------------------------------------------
# Traffic Sign Prediction - Initialsiation and Functions
# ------------------------------------------------------

# #Import Trained
# picklepath = './Trained_model.p'
# pickle_in = open(picklepath, 'rb')
# print('Check1')
# print(pickle_in)
# print('Check2')
# model = pickle.load(pickle_in)

# Probability Threshold
threshold = 0.75

# Labelling Font
font = cv2.FONT_HERSHEY_TRIPLEX

""" Image Processing """

def grayscale(img):
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    return img

def equalize(img):
    img = cv2.equalizeHist(img)
    return img

def preprocessing(img):
    img = grayscale(img)
    img = equalize(img)
    img = img / 255
    return img

def getClassName(classNo):
    if classNo == 1:
        return '30'  # Speed Limit 30 km/hr
    elif classNo == 3:
        return '60'  # Speed Limit 30 km/hr
    elif classNo == 14:
        return '0'  # STOP

#create path follower
path = veh.StraightLinePath(initLoc,finalLoc)
driver = veh.ChPathFollowerDriver(gator.GetVehicle(),path,"mypath",45)
temp =driver.GetSteeringController()
temp.SetLookAheadDistance(5)
temp.SetGains(0.8,0,0)
driver.GetSpeedController().SetGains(0.4,0,0)

driver.Initialize()
# ---------------
# Simulation loop
# ---------------
time = 0.0

while (time < tend) :
    time = gator.GetSystem().GetChTime()

    # Collect output data from modules (for inter-module communication)

    driver_inputs = driver.GetInputs()

    ###This is just to show changing speed works########################
    currentspeed  = driver.GetSpeedController().GetCurrentSpeed()

    if currentspeed >30 and time >tend/4:
        driver_inputs.m_braking = 0.5
        driver_inputs.m_throttle = driver_inputs.m_throttle -0.3
    else:
        driver_intputs = driver.GetInputs()

    terrain.Synchronize(time)
    gator.Synchronize(time, driver_inputs, terrain)
    driver.Synchronize(time)

    manager.Update()

    buffer = cam.GetMostRecentRGBA8Buffer()
    cam_data = buffer.GetRGBA8Data()

    if cam_data.size != 0:
        imgOriginal = cv2.cvtColor(cam_data,cv2.COLOR_BGR2RGB)
        imgOriginal = cv2.rotate(imgOriginal,cv2.ROTATE_180)

    #     # Image Processing
    #     img = np.asarray(imgOriginal)
    #     img = cv2.resize(img, (32, 32))
    #     img = preprocessing(img)
    #     # cv2.imshow("Processed Image", img)
    #     img = img.reshape(1, 32, 32, 1)
    #     cv2.putText(imgOriginal, "Traffic Sign:", (20, 35), font, 0.75, (255, 0, 0), 1, cv2.LINE_AA)
    #     cv2.putText(imgOriginal, "Probability:", (20, 75), font, 0.75, (255, 0, 0), 1, cv2.LINE_AA)

    #     # Image Prediction
    #     predictions = model.predict(img)
    #     classIndex = model.predict_classes(img)
    #     probabilityValue = np.amax(predictions)

    #     if probabilityValue > threshold:
    #         cv2.putText(imgOriginal, str(getClassName(classIndex)), (190, 35), font, 0.75, (255, 0, 0), 1, cv2.LINE_AA)
    #         cv2.putText(imgOriginal, str(round(probabilityValue*100, 2)) + "%", (180, 75), font, 0.75, (255, 0, 0), 1, cv2.LINE_AA)

    #         # ROS2 - To be published to the Perception node for Planning #
    #         vehicle_speed = str(getClassName(classIndex))
    #         print(vehicle_speed)
    #     else:
    #         print(vehicle_speed)

    # Advance simulation for one timestep for all modules
    terrain.Advance(step_size)
    gator.Advance(step_size)
    driver.Advance(step_size)


import argparse
import os
import shutil

description = r'Moves files from one directory to another, based on a manifest'
parser = argparse.ArgumentParser(description=description)
parser.add_argument('manifest', type=str,
                    help='Manifest: relative paths, on separate lines')
parser.add_argument('source', type=str,
                    help='Source Directory')
parser.add_argument('destination', type=str,
                    help='Destination Directory')
parser.add_argument('-n', '--dryrun', action='store_true', default=False,
                    help='Display result of command without executing')
parser.add_argument('-v', '--verbose', action='store_true', default=False,
                    help='Enable verbose logging')

args = parser.parse_args()

src_dir = args.source
dest_dir = args.destination
manifest = args.manifest
dryrun = args.dryrun
verbose = args.verbose

with open(manifest, 'r', encoding='utf-8') as file_handle:
    for line in file_handle:
        trimmed = line.rstrip()

        src_path = os.path.join(src_dir, trimmed)
        dest_path = os.path.join(dest_dir, trimmed)

        # Notify if a file in the manifest is missing
        if not os.path.isdir(src_path) and not os.path.isfile(src_path):
            print('Error '+str(src_path)+' not found.')

        # Move file found in manifest if it exists
        if os.path.isfile(src_path):
            if verbose or dryrun:
                print(src_path+' -> '+dest_path)

            if not dryrun:
                os.makedirs(os.path.dirname(dest_path), exist_ok=True)
                shutil.move(src_path, dest_path)

# Cleanup
for path in os.listdir(src_dir):
    relative_path = os.path.join(src_dir, path)
    if os.path.isdir(relative_path) and not os.listdir(relative_path):
        if verbose or dryrun:
            print(relative_path+' empty, removing')

        if not dryrun:
            os.rmdir(relative_path)


import configparser
import socket
import threading
from time import sleep
import logging
import datetime

######################################

class Server:

    #Server Constructor
    def __init__(self):
        logging.basicConfig(filename='src/server/logs/proxy.log', level=logging.DEBUG, format= '%(asctime)s:%(thread)d:%(name)s:%(message)s', filemode = 'a' )

        logging.info("[SERVER BOOTING] : ...")
        logging.info( "Server start time" + str(datetime.datetime.utcnow) )
        self.create_config_file()

        self.config = configparser.ConfigParser()
        self.config.read('config.ini')

        self.dead = False

        self.HOST_NAME = self.config['SERVERCONFIG']['HOST_NAME']
        self.BIND_PORT = int(self.config['SERVERCONFIG']['BIND_PORT'])
        self.MAX_REQUEST_LEN = int(self.config['SERVERCONFIG']['MAX_REQUEST_LEN'])
        self.CONNECTION_TIMEOUT = int(self.config['SERVERCONFIG']['CONNECTION_TIMEOUT'])

        # Create a TCP socket
        self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

        # Re-use the socket
        self.serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

        # bind the socket to a public host, and a port
        self.serverSocket.bind((self.HOST_NAME, self.BIND_PORT))

        self.serverSocket.listen(10) # become a server socket
        # self.__clients = {}

        logging.info(f"[SERVER LISTENING] : Server listening on port {self.BIND_PORT}")

    #Method called by view to establish connection and allow listening
    def establish_connection(self):
        while True:
            #Check for flag if main thread should be killed
            if self.dead == True:
                logging.debug(f"[MAIN THREAD INTERRUPTED] : Terminating listening stream connection")
                break

            # Establish the connection
            (clientSocket, client_address) = self.serverSocket.accept()
            logging.info(f"[NEW CONNECTION] : {client_address} connected.")
            logging.debug(f"[ACTIVE THREAD COUNT] : {threading.active_count()}")

            d = threading.Thread(target = self.proxy_thread, args=(clientSocket, client_address))
            d.setDaemon(True)
            d.start()
            sleep(1)

    def domain_block(self, webserver):
        sites_to_block = ['www.github.com', 'github.com']

        for site in sites_to_block:
            if site == webserver:
                print("Site blocked. Cannot access.")
                return True
            else:
                return False

    #Cleans up request and sends data back and forth between client, server and web host
    def proxy_thread(self, clientSocket, client_address):
        # get the request from browser
        request = clientSocket.recv(self.MAX_REQUEST_LEN)
        logging.debug(f"[REQUEST LENGTH] : {len(request)} from : {client_address}")

        # parse the first line
        first_line = request.split(b'\n')[0]

        # get url
        url = first_line.split(b' ')[1]

        http_pos = url.find(b"://") # find pos of ://
        if (http_pos==-1):
            temp = url
        else:
            temp = url[(http_pos+3):] # get the rest of url

        port_pos = temp.find(b":") # find the port pos (if any)

        # find end of web server
        webserver_pos = temp.find(b"/")
        if webserver_pos == -1:
            webserver_pos = len(temp)

        webserver = ""
        port = -1
        if (port_pos==-1 or webserver_pos < port_pos):

            # default port
            port = 80
            webserver = temp[:webserver_pos]

        else: # specific port
            port = int((temp[(port_pos+1):])[:webserver_pos-port_pos-1])
            webserver = temp[:port_pos]

            ################################
        logging.info(f"Making connection with webserver: {url} from client {client_address}")

        if self.domain_block(webserver) == False:

            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.settimeout(self.CONNECTION_TIMEOUT)
            s.connect((webserver, port))
            s_addr = (webserver, port)
            s.sendall(request)

            #Sends data until data sent completely sent or thread interrupted
            while 1:
                #Check for flag if thread should be killed
                if self.dead == True:
                    logging.debug(f"[THREAD INTERRUPTED] : Terminated during data transmission")
                    break

                # receive data from web server
                logging.info(f"[SERVER] : Retrieving data from {s_addr}")
                data = s.recv(self.MAX_REQUEST_LEN)
                logging.info(f"[DATA] : {len(data)} data from {s_addr}")

                if (len(data) > 0):
                    logging.info(f"[SERVER] : Sending data to {client_address}")
                    clientSocket.send(data) # send to browser/client
                else:
                    logging.info(f"[SERVER] : Data Successfully sent to {client_address}")
                    break

            clientSocket.close()
            s.close()
        else:
            clientSocket.close()

    def create_config_file(self):
            logging.debug("Creating config_file")
            config_object = configparser.ConfigParser()

            config_object["SERVERCONFIG"] = {
            "HOST_NAME": "",
            "BIND_PORT": "4444",
            "MAX_REQUEST_LEN": "4096",
            "CONNECTION_TIMEOUT": "60"

            }

            #Write the above sections to config.ini file
            with open('config.ini', 'w') as conf:
                config_object.write(conf)

######################################


import os

for dirname, _, filename in os.walk("../input"):

  for files in filename:

    print(os.path.join(dirname, files))
import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import seaborn as sns

from sklearn.model_selection import train_test_split

import tensorflow as tf

import cv2

from tqdm import tqdm_notebook as tqdm
train_data = pd.DataFrame(pd.read_csv("../input/plant-pathology-2020-fgvc7/train.csv"))

test_data = pd.DataFrame(pd.read_csv("../input/plant-pathology-2020-fgvc7/test.csv"))
print("Training data shape : = {}".format(train_data.shape))

print("Test data shape : = {}".format(test_data.shape))
train_data.head()
test_data.head()
image_folder_path = "../input/plant-pathology-2020-fgvc7/images/"
arr = train_data["image_id"]

train_images = [i for i in arr]

arr = test_data["image_id"]

test_images = [i for i in arr]
def load_image(image_id) :

  image_path = image_folder_path +image_id +".jpg"

  image = cv2.imread(image_path)

  image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

  return image

def resize(image, image_size):

  image = cv2.resize(image, (image_size[0], image_size[1]), interpolation = cv2.INTER_AREA)

  return image
def extract_classes(s):

  """

  s can be either of the four classes mentioned above.

  """

  t = train_data[train_data[s] == 1]

  arr = t["image_id"]

  images = [i for i in tqdm(arr)]

  train_images = [load_image(i) for i in tqdm(images)]

  return train_images

classes = ["healthy", "multiple_diseases", "rust", "scab"]
count_healthy = len(train_data[train_data["healthy"] == 1])

count_diseased = len(train_data[train_data["multiple_diseases"] == 1])

count_rust = len(train_data[train_data["rust"] == 1])

count_scab = len(train_data[train_data["scab"] == 1])

print(count_healthy)

print(count_diseased)

print(count_rust)

print(count_scab)

print(count_healthy + count_diseased + count_rust +  count_scab)
# observe number of cases present in each class

labels = ["Healthy", "Multiple Diseased", "Rust", "Scab"]

counts = [count_healthy, count_diseased, count_rust, count_scab]

explode = (0.05, 0.05, 0.05, 0.05)

fig, ax = plt.subplots(figsize = (20, 12))

ax.pie(counts, explode = explode, labels = labels, shadow = True, startangle = 90)

ax.axis("equal") # equal aspect ratio ensures pie graph is drawn as circle
red , green, blue = [], [], []
healthy = extract_classes("healthy")

for image in healthy :

    mean_red = np.mean(image[:,:,0])

    mean_green = np.mean(image[:,:,1])

    mean_blue = np.mean(image[:,:,2])

    red.append(mean_red)

    green.append(mean_green)

    blue.append(mean_blue)

healthy_image_1 = healthy[100]

healthy_image_2 = healthy[200]

healthy_image_3 = healthy[300]

del healthy # free memory

md = extract_classes("multiple_diseases")

for image in md :

    mean_red = np.mean(image[:,:,0])

    mean_green = np.mean(image[:,:,1])

    mean_blue = np.mean(image[:,:,2])

    red.append(mean_red)

    green.append(mean_green)

    blue.append(mean_blue)

md_image_1 = md[1]

md_image_2 = md[5]

md_image_3 = md[10]

del md # free memory

rust = extract_classes("rust")

for image in rust :

    mean_red = np.mean(image[:,:,0])

    mean_green = np.mean(image[:,:,1])

    mean_blue = np.mean(image[:,:,2])

    red.append(mean_red)

    green.append(mean_green)

    blue.append(mean_blue)

rust_image_1 = rust[10]

rust_image_2 = rust[20]

rust_image_3 = rust[30]

del rust # free memory

scab = extract_classes("healthy")

for image in scab :

    mean_red = np.mean(image[:,:,0])

    mean_green = np.mean(image[:,:,1])

    mean_blue = np.mean(image[:,:,2])

    red.append(mean_red)

    green.append(mean_green)

    blue.append(mean_blue)

scab_image_1 = scab[10]

scab_image_2 = scab[20]

scab_image_3 = scab[30]

del scab # free memory

image_collection = [healthy_image_1, healthy_image_2, healthy_image_3,

                   md_image_1, md_image_2, md_image_3,

                   rust_image_1, rust_image_2, rust_image_3,

                   scab_image_1, scab_image_2, scab_image_3]
fig, ax = plt.subplots(nrows = 4, ncols = 3, figsize = (25, 15))

for i in range(12):

    ax[i//3, i%3].imshow(image_collection[i])
# red channel plot

range_of_spread = max(red) - min(red)

plt.figure(figsize = (12, 8))

plt.rc('font', weight='bold')

sns.set_style("whitegrid")

fig = sns.distplot(red,  hist = True, kde = True, label = "Red Channel intensities", color = "r")

fig.set(xlabel = "Mean red channel intensities observed in each image (Sample size = 1000)", ylabel = "Probability Density")

plt.legend()

print("The range of spread = {:.2f}".format(range_of_spread))
# Green channel plot

range_of_spread = max(green) - min(green)

plt.figure(figsize = (12, 8))

plt.rc('font', weight='bold')

sns.set_style("whitegrid")

fig = sns.distplot(green,  hist = True, kde = True, label = "Green Channel intensities", color = "g")

fig.set(xlabel = "Mean green channel intensities observed in each image (Sample size = 1000)", ylabel = "Probability Density")

plt.legend()

print("The range of spread = {:.2f}".format(range_of_spread))
# Blue channel plot

range_of_spread = max(blue) - min(blue)

plt.figure(figsize = (12, 8))

plt.rc('font', weight='bold')

sns.set_style("whitegrid")

fig = sns.distplot(blue,  hist = True, kde = True, rug = False, label = "Blue Channel intensities", color = "b")

fig.set(xlabel = "Mean blue channel intensities observed in each image (Sample size = 1000)", ylabel = "Probability Density")

plt.legend()

print("The range of spread = {:.2f}".format(range_of_spread))
sample_image = rust_image_1

plt.figure(figsize = (12, 8))

plt.imshow(sample_image)
def non_local_means_denoising(image) :

    denoised_image = cv2.fastNlMeansDenoisingColored(image, None, 10, 10, 7, 21)

    return denoised_image
denoised_image = non_local_means_denoising(sample_image)

plt.figure(figsize = (12, 8))

plt.subplot(1,2,1)

plt.imshow(sample_image, cmap = "gray")

plt.grid(False)

plt.title("Normal Image")

plt.subplot(1,2,2)

plt.imshow(denoised_image, cmap = "gray")

plt.grid(False)

plt.title("Denoised image")

# Automatically adjust subplot parameters to give specified padding.

plt.tight_layout()
def sobel_edge_detection(image):

  """

  Using Sobel filter

  Sobel filter takes the following arguments :

  1. Original Image

  2. Depth of the destination image

  3. Order of derivative x

  4. Order of derivative y

  5. Kernel size for convolutions

  f(Image, depth, order_dx, order_dy, kernel_size)

  """

  sobel_x = cv2.Sobel(image, cv2.CV_64F, 1, 0, ksize = 5)

  sobel_y = cv2.Sobel(image, cv2.CV_64F, 0, 1, ksize = 5)

  return sobel_x, sobel_y
s_img_x, s_img_y = sobel_edge_detection(denoised_image)

plt.figure(figsize = (12, 8))

plt.subplot(2,2,1)

plt.imshow(sample_image, cmap = "gray")

plt.grid(False)

plt.title("Sample Image")

plt.subplot(2,2,2)

plt.imshow(denoised_image, cmap = "gray")

plt.grid(False)

plt.title("Denoised Image")

plt.subplot(2,2,3)

plt.imshow(s_img_x, cmap = "gray")

plt.grid(False)

plt.title("Sobel X filtered Image")

plt.subplot(2,2,4)

plt.imshow(s_img_y, cmap = "gray")

plt.grid(False)

plt.title("Sobel Y filtered Image")

# Automatically adjust subplot parameters to give specified padding.

plt.tight_layout()
from collections import deque

def canny_edge_detection(image):

  edges = cv2.Canny(image, 170, 200)

  return edges

def primary_roi(original_image, edge_image):

  edge_coordinates = deque()

  for i in tqdm(range(edge_image.shape[0])):

    for j in range(edge_image.shape[1]):

      if edge_image[i][j] != 0 :

        edge_coordinates.append((i, j))

  min_row = edge_coordinates[np.argsort([coordinate[0] for coordinate in edge_coordinates])[0]][0]

  max_row = edge_coordinates[np.argsort([coordinate[0] for coordinate in edge_coordinates])[-1]][0]

  min_col = edge_coordinates[np.argsort([coordinate[1] for coordinate in edge_coordinates])[0]][1]

  max_col = edge_coordinates[np.argsort([coordinate[1] for coordinate in edge_coordinates])[-1]][1]

  new_image = original_image.copy()

  new_edge_image = edge_image.copy()

  new_image[min_row - 10 : min_row + 10, min_col : max_col] = [255, 0, 0]

  new_image[max_row - 10 : max_row + 10, min_col : max_col] = [255, 0, 0]

  new_image[min_row : max_row , min_col - 10 : min_col + 10] = [255, 0, 0]

  new_image[min_row : max_row , max_col - 10 : max_col + 10] = [255, 0, 0]

  new_edge_image[min_row - 10 : min_row + 10, min_col : max_col] = [255]

  new_edge_image[max_row - 10 : max_row + 10, min_col : max_col] = [255]

  new_edge_image[min_row : max_row , min_col - 10 : min_col + 10] = [255]

  new_edge_image[min_row : max_row , max_col - 10 : max_col + 10] = [255]

  roi_image = new_image[min_row : max_row, min_col : max_col]

  edge_roi_image = new_edge_image[min_row : max_row, min_col : max_col]

  return roi_image, edge_roi_image
plt.figure(figsize = (12, 8))

plt.subplot(1,2,1)

plt.imshow(sample_image, cmap = "gray")

plt.grid(False)

plt.title("Sample Image")

edge_image = canny_edge_detection(sample_image)

plt.subplot(1,2,2)

plt.imshow(edge_image, cmap = "gray")

plt.grid(False)

plt.title("Canny Edge Image")

# Automatically adjust subplot parameters to give specified padding.

plt.tight_layout()
roi_image, edge_roi_image = primary_roi(sample_image, edge_image)

plt.figure(figsize = (12, 8))

plt.subplot(1,2,1)

plt.imshow(roi_image, cmap = "gray")

plt.grid(False)

plt.title("ROI Image")

plt.subplot(1,2,2)

plt.imshow(edge_roi_image, cmap = "gray")

plt.grid(False)

plt.title("Edge ROI Image")

# Automatically adjust subplot parameters to give specified padding.

plt.tight_layout()
def histogram_equalization(roi_image):

  image_ycrcb = cv2.cvtColor(roi_image, cv2.COLOR_RGB2YCR_CB)

  y_channel = image_ycrcb[:, :, 0] # apply histogram equalization on this channel

  cr_channel = image_ycrcb[:, :, 1]

  cb_channel = image_ycrcb[:, :, 2]

  # local histogram equalization

  clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))

  equalized = clahe.apply(y_channel)

  equalized_image = cv2.merge([equalized, cr_channel, cb_channel])

  equalized_image = cv2.cvtColor(equalized_image, cv2.COLOR_YCR_CB2RGB)

  return equalized_image
equalized_roi_image = histogram_equalization(roi_image)

plt.figure(figsize = (12, 8))

plt.subplot(1,2,1)

plt.imshow(roi_image, cmap = "gray")

plt.grid(False)

plt.title("ROI Image")

plt.subplot(1,2,2)

plt.imshow(equalized_roi_image, cmap = "gray")

plt.grid(False)

plt.title("Histogram Equalized ROI Image")

# Automatically adjust subplot parameters to give specified padding.

plt.tight_layout()
otsu_threshold, otsu_image = cv2.threshold(cv2.cvtColor(equalized_roi_image, cv2.COLOR_RGB2GRAY), 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)

plt.figure(figsize = (12, 8))

plt.subplot(1,2,1)

plt.imshow(equalized_roi_image, cmap = "gray")

plt.grid(False)

plt.title("equalized_roi_image")

plt.subplot(1,2,2)

plt.imshow(otsu_image, cmap = "gray")

plt.grid(False)

plt.title("Otsu's Thresholded Image")

# Automatically adjust subplot parameters to give specified padding.

plt.tight_layout()
def segmentation(image, k, attempts) :

    vectorized = np.float32(image.reshape((-1, 3)))

    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 20, 1.0)

    res , label , center = cv2.kmeans(vectorized, k, None, criteria, attempts, cv2.KMEANS_PP_CENTERS)

    center = np.uint8(center)

    res = center[label.flatten()]

    segmented_image = res.reshape((image.shape))

    return segmented_image
plt.figure(figsize = (12, 8))

plt.subplot(2,2,1)

plt.imshow(equalized_roi_image, cmap = "gray")

plt.grid(False)

plt.title("Histogram Equalized Image")

segmented_image = segmentation(equalized_roi_image, 3, 10) # k = 3, attempt = 10

plt.subplot(2,2,2)

plt.imshow(segmented_image, cmap = "gray")

plt.grid(False)

plt.title("Segmented Image with k = 3")

segmented_image = segmentation(equalized_roi_image, 4, 10) # k = 4, attempt = 10

plt.subplot(2,2,3)

plt.imshow(segmented_image, cmap = "gray")

plt.grid(False)

plt.title("Segmented Image with k = 4")

segmented_image = segmentation(equalized_roi_image, 5, 10) # k = 5, attempt = 10

plt.subplot(2,2,4)

plt.imshow(segmented_image, cmap = "gray")

plt.grid(False)

plt.title("Segmented Image with k = 5")
IMAGE_SIZE = (224, 224, 3)
train_image = []

for id in tqdm(train_data["image_id"]):

    image = load_image(id)

    image = resize(image, IMAGE_SIZE)

    edge_image = canny_edge_detection(image)

    roi_image, _ = primary_roi(image, edge_image)

    equalized_roi_image = histogram_equalization(roi_image)

    train_image.append(image)
#x_train = np.ndarray(shape = (len(train_image), IMAGE_SIZE[0], IMAGE_SIZE[1], 3), dtype = np.float32)

x_train = np.array(train_image)

x_train = x_train/255.0

y = train_data.copy()

del y["image_id"]

y_train = np.array(y.values)
print(x_train.shape)

print(y_train.shape)
sample = x_train[5]

sample.shape
plt.imshow(sample)

print("Label = ", y_train[5])
from imblearn.over_sampling import SMOTE

smote = SMOTE(random_state = 115)

x_train, y_train = smote.fit_resample(x_train.reshape((-1, IMAGE_SIZE[0] * IMAGE_SIZE[1] * 3)), y_train)

x_train = x_train.reshape((-1, IMAGE_SIZE[0], IMAGE_SIZE[1], 3))

print(x_train.shape)

print(y_train.shape)
def label_count(label) :

    count = 0

    for entry in y_train :

        if np.array_equal(label, entry):

            count += 1

    return count
healthy = np.array([1,0,0,0])

multiple_diseases = np.array([0,1,0,0])

rust = np.array([0,0,1,0])

scab = np.array([0,0,0,1])
count_healthy = label_count(healthy)

count_multiple_diseases = label_count(multiple_diseases)

count_rust = label_count(rust)

count_scab = label_count(scab)

labels = ["Healthy", "Multiple Diseased", "Rust", "Scab"]

counts = [count_healthy, count_multiple_diseases, count_rust, count_scab]

explode = (0.05, 0.05, 0.05, 0.05)

fig, ax = plt.subplots(figsize = (20, 12))

ax.pie(counts, explode = explode, labels = labels, shadow = True, startangle = 90)

ax.axis("equal") # equal aspect ratio ensures pie graph is drawn as circle
print(x_train.shape)

print(y_train.shape)
VALIDATION_FACTOR = 0.1

val_size = int(len(x_train) * VALIDATION_FACTOR)

train_x = x_train[: len(x_train) - val_size]

train_y = y_train[: len(y_train) - val_size] # len(x_train) = len(y_train)

val_x = x_train[len(x_train) - val_size : len(x_train)]

val_y = y_train[len(y_train) - val_size : len(y_train)]

print("Shape of training data = ", train_x.shape, train_y.shape)

print("Shape of validation data = ", val_x.shape, val_y.shape)
sample_image = train_x[250]

plt.imshow(sample_image)

plt.grid(False)

print("Image label = ", train_y[250])
monitor_es = tf.keras.callbacks.EarlyStopping(monitor = "val_loss", patience = 25,

                                              restore_best_weights = False, verbose = 0)
import os

os.environ["CUDA_VISIBLE_DEVICES"] = "0"

gpus = tf.config.list_physical_devices("GPU")

print(gpus)

if len(gpus) == 1 :

    strategy = tf.distribute.OneDeviceStrategy(device = "/gpu:0")

else:

    strategy = tf.distribute.MirroredStrategy()
tf.config.optimizer.set_experimental_options({"auto_mixed_precision" : True})

print("Mixed precision enabled")
vgg = tf.keras.applications.vgg16.VGG16(include_top = False, weights = "imagenet", input_shape = IMAGE_SIZE)
vgg.summary()
vgg.input
vgg.layers[-1].output
output = vgg.layers[-1].output

output = tf.keras.layers.GlobalAveragePooling2D()(output)

vgg_model = tf.keras.models.Model(vgg.input, output)
vgg_model.trainable = True

set_trainable = False

for layer in vgg_model.layers :

    if layer.name in ['block5_conv1', 'block4_conv1']:

        set_trainable = True

    if set_trainable:

        layer.trainable = True

    else:

        layer.trainable = False
pd.set_option("max_colwidth", -1)

info = [(layer, layer.name, layer.trainable) for layer in vgg_model.layers]

analysis = pd.DataFrame(info, columns = ["Layer Type", "Layer Name", "Train Status"])

analysis
def get_bottleneck_features(deep_learning_model, input_images) :

    #tf.keras.backend.clear_session()

    #with strategy.scope() : #with tf.device("/device:GPU:0"):

    with tf.device("/device:GPU:0"):

        bottleneck_features = deep_learning_model.predict(input_images, verbose = 1)

        return bottleneck_features

training_bottleneck_features = get_bottleneck_features(vgg_model, train_x)

validation_bottleneck_features = get_bottleneck_features(vgg_model, val_x)

print("Shape of training bottleneck features = ", training_bottleneck_features.shape)

print("Shape of validation bottleneck features = ", validation_bottleneck_features.shape)
vgg_model.output_shape[1]
input_shape = vgg_model.output_shape[1]

model = tf.keras.models.Sequential()

model.add(tf.keras.layers.InputLayer(input_shape = (input_shape, )))

model.add(tf.keras.layers.Dense(512, activation = "relu", input_dim = input_shape))

model.add(tf.keras.layers.Dropout(0.4))

model.add(tf.keras.layers.Dense(512, activation = "relu"))

model.add(tf.keras.layers.Dropout(0.4))

model.add(tf.keras.layers.Dense(4, activation = "softmax"))

model.compile(loss='categorical_crossentropy', optimizer = tf.optimizers.Adam(lr = 0.001), metrics=['accuracy'])

model.summary()
BATCH_SIZE = 32

EPOCHS = 100
with tf.device("/device:GPU:0"): #with tf.device("/device:GPU:0"):

    history = model.fit(x = training_bottleneck_features, y = train_y,

                        validation_data = (validation_bottleneck_features, val_y),

                        batch_size = BATCH_SIZE, epochs= EPOCHS, verbose = 1)
X = np.arange(0,EPOCHS,1)

plt.figure(1, figsize = (20, 12))

plt.subplot(1,2,1)

plt.xlabel("Epochs")

plt.ylabel("Loss")

plt.plot(X, history.history["loss"], label = "Training Loss")

plt.plot(X, history.history["val_loss"], label = "Validation Loss")

plt.grid(True)

plt.legend()

plt.subplot(1,2,2)

plt.xlabel("Epochs")

plt.ylabel("Accuracy")

plt.plot(X, history.history["accuracy"], label = "Training Accuracy")

plt.plot(X, history.history["val_accuracy"], label = "Validation Accuracy")

plt.grid(True)

plt.legend()
del train_x

del val_x

del train_y

del val_y
del training_bottleneck_features

del validation_bottleneck_features
import gc

gc.collect()
test_data
test_image = []

for id in tqdm(test_data["image_id"][:5]):

    image = load_image(id)

    image = resize(image, IMAGE_SIZE)

    edge_image = canny_edge_detection(image)

    roi_image, _ = primary_roi(image, edge_image)

    equalized_roi_image = histogram_equalization(roi_image)

    test_image.append(image)
test_x = np.array(test_image)

test_x = test_x/255.0
test_bottleneck_features = get_bottleneck_features(vgg_model, test_x)
y_pred = model.predict(test_bottleneck_features)

y_pred.shape
def convert_label(label) :

    m = max(label)

    index = list(label).index(m)

    if index == 0 :

        return "Healthy"

    elif index == 1 :

        return "Multiple Diseased"

    elif index == 2 :

        return "Rust"

    else:

        return "Scab"
plt.figure(figsize = (12, 8))

plt.subplot(1,2,1)

plt.imshow(test_x[0], cmap = "gray")

label = convert_label(y_pred[0])

plt.grid(False)

plt.title("First Test Image")

print("Label of first testing image", label)

plt.subplot(1,2,2)

plt.imshow(test_x[2], cmap = "gray")

label = convert_label(y_pred[2])

plt.grid(False)

plt.title("Second Test Image")

print("Label of first testing image", label)

plt.tight_layout()


import logging

class CLogRecoder:

    def __init__(self, logfile = 'log.log', format = '%(asctime)s : %(message)s', level = logging.DEBUG):
        logging.basicConfig(filename= logfile, level= level , format= format)
        self._ft = format

    def addStreamHandler(self):
        console = logging.StreamHandler()
        console.setLevel(logging.INFO)
        formater = logging.Formatter(self._ft)
        console.setFormatter(formater)
        logging.getLogger('').addHandler(console)
        return self

    def INFO(self, message):
        logging.info(message)
        return self

if __name__ == '__main__':
    lr = CLogRecoder().addStreamHandler()
    lr.INFO("test")