from typing import List
import requests
import logging

from medperf.enums import Status
import medperf.config as config
from medperf.comms.interface import Comms
from medperf.utils import (
    sanitize_json,
    log_response_error,
    format_errors_dict,
    filter_latest_associations,
)
from medperf.exceptions import (
    CommunicationError,
    CommunicationRetrievalError,
    CommunicationRequestError,
)

class REST(Comms):
    def __init__(self, source: str):
        self.server_url = self.parse_url(source)
        self.cert = config.certificate
        if self.cert is None:
            # No certificate provided, default to normal verification
            self.cert = True

    @classmethod
    def parse_url(cls, url: str) -> str:
        """Parse the source URL so that it can be used by the comms implementation.
        It should handle protocols and versioning to be able to communicate with the API.

        Args:
            url (str): base URL

        Returns:
            str: parsed URL with protocol and version
        """
        url_sections = url.split("://")
        api_path = f"/api/v{config.major_version}"
        # Remove protocol if passed
        if len(url_sections) > 1:
            url = "".join(url_sections[1:])

        return f"https://{url}{api_path}"

    def __auth_get(self, url, **kwargs):
        return self.__auth_req(url, requests.get, **kwargs)

    def __auth_post(self, url, **kwargs):
        return self.__auth_req(url, requests.post, **kwargs)

    def __auth_put(self, url, **kwargs):
        return self.__auth_req(url, requests.put, **kwargs)

    def __auth_req(self, url, req_func, **kwargs):
        token = config.auth.access_token
        return self.__req(
            url, req_func, headers={"Authorization": f"Bearer {token}"}, **kwargs
        )

    def __req(self, url, req_func, **kwargs):
        logging.debug(f"Calling {req_func}: {url}")
        if "json" in kwargs:
            logging.debug(f"Passing JSON contents: {kwargs['json']}")
            kwargs["json"] = sanitize_json(kwargs["json"])
        try:
            return req_func(url, verify=self.cert, **kwargs)
        except requests.exceptions.SSLError as e:
            logging.error(f"Couldn't connect to {self.server_url}: {e}")
            raise CommunicationError(
                "Couldn't connect to server through HTTPS. If running locally, "
                "remember to provide the server certificate through --certificate"
            )

    def __get_list(
        self,
        url,
        num_elements=None,
        page_size=config.default_page_size,
        offset=0,
        binary_reduction=False,
    ):
        """Retrieves a list of elements from a URL by iterating over pages until num_elements is obtained.
        If num_elements is None, then iterates until all elements have been retrieved.
        If binary_reduction is enabled, errors are assumed to be related to response size. In that case,
        the page_size is reduced by half until a successful response is obtained or until page_size can't be
        reduced anymore.

        Args:
            url (str): The url to retrieve elements from
            num_elements (int, optional): The desired number of elements to be retrieved. Defaults to None.
            page_size (int, optional): Starting page size. Defaults to config.default_page_size.
            start_limit (int, optional): The starting position for element retrieval. Defaults to 0.
            binary_reduction (bool, optional): Wether to handle errors by halfing the page size. Defaults to False.

        Returns:
            List[dict]: A list of dictionaries representing the retrieved elements.
        """
        el_list = []

        if num_elements is None:
            num_elements = float("inf")

        while len(el_list) < num_elements:
            paginated_url = f"{url}?limit={page_size}&offset={offset}"
            res = self.__auth_get(paginated_url)
            if res.status_code != 200:
                if not binary_reduction:
                    log_response_error(res)
                    details = format_errors_dict(res.json())
                    raise CommunicationRetrievalError(
                        f"there was an error retrieving the current list: {details}"
                    )

                log_response_error(res, warn=True)
                details = format_errors_dict(res.json())
                if page_size <= 1:
                    raise CommunicationRetrievalError(
                        f"Could not retrieve list. Minimum page size achieved without success: {details}"
                    )
                page_size = page_size // 2
                continue
            else:
                data = res.json()
                el_list += data["results"]
                offset += len(data["results"])
                if data["next"] is None:
                    break

        if isinstance(num_elements, int):
            return el_list[:num_elements]
        return el_list

    def __set_approval_status(self, url: str, status: str) -> requests.Response:
        """Sets the approval status of a resource

        Args:
            url (str): URL to the resource to update
            status (str): approval status to set

        Returns:
            requests.Response: Response object returned by the update
        """
        data = {"approval_status": status}
        res = self.__auth_put(url, json=data)
        return res

    def get_current_user(self):
        """Retrieve the currently-authenticated user information"""
        res = self.__auth_get(f"{self.server_url}/me/")
        return res.json()

    def get_benchmarks(self) -> List[dict]:
        """Retrieves all benchmarks in the platform.

        Returns:
            List[dict]: all benchmarks information.
        """
        bmks = self.__get_list(f"{self.server_url}/benchmarks/")
        return bmks

    def get_benchmark(self, benchmark_uid: int) -> dict:
        """Retrieves the benchmark specification file from the server

        Args:
            benchmark_uid (int): uid for the desired benchmark

        Returns:
            dict: benchmark specification
        """
        res = self.__auth_get(f"{self.server_url}/benchmarks/{benchmark_uid}")
        if res.status_code != 200:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRetrievalError(
                f"the specified benchmark doesn't exist: {details}"
            )
        return res.json()

    def get_benchmark_model_associations(self, benchmark_uid: int) -> List[int]:
        """Retrieves all the model associations of a benchmark.

        Args:
            benchmark_uid (int): UID of the desired benchmark

        Returns:
            list[int]: List of benchmark model associations
        """
        assocs = self.__get_list(f"{self.server_url}/benchmarks/{benchmark_uid}/models")
        return filter_latest_associations(assocs, "model_mlcube")

    def get_user_benchmarks(self) -> List[dict]:
        """Retrieves all benchmarks created by the user

        Returns:
            List[dict]: Benchmarks data
        """
        bmks = self.__get_list(f"{self.server_url}/me/benchmarks/")
        return bmks

    def get_cubes(self) -> List[dict]:
        """Retrieves all MLCubes in the platform

        Returns:
            List[dict]: List containing the data of all MLCubes
        """
        cubes = self.__get_list(f"{self.server_url}/mlcubes/")
        return cubes

    def get_cube_metadata(self, cube_uid: int) -> dict:
        """Retrieves metadata about the specified cube

        Args:
            cube_uid (int): UID of the desired cube.

        Returns:
            dict: Dictionary containing url and hashes for the cube files
        """
        res = self.__auth_get(f"{self.server_url}/mlcubes/{cube_uid}/")
        if res.status_code != 200:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRetrievalError(
                f"the specified cube doesn't exist {details}"
            )
        return res.json()

    def get_user_cubes(self) -> List[dict]:
        """Retrieves metadata from all cubes registered by the user

        Returns:
            List[dict]: List of dictionaries containing the mlcubes registration information
        """
        cubes = self.__get_list(f"{self.server_url}/me/mlcubes/")
        return cubes

    def upload_benchmark(self, benchmark_dict: dict) -> int:
        """Uploads a new benchmark to the server.

        Args:
            benchmark_dict (dict): benchmark_data to be uploaded

        Returns:
            int: UID of newly created benchmark
        """
        res = self.__auth_post(f"{self.server_url}/benchmarks/", json=benchmark_dict)
        if res.status_code != 201:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRetrievalError(f"Could not upload benchmark: {details}")
        return res.json()

    def upload_mlcube(self, mlcube_body: dict) -> int:
        """Uploads an MLCube instance to the platform

        Args:
            mlcube_body (dict): Dictionary containing all the relevant data for creating mlcubes

        Returns:
            int: id of the created mlcube instance on the platform
        """
        res = self.__auth_post(f"{self.server_url}/mlcubes/", json=mlcube_body)
        if res.status_code != 201:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRetrievalError(f"Could not upload the mlcube: {details}")
        return res.json()

    def get_datasets(self) -> List[dict]:
        """Retrieves all datasets in the platform

        Returns:
            List[dict]: List of data from all datasets
        """
        dsets = self.__get_list(f"{self.server_url}/datasets/")
        return dsets

    def get_dataset(self, dset_uid: int) -> dict:
        """Retrieves a specific dataset

        Args:
            dset_uid (int): Dataset UID

        Returns:
            dict: Dataset metadata
        """
        res = self.__auth_get(f"{self.server_url}/datasets/{dset_uid}/")
        if res.status_code != 200:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRetrievalError(
                f"Could not retrieve the specified dataset from server: {details}"
            )
        return res.json()

    def get_user_datasets(self) -> dict:
        """Retrieves all datasets registered by the user

        Returns:
            dict: dictionary with the contents of each dataset registration query
        """
        dsets = self.__get_list(f"{self.server_url}/me/datasets/")
        return dsets

    def upload_dataset(self, reg_dict: dict) -> int:
        """Uploads registration data to the server, under the sha name of the file.

        Args:
            reg_dict (dict): Dictionary containing registration information.

        Returns:
            int: id of the created dataset registration.
        """
        res = self.__auth_post(f"{self.server_url}/datasets/", json=reg_dict)
        if res.status_code != 201:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRequestError(f"Could not upload the dataset: {details}")
        return res.json()

    def get_results(self) -> List[dict]:
        """Retrieves all results

        Returns:
            List[dict]: List of results
        """
        res = self.__get_list(f"{self.server_url}/results")
        if res.status_code != 200:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRetrievalError(f"Could not retrieve results: {details}")
        return res.json()

    def get_result(self, result_uid: int) -> dict:
        """Retrieves a specific result data

        Args:
            result_uid (int): Result UID

        Returns:
            dict: Result metadata
        """
        res = self.__auth_get(f"{self.server_url}/results/{result_uid}/")
        if res.status_code != 200:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRetrievalError(
                f"Could not retrieve the specified result: {details}"
            )
        return res.json()

    def get_user_results(self) -> dict:
        """Retrieves all results registered by the user

        Returns:
            dict: dictionary with the contents of each result registration query
        """
        results = self.__get_list(f"{self.server_url}/me/results/")
        return results

    def get_benchmark_results(self, benchmark_id: int) -> dict:
        """Retrieves all results for a given benchmark

        Args:
            benchmark_id (int): benchmark ID to retrieve results from

        Returns:
            dict: dictionary with the contents of each result in the specified benchmark
        """
        results = self.__get_list(
            f"{self.server_url}/benchmarks/{benchmark_id}/results"
        )
        return results

    def upload_result(self, results_dict: dict) -> int:
        """Uploads result to the server.

        Args:
            results_dict (dict): Dictionary containing results information.

        Returns:
            int: id of the generated results entry
        """
        res = self.__auth_post(f"{self.server_url}/results/", json=results_dict)
        if res.status_code != 201:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRequestError(f"Could not upload the results: {details}")
        return res.json()

    def associate_dset(self, data_uid: int, benchmark_uid: int, metadata: dict = {}):
        """Create a Dataset Benchmark association

        Args:
            data_uid (int): Registered dataset UID
            benchmark_uid (int): Benchmark UID
            metadata (dict, optional): Additional metadata. Defaults to {}.
        """
        data = {
            "dataset": data_uid,
            "benchmark": benchmark_uid,
            "approval_status": Status.PENDING.value,
            "metadata": metadata,
        }
        res = self.__auth_post(f"{self.server_url}/datasets/benchmarks/", json=data)
        if res.status_code != 201:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRequestError(
                f"Could not associate dataset to benchmark: {details}"
            )

    def associate_cube(self, cube_uid: int, benchmark_uid: int, metadata: dict = {}):
        """Create an MLCube-Benchmark association

        Args:
            cube_uid (int): MLCube UID
            benchmark_uid (int): Benchmark UID
            metadata (dict, optional): Additional metadata. Defaults to {}.
        """
        data = {
            "approval_status": Status.PENDING.value,
            "model_mlcube": cube_uid,
            "benchmark": benchmark_uid,
            "metadata": metadata,
        }
        res = self.__auth_post(f"{self.server_url}/mlcubes/benchmarks/", json=data)
        if res.status_code != 201:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRequestError(
                f"Could not associate mlcube to benchmark: {details}"
            )

    def set_dataset_association_approval(
        self, benchmark_uid: int, dataset_uid: int, status: str
    ):
        """Approves a dataset association

        Args:
            dataset_uid (int): Dataset UID
            benchmark_uid (int): Benchmark UID
            status (str): Approval status to set for the association
        """
        url = f"{self.server_url}/datasets/{dataset_uid}/benchmarks/{benchmark_uid}/"
        res = self.__set_approval_status(url, status)
        if res.status_code != 200:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRequestError(
                f"Could not approve association between dataset {dataset_uid} and benchmark {benchmark_uid}: {details}"
            )

    def set_mlcube_association_approval(
        self, benchmark_uid: int, mlcube_uid: int, status: str
    ):
        """Approves an mlcube association

        Args:
            mlcube_uid (int): Dataset UID
            benchmark_uid (int): Benchmark UID
            status (str): Approval status to set for the association
        """
        url = f"{self.server_url}/mlcubes/{mlcube_uid}/benchmarks/{benchmark_uid}/"
        res = self.__set_approval_status(url, status)
        if res.status_code != 200:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRequestError(
                f"Could not approve association between mlcube {mlcube_uid} and benchmark {benchmark_uid}: {details}"
            )

    def get_datasets_associations(self) -> List[dict]:
        """Get all dataset associations related to the current user

        Returns:
            List[dict]: List containing all associations information
        """
        assocs = self.__get_list(f"{self.server_url}/me/datasets/associations/")
        return filter_latest_associations(assocs, "dataset")

    def get_cubes_associations(self) -> List[dict]:
        """Get all cube associations related to the current user

        Returns:
            List[dict]: List containing all associations information
        """
        assocs = self.__get_list(f"{self.server_url}/me/mlcubes/associations/")
        return filter_latest_associations(assocs, "model_mlcube")

    def set_mlcube_association_priority(
        self, benchmark_uid: int, mlcube_uid: int, priority: int
    ):
        """Sets the priority of an mlcube-benchmark association

        Args:
            mlcube_uid (int): MLCube UID
            benchmark_uid (int): Benchmark UID
            priority (int): priority value to set for the association
        """
        url = f"{self.server_url}/mlcubes/{mlcube_uid}/benchmarks/{benchmark_uid}/"
        data = {"priority": priority}
        res = self.__auth_put(url, json=data)
        if res.status_code != 200:
            log_response_error(res)
            details = format_errors_dict(res.json())
            raise CommunicationRequestError(
                f"Could not set the priority of mlcube {mlcube_uid} within the benchmark {benchmark_uid}: {details}"
            )


import os
import random
import json
import time
from dotenv import load_dotenv
from aes_prf import aes_prf
from adversary import Adversary

def hex_to_int(hex_str):
    hex_str = hex_str.replace(' ', '')
    return int(hex_str, 16)

def load_data(filename):
    with open(filename, 'r') as file:
        data = json.load(file)
    return data

class SubvertedProver:
    def __init__(self, p, q, g, x, A, backdoor_key, bit_number):
        self.protocol = (p, q, g)
        self.secret_key = x
        self.x_bits = [int(bit) for bit in bin(x)[2:]]
        self.public_key = A
        self.r = None
        self.bd_key = backdoor_key
        self.bit_number = bit_number

    def prover_commitment(self):
        p, q, g = self.protocol
        self.r = random.randint(1, q - 1)
        t = pow(g, self.r, p)

        # subverted commitment
        l, b = aes_prf(self.bd_key, t, self.secret_key.bit_length(), self.bit_number // 8)
        if self.x_bits[l] == b:
            return t, l

        self.r = random.randint(1, q - 1)
        t = pow(g, self.r, p)
        return t, l

    def prover_response(self, c):
        p, q, _ = self.protocol
        z = (self.r - c * self.secret_key) % q
        return z

class HonestVerifier:
    def __init__(self, p, q, g, A):
        self.protocol = (p, q, g)
        self.public_key = A
        self.c = None

    def challenge(self, bits):
        p, _, _ = self.protocol
        self.c = random.randint(1, pow(2, bits) - 1)
        return self.c

    def validate(self, t, z):
        # test if A is in range 1 - (p - 1)
        p, q, g = self.protocol
        if self.public_key < 1 or self.public_key > p - 1:
            return False

        # test if A ^ q = 1 mod p
        if pow(self.public_key, q, p) != 1:
            return False

        # test if t = g^z * A^c mod p
        left = t % p
        right = (pow(g, z, p) * pow(self.public_key, self.c, p)) % p
        return left == right

class SchnorrIdentificationProtocol:
    def __init__(self, backdoor_key):
        load_dotenv()
        self.bit_number = int(os.getenv('BIT_NUMBER')) # type: ignore
        self.num_rounds = 0

        # load the json data
        data = load_data('primes.json')
        p = hex_to_int(data[f'bit_{self.bit_number}'])
        q = (p - 1) // 2
        g = data['generator']

        x = random.randint(1, q - 1)
        A = pow(g, x, p)
        self.params = (p, q, g)
        self.secret_key = x
        self.public_key = A

        self.bd_key = backdoor_key
        self.counter = 0

        self.subverted_prover = SubvertedProver(p, q, g, x, A, backdoor_key, self.bit_number)
        self.honest_verifier = HonestVerifier(p, q, g, A)

    def simulate(self):

        transcripts = []
        start_time = time.time()
        time_values = []

        # counter for each bit poisition of the secret key
        bit_counters = [0] * self.secret_key.bit_length()
        transcripts_per_bit = 19
        min_counter = 0
        min_counter_old = 0

        # while min(bit_counters) < transcripts_per_bit:
        while min_counter < transcripts_per_bit:
            self.num_rounds += 1
            start_time = time.time()

            t, l = self.subverted_prover.prover_commitment()

            challenge_bits = 128 # number of bits in the challenge
            c = self.honest_verifier.challenge(challenge_bits)
            z = self.subverted_prover.prover_response(c)
            valid = self.honest_verifier.validate(t, z)

            total_time = time.time() - start_time
            time_values.append(total_time)

            l, b = aes_prf(self.bd_key, t, self.secret_key.bit_length(), self.bit_number // 8)

            if not valid:
                print("Verification failed")
                return

            transcripts.append((t, c, z))
            bit_counters[l] += 1
            min_counter = min(bit_counters)
            if min_counter > min_counter_old:
                min_counter_old = min_counter

        print("Number of rounds: ", self.num_rounds)

        return transcripts, time_values

def main():
    backdoor_key = os.urandom(32)  # Use a random 32-byte key as the backdoor key

    protocol = SchnorrIdentificationProtocol(backdoor_key)
    adversary = Adversary(protocol, backdoor_key)

    transcripts, time_values = protocol.simulate()

    recovered_x = adversary.obtain_secret(transcripts)

    adversary.save_attack(transcripts, protocol, time_values)

    # compare how many bits are different
    original_bits = [int(bit) for bit in bin(protocol.secret_key)[2:]]
    recovered_bits = recovered_x

    # compare
    diff = 0
    for i in range(len(original_bits)):
        if original_bits[i] != recovered_bits[i]:
            diff += 1

    print("Number of different bits: ", diff)

    recovered_x = int("".join([str(bit) for bit in recovered_x]), 2)

    print("Recovered secret key:", recovered_x)

    recovered_x = adversary.obtain_secret(transcripts)
    print("Original secret key number:", protocol.secret_key)
    print("Original secret key bits:", [int(bit) for bit in bin(protocol.secret_key)[2:]])
    print("Recovered secret key bits:", recovered_x)

    recovered_x = recovered_x[:len([int(bit) for bit in bin(protocol.secret_key)[2:]])]
    recovered_x = recovered_x[-1:] + recovered_x[:-1]
    recovered_x = int("".join([str(bit) for bit in recovered_x]), 2)

    print("Recovered secret key number:", recovered_x)

if __name__ == "__main__":
    main()


from copy import deepcopy
from typing import List
from typing import Optional
from typing import Tuple
from typing import Union

import numpy as np
from munch import Munch
from munch import munchify
from numpy import nan
from pandas import DataFrame
from plotly import graph_objects as go
from shapely.geometry import LineString
from shapely.geometry import MultiPoint
from shapely.geometry import MultiPolygon
from shapely.geometry import Point
from shapely.geometry import Polygon
from shapely.ops import nearest_points
from shapely.ops import unary_union

from app.lib.shapely_helper_functions import check_if_point_in_polygons
from app.lib.shapely_helper_functions import extend_line
from app.lib.shapely_helper_functions import extend_linestring
from app.lib.shapely_helper_functions import find_direction
from app.lib.shapely_helper_functions import find_perpendicular_direction
from app.lib.shapely_helper_functions import get_objects_in_polygon
from app.lib.shapely_helper_functions import get_unit_vector
from app.lib.shapely_helper_functions import get_unity_check_color
from app.lib.shapely_helper_functions import rotate_90_deg
from viktor.api_v1 import Entity
from viktor.api_v1 import EntityList
from viktor.core import UserException
from viktor.core import progress_message
from viktor.geometry import GeoPoint
from viktor.geometry import RDWGSConverter
from viktor.views import MapEntityLink
from viktor.views import MapFeature
from viktor.views import MapLabel
from viktor.views import MapPoint

from ..ditch.model import Ditch
from ..ditch.model import DitchHeffError
from ..ditch.model import DitchIntersectionLines
from ..ditch.model import DitchLargeBError
from ..ditch.utils import DitchPolygonIntersectionError
from ..ditch.utils import get_ditch_points_data
from ..dyke.dyke_model import LINE_SCALE
from ..dyke.dyke_model import Dyke
from ..exit_point.model import ExitPointProperties
from ..ground_model.model import build_combined_rep_and_exit_point_layout
from ..ground_model.tno_model import TNOGroundModel
from ..ground_model.tno_model import get_longitudinal_soil_layout
from ..ground_model.tno_model import get_min_max_permeabilities_from_soil_layout
from ..lib.ahn.ahn_helper_functions import get_xyz_df_from_multipoints
from ..lib.plotly_2d_profile_helper_functions import get_visualisation_along_trajectory
from ..lib.regis.regis_helper import get_longitudinal_regis_soil_layouts
from ..piping_tool.constants import PipingDataFrameColumns
from .constants import DEFAULT_PIPING_ERROR_RESULTS
from .constants import SPATIAL_RESOLUTION_SEGMENT_CHAINAGE
from .param_parser_functions import Scenario
from .param_parser_functions import get_piping_hydro_parameters
from .param_parser_functions import get_representative_soil_layouts
from .param_parser_functions import get_soil_scenario

TOL = 0.5  # meter tolerance to generate points along ditches lines

class Segment:
    def __init__(self, params: Munch, trajectory: LineString, dyke: Dyke, entry_line: Union[LineString, None]):
        self._params = params
        self.trajectory = trajectory  # coordinate system: RD
        self._dyke = dyke
        self._rotation_matrices = self._get_rotation_matrices
        self._entry_line = entry_line

    @property
    def interpolated_trajectory(self) -> LineString:
        """Returns the segmented trajectory, that is to say a LineString cut into portions of equal length based on
        the chainage resolution.
        This trajectory differs from the base trajectory of the segment that is uploaded but the two are very similar.
        This is not so much of a problem as long as the chainage resolution is small
        enough (25m in practice), which is acceptable.
        """
        base_trajectory = self.trajectory
        chainage_step = SPATIAL_RESOLUTION_SEGMENT_CHAINAGE
        nb_points = int(base_trajectory.length / chainage_step)

        # The last point of the base trajectory is added because it is not interpolated
        interpolated_trajectory = LineString(
            [base_trajectory.interpolate(n * chainage_step) for n in range(nb_points)] + [base_trajectory.coords[-1]]
        )
        return interpolated_trajectory

    def create_hinterland_hull(self, length: float, buffer_length_hinterland: float) -> list:
        """Return a shapely Polygon encapsulating the hinterland where length is the hinterland length, this polygon
        can potentially have interiors (i.e. holes)"""
        # TODO: consider memoizing this function?
        # note: vector switches direction if length is negative
        vector = length * self.perpendicular_unit_vector
        buffer_vector = buffer_length_hinterland * self.perpendicular_unit_vector

        # create the hinterland polygon
        hinterland = Polygon(
            [
                *(Point(np.array(point) + buffer_vector) for point in self.trajectory.coords),
                *(Point(np.array(point) + vector) for point in reversed(self.trajectory.coords)),
            ]
        )

        # remove the ditches if needed
        list_polygon = []
        if self._params.segment_ditches and self._params.segment_dry_ditches:
            for ditch in self._params.segment_ditches + self._params.segment_dry_ditches:
                poly_coord = ditch.ditch_polygon
                list_polygon.append(Polygon(poly_coord))
            ditch_polygons = MultiPolygon(list_polygon)
        else:
            ditch_polygons = MultiPolygon([])

        hinterland = hinterland.difference(ditch_polygons)

        if hinterland.geom_type == "MultiPolygon":
            hinterland_hull = list(hinterland.geoms)
        else:
            hinterland_hull = [hinterland]

        return hinterland_hull

    def create_foreland_hull(self, length: float, buffer_length_hinterland: float) -> Polygon:
        """create a geopolygon encapsulating the foreland"""
        # note: vector switches direction if length is negative
        vector = length * self.perpendicular_unit_vector * -1
        buffer_vector = buffer_length_hinterland * self.perpendicular_unit_vector

        # create the foreland polygon
        foreland = Polygon(
            [
                *(Point(np.array(point) + buffer_vector) for point in self.trajectory.coords),
                *(Point(np.array(point) + vector) for point in reversed(self.trajectory.coords)),
            ]
        )
        return foreland

    def create_ditches_hull(self, length: float) -> Polygon:
        """create geopolygon encapsulating the ditches buffer zone
        where length is the hinterland length"""
        # note: vector switches direction if length is negative
        vector = length * self.perpendicular_unit_vector

        # create the hinterland polygon
        hinterland = Polygon(
            [
                *(Point(point) for point in self.trajectory.coords),
                *(Point(np.array(point) + vector) for point in reversed(self.trajectory.coords)),
            ]
        )
        return hinterland

    @property
    def perpendicular_unit_vector(self) -> np.array:
        """get the unit vector perpendicular to the midpoint of the segment ** expressed  in RD ***
        in direction of hinterland"""
        middle_index = int(len(self.trajectory.coords) / 2)
        direction = find_perpendicular_direction(
            self.trajectory, middle_index, clockwise=not self._dyke.direction_to_river
        )  # not river = hinterland
        return get_unit_vector(self._origin_local_system_expressed_in_rd, direction)

    @property
    def parallel_unit_vector(self) -> np.array:
        """get the unit vector parallel to the midpoint of the segement ** expressed  in RD ***
        rotated clockwise from the perpendicular vector for right hand coordinate system"""
        midpoint = self._origin_local_system_expressed_in_rd
        direction = rotate_90_deg(midpoint, midpoint + self.perpendicular_unit_vector)
        return get_unit_vector(midpoint, direction)

    @property
    def _get_rotation_matrices(self) -> dict:
        """get the rotation matrix to transform between RD and the local coordinate system"""
        sin = self.parallel_unit_vector[0]
        cos = self.parallel_unit_vector[1]
        # because norm unit vector is 1 therefore it is on the unit circle

        return {"rd_to_local": np.array(((cos, -sin), (sin, cos))), "local_to_rd": np.array(((cos, sin), (-sin, cos)))}

    def transform_rd_to_local_coordinates(self, point_in_rd: np.array) -> np.array:
        """transform 2D point from RD to local coordinates, where the local system is defined by
        - origin at the midpoint of the segment
        - x parallel to the segment at the midpoint,
        - y perpendicular to the segment at the midpoint"""
        coords = self._rotation_matrices["rd_to_local"] @ (point_in_rd - self._origin_local_system_expressed_in_rd)
        return coords @ np.array([[0, -1], [1, 0]])

    def transform_local_coordinates_to_rd(self, point: np.array) -> np.array:
        """transform 2D point in local coordinates to RD"""
        coords = (self._rotation_matrices["local_to_rd"] @ point) @ np.array([[0, 1], [-1, 0]])
        return coords + self._origin_local_system_expressed_in_rd

    def fill_polygon_with_grid(self, polygon_hinterland: Polygon, delta_x: float, delta_y: float) -> MultiPoint:
        """Returns a multipoint containing the grid inside polygon to fill with a step of
        delta_x along the dyke and delta_y perpendicular to it"""
        polygon_to_fill = Polygon(polygon_hinterland.exterior)
        points_local_cs = [
            self.transform_rd_to_local_coordinates(np.array(point)) for point in polygon_to_fill.exterior.coords
        ]

        # find bounding box in local coordinate system
        x_min = min(p[0] for p in points_local_cs)
        x_max = max(p[0] for p in points_local_cs)
        y_min = min(p[1] for p in points_local_cs)
        y_max = max(p[1] for p in points_local_cs)

        # fill bounding box with points inside the hinterland polygon and outside the ditch
        exit_points = []
        for x in np.arange(x_min, x_max, delta_x):
            for y in np.arange(y_min, y_max, delta_y):
                pt = Point(self.transform_local_coordinates_to_rd(np.array([x, y])))
                if pt.within(
                    polygon_hinterland.buffer(0.1)
                ):  # this condition ALSO checks whether a point is inside a ditch or not
                    exit_points.append(pt)

        return polygon_to_fill.buffer(0.1).intersection(MultiPoint(exit_points))

    def get_exit_points_from_ditches(self) -> MultiPoint:
        if self._params.is_ditch_points:
            delta_ditch = self._params.delta_ditch
            exit_points = []

            for segment_ditch in self.all_ditches:
                ditch_line = LineString(segment_ditch.ditch_center_line)
                distances = np.arange(TOL, ditch_line.length - TOL, delta_ditch)
                points = [
                    ditch_line.interpolate(ditch_line.length - TOL)
                    if np.isclose(distance, ditch_line.length)
                    else ditch_line.interpolate(distance)
                    for distance in distances
                ]
                points = [(point.x, point.y) for point in points]
                exit_points.extend(points)
            return MultiPoint(exit_points)
        return MultiPoint([])

    def get_exit_points_from_grid_option_1(self) -> MultiPoint:
        """Return as a MultiPoint all the exit points generated from a grid. This grid generation ('option_1') is the
        first one implemented and has shown poor results. This generated has been kept here just in case."""
        if self._params.is_hinterland_grid:

            # transform hull to local coordinate system
            hinterland_polygon = self.create_hinterland_hull(
                self._params.exit_point_creation.exit_point_tab.length_hinterland,
                self._params.exit_point_creation.exit_point_tab.buffer_length_hinterland,
            )

            grid_points = []
            for polygon in hinterland_polygon:
                hinterland_grid = self.fill_polygon_with_grid(
                    polygon, self._params.hinterland_delta_x, self._params.hinterland_delta_y
                )
                for point in hinterland_grid.geoms:
                    grid_points.append(point)

            hinterland_grid = MultiPoint(grid_points)

            return hinterland_grid
        return MultiPoint([])

    def get_exit_points_from_manual_selection(self) -> MultiPoint:
        """Return as a MulitPoint all the exit points manually assigned on a MapView"""
        if self._params.manual_exit_point is None or not self._params.is_manual_exit_point:
            return MultiPoint([])
        exit_points = [Point(point.rd) for point in self._params.manual_exit_point.points]
        return MultiPoint(exit_points)

    def get_regular_grid_points_hinterland(
        self, resolution_x: float, resolution_y: float
    ) -> Union[MultiPoint, List[Point]]:
        """
        Return as a MultiPoint all the exit points generated from a grid. Points are generated more or less
        homogeneously in the hinterland region.
        :param resolution_x: resolution of the grid points in the direction parallel to the dike
        :param resolution_y: resolution of the grid points in the direction perpendicular to the dike
        :return:
        """

        # Get Hinterland start linestring

        _, _, direction = find_direction(
            self.trajectory, 0, clockwise=not self._dyke.direction_to_river
        )  # not river = hinterland
        side = "right" if direction > 0 else "left"
        hinterland_start_line = self.trajectory.parallel_offset(
            distance=self._params.exit_point_creation.exit_point_tab.buffer_length_hinterland, side=side
        )
        hinterland_end_line = self.trajectory.parallel_offset(
            distance=self._params.exit_point_creation.exit_point_tab.length_hinterland, side=side
        )

        # Partition of the hinterland start and end line
        distances = np.arange(0, hinterland_start_line.length, resolution_x)
        points_start_line = [hinterland_start_line.interpolate(distance) for distance in distances]
        multipoint_start_line = unary_union(points_start_line)
        points_end_line = [hinterland_end_line.interpolate(distance) for distance in distances]
        multipoint_end_line = unary_union(points_end_line)

        list_points = []
        for point_start, point_end in zip(multipoint_start_line, multipoint_end_line):
            perp_linestring = LineString([point_start, point_end])
            distances = np.arange(0, perp_linestring.length, resolution_y)
            points_on_line = [perp_linestring.interpolate(distance) for distance in distances]
            list_points.extend(points_on_line)

        return MultiPoint(list_points)

    def get_exit_points_from_grid_option_2(self) -> MultiPoint:
        if self._params.is_hinterland_grid:
            return self.get_regular_grid_points_hinterland(
                self._params.hinterland_delta_x, self._params.hinterland_delta_y
            )
        return MultiPoint([])

    def get_perpendicular_grid_points_hinterland(self, resolution_x: float, resolution_y: float) -> MultiPoint:
        """
        Return as a MultiPoint all the exit points generated from a grid. Points are generated on lines that are
        perpendicular to the segment's trajectory.
        :param resolution_x: resolution of the grid points in the direction parallel to the dike
        :param resolution_y: resolution of the grid points in the direction perpendicular to the dike
        :return:
        """
        LINE_OVER_EXTENSION = 500

        # Get Hinterland start linestring
        _, _, direction = find_direction(
            self.trajectory, 0, clockwise=not self._dyke.direction_to_river
        )  # not river = hinterland
        side = "right" if direction > 0 else "left"
        hinterland_start_line = self.trajectory.parallel_offset(
            distance=self._params.exit_point_creation.exit_point_tab.buffer_length_hinterland, side=side
        )
        hinterland_end_line = self.trajectory.parallel_offset(
            distance=self._params.exit_point_creation.exit_point_tab.length_hinterland, side=side
        )  # This line might be a bit short and must be extended on both sides to allow successful intersection

        hinterland_end_line = extend_line(
            line=hinterland_end_line,
            offset=20,
            side="start",
        )
        hinterland_end_line = extend_line(
            line=hinterland_end_line,
            offset=20,
            side="end",
        )

        # Partition of the hinterland start
        distances_start = np.arange(0, hinterland_start_line.length, resolution_x)
        points_start_line = [hinterland_start_line.interpolate(distance) for distance in distances_start]
        multipoint_start_line = unary_union(points_start_line)

        # Intersect the hinterland end line with perpendicular lines to the segment trajectory
        list_points = []
        for point_start in multipoint_start_line:

            projected_point_on_dike_trajectory = nearest_points(self.trajectory, point_start)[0]

            intersection_line = extend_linestring(
                LineString([projected_point_on_dike_trajectory, point_start]),
                LINE_OVER_EXTENSION,
            )

            projected_point_to_hinterland_endline = hinterland_end_line.intersection(intersection_line)
            if isinstance(projected_point_to_hinterland_endline, MultiPoint):  # error catching
                projected_point_to_hinterland_endline = projected_point_to_hinterland_endline[0]

            perp_linestring = LineString([point_start, projected_point_to_hinterland_endline])
            distances = np.arange(0, perp_linestring.length, resolution_y)
            points_on_line = [perp_linestring.interpolate(distance) for distance in distances]
            list_points.extend(points_on_line)
        return MultiPoint(list_points)

    def get_exit_points_from_grid_option_3(self) -> MultiPoint:
        if self._params.is_hinterland_grid:
            return self.get_perpendicular_grid_points_hinterland(
                self._params.hinterland_delta_x, self._params.hinterland_delta_y
            )
        return MultiPoint([])

    def find_lowest_point_hinterland(self) -> Tuple[float, float, float]:
        """Return the xyz coordinates of the lowest point in the hinterland from the AHN data"""
        all_grid_points = self.get_regular_grid_points_hinterland(
            self._params.exit_point_creation.exit_point_tab.resolution_lowest_point,
            self._params.exit_point_creation.exit_point_tab.resolution_lowest_point,
        )
        df_points = get_xyz_df_from_multipoints(all_grid_points)
        row_min_z = df_points.loc[df_points["z"].idxmin()]

        return row_min_z["x"], row_min_z["y"], row_min_z["z"]

    def get_all_draft_exit_point_locations(self) -> MultiPoint:
        """
        :return: Return all the draft exit points (both manually and automatically generated) as a single MultiPoint object.
        generated draft exit points are ordered according to the local coordinate system to facilitate proper naming
        """
        sorting_list = []

        if self._params.option_grid_hinterland == "option_1":
            exit_point_grids = self.get_exit_points_from_grid_option_1().geoms
        elif self._params.option_grid_hinterland == "option_2":
            exit_point_grids = self.get_exit_points_from_grid_option_2()
        elif self._params.option_grid_hinterland == "option_3":
            exit_point_grids = self.get_exit_points_from_grid_option_3()
        else:
            raise ValueError
        for point in exit_point_grids:
            x = point.x
            y = point.y
            x_local = round(self.transform_rd_to_local_coordinates(np.array([point.x, point.y]))[0], 1)
            y_local = round(self.transform_rd_to_local_coordinates(np.array([point.x, point.y]))[1], 1)
            sorting_list.append([x, y, x_local, y_local])

        # Sort draft exit points based on y_local (hence i[3])
        sorting_list = sorted(sorting_list, key=lambda i: i[3])

        points_list = [(Point(element[0], element[1])) for element in sorting_list]
        points_list.extend(self.get_exit_points_from_manual_selection().geoms)
        points_list.extend(self.get_exit_points_from_ditches().geoms)
        if self._params.is_lowest_point:
            x, y, _ = self.find_lowest_point_hinterland()
            points_list.extend([Point(x, y)])

        return MultiPoint(points_list)

    @property
    def _origin_local_system_expressed_in_rd(self) -> np.array:
        """coordinates of the origin of the local coordinate system in RD, at the midpoint of the segment"""
        middle_index = int(len(self.trajectory.coords) / 2)
        origin = np.array(self.trajectory.coords[middle_index])  # origin local coordinate system in RD
        return origin

    @property
    def all_ditches(self):
        if not self._params.segment_ditches and not self._params.segment_dry_ditches:
            raise UserException(
                "Geen sloten geselecteerd, gebruik de 'snij sloten' knop " "in Input Selectie> Selecteer Sloten"
            )

        all_ditches = []
        if self._params.segment_ditches is not None:
            all_ditches.extend(self._params.segment_dry_ditches)
        if self._params.segment_ditches is not None:
            all_ditches.extend(self._params.segment_ditches)
        return all_ditches

    @property
    def get_ditches_as_multipolygons(self) -> Tuple[MultiPolygon, MultiPolygon]:
        """Return Multipolygons containing the ditches if the Toggle 'params.select_with_buffer_zone' is switched on, or empty MultiPolygons otherwise."""
        if self._params.select_with_buffer_zone:
            if self._params.segment_ditches is None or self._params.segment_dry_ditches is None:
                raise UserException("Sloten dienen te worden gedefinieerd")

            segment_ditches_pol = MultiPolygon([Polygon(pol["ditch_polygon"]) for pol in self._params.segment_ditches])
            segment_dry_ditches_pol = MultiPolygon(
                [Polygon(pol["ditch_polygon"]) for pol in self._params.segment_dry_ditches]
            )
        else:
            segment_ditches_pol = MultiPolygon([])
            segment_dry_ditches_pol = MultiPolygon([])
        return segment_ditches_pol, segment_dry_ditches_pol

    def get_segment_2D_longitudinal_profile(
        self,
        tno_ground_model: TNOGroundModel,
        chainage_step: float,
        cpt_folder: Optional[Entity] = None,
        bore_folder: Optional[Entity] = None,
    ) -> go.Figure:
        """Get the 2D longitudinal figure of the segment trajectory"""
        buffer_polygon = self.interpolated_trajectory.buffer(self._dyke.params.buffer_zone_cpts_bore)
        if cpt_folder:
            cpt_entity_list = cpt_folder.children(entity_type_names=["CPT"])
            cpt_entity_list = get_objects_in_polygon(cpt_entity_list, buffer_polygon)
            cpt_params = dict(
                cpt_entity_list=cpt_entity_list,
                scale_measurement_cpt_qc=5 * LINE_SCALE,
                scale_measurement_cpt_rf=5 * LINE_SCALE,
                scale_measurement_cpt_u2=100 * LINE_SCALE,
            )
        else:
            cpt_params = None
        if bore_folder:
            bore_entity_list = bore_folder.children(entity_type_names=["Bore"])
            bore_entity_list = get_objects_in_polygon(bore_entity_list, buffer_polygon)
        else:
            bore_entity_list = None
        long_profile = get_longitudinal_soil_layout(self.interpolated_trajectory, tno_ground_model)
        min_perm_v, max_perm_v, min_perm_h, max_perm_h = get_min_max_permeabilities_from_soil_layout(long_profile)

        min_max_permeabilities = dict(
            min_perm_v=min_perm_v, max_perm_v=max_perm_v, min_perm_h=min_perm_h, max_perm_h=max_perm_h
        )
        regis_soil_layouts = get_longitudinal_regis_soil_layouts(
            buffer_polygon, self.interpolated_trajectory, self._params.input_selection.materials.bottom_level_query
        )

        return get_visualisation_along_trajectory(
            trajectory=self.interpolated_trajectory,
            longitudinal_soil_layout=long_profile,
            min_max_permeabilities=min_max_permeabilities,
            materials_table=self._params.input_selection.materials.table,
            chainage_step=chainage_step,
            classification_table=self._params.input_selection.materials.classification_table,
            vertical_line_location=self._params.input_selection.soil_schematization.distance_from_start,
            cpt_params=cpt_params,
            bore_entity_list=bore_entity_list,
            regis_layouts=regis_soil_layouts,
        )

    def get_ditch(self, coordinates: Tuple[float, float]) -> Optional[Ditch]:
        # Get ditch if Exit point is in a ditch
        segment_ditches_pol, segment_dry_ditches_pol = self.get_ditches_as_multipolygons
        point = Point(coordinates)
        check_bool, type_ditch = check_if_point_in_polygons(segment_ditches_pol, segment_dry_ditches_pol, point)
        if check_bool is True:
            if type_ditch == "wet":
                ditch_points, talu_slope = get_ditch_points_data(
                    self._params.segment_ditches, point, self._params.polder_level
                )
                ditch_param = {
                    "ditch_points": ditch_points,
                    "talu_slope": talu_slope,
                    "is_wet": True,
                }
            else:
                ditch_points, talu_slope = get_ditch_points_data(
                    self._params.segment_dry_ditches, point, self._params.polder_level
                )
                ditch_param = {
                    "ditch_points": ditch_points,
                    "talu_slope": talu_slope,
                    "is_wet": False,
                }

            if ditch_param.get("ditch_points") and ditch_param.get("is_wet") is not None:
                ditch = Ditch(
                    *ditch_param.get("ditch_points"),
                    is_wet=ditch_param.get("is_wet"),
                    talu_slope=ditch_param.get("talu_slope"),
                )
            else:
                ditch = None
        else:
            ditch = None
        return ditch

    def get_all_scenarios(
        self, leakage_length_array: List[Munch], geohyromodel: str, scenario_name: Optional[str] = None
    ) -> List:
        """
        Append the leakaage length to the scenario from the soil schematization and return the complete parametrization
        of the scenarios.
        :param leakage_length_array: DynamicArray containing the leakage lengths, structure is:
        [{"leakage_length_foreland_aquifer_1": float,
          "leakage_length_foreland_aquifer_2": float,
          "leakage_length_hinterland_aquifer_1": float,
          "leakage_length_hinterland_aquifer_2": float,
        }]
        :param geohyromodel: one of ['1', '2', '3']. Describe the level of detail for the geohydrological model.
        :param scenario_name: Optional, provide the index of a scenario if only this scenario should be returned.
        :return: list of Scenario objects

        """
        # If only a single scenario based on its index must be returned
        if scenario_name is not None:
            soil_scenario = get_soil_scenario(
                scenario_name, self._params.input_selection.soil_schematization.soil_scen_array
            )
            leakage_lengths = None

            # If model 2 selected, then fetch the corresponding leakage length row of the DynamicArray
            if geohyromodel == "2":
                for row in leakage_length_array:
                    if row.scenario == scenario_name:
                        leakage_lengths = row
                        break
                if leakage_lengths is None:
                    raise UserException("Selected scenario has no leakage lengths.")  # TODO TRANSLATE
            elif geohyromodel in ["0", "1"]:
                pass
            else:
                raise NotImplementedError
            try:
                return [Scenario(soil_scenario, geohyromodel, leakage_lengths)]
            except IndexError:
                raise UserException("Missing leakage length for at least one scenario")  # TODO TRANSLATE

        # if all scenarios should be returned in a list
        scenarios_soil = self._params.input_selection.soil_schematization.soil_scen_array
        if geohyromodel in ["0", "1"]:
            return [Scenario(scenario_soil, geohyromodel) for scenario_soil in scenarios_soil]
        elif geohyromodel == "2":
            compiled_scenarios = []
            for scenario in scenarios_soil:
                for leakage_length_row in leakage_length_array:
                    if leakage_length_row.scenario == scenario.name_of_scenario:
                        compiled_scenarios.append(Scenario(scenario, geohyromodel, leakage_length_row))
            return compiled_scenarios
        else:
            raise NotImplementedError

    def get_serialized_piping_results(self, exit_point_list: Union[EntityList, List[Entity]]) -> List[dict]:
        """
        Return all the piping results (Uplift, heave, Sellmeijer) for every aquifer of every exit point for every
        scenarios in a serialized format easily convertible into a DataFrame
        :param exit_point_list: List of ExitPoint entities to iterate
        :return:
        """
        scenario_index = (
            None
            if self._params.calculations.soil_profile.results_settings.composite_result_switch
            else self._params.calculations.soil_profile.results_settings.scenario_selection
        )
        scenarios = self.get_all_scenarios(
            self._params.soil_schematization.geohydrology.level2.leakage_length_array,
            geohyromodel=self._params.geohydrology_method,
            scenario_name=scenario_index,
        )
        result_list = []
        for j, scenario in enumerate(scenarios, 0):
            piping_hydro_parameters = munchify(get_piping_hydro_parameters(self._params))
            for i, exit_point in enumerate(exit_point_list, 1):
                progress_message(
                    f"{scenario.name_of_scenario} \n\n{exit_point.name}\n\n{i + j * len(exit_point_list)}/{len(exit_point_list) * len(scenarios)}"
                )

                exit_point_params = exit_point.last_saved_params
                coordinates = (
                    exit_point_params.exit_point_data.x_coordinate,
                    exit_point_params.exit_point_data.y_coordinate,
                )

                _, rep_soil_layout = get_representative_soil_layouts(self._params, scenario)
                soil_layout_piping = build_combined_rep_and_exit_point_layout(
                    exit_point_params.get("classified_soil_layout"), rep_soil_layout
                )
                try:
                    res_list = ExitPointProperties(
                        soil_layout_piping=soil_layout_piping.serialize(),
                        coordinates=coordinates,
                        dyke=self._dyke,
                        ditch=self.get_ditch(coordinates),
                        leakage_lengths=scenario.leakage_lengths,
                    ).get_exit_point_summary_piping_results(piping_hydro_parameters)
                    for piping_calculation in res_list:
                        piping_calculation[PipingDataFrameColumns.EXIT_POINT.value] = exit_point
                        piping_calculation["scenario"] = scenario
                        piping_calculation["scenario_name"] = scenario.name_of_scenario
                        result_list.append(piping_calculation)
                except (DitchHeffError, DitchLargeBError, DitchIntersectionLines, DitchPolygonIntersectionError):
                    piping_calculation = deepcopy(
                        DEFAULT_PIPING_ERROR_RESULTS
                    )  # deepcopy necessary here otherwise piping_calculation is getting overwritten
                    piping_calculation[PipingDataFrameColumns.EXIT_POINT.value] = exit_point
                    piping_calculation["scenario"] = scenario
                    piping_calculation["scenario_name"] = scenario.name_of_scenario
                    result_list.append(piping_calculation)

        return result_list

    def get_map_features_for_uncombined_piping_results(
        self,
        serialized_piping_res: List[dict],
        scenario_name: str,
        calculation_type: Optional[str] = None,
    ) -> Tuple[List[MapFeature], List[MapLabel]]:
        """
        Build the map features and labels for the uncombined piping calculations
        :param serialized_piping_res: Full piping results for all scenarios, all exit point and all aquifer
        :param scenario_name: name of the scenario for which the piping result nust be filtered
        :param calculation_type: one of ["uplift", "heave", "sellmeijer"]
        """
        map_features, map_labels = [], []

        # Filter the results for the selected scenario
        df_res = DataFrame.from_records(
            serialized_piping_res, columns=[col.value for col in PipingDataFrameColumns] + ["scenario_name"]
        )
        sorted_df = df_res[df_res["scenario_name"] == scenario_name]
        # Build the markers for the MapView
        description = ""
        uc_list = []
        for _, row in sorted_df.iterrows():  # iterrows() is code smell
            exit_point = row[PipingDataFrameColumns.EXIT_POINT.value]
            if row["aquifer"] == 1:
                description = f"## {exit_point.name} \n\n " + self.get_description_uncombined_results(
                    row, aquifer=1, calculation_type=calculation_type
                )
                uc_list = self.get_uc_list_uncombined_results(row, calculation_type=calculation_type)
            else:
                map_features.pop()
                map_labels.pop()
                description += self.get_description_uncombined_results(
                    row, aquifer=2, calculation_type=calculation_type
                )
                uc_list.extend(self.get_uc_list_uncombined_results(row, calculation_type=calculation_type))

            coordinates = (
                exit_point.last_saved_params.exit_point_data.x_coordinate,
                exit_point.last_saved_params.exit_point_data.y_coordinate,
            )
            map_features.append(
                MapPoint.from_geo_point(
                    GeoPoint.from_rd(coordinates),
                    color=get_unity_check_color(uc_list),
                    description=description,
                    entity_links=[MapEntityLink("Na Uitredepunt", entity_id=exit_point.id)],
                )
            )
            lat, lon = RDWGSConverter.from_rd_to_wgs(coordinates)
            map_labels.append(MapLabel(lat, lon, scale=17, text=f"{exit_point.name[13:]}"))

        return (
            map_features,
            map_labels,
        )

    @staticmethod
    def get_uc_list_uncombined_results(row, calculation_type: Optional[str] = None) -> List[float]:
        if calculation_type is None:
            uc_list = [row["uc_opbarsten"], row["uc_heave"], row["uc_sellmeijer"]]
        elif calculation_type == "uplift":
            uc_list = [row["uc_opbarsten"]]
        elif calculation_type == "heave":
            uc_list = [row["uc_heave"]]
        elif calculation_type == "sellmeijer":
            uc_list = [row["uc_sellmeijer"]]
        else:
            raise ValueError
        return uc_list

    @staticmethod
    def get_description_uncombined_results(row, aquifer: int, calculation_type: Optional[str] = None) -> str:
        if calculation_type is None:
            description = f"### Aquifer {aquifer}  \n\n Opbarsten: {cut_off_float(row['uc_opbarsten'])} \\\n Heave: {cut_off_float(row['uc_heave'])} \\\n Sellmeijer {cut_off_float(row['uc_sellmeijer'])} \n\n"
        elif calculation_type == "uplift":
            description = f"### Aquifer {aquifer}  \n\n Opbarsten: {cut_off_float(row['uc_opbarsten'])} \n\n"
        elif calculation_type == "heave":
            description = f"### Aquifer {aquifer}  \n\n Heave: {cut_off_float(row['uc_heave'])} \n\n"
        elif calculation_type == "sellmeijer":
            description = f"### Aquifer {aquifer}  \n\n Sellmeijer {cut_off_float(row['uc_sellmeijer'])} \n\n"
        else:
            raise ValueError
        return description

    def get_map_features_for_combined_piping_results(
        self, serialized_piping_res: List[dict], calculation_type: Optional[str] = None
    ) -> Tuple[List[MapFeature], List[MapLabel]]:
        """Build the map features and labels for the combined piping calculations
        :param serialized_piping_res: Full piping results for all scenarios, all exit point and all aquifer
        """
        # Filter the results for the selected scenario
        df_res = DataFrame.from_records(
            serialized_piping_res, columns=[col.value for col in PipingDataFrameColumns] + ["scenario_name"]
        )
        df_res = df_res.replace("nan", nan)

        # Add exit point name to dataframe
        def add_exit_point_name(row):
            return row.name

        df_res["exit_point_name"] = df_res[PipingDataFrameColumns.EXIT_POINT.value].apply(add_exit_point_name)
        unique_exit_point_entities = df_res.drop_duplicates(subset=["exit_point_name"])[
            PipingDataFrameColumns.EXIT_POINT.value
        ].to_list()
        exit_point_mapping = {exit_point.name: exit_point for exit_point in unique_exit_point_entities}
        sorted_df = df_res.sort_values(by=["exit_point_name"])

        # Separate first and second aquifer in different dataframes
        aq_1_df = sorted_df[sorted_df["aquifer"] == 1].sort_values(by=["scenario_name"])
        aq_2_df = sorted_df[sorted_df["aquifer"] == 2].sort_values(by=["scenario_name"])
        aq_1_df["w_sellmeijer"] = aq_1_df.apply(
            lambda x: (x["scenario"].weight_of_scenario * x["uc_sellmeijer"]), axis=1
        )
        aq_1_df["w_uplift"] = aq_1_df.apply(lambda x: (x["scenario"].weight_of_scenario * x["uc_opbarsten"]), axis=1)
        aq_1_df["w_heave"] = aq_1_df.apply(lambda x: (x["scenario"].weight_of_scenario * x["uc_heave"]), axis=1)

        if calculation_type is None:
            final_aq_1 = (
                aq_1_df.groupby("exit_point_name")["w_sellmeijer", "w_uplift", "w_heave"]
                .sum()
                .replace(0.0, nan)
                .to_dict("index")
            )
        else:
            final_aq_1 = (
                (aq_1_df.groupby("exit_point_name")["w_" + calculation_type].sum().replace(0.0, nan))
                .to_frame()
                .to_dict("index")
            )  # groupby returns a series when only one argument is provided

        if len(aq_2_df) > 0:
            aq_2_df["w_sellmeijer"] = aq_2_df.apply(
                lambda x: (x["scenario"].weight_of_scenario * x["uc_sellmeijer"]), axis=1
            )
            aq_2_df["w_uplift"] = aq_2_df.apply(
                lambda x: (x["scenario"].weight_of_scenario * x["uc_opbarsten"]), axis=1
            )
            aq_2_df["w_heave"] = aq_2_df.apply(lambda x: (x["scenario"].weight_of_scenario * x["uc_heave"]), axis=1)
            if calculation_type is None:
                final_aq_2 = (
                    aq_2_df.groupby("exit_point_name")["w_sellmeijer", "w_uplift", "w_heave"].sum().to_dict("index")
                )
            else:
                final_aq_2 = (
                    (aq_2_df.groupby("exit_point_name")["w_" + calculation_type].sum().replace(0.0, nan))
                    .to_frame()
                    .to_dict("index")
                )
            return make_marker_double_aquifer(final_aq_1, final_aq_2, exit_point_mapping, calculation_type)

        return make_marker_single_aquifer(final_aq_1, exit_point_mapping, calculation_type)

def make_marker_double_aquifer(
    final_aq_1: DataFrame, final_aq_2: DataFrame, exit_point_mapping: dict, calculation_type: Optional[str] = None
) -> Tuple[List[MapFeature], List[MapLabel]]:
    map_features, map_labels = [], []
    for key, aq_1, aq_2 in common_entries(final_aq_1, final_aq_2):
        exit_point = exit_point_mapping[key]
        coordinates = (
            exit_point.last_saved_params.exit_point_data.x_coordinate,
            exit_point.last_saved_params.exit_point_data.y_coordinate,
        )
        uc_list = list(aq_1.values()) + list(aq_2.values())

        map_features.append(
            MapPoint.from_geo_point(
                GeoPoint.from_rd(coordinates),
                color=get_unity_check_color(uc_list),
                description=get_marker_description_double_aquifer(exit_point, aq_1, aq_2, calculation_type),
                entity_links=[MapEntityLink("Na Uitredepunt", entity_id=exit_point.id)],
            )
        )
        lat, lon = RDWGSConverter.from_rd_to_wgs(coordinates)
        map_labels.append(MapLabel(lat, lon, scale=17, text=f"{exit_point.name[13:]}"))
    return (
        map_features,
        map_labels,
    )

def get_marker_description_double_aquifer(
    exit_point: Entity, aq_1: DataFrame, aq_2: DataFrame, calculation_type: Optional[str]
) -> str:
    if calculation_type is None:
        description = f"## {exit_point.name} \n\n ### Aquifer 1 \n\n Opbarsten: {cut_off_float(aq_1['w_uplift'])} \\\n  Heave: {cut_off_float(aq_1['w_heave'])} \\\n Sellmeijer {cut_off_float(aq_1['w_sellmeijer'])} \n\n"
        description += f"### Aquifer 2 \n\n Opbarsten: {cut_off_float(aq_2['w_uplift'])} \\\n  Heave: {cut_off_float(aq_2['w_heave'])} \\\n Sellmeijer {cut_off_float(aq_2['w_sellmeijer'])}"
    elif calculation_type == "uplift":
        description = f"## {exit_point.name} \n\n ### Aquifer 1 \n\n Opbarsten: {cut_off_float(aq_1['w_uplift'])} \n\n"
        description += f"### Aquifer 2 \n\n Opbarsten: {cut_off_float(aq_2['w_uplift'])}"
    elif calculation_type == "heave":
        description = f"## {exit_point.name} \n\n ### Aquifer 1 \n\n Heave: {cut_off_float(aq_1['w_heave'])} \n\n"
        description += f"### Aquifer 2 \n\n Heave: {cut_off_float(aq_2['w_heave'])}"
    elif calculation_type == "sellmeijer":
        description = (
            f"## {exit_point.name} \n\n ### Aquifer 1 \n\n Sellmeijer {cut_off_float(aq_1['w_sellmeijer'])} \n\n"
        )
        description += f"### Aquifer 2 \n\n Sellmeijer {cut_off_float(aq_2['w_sellmeijer'])}"
    else:
        raise ValueError
    return description

def make_marker_single_aquifer(
    final_aq_1: DataFrame, exit_point_mapping: dict, calculation_type: Optional[str] = None
) -> Tuple[List[MapFeature], List[MapLabel]]:
    map_features, map_labels = [], []
    for key, aq_1 in common_entries(final_aq_1):
        exit_point = exit_point_mapping[key]
        coordinates = (
            exit_point.last_saved_params.exit_point_data.x_coordinate,
            exit_point.last_saved_params.exit_point_data.y_coordinate,
        )
        uc_list = list(aq_1.values())

        map_features.append(
            MapPoint.from_geo_point(
                GeoPoint.from_rd(coordinates),
                color=get_unity_check_color(uc_list),
                description=get_marker_description_single_aquifer(exit_point, aq_1, calculation_type),
                entity_links=[MapEntityLink("Na Uitredepunt", entity_id=exit_point.id)],
            )
        )
        lat, lon = RDWGSConverter.from_rd_to_wgs(coordinates)
        map_labels.append(MapLabel(lat, lon, scale=17, text=f"{exit_point.name[13:]}"))
    return (
        map_features,
        map_labels,
    )

def get_marker_description_single_aquifer(exit_point: Entity, aq_1: DataFrame, calculation_type: Optional[str]) -> str:
    if calculation_type is None:
        description = f"## {exit_point.name} \n\n ### Aquifer 1 \n\n Opbarsten: {cut_off_float(aq_1['w_uplift'])} \\\n  Heave: {cut_off_float(aq_1['w_heave'])} \\\n Sellmeijer {cut_off_float(aq_1['w_sellmeijer'])} \n\n"
    elif calculation_type == "uplift":
        description = f"## {exit_point.name} \n\n ### Aquifer 1 \n\n Opbarsten: {cut_off_float(aq_1['w_uplift'])}"

    elif calculation_type == "heave":
        description = f"## {exit_point.name} \n\n ### Aquifer 1 \n\n Heave: {cut_off_float(aq_1['w_heave'])}"

    elif calculation_type == "sellmeijer":
        description = (
            f"## {exit_point.name} \n\n ### Aquifer 1 \n\n Sellmeijer {cut_off_float(aq_1['w_sellmeijer'])} \n\n"
        )

    else:
        raise ValueError
    return description

def common_entries(*dcts):
    """Make it easy to zip multiple dictionaries"""
    if not dcts:
        return
    for i in set(dcts[0]).intersection(*dcts[1:]):
        yield (i,) + tuple(d[i] for d in dcts)

def cut_off_float(value: Union[float, str]):
    if isinstance(value, float):
        return round(value, 2)
    elif isinstance(value, str):
        return value
    raise TypeError


from simulationAPI.serializers import TaskSerializer, \
    simulationSerializer, simulationSaveSerializer
from simulationAPI.tasks import process_task
from rest_framework.permissions import AllowAny, IsAuthenticated
from rest_framework.parsers import MultiPartParser, FormParser
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from django.conf import settings
from rest_framework.exceptions import ValidationError
from celery.result import AsyncResult
from saveAPI.models import StateSave
import uuid
from .models import runtimeStat, Limit, simulation
from ltiAPI.models import ltiSession
import celery.signals
from celery import current_task
import time
import math
import os
import logging

logger = logging.getLogger(__name__)

def saveNetlistDB(task_id, filepath, request):
    current_dir = settings.FILE_STORAGE_ROOT
    filepath = filepath.split('/')[-1]
    os.chdir(current_dir)
    f = open(filepath, "r")
    temp = f.read()
    if request.user.is_authenticated:
        owner = request.user.id
    else:
        owner = None
    if request.data.get('simulationType', None):
        simulation_type = request.data['simulationType']
    else:
        simulation_type = "NgSpiceSimulator"
    if request.data.get('save_id', None):
        if 'gallery' in request.data.get('save_id'):
            save_id = None
        else:
            save_id = StateSave.objects.get(
                save_id=request.data['save_id'],
                version=request.data['version'],
                branch=request.data['branch']).id
    else:
        save_id = None
    lti_session = None
    if request.data.get('lti_id', None):
        lti_session = ltiSession.objects.get(id=request.data['lti_id'])
    serialized = simulationSaveSerializer(
        data={"task": task_id, "netlist": temp, "owner": owner,
              "simulation_type": simulation_type, "schematic": save_id})
    if serialized.is_valid(raise_exception=True):
        serialized.save()
        if lti_session:
            lti_session.simulations.add(
                simulation.objects.get(id=serialized.data['id']))
        return
    else:
        return Response(serialized.errors)

class NetlistUploader(APIView):
    '''
    API for NetlistUpload

    Requires a multipart/form-data  POST Request with netlist file in the
    'file' parameter
    '''
    permission_classes = (AllowAny,)
    parser_classes = (MultiPartParser, FormParser,)

    def post(self, request, *args, **kwargs):
        logger.info('Got POST for netlist upload: ')
        logger.info(request.data)
        serializer = TaskSerializer(data=request.data, context={'view': self})

        limits = Limit.objects.all()
        TIME_LIMIT = 0
        if limits.exists():
            TIME_LIMIT = Limit.objects.all()[0].timeLimit
        # if timeLimit.objects.count() != 0:
        #     TIME_LIMIT = timeLimit.objects.all()[0]
        #     print('NOT NONE')
        # else:
        #     print('NONE')
        if serializer.is_valid():
            serializer.save()
            saveNetlistDB(
                serializer.data['task_id'], serializer.data['file'][0]['file'],
                request)
            task_id = serializer.data['task_id']
            if(TIME_LIMIT == 0):
                celery_task = process_task.apply_async(
                    kwargs={'task_id': str(task_id)}, task_id=str(task_id)
                )
            else:
                celery_task = process_task.apply_async(
                    kwargs={'task_id': str(task_id)}, task_id=str(task_id),
                    soft_time_limit=TIME_LIMIT)

            response_data = {
                'state': celery_task.state,
                'details': serializer.data,
            }
            return Response(response_data)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class CeleryResultView(APIView):
    """

    Returns Simulation results for 'task_id' provided after
    uploading the netlist
    /api/task/<uuid>

    """
    permission_classes = (AllowAny,)
    methods = ['GET']

    def get(self, request, task_id):

        if isinstance(task_id, uuid.UUID):
            celery_result = AsyncResult(str(task_id))
            response_data = {
                'state': celery_result.state,
                'details': celery_result.info
            }
            try:
                Output = simulation.objects.get(task__task_id=task_id)
                Output.result = celery_result.info
                Output.save()
            except simulation.DoesNotExist:
                pass
            return Response(response_data)
        else:
            raise ValidationError('Invalid uuid format')

class SimulationResults(APIView):
    permission_classes = (IsAuthenticated, )

    def get(self, request, save_id, sim, version, branch):
        if sim is None:
            sims = simulation.objects.filter(
                owner=self.request.user, schematic__save_id=save_id,
                schematic__version=version, schematic__branch=branch
            )
        else:
            sims = simulation.objects.filter(
                owner=self.request.user, schematic__save_id=save_id,
                schematic__version=version, schematic__branch=branch
            )
        serialized = simulationSerializer(sims, many=True)
        return Response(serialized.data, status=status.HTTP_200_OK)

class SimulationResultsForLTI(APIView):
    permission_classes = (IsAuthenticated, )

    def get(self, request, save_id, sim, version, branch):
        if sim is None:
            sims = simulation.objects.filter(
                owner=self.request.user, schematic__save_id=save_id
            )
        else:
            sims = simulation.objects.filter(
                owner=self.request.user, schematic__save_id=save_id
            )
        serialized = simulationSerializer(sims, many=True)
        return Response(serialized.data, status=status.HTTP_200_OK)

class SimulationResultsFromSimulator(APIView):
    permission_classes = (IsAuthenticated, )

    def get(self, request, sim):
        sims = simulation.objects.filter(
            owner=self.request.user, simulation_type=sim)
        serialized = simulationSerializer(sims, many=True)
        return Response(serialized.data, status=status.HTTP_200_OK)

class GetLTISimResults(APIView):
    permission_classes = (AllowAny, )

    def get(self, request, lti_id):
        try:
            session = ltiSession.objects.get(id=lti_id)
            serialized = simulationSerializer(
                session.simulations.all(), many=True)
            return Response(serialized.data, status=status.HTTP_200_OK)
        except ltiSession.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

@ celery.signals.task_prerun.connect
def statsd_task_prerun(task_id, **kwargs):
    current_task.start_time = time.time()

@ celery.signals.task_postrun.connect
def statsd_task_postrun(task_id, **kwargs):
    runtime = time.time() - current_task.start_time
    runtime = math.ceil(runtime)
    statObj, created = runtimeStat.objects.get_or_create(exec_time=runtime)
    statObj.qty += 1
    statObj.save()


import cv2
import math
import torch
import numpy as np
import segmentation_models_pytorch as smp
from torch import nn
from torch.optim.optimizer import Optimizer

UNET_CHANNELS = [256, 128, 64, 32, 16]
CNN_FEATURES = {
    'resnet34': 512,
    'resnet50': 2048,
    'efficientnet-b0': 320,
    'efficientnet-b1': 320,
    'efficientnet-b2': 352,
    'efficientnet-b3': 384,
    'efficientnet-b4': 448,
    'efficientnet-b5': 512,
    'se_resnext50_32x4d': 2048,
    'se_resnext101_32x4d': 4096,
}

class MultiTaskModel(nn.Module):
    def __init__(self, model, num_features, num_classes=4, topk=10):
        super(MultiTaskModel, self).__init__()
        self.encoder = model.encoder
        self.decoder = model.decoder
        self.topk = topk

    def forward(self, x):
        features = self.encoder(x)
        masks = self.decoder(features)
        logits = torch.topk(masks.view(masks.shape[0], masks.shape[1], -1), self.topk)[0].mean(2)
        return logits, masks

def get_model(encoder_name='efficientnet-b3', decoder_name='unet', unet_atten=False, fpn_dropout=.2):
    model_constructor = smp.Unet if decoder_name == 'unet' else smp.FPN
    if decoder_name == 'unet':
        model = model_constructor(encoder_name=encoder_name, classes=4, activation=None,
                                  attention_type='scse' if unet_atten else None, decoder_channels=UNET_CHANNELS)
    elif decoder_name == 'fpn':
        model = model_constructor(encoder_name=encoder_name, classes=4, activation=None, dropout=fpn_dropout)
    cnn_features = CNN_FEATURES[encoder_name]
    model = MultiTaskModel(model, cnn_features, num_classes=4)
    return model

def load_model(checkpoint, encoder_name='efficientnet-b3', decoder_name='unet', unet_atten=False, fpn_dropout=.2, use_gpu=True):
    model = get_model(encoder_name=encoder_name, decoder_name=decoder_name, unet_atten=unet_atten, fpn_dropout=fpn_dropout)
    model_state = torch.load(checkpoint)
    model.load_state_dict(model_state)
    if use_gpu:
        model = model.cuda()
    model = model.eval()
    return model

class RAdam(Optimizer):
    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, degenerated_to_sgd=True):
        if not 0.0 <= lr:
            raise ValueError("Invalid learning rate: {}".format(lr))
        if not 0.0 <= eps:
            raise ValueError("Invalid epsilon value: {}".format(eps))
        if not 0.0 <= betas[0] < 1.0:
            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
        if not 0.0 <= betas[1] < 1.0:
            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))

        self.degenerated_to_sgd = degenerated_to_sgd
        if isinstance(params, (list, tuple)) and len(params) > 0 and isinstance(params[0], dict):
            for param in params:
                if 'betas' in param and (param['betas'][0] != betas[0] or param['betas'][1] != betas[1]):
                    param['buffer'] = [[None, None, None] for _ in range(10)]
        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, buffer=[[None, None, None]
                                                                                for _ in range(10)])
        super(RAdam, self).__init__(params, defaults)

    def __setstate__(self, state):
        super(RAdam, self).__setstate__(state)

    def step(self, closure=None):

        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:

            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data.float()
                if grad.is_sparse:
                    raise RuntimeError('RAdam does not support sparse gradients')

                p_data_fp32 = p.data.float()

                state = self.state[p]

                if len(state) == 0:
                    state['step'] = 0
                    state['exp_avg'] = torch.zeros_like(p_data_fp32)
                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
                else:
                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)

                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
                beta1, beta2 = group['betas']

                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)
                exp_avg.mul_(beta1).add_(1 - beta1, grad)

                state['step'] += 1
                buffered = group['buffer'][int(state['step'] % 10)]
                if state['step'] == buffered[0]:
                    N_sma, step_size = buffered[1], buffered[2]
                else:
                    buffered[0] = state['step']
                    beta2_t = beta2 ** state['step']
                    N_sma_max = 2 / (1 - beta2) - 1
                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)
                    buffered[1] = N_sma

                    # more conservative since it's an approximated value
                    if N_sma >= 5:
                        step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])
                    elif self.degenerated_to_sgd:
                        step_size = 1.0 / (1 - beta1 ** state['step'])
                    else:
                        step_size = -1
                    buffered[2] = step_size

                # more conservative since it's an approximated value
                if N_sma >= 5:
                    if group['weight_decay'] != 0:
                        p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
                    denom = exp_avg_sq.sqrt().add_(group['eps'])
                    p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)
                    p.data.copy_(p_data_fp32)
                elif step_size > 0:
                    if group['weight_decay'] != 0:
                        p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
                    p_data_fp32.add_(-step_size * group['lr'], exp_avg)
                    p.data.copy_(p_data_fp32)

        return loss

class DiceLoss(nn.Module):
    def __init__(self, class_weights=None, epsilon=1e-6):
        super(DiceLoss, self).__init__()
        self.epsilon = epsilon
        self.class_weights = torch.from_numpy(np.array(class_weights)) if class_weights else None

    def forward(self, logits, labels):
        probas = logits.sigmoid()
        intersection = torch.sum(probas * labels, dim=[0, 2, 3])
        cardinality = torch.sum(probas + labels, dim=[0, 2, 3])
        dice_loss = 2. * intersection / (cardinality + self.epsilon)
        if self.class_weights:
            dice_loss = self.class_weights * dice_loss
        return (1 - dice_loss.mean())

def post_process(mask_i, proba_i, seg_threshold, clf_threshold, min_size=0):
    predictions = np.zeros((350, 525), np.float32)
    mask = (mask_i > seg_threshold).astype(np.float32)
    num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
    if proba_i > clf_threshold:
        predictions = mask
        for c in range(1, num_component):
            p = (component == c)
            if p.sum() > min_size:
                predictions[p] = 1
    return predictions

def tta(images, model):
    with torch.no_grad():
        clf_logits, seg_logits = model(images)
        clf_logits_h, seg_logits_h = model(images.flip(2))
        clf_logits_v, seg_logits_v = model(images.flip(3))
        clf_logits_hv, seg_logits_hv = model(images.flip(2, 3))
        seg_logits_h = seg_logits_h.flip(2)
        seg_logits_v = seg_logits_v.flip(3)
        seg_logits_hv = seg_logits_hv.flip(2, 3)
        clf_logits = (clf_logits + clf_logits_h + clf_logits_v + clf_logits_hv) / 4.
        seg_logits = (seg_logits + seg_logits_h + seg_logits_v + seg_logits_hv) / 4.
    return clf_logits, seg_logits


import os
import requests

#Example: 'C:/Users/John/Downloads/'
DOWNLOAD_PATH = "{YOUR_DOWNLOAD_PATH}"
URLS_FILE_PATH = "youtubeURLs.txt"

API_URL = "https://youtube-mp36.p.rapidapi.com/dl"

HEADERS = {
    "X-RapidAPI-Key": "{YOUR_RAPID_API_KEY}",
    "X-RapidAPI-Host": "youtube-mp36.p.rapidapi.com"
}

def read_file_lines(file_path):
    with open(file_path, 'r') as file:
        lines = file.readlines()
    return lines

def download_mp3(video_id):
    querystring = {"id": video_id}
    response = requests.get(API_URL, headers=HEADERS, params=querystring)

    json_response = response.json()

    if 'link' in json_response:
        download_link = json_response['link']

        response = requests.get(download_link)

        if response.status_code == 200:
            file_name = f"{json_response['title']}.mp3"
            file_path = os.path.join(DOWNLOAD_PATH, file_name)
            with open(file_path, 'wb') as file:
                file.write(response.content)
            print(f"Downloaded MP3 for video ID {video_id} and saved as {file_name}.")
        else:
            print(f"Failed to download the MP3 for video ID {video_id}.")
    else:
        print(f"Failed to get the download link for video ID {video_id}. Error: {json_response.get('error')}")

# Create the target directory if it does not exist
if not os.path.exists(DOWNLOAD_PATH):
    os.makedirs(DOWNLOAD_PATH)

lines_list = read_file_lines(URLS_FILE_PATH)
video_ids = [url.split('v=')[1].strip() for url in lines_list]

for video_id in video_ids:
    download_mp3(video_id)


from django.contrib import admin
from django.urls import path
from .views import *

app_name = "TTimes"

urlpatterns = [
    path('login/', loginview, name='login'),
    path('attendance/', attendanceview, name='attendance'),
    path('staffpayment/', staffpaymentview, name='staffpayment'),
    path('dailylist/', dailylistview, name='dailylist'),
    path('staff/record/<int:employee_number>/', staffrecordview, name='staffrecord'),
    path('attendance/manage/<int:pk>/',manageattendanceview, name='manage_attendance'),
    path('register/', RegisterStaffView.as_view(), name='register'),
    path('upload/', uploadview, name='upload'),
    path('download/', downloadview, name='download'),
]


import random
import json # use json to store bicycles.json and user data
import csv # use csv to store logs of rental
import datetime

import requests
import re

from admin import DEV_API_KEY

from telegram import (
    InlineKeyboardMarkup,
    InlineKeyboardButton,
    ParseMode,
    ReplyKeyboardMarkup,
    KeyboardButton
)

from telegram.ext import (
    Updater,
    CommandHandler,
    MessageHandler,
    Filters,
    ConversationHandler,
    PreCheckoutQueryHandler,
    ShippingQueryHandler,
    CallbackContext,
    CallbackQueryHandler,
    TypeHandler,
)

GMT = 8

class TeleBot:
    def __init__(self,api_key):
        self.api_key = api_key
        self.updater = Updater(token=api_key, use_context=True)
        self.dispatcher = self.updater.dispatcher

    def now(self,gmt=GMT):
        return datetime.datetime.utcnow() + datetime.timedelta(hours=gmt)

    def log_exception(self, e, text=""):
        if text:
            print(text)
        print(f'Error occured at {self.now()}. Error is \n{e}')

    def get_user(self,update=None,context=None,username=None)  -> dict or None:
        if username is not None:
            chat_id = self.get_user_table().get(username)
            if chat_id is None:
                return None
        else:
            chat_id = update.effective_chat.id
        try:
            with open(f'users/{chat_id}.json', 'r') as f:
                user_data = json.load(f)
        except FileNotFoundError:
            # User doesn't exist!
            user_data = None
        return user_data

    def update_user(self, user_data):
        chat_id = user_data.get('chat_id', None)
        if not chat_id:
            return None
        with open(f'users/{chat_id}.json', 'w') as f:
            json.dump(user_data, f, sort_keys=True, indent=4)

    def get_user_table(self) -> dict:
        table_data = dict()
        try:
            with open('users/table.json', 'r') as f:
                table_data = json.load(f)
        except FileNotFoundError:
            self.update_user_table({})
        return table_data

    def update_user_table(self, update_field):
        with open('users/table.json', 'w') as f:
            json.dump(update_field, f, sort_keys=True, indent=4)

    def get_bikes(self) -> dict:
        with open('bicycles.json', 'r') as f:
            bikes_data = json.load(f)
        return bikes_data

    def update_bikes(self, bikes_data) -> None:
        with open('bicycles.json', 'w') as f:
            json.dump(bikes_data, f, sort_keys=True, indent=4)

    def update_rental_log(self, update_list):
        """Updates rental logs with headers:
           bike,username,start_time,end_time"""
        with open('logs/rental.csv','a',newline='') as f:
            writer = csv.writer(f)
            writer.writerow(update_list)

    def update_report_log(self, update_list):
        """Updates report logs with headers:
           username,time,report"""
        with open('logs/report.csv','a',newline='') as f:
            writer = csv.writer(f)
            writer.writerow(update_list)

    def update_finance_log(self, update_list):
        """Updates finance logs with headers:
           username,time,initial_amt,change_amt,final_amt"""
        with open('logs/finance.csv', 'a', newline='') as f:
            writer = csv.writer(f)
            writer.writerow(update_list)

    def addnew(self,handler):
        self.dispatcher.add_handler(handler)

    def addcmd(self, command, methodname):
        handler = CommandHandler(command,methodname)
        self.addnew(handler)

    def addmsg(self, filters, methodname):
        handler = MessageHandler(filters, methodname)
        self.addnew(handler)

    def initialize(self):
        pass

    def main(self):
        print('Initializing bot...')
        self.initialize()
        self.updater.start_polling()
        self.updater.idle()

if __name__=="__main__":
    print('Running the TeleBot!')
    newbot = TeleBot(DEV_API_KEY)
    newbot.main()


# -*- coding: utf-8 -*-
"""
Created on Mon Aug  3 17:45:33 2020

@author: TANMAY MATHUR
"""

import collections as coll

#Counter
sample_cntr=coll.Counter('abcdefgabcdfrgg')
print(sample_cntr.most_common(3))

diff_test=coll.Counter(g=-2,a=3,b=1)
sample_cntr.subtract(diff_test)
print(sample_cntr)

#deque
sample_deq=coll.deque([1,2,3,4])
print(sample_deq)
sample_deq.append(5)
print(sample_deq)
sample_deq.appendleft(0)
print(sample_deq)
sample_deq.pop()
print(sample_deq)
sample_deq.popleft
print(sample_deq)

#defaultdict

sample_list_with_dup=[('yellow',1),('blue',2),('yellow',3),('blue',4),('red',1)]
dic_with_list=coll.defaultdict(list)
dic_with_set=coll.defaultdict(set)
for key,value in sample_list_with_dup:
    dic_with_list[key].append(value)
    dic_with_set[key].add(value)
print(dic_with_set,dic_with_list)

#names tuple
Color_dis=coll.namedtuple('color_dis',['r','g','b'])
clr_sample1 = Color_dis(24,100,98)
print(clr_sample1.r)
print(clr_sample1[1])
print(getattr(clr_sample1,'b'))


from flask import Flask, render_template, redirect
from flask_pymongo import PyMongo
import scrape_mars

# Create an instance of Flask
app = Flask(__name__)

# Use PyMongo to establish Mongo connection
mongo = PyMongo(app, uri="mongodb://localhost:27017/marsDB")

# Route to render index.html template using data from Mongo
@app.route("/")
def home():

    # Find one record of data from the mongo database
    scrape_data = mongo.db.scraped_data.find_one()

    # Return template and data
    return render_template("index.html", scrape_data= scrape_data)

# Route that will trigger the scrape function
@app.route("/scrape")

def scrape():
    "Main scrape functiona that will execute all other functions"
    "and return a dictionary"

     # Run the scrape function
    mars_data = scrape_mars.scrape_info()

    # Redirect back to home page
    return redirect("/")

if __name__ == "__main__":
    app.run(debug=True)


from collections import defaultdict
import argparse
import numpy as np
import json
import os
import networkx as nx
from networkx.readwrite import json_graph
import pickle

def parse_args():
    parser = argparse.ArgumentParser(description="Generate graphsage format from edgelist")
    parser.add_argument('--out_dir', default=None, help="Output directory")
    parser.add_argument('--prefix', default="karate", help="seed")
    parser.add_argument('--seed', default=121, type=int)
    return parser.parse_args()

def preprocess_edgelist(edgelist_pre, edgelist, edgelist_dic):
    if os.path.exists(edgelist):
        os.remove(edgelist)

    if os.path.exists(edgelist_dic):
        os.remove(edgelist_dic)

    d = {}
    c = 0
    fc = []
    f = open(edgelist_pre,'r')
    for l in f.readlines():
        ns = l.strip().split()[:2]
        for n in ns:
            if n not in d:
                d[n] = str(c)
                c += 1

        l = l.replace(ns[0], d[ns[0]]).replace(ns[1], d[ns[1]])
        l = " ".join(l.strip().split()[:2])

        fc.append(l)

    f.close()

    fc = "\n".join(fc)

    f = open(edgelist,'w')
    f.write(fc)
    f.close()

    with open(edgelist_dic, 'wb') as handle:
        pickle.dump(d, handle, protocol=pickle.HIGHEST_PROTOCOL)

def edgelist_to_graphsage(dir, seed=121):
    np.random.seed(seed)
    edgelist_pre = dir + "/edgelist/edgelist_pre"
    edgelist = dir + "/edgelist/edgelist"
    edgelist_dic = dir + "/edgelist/dic.pkl"
    preprocess_edgelist(edgelist_pre, edgelist, edgelist_dic)

    print(edgelist)
    G = nx.read_edgelist(edgelist)

    #get 2 core of largest connected component as suggested in comparison article
    #G = sorted(nx.connected_component_subgraphs(G), key=len, reverse=True)[0] #largest connected component
    #G = nx.algorithms.core.k_core(G, k=2) #2-core

    print(nx.info(G))
    num_nodes = len(G.nodes())
    rand_indices = np.random.permutation(num_nodes)
    train = rand_indices[:int(num_nodes * 0.81)]
    val = rand_indices[int(num_nodes * 0.81):int(num_nodes * 0.9)]
    test = rand_indices[int(num_nodes * 0.9):]

    id2idx = {}
    for i, node in enumerate(G.nodes()):
        id2idx[str(node)] = i

    res = json_graph.node_link_data(G)
    res['nodes'] = [
        {
            'id': node['id'],
            'val': id2idx[str(node['id'])] in val,
            'test': id2idx[str(node['id'])] in test
        }
        for node in res['nodes']]

    res['links'] = [
        {
            'source': link['source'],
            'target': link['target']
        }
        for link in res['links']]

    if not os.path.exists(dir + "/graphsage/"):
        os.makedirs(dir + "/graphsage/")

    with open(dir + "/graphsage/" + "G.json", 'w') as outfile:
        json.dump(res, outfile)
    with open(dir + "/graphsage/" + "id2idx.json", 'w') as outfile:
        json.dump(id2idx, outfile)

    print("GraphSAGE format stored in {0}".format(dir + "/graphsage/"))
    print("----------------------------------------------------------")

if __name__ == "__main__":
    args = parse_args()
    datadir = args.out_dir
    dataset = args.prefix
    edgelist_to_graphsage(datadir)


from typing import Callable, Awaitable, Any, Generic, TypeVar, Dict, List, \
    NamedTuple, Optional, Union
from asyncio import Queue
from mypy_extensions import VarArg, KwArg
from collections import MutableMapping

_T = TypeVar("_T")

FuncType = Callable[..., Any]
F = TypeVar('F', bound=FuncType)

class CacheInfo(NamedTuple('CacheInfo', [
    ('hits', int),
    ('misses', int),
    ('maxsize', int),
    ('currsize', int)
])): ...

class _cache_wrapper(Generic[_T]):
    __wrapped__ = ...  # type: Callable[..., Awaitable[_T]]
    def __call__(self, *args: Any, **kwargs: Any) -> Awaitable[_T]: ...
    def cache_info(self) -> CacheInfo: ...
    def cache_clear(self) -> None: ...

class async_cache():
    def __init__(
        self, maxsize: int=...,
        cache_obj: Optional[Callable[..., MutableMapping]]=...,
        cache_none: bool=..., **kwargs: Any,
    ) -> None: ...
    def __call__(self, f: F) -> F: ...

async def limit_gather(
    fs: List[Awaitable[_T]], n: int, task_timeout: int=...,
) -> List[Union[_T, BaseException]]: ...

async def process_queue_with_limit(
    queue: Queue[Awaitable[_T]], n: int,
    done_queue: Queue[Union[_T, BaseException]],
    task_timeout: int=...,
) -> None: ...