import os
from functools import partial
from typing import List, Tuple, Optional, Callable, Dict, Any, Union

import albumentations as A
import cv2
import numpy as np
import pandas as pd
import torch
from pytorch_toolbelt.datasets import INPUT_IMAGE_KEY, INPUT_INDEX_KEY
from pytorch_toolbelt.utils.torch_utils import image_to_tensor
from sklearn.neighbors import BallTree
from sklearn.utils import compute_sample_weight
from torch.utils.data import Sampler, WeightedRandomSampler
from torch.utils.data.dataloader import default_collate
from tqdm import tqdm

from .bboxer import (
    MultilabelCircleNetCoder,
    MultilabelCircleNetEncodeResult,
)
from .constants import *
from ..constants import *

__all__ = [
    "MultilabelCircleNetDataModule",
    "MultilabelCircleNetFixedCropDataset",
    "MultilabelCircleNetRandomCropDataset",
    "centernet_collate",
]

from ..dataset import (
    RandomCropFromImageDataset,
    RandomCropAroundEachObjectDataset,
    XView3DataModule,
    KeypointsDataset,
    FixedCropFromImageDataset,
    read_multichannel_image,
)
from ..dataset.random_crop_dataset import encode_labels
from ..metric import get_shoreline_shoreline_contours

def centernet_collate(batch, channels_last=False):
    skip_keys = [
        TARGET_CENTERS_KEY,
        TARGET_FISHING_KEY,
        TARGET_VESSEL_KEY,
        TARGET_LABELS_KEY,
        TARGET_LENGTHS_KEY,
        INPUT_SCENE_ID_KEY,
        INPUT_SCENE_CROP_KEY,
        INPUT_CONFIDENCE_KEY,
        INPUT_LOCATION_ID_KEY,
        INPUT_IS_NEAR_SHORE_KEY,
    ]
    excluded_items = [dict((k, v) for k, v in b.items() if k in skip_keys) for b in batch]
    included_items = [dict((k, v) for k, v in b.items() if k not in skip_keys) for b in batch]

    batch: dict = default_collate(included_items)
    for k in skip_keys:
        out = [item[k] for item in excluded_items if k in item]
        if len(out):
            batch[k] = out

    # for key in [
    #     INPUT_IMAGE_KEY,
    #     CENTERNET_TARGET_OBJECTNESS_MAP,
    #     CENTERNET_OUTPUT_CLASS_MAP,
    #     CENTERNET_TARGET_OFFSET,
    #     CENTERNET_TARGET_SIZE,
    #     CENTERNET_TARGET_FISHING_MAP,
    #     CENTERNET_TARGET_VESSEL_MAP,
    # ]:
    #     if key in batch:
    #         batch[key] = batch[key].to(memory_format=torch.contiguous_format)

    if channels_last:
        batch[INPUT_IMAGE_KEY] = batch[INPUT_IMAGE_KEY].to(memory_format=torch.channels_last)
    return batch

class MultilabelCircleNetTargetsMixin:
    box_coder: MultilabelCircleNetCoder

    def build_centernet_sample(
        self,
        image,
        centers,
        confidences,
        lengths,
        is_vessel,
        is_fishing,
        index,
        image_id,
        is_near_shore,
        location,
        folder: str,
        crop_coords: Tuple[Tuple[int, int], Tuple[int, int]],
    ) -> Dict[str, Any]:
        # Mask NaN values
        image[~np.isfinite(image)] = 0

        result = {
            INPUT_INDEX_KEY: index,
            INPUT_IMAGE_KEY: image_to_tensor(image),
            TARGET_CENTERS_KEY: centers.tolist(),  # .tolist to prevent Catalyst converting it to Tensor and move to GPU
            TARGET_VESSEL_KEY: is_vessel.tolist(),
            TARGET_FISHING_KEY: is_fishing.tolist(),
            TARGET_LENGTHS_KEY: lengths.tolist(),
            INPUT_CONFIDENCE_KEY: confidences.tolist(),
            INPUT_IS_NEAR_SHORE_KEY: is_near_shore.tolist(),
            INPUT_LOCATION_ID_KEY: location,
            INPUT_FOLDER_KEY: folder,
            INPUT_SCENE_ID_KEY: image_id,
            TARGET_SAMPLE_WEIGHT: {"train": 0.25, "validation": 1.0, "valid": 1.0, "test": 0.1}[folder],
        }
        if crop_coords is not None:
            result[INPUT_SCENE_CROP_KEY] = crop_coords

        box_coder = self.box_coder.box_coder_for_image_size(image.shape)
        centernet_targets: MultilabelCircleNetEncodeResult = box_coder.encode(
            centers=centers, confidences=confidences, lengths=lengths, is_vessel=is_vessel, is_fishing=is_fishing
        )
        result[CENTERNET_TARGET_OBJECTNESS_MAP] = centernet_targets.heatmap
        result[CENTERNET_TARGET_VESSEL_MAP] = centernet_targets.is_vessel
        result[CENTERNET_TARGET_FISHING_MAP] = centernet_targets.is_fishing
        result[CENTERNET_TARGET_SIZE] = centernet_targets.lengths
        result[CENTERNET_TARGET_OFFSET] = centernet_targets.offset
        return result

class MultilabelCircleNetFixedCropDataset(FixedCropFromImageDataset, MultilabelCircleNetTargetsMixin):
    def __init__(
        self,
        scenes: List[str],
        centers: List[np.ndarray],
        locations: List[str],
        confidences: List[np.ndarray],
        is_vessel: List[np.ndarray],
        is_fishing: List[np.ndarray],
        is_near_shore: List[np.ndarray],
        lengths: List[np.ndarray],
        crop_coords: List[Tuple],
        scene_sizes: List[Tuple[int, int]],
        individual_transforms: Dict[str, A.Compose],
        normalization: Dict[str, A.ImageOnlyTransform],
        input_channels: List[str],
        transform: A.Compose,
        read_image_fn: Callable,
        box_coder: MultilabelCircleNetCoder,
        channels_last: bool = False,
    ):
        super().__init__(
            scenes=scenes,
            centers=centers,
            locations=locations,
            confidences=confidences,
            is_vessel=is_vessel,
            is_fishing=is_fishing,
            is_near_shore=is_near_shore,
            lengths=lengths,
            crop_coords=crop_coords,
            scene_sizes=scene_sizes,
            individual_transforms=individual_transforms,
            normalization=normalization,
            input_channels=input_channels,
            transform=transform,
            read_image_fn=read_image_fn,
            channels_last=channels_last,
        )
        self.box_coder = box_coder

    def build_sample(
        self,
        image,
        centers,
        confidences,
        is_vessel,
        is_fishing,
        lengths,
        index,
        folder: str,
        image_id,
        is_near_shore,
        location: str,
        crop_coords,
    ) -> Dict[str, Any]:
        return self.build_centernet_sample(
            image=image,
            centers=centers,
            confidences=confidences,
            is_vessel=is_vessel,
            is_fishing=is_fishing,
            lengths=lengths,
            index=index,
            image_id=image_id,
            is_near_shore=is_near_shore,
            location=location,
            folder=folder,
            crop_coords=crop_coords,
        )

    def get_collate_fn(self) -> Callable:
        return partial(centernet_collate, channels_last=self.channels_last)

class MultilabelCircleNetRandomCropDataset(RandomCropFromImageDataset, MultilabelCircleNetTargetsMixin):
    def __init__(
        self,
        scenes: List[str],
        shoreline_contours: List[np.ndarray],
        locations: List[str],
        centers: List[np.ndarray],
        confidences: List[np.ndarray],
        lengths: List[np.ndarray],
        is_vessel: List[np.ndarray],
        is_fishing: List[np.ndarray],
        is_near_shore: List[np.ndarray],
        scene_sizes: List[Tuple[int, int]],
        individual_transforms: Dict[str, A.Compose],
        normalization: Dict[str, A.ImageOnlyTransform],
        input_channels: List[str],
        transform: A.Compose,
        read_image_fn: Callable,
        crop_size: Tuple[int, int],
        box_coder: MultilabelCircleNetCoder,
        crop_around_ship_p=0.9,
        balance_near_shore: Union[None, bool, float] = None,
        balance_crowd=None,
        balance_by_type=False,
        channels_last=False,
    ):
        super().__init__(
            scenes=scenes,
            shoreline_contours=shoreline_contours,
            locations=locations,
            centers=centers,
            confidences=confidences,
            is_vessel=is_vessel,
            is_fishing=is_fishing,
            is_near_shore=is_near_shore,
            lengths=lengths,
            scene_sizes=scene_sizes,
            individual_transforms=individual_transforms,
            normalization=normalization,
            input_channels=input_channels,
            transform=transform,
            read_image_fn=read_image_fn,
            crop_size=crop_size,
            crop_around_ship_p=crop_around_ship_p,
            balance_near_shore=balance_near_shore,
            balance_crowd=balance_crowd,
            balance_by_type=balance_by_type,
            channels_last=channels_last,
        )
        self.box_coder = box_coder

    def build_sample(
        self,
        image,
        centers,
        confidences,
        is_vessel,
        is_fishing,
        lengths,
        index,
        folder: str,
        image_id,
        is_near_shore,
        location: str,
        crop_coords,
    ) -> Dict[str, Any]:
        return self.build_centernet_sample(
            image=image,
            centers=centers,
            confidences=confidences,
            is_vessel=is_vessel,
            is_fishing=is_fishing,
            lengths=lengths,
            index=index,
            folder=folder,
            image_id=image_id,
            is_near_shore=is_near_shore,
            location=location,
            crop_coords=crop_coords,
        )

    def get_collate_fn(self) -> Callable:
        return partial(centernet_collate, channels_last=self.channels_last)

class MultilabelCircleNetRandomCropAroundEachObjectDataset(RandomCropAroundEachObjectDataset, MultilabelCircleNetTargetsMixin):
    def __init__(
        self,
        scenes: List[str],
        locations: List[str],
        crop_centers: np.ndarray,
        centers: List[np.ndarray],
        confidences: List[np.ndarray],
        lengths: List[np.ndarray],
        is_vessel: List[np.ndarray],
        is_fishing: List[np.ndarray],
        is_near_shore: List[np.ndarray],
        scene_sizes: List[Tuple[int, int]],
        individual_transforms: Dict[str, A.Compose],
        normalization: Dict[str, A.ImageOnlyTransform],
        input_channels: List[str],
        transform: A.Compose,
        read_image_fn: Callable,
        crop_size: Tuple[int, int],
        box_coder: MultilabelCircleNetCoder,
        channels_last=False,
    ):
        super().__init__(
            scenes=scenes,
            locations=locations,
            crop_centers=crop_centers,
            centers=centers,
            confidences=confidences,
            is_vessel=is_vessel,
            is_fishing=is_fishing,
            is_near_shore=is_near_shore,
            lengths=lengths,
            scene_sizes=scene_sizes,
            individual_transforms=individual_transforms,
            normalization=normalization,
            input_channels=input_channels,
            transform=transform,
            read_image_fn=read_image_fn,
            crop_size=crop_size,
            channels_last=channels_last,
        )
        self.box_coder = box_coder

    def build_sample(
        self,
        image,
        centers,
        confidences,
        is_vessel,
        is_fishing,
        lengths,
        index,
        folder: str,
        image_id,
        is_near_shore,
        location: str,
        crop_coords,
    ) -> Dict[str, Any]:
        return self.build_centernet_sample(
            image=image,
            centers=centers,
            confidences=confidences,
            is_vessel=is_vessel,
            is_fishing=is_fishing,
            is_near_shore=is_near_shore,
            lengths=lengths,
            index=index,
            folder=folder,
            image_id=image_id,
            location=location,
            crop_coords=crop_coords,
        )

    def get_collate_fn(self) -> Callable:
        return partial(centernet_collate, channels_last=self.channels_last)

class MultilabelCircleNetDataModule(XView3DataModule):
    @classmethod
    def get_keypoint_params(cls):
        return A.KeypointParams(format="xys", label_fields=["confidences", "is_vessel", "is_fishing", "is_near_shore"])

    @classmethod
    def get_random_crop_each_object_dataset(
        cls,
        train_df: pd.DataFrame,
        train_image_size: Tuple[int, int],
        input_channels: List[str],
        box_coder: MultilabelCircleNetCoder,
        normalization: Dict[str, A.ImageOnlyTransform],
        num_samples: int = 1024,
        individual_augmentations: Optional[Dict] = None,
        augmentations: Optional[List[A.BasicTransform]] = None,
        balance_crowd=False,
        balance_per_scene=True,
        balance_near_shore=None,
        balance_vessel_type=False,
        balance_fishing_type=False,
        balance_by_type=False,
        balance_location=False,
        channels_last=False,
        read_image_fn=read_multichannel_image,
    ) -> Tuple[KeypointsDataset, Optional[Sampler]]:
        scene_sizes = dict(
            [
                (row["scene_id"], (row["rows"], row["cols"]))
                for i, row in pd.read_csv(os.path.join(os.path.dirname(__file__), "..", "scene_sizes.csv")).iterrows()
            ]
        )

        all_images = []
        all_coordinates = []
        all_confidences = []
        all_lengths = []
        all_size = []
        all_is_vessel = []
        all_is_fishing = []
        all_is_near_shore = []

        all_scenes = []
        all_locations = []

        center_near_shore = []
        center_neighbors = []
        center_coordinates = []
        center_is_vessel = []
        center_is_fishing = []
        center_is_near_shore = []

        for scene_id in tqdm(train_df.scene_id.unique(), desc="Preparing training dataset"):
            # Get detections only for current scene
            scene_df = train_df[train_df.scene_id == scene_id]
            num_instances_in_scene = len(scene_df)
            scene_size = scene_sizes[scene_id]
            scene_path = scene_df.scene_path.values[0]
            location_id = scene_df.location.values[0]

            targets = cls.get_multilabel_targets_from_df(scene_df)
            distance_from_shore_km = scene_df.distance_from_shore_km
            near_shore = (distance_from_shore_km < 2).values

            tree = BallTree(targets.centers, leaf_size=8, metric="chebyshev")
            neighbors = tree.query_radius(targets.centers, r=max(train_image_size) // 2, count_only=True)

            all_scenes.extend([scene_id] * num_instances_in_scene)
            all_locations.extend([location_id] * num_instances_in_scene)
            all_images.extend([scene_path] * num_instances_in_scene)

            center_coordinates.extend(targets.centers)
            center_is_vessel.extend(targets.is_vessel)
            center_is_fishing.extend(targets.is_fishing)
            center_is_near_shore.extend(targets.is_near_shore)
            center_near_shore.extend(near_shore)
            center_neighbors.extend(neighbors)

            all_coordinates.extend([targets.centers] * num_instances_in_scene)
            all_confidences.extend([targets.confidences] * num_instances_in_scene)
            all_is_vessel.extend([targets.is_vessel] * num_instances_in_scene)
            all_is_fishing.extend([targets.is_fishing] * num_instances_in_scene)
            all_lengths.extend([targets.lengths] * num_instances_in_scene)
            all_is_near_shore.extend([targets.is_near_shore] * num_instances_in_scene)

            all_size.extend([scene_size] * num_instances_in_scene)

        if augmentations is None:
            augmentations = []

        transform = A.Compose(
            augmentations
            + [
                A.PadIfNeeded(
                    train_image_size[0],
                    train_image_size[1],
                    value=float("nan"),
                    border_mode=cv2.BORDER_CONSTANT,
                ),
            ],
            keypoint_params=cls.get_keypoint_params(),
        )
        train_ds = MultilabelCircleNetRandomCropAroundEachObjectDataset(
            scenes=all_images,
            locations=all_locations,
            centers=all_coordinates,
            confidences=all_confidences,
            crop_centers=np.array(center_coordinates),
            lengths=all_lengths,
            is_vessel=all_is_vessel,
            is_fishing=all_is_fishing,
            is_near_shore=all_is_near_shore,
            scene_sizes=all_size,
            individual_transforms=individual_augmentations,
            input_channels=input_channels,
            transform=transform,
            normalization=normalization,
            read_image_fn=read_image_fn,
            box_coder=box_coder,
            crop_size=train_image_size,
            channels_last=channels_last,
        )

        # weights = np.ones(len(all_coordinates), dtype=np.float64)
        # num_balancing = 0
        #
        # if balance_location:
        #     location_balance_weights = compute_sample_weight("balanced", all_locations)
        #     weights *= location_balance_weights
        #     num_balancing += 1
        #
        # if balance_crowd:
        #     inv_frequency = np.reciprocal(np.array(center_neighbors).astype(np.float32))
        #     crowd_balance_weights = np.sqrt(inv_frequency)  # Slightly dampen inverse proportional weight
        #     weights *= crowd_balance_weights
        #     num_balancing += 1
        #
        # if balance_per_scene:
        #     scene_balance_weights = compute_sample_weight("balanced", all_scenes)
        #     weights *= scene_balance_weights
        #     num_balancing += 1
        #
        # if balance_vessel_type:
        #     vessel_type_weights = compute_sample_weight({0: 0.45, 1: 0.45, IGNORE_LABEL: 0.1}, center_is_vessel)
        #     weights *= vessel_type_weights
        #     num_balancing += 1
        #
        # if balance_fishing_type:
        #     fishing_weights = compute_sample_weight({0: 0.45, 1: 0.45, IGNORE_LABEL: 0.1}, center_is_fishing)
        #     weights *= fishing_weights
        #     num_balancing += 1
        #
        # if balance_near_shore and len(np.bincount(center_near_shore)) > 1:
        #     shore_balance_weights = compute_sample_weight("balanced", center_near_shore)
        #     weights *= shore_balance_weights
        #     num_balancing += 1
        #
        # if num_balancing:
        #     weights **= 1.0 / num_balancing
        #

        weights = np.ones(len(all_coordinates), dtype=np.float64)
        num_balancing = 0

        if balance_location:
            location_balance_weights = compute_sample_weight("balanced", all_locations)
            weights += location_balance_weights
            num_balancing += 1

        if balance_crowd:
            inv_frequency = np.reciprocal(np.array(center_neighbors).astype(np.float32))
            crowd_balance_weights = inv_frequency
            if not np.all(np.isfinite(crowd_balance_weights)):
                raise RuntimeError()
            weights += crowd_balance_weights
            num_balancing += 1

        if balance_per_scene:
            scene_balance_weights = compute_sample_weight("balanced", all_scenes)
            weights += scene_balance_weights
            num_balancing += 1

        if balance_vessel_type:
            center_is_vessel = np.array(center_is_vessel)
            vessel_type_weights = compute_sample_weight(
                {
                    0: len(center_is_vessel) / (2.0 * np.sum(center_is_vessel == 0)),
                    1: len(center_is_vessel) / (2.0 * np.sum(center_is_vessel == 1)),
                    IGNORE_LABEL: 0,
                },
                center_is_vessel,
            )
            if not np.all(np.isfinite(vessel_type_weights)):
                raise RuntimeError()

            weights += vessel_type_weights
            num_balancing += 1

        if balance_fishing_type:
            center_is_fishing = np.array(center_is_fishing)
            fishing_weights = compute_sample_weight(
                {
                    0: len(center_is_fishing) / (2.0 * np.sum(center_is_fishing == 0)),
                    1: len(center_is_fishing) / (2.0 * np.sum(center_is_fishing == 1)),
                    IGNORE_LABEL: 0,
                },
                center_is_fishing,
            )
            if not np.all(np.isfinite(fishing_weights)):
                raise RuntimeError()
            weights += fishing_weights
            num_balancing += 1

        if balance_by_type:
            class_labels = encode_labels(center_is_vessel, center_is_fishing)
            type_balance_weights = compute_sample_weight("balanced", class_labels)
            weights += type_balance_weights
            num_balancing += 1

        if balance_near_shore and len(np.bincount(center_near_shore)) > 1:
            shore_balance_weights = compute_sample_weight("balanced", center_near_shore)
            weights += shore_balance_weights
            num_balancing += 1

        if num_balancing:
            weights /= float(num_balancing)

        train_sampler = WeightedRandomSampler(weights.astype(np.float32), num_samples=num_samples, replacement=True)
        return train_ds, train_sampler

    @classmethod
    def get_random_crop_training_dataset(
        cls,
        train_df: pd.DataFrame,
        train_image_size: Tuple[int, int],
        input_channels: List[str],
        box_coder: MultilabelCircleNetCoder,
        crop_around_ship_p: float,
        normalization: Dict[str, A.ImageOnlyTransform],
        shore_root: str,
        num_samples: int = 1024,
        individual_augmentations: Optional[Dict] = None,
        augmentations: Optional[List[A.BasicTransform]] = None,
        balance_near_shore=None,
        balance_crowd=None,
        balance_location=False,
        balance_by_type=False,
        balance_folder=False,
        channels_last=False,
        read_image_fn=read_multichannel_image,
    ) -> Tuple[KeypointsDataset, Optional[Sampler]]:
        scene_sizes = dict(
            [
                (row["scene_id"], (row["rows"], row["cols"]))
                for i, row in pd.read_csv(os.path.join(os.path.dirname(__file__), "..", "scene_sizes.csv")).iterrows()
            ]
        )

        all_images = []
        all_centers = []
        all_confidences = []
        all_lengths = []
        all_size = []
        all_is_vessel = []
        all_is_fishing = []
        all_is_near_shore = []
        all_near_shore = []
        all_locations = []
        all_shoreline_contours = []
        all_folders = []

        for scene_id in tqdm(train_df.scene_id.unique(), desc="Preparing training dataset"):
            # Get detections only for current scene
            scene_df = train_df[train_df.scene_id == scene_id]
            scene_size = scene_sizes[scene_id]
            scene_path = scene_df.scene_path.values[0]
            location_id = scene_df.location.values[0]
            folder_id = scene_df.folder.values[0]

            shoreline_contours = get_shoreline_shoreline_contours(shore_root, scene_id)

            targets = cls.get_multilabel_targets_from_df(scene_df)
            distance_from_shore_km = scene_df.distance_from_shore_km
            near_shore = (distance_from_shore_km < 2).values

            all_images.append(scene_path)
            all_shoreline_contours.append(shoreline_contours)

            all_centers.append(targets.centers)
            all_confidences.append(targets.confidences)
            all_is_vessel.append(targets.is_vessel)
            all_is_fishing.append(targets.is_fishing)
            all_is_near_shore.append(targets.is_near_shore)
            all_lengths.append(targets.lengths)
            all_size.append(scene_size)
            all_near_shore.append(near_shore)

            all_locations.append(location_id)
            all_folders.append(folder_id)

        if augmentations is None:
            augmentations = []

        transform = A.Compose(
            augmentations
            + [
                A.PadIfNeeded(
                    train_image_size[0],
                    train_image_size[1],
                    value=float("nan"),
                    border_mode=cv2.BORDER_CONSTANT,
                ),
            ],
            keypoint_params=cls.get_keypoint_params(),
        )
        train_ds = MultilabelCircleNetRandomCropDataset(
            scenes=all_images,
            shoreline_contours=all_shoreline_contours,
            locations=all_locations,
            centers=all_centers,
            confidences=all_confidences,
            lengths=all_lengths,
            is_vessel=all_is_vessel,
            is_fishing=all_is_fishing,
            is_near_shore=all_near_shore,
            scene_sizes=all_size,
            individual_transforms=individual_augmentations,
            input_channels=input_channels,
            transform=transform,
            normalization=normalization,
            read_image_fn=read_image_fn,
            box_coder=box_coder,
            crop_size=train_image_size,
            crop_around_ship_p=crop_around_ship_p,
            balance_near_shore=balance_near_shore,
            balance_crowd=balance_crowd,
            balance_by_type=balance_by_type,
            channels_last=channels_last,
        )

        weights = len(train_df)
        num_balancing = 0

        if balance_location:
            location_balance_weights = compute_sample_weight("balanced", all_locations)
            weights *= location_balance_weights
            num_balancing += 1

        if balance_folder:
            folder_balance_weights = compute_sample_weight("balanced", all_folders)
            weights *= folder_balance_weights
            num_balancing += 1

        if num_balancing:
            weights **= 1.0 / num_balancing

        train_sampler = WeightedRandomSampler(np.ones(len(train_ds)), num_samples=num_samples, replacement=True)
        return train_ds, train_sampler

    @classmethod
    def get_validation_dataset(
        cls,
        valid_df: pd.DataFrame,
        valid_crop_size: Tuple[int, int],
        input_channels: List[str],
        box_coder: MultilabelCircleNetCoder,
        normalization: Dict[str, A.ImageOnlyTransform],
        channels_last=False,
        read_image_fn=read_multichannel_image,
    ) -> KeypointsDataset:
        """
        For validation dataset we apply sliding crops
        :param valid_df:
        :param kwargs:
        :return:
        """

        all_images = []
        all_centers = []
        all_confidences = []
        all_lengths = []
        all_crop_coords = []
        all_is_vessel = []
        all_is_fishing = []
        all_scene_sizes = []
        all_locations = []
        all_is_near_shore = []

        scene_sizes = dict(
            [
                (row["scene_id"], (row["rows"], row["cols"]))
                for i, row in pd.read_csv(os.path.join(os.path.dirname(__file__), "..", "scene_sizes.csv")).iterrows()
            ]
        )

        for scene_id in tqdm(valid_df.scene_id.unique(), desc="Preparing validation dataset"):
            # Get detections only for current scene
            scene_df = valid_df[valid_df.scene_id == scene_id]
            scene_path = scene_df.scene_path.values[0]
            scene_size = scene_sizes[scene_id]
            location = scene_df.location.values[0]

            targets = cls.get_multilabel_targets_from_df(scene_df)
            for crop_coords in cls.iterate_crops(scene_size, valid_crop_size, valid_crop_size):
                all_images.append(scene_path)
                all_locations.append(location)
                all_centers.append(targets.centers)
                all_confidences.append(targets.confidences)
                all_is_vessel.append(targets.is_vessel)
                all_is_fishing.append(targets.is_fishing)
                all_is_near_shore.append(targets.is_near_shore)
                all_lengths.append(targets.lengths)
                all_crop_coords.append(crop_coords)
                all_scene_sizes.append(scene_size)

        individual_transforms = {}
        transform = A.Compose(
            [
                A.PadIfNeeded(
                    valid_crop_size[0],
                    valid_crop_size[1],
                    value=float("nan"),
                    border_mode=cv2.BORDER_CONSTANT,
                ),
            ],
            keypoint_params=cls.get_keypoint_params(),
        )
        valid_ds = MultilabelCircleNetFixedCropDataset(
            scenes=all_images,
            locations=all_locations,
            centers=all_centers,
            confidences=all_confidences,
            lengths=all_lengths,
            crop_coords=all_crop_coords,
            is_vessel=all_is_vessel,
            is_fishing=all_is_fishing,
            is_near_shore=all_is_near_shore,
            scene_sizes=all_scene_sizes,
            individual_transforms=individual_transforms,
            normalization=normalization,
            input_channels=input_channels,
            transform=transform,
            read_image_fn=read_image_fn,
            box_coder=box_coder,
            channels_last=channels_last,
        )
        return valid_ds


import pika
import uuid

class Sender:
    def __init__(self):
        """
        initialize connection, channel and declare a nameless queue and assign it to self.queue
        and at last basic consume on that queue bu auto acknowledgement and self.on_response method as callback
        """
        self.connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
        self.ch = self.connection.channel()
        result = self.ch.queue_declare(queue='', exclusive=True)
        self.qname = result.method.queue
        self.ch.basic_consume(queue=self.qname, on_message_callback=self.on_response, auto_ack=True)

    def on_response(self, ch, method, prop, body):
        """
        used as call callback on messages deliver and check if correlation code of instance equal to
        delivered message correlation id then assign self.response to message body
        :param ch: message channel
        :param method: message meta data
        :param prop: message properties
        :param body: message text body
        :return: no return just assign
        """
        if self.corr_id == prop.correlation_id:
            self.response = body

    def call(self, n: int):
        """
        set response to none
        create instance correlation id bu using uuid module
        publish with empty exchange (default=direct) and a named routing key with properties reply to assigned to queue
        name that initialized in init method and pass already created correlation id and pass n as a string for body
        and then wait for response from server by while loop
        at last return the self.response
        :param n: parameter as call method for doing some procedure on server
        :return: self.response
        """
        self.response = None
        self.corr_id = str(uuid.uuid4())
        self.ch.basic_publish(exchange='', routing_key='rpc_queue',
                              properties=pika.BasicProperties(reply_to=self.qname, correlation_id=self.corr_id),
                              body=str(n))
        print('Server is Processing...')

        while self.response is None:
            self.connection.process_data_events()

        return int(self.response)

send = Sender()
resp = send.call(int(input('Enter your Number: ')))
print(f'Response is {resp}')


"""
Image postprocessing module.
"""
import os

from typing import Union, Tuple
from time import time

import numpy as np
import cv2
from ds_utils import load, pack_rgb, unpack_rgb
from scipy.cluster.vq import kmeans, vq
from PIL import Image
from wbdlogger import WBDLogger

logger = WBDLogger()

def postprocessing(output_path: str, crop_weights: list, tmp_dir: str):
    """
    Постобработка включает 2 этапа:
       * Обрезка высветленного изображения
       * Перевод записей на белый фон с удалением шумов

    :param output_path: путь для итогового изображения.
        Будет модифицирован добавлением префикса postprocessing к имени файла.
    :param crop_weights: веса для обрезки кадра.
    :param tmp_dir: временная директория для хранения промежуточных этапов постобработки
    """
    logger.info("Начинаю постобработку")
    # Prepare image - crop it, to leave only the whiteboard
    img = cv2.imread(output_path)

    # crop_weights[0] - up/down coefficients.
    # crop_weights[1] - left/right coefficients.
    original_shape: tuple = img.shape
    crop_img = img[crop_weights[0][0]:original_shape[0] + crop_weights[0][1],
               crop_weights[1][0]:original_shape[1] + crop_weights[1][1]]

    tmp_filename = os.path.join(tmp_dir, f'tmp_{str(int(time()))}.png')
    cv2.imwrite(tmp_filename, crop_img)
    logger.info("Записан временный файл с обрезанным изображением")

    # Postprocessing
    img_np = load(tmp_filename)
    if img_np is None:
        raise IOError

    sample: np.ndarray = sample_pixels(img_np)
    palette: np.ndarray = get_palette(sample)
    _back_indx: int = output_path.rfind('/') + 1

    labels = apply_palette(img_np, palette)
    output_filename = output_path[:_back_indx] + 'postprocessing_' + output_path[_back_indx:]
    save(output_filename, labels, palette, (300, 300))

    logger.info(f"Postprocessing окончен. Изображение сохранено по адресу: {output_filename}")
    if os.path.exists(tmp_filename):
        os.remove(tmp_filename)

def unsharp_mask(image, kernel_size=(5, 5), sigma=1.0, amount=1.0, threshold=0):
    """Return a sharpened version of the image, using an unsharp mask."""
    logger.info("Увеличиваем чёткость изображения")
    blurred = cv2.GaussianBlur(image, kernel_size, sigma)
    sharpened = float(amount + 1) * image - float(amount) * blurred
    sharpened = np.maximum(sharpened, np.zeros(sharpened.shape))
    sharpened = np.minimum(sharpened, 255 * np.ones(sharpened.shape))
    sharpened = sharpened.round().astype(np.uint8)
    if threshold > 0:
        low_contrast_mask = np.absolute(image - blurred) < threshold
        np.copyto(sharpened, image, where=low_contrast_mask)
    return sharpened

def apply_brightness_contrast(input_img: np.ndarray, brightness: Union[int, float] = 0,
                              contrast: Union[int, float] = 0):
    """
    Функция увеличивает контраст и яркость исходного изображения без ухудшения качества пикселей.

    :param input_img: исходное изображение в формате массива.
    :param brightness: коэффициент яркости.
    :param contrast: коэффициент контраста.
    """
    if brightness != 0:
        logger.info(f"Увеличиваем яркость на {brightness}%")

        if brightness > 0:
            shadow = brightness
            highlight = 255
        else:
            shadow = 0
            highlight = 255 + brightness
        alpha_b = (highlight - shadow) / 255
        gamma_b = shadow

        buf = cv2.addWeighted(input_img, alpha_b, input_img, 0, gamma_b)
    else:
        buf = input_img.copy()

    if contrast != 0:
        logger.info(f"Увеличиваем контраст на {contrast}%")

        f = 131 * (contrast + 127) / (127 * (131 - contrast))
        alpha_c = f
        gamma_c = 127 * (1 - f)

        buf = cv2.addWeighted(buf, alpha_c, buf, 0, gamma_c)

    return buf

def sharpen(input_filename: str) -> str:
    im = cv2.imread(input_filename)
    im_bw = cv2.bilateralFilter(im, 9, 75, 75)
    kernel = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]])
    im_bw = cv2.filter2D(im_bw, -1, kernel)

    out_filename = f'tmp/tmp_{int(time())}.png'
    cv2.imwrite(out_filename, im_bw)

    return out_filename

def save(output_filename: str, labels, palette: np.ndarray, dpi: tuple):
    """
    Сохраняет label/palette пары как индексированное PNG изображение.
    Дополнительно насыщает палитру, сопоставляя наименьший цветовой
    компонент нулю, а наибольший - 255, а также дополнительно устанавливает
    цвет фона на чистый белый.

    :param output_filename:
    :param labels:
    :param palette: цветовая палитра
    :param dpi: dpi выходного изображения
    :return:
    """
    # Saturate img
    logger.info("Насыщение палитры - подбор более ярких цветов")
    palette = palette.astype(np.float32)
    pmin = palette.min()
    pmax = palette.max()
    palette = 255 * (palette - pmin) / (pmax - pmin)
    palette = palette.astype(np.uint8)

    # Make white background
    logger.info("Перевод фона в белый цвет")
    palette = palette.copy()
    palette[0] = (250, 250, 250)

    output_img = Image.fromarray(labels, 'P')
    output_img.putpalette(palette.flatten())
    if output_img.mode != 'RGB':
        output_img = output_img.convert('RGB')
    output_img.save(output_filename, dpi=dpi)

def apply_palette(img, palette):
    """
    Apply the pallete to the given image.
    The first step is to set all background pixels to the background color;
    then, nearest-neighbor matching is used to map each foreground color to the closest one in the palette.
    """
    logger.info("Применение палитры")
    bg_color = tuple(palette[0])

    fg_mask = get_foreground_mask(bg_color, img.reshape(-1, 3))
    orig_shape = img.shape
    logger.info("Фоновые пиксели отмечены")

    pixels = img.reshape((-1, 3))
    fg_mask = fg_mask.flatten()
    num_pixels = pixels.shape[0]

    logger.info("Сопоставление каждого цвета переднего плана с ближайшим цветом в палитре")
    labels = np.zeros(num_pixels, dtype=np.uint8)
    labels[fg_mask], _ = vq(pixels[fg_mask], palette)

    return labels.reshape(orig_shape[:-1])

def get_palette(sample: np.ndarray, num_colors: int = 20, kmeans_iter: int = 200) -> np.ndarray:
    """
    Позволяет получить палитру из num_colors наиболее часто встречающихся цветов

    :param sample: случайная выборка из исходного изображения
    :param num_colors: количество желаемых цветов в палитре
    :param kmeans_iter: количество итераций в алгоритме k-means

    :return: палитра цветов, найденных в sample, в количестве num_colors
    """
    logger.info("Подготовка палитры")

    bg_color: tuple = get_bg_color(img=sample, bits_per_channel=6)
    logger.info(f"Фон определён: {str(bg_color)}")

    fg_mask: np.ndarray = get_foreground_mask(bg_color, sample)

    centers, _ = kmeans(sample[fg_mask].astype(np.float32),
                        num_colors - 1,
                        iter=kmeans_iter)

    palette: np.ndarray = np.vstack((bg_color, centers)).astype(np.uint8)

    return palette

def get_foreground_mask(bg_color: tuple, sample: np.ndarray) -> np.ndarray:
    """
    Мы можем пометить пиксель как принадлежащий к фону, если выполняется один из этих критериев:
        * hsv value отличается от bg_color более чем на 0.25-0.3;
        * hsv saturation отличается от bg_color более чем на 0.2.

    :param bg_color: цвет фона в RGB
    :param sample: случайная выборка пикселей из исходного изображения

    :return: принадлежность пикселя из sample к фоновому
    """
    import colorsys

    _, s_bg, v_bg = colorsys.rgb_to_hsv(bg_color[0] / 255, bg_color[1] / 255, bg_color[2] / 255)

    s_smpl = list()
    v_smpl = list()

    for el in sample:
        _, s_s, v_s = colorsys.rgb_to_hsv(el[0] / 255, el[1] / 255, el[2] / 255)
        s_smpl.append(s_s)
        v_smpl.append(v_s)

    s_diff = np.abs(s_bg - np.array(s_smpl))
    v_diff = np.abs(v_bg - np.array(v_smpl))

    return ((v_diff >= 0.25) |
            (s_diff >= 0.2))

def sample_pixels(img: np.ndarray, pixels_percent: int = 40) -> np.ndarray:
    """
    Выбирает в случайном порядке набор пикселей, в размере options % от переданного img

    :param img: изображение в матричном формате
    :param pixels_percent: количество пикселей в %

    :return: выборка пикселей в размере pixels_percent % от исходного img
    """
    logger.info("Готовим sample для определения фона")
    pixels = img.reshape((-1, 3))
    num_pixels = pixels.shape[0]

    num_samples = int(num_pixels * pixels_percent / 100)
    idx = np.arange(num_pixels)

    np.random.shuffle(idx)
    return pixels[idx[:num_samples]]

def get_bg_color(img: np.ndarray, bits_per_channel=6) -> Tuple[int]:
    """
    Получает цвет фона из изображения или массива цветов RGB,
    группируя похожие цвета в ячейки и находя наиболее часто встречающийся.

    :param img: изображение в матричном формате
    :param bits_per_channel: желаемое количество бит на канал. По умолчанию - 6

    :return: 24-х битный RGB триплет с фоновым цветом
    """
    logger.info("Поиск цвета фона")
    # Проверка трёхслойности изображения
    assert img.shape[-1] == 3

    quantized = quantize(img, bits_per_channel).astype(int)
    packed = pack_rgb(quantized)

    unique, counts = np.unique(packed, return_counts=True)

    packed_mode = unique[counts.argmax()]

    return tuple(unpack_rgb(packed_mode))

def quantize(img: np.ndarray, bits_per_channel: int = 6):
    """
    Уменьшает количество бит на канал в переданном img.
    По сути, уменьшая битовую глубину, мы группируем похожие пиксели в более крупные «ячейки»,
    что упрощает поиск цвета фона.
    """
    assert img.dtype == np.uint8

    shift = 8 - bits_per_channel
    halfbin = (1 << shift) >> 1

    return ((img.astype(int) >> shift) << shift) + halfbin


import logging

from flask.views import MethodView
from mongoengine.errors import NotUniqueError, ValidationError

from .users_blp import users_blp
from ...models.user import User

from ...schemas.communs_schemas import PagingError
from ...schemas.users_schemas import (
    GetUsersListSchema,
    InputCreateUserSchema,
    UserResponseSchema
)

from ...helpers.errors_handler import BadRequest, ErrorHandler

logger = logging.getLogger('console')

@users_blp.route('/')
class RootUsersView(MethodView):

    @users_blp.doc(operationId='ListUsers')
    @users_blp.response(200, schema=GetUsersListSchema, description="List of users found in the database")
    def get(self):
        """Get a list of users"""
        users = User.objects()

        return {
            'users': users,
        }

    @users_blp.doc(operationId='CreateUser')
    @users_blp.arguments(InputCreateUserSchema)
    @users_blp.response(400, schema=PagingError, description="BadRequest")
    @users_blp.response(201, schema=UserResponseSchema, description="Infos of new user")
    def post(self, input_data: dict):
        """Create a new user"""
        user = User.create(input_data=input_data)

        try:
            user.save()
        except NotUniqueError:
            raise BadRequest(ErrorHandler.EMAIL_ALREADY_USED.value)

        return {
            'action': 'created',
            'user': user
        }


from flask import Flask, request, render_template, redirect, url_for, send_from_directory, abort, copy_current_request_context
from waitress import serve
from flask.templating import render_template
from flask_mail import Mail, Message
from cryptography.fernet import Fernet
import os
import configparser
import requests
import threading

#------------------------------------------------------------------Headers Server
project_root = os.path.dirname(os.path.realpath('__file__'))
template_path = os.path.join(project_root, 'app/templates')
static_path = os.path.join(project_root, 'app/static')
app = Flask(__name__, template_folder=template_path, static_folder=static_path)
#------------------------------------------------------------------

#Función para limpiar consola
def clearConsole():
    command = 'clear'
    if os.name in ('nt', 'dos'):  # If Machine is running on Windows, use cls
        command = 'cls'
    os.system(command)

#Clean Console
clearConsole()

#Correo
#Se carga archivo de configuración
config = configparser.ConfigParser()
config.read('./static/access/config.ini')

#acceso
fer = config['DEFAULT']['FER']
cypher = config['DEFAULT']['CYPHER']
fer = bytes(fer, 'utf-8')
cypher = bytes(cypher, 'utf-8')
siteKey = config['DEFAULT']['SITEKEY']
secretKey = config['DEFAULT']['SECRETKEY']
verifyURL = "https://www.google.com/recaptcha/api/siteverify"

#process
key = Fernet.generate_key()
f = Fernet(fer)
decypher = (f.decrypt(cypher)).decode()

def sendEmail(subject, name, email, phone, msg, to):
    message = Message(subject, sender = ('Consulta desde ortopediadrarreola.com', 'contacto@ortopediadrarreola.com'), recipients = [to])
    message.body = ".:: Recibimos la siguiente Consulta:\n\n-nombre: "+ name + "\n-correo: " + email + "\n-teléfono: " + phone + "\n-mensaje: " + msg
    mail.send(message)

def sendEmail2(subject2, name2, email2, phone2, msg2, to2):
    message2 = Message(subject2, sender = ('Dr. Atzin Arreola Ortopedista y Traumatólogo', 'contacto@ortopediadrarreola.com'), recipients = [to2])
    message2.body = ".:: Gracias por depositar su confianza en el Dr. Atzin Arreola\n\n:: Recibimos la siguiente Consulta:\n-nombre: "+ name2 + "\n-correo: " + email2 + "\n-teléfono: " + phone2 + "\n-mensaje: " + msg2 + "\n\n\nSu correo fue enviado correctamente, el Dr. se comunicara con usted en cuanto le sea posible.\n-= No es necesario responder este correo =-\n\n:: Calidad y Precisión en Cada Intervención ::"
    mail.send(message2)

#------------------------------------------------------------------Inicia aplicación

app = Flask(__name__)

#Control correo
app.config['MAIL_SERVER'] = 'mail.randrit.com'
app.config['MAIL_PORT'] = 465
app.config['MAIL_USERNAME'] = 'contacto@ortopediadrarreola.com'
app.config['MAIL_PASSWORD'] = decypher
app.config['MAIL_USE_TLS'] = False
app.config['MAIL_USE_SSL'] = True

mail = Mail(app)

#Carpeta de archivos temporales
app.config['UFtemp'] = 'static/temp'

#Limite de memoria disponible para carga de archivos
app.config['MAX_CONTENT_LENGTH'] = 50000

#------------------------------------------------------------------Inicio de sesión

@app.route('/')
def index():
    return redirect(url_for('home'))

@app.route('/home')
def home():
    return render_template('home.html', siteKey=siteKey)

@app.after_request
def after_request(response):
    response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
    return response

#------------------------------------------------------------------Links a secciones controladas

def status_401(error):
    return redirect(url_for('home'))

def status_404(error):
    return redirect(url_for('home'))

@app.route('/robots.txt')
@app.route('/sitemap.xml')
def static_from_root():
    return send_from_directory(app.static_folder, request.path[1:])

#------------------------------------------------------------------Función Correo

@app.route('/sent', methods = ['GET', 'POST'])
def send():
    if request.method == 'POST':

        name = request.form['nombre']
        email = request.form['email']
        msg = request.form['mensaje']

        if (name != "" and email != "" and msg != ""):

            secretResponse = request.form['g-recaptcha-response']
            verifyResponse = requests.post(url=f'{verifyURL}?secret={secretKey}&response={secretResponse}').json()

            if verifyResponse['success'] == False:
                abort(401)
            elif verifyResponse['score'] < 0.5:
                abort(401)
            else:
                # A correo de sitio
                subject = 'Correo enviado desde el Sitio Web'
                phone = request.form['telefono']
                to = 'contacto@ortopediadrarreola.com'

                # A correo de usuario
                subject2 = 'Correo del Dr Atzin Arreola'
                name2 = name
                email2 = email
                phone2 = phone
                msg2 = msg
                to2 = email2

                @copy_current_request_context
                def sendMessage(subject, name, email, phone, msg, to):
                    sendEmail(subject, name, email, phone, msg, to)

                sender1 = threading.Thread(name='mail_sender', target=sendMessage, args=(subject, name, email, phone, msg, to))
                sender1.start()

                @copy_current_request_context
                def sendMessage2(subject2, name2, email2, phone2, msg2, to2):
                    sendEmail2(subject2, name2, email2, phone2, msg2, to2)

                sender2 = threading.Thread(name='mail_sender_2', target=sendMessage2, args=(subject2, name2, email2, phone2, msg2, to2))
                sender2.start()

        return ('', 204)

@app.route('/mailok')
def mailok():
    return render_template("mailok.html")

#------------------------------------------------------------------Inicia de ejecución
if __name__ == '__main__':
    app.register_error_handler(401, status_401)
    app.register_error_handler(404, status_404)
    app.run(debug=True, port=5500)
    # serve(app, host='localhost', port=5500, threads=30)


# -*- coding: utf-8 -*-
import scrapy
from shiyanlougithub.items import ShiyanlougithubItem

class GithubSpider(scrapy.Spider):
    name = 'github'
    allowed_domains = ['https://github.com/shiyanlou?tab=repositories']
    start_urls = ['http://https://github.com/shiyanlou?tab=repositories/']

    @property
    def start_urls(self):
        url_tmp='https://github.com/shiyanlou?page={}&tab=repositories'
        return (url_tmp.format(i) for i in range(1,4))
    def parse(self, response):
        for course in response.css('li.public'):
            item=ShiyanlougithubItem()
            item["name"]=course.css("h3 a::text").re_first('[^\S]*(\S+)[^\S]*')
            item["update_time"]=course.css("div.f6.text-gray.mt-2 relative-time::attr(datetime)").extract_first()

            url=response.urljoin(course.css("a::attr(href)").extract_first())
            print('#'*20+url)
            #print(item)
            request=scrapy.Request(url,callback=self.parse_pro,dont_filter=True)
            request.meta['item']=item
            yield request
    def parse_pro(self,response):
        item=response.meta['item']
        print(item)
        for num in response.css('ul.numbers-summary li'):
            type_text=num.xpath('.//a/text()').re_first('\n\s*(.*)\n')
            number_text=num.xpath('.//span[@class="num text-emphasized"]/text()').re_first('\n\s*(.*)\n')
#            print("^^^^^^^^"+type_text)
#            print("--------"+number_text)
            if type_text and number_text:
                number_text=number_text.replace(',','')
                if type_text in ('commit','commits'):
                    item['commits']=int(number_text)
                elif type_text in ('branch','branches'):
                    item['branches']=int(number_text)
                elif type_text in ('release','releases'):
                    item['releases']=int(number_text)
        yield item


import torchvision

def get_backbone(name, pretrained=False):
    backbones = {
        "resnet18": torchvision.models.resnet18(pretrained=pretrained),
        "resnet50": torchvision.models.resnet50(pretrained=pretrained),
        "wide_resnet50_2": torchvision.models.wide_resnet50_2(pretrained=pretrained)
    }
    if name not in backbones.keys():
        raise KeyError(f"{name} is not a valid ResNet version")

    model = backbones[name]
    if name == 'resnet18' or 'resnet50':
        model.conv1 = nn.Conv2d(3, 64, 3, 1, 1, bias=False)
        model.maxpool = nn.Identity()
    return model


# This Python 3 environment comes with many helpful analytics libraries installed

# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python

# For example, here's several helpful packages to load in

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.

# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os

print(os.listdir("../"))

# Any results you write to the current directory are saved as output.
import pandas as pd

train_set = pd.read_csv('../input/train.csv')

test_set = pd.read_csv('../input/test.csv')

test_length = len(test_set)

train_set = pd.get_dummies(train_set,columns = ['diagnosis'])
train_image_path = "../input/train_images/"

test_image_path = "../input/test_images/"
display(train_set.head())
from skimage.io import imread

import matplotlib.pyplot as plt

path = train_image_path+train_set.iloc[2,0]+".png"

image = imread(path,as_gray=True)

img = imread(path)

plt.imshow(image,cmap='gray')

plt.imshow(img)
def get_train_input(image):

    path = train_image_path+image+".png"

    img = imread(path,as_gray=True)

    return img
def get_test_input(image):

    path = test_image_path+image+".png"

    img = imread(path,as_gray=True)

    return img
def get_output(image):

    df = train_set.loc[train_set['id_code'] == image]

    df = df.iloc[:,1:]

    return df.values[0]
from skimage.transform import resize,rotate

import cv2

def preprocess_input(image):

    kernel = np.ones((5,5),np.uint8)

    erosion = cv2.morphologyEx(image,cv2.MORPH_OPEN,kernel)

    erosion = resize(erosion,(100,100),anti_aliasing=False)

    erosion = rotate(erosion,np.random.randint(0,45))

    return resize(image,(100,100),anti_aliasing=False)

def train_image_data_generator(train_set,batch_size):

    while True:

        # Select files (paths/indices) for the batch

        batch_paths = np.random.choice(a = train_set.iloc[:,0], size = batch_size)

        batch_input = []

        batch_output = []

          # Read in each input, perform preprocessing and get labels

        for input_path in batch_paths:

            input = get_train_input(input_path)

            output = get_output(input_path)

            input = preprocess_input(image=input)

            input = np.reshape(input,(100,100,1))

            batch_input += [ input ]

            batch_output += [ output ]

          # Return a tuple of (input,output) to feed the network

        batch_x = np.array( batch_input )

        batch_y = np.array( batch_output )

        yield( batch_x, batch_y )
def test_image_data_generator(batch,batch_size):

        # Select files (paths/indices) for the batch

    for chunks in batch:

        batch_paths = chunks

        batch_input = []

            # Read in each input, perform preprocessing and get labels

        input = get_test_input(batch_paths[0])

        input = resize(image,(100,100),anti_aliasing=False)

        input = np.reshape(input,(100,100,1))

        batch_input += [ input ]

        # Return a tuple of (input,output) to feed the network

        batch_x = np.array( batch_input )

    return batch_x
from keras.layers import Conv2D,Dense,MaxPool2D,Flatten

from keras.models import Sequential

batch_size = 32

model = Sequential()

model.add(Conv2D(4,kernel_size = (3,3),padding = 'same',activation='relu',input_shape = (100,100,1)))

model.add(Conv2D(32,kernel_size = (3,3),activation='relu'))

model.add(Flatten())

model.add(Dense(64,activation = 'relu'))

model.add(Dense(32,activation = 'relu'))

model.add(Dense(5,activation = 'softmax'))

model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=["mae","accuracy"])

model.summary()
generator = train_image_data_generator(train_set,batch_size)

history = model.fit_generator(generator,steps_per_epoch = len(train_set)//batch_size,epochs= 15,verbose=1)
print("Loss :- "+ str(history.history['loss'][0]))

print("Mean absolute error :- "+ str(history.history['mean_absolute_error'][0]))

model.save_weights('model.h5')
import sys

test_set = pd.read_csv('../input/test.csv',chunksize= 32)

row_list = []

obj = {}

pred = pd.DataFrame(columns=['id_code','diagnosis'])

for chunks in test_set:

    data = chunks.values

    for item in data:

        image = get_test_input(item[0])

        kernel = np.ones((5,5),np.uint8)

        input = resize(image,(100,100),anti_aliasing=False)

        input = np.reshape(input,(1,100,100,1))

        out = model.predict(input)

        obj = {

            'id_code':item[0],

            'diagosis':np.argmax(out,axis=1)[0]

        }

        row_list.append(obj)

        sys.stdout.write(str(len(row_list)))

        sys.stdout.flush()

final_pred = pd.DataFrame(row_list)
final_pred.to_csv('submission.csv', index = False, header = True, sep = ',', encoding = 'utf-8')


"""This module does blah blah."""
import json

SEED = 12
NUM_THREADS = 8
WRITE_IMG_SUMMARY_EVERY = 100

with open('config.json', 'r') as f:
    CONFIG_JSON = json.load(f)

CONFIG = CONFIG_JSON[CONFIG_JSON['active_config']]

BASE_PATH = CONFIG['base_path']
DATA_PATH = CONFIG['data_path']
CHKPTS_PATH = CONFIG['chkpts_path']
GRAPHS_PATH = CONFIG['graphs_path']
OUTPUT_PATH = CONFIG['output_path']
FULL_OUTPUT_PATH = BASE_PATH + OUTPUT_PATH

DIM_X = CONFIG['output']['dimX']
DIM_Y = CONFIG['output']['dimY']
DIM_Z = CONFIG['output']['dimZ']

BATCH_SIZE = CONFIG['input']['batch_size']
Z_NOISE_DIM = CONFIG['input']['z_noise_dim']
Y_DIM = 2

F_DIM = CONFIG['training']['f_dim']
NUM_EPOCHS = CONFIG['training']['num_epochs']
D_LEARNING_RATE = CONFIG['training']['d_learning_rate']
G_LEARNING_RATE = CONFIG['training']['g_learning_rate']
BETA1 = CONFIG['training']['beta1']

SAVE_MODEL_EVERY = CONFIG['save_model_every']
SAVE_EXAMPLE_EVERY = CONFIG['save_example_every']
PRINT_INFO_EVERY = CONFIG['print_info_every']


import depthai as dai
import numpy as np
import cv2
try:
    import toolkit.methodspaths as methodspaths
except:
    import methodspaths

class PoseEdgeWorker:
    def __init__(self, method, detector):
        self._method = method
        self._detector = detector
        self._det_pool = ['yolox']
        self._cheatsheet = {'blazepose_lite': 0,
                            'blazepose_full': 0,
                            'blazepose_heavy': 0,
                            'hrnet_vpose': 1,
                            'hrnet_sbl': 1,
                            'hrnet_gcn': 1,
                            'hrnet_stgcn': 1,
                            'hrnet_poseaugvpose': 1,
                            'hrnet_poseaugsbl': 1,
                            'hrnet_poseauggcn': 1,
                            'hrnet_poseaugstgcn': 1}
        self._setup()

    def _setup(self):
        dt_ok = any(self._detector == word for word in self._det_pool)
        if dt_ok:
            try:
                _selector = self._cheatsheet[self._method]
                if _selector == 0:
                    self._est = PoseEdgeWorkerSingleStage(method=self._method,
                                                          detector=self._detector)
                elif _selector == 1:
                    x_mthd = self._method.split('_')
                    self._est = PoseEdgeWorkerTwoSteps(first_stage=x_mthd[0],
                                                       second_stage=x_mthd[1],
                                                       detector=self._detector)
            except:
                raise TypeError('selection not valid')
        else:
            raise TypeError('method or architecture not valid')

    def estimate(self, in_img):
        _out = self._est.estimate(in_img)
        return _out

class PoseEdgeWorkerSingleStage:
    def __init__(self, method, detector):
        self._method = method
        self._detector = detector
        self._setup()

    def _setup(self):
        self.path_to_detector = ''
        self.path_to_estimator = ''

    def estimate(self, in_img):
        return 0

class PoseEdgeWorkerTwoSteps:
    def __init__(self, first_stage, second_stage, detector):
        self._first_stage = first_stage
        self._second_stage = second_stage
        self._detector = detector
        self._setup()

    def _setup(self):
        zero_stage_paths = self._detector + '_Paths'
        self._paths0 = methodspaths.methodsDict[zero_stage_paths]
        first_stage_paths = self._first_stage + '_Paths'
        self._paths1 = methodspaths.methodsDict[first_stage_paths]
        second_stage_paths = self._second_stage + '_Paths'
        self._paths2 = methodspaths.methodsDict[second_stage_paths]

        self.path_to_detector = self._paths0.blob
        self.path_to_estimator = self._paths1.blob
        self.path_to_lifter = self._paths2.blob
        self._input_shape_detection = (640, 640)  # yolox, todo: move all constants to somewhere else
        self._input_shape_estimation = (256, 256)
        self._pipeline = self._setup_pipeline()

    def _setup_pipeline(self):
        nr_inference_threads = 1
        blocking_opt = True
        nr_per_threads = 1

        # ----------------------------------------------------------------

        l_pipeline = dai.Pipeline()
        l_pipeline.setOpenVINOVersion(dai.OpenVINO.Version.VERSION_2021_4)

        # setting up camera node
        cam_option = False
        if cam_option:
            camera_rgb = l_pipeline.create(dai.node.ColorCamera)
            camera_rgb.setPreviewSize(self._input_shape_detection[0], self._input_shape_detection[1])
            camera_rgb.setInterleaved(False)
            camera_rgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.RGB)
        else:
            xRGBIn = l_pipeline.create(dai.node.XLinkIn)
            xRGBIn.setStreamName("input_img")

        # setting up person detector stage
        detector_nn = l_pipeline.create(dai.node.NeuralNetwork)
        detector_nn.setBlobPath(self.path_to_detector)
        detector_nn.setNumInferenceThreads(nr_inference_threads)
        detector_nn.input.setBlocking(blocking_opt)
        detector_nn.setNumNCEPerInferenceThread(nr_per_threads)

        if False:
            # setting up preprocess for estimation part I/II (crop) and II/II (resize)
            pre_0_manip = l_pipeline.create(dai.node.ImageManip)
            pre_0_manip.initialConfig.setCropRect(0., 0., 1., 1.)
            pre_0_manip.initialConfig.setResize(self._input_shape_estimation[0], self._input_shape_estimation[0])
            pre_0_manip.setMaxOutputFrameSize(self._input_shape_estimation[0]*self._input_shape_estimation[1]*3)
            pre_0_manip.inputImage.setQueueSize(1)
            pre_0_manip.inputImage.setBlocking(blocking_opt)
            pre_0_manip.setWaitForConfigInput(False)

            # setting up 2d pose landmark estimator stage
            estimator_nn = l_pipeline.create(dai.node.NeuralNetwork)
            estimator_nn.setBlobPath(self.path_to_estimator)
            estimator_nn.setNumInferenceThreads(nr_inference_threads)
            estimator_nn.input.setBlocking(blocking_opt)
            estimator_nn.setNumNCEPerInferenceThread(nr_per_threads)

            # setting up 3d pose lifter stage
            lifter_nn = l_pipeline.create(dai.node.NeuralNetwork)
            lifter_nn.setBlobPath(self.path_to_lifter)
            lifter_nn.setNumInferenceThreads(nr_inference_threads)
            lifter_nn.input.setBlocking(blocking_opt)
            lifter_nn.setNumNCEPerInferenceThread(nr_per_threads)

        xout_bb = l_pipeline.create(dai.node.XLinkOut)
        xout_bb.setStreamName("bb_out")

        # linking stuff
        if cam_option:
            camera_rgb.preview.link(detector_nn.input)
        else:
            xRGBIn.out.link(detector_nn.input)
        detector_nn.out.link(xout_bb.input)

        return l_pipeline

    def estimate(self, in_img):
        probe_img_0 = cv2.resize(in_img, (self._input_shape_detection[0], self._input_shape_detection[1]),
                                 interpolation=cv2.INTER_AREA)
        input_i = np.moveaxis(probe_img_0, [0, 1, 2], [2, 1, 0])
        input_i = np.moveaxis(input_i, [0, 1, 2], [0, 2, 1])
        input_i = np.array([input_i, ]).astype(np.uint8)
        with dai.Device(self._pipeline) as device:

            device.setLogLevel(dai.LogLevel.WARN)
            device.setLogOutputLevel(dai.LogLevel.WARN)
            q_bb_out = device.getOutputQueue("bb_out", maxSize=1, blocking=False)
            q_img_in = device.getInputQueue("input_img")
            try:
                img_nn = dai.NNData()
                #planar_nn = input_i.astype(np.float64).flatten()
                planar_nn = input_i.flatten()
                img_nn.setLayer("input_1", planar_nn)
                q_img_in.send(img_nn)

                bb_outq = q_bb_out.get()
                if bb_outq is not None:
                    bb_ou3456yu = bb_outq.getAllLayers()
                    #bb_ou = bb_outq.getLayerFp16('902')
                    bb_ou = bb_outq.getLayerFp16('1049')
                    wally = 'here'
            except:
                raise TypeError('something')
        return 0


import io
import re

def __export_remote_functions():

    def substitute_vars(string, vars: dict):
        # Define a regular expression for placeholders
        placeholder_pattern = re.compile(r"\$\{?(\w+)(?:-(\w+))?\}?")
        # Define a function to replace the placeholders with keyword arguments
        def replace_placeholder(match):
            # Get the variable name and the default value from the match object
            var_name = match.group(1)
            default_value = match.group(2)
            # Check if the variable name is in the keyword arguments
            if var_name in vars:
                # Return the value of the keyword argument
                return vars[var_name]
            else:
                # Check if there is a default value
                if default_value is not None:
                    # Return the default value
                    return default_value
                else:
                    # Raise an exception if there is no default value
                    raise ValueError(f"Missing keyword argument: {var_name}")
        # Return the string with the placeholders replaced
        return placeholder_pattern.sub(replace_placeholder, string)

    def process_cp2k_macro(fp):
        # Define a regular expression for @SET directive with case insensitive flag
        set_pattern = re.compile(r"@SET\s+(\w+)\s+(.+)", re.IGNORECASE) # Added re.IGNORECASE flag
        # Initialize an empty dictionary for variables and a list for output lines
        variables = {}
        output_lines = []
        # Read line by line from the input file object
        for line in fp:
            # Strip whitespace and comments
            line = line.strip()
            line = line.split("#")[0]
            # Skip empty lines
            if not line:
                continue
            # Match @SET directive
            set_match = set_pattern.match(line)
            if set_match:
                # Get the variable name and value
                var_name = set_match.group(1)
                var_value = set_match.group(2).strip()

                # Check if the variable name is valid
                if var_name[0].isdigit():
                    # Raise an exception if the variable name starts with a number
                    raise ValueError(f"Invalid variable name: {var_name}")
                else:
                    # Assign the value to the variable in the dictionary
                    variables[var_name] = var_value
                # Skip the @SET line
                continue

            # Append the line to the output list
            output_lines.append(line)

        # Return the variables dictionary and the output list as a single string with newline characters
        return variables, "\n".join(output_lines)

    def parse_cp2k_input(fp):
        # Initialize an empty dictionary and a stack
        output = {}
        stack = []
        # Open the input file and read line by line
        for line in fp:
            # Strip whitespace and comments
            line = line.strip()
            line = line.split("#")[0]
            # Skip empty lines
            if not line:
                continue
            # Check if the line starts with &
            if line.startswith("&"):
                # Get the keyword name and strip the trailing whitespace
                keyword = line[1:].strip()
                # Split the keyword by whitespace and check if the first token is END
                tokens = keyword.split()
                if tokens[0] and tokens[0].upper() == "END":
                    # Pop the last section from the stack
                    stack.pop()
                else:
                    # If the stack is empty, add the keyword to the output dictionary
                    if not stack:
                        output[keyword] = {}
                        stack.append(keyword)
                    else:
                        # If the stack is not empty, get the current section from the output dictionary
                        current_section = output
                        for section in stack:
                            current_section = current_section[section]
                        # Add the keyword to the current section as a sub-dictionary
                        current_section[keyword] = {}
                        stack.append(keyword)
                # Continue to the next line
                continue
            # Split the line by whitespace
            tokens = line.split()
            # Get the value name and value
            value_name = tokens[0]
            value = " ".join(tokens[1:])
            # Get the current section from the output dictionary
            current_section = output
            for section in stack:
                current_section = current_section[section]
            # Add the value name and value to the current section
            current_section[value_name] = value
        return output

    # TODO: handle coords
    def load_cp2k_input(fp):
        variables, processed_text = process_cp2k_macro(fp)
        substituted_text = substitute_vars(processed_text, variables)
        return parse_cp2k_input(io.StringIO(substituted_text))

    def loads_cp2k_input(text):
        return load_cp2k_input(io.StringIO(text))

    # TODO: handle coords
    def dumps_cp2k_input(input_dict):
        # Initialize an empty list for output lines
        output_lines = []
        # Define a helper function to recursively dump the sections and values
        def dump_section(section_dict, indent=0):
            # Loop through the keys and values in the section dictionary
            for key, value in section_dict.items():
                # Check if the value is a sub-dictionary
                if isinstance(value, dict):
                    # Add a line with the section name and indentation
                    output_lines.append(" " * indent + f"&{key}")
                    # Recursively dump the sub-section with increased indentation
                    dump_section(value, indent + 3)
                    # Add a line with the end of section and indentation
                    output_lines.append(" " * indent + "&END")
                else:
                    # Add a line with the value name and value and indentation
                    output_lines.append(" " * indent + f"{key}  {value}")
        # Dump the input dictionary using the helper function
        dump_section(input_dict)
        # Return the output list as a single string with newline characters
        return "\n".join(output_lines)

    def dump_cp2k_input(input_dict, fp):
        fp.write(dumps_cp2k_input(input_dict))

    return dump_cp2k_input, dumps_cp2k_input, load_cp2k_input, loads_cp2k_input,

(
    dump_cp2k_input, dumps_cp2k_input,
    load_cp2k_input, loads_cp2k_input,
) = __export_remote_functions()


from .gui import *
import logging
import warnings
logging.basicConfig(level=logging.DEBUG, filename="mylog.txt")

class Box:
    def __init__(self, orientation, *args):
        self.mapped = False
        self.orientation = orientation
    def __repr__(self):
        return 'Py{}Box'.format(self.orientation)
    def ismapped(self):
        return self.mapped
    def map(self, *args):
        self.mapped = True
        return 1
    def unmap(self, *args):
        self.mapped = False
        return 1

class graphProto:
    def __init__(self, *args):
        self.expressions = []
        self.mode = 0
    def __repr__(self):
        return 'GraphProto'
    def addexpr(self, expr):
        self.expressions.append(expr)
        print(self.expressions)
        return 1
    def plot(self, *args):
        for e in self.expressions:
            print(e+"\n")
        return 1
    def __del__(self):
        print("goodbye Graph")

def graphmode(mode):
    print('obsolete graphmode')
    print(mode)
    return 0

class TextEditor:
    def __init__(self, param):
        self.param = param
    def __repr__(self):
        return f'TextEditor{self.param}'
    def name(self):
        return self.param

def list_browser(the_list):
    print('list_browser:')
    print(the_list)
    #for i in range(len(the_list)):
        #print(the_list.o(i))
    return 1

fn_map = {
    'xpanel': xpanel,
    'xlabel': xlabel,
    'xvalue': xvalue,
    'xcheckbox': xcheckbox,
    'xstatemenu': xstatebutton,
    'xbutton': xbutton,
    'xpvalue': xvalue,
    'xvarlabel': xvarlabel
}

class_map = {
    'HBox': HBox,
    'VBox': VBox,
    'Graph': Graph
}

def gui_callback(*args):
    global all_args
    all_args = args
    fn = args[0]
    obj = args[1]
    context = args[2]
    params = args[3:]
    logging.debug("fn: %s", fn)
    logging.debug("obj: %r", obj)
    logging.debug("params: %r", (params,))

    # testing
    if fn == 'List.browser':
        return list_browser(obj)

    elif fn == 'TextEditor':
        return TextEditor(*params)
    elif fn == 'TextEditor.text':
        return obj.name()

    elif fn == 'graphmode':
        return graphmode(*params)

    elif fn == 'boolean_dialog':
        print('boolean dialog')
        print(params[0])
        print('0:', params[2])
        print('1:', params[1])
        return float(input('your choice: '))

    # general redirection
    try:
        if not obj:
            if fn in fn_map:
                call = fn_map[fn]
                return call(*params) if context is None else call(*params, context=context)
            elif fn in class_map:
                return class_map[fn](*params)
        else:
            split = fn.split('.')
            call = getattr(obj, split[1])
            return call(*params) if context is None else call(*params, context=context)
    except:
        warnings.warn("redirection not set up for "+fn)

    return True


#!/usr/local/bin/python3

# *** NOTE: THIS SCRIPT MUST BE RUN AS ROOT ***

# python module imports
import os
import shutil
import nmap
import sys
import subprocess
from datetime import datetime

# variable containing the filepath of the nmap scan results file
mac_list = os.environ['HOME'] + "/maclist.log"

# variable containing the filepath of the approved mac addresses on the LAN file
masterfile = os.environ['HOME'] + "/master_mac"

# variable containing the filepath of the log file which documents every scan result
scanlog = os.environ['HOME'] + "/macscan.log"

# If a file from previous nmap scans exists, create a backup of the file
if os.path.exists(mac_list):
    shutil.copyfile(mac_list, os.environ['HOME'] + '/maclist_' + datetime.now().strftime("%Y_%m_%H:%M") + '.log.bk')

# Open a new file for the new nmap scan results
f = open(mac_list, "w+")

# Print to console this warning
print("Don't forget to update your network in the nmap scan of this script")

# This block of code will scan the network and extract the IP and MAC address
# Create port scanner object, nm
nm = nmap.PortScanner()
# Perform: nmap -oX - -n -sn 192.168.1.69/24
# *** NOTE: -sP has changed to -sn for newer versions of nmap! ***
nm.scan(hosts='192.168.1.69/24', arguments='-n -sn')
# Retrieve results for all hosts found
nm.all_hosts()
# For every host result, write their mac address and ip to the $mac_list file
for host in nm.all_hosts():
    # If 'mac' expression is found in results write results to file
    if 'mac' in nm[host]['addresses']:
        f.write(str(nm[host]['addresses']) + "\n")
    # If 'mac' expression isn't found in results print warning
    elif nm[host]['status']['reason'] != "localhost-response":
        print("MAC addresses not found in results, make sure you run this script as root!")

# Close file for editing
f.close()
# Open file for reading
f = open(mac_list, "r")
# Read each line of file and store it in a list
mac_addresses = f.read().splitlines()
# Close file for reading
f.close()
# Open masterfile for reading
if os.path.isfile(masterfile):
    f2 = open(masterfile, "r")
else:
    print("Could not find master_mac file! Verify file path is correct!")
    sys.exit()

# Read each line of file and store it in a list
master_mac_addresses = f2.read().splitlines()
# Close file for reading
f2.close()

# Create empty list of new devices found on network
new_devices = []

# For every list entry in the mac_addresses list
for i in mac_addresses:
    # Convert the list index from a string to a dictionary so we can parse out mac address for comparison
    dic = eval(i)
    # Compare mac address portion of dictionary to mac addresses in the master_mac_addresses file
    # If the scanned mac address is not in the master_mac_addresses file
    if dic['mac'] not in master_mac_addresses:
        # Add scanned mac address to new devices list
        new_devices.append(dic)

# Open scanlog file for appending
if os.path.isfile(scanlog):
    f3 = open(scanlog, 'a')
else:
    print("Could not find macscan.log file! Verify file path is correct!")
    sys.exit()

# If the new_devices list is empty
if len(new_devices) == 0:
    # Write in scanlog file: All is well on: [date]
    f3.write("\n- - - - All is well on: " + datetime.now().strftime("%Y_%m_%H:%M") + " - - - -\n")
else:
    warning = "\nWARNING!! NEW DEVICE(S) ON THE LAN!! - UNKNOWN MAC ADDRESS(ES): " + str(new_devices) + "\n"
    print(warning)
    # Write in scanlog file: New Devices on the LAN: {LIST OF DEVICES}
    f3.write(warning)

    try:
        # First command which gets PIPEd to second command
        p1 = subprocess.Popen(['echo', '-e', '\nWARNING!!'], stderr=subprocess.PIPE, universal_newlines=True,
                              stdout=subprocess.PIPE)
        # Second command which takes input from first command
        p2 = subprocess.Popen(['mutt', '-e', 'my_hdr From:user@email.com', '-s', 'WARNING!! NEW DEVICE ON THE '
                                                                                'LAN',
                               '-i', 'maclist.log', 'user@email.com'], stdin=p1.stdout, stdout=subprocess.PIPE)
        output = p2.communicate()
        # Print output of running command
        print(output)

        # Another way to send email with mutt using shell with python, for future reference.
        # ------------------------------------------------------
        #p1 = subprocess.Popen(['echo', '-e', '\nWARNING!!'], stderr=subprocess.PIPE, universal_newlines=True,
        #                      stdout=subprocess.PIPE)

        #subject = "WARNING, new device on LAN!"
        #content = "There is a new, unknown device on the LAN, see attached."
        #m1 = subprocess.Popen('echo "{content}" | mutt -a "{scanlog}" -s "{subject}" -- adambensee@gmail.com'.format(
        #    content=content, scanlog=scanlog, subject=subject), shell=True)
        #output = m1.communicate()
        # Print output of running command
        #print(output)
        # ---------------------------------------------------------

    except OSError as e:
        print("Error sending email: {0}".format(e))
    except subprocess.SubprocessError as se:
        print("Error sending email: {0}".format(se))

# Close the scanlog file
f3.close()


import requests
import time
import json
from constants import *

# initialize the scryfall API link and pull the data from the website
scryfallAPILink = (f"https://api.scryfall.com/cards/search?q=set:{setCode}"
                   f"+or+set:{bonusSheetCode}")
scryfallDataPath = 'scryfall.json'

downloadData = input("Do you really want to download the data? yes/no ")

if downloadData.lower() != "yes":
    raise ValueError("You called this file but didn't want to download any data")

def getScryfallData(link):
    # Send a GET requests to the URL
    scryfallData = requests.get(link)

    # Check if the requests was successful (status code 200)
    if scryfallData.status_code == 200:
        # Get the JSON data from the response
        scryfallJSON = scryfallData.json()

        if scryfallJSON["has_more"]:
            time.sleep(1)
            nextPage = getScryfallData(scryfallJSON["next_page"])
            return scryfallJSON["data"] + nextPage

        return scryfallJSON["data"]
    else:
        print("Warning: no data available. Please refrain from using "
              "the '!cardName' command.")

with open(scryfallDataPath, 'w', encoding="utf-8") as scryfall:
    json_data = scryfall.write(json.dumps(getScryfallData(scryfallAPILink)))

def get17LDataIntoFile(url, filePath):
    time.sleep(1)
    # get the 17L data
    response = requests.get(url)

    # if the query was successful, write it into the card rating JSON
    if response.status_code == 200:
        text_content = response.text

        with open(filePath, 'w') as file:
            file.write(text_content)
            print(f"ðŸ“ˆ stats for file path {filePath} loaded!")
    else:
        print("Request failed with status code:", response.status_code)

topURL = (f"https://www.17lands.com/card_ratings/data?"
          f"expansion={setCode.upper()}&"
          f"format=PremierDraft&"
          f"user_group=top"
          )

allURL = (f"https://www.17lands.com/card_ratings/data?"
          f"expansion={setCode.upper()}&"
          f"format=PremierDraft&"
          )

for pair in colorPairs:
    pairURL = (f"https://www.17lands.com/card_ratings/data?"
               f"expansion={setCode.upper()}&"
               f"format=PremierDraft&"
               f"colors={pair}"
               )

    topPairURL = (f"https://www.17lands.com/card_ratings/data?"
                  f"expansion={setCode.upper()}&"
                  f"format=PremierDraft&"
                  f"colors={pair}&"
                  f"user_group=top"
                  )

    get17LDataIntoFile(pairURL, f'requests/all/{pair}-card-ratings.json')
    get17LDataIntoFile(topPairURL, f'requests/top/{pair}-card-ratings.json')

get17LDataIntoFile(topURL, 'requests/top/card-ratings.json')
get17LDataIntoFile(allURL, 'requests/all/card-ratings.json')

print(f"ðŸ”® all stats loaded!")


from typing import Optional

import numpy as np

__all__ = ["Results"]

def _convert_arrays_to_lists(data: dict) -> dict:
    """If the input dictionary contains numpy arrays, this function will
    cast them to lists and return the same dictionary with the lists instead
    of the numpy arrays.

    Parameters
    ----------
    data : dict
        The dictionary to clean.

    Returns
    -------
    new_data : dict
        The dictionary with arrays casted to lists.
    """
    new_data = dict()
    for k, v in data.items():
        if isinstance(v, dict):
            new_data[k] = _convert_arrays_to_lists(v)
        elif isinstance(v, np.ndarray):
            new_data[k] = v.tolist()
        else:
            new_data[k] = v
    return new_data

class Results:
    """Common class for GLM results."""

    def to_dict(self) -> dict:
        """Returns the `Results` instance in dict format.

        Private attributes and all methods are not returned.

        This function does not perform any casting.

        Returns
        -------
        data : dict
            Resulting dictionary.
        """
        import inspect

        data = dict()
        for attribute in inspect.getmembers(self):
            name, value = attribute
            if not name.startswith("_"):
                if not inspect.ismethod(value):
                    if hasattr(value, "to_dict"):
                        data[name] = value.to_dict()
                    else:
                        data[name] = value
        return data

    def to_json(self, indent: Optional[int] = 4) -> str:
        """Returns the json of the `Results` instance.

        Parameters
        ----------
        indent : int, optional
            Indent to use. Default=4.

        Returns
        -------
        str :
            The JSON dumps of the results.
        """
        import json

        return json.dumps(_convert_arrays_to_lists(self.to_dict()), indent=indent)


# convert group table in to splitted table with 80+ columns

import pandas as pd
import json
from pprint import pprint

with open('results-20181108-202123.json') as f:
    data = json.load(f)
ages = ["age_0","age_20","age_30","age_40","age_50","age_60","age_70","age_80","age_99"]
sexes = ["F_SEX","M_SEX"]
final_df = None
for item in data:
    csv_columns = {"FLX_ANN_MOI":0 , "M_SEX":0, "F_SEX":0 ,"age_0":0,"age_20":0,"age_30":0,
    "age_40":0,"age_50":0,"age_60":0,"age_70":0,"age_80":0,"age_99":0}
    csv_columns["FLX_ANN_MOI"] = item["FLX_ANN_MOI"]
    for age in ages:
        som = 0.0
        for i in item[age]:
            som+= float(i)
        csv_columns[age]+=som
    for sex in sexes:
        som = 0.0
        for i in item[sex]:
            som+= float(i)
        csv_columns[sex]+=som
    for i in range(len(item["maladie"])):
        n_c = item["reg"][i]+"_"+item["maladie"][i]
        if n_c in csv_columns:
            csv_columns[n_c]+=float(item["dep"][i])
        else:
            csv_columns[n_c]=float(item["dep"][i])
    df = pd.DataFrame(csv_columns,index=[0])
    try:
        if final_df == None:
            final_df = df
        else:
            print("append done IN ELSE")
            final_df = final_df.append(df,sort=True)
    except TypeError:
        print("append done IN EXCEPT")
        final_df = final_df.append(df,sort=True)
    #print(final_df)

final_df.to_csv("csv_file_01.csv")
#print(final_df)

# pour chaque couple reg et mnt

# on prends la valeure a partir de l'index
# reg = "86", mnt = 232, maladie = "12"
# average des sex

'''
pprint(data)
with open("./csv_test.csv") as csvfile:
    spamreader = csv.reader(csvfile, delimiter=',')
    for row in spamreader:
'''


import csv
import os
from contextlib import suppress

from PySide2 import QtCore, QtGui
from PySide2.QtGui import QStandardItemModel, QStandardItem
from PySide2.QtWidgets import *

from modbus_client.gui.style.custom_elements import CenterDelegate
from modbus_client.resources.codes import ErrorCodes

class HistorianWidget(QGroupBox):

    def __init__(self):
        super(HistorianWidget, self).__init__()
        self.existing_responses = set()
        self.existing_requests = set()
        self.loaded = False

        layout = QHBoxLayout()

        request_box = QGroupBox('REQUEST HISTORY')
        request_box.setAlignment(QtCore.Qt.AlignCenter)
        self.export_request_history = QPushButton('Export to CSV')
        self.request_history = QTableView()
        self.request_rows = QStandardItemModel()
        self.request_history.setSortingEnabled(True)
        # self.request_history.verticalHeader().hide()
        self.request_history.setItemDelegate(CenterDelegate())
        # self.request_history.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
        self.request_history.horizontalHeader().setStretchLastSection(True)
        self.request_history.setEditTriggers(QAbstractItemView.NoEditTriggers)
        self.request_header_labels = ['Timestamp', 'Transaction ID', 'Unit Address', 'Function Code', 'Data']
        self.request_rows.setHorizontalHeaderLabels(self.request_header_labels)
        req_layout = QVBoxLayout()
        req_layout.addWidget(self.request_history)
        req_layout.addWidget(self.export_request_history)
        request_box.setLayout(req_layout)

        response_box = QGroupBox('RESPONSE HISTORY')
        response_box.setAlignment(QtCore.Qt.AlignCenter)
        self.export_response_history = QPushButton('Export to CSV')
        self.response_history = QTableView()
        self.response_rows = QStandardItemModel()
        self.response_history.setSortingEnabled(True)
        # self.response_history.verticalHeader().hide()
        self.response_history.setItemDelegate(CenterDelegate())
        # self.response_history.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
        self.response_history.horizontalHeader().setStretchLastSection(True)
        self.response_history.setEditTriggers(QAbstractItemView.NoEditTriggers)
        self.response_header_labels = ['Timestamp', 'Transaction ID', 'Unit Address', 'Function Code', 'Data']
        self.response_rows.setHorizontalHeaderLabels(self.response_header_labels)
        res_layout = QVBoxLayout()
        res_layout.addWidget(self.response_history)
        res_layout.addWidget(self.export_response_history)
        response_box.setLayout(res_layout)

        layout.addWidget(request_box)
        layout.addWidget(response_box)
        self.setLayout(layout)

    def load(self, history):

        responses = history['response_history']

        self.response_history.setSortingEnabled(False)

        for response in responses:
            if response not in self.existing_responses:
                self.response_rows.insertRow(0)
                timestamp = QStandardItem()
                timestamp.setData(response[0], QtCore.Qt.EditRole)
                self.response_rows.setItem(0, 0, timestamp)
                transaction_id = QStandardItem()
                transaction_id.setData(response[1], QtCore.Qt.EditRole)
                self.response_rows.setItem(0, 1, transaction_id)
                unit_address = QStandardItem()
                unit_address.setData(response[2], QtCore.Qt.EditRole)
                self.response_rows.setItem(0, 2, unit_address)
                function_code = QStandardItem()
                function_code.setData(response[3], QtCore.Qt.EditRole)
                self.response_rows.setItem(0, 3, function_code)
                data = QStandardItem()
                data.setData(str(response[4]), QtCore.Qt.EditRole)
                self.response_rows.setItem(0, 4, data)
                if response[3] in [x.value for x in ErrorCodes]:
                    error_color = QtGui.QColor(255, 114, 111)
                    timestamp.setBackground(error_color)
                    transaction_id.setBackground(error_color)
                    unit_address.setBackground(error_color)
                    function_code.setBackground(error_color)
                    data.setBackground(error_color)

        self.response_history.setModel(self.response_rows)

        self.existing_responses = self.existing_responses | set(responses)
        self.response_history.setSortingEnabled(True)

        requests = history['request_history']
        self.request_history.setSortingEnabled(False)
        for request in requests:
            if request not in self.existing_requests:
                self.request_rows.insertRow(0)
                timestamp = QStandardItem()
                timestamp.setData(request[0], QtCore.Qt.EditRole)
                self.request_rows.setItem(0, 0, timestamp)
                transaction_id = QStandardItem()
                transaction_id.setData(request[1], QtCore.Qt.EditRole)
                self.request_rows.setItem(0, 1, transaction_id)
                unit_address = QStandardItem()
                unit_address.setData(request[2], QtCore.Qt.EditRole)
                self.request_rows.setItem(0, 2, unit_address)
                function_code = QStandardItem()
                function_code.setData(request[3], QtCore.Qt.EditRole)
                self.request_rows.setItem(0, 3, function_code)
                data = QStandardItem()
                data.setData(str(request[4]), QtCore.Qt.EditRole)
                self.request_rows.setItem(0, 4, data)
                if request[3] in [x.value for x in ErrorCodes]:
                    error_color = QtGui.QColor(255, 114, 111)
                    timestamp.setBackground(error_color)
                    transaction_id.setBackground(error_color)
                    unit_address.setBackground(error_color)
                    function_code.setBackground(error_color)
                    data.setBackground(error_color)

        self.request_history.setModel(self.request_rows)

        self.existing_requests = self.existing_requests | set(requests)
        self.request_history.setSortingEnabled(True)

    def export_request_history_to_csv(self, requests):
        with suppress(FileNotFoundError):
            path = os.path.abspath(__file__ + '/../')
            request_history_name = QFileDialog.getSaveFileName(self, 'Save file', path + '/request_history.csv',
                                                               'CSV (*.csv)',
                                                               options=QFileDialog.DontUseNativeDialog)
            request_history = open(request_history_name[0], 'w')
            with request_history:
                write = csv.writer(request_history)
                write.writerow(self.request_header_labels)
                write.writerows(requests)

    def export_response_history_to_csv(self, responses):
        with suppress(FileNotFoundError):
            path = os.path.abspath(__file__ + '/../')
            response_history_name = QFileDialog.getSaveFileName(self, 'Save file', path + '/response_history.csv',
                                                                'CSV (*.csv)',
                                                                options=QFileDialog.DontUseNativeDialog)
            response_history = open(response_history_name[0], 'w')
            with response_history:
                write = csv.writer(response_history)
                write.writerow(self.response_header_labels)
                write.writerows(responses)


from bson import ObjectId
from pymongo import ASCENDING,MongoClient
from flask import g

class SettingModel():
    def __init__(self, settingId=None):
        self._id = ObjectId(settingId)
        self.collection = getattr(g,'database',MongoClient('localhost', 27017)).db_meta.settings
        self.collection_config = getattr(g,'database',MongoClient('localhost', 27017)).db_meta.configuration

    def get_all(self):
        result = []
        req = self.collection.find({}).sort('group', ASCENDING)
        if req.count() > 0:
            result = []
            for r in req:
                r['_id'] = str(r['_id'])
                result.append(r)
        return result

    def get_version(self):
        req = self.collection_config.find_one()
        req['_id']=str(req['_id'])
        return req


import os
import csv
import unicodedata
import datetime
import eel

### 引数textの全角文字数カウント
def get_zen_count(text):
	count=0
	for c in text:
		if unicodedata.east_asian_width(c) in "FWA":
			count +=1
	return count

### 商品クラス
class Item:
	def __init__(self,code,name,price,cnt):
		self.item_code=code
		self.item_name=name
		self.item_price=price
		self.item_cnt=cnt

### 注文クラス
class Order:
	def __init__(self):
		self.order_list=[]
		self.order_cnt = 0
		self.order_price = 0

	def item_data_to_text(self,co,na,pr,n):
		na_width = 10-get_zen_count(na)
		pr_total = str( int(pr)*int(n) ).rjust(5)
		return co.ljust(4) + na.ljust(na_width) + str(pr).rjust(4)+"円" + str(n).rjust(3)+"個   = " + pr_total+"円\n"

	def add_order_list(self,code,name,price,cnt):
		add_flg = 0
		for item in self.order_list:
			if item.item_code == code:
				item.item_code = code
				item.item_name = name
				item.item_price = price
				item.item_cnt = str(int(item.item_cnt) + int(cnt))
				self.order_cnt += int(cnt)
				self.order_price += int(price)*int(cnt)
				add_flg = 1
			# JS に書き出し
			eel.order_list_add_write_js(self.item_data_to_text(item.item_code,item.item_name,item.item_price,item.item_cnt))

		if add_flg == 0:
			self.order_cnt += int(cnt)
			self.order_price += int(price)*int(cnt)
			self.order_list.append(Item(code,name,price,cnt))
			# JS に書き出し
			eel.order_list_add_write_js(self.item_data_to_text(code,name,price,cnt))
		eel.order_list_add_write_js("-------------------------------------\n")
		eel.order_list_add_write_js("合計                "+str(self.order_cnt).rjust(3)+"個     "+str(self.order_price).rjust(5)+"円\n")

	def del_order_list(self,code, name, price, cnt):
		del_flg = -1
		del_cnt = 0
		i = 0
		# price:str、cnt:str、item_price:str、item_cnt:str
		for item in self.order_list:
			if item.item_code == code:
				item.item_code = code
				item.item_name = name
				item.item_price = price
				if int(item.item_cnt) <= int(cnt):
					del_cnt = int(item.item_cnt)
					del_flg = i
				else:
					del_cnt = int(cnt)
					del_flg = -2
				item.item_cnt = str(int(item.item_cnt) - del_cnt)
				self.order_cnt -= del_cnt
				self.order_price -= int(price)*del_cnt
			# JS に書き出し
			if item.item_cnt != "0":
				eel.order_list_add_write_js(self.item_data_to_text(item.item_code,item.item_name,item.item_price,item.item_cnt))
			i += 1
		if del_flg == -1:
			i = name.find("　")
			eel.alert_js(f"「{name[:i]}」はかごに入っていません")
		if del_flg >= 0:
			self.order_list.pop(del_flg)
		if len(self.order_list) != 0:
			eel.order_list_add_write_js("-------------------------------------\n")
			eel.order_list_add_write_js("合計                "+str(self.order_cnt).rjust(3)+"個     "+str(self.order_price).rjust(5)+"円\n")

	def order_payment(self,cash):
		# print(type(cash))   str
		# print(type(self.order_cnt))   int
		# print(type(self.order_price))   int
		cash = int(cash)
		if cash < self.order_price:
			eel.payment_result_js(cash-self.order_price)
		else:
			# おつり計算と表示
			eel.order_list_add_write_js(f"お預かり                      {str(cash).rjust(5)}円\n")
			if cash == self.order_price:
				eel.alert_js(f"{cash}円丁度、お預かりします。")
			else:
				eel.order_list_add_write_js(f"おつり                        {str(int(cash) -self.order_price).rjust(5)}円\n")
				eel.alert_js(f"{cash}円お預かりし、{cash-self.order_price}円のおつりになります。")
			self.receipt_txt(cash)
			eel.payment_result_js(0)

	#レシートをファイルに出力
	def receipt_txt(self,cash):
		order_time = datetime.datetime.now()
		file_name = order_time.strftime("order_%Y-%m-%d_%H%M")
		if not os.path.isfile(file_name):
			with open( file_name+".txt", encoding="UTF-8", mode="w") as f:
				# ファイルに書き込む
				f.write("\n" + file_name+"\n")
				f.write("-------------------------------------\n")
				for item in self.order_list:                         #注文の合計を表示
					f.write(self.item_data_to_text(item.item_code,item.item_name,item.item_price,item.item_cnt))

				# 合計点数、合計金額の表示
				f.write("-------------------------------------\n")
				f.write(f"合計                "+str(self.order_cnt).rjust(3)+"個     "+str(self.order_price).rjust(5)+"円\n")
				f.write(f"お預かり                      {str(cash).rjust(5)}円\n")
				f.write(f"おつり                        {str(int(cash) -self.order_price).rjust(5)}円\n")
				f.write("-------------------------------------\n")
				f.write("\n「お買い上げありがとうございました。」\n")
				return file_name
		else:
			return "Error"

### csvファイルの読み込み
def set_item_master_csv(csv_file):
	with open(csv_file,encoding="utf_8",newline="") as items_csv:
		item_master = csv.reader(items_csv)
		next(item_master)

		# 商品リストを読み込む前にテキストエリアを空にする
		eel.item_list_add_item_js()
		for item in item_master:
			eel.item_list_add_item_js(item[0],item[1],item[2])