import logging
import math
from typing import List, Dict, Optional

import torch
from adet.layers import DFConv2d, NaiveGroupNorm
from adet.utils.comm import compute_locations
from detectron2.layers import ShapeSpec, NaiveSyncBatchNorm
from detectron2.modeling.proposal_generator.build import PROPOSAL_GENERATOR_REGISTRY
from sylph.modeling.meta_fcos.fcos_outputs import FCOSOutputs
from sylph.modeling.utils import init_norm
from torch import nn
from torch.nn import functional as F
from detectron2.utils.file_io import PathManager
from detectron2.data import MetadataCatalog
from .head_utils import CondConv, CondConvBasic

from .head_utils import CondConvBlock, Scale

logger = logging.getLogger(__name__)

__all__ = ["MetaFCOS"]

INF = 100000000

def load_pretrained_class_conv_kernel_weights(weight_path: str):
    if not PathManager.exists(weight_path):
        logger.info(f"Weight path: {weight_path} does not exist")
        return None, None

    local_weight_path = PathManager.get_local_path(weight_path)
    pretrained_model = torch.load(
        local_weight_path, map_location=torch.device("cpu")
    )
    # find : proposal_generator.fcos_head.cls_logits.weight
    weight, bias = None, None
    for key, value in pretrained_model["model"].items():
        # print(f"key {key}, value: {value}")
        if "proposal_generator.fcos_head.cls_logits" in key:
            if "weight" in key:
                weight = value
            elif "bias" in key:
                bias = value
    return weight, bias

class ModuleListDial(nn.ModuleList):
    def __init__(self, modules=None):
        super(ModuleListDial, self).__init__(modules)
        self.cur_position = 0

    def forward(self, x):
        result = self[self.cur_position](x)
        self.cur_position += 1
        if self.cur_position >= len(self):
            self.cur_position = 0
        return result

def _build_tower_module(
    head_configs: Dict,
    in_channels: int,
    norm: str,
    num_feature_levels: int,
):
    head_tower_sequential = {}
    for head, head_config in head_configs.items():
        num_convs, use_deformable = head_config
        tower = []
        for i in range(num_convs):
            if use_deformable and i == num_convs - 1:
                conv_func = DFConv2d
            else:
                conv_func = nn.Conv2d
            tower.append(
                conv_func(
                    in_channels,
                    in_channels,
                    kernel_size=3,
                    stride=1,
                    padding=1,
                    bias=True,
                )
            )
            if norm == "GN":
                tower.append(nn.GroupNorm(32, in_channels))
            elif norm == "NaiveGN":
                tower.append(NaiveGroupNorm(32, in_channels))
            elif norm == "BN":
                tower.append(
                    ModuleListDial(
                        [nn.BatchNorm2d(in_channels)
                         for _ in range(num_feature_levels)]
                    )
                )
            elif norm == "SyncBN":
                tower.append(
                    ModuleListDial(
                        [
                            NaiveSyncBatchNorm(in_channels)
                            for _ in range(num_feature_levels)
                        ]
                    )
                )
            tower.append(nn.ReLU())
        # pop out the last layer's ReLU in cls_tower
        # if use_l2_norm_conv2d and head == "cls":
        #     tower.pop()
        head_tower_sequential[head] = nn.Sequential(*tower)
    return head_tower_sequential

class BaseFewShotDetectorHead(nn.Module):
    def __init__(self, cfg, input_shape: List[ShapeSpec]):
        super().__init__()
        self.cfg = cfg
        self.input_shape = input_shape
        # get submodule configs
        self._init_base_detector_config()
        # get meta_learning
        self.episodic_learning = cfg.MODEL.META_LEARN.EPISODIC_LEARNING
        if self.episodic_learning:
            self._init_code_generator_config(
                name=cfg.MODEL.META_LEARN.CODE_GENERATOR.NAME
            )
            self._init_merge_module_config()

    def _init_base_detector_config(self, name: Optional[str] = None):
        pass

    def _init_code_generator_config(self, name: Optional[str] = None):
        pass

    def _init_merge_module_config(self, name: Optional[str] = None):
        pass

    #  forward
    # def forward_base_train(self, x, top_module=None, yield_bbox_towers=False):
    #     raise NotImplementedError()

    # def forward(self, x: List[Any],top_module=None,yield_bbox_towers: bool =False, support_set_class_codes: Dict[torch.Tensor]=None):
    #     raise NotImplementedError()

# Head outputs calls few-shot predictor head
@PROPOSAL_GENERATOR_REGISTRY.register()
class MetaFCOS(nn.Module):  # TODO: convert it to Base Few Shot Detector Head
    """
    Implement Meta FCOS.
    FCOS: (https://arxiv.org/abs/1904.01355).
    """

    def __init__(self, cfg, input_shape: Dict[str, ShapeSpec]):
        """
        DetectorHead (forward)
        Detector outputs (losses or proposals)
        """
        super().__init__()
        self.cfg = cfg
        self.episodic_learning = cfg.MODEL.META_LEARN.EPISODIC_LEARNING
        self.in_features = cfg.MODEL.FCOS.IN_FEATURES
        self.fpn_strides = cfg.MODEL.FCOS.FPN_STRIDES
        self.yield_proposal = cfg.MODEL.FCOS.YIELD_PROPOSAL

        # TODO: test more modualized head
        self.fcos_head = MetaFCOSHead(
            cfg, [input_shape[f] for f in self.in_features])
        self.in_channels_to_top_module = self.fcos_head.in_channels_to_top_module

        self.fcos_outputs = FCOSOutputs(cfg)

    def forward(
        self,
        images,
        features,
        gt_instances=None,
        top_module=None,
        support_set_per_class_code=None,
        support_set_targets=None,
    ):
        """
        Arguments:
            images (list[Tensor] or ImageList): query set images to be processed
            features: query set features.
            targets (list[BoxList]): ground-truth boxes present in the image (optional)

        Returns:
            result (list[BoxList] or dict[Tensor]): the output from the model.
                During training, it returns a dict[Tensor] which contains the losses.
                During testing, it returns list[BoxList] contains additional fields
                like `scores`, `labels` and `mask` (for Mask R-CNN models).

        """

        features = [features[f] for f in self.in_features]
        locations = self.compute_locations(features)
        logits_pred, reg_pred, ctrness_pred, iou_pred, top_feats, bbox_towers = self.fcos_head(
            features, top_module, self.yield_proposal, support_set_per_class_code
        )

        results = {}
        if self.yield_proposal:
            results["features"] = {f: b for f, b in zip(
                self.in_features, bbox_towers)}

        if self.training:
            # TODO(neogong): This need to be optimized. Find out a way to copy params after loading the checkpoint.
            if (
                self.episodic_learning
                and self.cfg.MODEL.META_LEARN.CODE_GENERATOR.DISTILLATION_LOSS_WEIGHT
            ):
                cls_logits_kernel = (
                    self.fcos_head.cls_logits.weight.data.clone().detach(),
                    self.fcos_head.cls_logits.bias.data.clone().detach(),
                )
            else:
                cls_logits_kernel = None
            # If support_set_targets is not None, it goes to meta_loss, else, pretraining loss

            results, losses = self.fcos_outputs.losses(
                logits_pred,
                reg_pred,
                ctrness_pred,
                iou_pred,
                locations,
                gt_instances,
                top_feats,
                support_set_targets=support_set_targets,
                support_set_per_class_code=support_set_per_class_code,
                cls_logits_kernel=cls_logits_kernel,
            )

            if self.yield_proposal:
                with torch.no_grad():
                    results["proposals"] = self.fcos_outputs.predict_proposals(
                        logits_pred,
                        reg_pred,
                        ctrness_pred,
                        iou_pred,
                        locations,
                        images.image_sizes,
                        top_feats,
                    )
            return results, losses
        else:
            results = self.fcos_outputs.predict_proposals(
                logits_pred,
                reg_pred,
                ctrness_pred,
                iou_pred,
                locations,
                images.image_sizes,
                top_feats,
            )

            return results, {}

    def compute_locations(self, features):
        """
        For each location(x,y)on the feature map Fi, we can map it back onto the input image as(⌊s/2⌋+xs,⌊s/2⌋+ys)
        locations: it has dimension [h*w, 2]
        """
        locations = []
        for level, feature in enumerate(features):
            h, w = feature.size()[-2:]
            locations_per_level = compute_locations(
                h, w, self.fpn_strides[level], feature.device
            )
            locations.append(locations_per_level)
        return locations

class MetaFCOSHead(BaseFewShotDetectorHead):
    """
    Define Meta-FCOS heads,  it initalize the base detector for training on base detector
    and it also prepare "class-conditional conv" for the meta-learning stage.
    """

    def __init__(self, cfg, input_shape: List[ShapeSpec]):
        super().__init__(cfg, input_shape)

    def _evaluate_with_base_class_codes(self, eval_dataset_name: str):
        if "all" not in eval_dataset_name:
            print(
                f"tried to eval with base class codes, but the dataset split is not 'all'")
            return None, None
        if self.base_weight is not None and self.base_bias is not None:
            print(f"evaluating with base weights.")
            base_dataset_name = self.cfg.DATASETS.BASE_CLASSES_SPLIT
            novel_dataset_name = self.cfg.DATASETS.NOVEL_CLASSES_SPLIT
            if base_dataset_name == '' or novel_dataset_name == '':
                print(f"evaluating with base weights.")
                return None, None
            assert base_dataset_name != '', "base data set name is not provided"
            assert novel_dataset_name != '', "novel data set name is not provided"
            # base_classes = MetadataCatalog.get(base_dataset_name).thing_classes
            base_dataset_id_to_continous_id = MetadataCatalog.get(
                base_dataset_name).thing_dataset_id_to_contiguous_id
            # training data classes
            novel_dataset_id_to_continous_id = MetadataCatalog.get(
                novel_dataset_name).thing_dataset_id_to_contiguous_id
            # combine base class codes with current class_codes

            current_dataset_id_to_continous_id = MetadataCatalog.get(
                eval_dataset_name).thing_dataset_id_to_contiguous_id
            N = len(current_dataset_id_to_continous_id)
            assert self.base_weight.size(1) == self.cls_logits.weight.size(1)
            assert self.base_weight.size(2) == self.cls_logits.weight.size(2)
            assert self.base_weight.size(3) == self.cls_logits.weight.size(3)
            new_weights = torch.zeros(N, self.base_weight.size(
                1), self.base_weight.size(2), self.base_weight.size(3))
            new_bias = torch.zeros(N)
            # new_weights = self.cls_logits.weight.data
            # new_bias = self.cls_logits.bias.data
            # copy base classes
            for base_dataset_id, base_index in base_dataset_id_to_continous_id.items():
                if base_dataset_id in current_dataset_id_to_continous_id.keys():
                    current_index = current_dataset_id_to_continous_id[base_dataset_id]
                    new_weights[current_index] = self.base_weight[base_index]
                    new_bias[current_index] = self.base_bias[base_index]
            # copy novel classes, save the novels,
            for novel_dataset_id, novel_index in novel_dataset_id_to_continous_id.items():
                if novel_dataset_id in current_dataset_id_to_continous_id.keys():
                    current_index = current_dataset_id_to_continous_id[novel_dataset_id]
                    new_weights[current_index] = self.cls_logits.weight[novel_index]
                    new_bias[current_index] = self.cls_logits.bias[novel_index]
            new_weights = new_weights.to(self.cls_logits.weight.device)
            new_bias = new_bias.to(self.cls_logits.weight.device)
            print(f"final weights: {new_weights.size()}")
        return new_weights, new_bias

    def _preload_cls_logits_weights(self):
        use_pretrained_base_cls_logits = self.cfg.MODEL.TFA.USE_PRETRAINED_BASE_CLS_LOGITS
        # if not self.cfg.MODEL.TFA.FINETINE:
        #     return
        if not use_pretrained_base_cls_logits:
            return
        # weight_path = self.cfg.MODEL.WEIGHTS
        # print(f"load class logits from weights path: {weight_path}")
        # base_weight, base_bias = load_pretrained_class_conv_kernel_weights(weight_path)
        if self.base_weight is not None and self.base_bias is not None:
            base_dataset_name = self.cfg.DATASETS.BASE_CLASSES_SPLIT
            novel_dataset_name = self.cfg.DATASETS.NOVEL_CLASSES_SPLIT
            print(f"Loading pretrained weights on cls_logits")
            if base_dataset_name == '' or novel_dataset_name == '':
                return
            assert base_dataset_name != '', "base data set name is not provided"
            assert novel_dataset_name != '', "novel data set name is not provided"
            # base_classes = MetadataCatalog.get(base_dataset_name).thing_classes
            base_dataset_id_to_continous_id = MetadataCatalog.get(
                base_dataset_name).thing_dataset_id_to_contiguous_id
            # replace
            new_weights = self.cls_logits.weight.data
            new_bias = self.cls_logits.bias.data
            current_dataset_id_to_continous_id = MetadataCatalog.get(
                self.cfg.DATASETS.TRAIN[0]).thing_dataset_id_to_contiguous_id
            # replace base dataset id to current weights
            for base_dataset_id, base_index in base_dataset_id_to_continous_id.items():
                if base_dataset_id in current_dataset_id_to_continous_id.keys():
                    current_index = current_dataset_id_to_continous_id[base_dataset_id]
                    new_weights[current_index] = self.base_weight[base_index]
                    new_bias[current_index] = self.base_bias[base_index]
            with torch.no_grad():
                self.cls_logits.weight = nn.Parameter(new_weights)
                self.cls_logits.bias = nn.Parameter(new_bias)
                print(
                    f"done Loading pretrained weights on cls_logits, number of classes: {self.cls_logits.weight.data.size(0)}")
        return

    def _init_base_detector_config(self, name: Optional[str] = None):
        """
        Extract useful FCOS config
        """
        self.num_classes = self.cfg.MODEL.FCOS.NUM_CLASSES
        self.fpn_strides = self.cfg.MODEL.FCOS.FPN_STRIDES
        head_configs = {
            "cls": (
                self.cfg.MODEL.FCOS.NUM_CLS_CONVS,
                self.cfg.MODEL.FCOS.USE_DEFORMABLE,
            ),
            "bbox": (
                self.cfg.MODEL.FCOS.NUM_BOX_CONVS,
                self.cfg.MODEL.FCOS.USE_DEFORMABLE,
            ),
            "share": (self.cfg.MODEL.FCOS.NUM_SHARE_CONVS, False),
        }
        norm = None if self.cfg.MODEL.FCOS.NORM == "none" else self.cfg.MODEL.FCOS.NORM
        self.num_levels = len(self.input_shape)
        in_channels = [s.channels for s in self.input_shape]
        assert len(set(in_channels)
                   ) == 1, "Each level must have the same channel!"
        in_channels = in_channels[0]

        self.in_channels_to_top_module = in_channels
        # Build all layers befor the predictor layer
        head_tower_sequential = _build_tower_module(
            head_configs, in_channels, norm, self.num_levels,
        )
        for head in head_configs.keys():
            self.add_module(f"{head}_tower", head_tower_sequential[head])
        # The original clssification head
        self.use_l2_norm_conv2d = self.cfg.MODEL.FCOS.L2_NORM_CLS_WEIGHT
        if self.use_l2_norm_conv2d:  # will not be used
            from .head_utils import CosineSimilarityConv2d

            self.cls_logits = CosineSimilarityConv2d(
                in_channels, out_channel=self.num_classes
            )
        else:
            self.cls_logits = nn.Conv2d(
                in_channels,
                self.num_classes,
                kernel_size=self.cfg.MODEL.FCOS.CLS_LOGITS_KERNEL_SIZE,
                stride=1,
                padding=self.cfg.MODEL.FCOS.CLS_LOGITS_KERNEL_SIZE // 2,
            )  # default is 3 x 3

        self.bbox_pred = nn.Conv2d(
            in_channels, 4, kernel_size=3, stride=1, padding=1)
        self.ctrness = nn.Conv2d(
            in_channels, 1, kernel_size=3, stride=1, padding=1)
        self.iou_overlap = nn.Conv2d(
            in_channels, 1, kernel_size=3, stride=1, padding=1)

        if self.cfg.MODEL.FCOS.USE_SCALE:
            self.scales = nn.ModuleList(
                [Scale(init_value=1.0) for _ in range(self.num_levels)]
            )
        else:
            self.scales = None

        # Module initialization
        for modules in [
            self.cls_tower,
            self.bbox_tower,
            self.share_tower,
            self.cls_logits,
            self.bbox_pred,
            self.ctrness,
            self.iou_overlap
        ]:
            if modules is None:
                continue
            for layer in modules.modules():
                if isinstance(layer, nn.Conv2d):
                    print(f"init {layer} in {modules.modules()}")
                    torch.nn.init.normal_(layer.weight, std=0.01)
                    if layer.bias is not None:
                        torch.nn.init.constant_(layer.bias, 0)

        # initialize the bias for focal loss
        prior_prob = self.cfg.MODEL.FCOS.PRIOR_PROB
        bias_value = -math.log((1 - prior_prob) / prior_prob)
        if not self.use_l2_norm_conv2d:
            torch.nn.init.constant_(self.cls_logits.bias, bias_value)
        else:
            torch.nn.init.constant_(self.cls_logits.cls_layer.bias, bias_value)

        # TFA: load pretrained weights
            # print(f"load class logits from weights path: {weight_path}")
        # Always save it
        weight_path = self.cfg.MODEL.WEIGHTS
        print(f"weight path: {weight_path}")
        self.base_weight, self.base_bias = load_pretrained_class_conv_kernel_weights(
            weight_path)
        self._preload_cls_logits_weights()
        # save
        self.evaluate_with_base_class_codes = self.cfg.MODEL.TFA.EVAL_WITH_PRETRAINED_BASE_CLS_LOGITS
        print(
            f"eval with clss codes path: {self.evaluate_with_base_class_codes}")
        if self.evaluate_with_base_class_codes:
            self.eval_conv = CondConvBasic()

    def _init_code_generator_config(self, name: Optional[str]):
        """
        Will only be called while episodic learning is set to True
        """
        # all code generator has some basic configs

        if name == "CodeGenerator":
            # self.cls_logits will not be updated in episodic_learning.
            self.class_code_channel = (
                self.cfg.MODEL.META_LEARN.CODE_GENERATOR.OUT_CHANNEL
            )
            for param in self.cls_logits.parameters():
                param.requires_grad = False
            k_s = self.cfg.MODEL.META_LEARN.CODE_GENERATOR.CLS_LAYER[2]
            padding_size = k_s // 2
            # conditional classification head (takes in weight and bias)

            # self.cond_cls_logits = CondConvBlock(
            #     padding=padding_size, weight_len=self.class_code_channel
            # )

            self.cond_cls_logits = CondConvBasic(
                padding=padding_size,
                use_bias=self.cfg.MODEL.META_LEARN.CODE_GENERATOR.USE_BIAS
            )
            # self.score_scale = Scale(init_value=1.0)

            # CondConv(
            #     padding=padding_size,
            #     scale_score=True,
            #     l2_norm_weight=self.use_l2_norm_conv2d,
            #     scale_value=1.0,
            # )

        elif name == "ROIEncoder":
            self.class_code_channel = (
                self.cfg.MODEL.META_LEARN.CODE_GENERATOR.HEAD.OUTPUT_DIM
            )
            k_s = 1
            padding_size = 0
            # conditional classification head (takes in weight and bias)
            self.cond_cls_logits = CondConvBlock(
                padding=padding_size, weight_len=self.class_code_channel
            )
        else:
            raise NotImplementedError(f"{name} is not implemented")

    # def _build_base_detector(self, *args, **kwargs):
    #     # Build all layers befor the predictor layer
    #     self._build_tower_module(head_configs, in_channels, norm)

    # def _build_code_generator(self, *args, **kwargs):
    #     raise NotImplementedError()

    # def _build_merge_module(self, *args, **kwargs):
    #     raise NotImplementedError()

    #  forward
    def forward_base_train(self, x, top_module=None, yield_bbox_towers=False):
        logits = []
        bbox_reg = []
        ctrness = []
        ious = []
        top_feats = []
        bbox_towers = []
        for level, feature in enumerate(x):
            feature = self.share_tower(feature)
            cls_tower = self.cls_tower(feature)
            bbox_tower = self.bbox_tower(feature)
            if yield_bbox_towers:
                bbox_towers.append(bbox_tower)
            # TODO: only support the first evaluation dataset and it has to be split "all"
            # slow now because every single time, it has to redo the loading.
            if not self.training and self.evaluate_with_base_class_codes:
                new_weight, new_bias = self._evaluate_with_base_class_codes(
                    self.cfg.DATASETS.TEST[0])
                if new_weight is not None:
                    logger.info(
                        f"evaluating using new weight and new bias, in total {new_weight.size(0)}")
                    logits.append(self.eval_conv(
                        feature=cls_tower, weight=new_weight, bias=new_bias))
                else:
                    logits.append(self.cls_logits(cls_tower))
            else:
                logits.append(self.cls_logits(cls_tower))
            reg = self.bbox_pred(bbox_tower)
            ctrness.append(self.ctrness(bbox_tower))
            ious.append(self.iou_overlap(bbox_tower))
            if self.scales is not None:
                reg = self.scales[level](reg)
            # Note that we use relu, as in the improved FCOS, instead of exp.
            bbox_reg.append(F.relu(reg))
            if top_module is not None:
                top_feats.append(top_module(bbox_tower))
        return logits, bbox_reg, ctrness, ious, top_feats, bbox_towers

    def forward(
        self,
        x,
        top_module=None,
        yield_bbox_towers=False,
        support_set_per_class_code=None,
    ):
        """
        support_set_per_class_code a tensor of dimension t*c, 256, 1, 1
        """
        if support_set_per_class_code is None:
            return self.forward_base_train(x, top_module, yield_bbox_towers)
        logits = []
        bbox_reg = []
        ctrness = []
        ious = []

        top_feats = []
        bbox_towers = []

        support_set_per_class_conv = support_set_per_class_code["cls_conv"]
        S = support_set_per_class_conv.size(0)
        bias = support_set_per_class_code["cls_bias"]
        box_weight = None
        # replace the base class codes
        #print(f"training stage: {self.training}, self.evaluate_with_base_class_codes: {self.evaluate_with_base_class_codes}")
        if (not self.training) and self.evaluate_with_base_class_codes:
            #print(f"replacing {self.training}")
            new_weights, new_bias = self._evaluate_with_base_class_codes(
                self.cfg.DATASETS.TEST[0])
            if new_weights is not None:
                print(
                    f"evaluating using new weight and new bias, in total {new_weights.size(0)}")
                support_set_per_class_conv = new_weights
                bias = new_bias

        if "loc_conv" in support_set_per_class_code:
            box_weight = support_set_per_class_code["loc_conv"]
            S, C, H, W = box_weight.size()
            # assert H * W == 4
            box_weight = box_weight.view(S, int(C // 4), 4, 1, 1)
            box_weight = box_weight.permute(0, 2, 1, 3, 4)  # S, 4, C, 1, 1

        for level, feature in enumerate(x):
            feature = self.share_tower(feature)
            cls_tower = self.cls_tower(feature)
            bbox_tower = self.bbox_tower(feature)
            if yield_bbox_towers:
                bbox_towers.append(bbox_tower)

            # logit for pretraining and meta-finetuning differs only at dimension 1, it has a size t*c
            # support_set_per_class_code CONV cls_tower
            if self.episodic_learning and support_set_per_class_code is not None:
                logit = self.cond_cls_logits(
                    cls_tower, support_set_per_class_conv, bias
                )
                # logit = self.score_scale(logit)
            else:
                assert self.cls_logits is not None, "cls_logits is None"
                logit = self.cls_logits(cls_tower)

            # location reg
            # if self.box_on and box_weight is not None:
            #     reg_all_class = []
            #     for i in range(S):
            #         reg = self.bbox_pred(bbox_tower, box_weight[i])
            #         if self.scales is not None:
            #             reg = self.scales[level](reg)
            #         reg = F.relu(reg)
            #         reg_all_class.append(reg)
            #     reg_all_class = torch.cat(reg_all_class, dim=1)
            #     # Note that we use relu, as in the improved FCOS, instead of exp.
            #     bbox_reg.append(reg_all_class)  # a list of dim b, c X4, h, w
            # else:
            reg = self.bbox_pred(bbox_tower)
            if self.scales is not None:
                reg = self.scales[level](reg)
            # Note that we use relu, as in the improved FCOS, instead of exp.
            bbox_reg.append(F.relu(reg))  # a list of dim b, 4, h, w

            logits.append(logit)
            ctrness.append(self.ctrness(bbox_tower))
            ious.append(self.iou_overlap(bbox_tower))
            if top_module is not None:
                top_feats.append(top_module(bbox_tower))
        return logits, bbox_reg, ctrness, ious, top_feats, bbox_towers

class FCOSHead(nn.Module):
    def __init__(self, cfg, input_shape: List[ShapeSpec]):
        """
        Arguments:
            in_channels (int): number of channels of the input feature
        """
        super().__init__()
        self.num_classes = cfg.MODEL.FCOS.NUM_CLASSES
        self.fpn_strides = cfg.MODEL.FCOS.FPN_STRIDES
        head_configs = {
            "cls": (cfg.MODEL.FCOS.NUM_CLS_CONVS, cfg.MODEL.FCOS.USE_DEFORMABLE),
            "bbox": (cfg.MODEL.FCOS.NUM_BOX_CONVS, cfg.MODEL.FCOS.USE_DEFORMABLE),
            "share": (cfg.MODEL.FCOS.NUM_SHARE_CONVS, False),
        }
        self.box_on = cfg.MODEL.META_LEARN.CODE_GENERATOR.BOX_ON
        logger.info(f"box_on: {self.box_on}")

        norm = None if cfg.MODEL.FCOS.NORM == "none" else cfg.MODEL.FCOS.NORM
        self.num_levels = len(input_shape)

        in_channels = [s.channels for s in input_shape]
        assert len(set(in_channels)
                   ) == 1, "Each level must have the same channel!"
        in_channels = in_channels[0]

        self.in_channels_to_top_module = in_channels

        # Build all layers befor the predictor layer
        self._build_tower_module(head_configs, in_channels, norm)
        # Join predictor head with support set, Only replace class right now, no bias right now
        self.episodic_learning = cfg.MODEL.META_LEARN.EPISODIC_LEARNING

        # The original classification head will be loaded in meta-learning stage
        self.cls_logits = nn.Conv2d(
            in_channels,
            self.num_classes,
            kernel_size=cfg.MODEL.FCOS.CLS_LOGITS_KERNEL_SIZE,
            stride=1,
            padding=cfg.MODEL.FCOS.CLS_LOGITS_KERNEL_SIZE // 2,
        )  # default is 3 x 3
        if self.episodic_learning:
            self.class_code_channel = cfg.MODEL.META_LEARN.CODE_GENERATOR.OUT_CHANNEL
            # self.cls_logits will not be updated in episodic_learning.
            for param in self.cls_logits.parameters():
                param.requires_grad = False
            k_s = cfg.MODEL.META_LEARN.CODE_GENERATOR.CLS_LAYER[2]
            padding_size = k_s // 2
            # conditional classification head
            self.cond_cls_logits = CondConvBlock(
                padding=padding_size, weight_len=self.class_code_channel
            )

        if self.episodic_learning and self.box_on:
            self.bbox_pred = CondConvBlock()
        else:
            self.bbox_pred = nn.Conv2d(
                in_channels, 4, kernel_size=3, stride=1, padding=1
            )

        self.ctrness = nn.Conv2d(
            in_channels, 1, kernel_size=3, stride=1, padding=1)

        if cfg.MODEL.FCOS.USE_SCALE:
            self.scales = nn.ModuleList(
                [Scale(init_value=1.0) for _ in range(self.num_levels)]
            )
        else:
            self.scales = None

        # Module initialization
        for modules in [
            self.cls_tower,
            self.bbox_tower,
            self.share_tower,
            self.cls_logits,
            self.bbox_pred,
            self.ctrness,
        ]:
            if modules is None:
                continue
            for layer in modules.modules():
                if isinstance(layer, nn.Conv2d):
                    torch.nn.init.normal_(layer.weight, std=0.01)
                    if layer.bias is not None:
                        torch.nn.init.constant_(layer.bias, 0)

                if cfg.MODEL.META_LEARN.CODE_GENERATOR.INIT_NORM_LAYER:
                    init_norm(layer)

        # initialize the bias for focal loss
        prior_prob = cfg.MODEL.FCOS.PRIOR_PROB
        bias_value = -math.log((1 - prior_prob) / prior_prob)
        torch.nn.init.constant_(self.cls_logits.bias, bias_value)

    def _build_tower_module(self, head_configs: Dict, in_channels: int, norm: str):
        for head, head_config in head_configs.items():
            num_convs, use_deformable = head_config
            tower = []
            for i in range(num_convs):
                if use_deformable and i == num_convs - 1:
                    conv_func = DFConv2d
                else:
                    conv_func = nn.Conv2d
                tower.append(
                    conv_func(
                        in_channels,
                        in_channels,
                        kernel_size=3,
                        stride=1,
                        padding=1,
                        bias=True,
                    )
                )
                if norm == "GN":
                    tower.append(nn.GroupNorm(32, in_channels))
                elif norm == "NaiveGN":
                    tower.append(NaiveGroupNorm(32, in_channels))
                elif norm == "BN":
                    tower.append(
                        ModuleListDial(
                            [
                                nn.BatchNorm2d(in_channels)
                                for _ in range(self.num_levels)
                            ]
                        )
                    )
                elif norm == "SyncBN":
                    tower.append(
                        ModuleListDial(
                            [
                                NaiveSyncBatchNorm(in_channels)
                                for _ in range(self.num_levels)
                            ]
                        )
                    )
                tower.append(nn.ReLU())

            self.add_module(f"{head}_tower", nn.Sequential(*tower))

    def forward_base_train(self, x, top_module=None, yield_bbox_towers=False):
        logits = []
        bbox_reg = []
        ctrness = []
        top_feats = []
        bbox_towers = []
        for level, feature in enumerate(x):
            feature = self.share_tower(feature)
            cls_tower = self.cls_tower(feature)
            bbox_tower = self.bbox_tower(feature)
            if yield_bbox_towers:
                bbox_towers.append(bbox_tower)

            logits.append(self.cls_logits(cls_tower))
            ctrness.append(self.ctrness(bbox_tower))
            reg = self.bbox_pred(bbox_tower)
            if self.scales is not None:
                reg = self.scales[level](reg)
            # Note that we use relu, as in the improved FCOS, instead of exp.
            bbox_reg.append(F.relu(reg))
            if top_module is not None:
                top_feats.append(top_module(bbox_tower))
        return logits, bbox_reg, ctrness, top_feats, bbox_towers

    def forward(
        self,
        x,
        top_module=None,
        yield_bbox_towers=False,
        support_set_per_class_code=None,
    ):
        """
        support_set_per_class_code a tensor of dimension t*c, 256, 1, 1
        """
        if support_set_per_class_code is None:
            return self.forward_base_train(x, top_module, yield_bbox_towers)
        logits = []
        bbox_reg = []
        ctrness = []
        top_feats = []
        bbox_towers = []

        support_set_per_class_conv = support_set_per_class_code["cls_conv"]
        S = support_set_per_class_conv.size(0)
        bias = support_set_per_class_code["cls_bias"]
        box_weight = None

        if "loc_conv" in support_set_per_class_code:
            box_weight = support_set_per_class_code["loc_conv"]
            S, C, H, W = box_weight.size()
            # assert H * W == 4
            box_weight = box_weight.view(S, int(C // 4), 4, 1, 1)
            box_weight = box_weight.permute(0, 2, 1, 3, 4)  # S, 4, C, 1, 1

        for level, feature in enumerate(x):
            feature = self.share_tower(feature)
            cls_tower = self.cls_tower(feature)
            bbox_tower = self.bbox_tower(feature)
            if yield_bbox_towers:
                bbox_towers.append(bbox_tower)

            # logit for pretraining and meta-finetuning differs only at dimension 1, it has a size t*c
            # support_set_per_class_code CONV cls_tower
            if self.episodic_learning and support_set_per_class_code is not None:
                logit = self.cond_cls_logits(
                    cls_tower, support_set_per_class_conv, bias
                )
                # logit = self.score_scale
            else:
                assert self.cls_logits is not None, "cls_logits is None"
                logit = self.cls_logits(cls_tower)

            # location reg
            if self.box_on and box_weight is not None:
                reg_all_class = []
                for i in range(S):
                    reg = self.bbox_pred(bbox_tower, box_weight[i])
                    if self.scales is not None:
                        reg = self.scales[level](reg)
                    reg = F.relu(reg)
                    reg_all_class.append(reg)
                reg_all_class = torch.cat(reg_all_class, dim=1)
                # Note that we use relu, as in the improved FCOS, instead of exp.
                bbox_reg.append(reg_all_class)  # a list of dim b, c X4, h, w
            else:
                reg = self.bbox_pred(bbox_tower)
                if self.scales is not None:
                    reg = self.scales[level](reg)
                # Note that we use relu, as in the improved FCOS, instead of exp.
                bbox_reg.append(F.relu(reg))  # a list of dim b, 4, h, w

            logits.append(logit)
            ctrness.append(self.ctrness(bbox_tower))
            if top_module is not None:
                top_feats.append(top_module(bbox_tower))
        return logits, bbox_reg, ctrness, top_feats, bbox_towers


class Sieve:
    def __init__(self, n: int, sieve_max=100_000_000):
        self.sieve_max = sieve_max
        self.values = self.sieve(n)

    def sieve(self, n: int) -> [int]:
        """takes n and returns a list of all primes from 2 to n using
        the sieve of Eratosthenes"""
        if n > self.sieve_max:
            raise ValueError(f"{n} exceeds max sieve value ({self.sieve_max}). To override, manually set Sieve's `sieve_max` attribute")

        print(f'setting search_max to {n}')
        self.search_max = n

        if n < 2:
            return []
        arr = [True] * (n+1)
        for i in range(2, n+1):
            if arr[i]:
                for j in range(i ** 2, n+1, i):
                    arr[j] = False

        return [i for i in range(2,n+1) if arr[i]]

    def is_prime(self, n: int) -> bool:
#         print(f'checking if {n} is prime')
        if n < 2:
            return False

        if n <= self.search_max:
            return n in self.values

        print(f"{n} exceeds search max ({self.search_max})")

        # Re-perform sieve and update self.values
        self.values = self.sieve(n)
        return n in self.values

    def prime_factors(self, n: int) -> [int]:
        print(f'looking for factors of {n}')

        if n < 2:  # no prime factors
            return []

        if n > self.search_max:
            print(f'expanding search max from {self.search_max}...')
            self.values = self.sieve(n)
            print(f'...done')

        for prime in self.values:
            if n % prime == 0:
                print(prime)
                return [prime] + self.prime_factors(int(n/prime))

        return []

# Example usage:
# --------------
MAX_VALUE = 1_000

sieve = Sieve(MAX_VALUE)

sieve.search_max  # 1000
sieve.values  # all the primes <= 1000, [2,3,5,...,983,991,997]

sieve.is_prime(999_999)  # False
sieve.values  # all the primes <= 999,999 [2,3,5,...,etc]

pf = sieve.prime_factors(231_510_111)  # ValueError: n exceeds max sieve value

sieve.sieve_max = 250_000_000  # produces about 12.7M primes, might take up to a minute or so.
pf = sieve.prime_factors(231_510_111)  # [3, 7, 23, 479317]

from functools import reduce
reduce(lambda x,y: x*y, pf)  # 231,510,111


'''
INFO 523 Final Project DBSCAN code
https://www.kaggle.com/code/rpsuraj/outlier-detection-techniques-simplified
'''

import pandas as pd
from sklearn.cluster import DBSCAN
import matplotlib.pyplot as plt

df = pd.read_csv("fetal_health.csv")
X = df[['abnormal_short_term_variability', 'baseline value']].values
db = DBSCAN(eps=3.0, min_samples=10).fit(X)

labels = db.labels_
#print(pd.Series(labels).value_counts())

plt.figure()
unique_labels = set(labels)
colors = ['blue', 'red']

for color, label in zip(colors, unique_labels):
    sample_mask = [True if l == label else False for l in labels]
    plt.plot(X[:,0][sample_mask], X[:, 1][sample_mask], 'o', color=color)

plt.xlabel('abnormal_short_term_variability')
plt.ylabel('baseline value')
plt.show()


import os
from google.cloud.devtools import cloudbuild_v1

client = cloudbuild_v1.services.cloud_build.CloudBuildClient()

def validate(event, context):
    """Triggered by a change to a Cloud Storage bucket.
    Args:
        event (dict): Event payload.
        context (google.cloud.functions.Context): Metadata for the event.
    """

    build = cloudbuild_v1.Build()

    core_org_id = os.environ.get('ORG_ID')
    project_id = os.environ.get('PROJECT_ID')
    region_name = os.environ.get('REGION')
    trigger_branch = os.environ.get('BRANCH')
    csr_repo_name = os.environ.get('REPO_NAME')
    asset_inventory_bucket_name = os.environ.get('ASSET_INVENTORY_GCS_BUCKET')
    guardrails_artifact_registry_name = os.environ.get('GUARDRAILS_ARTIFACT_REGISTRY_NAME')
    guardrails_report_bucket_name = os.environ.get('GUARDRAILS_REPORTS_GCS_BUCKET')
    guardrails_report_bucket_name_suffix = os.environ.get('GUARDRAILS_REPORTS_GCS_BUCKET_SUFFIX')
    guardrails_policies_container_name = os.environ.get('GUARDRAILS_POLICIES_CONTAINER_NAME')
    guardrails_policies_container_tag = os.environ.get('GUARDRAILS_POLICIES_CONTAINER_TAG')

    folder = event['name'].split('/')[0]
    filename = event['name'].split('/')[1]
    inventory_org_id = filename.split('.')[0]

    build.steps = [
        {
            "name": "gcr.io/cloud-builders/gcloud",
            "args": ["cp","gs://{}/{}".format(asset_inventory_bucket_name,event['name']),"/assets/asset_inventory.json"],
            "entrypoint": "gsutil"
        },
        {
            "name": "gcr.io/cloud-builders/docker",
            "args": ["-c","cp /assets/asset_inventory.json assets"],
            "entrypoint": "bash"
        },
        {
            "name": "gcr.io/cloud-builders/docker",
            "args": ["run","-v","/workspace/guardrails-validation:/app","{}-docker.pkg.dev/{}/{}/{}:{}".format(region_name,project_id,guardrails_artifact_registry_name,guardrails_policies_container_name,guardrails_policies_container_tag)],
        },
        {
            "name": "gcr.io/cloud-builders/docker",
            "args": ["-c","cat report.txt"],
            "entrypoint": "bash"
        },
        {
            "name": "gcr.io/cloud-builders/docker",
            "args": ["-c","cp report.txt /assets/{}".format(filename)],
            "entrypoint": "bash"
        },
        {
            "name": "gcr.io/cloud-builders/gcloud",
            "args": ["cp","/assets/{}".format(filename), "gs://{}/{}/{}.txt".format(guardrails_report_bucket_name,folder,inventory_org_id)],
            "entrypoint": "gsutil"
        }
    ]

    # Send to remote bucket if scanning remote organization
    if core_org_id.strip() != inventory_org_id.strip():
        remote_report_bucket_name = "{}-{}".format(inventory_org_id,guardrails_report_bucket_name_suffix)
        build.steps.append({
            "name": "gcr.io/cloud-builders/gcloud",
            "args": ["cp","/assets/{}".format(filename), "gs://{}/{}/{}.txt".format(remote_report_bucket_name,folder,inventory_org_id)],
            "entrypoint": "gsutil"
        })

    build.options = {
        "volumes": [
            {
                "name": "assets",
                "path": "/assets"
            }
        ]
    }

    build.source = {
        "repo_source":{
            "project_id": project_id,
            "repo_name": csr_repo_name,
            "branch_name": trigger_branch,
            "dir_": "guardrails-validation"
        }
    }

    result = client.create_build(project_id=project_id, build=build)
    return result


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score

# Función para estructurar los datos recibidos de txt a un dataframe
def structure_data(file):
    data = {"E": [], "J": []}

    with open(file) as f:
        for line in f.readlines():
            line_array = line.split()
            for i in range(len(line_array)):

                if i == 1:
                    data["J"].append(float(line_array[i]))

                if i == 0:
                    data["E"].append(float(line_array[i]))

    return pd.DataFrame(data)

# Función para analizar eléctricamente una determinada muestra, ajustando los datos al modelo
def sample_analysis(df):

    # Instanciar las variables de corte
    x_inicial = 0
    x_prerruptura = 10
    x_ruptura = 20
    x_final = len(df)

    # Variables de ajuste
    min_r2_exponential = 0.99
    min_r2_lineal = 0.80

    # Ajuste de curva lineal : Evaluación para determinar el inicio del punto de ruptura
    # El ajuste se explora fijando un punto en la izquierda y recorriendo a la derecha
    # la operación termina una vex que el score alcanza un valor adecuado (0.95)

    err_lineal = 100
    for i in range(len(df) - 1):
        # Generar el modelo lineal
        x_lineal = np.array(df[x_inicial: x_final - i]["E"]).reshape((-1, 1))
        y_lineal = np.array(df[x_inicial: x_final - i]["J"])
        lineal_reg = LinearRegression().fit(x_lineal, y_lineal)

        # Predecir la recta y evaluar el error recorriendo la curva a la derecha
        y_lineal_fit = lineal_reg.predict(x_lineal)
        score = "{0:.2f}".format(lineal_reg.score(x_lineal, y_lineal))
        lineal_coefficient = lineal_reg.coef_[0]
        lineal_intercept = lineal_reg.intercept_

        if float(score) >= min_r2_lineal:
            print(score)
            x_prerruptura = x_final - i
            break

    # Ajuste de curva exponencial : En este caso, el ajuste de curva se realiza en el sentido contrario,
    # Fijando el punto al final de la gráfica y recortando los datos de la izquierda hasta llegar a una
    # evaluación adecuada (0.90). Es necesario establecer que no se considera el efecto joule

    err_exponecial = 100
    for i in range(x_prerruptura + 1, len(df)):
        # Calcular el valor de alfa
        alfa_array = []
        for j in range(i, len(df) - 1):
            try:
                alfa = ((np.log(df["J"][j + 1]) - np.log(df["J"][j])) / (np.log(df["E"][j + 1]) - np.log(df["E"][j])))
                alfa_array.append(alfa)
            except:
                print("Error in alfa calculation")

        x_exponential = df[i:x_final]["E"]
        y_exponential = df[i:x_final]["J"]

        log_y = np.log(y_exponential)

        fit = np.polyfit(x_exponential, log_y, 1)
        y_exponential_fit = np.exp(fit[1]) * np.exp(fit[0]*x_exponential)

        score = r2_score(y_exponential, y_exponential_fit)

        if score > min_r2_exponential:
            x_ruptura = i
            print("hola" + str(np.exp(fit[1]))+ " " + str(fit[0]))

            print("MEDIA DE ALFA:", np.mean(alfa_array))
            print("DISTRIBUCIÓN ESTÁNDAR DE ALFA:", np.std(alfa_array))
            print("MÍNIMO DE ALFA:", min(alfa_array))
            print("MÁXIMO DE ALFA:", max(alfa_array))
            print("- - - - - - - - - - - - - - - - - - -")

            break

    # Impresión de las zonas
    print("ZONA ÓHMICA: ", str(x_inicial), ": ", str(x_prerruptura))
    print("ZONA DE TRANSICIÓN: ", str(x_prerruptura + 1), ": ", str(x_ruptura))
    print("ZONA NO-ÓHMICA: ", str(x_ruptura + 1), ": ", str(x_final))
    print("- - - - - - - - - - - - - - - - - - -")

    # Graficar los ajustes del modelo
    plt.scatter(df["E"], df["J"], c="r")
    # Definición de las ecuaciones de ajuste
    try:
        E_entre_R = np.power(y_exponential_fit, (1/np.mean(alfa_array)))   # sentido físico de la exponencial
        physical_exponential_fit = np.array(E_entre_R)**np.mean(alfa_array)
        plt.plot(x_exponential, physical_exponential_fit, c="b",
        label="Exponential Fit2:" + "(V/R)^" + str("{0:.2f}".format(np.mean(alfa_array))))
    except:
        print("NO HAY COMPORTAMIENTO EXPONENCIAL")

    try:
        plt.plot(x_lineal, y_lineal_fit, c="b", label="Lineal Fit: " + str("{0:.2f}".format(lineal_coefficient)) + "x"
                                                 + " + " + str("{0:.2f}".format(lineal_intercept)))
        print(lineal_intercept)
    except:
        print("NO HAY COMPORTAMIENTO LINEAL")
    plt.grid()
    plt.legend()
    plt.title("CAMPO ELÉCTRICO CONTRA DENSIDAD DE CORRIENTE")
    plt.xlabel("CAMPO ELÉCTRICO (V/cm)")
    plt.ylabel("DENSIDAD DE CORRIENTE (A/cm^2)")
    plt.show()

# Función para graficar la data que se recibe
def graph_data(df):
    plt.plot(df["E"], df["J"])
    plt.scatter(df["E"], df["J"], c="r")
    plt.grid()
    plt.title("CAMPO ELÉCTRICO CONTRA DENSIDAD DE CORRIENTE")
    plt.xlabel("CAMPO ELÉCTRICO (V/cm)")
    plt.ylabel("DENSIDAD DE CORRIENTE (A/cm^2)")
    plt.show()

file0 = 'Pruebas_electricas_1.txt'
file1 = 'Prueba1Eq1.txt'
file2 = 'Prueba2Eq1.txt'
file3 = 'Prueba3Eq1.txt'
file4 = 'Prueba4Eq1.txt'

df = structure_data(file0)
sample_analysis(df)


import torch
import torch.nn as nn
from torchsummary import summary
from dataclasses import asdict
from typing import Optional

from .configs.dqn import DQNConfig
from .utils.layer_params import LinearParams, ConvParams

def conv(params: ConvParams, pool: bool = True) -> nn.Module:
    layers = [
        nn.Conv2d(**asdict(params)),
        nn.ReLU(inplace=True),
        nn.BatchNorm2d(params.out_channels),
    ]
    if pool is True:
        layers.append(nn.MaxPool2d(2, 2))
    return nn.Sequential(*layers)

def linear(params: LinearParams, dropout: Optional[float] = None) -> nn.Module:
    layers = [nn.Linear(**asdict(params))]
    if dropout is not None:
        layers.append(nn.Dropout(dropout))
    layers.append(nn.ReLU(inplace=True))
    return nn.Sequential(*layers)

class DQN(nn.Module):
    def __init__(self, config: DQNConfig) -> None:
        super().__init__()
        num_features = self.__get_num_features(config.width, config.height)
        self.network = nn.Sequential(
            conv(ConvParams(config.in_channels, 8)),
            conv(ConvParams(8, 16)),
            conv(ConvParams(16, 32)),
            nn.Flatten(),
            linear(LinearParams(num_features, 64), dropout=0.5),
            linear(LinearParams(64, config.out_features)),
        )
        self.config = config

    def __get_num_features(self, width: int, height: int) -> int:
        num_channles, reduce_by = 32, 8
        width = width // reduce_by
        height = height // reduce_by
        return int(width * height * num_channles)

    def forward(self, state: torch.Tensor) -> torch.Tensor:
        return self.network(state)

if __name__ == "__main__":
    shape = in_channels, width, height = 4, 24, 32
    model = DQN(width, height, in_channels)
    summary(model, shape, device="cpu")


# ---
# Created by aitirga at 09/10/2019
# Description: This module contains the gradient descent class
# ---
import os
import time

import matplotlib.pyplot as plt
import numpy as np

from ann_solver.ann_core import ANN
from ann_solver.constants import *

class GradientDescent(ANN):
    def initialize(self, alpha=1e-4, ATOL=ATOL, RTOL=RTOL, always_decrease=False,
                   n=100, plotting=True, n_plot=False, NMAX=NMAX, lambda0=0.0, input_set_type="full",
                   momentum_g=0.8, norm=False, n_batch=25, plot_contour=False, n_contour=(100, 100),
                   momentum=False, avoid_cf=False):
        """

        :param always_decrease:
        :param norm:
        :param n_batch:
        :param alpha: initial learning rate
        :param ATOL: absolute tolerance, if the change in cost function between consecutive steps is less than ATOL,
        the algorithm converges
        :param RTOL: relative tolerance, if the change in costa function compared to the initial one is less than RTOL,
        the algorithm converges
        not do that, an error is raised
        :param n: parameter that controls the number of steps before updating the plots
        :param plotting:
        :param n_plot:
        :param NMAX: maximum number of timesteps, if the algorithm performs more timesteps, it raises an error
        :param lambda0: regularization parameter
        :param input_set_type: sets the criteria to use the input file in the calculation of the gradient descent:
        "full" to use the whole dataset, "batch" to do batches of n_batch elements and "stochastic" to use one
        input every step
        :param momentum_g: sets the momentum constant
        :param plot_contour: Set it True to dinamically plot a contour plot of the results
        :param n_contour: number of elements on the contour plot
        :param momentum: Set it True to use momentum and "Nesterov" to use the Nesterov momentum
        :param avoid_cf: Set it True to only compute the cost function at the plotting timesteps set by n
        :return:
        """
        # Perform gradient descent minimization algorithm using the provided learning rate.
        # It uses a fixed number of learning steps
        # Parameters for live plotting
        self.input_set_type = input_set_type
        self.n_batch = n_batch
        self.plotting = plotting
        self.time_start = time.time()
        self.time_0 = self.time_start
        self.lambda0 = lambda0  # Regularization term
        self.gamma = momentum_g  # The momentum term
        self.alpha = alpha  # Step size
        self.alpha0 = alpha  # Initial step size (default)
        self.ATOL = ATOL
        self.RTOL = RTOL
        self.NMAX = NMAX
        self.momentum = momentum
        self.tol["AlwaysDecrease"]["Value"] = always_decrease
        self.x_backup = self.x
        self.avoid_CF = avoid_cf
        self.plotting = plotting
        self.plot_contour = plot_contour
        self.n_contour = n_contour
        self.n = n

        if not n_plot:
            self.n_plot = self.n
        else:
            self.n_plot = n_plot
        self.Time = []
        self.CF = []

        # Automatic tolerance stop (ATOL)
        self.nsim = 0
        # Velocities matrix for the momentum implementation

        if norm:  # Specifies if the input X vector should be normalized
            if not self.normalized:
                ANN.normalize_x_stdmean(self)

    def run_gradient_descent(self):
        line1 = []
        io.print_gradient_descent_intro(self)
        self.n_plot_list = []
        self.CF_plot_list = []
        # First timestep cf calculation
        temp_CF = ANN.cost_function(self)
        self.n_plot_list.append(self.nsim)
        self.CF_plot_list.append(temp_CF)
        line1 = ANN.plot_CF(self, line1)
        contour = []
        while 1:
            if self.adaptive_learning_rate:
                self.alpha = GradientDescent.adaptive_learning_rate(self)
            GradientDescent.compute_gradient_and_update_weights(self)
            # ANN.update_weights(self)
            if GradientDescent.evaluate_tolerances(self, self.ATOL, self.RTOL, self.NMAX):
                break
            if self.avoid_CF:
                pass
            else:
                temp_CF = ANN.cost_function(self)
                self.CF.append(temp_CF)
            GradientDescent.evaluate_tolerances(self, self.ATOL, self.RTOL, self.NMAX)
            if float(self.nsim) % self.n == 0.0:
                if self.avoid_CF:
                    temp_CF = ANN.cost_function(self)
                    self.CF.append(temp_CF)
                print("Learning step %s, cost function value: %s" % (self.nsim, temp_CF))
                ANN.print_tolerances(self)
                if self.plot_contour:
                    contour = ANN.live_plotter_contour(self, contour, self.n_contour)
                # ANN.check_gradient(self)
            self.nsim += 1
            if self.plotting:
                if float(self.nsim) % self.n == 0:
                    self.n_plot_list.append(self.nsim)
                    self.CF_plot_list.append(temp_CF)
                    line1 = ANN.plot_CF(self, line1)
        io.sim_status(self)
        plt.ioff()

    def adaptive_learning_rate(self):
        return self.dict_learning_rate[self.adaptive_learning_rate_type](self, **self.adaptive_learning_rate_args)

    def compute_gradient_and_update_weights(self):
        if not self.momentum == "nesterov":
            ANN.compute_gradient(self)
        if self.momentum == "nesterov":
            self.Theta += np.multiply(self.alpha, self.v_momentum)
            ANN.compute_gradient(self)
        if self.momentum == True or self.momentum == "yes":
            self.v_momentum = np.multiply(self.gamma, self.v_momentum) - np.multiply(self.alpha, self.D)
            self.Theta += self.v_momentum
        if self.momentum == "nesterov":
            self.v_momentum = np.multiply(self.gamma, self.v_momentum) - np.multiply(self.alpha, self.D)
            self.Theta += self.v_momentum
        if not self.momentum:
            self.Theta -= np.multiply(self.alpha, self.D)

    def evaluate_tolerances(self, ATOL=1E-2, RTOL=1E-4, NMAX=100000, AlwaysDecrease=False):
        if len(self.CF) > 1:
            self.tol["ATOL"]["Value"] = abs(self.CF[-2] - self.CF[-1])
            self.tol["RTOL"]["Value"] = abs(self.CF[-2] - self.CF[-1]) / (abs(self.CF[-1] - self.CF[0] + self.eps))
            self.tol["NMAX"]["Value"] = self.nsim
            if (self.tol["AlwaysDecrease"]["Value"] == 1.0) and (self.CF[-1] - self.CF[-2] > 0.0):
                self.tol["AlwaysDecrease"]["Status"] = True
            if self.tol["ATOL"]["Value"] < ATOL:
                self.tol["ATOL"]["Status"] = True
            if self.tol["RTOL"]["Value"] < RTOL:
                self.tol["RTOL"]["Status"] = True
            if self.tol["NMAX"]["Value"] > NMAX:
                self.tol["NMAX"]["Status"] = True
        for i in self.tol:
            if self.tol[i]["Status"]:
                return True

class io(GradientDescent):
    """Input/output control class for GradientDescent
    This class contains the functions to print output data regarding the evolution of the GradientDescent algorithm
    """

    def sim_status(self):
        reason = {}
        for i in self.tol:
            if self.tol[i]["Status"]:
                reason[i] = self.tol[i]
        print("************ Training finished ************\n")
        print("*******************************************")
        print("********** ANN training status ************")
        print("*******************************************")
        print("The training of the neural network has finished")
        print("Total training time: %s s" % (time.time() - self.time_start))
        print("Simulation steps: %s" % self.nsim)
        print("Value of the minimized Cost Function %s" % self.CF[-1])
        print("Reason for termination: %s" % reason)
        print("****************** END ********************\n")
        os.system('pause')

    def print_gradient_descent_intro(self):
        print("*******************************************")
        print("**** Gradient descent training utility ****")
        print("*******************************************")
        print("Gradient descent utility properties:")
        print("\tLearning rate: %s" % self.alpha)
        if self.plotting:
            print("\tPlotting CF evolution")
        print("Regularization term: %s" % self.lambda0)
        print("Input set for gradient calculation: %s" % self.input_set_type)
        if self.input_set_type == "batch":
            print("Batch elements: %s" % self.n_batch)
        print("Gradient acceleration: %s" % self.momentum)
        print("\tTolerances:")
        print("\t\tATOL: %s" % self.ATOL)
        print("\t\tRTOL: %s" % self.RTOL)
        print("\t\tNMAX: %s" % self.NMAX)
        print("************ Training started *************")


"""
Authors: Bulelani Nkosi, Caryn Pialat
Main api to run chatbot
"""

# Imports

import os
import logging
from flask import Flask
from slack import WebClient
from slackeventsapi import SlackEventAdapter
from controllers.templates import QuestionAnswering

# Initialize a Flask app to host the events adapter
app = Flask(__name__)
slack_events_adapter = SlackEventAdapter(os.environ['SLACK_SIGNING_SECRET'], "/slack/events", app)
# slack_events_adapter.on()
# Initialize a Web API client
slack_web_client = WebClient(token=os.environ['SLACK_BOT_TOKEN'])
# Keep track of past messages
messages_received = {}

def answer_question(user_id: str, channel: str, question: str):
    """
    Function to answer questions
    """
    # Search for the answer
    question_answer = QuestionAnswering(channel, question)
    # Build answer payload
    answer = question_answer.get_answer_payload()

    # Post answer in slack
    response = slack_web_client.chat_postMessage(**answer)

    # Capture the timestamp of the message we've just posted so
    # we can use it to update the message after a user
    # has completed a task.
    question_answer.timestamp = response["ts"]

@slack_events_adapter.on("message")
def handle_message(payload):
    event = payload.get("event", {})
    text = event.get("text")
    print("++++++++++++++++++++++")
    print(event)
    # If the incoming message contains "hi", then respond with a "Hello" message
    if event.get("subtype") is None and "hi" == text:
        channel = event.get("channel")
        message = "Hello <@%s>! :tada:" % event.get("user")
        slack_web_client.chat_postMessage(channel=channel, text=message)

    elif event.get("subtype") is None and text.endswith("?"):
        event = payload.get("event", {})
        channel = event.get("channel")
        user = event.get("user")
        return answer_question(user, channel, text)

if __name__ == "__main__":
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    logger.addHandler(logging.StreamHandler())
    app.run(port=3000)


import csv
import requests
from bs4 import BeautifulSoup
import write_db as wdb

def check_in_file(file_csv,line_csv):
    open(f'{file_csv}','a')
    with open(f'{file_csv}',encoding='utf-16', newline="") as file:
        reader = csv.reader(file)
        for line in reader:
            if line == line_csv:
                return False
        return True

def get_quotes(soup):
    quotes = soup.find_all('span', class_='text')
    authors = soup.find_all('small', class_='author')
    for i in range(len(quotes)):
        quote = [quotes[i].text,authors[i].text]
        if check_in_file('quotes.csv',quote):
            with open('quotes.csv', "a",encoding='utf-16', newline="") as file:
                writer = csv.writer(file)
                writer.writerow(quote)

def get_authors(soup):
    authors = soup.find_all('small', class_='author')
    for author in authors:
        author = [author.text]
        if check_in_file('authors.csv',author):
            with open('authors.csv', "a",encoding='utf-16', newline="") as file:
                writer = csv.writer(file)
                writer.writerow(author)

def get_about_author(param):
    url = f'https://quotes.toscrape.com/{param}'
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'lxml')
    names = soup.find('h3', class_='author-title')
    born = soup.find('span', class_='author-born-date')
    born_loc = soup.find('span', class_='author-born-location')
    description = soup.find('div', class_='author-description')
    about_auth = [names.get_text().strip(), born.get_text(), born_loc.get_text(), description.get_text().replace("\n","")]
    if check_in_file('about_authors.csv',about_auth):
        with open('about_authors.csv','a',encoding='utf-16',newline="") as file:
            writer = csv.writer(file)
            writer.writerow(about_auth)

def href_author(soup):
    name_author = soup.find_all('small', class_='author')
    if check_in_file('authors.csv',name_author):
        for link in soup.find_all('a', href = True, text = '(about)'):
            get_about_author(link.get('href'))

def main():
    param = 'page/1/'
    while param != None:
        url = f'https://quotes.toscrape.com/{param}'
        response = requests.get(url)
        soup = BeautifulSoup(response.text, 'lxml')
        get_quotes(soup)
        get_authors(soup)
        href_author(soup)
        try:
            page = soup.find('li', class_='next')
            for i in page.find_all('a'):
                page = i['href']
        except:
            break
        param = page

main()
wdb.main()


#%%
import gspread

import pandas as pd
import re

gc = gspread.service_account()
# %%
worksheet = gc.open('Total partidas').sheet1

rows = worksheet.get_all_values()

data = pd.DataFrame.from_records(rows)
data.columns = data.iloc[0]
data.drop(0, inplace=True)
display(data.info())
display(data.describe())

#%%


# -----------------------------------------------------------
# Creating a candlestick chart of stock-market data using python.
#
# (C) 2020 Sandra VS Nair, Trivandrum
# email sandravsnair@gmail.com
# -----------------------------------------------------------

from pandas_datareader import data
from bokeh.models.annotations import Title
from bokeh.plotting import figure,show,output_file
import datetime

#The time period for which you need the data.
starttime=datetime.datetime(2018,5,2)
endttime=datetime.datetime(2018,5,20)

#pandas_datareader imports data from a number of online sources.
#Parameters: Stock symbol, Data source, Start time and End time.
stock_data=data.DataReader(name="AAPL", data_source="yahoo", start=starttime, end=endttime)

#Setting properties of the plot.
plot=figure(x_axis_type='datetime',width=1000,height=300)
t = Title()
t.text = 'Candlestick chart'
plot.title = t
plot.grid.grid_line_alpha=0.3

#Twelve horus in milliseconds.
twelve_hours=12*60*60*1000

#Updating status of the stock.
def status_update(open_price,close_price):
    if close_price > open_price:
        value="Increase"
    elif open_price > close_price:
        value="Decrease"
    else:
        value="Equal"
    return value

#Finding mean value of opening price and closing price.
def middle_value(open_price,close_price):
    value=(open_price+close_price)/2
    return value

#Finding the difference between opening price and closing price.
def height(open_price,close_price):
    value=abs(close_price-open_price)
    return value

#Adding three more columns to the table.
stock_data["Status"]=[status_update(open_price,close_price) for open_price,close_price in zip(stock_data.Open,stock_data.Close)]
stock_data["Middle"]=[middle_value(open_price,close_price) for open_price,close_price in zip(stock_data.Open,stock_data.Close)]
stock_data["Height"]=[height(open_price,close_price) for open_price,close_price in zip(stock_data.Open,stock_data.Close)]

#Adding rectangles and segments to the plot

#Four mandatory parameters
#1. x-coordinate of central point of rectangles
#2. y-coordinate of central point of rectangles
#3. Width
#4. Height

plot.rect(stock_data.index[stock_data.Status=="Increase"], stock_data.Middle[stock_data.Status=="Increase"], \
          twelve_hours, stock_data.Height[stock_data.Status=="Increase"],fill_color="green",line_color="black")
plot.rect(stock_data.index[stock_data.Status=="Decrease"], stock_data.Middle[stock_data.Status=="Decrease"], \
          twelve_hours,stock_data.Height[stock_data.Status=="Decrease"],fill_color="red",line_color="black")

#Four mandatory parameters
#1. Highest x-coordinate.
#2. Highest y-coordinate.
#3. Lowest x-coordinate.
#4. Lowest y-coordinate.

plot.segment(stock_data.index,stock_data.High,stock_data.index,stock_data.Low,color="black")

#Saving the plot.
output_file("Candlestick.html")

#Displaying the plot.
show(plot)


import numpy as np
import matplotlib.pyplot as plt
import time

def integrate(f, a, b, N):
    """ Uses pure python to integrate function using Riemann sum

        Example usage:
        integrate(f=lambda x: x**2 , a=0, b=1, N=6000000)
    """
    y = np.zeros(1)
    if isinstance(f(y), (int, float)): #Checking if the function is constant
        return(f(y))
    else:
        x = np.linspace(a, b, N + 1)
        _sum = 0
        dx = float(b - a)/N
        for xi in x:
            _sum += f(xi)*dx
        return _sum

def midpoint_integrate(f, a, b, N):
    """ Uses pure python to integrate function using midpoint method

        Example usage:
        midpoint_integrate(f=lambda x: x**2 , a=0, b=1, N=6000000)
    """
    y = np.zeros(1)
    if isinstance(f(y), (int, float)):
        return(f(y))

    else:
        _sum = 0
        dx = float(b-a)/N
        dx_midpoint = dx*0.5
        x = np.linspace(a, b, N + 1) - dx_midpoint #Creating an array with the midpoints
        x[0] = 0 #Replace first element with zero
        for xi in x:
            _sum += f(xi)*dx
        return _sum

def plot_error(f = lambda x: x**2, a=0, b=1, Nmax=500, Nmin=10, stepsize=10, plot = True):
    "Plots the error of a x^2 compared to the real answer which is 1/3 "
    error = []
    mesh = []

    for N in range(Nmin, Nmax + 1, stepsize):
        mesh.append(N)
        error.append(abs(1./3 - integrate(f, a, b, N)))
    if plot:
        fig = plt.figure(figsize=[7,5])
        ax = plt.subplot(111)
        ax.plot(mesh, error)
        ax.set_xlabel('Number of mesh points (N)')
        ax.set_ylabel('Error')
        ax.set_title("Error for nummerical integral of f(x) = x^2")
        fig.savefig("quadratic_error.png")
        plt.show()
if __name__ == "__main__":
    for i in range(4):

        t_start = time.clock()
        integrate(f = lambda x: x**2, a=0, b=1, N = 10000)
        t_stop = time.clock()
        print("Runtime for pure python integral: {}".format(t_stop - t_start))
        # t_start = time.clock()
        # midpoint_integrate(f = lambda x: x**2, a=0, b=1, N = 2000)
        # t_stop = time.clock()
        # print("Runtime for pure python midpoint integral: {}".format(t_stop - t_start))
    #plot_error()


from django import template

register = template.Library()

@register.filter(nome="remover_texto")
def remover(texto, r):
    return texto.replace(r, "")

@register.filter(name="verificardddpr")
def verificardddpr(telefone):
    ddd = telefone[0:4]
    if ddd == "(44)":
        return True
    else:
        return False

@register.filter(name="verificaruser")
def verificausuario(usuario, nome_do_grupo):
    if usuario.groups.filter(name=nome_do_grupo):
        return True
    else:
        return False

@register.filter(name="verificaplaca")
def verificaplaca(usuario, nome_do_grupo):
    if usuario.groups.filter(name=nome_do_grupo):
        return True
    else:
        return False


import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import VarianceThreshold
from sklearn.metrics import r2_score, mean_squared_error
from rdkit import Chem
from rdkit.Chem import AllChem
import dill
import pymc3 as pm
import arviz as az

## Helper Functions

def variance_threshold_selector(data, threshold=0.00):
	selector = VarianceThreshold(threshold)
	selector.fit(data)
	return data[data.columns[selector.get_support(indices=True)]]

def get_ci_error(model, x, y, pred, error):
    x_dist = model.μ.distribution.predict(np.array(x))
    x_reshape = x_dist.T
    ci = list(az.hdi(np.array(x), hxdi_prob=0.89) for x in x_reshape)
    ci_flat =np.stack(ci)
    ci_errors = (np.abs(ci_flat - pred.reshape(len(pred),1)))

    error_lim = np.log10(10**y+np.array(error).reshape(1,-1))-y

    return x_dist, ci, ci_errors, error_lim

## Load in data

data = pd.read_csv("../data/raw/data.csv")
train = np.where(data.label == "train")[0]
test = np.where(data.label == "test")[0]
val = np.where(data.label == "val")[0]

smiles_full = data.SMILES
mols_full = [Chem.MolFromSmiles(x) for x in smiles_full]
g = data.g
gdbl = data.gdbl

gtrain_error = data.err_g[train]
gtest_error = data.err_g[test]
gval_error = data.err_g[val]
gdbltrain_error = data.err_gdbl[train]
gdbltest_error = data.err_gdbl[test]
gdblval_error = data.err_gdbl[val]

## Get Descriptors

df = pd.DataFrame()
df["num_rings"] = [Chem.rdMolDescriptors.CalcNumRings(x) for x in mols_full]

fcfp = []
for mol in mols_full:
    ar = np.zeros((1,))
    fp = AllChem.GetMorganFingerprintAsBitVect(mol, 2, 2048, useFeatures=True)
    Chem.DataStructs.ConvertToNumpyArray(fp,ar)
    fcfp.append(ar)
fcfp = pd.DataFrame(np.stack(fcfp), columns=["FCFP_"+str(i) for i in range(2048)])
ecfp = []
for mol in mols_full:
    ar = np.zeros((1,))
    fp = AllChem.GetMorganFingerprintAsBitVect(mol, 2, 2048, useFeatures=False)
    Chem.DataStructs.ConvertToNumpyArray(fp,ar)
    ecfp.append(ar)
ecfp = pd.DataFrame(np.stack(ecfp), columns=["ECFP_"+str(i) for i in range(2048)])
df = pd.concat([df,fcfp,ecfp],axis=1)

for col in df.columns:
    df[col] = df[col]*data.conc

## Split into predetermined train, test and validation sets

xtrain = df.iloc[train]
xtest = df.iloc[test].reset_index(drop=True)
xval = df.iloc[val].reset_index(drop=True)

gtrain = g.iloc[train]
gtest = g.iloc[test]
gval = g.iloc[val]

gdbltrain = gdbl.iloc[train]
gdbltest = gdbl.iloc[test]
gdblval = gdbl.iloc[val]

## Remove correlated and zero variance descriptors

xtrain = variance_threshold_selector(xtrain)
corr_matrix = xtrain.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.9)]+["FCFP_0"] ## Added FCFP_0 due to being set in all molecules
xtrain = xtrain.drop(xtrain[to_drop], axis=1)
xtest = xtest[xtrain.columns]
xval = xval[xtrain.columns]

scaler = StandardScaler()
xtrain = pd.DataFrame(scaler.fit_transform(xtrain), columns = xtrain.columns)
xtest = pd.DataFrame(scaler.transform(xtest), columns = xtrain.columns)
xval = pd.DataFrame(scaler.transform(xval), columns = xtrain.columns)

with open("../models/scaler.pkl", "wb") as f:
    dill.dump(scaler, f)

## Save split data

xtrain.to_csv("../data/split/xtrain.csv", index=False)
xtest.to_csv("../data/split/xtest.csv", index=False)
xval.to_csv("../data/split/xval.csv", index=False)

gtrain.to_csv("../data/split/gtrain.csv", index=False)
gtest.to_csv("../data/split/gtest.csv", index=False)
gval.to_csv("../data/split/gval.csv", index=False)

gdbltrain.to_csv("../data/split/gdbltrain.csv", index=False)
gdbltest.to_csv("../data/split/gdbltest.csv", index=False)
gdblval.to_csv("../data/split/gdblval.csv", index=False)

## BART requires y data as np array

gtrain = np.ravel(gtrain)
gtest = np.ravel(gtest)
gval = np.ravel(gval)

gdbltrain = np.ravel(gdbltrain)
gdbltest = np.ravel(gdbltest)
gdblval = np.ravel(gdblval)

## Build and save models

models = []
traces = []
for y,m,alpha in [(gtrain,10,0.9), (gdbltrain,25,0.8)]:
    with pm.Model() as model:
        σ = pm.HalfNormal('σ', 1)
        μ = pm.BART('μ', xtrain, np.array(y), m=m, alpha=alpha)
        y = pm.Normal('y', μ, σ, observed=np.array(y))
        trace = pm.sample(2000, cores=1, tune=1000, return_inferencedata=False, random_seed = 42)

    models.append(model)
    traces.append(trace)

model_g = models[0]
trace_g = traces[0]
model_gdbl = models[1]
trace_gdbl = traces[1]

with open("../models/g.pkl","wb") as f:
    dill.dump(model_g, f)

with open("../models/gdbl.pkl","wb") as f:
    dill.dump(model_gdbl, f)

with open("../models/g_trace.pkl","wb") as f:
    dill.dump(trace_g, f)

with open("../models/gdbl_trace.pkl","wb") as f:
    dill.dump(trace_gdbl, f)

## Get Predictions and produce report

gtrain_pred = model_g.µ.distribution.predict(np.array(xtrain)).mean(axis=0)
gdbltrain_pred = model_gdbl.µ.distribution.predict(np.array(xtrain)).mean(axis=0)
gtest_pred = model_g.µ.distribution.predict(np.array(xtest)).mean(axis=0)
gdbltest_pred = model_gdbl.µ.distribution.predict(np.array(xtest)).mean(axis=0)
gval_pred = model_g.µ.distribution.predict(np.array(xval)).mean(axis=0)
gdblval_pred = model_gdbl.µ.distribution.predict(np.array(xval)).mean(axis=0)

gtrain_pred_res = np.abs(gtrain_pred - gtrain)
gdbltrain_pred_res = np.abs(gdbltrain_pred - gdbltrain)
gtest_pred_res = np.abs(gtest_pred - gtest)
gdbltest_pred_res = np.abs(gdbltest_pred - gdbltest)
gval_pred_res = np.abs(gval_pred - gval)
gdblval_pred_res = np.abs(gdblval_pred - gdblval)

gtrain_dist, gtrain_ci, gtrain_ci_errors, gtrain_exp_errors = get_ci_error(model_g, xtrain, gtrain, gtrain_pred, gtrain_error)
gtest_dist, gtest_ci, gtest_ci_errors, gtest_exp_errors = get_ci_error(model_g, xtest, gtest, gtest_pred, gtest_error)
gval_dist, gval_ci, gval_ci_errors, gval_exp_errors = get_ci_error(model_g, xval, gval, gval_pred, gval_error)
gdbltrain_dist, gdbltrain_ci, gdbltrain_ci_errors, gdbltrain_exp_errors = get_ci_error(model_gdbl, xtrain, gdbltrain, gdbltrain_pred, gdbltrain_error)
gdbltest_dist, gdbltest_ci, gdbltest_ci_errors, gdbltest_exp_errors = get_ci_error(model_gdbl, xtest, gdbltest, gdbltest_pred, gdbltest_error)
gdblval_dist, gdblval_ci, gdblval_ci_errors, gdblval_exp_errors = get_ci_error(model_gdbl, xval, gdblval, gdblval_pred, gdblval_error)

ci_errors_fullg = np.concatenate([gtrain_ci_errors, gtest_ci_errors, gval_ci_errors])
ci_errors_1g = []
ci_errors_2g = []
for i in range(len(ci_errors_fullg)):
    ci_errors_1g.append(ci_errors_fullg[i][0])
    ci_errors_2g.append(ci_errors_fullg[i][1])

ci_errors_fullgdbl = np.concatenate([gdbltrain_ci_errors, gdbltest_ci_errors, gdblval_ci_errors])
ci_errors_1gdbl = []
ci_errors_2gdbl = []
for i in range(len(ci_errors_fullgdbl)):
    ci_errors_1gdbl.append(ci_errors_fullgdbl[i][0])
    ci_errors_2gdbl.append(ci_errors_fullgdbl[i][1])

ci_errors_1g = pd.Series(ci_errors_1g, name="ci1_g")
ci_errors_2g = pd.Series(ci_errors_2g, name="ci2_g")
ci_errors_1gdbl = pd.Series(ci_errors_1g, name="ci1_gdbl")
ci_errors_2gdbl = pd.Series(ci_errors_2g, name="ci2_gdbl")

all_errg = pd.Series(list(gtrain_exp_errors[0]) + list(gtest_exp_errors[0]) + list(gval_exp_errors[0]), name = "err_g")
all_errgdbl = pd.Series(list(gdbltrain_exp_errors[0]) + list(gdbltest_exp_errors[0]) + list(gdblval_exp_errors[0]), name = "err_gdbl")

train_results = pd.DataFrame([gtrain, gtrain_pred, gtrain_pred_res, gdbltrain, gdbltrain_pred, gdbltrain_pred_res], index=["g", "g_pred", "g_pred_res", "gdbl", "gdbl_pred", "gdbl_pred_res"]).T
train_results["label"] = ["train" for x in range(len(train_results))]
test_results = pd.DataFrame([gtest, gtest_pred, gtest_pred_res, gdbltest, gdbltest_pred, gdbltest_pred_res], index=["g", "g_pred", "g_pred_res", "gdbl", "gdbl_pred", "gdbl_pred_res"]).T
test_results["label"] = ["test" for x in range(len(test_results))]
val_results = pd.DataFrame([gval, gval_pred, gval_pred_res, gdblval, gdblval_pred, gdblval_pred_res], index=["g", "g_pred", "g_pred_res", "gdbl", "gdbl_pred", "gdbl_pred_res"]).T
val_results["label"] = ["val" for x in range(len(val_results))]

results = pd.concat([train_results, test_results, val_results], axis=0).reset_index(drop=True)

results = pd.concat([results, all_errg, ci_errors_1g, ci_errors_2g, all_errgdbl, ci_errors_1gdbl, ci_errors_2gdbl, smiles_full],axis=1)
results.to_csv("../reports/predictions.csv", index=False)

prediction_summary = pd.DataFrame()
for ind in [train, test, val]:
    gr2 = r2_score(results.g[ind], results.g_pred[ind])
    grmse = np.sqrt(mean_squared_error(results.g[ind], results.g_pred[ind]))
    gdblr2 = r2_score(results.gdbl[ind], results.gdbl_pred[ind])
    gdblrmse = np.sqrt(mean_squared_error(results.gdbl[ind], results.gdbl_pred[ind]))
    pred_rep = pd.DataFrame([gr2, grmse, gdblr2, gdblrmse],index=["gr2", "grmse", "gdblr2", "gdblrmse"]).T
    prediction_summary = pd.concat([prediction_summary, pred_rep], axis=0).reset_index(drop=True)
prediction_summary.rename({0:"train",1:"test",2:"val"}, axis=0, inplace=True)
prediction_summary.to_csv("../reports/prediction_summary.csv", index=False)


#!/usr/bin/env python
import os
import sys

from django.core.exceptions import ImproperlyConfigured

def run_gunicorn(is_production: bool):
    from django_events.wsgi import application  # noqa
    from django_docker_helpers.management import run_gunicorn  # noqa

    gunicorn_module_name = 'gunicorn_prod' if is_production else 'gunicorn_dev'
    run_gunicorn(application, gunicorn_module_name=gunicorn_module_name)

def _prepare():
    from django_docker_helpers.db import ensure_databases_alive, ensure_caches_alive, migrate  # noqa
    from django_docker_helpers.files import collect_static
    from django_docker_helpers.management import create_admin

    collect_static()
    ensure_databases_alive(100)
    ensure_caches_alive(100)
    migrate()
    create_admin()

if __name__ == '__main__':
    BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))

    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_events.settings')

    IS_PRODUCTION = bool(int(os.environ.get('PRODUCTION', 0) or 0))
    IS_DOCKERIZED = bool(int(os.environ.get('DOCKERIZED', 0) or 0))

    if IS_DOCKERIZED:
        _prepare()

    if len(sys.argv) == 2:
        if sys.argv[1] == 'runuwsgi':
            from django.conf import settings

            os.environ.setdefault('PORT', str(settings.COMMON_BASE_PORT))
            safe = getattr(settings, 'UWSGI_STATIC_SAFE', None)
            if safe or safe == '':
                if IS_PRODUCTION:
                    raise ImproperlyConfigured(r"""
                        WARNING: UWSGI_STATIC_SAFE is the unsafe security option! DO NOT USE IT ON PRODUCTION!!!
                        (We treat an empty string and non-falsy values as 'enable static-safe option')""")

                os.environ.setdefault('UWSGI_STATIC_SAFE', str(settings.UWSGI_STATIC_SAFE))

        if sys.argv[1] == 'gunicorn':
            run_gunicorn(IS_PRODUCTION)
            exit()

    from django.core.management import execute_from_command_line
    execute_from_command_line(sys.argv)


# -*- coding: utf-8 -*-

import asyncio
import gi
gi.require_version("GUPnP", "1.0")
from gi.repository import GUPnP
import urllib.request
import urllib.parse
from xml.etree.ElementTree import XML, XMLParser
from xmlutils import StripNamespace

from cameraremoteapi import CameraRemoteApi
# from utils import debug_trace

class CameraRemoteControl(object):

    def __init__(self, friendly_name, callback):
        self.__friendly_name = friendly_name
        self.__device_available_callback = callback

        ctx = GUPnP.Context.new(None, None, 0)
        # caution : keep cp as an attribute !
        self.cp = GUPnP.ControlPoint.new(ctx, "upnp:rootdevice")
        self.cp.set_active(True)
        self.cp.connect("device-proxy-available", self.__device_available)

    def __device_available(self, cp, proxy):
        proxy_friendly_name = proxy.get_friendly_name()
        # print("Found " + proxy_friendly_name)
        if proxy_friendly_name.startswith(self.__friendly_name):
            service = proxy.get_service("urn:schemas-sony-com:service:ScalarWebAPI:1")
            if service is not None:

                # browse the xml location file and search for the camera service
                # and set the action list uri
                location = proxy.get_location()
                xd = urllib.request.urlopen(location)
                s = xd.read()

                target = StripNamespace()
                parser = XMLParser(target=target)
                root = XML(s, parser=parser)

                sl = root.findall("device/X_ScalarWebAPI_DeviceInfo/X_ScalarWebAPI_ServiceList/X_ScalarWebAPI_Service")
                for s in sl:
                    service_type = s.find("X_ScalarWebAPI_ServiceType").text
                    if service_type == CameraRemoteApi.SERVICE_NAME:
                        camera_action_list = s.find("X_ScalarWebAPI_ActionList_URL").text
                        if not camera_action_list.endswith('/'):
                            camera_action_list += '/'
                        endpoint_url = urllib.parse.urljoin(camera_action_list, CameraRemoteApi.SERVICE_NAME)
                        asyncio.ensure_future(self.__device_available_callback(proxy_friendly_name, endpoint_url))
                        break


from keras.models import Sequential
from keras.models import model_from_json
from keras.layers import Dense
import numpy
import os
numpy.random.seed(7)

# DATASET

train_dataset = numpy.loadtxt("pima-indians-diabetes.train_data.txt", delimiter=",")
X = train_dataset[:,0:8]
Y = train_dataset[:,8]

eval_dataset = numpy.loadtxt("pima-indians-diabetes.eval_data.txt", delimiter=",")

evalX = eval_dataset[:,0:8]
evalY = eval_dataset[:,8]

# NETWORK
model = Sequential()

# 8 inputs, 12 outputs
# Rectifier activation layer
# https://en.wikipedia.org/wiki/Rectifier_(neural_networks)
model.add(Dense(12, input_dim=8, activation='relu'))

# 12 inputs, 8 outputs
model.add(Dense(8, activation='relu'))

# ensure output is in range (0,1)
# Sigmoid activation layer
# https://en.wikipedia.org/wiki/Sigmoid_function
model.add(Dense(1, activation='sigmoid'))

# COMPILE MODEL

# logarithmic loss = binary cross entropy (https://en.wikipedia.org/wiki/Cross_entropy)
# gradient descent algorithm = adam (http://arxiv.org/abs/1412.6980)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

# TRAINING

# epochs is the number of iterations
# batch size is the number of instances evaluated before weight update in network
model.fit(X, Y, epochs=150, batch_size=10)

# EVALUATION (should be another dataset than we trained on)
scores = model.evaluate(X, Y)

print("\n%s: %.2f%%" % (model.metrics_names[1], scores[1]*100))

predictions = model.predict(evalX)
rounded = [round(x[0]) for x in predictions]
print("Result",rounded)

# Serialize the model to JSON
model_json = model.to_json()
with open("model/model.json","w") as json_file:
    json_file.write(model_json)
# Serialize weights to HDF5
model.save_weights("model/model.h5")
print("Saved model to disk")


import json

products_suppliers = dict()   # { product_name : supplier_id}

def filling_tables(connect,file):
    """ Функция заполняет новыми данными таблицу suppliers из файла  и добавляет suppliers_id в products"""
    with connect.cursor() as cursor:
        with open(f'{file}') as file:
            suppliers = json.load(file)
            for supplier in suppliers:
                new_address = supplier["address"].split(';')     # Создаем список из "address", будем делать разные колонки (country,region,postal_code,city,address)
                cursor.execute('INSERT INTO suppliers (company_name,contact,country,region,postal_code,city,address,phone,fax,homepage ) VALUES (%s, %s, %s, %s, %s,%s, %s, %s, %s, %s)  RETURNING suppliers_id',(supplier["company_name"],supplier["contact"],new_address[0],new_address[1],new_address[2],new_address[3],new_address[4], supplier["phone"],supplier["fax"],supplier["homepage"]))
                id_of_new_row = cursor.fetchone()[0]     # Получаем назад supplier_id
                for product in supplier["products"]:     # Делаем словарик product_name : supplier_id
                    products_suppliers[product] = id_of_new_row
            big_sql_row =''                         # Делаем один большой запрос (обновляем базу)
            for i,v in products_suppliers.items():
                name = i
                name = name.replace("'", "''")       # В запросе есть строки с " ' ", заменяем или будет ошибка
                big_sql_row += f"UPDATE products SET suppliers_id = {v} WHERE product_name = '{name}';"
            cursor.execute(big_sql_row)
            connect.commit()


import numpy as np

from math import sqrt

from utils.transform import list2str, str2list

def combine(parts):
    if len(parts) == 1:
        return parts[0]
    columns = []
    for i in range(0, int(sqrt(len(parts)))):
        columns.append([])
        key = len(columns) - 1
        for j in range(i, len(parts), int(sqrt(len(parts)))):
            for line in parts[j].split('/'):
                columns[key].append(line)
    new = np.column_stack(columns).tolist()
    return list2str(new)

def divide(pattern):
    """
    @url https://stackoverflow.com/questions/16856788/slice-2d-array-into-smaller-2d-arrays
    """
    if isinstance(pattern, str):
        pattern = str2list(pattern)
    arr = np.asarray(pattern)
    h, w = arr.shape
    if h % 2 == 0:
        by = 2
    else:
        by = 3
    return arr.reshape(h//by, by, -1, by).swapaxes(1, 2).reshape(-1, by, by)


# -*- coding: utf-8 -*-
# @Author: Lishi
# @Date:   2017-11-09 18:41:13
# @Last Modified by:   Lishi
# @Last Modified time: 2017-11-14 20:00:48

from __future__ import print_function
import torch as t
import numpy as np

from torch.autograd import Variable  #

import os, sys
import shutil
import pdb

def testTensor():
    x = t.Tensor(5, 3)  # 仅仅分配了空间，并未初始化
    x = t.rand(5, 3)  # 使用 [0,1] 均匀分布随机初始化二维数组
    print(x.size())

    y = t.rand(5, 3)

    # 进行加法计算
    result = t.Tensor(5, 3)
    t.add(x, y, out=result)  # 输出到result

    ## 函数名后面带下划线_ 的函数会修改Tensor本身,如 add_()
    y.add(x)  # 普通加法，不会改变y的值
    y.add_(x)  #inplace 加法，y改变

    ## 类型转化
    # numpy 可以处理 Tensor不可执行的
    a_t = t.ones(5)
    b_np = a.numpy()  #Tensor => Numpy
    a_t = t.from_numpy(b)  # Numpy => Tensor

    # Tensor => GPU Tensor,加速
    if t.cuda.is_available():
        x = x.cuda()
        y = y.cuda()
        x + y

    # Autograd : 自动微分
    x = Variable(t.ones(2, 2), requires_grad=True)
    y = x.sum()
    y.grad_fn
    y.backward()  # 反向传播，计算梯度;

    # y = x.sum() = (x[0][0] + x[0][1] + x[1][0] + x[1][1])
    # 每个值的梯度都为1
    x.grad

def main():
    print("this is a test {}".format(os.getcwd()))

if __name__ == '__main__':
    main()


"""Simulations module

simulation_1():
    "Preliminary observations" - There are three possible cases.

simulation_2():
    "Homogeneous case" - Calculates asymptotic eigenvalues and alignments of eigenvectors.

simulation_3():
    "Homogeneous case" - Calculates asymptotic alignment of eigenvectors for growing n.

simulation_4():
    "Homogeneous case" - Performs community detection.
"""

from model import Model
from utils import *
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
import warnings
warnings.filterwarnings('ignore')  # use only after debugging

def simulation_1():
    """Shows spectrum and extremal eigenvectors for the chosen case.
    """

    # Input basic variables
    n = int(input('Enter number of points (n, default 1000): ') or "1000")
    k = 3
    case = int(input('Enter case desired (1, 2, or 3): '))

    # Build Q from case
    q = 0*np.ones(n)  # initialise

    # CASE 1
    if case == 1:
        q_0 = float(input("Enter q_0 (default 0.05): ") or "0.05")
        q = q_0*np.ones(n)

    # CASE 2
    if case == 2:
        epsilon = 0.25
        q_0 = 0.5
        q = np.random.uniform(low=q_0-epsilon, high=q_0+epsilon, size=n)

    # CASE 3
    if case == 3:
        q_1 = float(input("Enter q_1 (default 0.5): ") or "0.5")
        q_2 = float(input("Enter q_2 (default 0.05): ") or "0.05")
        q = np.random.choice(np.array([q_1,q_2]), size = n)

    Q = np.diag(q)

    # Build the model and take eigenvalues
    model = Model(n, k, Q)

    # Graphical representations
    # Spectrum
    isolated_evals = isolated_eigenvalues(model)
    plt.hist(model.B_evals, density=True, bins=100)
    plt.title(r'Empirical spectral distribution of $B/\sqrt{n}$')
    plt.xlabel(r'Spectrum of $B/\sqrt{n}$')
    plt.ylabel('Density')
    annotation_string = f'n = {n} \nM ~ {int(model.order)} \nq_0 \
= {np.round(model.Q[0,0],2)} \nn. of spikes = {len(isolated_evals)}'
    plt.text(0.75, 0.8, annotation_string, transform=plt.gca().transAxes)
    # Indicate spikes clearly, if any (attention: only for homogeneous case)
    if case == 1:
        for _, val in enumerate(isolated_evals):
            plt.annotate('Spike', xy=(val, 0), xycoords='data',
                    xytext=(0, 10), textcoords='offset points',
                    ha='center', color='red', rotation=45
                    )

    # Eigenvectors
    plt.figure()
    plt.title(f'Extremal ({model.k} largest) eigenvectors')  # not showing
    for i in range(model.k):
        plt.subplot(model.k, 1, i+1)
        plt.plot(model.B_evecs[:,-i-1])
        plt.annotate(r'$\lambda =$ '+f'{np.round(model.B_evals[-i-1], 4)}',
                xy=(0.9,0.9), xycoords='axes points')
        for j in range(model.k):
            plt.axvline(x=np.sum(model.class_sizes[:j+1]), c='gray', linestyle='--')
        if i < model.k-1:
            plt.xticks([])
    plt.xlabel('Dimensions')
    plt.show()

def simulation_2():
    """Calculates the asymptotics of eigenvalues.
    """

    # Input basic variables
    n = int(input('Enter number of points (n, default 1000): ') or "1000")
    k = 3
    q_0 = float(input('Enter q_0 (default 0.05): ') or '0.05')
    Q = np.diag(q_0*np.ones(n))

    # Build model
    model = Model(n, k, Q)

    # Eigenvalues
    lambd = isolated_eigenvalues(model)
    n_isolated_eigvals = len(lambd)

    # Graphical representation (Eigenvalues)
    color = iter(plt.cm.rainbow(np.linspace(0, 1, len(lambd))))
    plt.hist(model.B_evals, density=True, bins=100)
    for i in range(len(lambd)):
        plt.axvline(lambd[i], c=next(color), linestyle='dashed')
    plt.title(f"Number of isolated eigenvalues: {n_isolated_eigvals}.")
    plt.show()

def simulation_3():
    """Calculates alignements of eigenvectors for growing n.
    """

    # Basic variables
    min_n = int(input("Enter minimal n (default 200): ") or "200")
    max_n = int(input("Enter maximal n (default 2000): ") or "2000")+200
    step_n = int(input("Enter step for n (default 200): ") or "200")
    list_n = list(range(min_n, max_n, step_n))
    k = 3
    q_0 = float(input("Enter q_0 (default 0.05): ") or "0.05")

    # Fix diagonal M
    order = float(input("Enter an order for M (default 100): ") or "100")
    m = order*np.random.rand(k)
    M = np.diag(m)

    # For saving results
    alignments_final = []
    alignments_theoretical = []

    # Main loop
    for i, n in enumerate(list_n):
        # Create model with n points but fixed M
        q = q_0*np.ones(n)
        Q = np.diag(q)
        model_n = Model(n, k, Q, M, ask_priors=False)

        # Isolated eigenvectors
        alignments_n, theoretical_alignments_n = alignments(model_n)

        np.set_printoptions(precision=3, suppress=True)
        print(f'n = {n} \n')
        print('Observed alignments: ', np.array_str(alignments_n, precision=4), '\n')
        print('Theoret. alignments: ', np.array_str(theoretical_alignments_n, precision=4), '\n')

        # Register iteration
        alignments_final.append(alignments_n)
        alignments_theoretical.append(theoretical_alignments_n)

    # Transform into convenient format
    alignments_final = np.array(alignments_final)
    alignments_theoretical = np.array(alignments_theoretical)

    # Print M to know its influence on observations
    print('M: ', m)

    # Graphical representation
    # Observed alignments
    for i in range(k):
        plt.plot(list_n, alignments_final[:,i], label=f'Observed alignments {i+1}')
        plt.plot(list_n, alignments_theoretical[:, i], linestyle='dashed', label=f'Predicted {i+1}')
        plt.ylim([0,1])
        plt.legend()
    plt.show()

def simulation_4():
    """Performs community detection by K-means on rows of isolated eigenvectors.
    """

    # Take isolated eigenvectors
    n = int(input('Enter number of points (n, default 1000): ') or "1000")
    k = 3
    q_0 = float(input('Enter q_0 (default 0.1): ') or '0.1')
    Q = np.diag(q_0*np.ones(n))

    # Build model
    model = Model(n, k, Q)

    # Perform K-means clustering on the rows of k largest eigenvectors.
    kmeans_predictions = KMeans(n_clusters=k).fit_predict(model.B_evecs[:,-k:])
    print("Accuracy: ", np.mean(kmeans_predictions == model.indicatrix))

    # For n=1000, q_0=0.1, M=150, I get 0.968 accuracy.


'''
Created on May 15, 2012

@author: lwoydziak
'''
import unittest
from trackeritemstatus import TrackerItemStatus
from jiraticket import JiraTicket
from mockito.mockito import verify, when
from mockito.mocking import mock
from mockito import inorder
from mockito.verification import never
from mockito.matchers import any

class TrackerItemStatusTests(unittest.TestCase):
    def test_whenSeedingWithRemoteStatusForJira(self):
        ticket = JiraTicket()
        statusId = "1234"
        closed = "Closed"
        ticket.setStatus(statusId)
        mapObject = mock()
        when(mapObject).translateStatusTo('jiraStatusName', statusId).thenReturn([closed])
        status = TrackerItemStatus(ticket, apiObject=mapObject)
        status.pivotal()
        inorder.verify(mapObject).translateStatusTo('jiraStatusName', statusId)
        inorder.verify(mapObject).translateStatusTo('pivotal', closed)
        self.assertEqual(status.jira(), [closed])

    def test_whenSeedingWithPivotalStatus(self):
        pivotalStatus = "Started"
        mapObject = mock()
        status = TrackerItemStatus(pivotalStatus, apiObject=mapObject)
        status.jira()
        verify(mapObject).translateStatusTo('jira', pivotalStatus)
        self.assertEqual(status.pivotal(), pivotalStatus)

    def test_canCreateDefaultStatus(self):
        status = TrackerItemStatus()
        self.assertEqual(status.jira(), None)
        self.assertEqual(status.pivotal(), None)

    def test_equalityOfStatuses(self):
        status = TrackerItemStatus()
        statusEqual = TrackerItemStatus()
        self.assertTrue(status == statusEqual)
        self.assertFalse(status is statusEqual)

    def test_inequalityOfStatused(self):
        status = TrackerItemStatus()
        garbage = "garbage"
        pivotalStatus = "Started"
        mapObject = mock()
        statusNotEqual = TrackerItemStatus(pivotalStatus, apiObject=mapObject)
        self.assertTrue(status != statusNotEqual)
        self.assertFalse(status is statusNotEqual)
        self.assertTrue(status != garbage)

    def test_whenJiraIdIsZeroNameIsNone(self):
        jiraStatus = mock()
        mapObject = mock()
        when(jiraStatus).status().thenReturn("")
        status = TrackerItemStatus(jiraStatus, mapObject)
        verify(mapObject, never).translateStatusTo(any(), any())
        self.assertEqual(status.jira(), None)

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()


import sys
from PIL import Image
import struct

if len(sys.argv)!=2:
  print("Usage: python read_bin.py in_bin_file")

infile = open(sys.argv[1],'rb')
# infile = open('y.bin','rb')
w = int(struct.unpack("i",infile.read(4))[0])
h = int(struct.unpack("i",infile.read(4))[0])
print("The picture\'s size is:")
print(w,h)

# short
str = 'h'
num = 2

pixel_r = (struct.unpack(str*h*w,infile.read(num*h*w)))
pixel_g = (struct.unpack(str*h*w,infile.read(num*h*w)))
pixel_b = (struct.unpack(str*h*w,infile.read(num*h*w)))
pixel_a = (struct.unpack(str*h*w,infile.read(num*h*w)))

infile.close()

print("Putting pixels to new images...")
cnt = 0
new_img_r = Image.new('L',(w,h))
new_img_g = Image.new('L',(w,h))
new_img_b = Image.new('L',(w,h))
new_img_a = Image.new('L',(w,h))
new_img_rgb = Image.new('RGB',(w,h))
new_img_0 = Image.new('RGBA',(w,h))

img_r = new_img_r.load()
img_g = new_img_g.load()
img_b = new_img_b.load()
img_a = new_img_a.load()
img_0 = new_img_0.load()
img_rgb = new_img_rgb.load()

for i in range(h):
  for j in range(w):
    temp_r = int(pixel_r[cnt])
    temp_g = int(pixel_g[cnt])
    temp_b = int(pixel_b[cnt])
    temp_a = int(pixel_a[cnt])

    img_r[j,i] = (temp_r)
    img_g[j,i] = (temp_g)
    img_b[j,i] = (temp_b)
    img_a[j,i] = (temp_a)
    img_rgb[j,i] = (temp_r,temp_g,temp_b)
    img_0[j,i] = (temp_r,temp_g,temp_b,temp_a)
    cnt+=1

print("Creating png files...")
new_img_r.save("output/out_image_r.png","png")
new_img_g.save("output/out_image_g.png","png")
new_img_b.save("output/out_image_b.png","png")
new_img_a.save("output/out_image_a.png","png")
new_img_rgb.save("output/out_image_rgb.png","png")
new_img_0.save("output/out_im_rgba.png","png")