import torch
from torch import nn
import torch.nn.functional as F
import math
import copy
from util.box_ops import box_cxcywh_to_xyxy, generalized_box_iou
from util.misc import (NestedTensor, nested_tensor_from_tensor_list,
                       accuracy, get_world_size, interpolate,
                       is_dist_avail_and_initialized, inverse_sigmoid)
import numpy as np
import clip
from datasets.hico_text_label import hico_text_label, hico_obj_text_label, hico_unseen_index
from datasets.vcoco_text_label import vcoco_hoi_text_label, vcoco_obj_text_label

from .backbone import build_backbone
from .matcher import build_matcher
from .deformable_transformer import build_deformable_transformer
from .utils import sigmoid_focal_loss, MLP

import sys
from omegaconf import OmegaConf
from einops import rearrange, repeat

from ldm.util import instantiate_from_config
from SD_Extractor import UNetWrapper, TextAdapter

def _sigmoid(x):
    y = torch.clamp(x.sigmoid(), min=1e-4, max=1 - 1e-4)
    return y

class DiffHOI_L(nn.Module):
    """ This is the Cross-Attention Detector module that performs object detection """

    def __init__(self, backbone, transformer, num_classes, num_queries,
                 aux_loss=False, iter_update=False,
                 query_dim=2,
                 random_refpoints_xy=False,
                 fix_refpoints_hw=-1,
                 num_feature_levels=1,
                 nheads=8,
                 # two stage
                 two_stage_type='no',  # ['no', 'standard']
                 two_stage_add_query_num=0,
                 dec_pred_class_embed_share=True,
                 dec_pred_bbox_embed_share=True,
                 two_stage_class_embed_share=True,
                 two_stage_bbox_embed_share=True,
                 decoder_sa_type='sa',
                 num_patterns=0,
                 dn_number=100,
                 dn_box_noise_scale=0.4,
                 dn_label_noise_ratio=0.5,
                 dn_labelbook_size=100,
                 args=None,
                 unet_config = dict()
                 ):
        """ Initializes the model.
        Parameters:
            backbone: torch module of the backbone to be used. See backbone.py
            transformer: torch module of the transformer architecture. See transformer.py
            num_classes: number of object classes
            num_queries: number of object queries, ie detection slot. This is the maximal number of objects
                         Conditional DETR can detect in a single image. For COCO, we recommend 100 queries.
            aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used.

            fix_refpoints_hw: -1(default): learn w and h for each box seperately
                                >0 : given fixed number
                                -2 : learn a shared w and h
        """
        super().__init__()

        config_path = os.environ.get("SD_Config")
        ckpt_path = os.environ.get("SD_ckpt")
        config = OmegaConf.load(config_path)
        config.model.params.ckpt_path = ckpt_path
        config.model.params.cond_stage_config.target = 'ldm.modules.encoders.modules.AbstractEncoder'
        sd_model = instantiate_from_config(config.model)
        self.encoder_vq = sd_model.first_stage_model
        self.unet = UNetWrapper(sd_model.model, **unet_config)
        device = "cuda" if torch.cuda.is_available() else "cpu"
        self.unet.to(device)
        # self.unet.eval()
        sd_model.model = None
        sd_model.first_stage_model = None
        del sd_model.cond_stage_model
        del self.encoder_vq.decoder
        self.sd_model = sd_model
        self.text_adapter = nn.Linear(args.clip_embed_dim, 768)
        self.clip_adapter = nn.Linear(args.clip_embed_dim,args.clip_embed_dim)

        self.num_queries = num_queries
        self.transformer = transformer
        self.num_classes = num_classes
        self.hidden_dim = hidden_dim = transformer.d_model
        self.num_feature_levels = num_feature_levels
        self.nheads = nheads
        self.label_enc = nn.Embedding(dn_labelbook_size + 1, hidden_dim)
        self.args=args
        # setting query dim
        self.query_dim = query_dim
        assert query_dim == 4
        self.random_refpoints_xy = random_refpoints_xy
        self.fix_refpoints_hw = fix_refpoints_hw

        # for dn training
        self.num_patterns = num_patterns
        self.dn_number = dn_number
        self.dn_box_noise_scale = dn_box_noise_scale
        self.dn_label_noise_ratio = dn_label_noise_ratio
        self.dn_labelbook_size = dn_labelbook_size

        # prepare input projection layers
        if num_feature_levels > 1:
            num_backbone_outs = len(backbone.num_channels)
            input_proj_list = []
            for _ in range(num_backbone_outs):
                in_channels = backbone.num_channels[_]
                input_proj_list.append(nn.Sequential(
                    nn.Conv2d(in_channels, hidden_dim, kernel_size=1),
                    nn.GroupNorm(32, hidden_dim),
                ))
            for _ in range(num_feature_levels - num_backbone_outs):
                input_proj_list.append(nn.Sequential(
                    nn.Conv2d(in_channels, hidden_dim, kernel_size=3, stride=2, padding=1),
                    nn.GroupNorm(32, hidden_dim),
                ))
                in_channels = hidden_dim
            self.input_proj = nn.ModuleList(input_proj_list)
        else:
            assert two_stage_type == 'no', "two_stage_type should be no if num_feature_levels=1 !!!"
            self.input_proj = nn.ModuleList([
                nn.Sequential(
                    nn.Conv2d(backbone.num_channels[-1], hidden_dim, kernel_size=1),
                    nn.GroupNorm(32, hidden_dim),
                )])

        self.backbone = backbone
        self.aux_loss = aux_loss
        self.box_pred_damping = box_pred_damping = None

        self.iter_update = iter_update
        assert iter_update, "Why not iter_update?"

        # prepare pred layers
        self.dec_pred_class_embed_share = dec_pred_class_embed_share
        self.dec_pred_bbox_embed_share = dec_pred_bbox_embed_share
        # prepare class & box embed
        _class_embed = nn.Linear(hidden_dim, num_classes)
        _bbox_embed = MLP(hidden_dim, hidden_dim, 4, 3)

        # init the two embed layers
        prior_prob = 0.01
        bias_value = -math.log((1 - prior_prob) / prior_prob)
        _class_embed.bias.data = torch.ones(self.num_classes) * bias_value
        nn.init.constant_(_bbox_embed.layers[-1].weight.data, 0)
        nn.init.constant_(_bbox_embed.layers[-1].bias.data, 0)

        if dec_pred_bbox_embed_share:
            box_embed_layerlist = [_bbox_embed for i in range(transformer.num_decoder_layers -3 )]
        else:
            box_embed_layerlist = [copy.deepcopy(_bbox_embed) for i in range(transformer.num_decoder_layers -3  )]
        if dec_pred_class_embed_share:
            class_embed_layerlist = [_class_embed for i in range(transformer.num_decoder_layers -3  )]
        else:
            class_embed_layerlist = [copy.deepcopy(_class_embed) for i in range(transformer.num_decoder_layers -3 )]

        self.bbox_embed = nn.ModuleList(box_embed_layerlist)
        self.class_embed = nn.ModuleList(class_embed_layerlist)
        self.transformer.decoder.bbox_embed = self.bbox_embed
        self.transformer.decoder.class_embed = self.class_embed

        # two stage
        self.two_stage_type = two_stage_type
        self.two_stage_add_query_num = two_stage_add_query_num
        assert two_stage_type in ['no', 'standard'], "unknown param {} of two_stage_type".format(two_stage_type)
        if two_stage_type != 'no':
            if two_stage_bbox_embed_share:
                assert dec_pred_class_embed_share and dec_pred_bbox_embed_share
                self.transformer.enc_out_bbox_embed = _bbox_embed
            else:
                self.transformer.enc_out_bbox_embed = copy.deepcopy(_bbox_embed)

            if two_stage_class_embed_share:
                assert dec_pred_class_embed_share and dec_pred_bbox_embed_share
                self.transformer.enc_out_class_embed = _class_embed
            else:
                self.transformer.enc_out_class_embed = copy.deepcopy(_class_embed)

            self.refpoint_embed = None
            if self.two_stage_add_query_num > 0:
                self.init_ref_points(two_stage_add_query_num)

        self.decoder_sa_type = decoder_sa_type
        assert decoder_sa_type in ['sa', 'ca_label', 'ca_content']
        if decoder_sa_type == 'ca_label':
            self.label_embedding = nn.Embedding(num_classes, hidden_dim)
            for layer in self.transformer.decoder.layers:
                layer.label_embedding = self.label_embedding
        else:
            for layer in self.transformer.decoder.layers:
                layer.label_embedding = None
            self.label_embedding = None

        self.dec_layers = 3

        self._reset_parameters()

        self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
        self.obj_logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))

        if self.args.dataset_file == 'hico':
            hoi_text_label = hico_text_label
            obj_text_label = hico_obj_text_label
            unseen_index = hico_unseen_index
        elif self.args.dataset_file == 'vcoco':
            hoi_text_label = vcoco_hoi_text_label
            obj_text_label = vcoco_obj_text_label
            unseen_index = None

        clip_label, obj_clip_label, v_linear_proj_weight, hoi_text, obj_text, train_clip_label = \
            self.init_classifier_with_CLIP(hoi_text_label, obj_text_label, unseen_index)
        num_obj_classes = len(obj_text) - 1  # del nothing

        self.hoi_class_fc = nn.Sequential(
            nn.Linear(hidden_dim, args.clip_embed_dim),
            nn.LayerNorm(args.clip_embed_dim),
        )

        if args.with_clip_label:
            self.visual_projection = nn.Linear(args.clip_embed_dim, len(hoi_text))
            self.visual_projection.weight.data = train_clip_label / train_clip_label.norm(dim=-1, keepdim=True)
            if self.args.dataset_file == 'hico' and self.args.zero_shot_type != 'default':
                self.eval_visual_projection = nn.Linear(args.clip_embed_dim, 600)
                self.eval_visual_projection.weight.data = clip_label / clip_label.norm(dim=-1, keepdim=True)

        if args.with_obj_clip_label:
            self.obj_class_fc = nn.Sequential(
                nn.Linear(hidden_dim, args.clip_embed_dim),
                nn.LayerNorm(args.clip_embed_dim),
            )
            self.obj_visual_projection = nn.Linear(args.clip_embed_dim, num_obj_classes + 1)
            self.obj_visual_projection.weight.data = obj_clip_label / obj_clip_label.norm(dim=-1, keepdim=True)
        device = "cuda" if torch.cuda.is_available() else "cpu"
        self.clip_model, _ = clip.load(args.clip_model, device=device)

        self.image_adapter_4 = nn.Conv2d(1280, hidden_dim, kernel_size=1)

        self.image_adapter_3 = nn.Conv2d(1280, hidden_dim, kernel_size=1)

        self.image_adapter_2 = nn.Conv2d(640, hidden_dim, kernel_size=1)

        self.image_adapter_1 = nn.Conv2d(320, hidden_dim, kernel_size=1)

        self.clip_adapter = nn.Linear(args.clip_embed_dim,args.clip_embed_dim)
        # import pdb
        # pdb.set_trace()

        self.freeze()

    def freeze(self):
        for param in self.class_embed.parameters():
            param.requires_grad = False

        for param in self.transformer.enc_out_bbox_embed.parameters():
            param.requires_grad = False

    def _reset_parameters(self):
        # init input_proj
        for proj in self.input_proj:
            nn.init.xavier_uniform_(proj[0].weight, gain=1)
            nn.init.constant_(proj[0].bias, 0)

    def init_classifier_with_CLIP(self, hoi_text_label, obj_text_label, unseen_index):
        device = "cuda" if torch.cuda.is_available() else "cpu"
        text_inputs = torch.cat([clip.tokenize(hoi_text_label[id]) for id in hoi_text_label.keys()])
        if self.args.del_unseen and unseen_index is not None:
            hoi_text_label_del = {}
            unseen_index_list = unseen_index.get(self.args.zero_shot_type, [])
            for idx, k in enumerate(hoi_text_label.keys()):
                if idx in unseen_index_list:
                    continue
                else:
                    hoi_text_label_del[k] = hoi_text_label[k]
        else:
            hoi_text_label_del = hoi_text_label.copy()
        text_inputs_del = torch.cat(
            [clip.tokenize(hoi_text_label[id]) for id in hoi_text_label_del.keys()])

        obj_text_inputs = torch.cat([clip.tokenize(obj_text[1]) for obj_text in obj_text_label])
        clip_model, preprocess = clip.load(self.args.clip_model, device=device)
        with torch.no_grad():
            text_embedding = clip_model.encode_text(text_inputs.to(device))
            text_embedding_del = clip_model.encode_text(text_inputs_del.to(device))
            obj_text_embedding = clip_model.encode_text(obj_text_inputs.to(device))
            v_linear_proj_weight = clip_model.visual.proj.detach()

        del clip_model

        return text_embedding.float(), obj_text_embedding.float(), v_linear_proj_weight.float(), \
               hoi_text_label_del, obj_text_inputs, text_embedding_del.float()

    def init_ref_points(self, use_num_queries):
        self.refpoint_embed = nn.Embedding(use_num_queries, self.query_dim)
        if self.random_refpoints_xy:
            self.refpoint_embed.weight.data[:, :2].uniform_(0, 1)
            self.refpoint_embed.weight.data[:, :2] = inverse_sigmoid(self.refpoint_embed.weight.data[:, :2])
            self.refpoint_embed.weight.data[:, :2].requires_grad = False

        if self.fix_refpoints_hw > 0:
            print("fix_refpoints_hw: {}".format(self.fix_refpoints_hw))
            assert self.random_refpoints_xy
            self.refpoint_embed.weight.data[:, 2:] = self.fix_refpoints_hw
            self.refpoint_embed.weight.data[:, 2:] = inverse_sigmoid(self.refpoint_embed.weight.data[:, 2:])
            self.refpoint_embed.weight.data[:, 2:].requires_grad = False
        elif int(self.fix_refpoints_hw) == -1:
            pass
        elif int(self.fix_refpoints_hw) == -2:
            print('learn a shared h and w')
            assert self.random_refpoints_xy
            self.refpoint_embed = nn.Embedding(use_num_queries, 2)
            self.refpoint_embed.weight.data[:, :2].uniform_(0, 1)
            self.refpoint_embed.weight.data[:, :2] = inverse_sigmoid(self.refpoint_embed.weight.data[:, :2])
            self.refpoint_embed.weight.data[:, :2].requires_grad = False
            self.hw_embed = nn.Embedding(1, 1)
        else:
            raise NotImplementedError('Unknown fix_refpoints_hw {}'.format(self.fix_refpoints_hw))

    def extract_feat(self, img,targets):
        """Extract features from images."""
        target_clip_inputs = torch.cat([t['clip_inputs'].unsqueeze(0) for t in targets])
        target_sd_inputs = torch.cat([t['sd_inputs'].unsqueeze(0) for t in targets])
        with torch.no_grad():
            clip_feats = self.clip_model.encode_image(target_clip_inputs)
        clip_feats=self.text_adapter(clip_feats.unsqueeze(1).float())
        t = torch.zeros((img.shape[0],), device=img.device).long()
        with torch.no_grad():
            latents = self.encoder_vq.encode(target_sd_inputs)
            latents = latents.mode().detach()
            outs = self.unet(latents, t, c_crossattn=[clip_feats])
        return outs

    def forward(self, samples: NestedTensor, targets , is_training=True):
        """ The forward expects a NestedTensor, which consists of:
               - samples.tensor: batched images, of shape [batch_size x 3 x H x W]
               - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels

            It returns a dict with the following elements:
               - "pred_logits": the classification logits (including no-object) for all queries.
                                Shape= [batch_size x num_queries x num_classes]
               - "pred_boxes": The normalized boxes coordinates for all queries, represented as
                               (center_x, center_y, width, height). These values are normalized in [0, 1],
                               relative to the size of each individual image (disregarding possible padding).
                               See PostProcess for information on how to retrieve the unnormalized bounding box.
               - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of
                                dictionnaries containing the two above keys for each decoder layer.
        """
        if isinstance(samples, (list, torch.Tensor)):
            samples = nested_tensor_from_tensor_list(samples)
        features, poss = self.backbone(samples)

        srcs = []
        masks = []
        for l, feat in enumerate(features):
            src, mask = feat.decompose()
            srcs.append(self.input_proj[l](src))
            masks.append(mask)
            assert mask is not None
        if self.num_feature_levels > len(srcs):
            _len_srcs = len(srcs)
            for l in range(_len_srcs, self.num_feature_levels):
                if l == _len_srcs:
                    src = self.input_proj[l](features[-1].tensors)
                else:
                    src = self.input_proj[l](srcs[-1])
                m = samples.mask
                mask = F.interpolate(m[None].float(), size=src.shape[-2:]).to(torch.bool)[0]
                pos_l = self.backbone[1](NestedTensor(src, mask)).to(src.dtype)
                srcs.append(src)
                masks.append(mask)
                poss.append(pos_l)

        input_query_bbox = input_query_label = attn_mask = dn_meta = None

        sd_feat = self.extract_feat(samples.decompose()[0],targets)
        sd_decoder_4 = self.image_adapter_4(sd_feat[-1])
        sd_decoder_3 = self.image_adapter_3(sd_feat[-2])
        sd_decoder_2 = self.image_adapter_2(sd_feat[-3])
        sd_decoder_1 = self.image_adapter_1(sd_feat[-4])
        sd_decoder_4 = torch.nn.functional.interpolate(sd_decoder_4,size=[srcs[-1].shape[-2],srcs[-1].shape[-1]])
        sd_decoder_3 = torch.nn.functional.interpolate(sd_decoder_3,size=[srcs[-2].shape[-2],srcs[-2].shape[-1]])
        sd_decoder_2 = torch.nn.functional.interpolate(sd_decoder_2,size=[srcs[-3].shape[-2],srcs[-3].shape[-1]])
        sd_decoder_1 = torch.nn.functional.interpolate(sd_decoder_1,size=[srcs[-4].shape[-2],srcs[-4].shape[-1]])
        sd_decoder = [sd_decoder_1,sd_decoder_2,sd_decoder_3,sd_decoder_4]

        hs, hs_interaction , reference, hs_enc, ref_enc, init_box_proposal = self.transformer(srcs, masks, input_query_bbox, poss,
                                                                             input_query_label,sd_decoder, attn_mask)
        # In case num object=0
        hs[0] += self.label_enc.weight[0, 0] * 0.0

        hs_interaction = torch.stack(hs_interaction)
        # deformable-detr-like anchor update
        # reference_before_sigmoid = inverse_sigmoid(reference[:-1]) # n_dec, bs, nq, 4
        outputs_coord_list = []
        for dec_lid, (layer_ref_sig, layer_bbox_embed, layer_hs) in enumerate(zip(reference[:-1], self.bbox_embed, hs)):
            layer_delta_unsig = layer_bbox_embed(layer_hs)
            layer_outputs_unsig = layer_delta_unsig + inverse_sigmoid(layer_ref_sig)
            layer_outputs_unsig = layer_outputs_unsig.sigmoid()
            outputs_coord_list.append(layer_outputs_unsig)
        outputs_coord_list = torch.stack(outputs_coord_list)

        # clip_feat
        target_clip_inputs = torch.cat([t['clip_inputs'].unsqueeze(0) for t in targets])
        with torch.no_grad():
            target_clip_feats = self.clip_model.encode_image(target_clip_inputs)
        target_clip_feats = self.clip_adapter(target_clip_feats.float())

        if self.args.with_obj_clip_label:
            obj_logit_scale = self.obj_logit_scale.exp()
            o_hs = self.obj_class_fc(torch.stack(hs))
            o_hs = o_hs / o_hs.norm(dim=-1, keepdim=True)
            outputs_obj_class = obj_logit_scale * self.obj_visual_projection(o_hs)

        if self.args.with_clip_label:
            logit_scale = self.logit_scale.exp()
            inter_hs = self.hoi_class_fc(hs_interaction)
            inter_hs = inter_hs + target_clip_feats[None,:,None,:].repeat(inter_hs.shape[0],1,inter_hs.shape[2],1)
            inter_hs = inter_hs / inter_hs.norm(dim=-1, keepdim=True)
            if self.args.dataset_file == 'hico' and self.args.zero_shot_type != 'default' \
                    and (self.args.eval or not is_training):
                outputs_hoi_class = logit_scale * self.eval_visual_projection(inter_hs)
            else:
                outputs_hoi_class = logit_scale * self.visual_projection(inter_hs)

        outputs_coord_list = outputs_coord_list.view(outputs_coord_list.shape[0],outputs_coord_list.shape[1],int(self.num_queries/2),2,4)

        outputs_class = outputs_obj_class.view(outputs_obj_class.shape[0],outputs_obj_class.shape[1],int(self.num_queries/2),2,outputs_obj_class.shape[-1])

        out = {'pred_hoi_logits': outputs_hoi_class[-1], 'pred_obj_logits': outputs_class[:,:,:,1,:][-1],
               'pred_sub_boxes': outputs_coord_list[:,:,:,0,:][-1], 'pred_obj_boxes': outputs_coord_list[:,:,:,1,:][-1],}

        if self.aux_loss:
            out['aux_outputs'] = self._set_aux_loss_triplet(outputs_hoi_class, outputs_class[:,:,:,1,:],
                                                            outputs_coord_list[:,:,:,0,:], outputs_coord_list[:,:,:,1,:])

        out['dn_meta'] = dn_meta

        return out

    @torch.jit.unused
    def _set_aux_loss(self, outputs_class, outputs_coord):
        # this is a workaround to make torchscript happy, as torchscript
        # doesn't support dictionary with non-homogeneous values, such
        # as a dict having both a Tensor and a list.
        return [{'pred_logits': a, 'pred_boxes': b}
                for a, b in zip(outputs_class[:-1], outputs_coord[:-1])]

    @torch.jit.unused
    def _set_aux_loss_triplet(self, outputs_hoi_class, outputs_obj_class,
                              outputs_sub_coord, outputs_obj_coord, outputs_inter_hs=None):

        aux_outputs = {'pred_hoi_logits': outputs_hoi_class[-self.dec_layers: -1],
                       'pred_obj_logits': outputs_obj_class[-self.dec_layers: -1],
                       'pred_sub_boxes': outputs_sub_coord[-self.dec_layers: -1],
                       'pred_obj_boxes': outputs_obj_coord[-self.dec_layers: -1]}
        if outputs_inter_hs is not None:
            aux_outputs['inter_memory'] = outputs_inter_hs[-self.dec_layers: -1]
        outputs_auxes = []
        for i in range(self.dec_layers - 1):
            output_aux = {}
            for aux_key in aux_outputs.keys():
                output_aux[aux_key] = aux_outputs[aux_key][i]
            outputs_auxes.append(output_aux)
        return outputs_auxes

class MLP(nn.Module):
    """ Very simple multi-layer perceptron (also called FFN)"""

    def __init__(self, input_dim, hidden_dim, output_dim, num_layers):
        super().__init__()
        self.num_layers = num_layers
        h = [hidden_dim] * (num_layers - 1)
        self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]))

    def forward(self, x):
        for i, layer in enumerate(self.layers):
            x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
        return x

class SetCriterionHOI(nn.Module):

    def __init__(self, num_obj_classes, num_queries, num_verb_classes, matcher, weight_dict, eos_coef, losses, args):
        super().__init__()

        self.num_obj_classes = num_obj_classes
        self.num_queries = num_queries
        self.num_verb_classes = num_verb_classes
        self.matcher = matcher
        self.weight_dict = weight_dict
        self.eos_coef = eos_coef
        self.losses = losses
        empty_weight = torch.ones(self.num_obj_classes + 1)
        empty_weight[-1] = self.eos_coef
        self.register_buffer('empty_weight', empty_weight)
        device = "cuda" if torch.cuda.is_available() else "cpu"
        self.clip_model, _ = clip.load(args.clip_model, device=device)
        self.alpha = args.alpha

    def loss_obj_labels(self, outputs, targets, indices, num_interactions, log=True):
        assert 'pred_obj_logits' in outputs
        src_logits = outputs['pred_obj_logits']

        idx = self._get_src_permutation_idx(indices)
        target_classes_o = torch.cat([t['obj_labels'][J] for t, (_, J) in zip(targets, indices)])
        target_classes = torch.full(src_logits.shape[:2], self.num_obj_classes,
                                    dtype=torch.int64, device=src_logits.device)
        target_classes[idx] = target_classes_o

        loss_obj_ce = F.cross_entropy(src_logits.transpose(1, 2), target_classes, self.empty_weight)
        losses = {'loss_obj_ce': loss_obj_ce}

        if log:
            losses['obj_class_error'] = 100 - accuracy(src_logits[idx], target_classes_o)[0]
        return losses

    @torch.no_grad()
    def loss_obj_cardinality(self, outputs, targets, indices, num_interactions):
        pred_logits = outputs['pred_obj_logits']
        device = pred_logits.device
        tgt_lengths = torch.as_tensor([len(v['obj_labels']) for v in targets], device=device)
        card_pred = (pred_logits.argmax(-1) != pred_logits.shape[-1] - 1).sum(1)
        card_err = F.l1_loss(card_pred.float(), tgt_lengths.float())
        losses = {'obj_cardinality_error': card_err}
        return losses

    def loss_verb_labels(self, outputs, targets, indices, num_interactions):
        assert 'pred_verb_logits' in outputs
        src_logits = outputs['pred_verb_logits']

        idx = self._get_src_permutation_idx(indices)
        target_classes_o = torch.cat([t['verb_labels'][J] for t, (_, J) in zip(targets, indices)])
        target_classes = torch.zeros_like(src_logits)
        target_classes[idx] = target_classes_o

        src_logits = src_logits.sigmoid()
        loss_verb_ce = self._neg_loss(src_logits, target_classes, weights=None, alpha=self.alpha)
        losses = {'loss_verb_ce': loss_verb_ce}
        return losses

    def loss_hoi_labels(self, outputs, targets, indices, num_interactions, topk=5):
        assert 'pred_hoi_logits' in outputs
        src_logits = outputs['pred_hoi_logits']

        idx = self._get_src_permutation_idx(indices)
        target_classes_o = torch.cat([t['hoi_labels'][J] for t, (_, J) in zip(targets, indices)])
        target_classes = torch.zeros_like(src_logits)
        target_classes[idx] = target_classes_o
        src_logits = _sigmoid(src_logits)
        loss_hoi_ce = self._neg_loss(src_logits, target_classes, weights=None, alpha=self.alpha)
        losses = {'loss_hoi_labels': loss_hoi_ce}

        _, pred = src_logits[idx].topk(topk, 1, True, True)
        acc = 0.0
        for tid, target in enumerate(target_classes_o):
            tgt_idx = torch.where(target == 1)[0]
            if len(tgt_idx) == 0:
                continue
            acc_pred = 0.0
            for tgt_rel in tgt_idx:
                acc_pred += (tgt_rel in pred[tid])
            acc += acc_pred / len(tgt_idx)
        rel_labels_error = 100 - 100 * acc / max(len(target_classes_o), 1)
        losses['hoi_class_error'] = torch.from_numpy(np.array(
            rel_labels_error)).to(src_logits.device).float()
        return losses

    def loss_sub_obj_boxes(self, outputs, targets, indices, num_interactions):
        assert 'pred_sub_boxes' in outputs and 'pred_obj_boxes' in outputs
        idx = self._get_src_permutation_idx(indices)
        src_sub_boxes = outputs['pred_sub_boxes'][idx]
        src_obj_boxes = outputs['pred_obj_boxes'][idx]
        target_sub_boxes = torch.cat([t['sub_boxes'][i] for t, (_, i) in zip(targets, indices)], dim=0)
        target_obj_boxes = torch.cat([t['obj_boxes'][i] for t, (_, i) in zip(targets, indices)], dim=0)

        exist_obj_boxes = (target_obj_boxes != 0).any(dim=1)

        losses = {}
        if src_sub_boxes.shape[0] == 0:
            losses['loss_sub_bbox'] = src_sub_boxes.sum()
            losses['loss_obj_bbox'] = src_obj_boxes.sum()
            losses['loss_sub_giou'] = src_sub_boxes.sum()
            losses['loss_obj_giou'] = src_obj_boxes.sum()
        else:
            loss_sub_bbox = F.l1_loss(src_sub_boxes, target_sub_boxes, reduction='none')
            loss_obj_bbox = F.l1_loss(src_obj_boxes, target_obj_boxes, reduction='none')
            losses['loss_sub_bbox'] = loss_sub_bbox.sum() / num_interactions
            losses['loss_obj_bbox'] = (loss_obj_bbox * exist_obj_boxes.unsqueeze(1)).sum() / (
                    exist_obj_boxes.sum() + 1e-4)
            loss_sub_giou = 1 - torch.diag(generalized_box_iou(box_cxcywh_to_xyxy(src_sub_boxes),
                                                               box_cxcywh_to_xyxy(target_sub_boxes)))
            loss_obj_giou = 1 - torch.diag(generalized_box_iou(box_cxcywh_to_xyxy(src_obj_boxes),
                                                               box_cxcywh_to_xyxy(target_obj_boxes)))
            losses['loss_sub_giou'] = loss_sub_giou.sum() / num_interactions
            losses['loss_obj_giou'] = (loss_obj_giou * exist_obj_boxes).sum() / (exist_obj_boxes.sum() + 1e-4)
        return losses

    def mimic_loss(self, outputs, targets, indices, num_interactions):
        src_feats = outputs['inter_memory']
        src_feats = torch.mean(src_feats, dim=1)

        target_clip_inputs = torch.cat([t['clip_inputs'].unsqueeze(0) for t in targets])
        with torch.no_grad():
            target_clip_feats = self.clip_model.encode_image(target_clip_inputs)
        loss_feat_mimic = F.l1_loss(src_feats, target_clip_feats)
        losses = {'loss_feat_mimic': loss_feat_mimic}
        return losses

    def _neg_loss(self, pred, gt, weights=None, alpha=0.25):
        ''' Modified focal loss. Exactly the same as CornerNet.
          Runs faster and costs a little bit more memory
        '''
        pos_inds = gt.eq(1).float()
        neg_inds = gt.lt(1).float()

        loss = 0

        pos_loss = alpha * torch.log(pred) * torch.pow(1 - pred, 2) * pos_inds
        if weights is not None:
            pos_loss = pos_loss * weights[:-1]

        neg_loss = (1 - alpha) * torch.log(1 - pred) * torch.pow(pred, 2) * neg_inds

        num_pos = pos_inds.float().sum()
        pos_loss = pos_loss.sum()
        neg_loss = neg_loss.sum()

        if num_pos == 0:
            loss = loss - neg_loss
        else:
            loss = loss - (pos_loss + neg_loss) / num_pos
        return loss

    def _get_src_permutation_idx(self, indices):
        # permute predictions following indices
        batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)])
        src_idx = torch.cat([src for (src, _) in indices])
        return batch_idx, src_idx

    def get_loss(self, loss, outputs, targets, indices, num, **kwargs):
        if 'pred_hoi_logits' in outputs.keys():
            loss_map = {
                'hoi_labels': self.loss_hoi_labels,
                'obj_labels': self.loss_obj_labels,
                'sub_obj_boxes': self.loss_sub_obj_boxes,
                'feats_mimic': self.mimic_loss
            }
        else:
            loss_map = {
                'obj_labels': self.loss_obj_labels,
                'obj_cardinality': self.loss_obj_cardinality,
                'verb_labels': self.loss_verb_labels,
                'sub_obj_boxes': self.loss_sub_obj_boxes
            }
        assert loss in loss_map, f'do you really want to compute {loss} loss?'
        return loss_map[loss](outputs, targets, indices, num, **kwargs)

    def forward(self, outputs, targets):
        outputs_without_aux = {k: v for k, v in outputs.items() if k != 'aux_outputs'}

        # Retrieve the matching between the outputs of the last layer and the targets
        indices = self.matcher(outputs_without_aux, targets)

        num_interactions = sum(len(t['hoi_labels']) for t in targets)
        num_interactions = torch.as_tensor([num_interactions], dtype=torch.float,
                                           device=next(iter(outputs.values())).device)
        if is_dist_avail_and_initialized():
            torch.distributed.all_reduce(num_interactions)
        num_interactions = torch.clamp(num_interactions / get_world_size(), min=1).item()

        # Compute all the requested losses
        losses = {}
        for loss in self.losses:
            losses.update(self.get_loss(loss, outputs, targets, indices, num_interactions))

        # In case of auxiliary losses, we repeat this process with the output of each intermediate layer.
        if 'aux_outputs' in outputs:
            for i, aux_outputs in enumerate(outputs['aux_outputs']):
                indices = self.matcher(aux_outputs, targets)
                for loss in self.losses:
                    kwargs = {}
                    if loss == 'obj_labels':
                        # Logging is enabled only for the last layer
                        kwargs = {'log': False}
                    l_dict = self.get_loss(loss, aux_outputs, targets, indices, num_interactions, **kwargs)
                    l_dict = {k + f'_{i}': v for k, v in l_dict.items()}
                    losses.update(l_dict)

        return losses

class PostProcessHOITriplet(nn.Module):

    def __init__(self, args):
        super().__init__()
        self.subject_category_id = args.subject_category_id

    @torch.no_grad()
    def forward(self, outputs, target_sizes):
        out_hoi_logits = outputs['pred_hoi_logits']
        out_obj_logits = outputs['pred_obj_logits']
        out_sub_boxes = outputs['pred_sub_boxes']
        out_obj_boxes = outputs['pred_obj_boxes']

        assert len(out_hoi_logits) == len(target_sizes)
        assert target_sizes.shape[1] == 2

        hoi_scores = out_hoi_logits.sigmoid()
        obj_scores = out_obj_logits.sigmoid()
        obj_labels = F.softmax(out_obj_logits, -1)[..., :-1].max(-1)[1]

        img_h, img_w = target_sizes.unbind(1)
        scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1).to(hoi_scores.device)
        sub_boxes = box_cxcywh_to_xyxy(out_sub_boxes)
        sub_boxes = sub_boxes * scale_fct[:, None, :]
        obj_boxes = box_cxcywh_to_xyxy(out_obj_boxes)
        obj_boxes = obj_boxes * scale_fct[:, None, :]

        results = []
        for index in range(len(hoi_scores)):
            hs, os, ol, sb, ob = hoi_scores[index], obj_scores[index], obj_labels[index], sub_boxes[index], obj_boxes[
                index]
            sl = torch.full_like(ol, self.subject_category_id)
            l = torch.cat((sl, ol))
            b = torch.cat((sb, ob))
            results.append({'labels': l.to('cpu'), 'boxes': b.to('cpu')})

            ids = torch.arange(b.shape[0])

            results[-1].update({'hoi_scores': hs.to('cpu'), 'obj_scores': os.to('cpu'),
                                'sub_ids': ids[:ids.shape[0] // 2], 'obj_ids': ids[ids.shape[0] // 2:]})

        return results

def build_diffhoi_l(args):
    device = torch.device(args.device)

    num_classes = args.num_classes
    device = torch.device(args.device)

    backbone = build_backbone(args)

    transformer = build_deformable_transformer(args)

    try:
        match_unstable_error = args.match_unstable_error
        dn_labelbook_size = args.dn_labelbook_size
    except:
        match_unstable_error = True
        dn_labelbook_size = num_classes

    try:
        dec_pred_class_embed_share = args.dec_pred_class_embed_share
    except:
        dec_pred_class_embed_share = True
    try:
        dec_pred_bbox_embed_share = args.dec_pred_bbox_embed_share
    except:
        dec_pred_bbox_embed_share = True

    model = DiffHOI_L(
        backbone,
        transformer,
        num_classes=num_classes,
        num_queries=args.num_queries,
        aux_loss=True,
        iter_update=True,
        query_dim=4,
        random_refpoints_xy=args.random_refpoints_xy,
        fix_refpoints_hw=args.fix_refpoints_hw,
        num_feature_levels=args.num_feature_levels,
        nheads=args.nheads,
        dec_pred_class_embed_share=dec_pred_class_embed_share,
        dec_pred_bbox_embed_share=dec_pred_bbox_embed_share,
        # two stage
        two_stage_type=args.two_stage_type,
        # box_share
        two_stage_bbox_embed_share=args.two_stage_bbox_embed_share,
        two_stage_class_embed_share=args.two_stage_class_embed_share,
        decoder_sa_type=args.decoder_sa_type,
        num_patterns=args.num_patterns,
        dn_number = args.dn_number if args.use_dn else 0,
        dn_box_noise_scale = args.dn_box_noise_scale,
        dn_label_noise_ratio = args.dn_label_noise_ratio,
        dn_labelbook_size = dn_labelbook_size,
        args=args
    )
    matcher = build_matcher(args)
    weight_dict = {}
    if args.with_clip_label:
        weight_dict['loss_hoi_labels'] = args.hoi_loss_coef
        weight_dict['loss_obj_ce'] = args.obj_loss_coef
    else:
        weight_dict['loss_hoi_labels'] = args.hoi_loss_coef
        weight_dict['loss_obj_ce'] = args.obj_loss_coef

    weight_dict['loss_sub_bbox'] = args.bbox_loss_coef
    weight_dict['loss_obj_bbox'] = args.bbox_loss_coef
    weight_dict['loss_sub_giou'] = args.giou_loss_coef
    weight_dict['loss_obj_giou'] = args.giou_loss_coef
    # if args.with_mimic:
    #     weight_dict['loss_feat_mimic'] = args.mimic_loss_coef
    if args.aux_loss:
        aux_weight_dict = {}
        for i in range(args.dec_layers - 1):
            aux_weight_dict.update({k + f'_{i}': v for k, v in weight_dict.items()})
        weight_dict.update(aux_weight_dict)
    losses = ['hoi_labels', 'obj_labels', 'sub_obj_boxes']
    if args.with_mimic:
        losses.append('feats_mimic')

    criterion = SetCriterionHOI(args.num_obj_classes, args.num_queries, args.num_verb_classes, matcher=matcher,
                                weight_dict=weight_dict, eos_coef=args.eos_coef, losses=losses,
                                args=args)
    criterion.to(device)
    postprocessors = {'hoi': PostProcessHOITriplet(args)}

    return model, criterion, postprocessors


'''bdpy.evals.metrics'''

import warnings

import numpy as np
from scipy.spatial.distance import cdist

def profile_correlation(x, y):
    '''Profile correlation.'''

    sample_axis = 0

    orig_shape = x.shape
    n_sample = orig_shape[sample_axis]

    _x = x.reshape(n_sample, -1)
    _y = y.reshape(n_sample, -1)

    n_feat = _y.shape[1]

    r = np.array(
        [
            np.corrcoef(
                _x[:, j].flatten(),
                _y[:, j].flatten()
            )[0, 1]
            for j in range(n_feat)
        ]
    )

    r = r.reshape((1,) + orig_shape[1:])

    return r

def pattern_correlation(x, y, mean=None, std=None, remove_nan=True):
    '''Pattern correlation.'''

    sample_axis = 0

    orig_shape = x.shape
    n_sample = orig_shape[sample_axis]

    _x = x.reshape(n_sample, -1)
    _y = y.reshape(n_sample, -1)

    if mean is not None and std is not None:
        m = mean.reshape(-1)
        s = std.reshape(-1)

        _x = (_x - m) / s
        _y = (_y - m) / s

    if remove_nan:
        # Remove nan columns based on the decoded features
        nan_cols = np.isnan(_x).any(axis=0) | np.isnan(_y).any(axis=0)
        if nan_cols.any():
            warnings.warn('NaN column removed ({})'.format(np.sum(nan_cols)))
        _x = _x[:, ~nan_cols]
        _y = _y[:, ~nan_cols]

    r = np.array(
        [
            np.corrcoef(
                _x[i, :].flatten(),
                _y[i, :].flatten()
            )[0, 1]
            for i in range(n_sample)
        ]
    )

    return r

def pairwise_identification(pred, true, metric='correlation', remove_nan=True, remove_nan_dist=True, single_trial=False, pred_labels=None, true_labels=None):
    '''Pair-wise identification.'''

    p = pred.reshape(pred.shape[0], -1)
    t = true.reshape(true.shape[0], -1)

    if remove_nan:
        # Remove nan columns based on the decoded features
        nan_cols = np.isnan(p).any(axis=0) | np.isnan(t).any(axis=0)
        if nan_cols.any():
            warnings.warn('NaN column removed ({})'.format(np.sum(nan_cols)))
        p = p[:, ~nan_cols]
        t = t[:, ~nan_cols]

    if single_trial:
        cr = []
        for i in range(p.shape[0]):
            d = 1 - cdist(p[i][np.newaxis], t, metric=metric)
            # label の情報
            ind = np.where(np.array(true_labels) == pred_labels[i])[0][0]

            s = (d - d[0, ind]).flatten()
            if remove_nan_dist and np.isnan(s).any():
                warnings.warn('NaN value detected in the distance matrix ({}).'.format(np.sum(np.isnan(s))))
                s = s[~np.isnan(s)]
            ac = np.sum(s < 0) / (len(s) - 1)
            cr.append(ac)
        cr = np.asarray(cr)
    else:
        d = 1 - cdist(p, t, metric=metric)

        if remove_nan_dist:
            cr = []
            for d_ind in range(d.shape[0]):
                pef = d[d_ind, :] - d[d_ind, d_ind]
                if np.isnan(pef).any():
                    warnings.warn('NaN value detected in the distance matrix ({}).'.format(np.sum(np.isnan(pef))))
                    pef = pef[~np.isnan(pef)] # Remove nan value from the comparison for identification
                pef = np.sum(pef < 0) / (len(pef) - 1)
                cr.append(pef)
            cr = np.asarray(cr)
        else:
            cr = np.sum(d - np.diag(d)[:, np.newaxis] < 0, axis=1) / (d.shape[1] - 1)

    return cr

def remove_nan_value(array, nan_flag=None, return_nan_flag=False):
    '''Helper function:
    Remove columns (units) which contain nan values

    array (numpy.array) ... shape should be [sample x units]
    nan_flag (numpy.array or list) ... if exist, remove columns according to the nan_flag
    return_nan_flag (bool) ... if True, return nan_flag to remove columns of the array.

    '''

    if nan_flag is None:
        nan_flag = np.isnan(array).any(axis=0)
    nan_removed_array = array[:, ~nan_flag]

    if return_nan_flag:
        return nan_removed_array, nan_flag
    else:
        return nan_removed_array


#!/usr/bin/env python
# coding: utf-8

# In[1]:

# -*- coding: utf-8 -*-
"""
Created on Tue Jan 26 16:35:58 2021

@author: Malcor
"""

# Python program to convert text
# file to JSON
import os
import base64
import boto3
from flask import Flask, request
import PIL
import json
import os
import csv
client = boto3.client('s3')
#client = boto3.client(
#    's3',
#    aws_access_key_id=ACCESS_KEY,
#    aws_secret_access_key=SECRET_KEY,
#    aws_session_token=SESSION_TOKEN
#)

UPLOAD_FOLDER = './static/jason/'
app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

def upload_files(file_name, bucket, object_name= None, args= None):
    if object_name is None:
        object_name = file_name
    response = client.upload_file(file_name, bucket, object_name, ExtraArgs = args)
    print(response)

def make_json(csvFilePath, jsonFilePath):
            i = 0
            # create a dictionary
            data = {}

            # Open a csv reader called DictReader
            with open(csvFilePath) as csvf:
                csvReader = csv.DictReader(csvf)

                # Convert each row into a dictionary
                # and add it to data
                for rows in csvReader:

                    # Assuming a column named 'No' to
                    # be the primary key

                    data[i] = rows
                    i += 1
                    #print(i)

            # Open a json writer, and use the json.dumps()
            # function to dump data
#            with open(jsonFilePath, 'w', encoding='utf-8') as jsonf:
                csvf.write(json.dumps(data, indent=4))
# In[3]:

@app.route('/depot', methods=['GET', 'POST'])
def txt_to_json():

    if not os.path.isdir(app.config['UPLOAD_FOLDER']):

        os.makedirs(app.config['UPLOAD_FOLDER'])

    # the file to be converted
    path_origin = request.files['file']
    noms_du_fichier = request.files['file'].filename
    print(noms_du_fichier)
    # resultant dictionary
    dict1 = {}
    L=[]

    path_origin.save(os.path.join(app.config['UPLOAD_FOLDER'], noms_du_fichier))

    prepa_extension = os.path.splitext(noms_du_fichier)
    extension = prepa_extension[1]

    if extension == '.jpg':

        data = {}
        with open('./static/jason/' + noms_du_fichier, mode='rb') as file:
            img = file.read()
        pouet = './photo_test.jpg'
        extension = os.path.splitext(pouet)

        print(extension[-1])
        data['file.filename'] = base64.encodebytes(img).decode('utf-8')
        nom_du_fichier_propre = os.path.splitext(noms_du_fichier)[0]
        out_file = open('./static/jason/' + nom_du_fichier_propre + ".json", "w")
        json.dump(data, out_file, indent = 4)
        out_file.close()
        upload_files('./static/jason/' + nom_du_fichier_propre + ".json", 'fil-rouge-storage', nom_du_fichier_propre + ".json")
        return ('Conversion réussie !')

    if extension == '.txt':

        with open('./static/jason/' + noms_du_fichier) as fh:
            # count variable for employee id creation
            l=0
            for m, line in enumerate(fh):
                if m == 0:
                    L = list(line.strip().split())

                else :
                    fields=len(L)
                    # reading line by line from the text file
                    description = list( line.strip().split())
                    # for output see below
                    print(description)

            		# for automatic creation of id for each employee
                    sno ='key_No'+str(l)

            		# loop variable
                    i = 0
            		# intermediate dictionary
                    dict2 = {}
                    while i<fields:

            				# creating dictionary for each employee
                            dict2[str(L[i])]= description[i]
                            i = i + 1

            		# appending the record of each employee to
            		# the main dictionary
                    dict1[sno]= dict2
                    l = l + 1

        # creating json file
        nom_du_fichier_propre = os.path.splitext(noms_du_fichier)[0]
        out_file = open('./static/jason/' + nom_du_fichier_propre + ".json", "w")
        json.dump(dict1, out_file, indent = 4)
        out_file.close()
        upload_files('./static/jason/' + nom_du_fichier_propre + ".json", 'fil-rouge-storage', nom_du_fichier_propre + ".json")
        return ('Conversion réussie !')

    if extension == '.csv':
                make_json('./static/jason/' + noms_du_fichier, './static/jason/' + nom_du_fichier_propre + ".json")
                upload_files('./static/jason/' + nom_du_fichier_propre + ".json", 'fil-rouge-storage', nom_du_fichier_propre + ".json")
                return ('Conversion réussie !')

if __name__ == "__main__":
        app.run(host='0.0.0.0', port=5000)


import nltk
from nltk.corpus import stopwords
from nltk.corpus import sentiwordnet as swn

class DictionarySentiment(object):
    def __init__(self, trainData, l, r):
        self.tweets, self.labels = trainData['TweetText'], trainData['Sentiment']
        self.data = trainData
        self.l = l
        self.r = r
        self.wnl = nltk.WordNetLemmatizer()
        self.sw = stopwords.words('english')

    def classify(self, message):
        message = message.strip()
        #message = ''.join([char for char in message if char not in '@#"\''])
        message = " ".join([word for word in message.split()
                            if 'http' not in word
                            ])

        sentences = nltk.sent_tokenize(message)
        stokens = [nltk.word_tokenize(sent) for sent in sentences]
        stokens = [sent for sent in stokens if sent not in self.sw]

        taggedlist = []
        for stoken in stokens:
            taggedlist.append(nltk.pos_tag(stoken))

        score_list = []
        for idx, taggedsent in enumerate(taggedlist):
            score_list.append([])
            for idx2, t in enumerate(taggedsent):
                lemmatized = self.wnl.lemmatize(t[0])
                if t[1].startswith('NN'):
                    newtag = 'n'
                elif t[1].startswith('JJ'):
                    newtag = 'a'
                elif t[1].startswith('V'):
                    newtag = 'v'
                elif t[1].startswith('R'):
                    newtag = 'r'
                else:
                    newtag = ''
                if (newtag != ''):
                    synsets = list(swn.senti_synsets(lemmatized, newtag))
                    score = 0
                    if (len(synsets) > 0):
                        for syn in synsets:
                            score += syn.pos_score() - syn.neg_score()
                        score_list[idx].append(score / len(synsets))

        ans = 0

        for score_sent in score_list:
            if (len(score_sent) != 0):
                ans += sum([word_score for word_score in score_sent]) / len(score_sent)

        if ans > self.r:
            return 'positive'
        if ans < self.l:
            return 'negative'
        return 'neutral'

    def predict(self, testData):
        result = []

        for i, row in testData.iterrows():
            result.append(self.classify(row['TweetText']))

        return result


import os
import requests
import datetime
from dateutil.relativedelta import relativedelta

translations = {"Ю": "S", "С": "N", "З": "W", "В": "E", "м": "m", "с": "s"}

def getSimpleParts(source, beginning, ending):
    data = ""
    BrIndex = source.find("
")
    positiveIndex = source.find("")
    # if positive Found
    nullIndex = source.find("label_icon label_small screen_icon")
    if nullIndex != -1:
        data = "null"
    elif positiveIndex != -1:
        startIndex = source.find("")
        endIndex = source.find(ending, startIndex)
        data = source[startIndex + len("") : endIndex]
    # if
 is not found
    elif BrIndex == -1:
        startIndex = source.find(beginning)
        endIndex = source.find(ending, startIndex)
        data = source[startIndex + len(beginning) : endIndex]
    # if
 is found (there won't be other conditins here)
    else:
        BrSpanIndex = source.find("	")
        data = source[BrIndex + len("
") : BrSpanIndex]
    return data

def russianWindNotationsToEnglish(source):
    data = ""
    for str in source:
        isFound = False
        savedValue = ""
        for key in translations:
            if key == str:
                isFound = True
                savedValue = translations[key]
        data += savedValue if isFound else str

    return data

# if it exists, delete it
if os.path.exists("dataset.csv"):
    os.remove("dataset.csv")

# save current Date
currentDate = datetime.date.today()

# open file in append Mode (why not)
with open("dataset.csv", "a") as dataset:
    for day in range(30):
        currentDate -= (
            relativedelta(months=0) if (day == 0) else relativedelta(months=1)
        )

        response = requests.get(
            f"https://www.gismeteo.ru/diary/4980/{currentDate.year}/{currentDate.month}/",
            headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
            },
        )

        startIndex = response.text.find("
")
        endIndex = response.text.find("
", startIndex)
        finalString = response.text[startIndex : endIndex + len("
")]

        currentposition = 0
        appendData = []

        while True:
            toBeAppended = ""

            startCenter = finalString.find('
', currentposition)
            if startCenter == -1:
                break
            endCenter = finalString.find("
", startCenter)
            currentposition = endCenter + len("
")
            finalCenter = finalString[startCenter : endCenter + len("
")]

            arr = []
            incremental = 0

            while finalCenter.find("<td", incremental) != -1:
                beginningIndex = finalCenter.find("<td", incremental)
                endingIndex = finalCenter.find("	", beginningIndex)
                incremental = endingIndex
                string = finalCenter[beginningIndex : endingIndex + len("	")]
                arr.append(string)

            # date
            toBeAppended += f"{currentDate.year}/{currentDate.month}" + ","
            # 0 counter
            toBeAppended += getSimpleParts(arr[0], "", "	") + ","
            # 1D temp
            toBeAppended += (
                getSimpleParts(arr[1], "", "	")
                + ","
            )
            # 2D pressure
            toBeAppended += getSimpleParts(arr[2], "", "	") + ","
            # 5D wind
            toBeAppended += (
                russianWindNotationsToEnglish(
                    getSimpleParts(arr[5], "", "	")
                )
                + ","
            )
            # 6N temp
            toBeAppended += (
                getSimpleParts(arr[6], "", "	")
                + ","
            )
            # 7N pressure
            toBeAppended += getSimpleParts(arr[7], "", "	") + ","
            # 10N wind
            toBeAppended += (
                russianWindNotationsToEnglish(
                    getSimpleParts(arr[10], "", "	")
                )
                + "\n"
            )
            appendData.append(toBeAppended)

        for item in appendData:
            dataset.write(item)


import sympy
import numpy as np
def Hessen(x):
    f = "(x2-x1**2)**2+(1-x1)**2"
    vars = sympy.symbols('x1 x2')
    x1= sympy.symbols('x1')
    x2= sympy.symbols('x2')
    f = sympy.sympify([f])
    H = sympy.zeros(len(vars),len(vars))
    for i ,fi in enumerate(f):
        for j,r in enumerate(vars):
            for k,s in enumerate(vars):
                H[j,k] = sympy.diff(sympy.diff(fi,r), s)
    H = H.subs({x1:x[0],x2:x[1]})
    return np.array(H)
def objfun(x):
    return (x[1,0]-x[0,0]**2)**2+(1-x[0,0])**2
def gradobjfun(x):
    return np.array([[4*x[0,0]**3-4*x[0,0]*x[1,0]+2*x[0,0]-2],[2*x[1,0]-2*x[0,0]**2]])
def phi(alpha):
    Dk = np.linalg.inv(Hessen(x))
    gk = gradobjfun(x)
    dk = -1.0*np.dot(Dk,gk)
    return objfun(x+alpha*dk)
def GoldenSearch(x):
    epsilon = 1e-5
    tua = 0.618;ak = 0;bk = 2
    while True:
        if bk - ak < epsilon:
            alpha = (ak+bk)/2
            return alpha
        else:
            landa = ak + (1-tua)*(bk - ak)
            miu = ak + tua*(bk - ak)
            flanda = phi(landa)
            fmiu = phi(miu)
            if flanda>fmiu:
                ak = landa
            else:
                bk = miu
def Quasi_Newton_methods(x,epsilon):
    k = 0
    Dk = np.linalg.inv(Hessen(x))
    x0 = x
    rho = 0.5
    while True:
        gk = gradobjfun(x0)
        if np.linalg.norm(gk) < epsilon:
            return np.hstack((x0[0],x0[1])),objfun(x0),k
        dk = -1.0*np.dot(Dk,gk)
        alpha = GoldenSearch(x0)
        x = x0 + alpha*dk
        sk = x - x0
        yk = gradobjfun(x) - gk
        if np.dot(sk.T,yk)[0,0] > 0:
            ak = np.dot(sk,sk.T)
            bk = np.dot(sk.T,yk)[0,0]
            ck = np.dot(np.dot(np.dot(Dk,yk),yk.T),Dk)
            dk = np.dot(np.dot(yk.T,Dk),yk)[0,0]
            Dk = Dk + ak/bk - ck/dk
        x0 = x
        k += 1

if __name__ == "__main__":
    epsilon = float(input("阀值："))
    x = np.array([[3],[2]])
    x_min, f_min, k = Quasi_Newton_methods(x,epsilon)
    print("x_min:",x_min)
    print("f_min:",f_min)
    print("迭代数:",k)


import math
import random
import numpy as np
import pandas as pd
import nltk

def split_to_sentences(data):
    """
    Split data by line break "\n"

    Args:
        data: str

    Returns:
        A list of sentences
    """
    # Splitting data by line break
    sentences = data.split('\n')

    sentences = [s.strip() for s in sentences] # removing leading and trailing spaces from each sentence
    sentences = [s for s in sentences if len(s) > 0] # dropping sentences if they are empty strings

    return sentences

def tokenize_sentences(sentences):
    """
    Tokenize sentences into tokens (words)

    Args:
        sentences: List of strings

    Returns:
        List of lists of tokens
    """

    # Initializing the list of lists of tokenized sentences
    tokenized_sentences = []

    # Going through each sentence
    for sentence in sentences:
        sentence = sentence.lower() # converting to lowercase letters
        tokenized = nltk.word_tokenize(sentence) # converting into a list of words
        tokenized_sentences.append(tokenized) # appending the list of words to the list of lists

    return tokenized_sentences

def get_tokenized_data(data):
    """
    Make a list of tokenized sentences

    Args:
        data: String

    Returns:
        List of lists of tokens
    """

    # Getting the sentences by splitting up the data
    sentences = split_to_sentences(data)

    # Getting the list of lists of tokens by tokenizing the sentences
    tokenized_sentences = tokenize_sentences(sentences)

    return tokenized_sentences

def count_words(tokenized_sentences):
    """
    Count the number of word appearence in the tokenized sentences

    Args:
        tokenized_sentences: List of lists of strings

    Returns:
        dict that maps word (str) to the frequency (int)
    """

    word_counts = {}

    # Looping through each sentence
    for sentence in tokenized_sentences:
        # Going through each token in the sentence
        for token in sentence:
            # If the token is not in the dictionary yet, setting the count to 1
            if token not in word_counts:
                word_counts[token] = 1

            # If the token is already in the dictionary, incrementing the count by 1
            else:
                word_counts[token] += 1

    return word_counts

def get_words_with_nplus_frequency(tokenized_sentences, count_threshold):
    """
    Find the words that appear N times or more

    Args:
        tokenized_sentences: List of lists of sentences
        count_threshold: minimum number of occurrences for a word to be in the closed vocabulary.

    Returns:
        List of words that appear N times or more
    """
    # Initializing an empty list to contain the words that appear at least 'count_threshold' times.
    closed_vocab = []

    # Getting the word counts of the tokenized sentences
    word_counts = count_words(tokenized_sentences)

    # Looping through word-count key-value pairs
    for word, cnt in word_counts.items():
        if cnt >= count_threshold: # checking that the count is equal or more than the threshold
            closed_vocab.append(word) # appending the word to the list

    return closed_vocab

def replace_oov_words_by_unk(tokenized_sentences, vocabulary, unknown_token="<unk>"):
    """
    Replace words not in the given vocabulary with '<unk>' token.

    Args:
        tokenized_sentences: List of lists of strings
        vocabulary: List of strings that we will use
        unknown_token: A string representing unknown (out-of-vocabulary) words

    Returns:
        List of lists of strings, with words not in the vocabulary replaced
    """

    # Place vocabulary into a set for faster search
    vocabulary = set(vocabulary)

    # Initializing a list that will hold the sentences after less frequent words are replaced by the unknown token
    replaced_tokenized_sentences = []

    # Going through each sentence
    for sentence in tokenized_sentences:

        # Initializing the list that will contain a single sentence with "unknown_token" replacements
        replaced_sentence = []

        # Going through each token in the sentence
        for token in sentence:
            if token in vocabulary: # checking if the token is in the closed vocabulary
                replaced_sentence.append(token) # appending the word to the replaced sentence
            else:
                replaced_sentence.append(unknown_token) # otherwise, appending the unknown token

        # Appending the list of tokens to the list of lists
        replaced_tokenized_sentences.append(replaced_sentence)

    return replaced_tokenized_sentences

def preprocess_data(train_data, test_data, count_threshold):
    """
    Preprocess data, i.e.,
        - Find tokens that appear at least N times in the training data.
        - Replace tokens that appear less than N times by "<unk>" both for training and test data.
    Args:
        train_data, test_data: List of lists of strings.
        count_threshold: Words whose count is less than this are
                      treated as unknown.

    Returns:
        Tuple of
        - training data with low frequent words replaced by "<unk>"
        - test data with low frequent words replaced by "<unk>"
        - vocabulary of words that appear n times or more in the training data
    """

    # Getting the closed vocabulary using the training data
    vocabulary = get_words_with_nplus_frequency(train_data, count_threshold)

    # For the training data, replacing less common words with "<unk>"
    train_data_replaced = replace_oov_words_by_unk(train_data, vocabulary, unknown_token="<unk>")

    # For the testing data, replacing less common words with "<unk>"
    test_data_replaced = replace_oov_words_by_unk(test_data, vocabulary, unknown_token="<unk>")

    return train_data_replaced, test_data_replaced, vocabulary

def count_n_grams(data, n, start_token='<s>', end_token = '<e>'):
    """
    Count all n-grams in the data

    Args:
        data: List of lists of tokens
        n: number of tokens in a sequence for the n-gram model

    Returns:
        A dictionary that maps a tuple of n-words to its frequency
    """

    # Initializing dictionary of n-grams and their counts
    n_grams = {}

    # Going through each sentence in the data
    for sentence in data:

        # Prepending start token n times, and  appending <e> one time (kind of padding for this model)
        sentence = [start_token] * n + sentence + [end_token]

        # Convert list to tuple so that the sequence of words can be used as a key in the dictionary
        sentence = tuple(sentence)

        for i in range(len(sentence) - n + 1): # looping through tokens to get the n-grams

            # Getting the n-gram from i to i+n
            n_gram = sentence[i:i+n]

            # Checking if the n-gram is in the dictionary
            if n_gram in n_grams:
                n_grams[n_gram] += 1 # incrementing the count for this n-gram
            else:
                n_grams[n_gram] = 1 # initializing this n-gram count to 1

    return n_grams

def estimate_probability(word, previous_n_gram, n_gram_counts, n_plus1_gram_counts, vocabulary_size, k=1.0):
    """
    Estimate the probabilities of a next word using the n-gram counts with k-smoothing

    Args:
        word: next word
        previous_n_gram: A sequence of words of length n
        n_gram_counts: Dictionary of counts of n-grams
        n_plus1_gram_counts: Dictionary of counts of (n+1)-grams
        vocabulary_size: number of words in the vocabulary
        k: positive constant, smoothing parameter

    Returns:
        A probability
    """
    # Converting list to tuple to use it as a dictionary key
    previous_n_gram = tuple(previous_n_gram)

    # If the previous n-gram exists in the dictionary of n-gram counts, we get its count; otherwise, we set the count to zero
    previous_n_gram_count = n_gram_counts.get(previous_n_gram, 0)

    # Calculating the denominator using the count of the previous n gram and applying k-smoothing
    denominator = previous_n_gram_count + k * vocabulary_size

    # Defining n plus 1 gram as the previous n-gram plus the current word as a tuple
    n_plus1_gram = previous_n_gram + (word,)

    # Setting the count to the count in the dictionary; otherwise, 0 if not in the dictionary
    n_plus1_gram_count = n_plus1_gram_counts.get(n_plus1_gram, 0)

    # Defining the numerator using the count of the n-gram plus current word and applying smoothing
    numerator = n_plus1_gram_count + k

    # Calculating the probability as the numerator divided by denominator
    probability = numerator / denominator

    return probability

def estimate_probabilities(previous_n_gram, n_gram_counts, n_plus1_gram_counts, vocabulary, k=1.0):
    """
    Estimate the probabilities of next words using the n-gram counts with k-smoothing

    Args:
        previous_n_gram: A sequence of words of length n
        n_gram_counts: Dictionary of counts of n-grams
        n_plus1_gram_counts: Dictionary of counts of (n+1)-grams
        vocabulary: List of words
        k: positive constant, smoothing parameter

    Returns:
        A dictionary mapping from next words to the probability.
    """

    # Converting list to tuple to use it as a dictionary key
    previous_n_gram = tuple(previous_n_gram)

    # Adding <e> <unk> to the vocabulary
    vocabulary = vocabulary + ["<e>", "<unk>"]
    vocabulary_size = len(vocabulary)

    probabilities = {} # initializing the dictionary where we will store the word with each probability
    for word in vocabulary:
        probability = estimate_probability(word, previous_n_gram, n_gram_counts, n_plus1_gram_counts, vocabulary_size, k=k) # estimating probability of word
        probabilities[word] = probability # the value of the word (key) will be the probability itself

    return probabilities

def suggest_a_word(previous_tokens, n_gram_counts, n_plus1_gram_counts, vocabulary, k=1.0, start_with=None):
    """
    Get suggestion for the next word

    Args:
        previous_tokens: The sentence you input where each token is a word. Must have length > n
        n_gram_counts: Dictionary of counts of n-grams
        n_plus1_gram_counts: Dictionary of counts of (n+1)-grams
        vocabulary: List of words
        k: positive constant, smoothing parameter
        start_with: If not None, specifies the first few letters of the next word

    Returns:
        A tuple of
          - string of the most likely next word
          - corresponding probability
    """

    # Length of previous words
    n = len(list(n_gram_counts.keys())[0])

    # From the words that the user already typed, getting the most recent 'n' words as the previous n-gram
    previous_n_gram = previous_tokens[-n:]

    # Estimating the probabilities that each word in the vocabulary is the next word
    probabilities = estimate_probabilities(previous_n_gram, n_gram_counts, n_plus1_gram_counts, vocabulary, k=k)

    # Initializing suggested word to None; this will be set to the word with highest probability
    suggestion = None

    # Initializing the highest word probability to 0; this will be set to the highest probabilityof all words to be suggested
    max_prob = 0

    # Looping through every word and probability
    for word, prob in probabilities.items():

        # If the optional start_with string is set
        if start_with != None:
            if word.startswith(start_with) is False: # checking if the beginning of the word does not match with the letters in 'start_with'
                continue # if they don't match, skip this word and move onto the next word

        if prob > max_prob: # checking if this word's probability is greater than the current maximum probability
            suggestion = word # the suggestion becomes the word itself
            max_prob = prob # saving the new maximum probability

    return suggestion, max_prob

def get_suggestions(previous_tokens, n_gram_counts_list, vocabulary, k=1.0, start_with=None):
    """
    Get suggestions from the input sequence

    Args:
        previous_tokens: The sentence you input where each token is a word. Must have length > n
        n_gram_counts_list: List of dictionaries with n-gram counts
        vocabulary: List of words
        k: positive constant, smoothing parameter
        start_with: If not None, specifies the first few letters of the next word

    Returns:
        A list of suggestions in the form of a tuple for each entry (suggestion, probability)
    """
    model_counts = len(n_gram_counts_list)
    suggestions = []
    for i in range(model_counts-1):
        n_gram_counts = n_gram_counts_list[i] # getting n-gram counts of each model
        n_plus1_gram_counts = n_gram_counts_list[i+1] # getting n-gram plus 1 counts of each model

        suggestion = suggest_a_word(previous_tokens, n_gram_counts, n_plus1_gram_counts, vocabulary, k=k, start_with=start_with) # getting suggestion
        suggestions.append(suggestion) # appending to suggestions
    return suggestions


############################################################
#
# craft_poisons_fc.py
# Feature Collision Attack
# June 2020
#
# Reference: A. Shafahi, W. R. Huang, M. Najibi, O. Suciu, C. Studer,
#     T. Dumitras, and T. Goldstein. Poison frogs! targeted clean-label
#     poisoning attacks on neural networks.
#     In Advances in Neural Information Processing Systems, pages 6103-6113, 2018.
############################################################
import argparse
import copy
import os
import pickle
import sys

import torch
import torchvision
import torchvision.transforms as transforms

sys.path.append(os.path.realpath("."))
from learning_module import get_transform
from learning_module import (
    TINYIMAGENET_ROOT,
    to_log_file,
    now,
    normalize_data,
    un_normalize_data,
    load_model_from_checkpoint,
)
from tinyimagenet_module import TinyImageNet

def main(args):
    """Main function to generate the FC poisons
    inputs:
        args:           Argparse object
    reutrn:
        void
    """
    print(now(), "craft_poisons_fc.py main() running.")

    craft_log = "craft_log.txt"
    to_log_file(args, args.output, craft_log)

    # Set device
    device = "cuda" if torch.cuda.is_available() else "cpu"

    ####################################################
    #               Dataset
    if args.dataset.lower() == "cifar10":
        transform_test = get_transform(args.normalize, False)
        trainset = torchvision.datasets.CIFAR10(
            root="./data", train=True, download=True, transform=transform_test
        )
        testset = torchvision.datasets.CIFAR10(
            root="./data", train=False, download=True, transform=transform_test
        )
    elif args.dataset.lower() == "tinyimagenet_first":
        transform_test = get_transform(args.normalize, False, dataset=args.dataset)
        trainset = TinyImageNet(
            TINYIMAGENET_ROOT,
            split="train",
            transform=transform_test,
            classes="firsthalf",
        )
        testset = TinyImageNet(
            TINYIMAGENET_ROOT,
            split="val",
            transform=transform_test,
            classes="firsthalf",
        )
    elif args.dataset.lower() == "tinyimagenet_last":
        transform_test = get_transform(args.normalize, False, dataset=args.dataset)
        trainset = TinyImageNet(
            TINYIMAGENET_ROOT,
            split="train",
            transform=transform_test,
            classes="lasthalf",
        )
        testset = TinyImageNet(
            TINYIMAGENET_ROOT,
            split="val",
            transform=transform_test,
            classes="lasthalf",
        )
    elif args.dataset.lower() == "tinyimagenet_all":
        transform_test = get_transform(args.normalize, False, dataset=args.dataset)
        trainset = TinyImageNet(
            TINYIMAGENET_ROOT,
            split="train",
            transform=transform_test,
            classes="all",
        )
        testset = TinyImageNet(
            TINYIMAGENET_ROOT,
            split="val",
            transform=transform_test,
            classes="all",
        )
    else:
        print("Dataset not yet implemented. Exiting from craft_poisons_fc.py.")
        sys.exit()
    ###################################################

    ####################################################
    #          Craft and insert poison image
    feature_extractors = []
    for i in range(len(args.model)):
        feature_extractors.append(
            load_model_from_checkpoint(
                args.model[i], args.model_path[i], args.pretrain_dataset
            )
        )

    for i in range(len(feature_extractors)):
        for param in feature_extractors[i].parameters():
            param.requires_grad = False
        feature_extractors[i].eval()
        feature_extractors[i] = feature_extractors[i].to(device)

    with open(args.poison_setups, "rb") as handle:
        setup_dicts = pickle.load(handle)
    setup = setup_dicts[args.setup_idx]

    target_img_idx = (
        setup["target index"] if args.target_img_idx is None else args.target_img_idx
    )
    base_indices = (
        setup["base indices"] if args.base_indices is None else args.base_indices
    )
    # Craft poisons
    poison_iterations = args.crafting_iters
    poison_perturbation_norms = []

    # get single target
    target_img, target_label = testset[target_img_idx]

    # get multiple bases
    base_imgs = torch.stack([trainset[i][0] for i in base_indices])
    base_labels = torch.LongTensor([trainset[i][1] for i in base_indices])

    # log target and base details
    to_log_file("base indices: " + str(base_indices), args.output, craft_log)
    to_log_file("base labels: " + str(base_labels), args.output, craft_log)
    to_log_file("target_label: " + str(target_label), args.output, craft_log)
    to_log_file("target_index: " + str(target_img_idx), args.output, craft_log)

    # fill list of tuples of poison images and labels
    poison_tuples = []
    target_img = (
        un_normalize_data(target_img, args.dataset) if args.normalize else target_img
    )
    beta = 4.0 if args.normalize else 0.1

    base_tuples = list(zip(base_imgs, base_labels))
    for base_img, label in base_tuples:
        # unnormalize the images for optimization
        b_unnormalized = (
            un_normalize_data(base_img, args.dataset) if args.normalize else base_img
        )
        objective_vals = [10e8]
        step_size = args.step_size

        # watermarking
        x = copy.deepcopy(b_unnormalized)
        x = args.watermark_coeff * target_img + (1 - args.watermark_coeff) * x

        # feature collision optimization
        done_with_fc = False
        i = 0
        while not done_with_fc and i < poison_iterations:
            x.requires_grad = True
            if args.normalize:
                mini_batch = torch.stack(
                    [
                        normalize_data(x, args.dataset),
                        normalize_data(target_img, args.dataset),
                    ]
                ).to(device)
            else:
                mini_batch = torch.stack([x, target_img]).to(device)

            loss = 0
            for feature_extractor in feature_extractors:
                feats = feature_extractor.penultimate(mini_batch)
                loss += torch.norm(feats[0, :] - feats[1, :]) ** 2
            grad = torch.autograd.grad(loss, [x])[0]
            x_hat = x.detach() - step_size * grad.detach()
            if not args.l2:
                pert = (x_hat - b_unnormalized).clamp(-args.epsilon, args.epsilon)
                x_new = b_unnormalized + pert
                x_new = x_new.clamp(0, 1)
                obj = loss

            else:
                x_new = (
                    x_hat.detach() + step_size * beta * b_unnormalized.detach()
                ) / (1 + step_size * beta)
                x_new = x_new.clamp(0, 1)
                obj = beta * torch.norm(x_new - b_unnormalized) ** 2 + loss

            if obj > objective_vals[-1]:
                step_size *= 0.2

            else:
                if torch.norm(x - x_new) / torch.norm(x) < 1e-5:
                    done_with_fc = True
                x = copy.deepcopy(x_new)
                objective_vals.append(obj)
            i += 1
        poison_tuples.append((transforms.ToPILImage()(x), label.item()))
        poison_perturbation_norms.append(
            torch.max(torch.abs(x - b_unnormalized)).item()
        )
        x.requires_grad = False
    ####################################################

    ####################################################
    #        Save Poisons
    print(now(), "Saving poisons...")
    if not os.path.isdir(args.poisons_path):
        os.makedirs(args.poisons_path)
    with open(os.path.join(args.poisons_path, "poisons.pickle"), "wb") as handle:
        pickle.dump(poison_tuples, handle, protocol=pickle.HIGHEST_PROTOCOL)
    with open(
        os.path.join(args.poisons_path, "perturbation_norms.pickle"), "wb"
    ) as handle:
        pickle.dump(poison_perturbation_norms, handle, protocol=pickle.HIGHEST_PROTOCOL)
    with open(os.path.join(args.poisons_path, "base_indices.pickle"), "wb") as handle:
        pickle.dump(base_indices, handle, protocol=pickle.HIGHEST_PROTOCOL)
    with open(os.path.join(args.poisons_path, "target.pickle"), "wb") as handle:
        pickle.dump(
            (transforms.ToPILImage()(target_img), target_label),
            handle,
            protocol=pickle.HIGHEST_PROTOCOL,
        )
    to_log_file("poisons saved.", args.output, craft_log)
    ####################################################

    print(now(), "craft_poisons_fc.py done.")
    return

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Feature Collision Poison Attack")
    parser.add_argument(
        "--model", default=["resnet18"], nargs="+", type=str, help="model for training"
    )
    parser.add_argument("--dataset", default="CIFAR10", type=str, help="dataset")
    parser.add_argument(
        "--pretrain_dataset", default="CIFAR100", type=str, help="dataset"
    )
    parser.add_argument(
        "--model_path",
        default=["pretrained_models/ResNet18_CIFAR100_A.pth"],
        nargs="+",
        type=str,
        help="Model path",
    )
    parser.add_argument("--normalize", dest="normalize", action="store_true")
    parser.add_argument("--no-normalize", dest="normalize", action="store_false")
    parser.set_defaults(normalize=True)
    parser.add_argument(
        "--output", default="output_default", type=str, help="output subdirectory"
    )
    parser.add_argument(
        "--crafting_iters",
        default=120,
        type=int,
        help="How many iterations when crafting poison?",
    )
    parser.add_argument(
        "--watermark_coeff", default=0.3, type=float, help="Opacity of watermark."
    )
    parser.add_argument(
        "--step_size", default=0.001, type=float, help="Step size in optimization"
    )
    parser.add_argument(
        "--l2", action="store_true", help="use l-2 constrained version of attack."
    )
    parser.add_argument(
        "--epsilon", default=8 / 255, type=float, help="poison perturbation allowance"
    )
    parser.add_argument(
        "--poisons_path",
        default="poison_examples/fc_poisons",
        type=str,
        help="Where to save the poisons?",
    )
    parser.add_argument(
        "--poison_setups",
        type=str,
        default="poison_setups_transfer_learning.pickle",
        help="poison setup pickle file",
    )
    parser.add_argument("--setup_idx", type=int, default=0, help="Which setup to use")
    parser.add_argument(
        "--target_img_idx",
        default=None,
        type=int,
        help="Index of the target image in the claen set.",
    )
    parser.add_argument(
        "--base_indices", nargs="+", default=None, type=int, help="which base images"
    )

    args = parser.parse_args()

    main(args)


#!/usr/bin/env Python
from sys import argv
import itertools
import math

with open(argv[1],'r') as in_file:
  in_data = in_file.read().splitlines()
in_file.closed

n =int(in_data[0])

permute = []
for i in range(n):
  permute.append(i+1)

with open("enumerating_gene_orders_output.txt",'w') as out_file:
  out_file.write(str(math.factorial(n))+"\n")
  for i in set(itertools.permutations(permute)):
    for j in i:
      out_file.write(str(j)+" ")
    out_file.write("\n")
out_file.closed


from django.shortcuts import redirect, render
from .forms import Registeration

def registeration(request):
    if request.method == 'POST':
        form = Registeration(request.POST)
        if form.is_valid():
            form.save()
            print('New user has been created')
            return redirect('login')
    else:
        form = Registeration()
    context = {
        'form':form
        }
    return render(request, 'users/registeration.html', context)


from django.shortcuts import render
import json
from rest_framework_jwt.utils import jwt_decode_handler
from django.views.decorators.csrf import csrf_exempt
from django.http import JsonResponse, HttpResponse
from .database.orderOFdiagram import orderDate

from collections import Counter
import json
import datetime
from .database import analiticsQuery,TopClient
from .consts import  getinterval,GetJsonDSOC,getmaxel, CountSumOfDict
# Create your views here.

@csrf_exempt
def TopClients(request):

    body = json.loads(request.body)
    userID = jwt_decode_handler(body["access"])['sub']
    date_start = body['datastart']
    date_end = body['dataend']
    statistic = TopClient.GetTopClient(analiticsQuery.GetCompany(userID),date_start,date_end)
    data = []
    for i in statistic:
        block_data = {
            'client': i[0],
            'count': i[1],
            'sum': i[2],
        }
        data.append(block_data)

    return JsonResponse(
        {'count': len(statistic),
            'data': data },
        headers={
            "Access-Control-Allow-Origin": "*"
        }
    )

@csrf_exempt
def analytics(request):
    body = json.loads(request.body)
    userID = jwt_decode_handler(body["access"])

    date_start = body['datastart']
    date_end = body['dataend']

    statistic = analiticsQuery.getAnaliz(analiticsQuery.GetCompany(userID['sub']),date_start,date_end)
    data = []
    all_sum =0
    for i in statistic:
        block_data = {
            'name': i[0],
            'total': i[1],
            'avg':float(i[2]),
            "workload":  float(i[3])
        }
        data.append(block_data)
        all_sum+=i[1]

    return JsonResponse(
    {
        'sum':all_sum,
        'count': len(statistic),
        'data': data
    },
        headers={
            "Access-Control-Allow-Origin": "*"
        }
    )

@csrf_exempt
def diagram(request):
    body = json.loads(request.body)
    userID = jwt_decode_handler(body["access"])['sub']
    print(analiticsQuery.GetCompany(userID))
    date_start = body['datastart']
    date_end = body['dataend']

    dates = getinterval(date_start,date_end)
    statistic = orderDate(analiticsQuery.GetCompany(userID),date_start,date_end)
    station_list = []
    for j in statistic:
        if not (j[0] in station_list):
            station_list.append(j[0])
    print(station_list)

    i = 0

    masvalue = [[]* 1 for _ in range((len(station_list)))]
    dailyIncomeMas  = [[] * 1 for _ in range((len(station_list)))]
    print(masvalue)
    negr = []
    hui = []
    formated_dates= []
    for date in dates:

        formated_dates.append( date[5:])
        for stat in statistic:

            if str(stat[1].date()) == date:
                hui.append(stat[0])

                negr.append({
                    f'{stat[0]}': stat[2]
                })
                print(stat)
                i += 1
        print(negr)
        if hui == []:
            j = 0
            for x in station_list:
                masvalue[j].append(0)
                dailyIncomeMas[j].append(0)
                j += 1
        else:
            j = 0
            sumOrders = CountSumOfDict(station_list,negr)
            for x in station_list:
                carwarshIndex = station_list.index(x)
                counts = Counter(hui)
                masvalue[j].append(counts[x])
                dailyIncomeMas[j].append(sumOrders[carwarshIndex][x])
                j += 1

        hui = []
        negr=[]
    print(dailyIncomeMas)
    print(getmaxel(masvalue))
    #orders.append( GetJsonDSOC(date,hui, station_list))

    return JsonResponse(
        { 'stationList': station_list,
            'dates': formated_dates,
            'values': masvalue,
            'maxEl': getmaxel(masvalue),
            'dailyIncomes' : dailyIncomeMas,
            'maxIncome': getmaxel(dailyIncomeMas)
          },
        headers={
            "Access-Control-Allow-Origin": "*"
        }
    )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Multithreaded simple HTTP directory server.
#
# The files can optionally be aes-256-cbc encrypted in a way that's compatible with:
#   openssl enc -aes-256-cbc -d -in encrypted.data -out decrypted.data

import argparse
import hashlib
import os
import sys
from threading import Thread
from socketserver import ThreadingMixIn
from http.server import HTTPServer, SimpleHTTPRequestHandler
from Crypto.Cipher import AES

from malcolm_utils import str2bool, eprint, EVP_KEY_SIZE, PKCS5_SALT_LEN, OPENSSL_ENC_MAGIC, EVP_BytesToKey

###################################################################################################
args = None
debug = False
script_name = os.path.basename(__file__)
script_path = os.path.dirname(os.path.realpath(__file__))
orig_path = os.getcwd()

###################################################################################################
#
class HTTPHandler(SimpleHTTPRequestHandler):
    # return full path based on server base path and requested path
    def translate_path(self, path):
        path = SimpleHTTPRequestHandler.translate_path(self, path)
        relpath = os.path.relpath(path, os.getcwd())
        fullpath = os.path.join(self.server.base_path, relpath)
        return fullpath

    # override do_GET so that files are encrypted, if requested
    def do_GET(self):
        global debug
        global args

        fullpath = self.translate_path(self.path)

        if (not args.encrypt) or os.path.isdir(fullpath):
            # unencrypted, just use default implementation
            SimpleHTTPRequestHandler.do_GET(self)

        else:
            # encrypt file transfers
            if os.path.isfile(fullpath) or os.path.islink(fullpath):
                self.send_response(200)
                self.send_header('Content-type', 'application/octet-stream')
                self.send_header('Content-Disposition', f'attachment; filename={os.path.basename(fullpath)}.encrypted')
                self.end_headers()
                salt = os.urandom(PKCS5_SALT_LEN)
                key, iv = EVP_BytesToKey(EVP_KEY_SIZE, AES.block_size, hashlib.sha256, salt, args.key.encode('utf-8'))
                cipher = AES.new(key, AES.MODE_CBC, iv)
                encrypted = b""
                encrypted += OPENSSL_ENC_MAGIC
                encrypted += salt
                self.wfile.write(encrypted)
                with open(fullpath, 'rb') as f:
                    padding = b''
                    while True:
                        chunk = f.read(cipher.block_size)
                        if len(chunk) < cipher.block_size:
                            remaining = cipher.block_size - len(chunk)
                            padding = bytes([remaining] * remaining)
                        self.wfile.write(cipher.encrypt(chunk + padding))
                        if padding:
                            break

            else:
                self.send_error(404, "Not Found")

###################################################################################################
#
class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
    def __init__(self, base_path, server_address, RequestHandlerClass=HTTPHandler):
        self.base_path = base_path
        HTTPServer.__init__(self, server_address, RequestHandlerClass)

###################################################################################################
#
def serve_on_port(path: str, port: int):
    server = ThreadingHTTPServer(path, ("", port))
    print(f"serving {path} at port {port}")
    server.serve_forever()

###################################################################################################
# main
def main():
    global args
    global debug
    global orig_path

    defaultDebug = os.getenv('EXTRACTED_FILE_HTTP_SERVER_DEBUG', 'false')
    defaultEncrypt = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ENCRYPT', 'false')
    defaultPort = int(os.getenv('EXTRACTED_FILE_HTTP_SERVER_PORT', 8440))
    defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'quarantined')
    defaultDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_PATH', orig_path)

    parser = argparse.ArgumentParser(
        description=script_name, add_help=False, usage='{} <arguments>'.format(script_name)
    )
    parser.add_argument(
        '-v',
        '--verbose',
        dest='debug',
        type=str2bool,
        nargs='?',
        const=True,
        default=defaultDebug,
        metavar='true|false',
        help=f"Verbose/debug output ({defaultDebug})",
    )
    parser.add_argument(
        '-p',
        '--port',
        dest='port',
        help=f"Server port ({defaultPort})",
        metavar='<port>',
        type=int,
        default=defaultPort,
    )
    parser.add_argument(
        '-d',
        '--directory',
        dest='serveDir',
        help=f'Directory to serve ({defaultDir})',
        metavar='<directory>',
        type=str,
        default=defaultDir,
    )
    parser.add_argument(
        '-e',
        '--encrypt',
        dest='encrypt',
        type=str2bool,
        nargs='?',
        const=True,
        default=defaultEncrypt,
        metavar='true|false',
        help=f"Encrypt files with aes-256-cbc ({defaultEncrypt})",
    )
    parser.add_argument(
        '-k',
        '--key',
        dest='key',
        help="File encryption key",
        metavar='<str>',
        type=str,
        default=defaultKey,
    )
    try:
        parser.error = parser.exit
        args = parser.parse_args()
    except SystemExit:
        parser.print_help()
        exit(2)

    debug = args.debug
    if debug:
        eprint(os.path.join(script_path, script_name))
        eprint("Arguments: {}".format(sys.argv[1:]))
        eprint("Arguments: {}".format(args))
    else:
        sys.tracebacklimit = 0

    Thread(target=serve_on_port, args=[args.serveDir, args.port]).start()

###################################################################################################
if __name__ == '__main__':
    main()


import logging
from telethon import sync, TelegramClient, events
from telethon.tl.types import InputPeerChannel
from telethon.tl.types import InputPeerUser
from telethon.tl.functions.channels import InviteToChannelRequest
from telethon.errors.rpcerrorlist import PeerFloodError, UserPrivacyRestrictedError, FloodWaitError
import time
import traceback
import datetime
import os
import json

def get_group_by_id(groups, group_id):
    for group in groups:
        if (group_id == int(group['group_id'])):
            return group
    return None

root_path = os.path.dirname(os.path.abspath(__file__))
print(root_path)

start_time = datetime.datetime.now()
logging.basicConfig(level=logging.WARNING)

with open('config.json', 'r', encoding='utf-8') as f:
    config = json.loads(f.read())

account = config['account']
print("Total account: " + str(len(account)))
folder_session = 'session/'

# group target
group_target_id = int(config['group_target'])
# group source
group_source_id = int(config['group_source'])
# date_online_from
from_date_active = '19700101'
if 'from_date_active' in config:
    from_date_active = config['from_date_active']

# list client
clients = []

api_id = account['api_id']
api_hash = account['api_hash']
phone = account['phone']

client = TelegramClient(folder_session + phone, api_id, api_hash)

client.connect()

if client.is_user_authorized():
    print(phone + ' login success')
    clients.append({
        'phone': phone,
        'client': client
    })
else:
    print(phone + ' login fail')

filter_clients = []

for my_client in clients:
    phone = my_client['phone']
    path_group = root_path + '/data/group/' + phone + '.json'
    if os.path.isfile(path_group):

        with open(path_group, 'r', encoding='utf-8') as f:
            groups = json.loads(f.read())

        current_target_group = get_group_by_id(groups, group_target_id)

        if current_target_group:
            group_access_hash = int(current_target_group['access_hash'])
            target_group_entity = InputPeerChannel(
                group_target_id, group_access_hash)

            path_group_user = root_path + '/data/user/' + \
                phone + "_" + str(group_source_id) + '.json'
            if os.path.isfile(path_group_user):
                # add target_group_entity key value
                my_client['target_group_entity'] = target_group_entity
                with open(path_group_user, encoding='utf-8') as f:
                    my_client['users'] = json.loads(f.read())

                filter_clients.append(my_client)
            else:
                print('This account with phone ' +
                      str(phone) + ' is not in source group')
        else:
            print('This account with phone ' +
                  str(phone) + ' is not in target group')
    else:
        print(
            'This account with phone do not have data. Please run get_data or init_session')

# run
previous_count = 0
count_add = 0

try:
    with open(root_path + '/current_count.txt') as f:
        previous_count = int(f.read())
except Exception as e:
    pass

print('From index: ' + str(previous_count))
total_client = len(filter_clients)

total_user = filter_clients[0]['users'].__len__()

i = 0
while i < total_user:

    # previous run
    if i < previous_count:
        i += 1
        continue

    # count_add if added 35 user
    if count_add % (35 * total_client) == (35 * total_client - 1):
        print('sleep 15 minute')
        time.sleep(15 * 60)

    total_client = filter_clients.__len__()
    print("remain client: " + str(total_client))
    if total_client == 0:
        with open(root_path + '/current_count.txt', 'w') as g:
            g.write(str(i))
            g.close()

        print('END: accounts is empty')
        break

    current_index = count_add % total_client
    print("current_index: " + str(current_index))
    current_client = filter_clients[current_index]
    client = current_client['client']
    user = current_client['users'][i]

    if user['date_online'] != 'online' and user['date_online'] < from_date_active:
        i += 1
        print('User ' + user['user_id'] + ' has time active: ' +
              user['date_online'] + ' is overdue')
        continue

    target_group_entity = current_client['target_group_entity']

    try:
        print('add member: ' + user['user_id'])
        user_to_add = InputPeerUser(
            int(user['user_id']), int(user['access_hash']))
        client(InviteToChannelRequest(target_group_entity, [user_to_add]))
        print('Add member ' + user['user_id'] + ' success')
        count_add += 1
        print('sleep: ' + str(120 / total_client))
        time.sleep(120 / total_client)

    except PeerFloodError as e:
        print("Error Fooling cmnr")
        traceback.print_exc()
        print("remove client: " + current_client['phone'])
        client.disconnect()
        filter_clients.remove(current_client)

        # not increate i
        continue
    except UserPrivacyRestrictedError:
        print("Error Privacy")
    except:
        print("Error other")
    # break

    i += 1

with open(root_path + '/current_count.txt', 'w') as g:
    g.write(str(i))
    g.close()
print("disconnect")
for cli in clients:
    cli['client'].disconnect()
end_time = datetime.datetime.now()
print("total: " + str(count_add))
print("total time: " + str(end_time - start_time))


from lib.external_api.base_wrapper import BaseClient
from model.request import SystemName

class StarLineClient(BaseClient):
    DEFAULT_HOST = 'starline.ru'
    DEV_HOST = 'https://dev-id.{}/:4444'.format(DEFAULT_HOST)

    def __init__(self):
        super(StarLineClient, self).__init__()
        self.slid = None
        self.url = self.DEV_HOST
        self.external_system = SystemName.STARLINE

    async def authorize(self):  # Placeholder for authorization method
        self.cookies = ''
        return True

    async def get_device_state(self):  # Get current state of device sensors
        method_url = '/json/v1/device/{}/state'.format(self.slid)
        self.url = '{}{}'.format(self.DEV_HOST, method_url)
        response = await self.get()
        return response

    async def device_sync(self, command, value):  # Temporary naming
        method_url = '/json/v1/device/{}/set_param'.format(self.slid)
        self.url = '{}{}'.format(self.DEV_HOST, method_url)
        self.data = {
            'id': self.slid,
            'type': command,
            command: value,
        }
        response = await self.post()
        return response


from peephole.util import IdleObject
import logging
from gettext import gettext as _
import gobject

# yes, I am polling instead of actually using asyncronous I/O.
# PyUSB doesn't support asynchronous operation, even though libusb
# has fairly comprehensive asynchronous support.
class EventListener(gobject.GObject):

    __gsignals__ = {
        'buttonPressed' : (gobject.SIGNAL_RUN_LAST, gobject.TYPE_NONE,
                           (gobject.TYPE_INT,))}

    def __init__(self, lcd):
        gobject.GObject.__init__(self)
        self.lcd = lcd
        self.please_stop = False
        self.timer = None

    def start(self):
        # register glib timeout
        self.timer = gobject.timeout_add(50, self.check_button_press)

    def check_button_press(self):
        button = self.lcd.get_event_nonblock()
        if button is not None:
            logging.debug("Button was: %s" % button)
            self.emit('buttonPressed', button)
        return True

#     def run(self):
#         '''This is what is run in the Thread when it is started.'''
#         logging.debug(_("PicoLCD button listener thread now running."))
#         while True:
#             if self.please_stop:
#                 return
#             button = self.lcd.get_event(self.check_if_time_to_stop)
#             if button is not None:
#                 logging.debug("Button was: %s" % button)
#                 self.emit('buttonPressed', button)

    def shutdown(self):
        # unregister glib timeout
        if self.timer is not None:
            gobject.source_remove(self.timer)