from __future__ import print_function, division

import math
import numpy as np

import torch
from torch import nn, Tensor
import torch.nn.functional as F
from typing import Optional, List
from functools import partial

from torch.nn.modules.loss import _Loss
try:
    from itertools import ifilterfalse
except ImportError:  # py3k
    from itertools import filterfalse as ifilterfalse

#: Loss binary mode suppose you are solving binary segmentation task.
#: That mean yor have only one class which pixels are labled as **1**,
#: the rest pixels are background and labeled as **0**.
#: Target mask shape - (N, H, W), model output mask shape (N, 1, H, W).
BINARY_MODE: str = "binary"

#: Loss multiclass mode suppose you are solving multi-**class** segmentation task.
#: That mean you have *C = 1..N* classes which have unique label values,
#: classes are mutually exclusive and all pixels are labeled with theese values.
#: Target mask shape - (N, H, W), model output mask shape (N, C, H, W).
MULTICLASS_MODE: str = "multiclass"

#: Loss multilabel mode suppose you are solving multi-**label** segmentation task.
#: That mean you have *C = 1..N* classes which pixels are labeled as **1**,
#: classes are not mutually exclusive and each class have its own *channel*,
#: pixels in each channel which are not belong to class labeled as **0**.
#: Target mask shape - (N, C, H, W), model output mask shape (N, C, H, W).
MULTILABEL_MODE: str = "multilabel"

__all__ = [
    "focal_loss_with_logits",
    "softmax_focal_loss_with_logits",
    "soft_jaccard_score",
    "soft_dice_score",
    "wing_loss",
]

def to_tensor(x, dtype=None) -> torch.Tensor:
    if isinstance(x, torch.Tensor):
        if dtype is not None:
            x = x.type(dtype)
        return x
    if isinstance(x, np.ndarray):
        x = torch.from_numpy(x)
        if dtype is not None:
            x = x.type(dtype)
        return x
    if isinstance(x, (list, tuple)):
        x = np.array(x)
        x = torch.from_numpy(x)
        if dtype is not None:
            x = x.type(dtype)
        return x

def focal_loss_with_logits(
    output: torch.Tensor,
    target: torch.Tensor,
    gamma: float = 2.0,
    alpha: Optional[float] = 0.25,
    reduction: str = "mean",
    normalized: bool = False,
    reduced_threshold: Optional[float] = None,
    eps: float = 1e-6,
) -> torch.Tensor:
    """Compute binary focal loss between target and output logits.
    See :class:`~pytorch_toolbelt.losses.FocalLoss` for details.
    Args:
        output: Tensor of arbitrary shape (predictions of the model)
        target: Tensor of the same shape as input
        gamma: Focal loss power factor
        alpha: Weight factor to balance positive and negative samples. Alpha must be in [0...1] range,
            high values will give more weight to positive class.
        reduction (string, optional): Specifies the reduction to apply to the output:
            'none' | 'mean' | 'sum' | 'batchwise_mean'. 'none': no reduction will be applied,
            'mean': the sum of the output will be divided by the number of
            elements in the output, 'sum': the output will be summed. Note: :attr:`size_average`
            and :attr:`reduce` are in the process of being deprecated, and in the meantime,
            specifying either of those two args will override :attr:`reduction`.
            'batchwise_mean' computes mean loss per sample in batch. Default: 'mean'
        normalized (bool): Compute normalized focal loss (https://arxiv.org/pdf/1909.07829.pdf).
        reduced_threshold (float, optional): Compute reduced focal loss (https://arxiv.org/abs/1903.01347).
    References:
        https://github.com/open-mmlab/mmdetection/blob/master/mmdet/core/loss/losses.py
    """
    target = target.type(output.type())

    logpt = F.binary_cross_entropy_with_logits(output, target, reduction="none")
    pt = torch.exp(-logpt)

    # compute the loss
    if reduced_threshold is None:
        focal_term = (1.0 - pt).pow(gamma)
    else:
        focal_term = ((1.0 - pt) / reduced_threshold).pow(gamma)
        focal_term[pt < reduced_threshold] = 1

    loss = focal_term * logpt

    if alpha is not None:
        loss *= alpha * target + (1 - alpha) * (1 - target)

    if normalized:
        norm_factor = focal_term.sum().clamp_min(eps)
        loss /= norm_factor

    if reduction == "mean":
        loss = loss.mean()
    if reduction == "sum":
        loss = loss.sum()
    if reduction == "batchwise_mean":
        loss = loss.sum(0)

    return loss

def softmax_focal_loss_with_logits(
                                    output: torch.Tensor,
                                    target: torch.Tensor,
                                    gamma: float = 2.0,
                                    reduction="mean",
                                    normalized=False,
                                    reduced_threshold: Optional[float] = None,
                                    eps: float = 1e-6,
                                ) -> torch.Tensor:
    """Softmax version of focal loss between target and output logits.
    See :class:`~pytorch_toolbelt.losses.FocalLoss` for details.
    Args:
        output: Tensor of shape [B, C, *] (Similar to nn.CrossEntropyLoss)
        target: Tensor of shape [B, *] (Similar to nn.CrossEntropyLoss)
        reduction (string, optional): Specifies the reduction to apply to the output:
            'none' | 'mean' | 'sum' | 'batchwise_mean'. 'none': no reduction will be applied,
            'mean': the sum of the output will be divided by the number of
            elements in the output, 'sum': the output will be summed. Note: :attr:`size_average`
            and :attr:`reduce` are in the process of being deprecated, and in the meantime,
            specifying either of those two args will override :attr:`reduction`.
            'batchwise_mean' computes mean loss per sample in batch. Default: 'mean'
        normalized (bool): Compute normalized focal loss (https://arxiv.org/pdf/1909.07829.pdf).
        reduced_threshold (float, optional): Compute reduced focal loss (https://arxiv.org/abs/1903.01347).
    """
    log_softmax = F.log_softmax(output, dim=1)

    loss = F.nll_loss(log_softmax, target, reduction="none")
    pt = torch.exp(-loss)

    # compute the loss
    if reduced_threshold is None:
        focal_term = (1.0 - pt).pow(gamma)
    else:
        focal_term = ((1.0 - pt) / reduced_threshold).pow(gamma)
        focal_term[pt < reduced_threshold] = 1

    loss = focal_term * loss

    if normalized:
        norm_factor = focal_term.sum().clamp_min(eps)
        loss = loss / norm_factor

    if reduction == "mean":
        loss = loss.mean()
    if reduction == "sum":
        loss = loss.sum()
    if reduction == "batchwise_mean":
        loss = loss.sum(0)

    return loss

def soft_jaccard_score(
                        output: torch.Tensor,
                        target: torch.Tensor,
                        smooth: float = 0.0,
                        eps: float = 1e-7,
                        dims=None,
                    ) -> torch.Tensor:
    assert output.size() == target.size()
    if dims is not None:
        intersection = torch.sum(output * target, dim=dims)
        cardinality = torch.sum(output + target, dim=dims)
    else:
        intersection = torch.sum(output * target)
        cardinality = torch.sum(output + target)

    union = cardinality - intersection
    jaccard_score = (intersection + smooth) / (union + smooth).clamp_min(eps)
    return jaccard_score

def soft_dice_score(
                    output: torch.Tensor,
                    target: torch.Tensor,
                    smooth: float = 0.0,
                    eps: float = 1e-7,
                    dims=None,
                ) -> torch.Tensor:
    assert output.size() == target.size()
    if dims is not None:
        intersection = torch.sum(output * target, dim=dims)
        cardinality = torch.sum(output + target, dim=dims)
    else:
        intersection = torch.sum(output * target)
        cardinality = torch.sum(output + target)
    dice_score = (2.0 * intersection + smooth) / (cardinality + smooth).clamp_min(eps)
    return dice_score

def soft_tversky_score(
                        output: torch.Tensor,
                        target: torch.Tensor,
                        alpha: float,
                        beta: float,
                        smooth: float = 0.0,
                        eps: float = 1e-7,
                        dims=None,
                    ) -> torch.Tensor:
    assert output.size() == target.size()
    if dims is not None:
        intersection = torch.sum(output * target, dim=dims)  # TP
        fp = torch.sum(output * (1.0 - target), dim=dims)
        fn = torch.sum((1 - output) * target, dim=dims)
    else:
        intersection = torch.sum(output * target)  # TP
        fp = torch.sum(output * (1.0 - target))
        fn = torch.sum((1 - output) * target)

    tversky_score = (intersection + smooth) / (intersection + alpha * fp + beta * fn + smooth).clamp_min(eps)
    return tversky_score

def wing_loss(output: torch.Tensor, target: torch.Tensor, width=5, curvature=0.5, reduction="mean"):
    """Wing loss
    References:
        https://arxiv.org/pdf/1711.06753.pdf
    """
    diff_abs = (target - output).abs()
    loss = diff_abs.clone()

    idx_smaller = diff_abs < width
    idx_bigger = diff_abs >= width

    loss[idx_smaller] = width * torch.log(1 + diff_abs[idx_smaller] / curvature)

    C = width - width * math.log(1 + width / curvature)
    loss[idx_bigger] = loss[idx_bigger] - C

    if reduction == "sum":
        loss = loss.sum()

    if reduction == "mean":
        loss = loss.mean()

    return loss

def label_smoothed_nll_loss(
                            lprobs: torch.Tensor,
                            target: torch.Tensor,
                            epsilon: float,
                            ignore_index=None,
                            reduction="mean",
                            dim=-1,
                        ) -> torch.Tensor:
    """NLL loss with label smoothing
    References:
        https://github.com/pytorch/fairseq/blob/master/fairseq/criterions/label_smoothed_cross_entropy.py
    Args:
        lprobs (torch.Tensor): Log-probabilities of predictions (e.g after log_softmax)
    """
    if target.dim() == lprobs.dim() - 1:
        target = target.unsqueeze(dim)

    if ignore_index is not None:
        pad_mask = target.eq(ignore_index)
        target = target.masked_fill(pad_mask, 0)
        nll_loss = -lprobs.gather(dim=dim, index=target)
        smooth_loss = -lprobs.sum(dim=dim, keepdim=True)

        # nll_loss.masked_fill_(pad_mask, 0.0)
        # smooth_loss.masked_fill_(pad_mask, 0.0)
        nll_loss = nll_loss.masked_fill(pad_mask, 0.0)
        smooth_loss = smooth_loss.masked_fill(pad_mask, 0.0)
    else:
        nll_loss = -lprobs.gather(dim=dim, index=target)
        smooth_loss = -lprobs.sum(dim=dim, keepdim=True)

        nll_loss = nll_loss.squeeze(dim)
        smooth_loss = smooth_loss.squeeze(dim)

    if reduction == "sum":
        nll_loss = nll_loss.sum()
        smooth_loss = smooth_loss.sum()
    if reduction == "mean":
        nll_loss = nll_loss.mean()
        smooth_loss = smooth_loss.mean()

    eps_i = epsilon / lprobs.size(dim)
    loss = (1.0 - epsilon) * nll_loss + eps_i * smooth_loss
    return loss

class DiceLoss(_Loss):
    def __init__(
        self,
        mode: str,
        classes: Optional[List[int]] = None,
        log_loss: bool = False,
        from_logits: bool = True,
        smooth: float = 0.0,
        ignore_index: Optional[int] = None,
        eps: float = 1e-7,
    ):
        """Dice loss for image segmentation task.
        It supports binary, multiclass and multilabel cases
        Args:
            mode: Loss mode 'binary', 'multiclass' or 'multilabel'
            classes:  List of classes that contribute in loss computation. By default, all channels are included.
            log_loss: If True, loss computed as `- log(dice_coeff)`, otherwise `1 - dice_coeff`
            from_logits: If True, assumes input is raw logits
            smooth: Smoothness constant for dice coefficient (a)
            ignore_index: Label that indicates ignored pixels (does not contribute to loss)
            eps: A small epsilon for numerical stability to avoid zero division error
                (denominator will be always greater or equal to eps)
        Shape
             - **y_pred** - torch.Tensor of shape (N, C, H, W)
             - **y_true** - torch.Tensor of shape (N, H, W) or (N, C, H, W)
        Reference
            https://github.com/BloodAxe/pytorch-toolbelt
        """
        assert mode in {BINARY_MODE, MULTILABEL_MODE, MULTICLASS_MODE}
        super(DiceLoss, self).__init__()
        self.mode = mode
        if classes is not None:
            assert mode != BINARY_MODE, "Masking classes is not supported with mode=binary"
            classes = to_tensor(classes, dtype=torch.long)

        self.classes = classes
        self.from_logits = from_logits
        self.smooth = smooth
        self.eps = eps
        self.log_loss = log_loss
        self.ignore_index = ignore_index

    def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:

        assert y_true.size(0) == y_pred.size(0)

        if self.from_logits:
            # Apply activations to get [0..1] class probabilities
            # Using Log-Exp as this gives more numerically stable result and does not cause vanishing gradient on
            # extreme values 0 and 1
            if self.mode == MULTICLASS_MODE:
                y_pred = y_pred.log_softmax(dim=1).exp()
            else:
                y_pred = F.logsigmoid(y_pred).exp()

        bs = y_true.size(0)
        num_classes = y_pred.size(1)
        dims = (0, 2)

        if self.mode == BINARY_MODE:
            y_true = y_true.view(bs, 1, -1)
            y_pred = y_pred.view(bs, 1, -1)

            if self.ignore_index is not None:
                mask = y_true != self.ignore_index
                y_pred = y_pred * mask
                y_true = y_true * mask

        if self.mode == MULTICLASS_MODE:
            y_true = y_true.view(bs, -1)
            y_pred = y_pred.view(bs, num_classes, -1)

            if self.ignore_index is not None:
                mask = y_true != self.ignore_index
                y_pred = y_pred * mask.unsqueeze(1)

                y_true = F.one_hot((y_true * mask).to(torch.long), num_classes)  # N,H*W -> N,H*W, C
                y_true = y_true.permute(0, 2, 1) * mask.unsqueeze(1)  # N, C, H*W
            else:
                y_true = F.one_hot(y_true.long(), num_classes)  # N,H*W -> N,H*W, C
                y_true = y_true.permute(0, 2, 1)  # N, C, H*W

        if self.mode == MULTILABEL_MODE:
            y_true = y_true.view(bs, num_classes, -1)
            y_pred = y_pred.view(bs, num_classes, -1)

            if self.ignore_index is not None:
                mask = y_true != self.ignore_index
                y_pred = y_pred * mask
                y_true = y_true * mask

        scores = self.compute_score(y_pred, y_true.type_as(y_pred), smooth=self.smooth, eps=self.eps, dims=dims)

        if self.log_loss:
            loss = -torch.log(scores.clamp_min(self.eps))
        else:
            loss = 1.0 - scores

        # Dice loss is undefined for non-empty classes
        # So we zero contribution of channel that does not have true pixels
        # NOTE: A better workaround would be to use loss term `mean(y_pred)`
        # for this case, however it will be a modified jaccard loss

        mask = y_true.sum(dims) > 0
        loss *= mask.to(loss.dtype)

        if self.classes is not None:
            loss = loss[self.classes]

        return self.aggregate_loss(loss)

    def aggregate_loss(self, loss):
        return loss.mean()

    def compute_score(self, output, target, smooth=0.0, eps=1e-7, dims=None) -> torch.Tensor:
        return soft_dice_score(output, target, smooth, eps, dims)

class FocalLoss(_Loss):
    def __init__(
        self,
        mode: str,
        alpha: Optional[float] = None,
        gamma: Optional[float] = 2.0,
        ignore_index: Optional[int] = None,
        reduction: Optional[str] = "mean",
        normalized: bool = False,
        reduced_threshold: Optional[float] = None,
    ):
        """Compute Focal loss
        Args:
            mode: Loss mode 'binary', 'multiclass' or 'multilabel'
            alpha: Prior probability of having positive value in target.
            gamma: Power factor for dampening weight (focal strength).
            ignore_index: If not None, targets may contain values to be ignored.
                Target values equal to ignore_index will be ignored from loss computation.
            normalized: Compute normalized focal loss (https://arxiv.org/pdf/1909.07829.pdf).
            reduced_threshold: Switch to reduced focal loss. Note, when using this mode you
                should use `reduction="sum"`.
        Shape
             - **y_pred** - torch.Tensor of shape (N, C, H, W)
             - **y_true** - torch.Tensor of shape (N, H, W) or (N, C, H, W)
        Reference
            https://github.com/BloodAxe/pytorch-toolbelt
        """
        assert mode in {BINARY_MODE, MULTILABEL_MODE, MULTICLASS_MODE}
        super().__init__()

        self.mode = mode
        self.ignore_index = ignore_index
        self.focal_loss_fn = partial(
            focal_loss_with_logits,
            alpha=alpha,
            gamma=gamma,
            reduced_threshold=reduced_threshold,
            reduction=reduction,
            normalized=normalized,
        )

    def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:

        if self.mode in {BINARY_MODE, MULTILABEL_MODE}:
            y_true = y_true.view(-1)
            y_pred = y_pred.view(-1)

            if self.ignore_index is not None:
                # Filter predictions with ignore label from loss computation
                not_ignored = y_true != self.ignore_index
                y_pred = y_pred[not_ignored]
                y_true = y_true[not_ignored]

            loss = self.focal_loss_fn(y_pred, y_true)

        elif self.mode == MULTICLASS_MODE:

            num_classes = y_pred.size(1)
            loss = 0

            # Filter anchors with -1 label from loss computation
            if self.ignore_index is not None:
                not_ignored = y_true != self.ignore_index

            for cls in range(num_classes):
                cls_y_true = (y_true == cls).long()
                cls_y_pred = y_pred[:, cls, ...]

                if self.ignore_index is not None:
                    cls_y_true = cls_y_true[not_ignored]
                    cls_y_pred = cls_y_pred[not_ignored]

                loss += self.focal_loss_fn(cls_y_pred, cls_y_true)

        return loss

class JaccardLoss(_Loss):
    def __init__(
        self,
        mode: str,
        classes: Optional[List[int]] = None,
        log_loss: bool = False,
        from_logits: bool = True,
        smooth: float = 0.0,
        eps: float = 1e-7,
    ):
        """Jaccard loss for image segmentation task.
        It supports binary, multiclass and multilabel cases
        Args:
            mode: Loss mode 'binary', 'multiclass' or 'multilabel'
            classes:  List of classes that contribute in loss computation. By default, all channels are included.
            log_loss: If True, loss computed as `- log(jaccard_coeff)`, otherwise `1 - jaccard_coeff`
            from_logits: If True, assumes input is raw logits
            smooth: Smoothness constant for dice coefficient
            eps: A small epsilon for numerical stability to avoid zero division error
                (denominator will be always greater or equal to eps)
        Shape
             - **y_pred** - torch.Tensor of shape (N, C, H, W)
             - **y_true** - torch.Tensor of shape (N, H, W) or (N, C, H, W)
        Reference
            https://github.com/BloodAxe/pytorch-toolbelt
        """
        assert mode in {BINARY_MODE, MULTILABEL_MODE, MULTICLASS_MODE}
        super(JaccardLoss, self).__init__()

        self.mode = mode
        if classes is not None:
            assert mode != BINARY_MODE, "Masking classes is not supported with mode=binary"
            classes = to_tensor(classes, dtype=torch.long)

        self.classes = classes
        self.from_logits = from_logits
        self.smooth = smooth
        self.eps = eps
        self.log_loss = log_loss

    def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:

        assert y_true.size(0) == y_pred.size(0)

        if self.from_logits:
            # Apply activations to get [0..1] class probabilities
            # Using Log-Exp as this gives more numerically stable result and does not cause vanishing gradient on
            # extreme values 0 and 1
            if self.mode == MULTICLASS_MODE:
                y_pred = y_pred.log_softmax(dim=1).exp()
            else:
                y_pred = F.logsigmoid(y_pred).exp()

        bs = y_true.size(0)
        num_classes = y_pred.size(1)
        dims = (0, 2)

        if self.mode == BINARY_MODE:
            y_true = y_true.view(bs, 1, -1)
            y_pred = y_pred.view(bs, 1, -1)

        if self.mode == MULTICLASS_MODE:
            y_true = y_true.view(bs, -1)
            y_pred = y_pred.view(bs, num_classes, -1)

            y_true = F.one_hot(y_true.long(), num_classes)  # N,H*W -> N,H*W, C
            y_true = y_true.permute(0, 2, 1)  # H, C, H*W

        if self.mode == MULTILABEL_MODE:
            y_true = y_true.view(bs, num_classes, -1)
            y_pred = y_pred.view(bs, num_classes, -1)

        scores = soft_jaccard_score(
            y_pred,
            y_true.type(y_pred.dtype),
            smooth=self.smooth,
            eps=self.eps,
            dims=dims,
        )

        if self.log_loss:
            loss = -torch.log(scores.clamp_min(self.eps))
        else:
            loss = 1.0 - scores

        # IoU loss is defined for non-empty classes
        # So we zero contribution of channel that does not have true pixels
        # NOTE: A better workaround would be to use loss term `mean(y_pred)`
        # for this case, however it will be a modified jaccard loss

        mask = y_true.sum(dims) > 0
        loss *= mask.float()

        if self.classes is not None:
            loss = loss[self.classes]

        return loss.mean()

def _lovasz_grad(gt_sorted):
    """Compute gradient of the Lovasz extension w.r.t sorted errors
    See Alg. 1 in paper
    """
    p = len(gt_sorted)
    gts = gt_sorted.sum()
    intersection = gts - gt_sorted.float().cumsum(0)
    union = gts + (1 - gt_sorted).float().cumsum(0)
    jaccard = 1.0 - intersection / union
    if p > 1:  # cover 1-pixel case
        jaccard[1:p] = jaccard[1:p] - jaccard[0:-1]
    return jaccard

def _lovasz_hinge(logits, labels, per_image=True, ignore=None):
    """
    Binary Lovasz hinge loss
        logits: [B, H, W] Logits at each pixel (between -infinity and +infinity)
        labels: [B, H, W] Tensor, binary ground truth masks (0 or 1)
        per_image: compute the loss per image instead of per batch
        ignore: void class id
    """
    if per_image:
        loss = mean(
            _lovasz_hinge_flat(*_flatten_binary_scores(log.unsqueeze(0), lab.unsqueeze(0), ignore))
            for log, lab in zip(logits, labels)
        )
    else:
        loss = _lovasz_hinge_flat(*_flatten_binary_scores(logits, labels, ignore))
    return loss

def _lovasz_hinge_flat(logits, labels):
    """Binary Lovasz hinge loss
    Args:
        logits: [P] Logits at each prediction (between -infinity and +infinity)
        labels: [P] Tensor, binary ground truth labels (0 or 1)
        ignore: label to ignore
    """
    if len(labels) == 0:
        # only void pixels, the gradients should be 0
        return logits.sum() * 0.0
    signs = 2.0 * labels.float() - 1.0
    errors = 1.0 - logits * signs
    errors_sorted, perm = torch.sort(errors, dim=0, descending=True)
    perm = perm.data
    gt_sorted = labels[perm]
    grad = _lovasz_grad(gt_sorted)
    loss = torch.dot(F.relu(errors_sorted), grad)
    return loss

def _flatten_binary_scores(scores, labels, ignore=None):
    """Flattens predictions in the batch (binary case)
    Remove labels equal to 'ignore'
    """
    scores = scores.view(-1)
    labels = labels.view(-1)
    if ignore is None:
        return scores, labels
    valid = labels != ignore
    vscores = scores[valid]
    vlabels = labels[valid]
    return vscores, vlabels

# --------------------------- MULTICLASS LOSSES ---------------------------

def _lovasz_softmax(probas, labels, classes="present", per_image=False, ignore=None):
    """Multi-class Lovasz-Softmax loss
    Args:
        @param probas: [B, C, H, W] Class probabilities at each prediction (between 0 and 1).
        Interpreted as binary (sigmoid) output with outputs of size [B, H, W].
        @param labels: [B, H, W] Tensor, ground truth labels (between 0 and C - 1)
        @param classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average.
        @param per_image: compute the loss per image instead of per batch
        @param ignore: void class labels
    """
    if per_image:
        loss = mean(
            _lovasz_softmax_flat(*_flatten_probas(prob.unsqueeze(0), lab.unsqueeze(0), ignore), classes=classes)
            for prob, lab in zip(probas, labels)
        )
    else:
        loss = _lovasz_softmax_flat(*_flatten_probas(probas, labels, ignore), classes=classes)
    return loss

def _lovasz_softmax_flat(probas, labels, classes="present"):
    """Multi-class Lovasz-Softmax loss
    Args:
        @param probas: [P, C] Class probabilities at each prediction (between 0 and 1)
        @param labels: [P] Tensor, ground truth labels (between 0 and C - 1)
        @param classes: 'all' for all, 'present' for classes present in labels, or a list of classes to average.
    """
    if probas.numel() == 0:
        # only void pixels, the gradients should be 0
        return probas * 0.0
    C = probas.size(1)
    losses = []
    class_to_sum = list(range(C)) if classes in ["all", "present"] else classes
    for c in class_to_sum:
        fg = (labels == c).type_as(probas)  # foreground for class c
        if classes == "present" and fg.sum() == 0:
            continue
        if C == 1:
            if len(classes) > 1:
                raise ValueError("Sigmoid output possible only with 1 class")
            class_pred = probas[:, 0]
        else:
            class_pred = probas[:, c]
        errors = (fg - class_pred).abs()
        errors_sorted, perm = torch.sort(errors, 0, descending=True)
        perm = perm.data
        fg_sorted = fg[perm]
        losses.append(torch.dot(errors_sorted, _lovasz_grad(fg_sorted)))
    return mean(losses)

def _flatten_probas(probas, labels, ignore=None):
    """Flattens predictions in the batch"""
    if probas.dim() == 3:
        # assumes output of a sigmoid layer
        B, H, W = probas.size()
        probas = probas.view(B, 1, H, W)

    C = probas.size(1)
    probas = torch.movedim(probas, 1, -1)  # [B, C, Di, Dj, ...] -> [B, Di, Dj, ..., C]
    probas = probas.contiguous().view(-1, C)  # [P, C]

    labels = labels.view(-1)
    if ignore is None:
        return probas, labels
    valid = labels != ignore
    vprobas = probas[valid]
    vlabels = labels[valid]
    return vprobas, vlabels

# --------------------------- HELPER FUNCTIONS ---------------------------
def isnan(x):
    return x != x

def mean(values, ignore_nan=False, empty=0):
    """Nanmean compatible with generators."""
    values = iter(values)
    if ignore_nan:
        values = ifilterfalse(isnan, values)
    try:
        n = 1
        acc = next(values)
    except StopIteration:
        if empty == "raise":
            raise ValueError("Empty mean")
        return empty
    for n, v in enumerate(values, 2):
        acc += v
    if n == 1:
        return acc
    return acc / n

class LovaszLoss(_Loss):
    def __init__(
        self,
        mode: str,
        per_image: bool = False,
        ignore_index: Optional[int] = None,
        from_logits: bool = True,
    ):
        """Lovasz loss for image segmentation task.
        It supports binary, multiclass and multilabel cases
        Args:
            mode: Loss mode 'binary', 'multiclass' or 'multilabel'
            ignore_index: Label that indicates ignored pixels (does not contribute to loss)
            per_image: If True loss computed per each image and then averaged, else computed per whole batch
        Shape
             - **y_pred** - torch.Tensor of shape (N, C, H, W)
             - **y_true** - torch.Tensor of shape (N, H, W) or (N, C, H, W)
        Reference
            https://github.com/BloodAxe/pytorch-toolbelt
        """
        assert mode in {BINARY_MODE, MULTILABEL_MODE, MULTICLASS_MODE}
        super().__init__()

        self.mode = mode
        self.ignore_index = ignore_index
        self.per_image = per_image

    def forward(self, y_pred, y_true):

        if self.mode in {BINARY_MODE, MULTILABEL_MODE}:
            loss = _lovasz_hinge(y_pred, y_true, per_image=self.per_image, ignore=self.ignore_index)
        elif self.mode == MULTICLASS_MODE:
            y_pred = y_pred.softmax(dim=1)
            loss = _lovasz_softmax(y_pred, y_true, per_image=self.per_image, ignore=self.ignore_index)
        else:
            raise ValueError("Wrong mode {}.".format(self.mode))
        return loss

class MCCLoss(_Loss):
    def __init__(self, eps: float = 1e-5):
        """Compute Matthews Correlation Coefficient Loss for image segmentation task.
        It only supports binary mode.
        Args:
            eps (float): Small epsilon to handle situations where all the samples in the dataset belong to one class
        Reference:
            https://github.com/kakumarabhishek/MCC-Loss
        """
        super().__init__()
        self.eps = eps

    def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:
        """Compute MCC loss
        Args:
            y_pred (torch.Tensor): model prediction of shape (N, H, W) or (N, 1, H, W)
            y_true (torch.Tensor): ground truth labels of shape (N, H, W) or (N, 1, H, W)
        Returns:
            torch.Tensor: loss value (1 - mcc)
        """

        bs = y_true.shape[0]

        y_true = y_true.view(bs, 1, -1)
        y_pred = y_pred.view(bs, 1, -1)

        tp = torch.sum(torch.mul(y_pred, y_true)) + self.eps
        tn = torch.sum(torch.mul((1 - y_pred), (1 - y_true))) + self.eps
        fp = torch.sum(torch.mul(y_pred, (1 - y_true))) + self.eps
        fn = torch.sum(torch.mul((1 - y_pred), y_true)) + self.eps

        numerator = torch.mul(tp, tn) - torch.mul(fp, fn)
        denominator = torch.sqrt(torch.add(tp, fp) * torch.add(tp, fn) * torch.add(tn, fp) * torch.add(tn, fn))

        mcc = torch.div(numerator.sum(), denominator.sum())
        loss = 1.0 - mcc

        return loss

class SoftBCEWithLogitsLoss(nn.Module):

    __constants__ = [
        "weight",
        "pos_weight",
        "reduction",
        "ignore_index",
        "smooth_factor",
    ]

    def __init__(
        self,
        weight: Optional[torch.Tensor] = None,
        ignore_index: Optional[int] = -100,
        reduction: str = "mean",
        smooth_factor: Optional[float] = None,
        pos_weight: Optional[torch.Tensor] = None,
    ):
        """Drop-in replacement for torch.nn.BCEWithLogitsLoss with few additions: ignore_index and label_smoothing
        Args:
            ignore_index: Specifies a target value that is ignored and does not contribute to the input gradient.
            smooth_factor: Factor to smooth target (e.g. if smooth_factor=0.1 then [1, 0, 1] -> [0.9, 0.1, 0.9])
        Shape
             - **y_pred** - torch.Tensor of shape NxCxHxW
             - **y_true** - torch.Tensor of shape NxHxW or Nx1xHxW
        Reference
            https://github.com/BloodAxe/pytorch-toolbelt
        """
        super().__init__()
        self.ignore_index = ignore_index
        self.reduction = reduction
        self.smooth_factor = smooth_factor
        self.register_buffer("weight", weight)
        self.register_buffer("pos_weight", pos_weight)

    def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:
        """
        Args:
            y_pred: torch.Tensor of shape (N, C, H, W)
            y_true: torch.Tensor of shape (N, H, W)  or (N, 1, H, W)
        Returns:
            loss: torch.Tensor
        """

        if self.smooth_factor is not None:
            soft_targets = (1 - y_true) * self.smooth_factor + y_true * (1 - self.smooth_factor)
        else:
            soft_targets = y_true

        loss = F.binary_cross_entropy_with_logits(
            y_pred,
            soft_targets,
            self.weight,
            pos_weight=self.pos_weight,
            reduction="none",
        )

        if self.ignore_index is not None:
            not_ignored_mask = y_true != self.ignore_index
            loss *= not_ignored_mask.type_as(loss)

        if self.reduction == "mean":
            loss = loss.mean()

        if self.reduction == "sum":
            loss = loss.sum()

        return loss

class SoftCrossEntropyLoss(nn.Module):

    __constants__ = ["reduction", "ignore_index", "smooth_factor"]

    def __init__(
        self,
        reduction: str = "mean",
        smooth_factor: Optional[float] = None,
        ignore_index: Optional[int] = -100,
        dim: int = 1,
    ):
        """Drop-in replacement for torch.nn.CrossEntropyLoss with label_smoothing
        Args:
            smooth_factor: Factor to smooth target (e.g. if smooth_factor=0.1 then [1, 0, 0] -> [0.9, 0.05, 0.05])
        Shape
             - **y_pred** - torch.Tensor of shape (N, C, H, W)
             - **y_true** - torch.Tensor of shape (N, H, W)
        Reference
            https://github.com/BloodAxe/pytorch-toolbelt
        """
        super().__init__()
        self.smooth_factor = smooth_factor
        self.ignore_index = ignore_index
        self.reduction = reduction
        self.dim = dim

    def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:
        log_prob = F.log_softmax(y_pred, dim=self.dim)
        return label_smoothed_nll_loss(
            log_prob,
            y_true,
            epsilon=self.smooth_factor,
            ignore_index=self.ignore_index,
            reduction=self.reduction,
            dim=self.dim,
        )

class TverskyLoss(DiceLoss):
    """Tversky loss for image segmentation task.
    Where FP and FN is weighted by alpha and beta params.
    With alpha == beta == 0.5, this loss becomes equal DiceLoss.
    It supports binary, multiclass and multilabel cases
    Args:
        mode: Metric mode {'binary', 'multiclass', 'multilabel'}
        classes: Optional list of classes that contribute in loss computation;
        By default, all channels are included.
        log_loss: If True, loss computed as ``-log(tversky)`` otherwise ``1 - tversky``
        from_logits: If True assumes input is raw logits
        smooth:
        ignore_index: Label that indicates ignored pixels (does not contribute to loss)
        eps: Small epsilon for numerical stability
        alpha: Weight constant that penalize model for FPs (False Positives)
        beta: Weight constant that penalize model for FNs (False Negatives)
        gamma: Constant that squares the error function. Defaults to ``1.0``
    Return:
        loss: torch.Tensor
    """

    def __init__(
        self,
        mode: str,
        classes: List[int] = None,
        log_loss: bool = False,
        from_logits: bool = True,
        smooth: float = 0.0,
        ignore_index: Optional[int] = None,
        eps: float = 1e-7,
        alpha: float = 0.5,
        beta: float = 0.5,
        gamma: float = 1.0,
    ):

        assert mode in {BINARY_MODE, MULTILABEL_MODE, MULTICLASS_MODE}
        super().__init__(mode, classes, log_loss, from_logits, smooth, ignore_index, eps)
        self.alpha = alpha
        self.beta = beta
        self.gamma = gamma

    def aggregate_loss(self, loss):
        return loss.mean() ** self.gamma

    def compute_score(self, output, target, smooth=0.0, eps=1e-7, dims=None) -> torch.Tensor:
        return soft_tversky_score(output, target, self.alpha, self.beta, smooth, eps, dims)


import argparse
import logging
from concurrent import futures

import grpc
import info_pb2
import info_pb2_grpc
from transformers import pipeline, set_seed

class TextGenerationServiceServicer(info_pb2_grpc.TextGenerationServiceServicer):
    def __init__(self):
        set_seed(42)
        self.generator = pipeline("text-generation", model="gpt2")

    def GenerateText(self, request, context):
        text =  self.generator(request.text, max_length=30, num_return_sequences=1)[0][
            "generated_text"
        ]
        return info_pb2.Message(text=text)

def serve(args):
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    info_pb2_grpc.add_TextGenerationServiceServicer_to_server(
        TextGenerationServiceServicer(), server)
    address = f'{args.ip}:{args.port}'
    port = server.add_insecure_port(address)
    print(f'serving on port {port}')
    server.start()
    server.wait_for_termination()

if __name__ == '__main__':
    logging.basicConfig()

    parser = argparse.ArgumentParser(description="Parse arguments")
    parser.add_argument("--ip", type = str, default="[::]", help="port")
    parser.add_argument("--port", type = str, default="50051", help="port")
    args = parser.parse_args()

    serve(args)


import telegramModule
import os
import sys
import time
from datetime import date
from datetime import timedelta
import sqlite3

import hiworks

# 환경설정 파일 읽기.
exefilePath = os.path.dirname(sys.argv[0])
dbFile = exefilePath+'/holyday.db'

# 텔래그램 아이디를 가지고 오자.
notiGroupTelegramId = os.environ.get("TELEGRAM_GROUP_ID_OF_ADMIN_FOR_HIWORKS_NOTI")
# notiGroupTelegramId = os.environ.get("TELEGRAM_ID_OF_ADMIN_FOR_HIWORKS_NOTI")

# 세션 아이디를 가지고 오자.
sessionid = os.environ.get("SESSION_ID_FOR_HIWORKS_NOTI")
# 로그인 해더를 추가한다.
hiworks.requestHeader = {"Cookie": "PHPSESSID={0};".format(sessionid),
             "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
             }

#검사해야 하는 이름
user1Name = os.environ.get("USER_1_NAME_FOR_HIWORKS_NOTI")
user2Name = os.environ.get("USER_2_NAME_FOR_HIWORKS_NOTI")

holyday = set()

# 모든 주말이 아닌 휴일을 holyday 전역 변수에 담는다.
def getHolydayList():
    conn = sqlite3.connect(dbFile)
    # Connection 으로부터 Cursor 생성
    cur = conn.cursor()

    # SQL 쿼리 실행
    cur.execute("select strDate from holyday")

    # 데이타 Fetch
    rows = cur.fetchall()
    for row in rows:
        holyday.add(row[0])

    # Connection 닫기
    conn.close()

# 전달받은 날짜가 일하는 날인지 판단한다.
# 전달 받은 객채가 date객채가 아니면 항상 False를 반환한다.
def getWorkingDay(d):
    if type(d) is not date:
        return False

    if d.weekday() == 5 or d.weekday() == 6:
        return False

    tempDateString = d.strftime("%Y.%m.%d")
    if tempDateString in holyday:
        return False

    return True

# 오늘 이 일하는 날이라면..
# 다음 일하는 날을 골라서 휴가자를 확인하여 전달한다.
def sendVacationUser():
    #휴일을 일단 불러온다.
    getHolydayList()

    oneDay = timedelta(days=1)

    t = date.today()

    targetDateString = ""

    newFileWrites = False

    # 오늘이 일하는 날이라면...
    if getWorkingDay(t):
        #메세지를 보내야 한다.
        #다음 일하는 날짜를 찾는다.
        while targetDateString == "":
            # 하루 추가.
            t = t + oneDay
            #일하는 날짜라면..
            if getWorkingDay(t):
                #목표날짜로 지정.
                targetDateString = t.strftime("%Y.%m.%d")

        print("휴가자 검사중... 검사날짜 : {0}".format(targetDateString))
        #while 문이 끝났다면 목표날짜를 찾음...
        #목표 날짜의 휴가자를 모두 구함.
        (total, month, day, userList) = hiworks.getTargetDayVacationUserList(targetDateString)

        # 휴가자가 있다면..
        if int(total) > 0:
            # 텔레그램으로 전달할 메세지를 작성하기 시작한다.
            resultString = "{0} {1} 총 휴가자 {2}명 내역입니다. \n\n".format(month,day,total)
            # 가나다 순으로 정렬하자 ... 개발팀이 가장 먼저 온다.
            userList.sort()
            # 각각을 개행으로 구분하여 써준다.
            for team, name, duration, approvalStatus in userList:
                if duration != "종일":
                    resultString += f"{team} {name}({duration}) : {approvalStatus}\n"
                else:
                    resultString += f"{team} {name} : {approvalStatus}\n"

                if duration == "종일" or duration == "오전":
                    if user1Name == name:
                        newFileWrites = True
                        vacFilePath = exefilePath+"/user1.vac"
                        with open(vacFilePath, 'w') as fp:
                            fp.write(targetDateString)

                    if user2Name == name:
                        newFileWrites = True
                        vacFilePath = exefilePath+"/user2.vac"
                        with open(vacFilePath, 'w') as fp:
                            fp.write(targetDateString)

            # 텔레그램으로 전달.
            telegramModule.sendMessage(notiGroupTelegramId, resultString)
            print("휴가자 전달\n전달 텔레그램 번호 : {0}\n{1}".format(notiGroupTelegramId,resultString))

        print("휴가자 검사끝... 검사날짜 : {0}".format(targetDateString))

    return newFileWrites

newFileWrites = sendVacationUser()
print(f"::set-output name=new_file::{newFileWrites}")


from typing import List

class QAPair:

    def __init__(self, task: str):
        self.task = task

    def from_row(self, row):
        self.gif_name: str = row.gif_name
        self.question: str = row.question.lower()

        if self.task in ['action', 'trans']:
            self.answers: List[str] = [
                row.a0, row.a1, row.a2, row.a3, row.a4
            ]
            self.answer_index: int = row.answer

        else:
            self.answer_index: str = row.answer


import collections
import os  # for testing if configs are valid paths.
from tinydb import TinyDB, Query
from tinydb.operations import set
import consoleprint as console
from webconfig import WebConfig

class Singleton(type):
    _instances = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
        return cls._instances[cls]

class ActualConfig:
    def __init__(self):
        self.logger = ''

        path = os.path.expanduser('~/.config/unrar-angel/')
        if not os.path.exists(path):
            os.makedirs(path)

        self.config_path = path

        self.globals = TinyDB(path + 'global.json')
        self.watchers = TinyDB(path + 'watchers.json')

        self.config_items = ConfigItems()

        self.verify_global_configs_are_present()
        self.verify_watcher_configs_are_present()

        self.var_watchers_should_restart = False
        self.web_config = ''

    def set_logger(self, logger):
        self.logger = logger
        self.logger.info('Config', 'Loaded config from directory {}'.format(self.config_path))

    def start_web_server(self):
        self.web_config = WebConfig(self.logger, self)
        self.web_config.start_web_servers()

    def kill_web_server(self):
        self.web_config.shutdown_web_servers()

    # Global related items
    # - Setting global settings
    # - Verifying that they are set

    def search_and_set_globals(self, key, value):
        # A key and value are a different record
        setting = Query()
        if len(self.globals.search(setting.key == key)) == 0:
            self.globals.insert({'key': key, 'value': value})

    def verify_global_configs_are_present(self):
        global_items = self.config_items.get_global_config_options()
        for global_item in global_items:
            self.search_and_set_globals(global_item[0], global_item[2])

    def get_all_global_settings(self):
        return self.globals

    def get_config_global(self, key):
        setting = Query()
        if len(self.globals.search(setting.key == key)) != 0:
            return self.globals.search(setting.key == key)[0]['value']
        else:
            self.logger.critical('Config', 'Invalid global config key requested. {}'.format(key))
            return False

    def set_config_global(self, value, keys):
        setting = Query()
        self.globals.update({'value': value}, setting.key == keys)

    # Watcher settings
    # - Searching and setting settings
    # - Verifying that they are set
    # - Getting the amount of watchers
    # - Copying a watcher
    # - Verifying whether they should restart

    def search_and_set_watcher(self, key, value):
        # A key and value are on the same record
        setting = Query()
        for item in self.watchers:
            if key not in item:
                self.watchers.update(set(key, value), setting.uid == item['uid'])

    def get_amount_of_watchers(self):
        return len(self.watchers)

    def check_amount_of_watchers(self):
        if len(self.watchers) == 0:
            self.watchers.insert({'uid': 1})

    def verify_watcher_configs_are_present(self):
        self.check_amount_of_watchers()
        watcher_items = self.config_items.get_watcher_config_options()
        for watcher_item in watcher_items:
            self.search_and_set_watcher(watcher_item[0], watcher_item[2])

    def add_new_watchers(self):
        self.add_new_watcher_by_uid(self.get_highest_uid_watcher()+1)

    def add_new_watcher_by_uid(self, uid):
        self.watchers.insert({'uid': uid,
                              'name': 'Watcher config ' + str(self.get_highest_uid_watcher() + 1)})

    def remove_watcher(self, w_uid):
        setting = Query()
        self.watchers.remove(setting.uid == int(w_uid))

    def get_highest_uid_watcher(self):
        highest_uid = 0
        for item in self.watchers:
            highest_uid = max(item['uid'], highest_uid)
        return int(highest_uid)

    def get_config_watcher(self, uid, key):
        setting = Query()
        item = self.watchers.search(setting.uid == int(uid))
        if len(item) == 0:
            self.logger.critical('Config', 'Invalid watcher uid given. {} : {}'.format(uid, key))
        elif key not in item[0]:
            self.logger.critical('Config', 'Invalid watcher config key requested. {} : {}'.format(uid, key))
        else:
            return item[0][key]
        return False  # In any other case

    def get_all_watchers(self):
        return self.watchers

    def get_watcher_by_uid(self, uid_d):
        setting = Query()
        return self.watchers.search(setting.uid == uid_d)

    def get_all_active_watchers(self):
        setting = Query()
        return self.watchers.search(setting.on_or_off == 1)

    def get_amount_of_active_watchers(self):
        return len(self.get_all_active_watchers())

    def watchers_should_restart(self, new_value):
        self.var_watchers_should_restart = new_value

    def should_watchers_restart(self):
        return self.var_watchers_should_restart

    def set_config_watcher(self, uid, value, key):
        setting = Query()
        self.watchers.update({key: value}, setting.uid == int(uid))

    # Testing settings
    # - Testing whether the values actually match with the intended value.

    @staticmethod
    def test_global_int(value, key):
        if not isinstance(value, int):
            console.print_conf_error('Global config has invalid value at {}.\n\tShould be an int'.format(key))
            return False
        return True

    @staticmethod
    def test_global_string(value, key):
        if not isinstance(value, str):
            console.print_conf_error('Global config has invalid value at {}.\n\tShould be a string'.format(key))
            return False
        return True

    def test_global_path(self, value, key):
        self.test_global_string(value, key)
        path_without_file = value.split('/')
        path_without_file = path_without_file[0:len(path_without_file)-1]
        path_without_file = "/".join(path_without_file) + "/"
        if os.access(path_without_file, os.W_OK) is not True:
            console.print_conf_error(
                'Global config has invalid value at {}.\n\tShould be a valid path with write access '
                '(/example/of/link.ext)'.format(key))
            return False
        return True

    @staticmethod
    def test_global_or(value, key, values):
        if value not in values:
            console.print_conf_error(
                'Global config has invalid value at {}.\n\tShould be one of the following strings: {}'
                .format(key, ", ".join(str(x) for x in values)))
            return False
        return True

    @staticmethod
    def test_watcher_boolean(value, key):
        if not (value or not value):
            console.print_conf_error('Watcher config has invalid value at {}.\n\tShould be 0 for off or 1 for on'
                                     .format(key))
            return False
        return True

    @staticmethod
    def test_watcher_int(value, key):
        if not isinstance(value, int):
            console.print_conf_error('Watcher config has invalid value at {}.\n\tShould be an integer value'
                                     .format(key))
            return False
        return True

    @staticmethod
    def test_watcher_str(value, key):
        if not isinstance(value, str):
            console.print_conf_error('Watcher config has invalid value at {}.\n\tShould be a string'.format(key))
            return False
        return True

    @staticmethod
    def test_watcher_path(value, key):
        if os.access(value, os.W_OK) is not True:
            console.print_conf_error(
                'Watcher config has invalid value at {}.\n\tShould be a valid path with write access'
                .format(key))
            return False
        return True

    @staticmethod
    def test_watcher_or(value, key, values):
        if value not in values:
            console.print_conf_error(
                'Global config has invalid value at {}.\n\tShould be one of the following strings: {}'
                .format(key, ", ".join(str(x) for x in values)))
            return False
        return True

    # Combining all the tests in one function

    def test_each_individual_setting(self):
        print('Verifying all global settings')

        global_items = self.config_items.get_global_config_options()
        for global_item in global_items:
            if global_item[1] == 'str':
                self.test_global_string(self.get_config_global(global_item[0]), global_item[0])
            if global_item[1] == 'int':
                self.test_global_int(self.get_config_global(global_item[0]), global_item[0])
            if global_item[1] == 'path':
                self.test_global_path(self.get_config_global(global_item[0]), global_item[0])
            if global_item[1] == 'bool':
                self.test_global_or(self.get_config_global(global_item[0]), global_item[0], [0, 1])
            if global_item[1] == 'or':
                if len(global_item) < 4:
                    print('Critical error! OR method does not have selection criteria.')
                self.test_global_or(self.get_config_global(global_item[0]), global_item[0], global_items[3])

        console.print_ok('Verified all global settings')
        print('Verifying all watcher settings')

        uid_list = []
        watcher_items = self.config_items.get_watcher_config_options()
        for watcher in self.get_all_watchers():
            for watcher_item in watcher_items:
                if watcher_item[1] == 'str':
                    self.test_watcher_str(watcher[watcher_item[0]], watcher_item[0])
                if watcher_item[1] == 'int':
                    self.test_watcher_int(watcher[watcher_item[0]], watcher_item[0])
                if watcher_item[1] == 'path':
                    self.test_watcher_path(watcher[watcher_item[0]], watcher_item[0])
                if watcher_item[1] == 'bool':
                    self.test_watcher_boolean(watcher[watcher_item[0]], watcher_item[0])
                if watcher_item[1] == 'or':
                    if len(watcher_item) < 4:
                        print('Critical error! OR method does not have selection criteria.')
                    self.test_watcher_or(watcher[watcher_item[0]], watcher_item[0], watcher_items[3])
                if watcher_item[0] == 'uid':
                    if watcher[watcher_item[0]] in uid_list:
                        console.print_conf_error(
                            'All watchers should have a distinct UID, {} is defined multiple times.'
                            .format(watcher[watcher_item[0]]))
                    uid_list.append(watcher[watcher_item[0]])

        console.print_ok('Verified all watcher settings')

    # Function to convert TinyDB items to JSON.
    def get_all_global_settings_json(self):
        arr = collections.defaultdict(dict)
        i = 0
        for global_set in self.config_items.get_global_config_options():
            arr[i]['name'] = global_set[0]
            arr[i]['type'] = global_set[1]
            arr[i]['default'] = global_set[2]
            if len(global_set) > 3:
                arr[i]['enum'] = global_set[3]
            else:
                arr[i]['enum'] = None
            arr[i]['value'] = self.get_config_global(global_set[0])
            i += 1
        return arr

    def get_all_watcher_settings_json(self, watcher_uid=''):
        arr = collections.defaultdict(dict)
        if watcher_uid == '':
            watchers = self.get_all_watchers()
        else:
            watchers = self.get_watcher_by_uid(int(watcher_uid))
        first = True
        i = 0
        for watcher_setting in self.config_items.get_watcher_config_options():
            setting_key = watcher_setting[0]
            setting_type = watcher_setting[1]
            setting_default = watcher_setting[2]
            setting_enum = watcher_setting[3] if len(watcher_setting) > 3 else None
            for watcher in watchers:
                uid = watcher['uid']
                if first:
                    arr[uid] = collections.defaultdict(dict)
                    first = False
                arr[uid][i] = collections.defaultdict(dict)
                arr[uid][i]['name'] = setting_key
                arr[uid][i]['type'] = setting_type
                arr[uid][i]['default'] = setting_default
                arr[uid][i]['enum'] = setting_enum
                arr[uid][i]['value'] = self.get_config_watcher(uid, setting_key)
            i += 1
        return arr

    def get_config_item_type(self, key):
        for setting in self.config_items.get_both_global_config_options():
            if setting[0] == key:
                if len(setting) > 3:
                    return setting[1], setting[3]
                else:
                    return setting[1], None
        return None, None

class Config(ActualConfig, metaclass=Singleton):
    pass

# Containing all the default values and what they are for sort of value.
class ConfigItems:
    def __init__(self):
        self.gci = []  # Global config items
        self.wci = []  # Watcher config items

        self.gci.append(['personal_name', 'str', 'John Doe'])
        self.gci.append(['program_name', 'str', 'UnRAR angel'])
        self.gci.append(['logging_path', 'path', '/tmp/angel-logger.log'])
        self.gci.append(['logging_path_new_files', 'path', '/tmp/angel-logger-new-files.log'])
        self.gci.append(['logging_level', 'option', 'INFO'])
        self.gci[len(self.gci) - 1].append(['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'])
        self.gci.append(['angel_pid_path', 'path', '/tmp/unrar-angel.pid'])
        self.gci.append(['update_delay_in_seconds', 'int', 10])
        self.gci.append(['web_on_or_off', 'bool', 1])
        self.gci.append(['web_config_host_ip', 'str', 'localhost'])
        self.gci.append(['web_config_site_port', 'int', 5000])
        # self.gci.append(['web_config_api_port', 'int', 5001])
        self.gci.append(['web_password', 'str', 'unrar_angel'])

        self.wci.append(['uid', 'int', 1])  # D
        self.wci.append(['on_or_off', 'bool', 0])  # D
        self.wci.append(['name', 'str', 'Watcher config 1'])  # D

        self.wci.append(['source_path', 'path', '/home/user/Downloads/'])  # D
        self.wci.append(['destination_path', 'path', '/home/user/Downloads_unrar/'])  # D
        self.wci.append(['recursive_searching', 'bool', 1])  # D
        self.wci.append(['recursive_directory_building_for_new_file', 'bool', 1])  # D
        self.wci.append(['copy_or_unrar', 'option', 'unrar'])  # D
        self.wci[len(self.wci) - 1].append(['copy', 'unrar'])
        self.wci.append(['remove_after_finished', 'bool', 0])  # D
        self.wci.append(['copy_match_everything', 'bool', 1])  # D
        self.wci.append(['copy_not_everything_but_match_regexp', 'str', '.'])
        self.wci.append(['copy_actually_dont_do_shit', 'bool', 0])  # D

        self.wci.append(['unrar_match_only_rar_extension', 'bool', 1])
        self.wci.append(['unrar_not_rar_but_match_regexp', 'str', '.'])
        self.wci.append(['unrar_dir_write_permission', 'bool', 0])
        self.wci.append(['move_into_serie_maps', 'bool', 0])

        self.wci.append(['plex_on_or_off', 'bool', 0])  # D
        self.wci.append(['plex_ip_port', 'str', 'localhost:32400'])  # D
        self.wci.append(['plex_token', 'str', 'abcdef'])  # D
        self.wci.append(['plex_library_name', 'str', 'TV Series'])  # D

        self.wci.append(['kodi_on_or_off', 'bool', 0])  # D
        self.wci.append(['kodi_user', 'str', 'kodi'])  # D
        self.wci.append(['kodi_pass', 'str', 'kodi'])  # D
        self.wci.append(['kodi_ip_port', 'str', 'localhost:80'])  # D

    def get_global_config_options(self):
        return self.gci

    def get_watcher_config_options(self):
        return self.wci

    def get_both_global_config_options(self):
        return self.gci + self.wci


import re
import time

from addressnet.predict import predict
import pandas as pd
from collections import Counter
from dbhepler.mysqlhelper import DBPipeline
from apscheduler.schedulers.blocking import BlockingScheduler

from excel_re import read_excel
import os
import glob

import setting as setting

total_path = glob.glob(os.path.join('*.xls*'))
product_io = '/Users/loctek/Desktop/loctek/address-net/addressnet/乐仓产品数据2.xlsx'
# product_io = 'C:\\addressnet\\乐仓产品数据(1).xlsx'

product_data = pd.read_excel(product_io, dtype=str)

none_product = []

name_list = setting.ssh_name_list
data_country = setting.data_country
word_number = setting.word_number
champion_name = setting.champion_name

def fun_isdigit(aString):
    try:
        return int(aString)
    except ValueError as e:
        return ''

def function(text):
    if 'OT' in text:
        return 'OTTO'
    elif 'AM' in text:
        return 'AMAZON'
    else:
        return 'OTHER'

def get_country(text):
    if pd.isnull(text):
        return ''
    text = str.upper(text)
    if text in data_country.keys():
        return data_country[text]
    else:
        return text

# PR1206-Ebony
def get_delivery(sku, country):  # 物流方式
    try:
        if int(product_data[product_data.SKU == sku]['is_GLS'].values[0]):
            country_list = product_data[product_data.SKU == sku]['国家'].values[0]
            if 'GLOBAL' in country_list:
                return 'GLS_BIZ_PCL'
            if country in country_list:
                return 'GLS_BIZ_PCL'
        product_size = float(product_data[product_data.SKU == sku]['围长'].values[0])  # 对应SKU的围长
        if country == 'DE' and int(product_size) <= 360:
            return 'DHL_DE_PAKET'
        elif ((country == 'DE' or country == 'AT') and product_size > 360) or (
                (country != 'DE' and country != 'AT') and (product_size > 300)):
            return 'DHL_FRIGHT'
        else:
            return 'DHL_DE_INT_PAKET'
    except:
        if sku not in none_product:
            none_product.append(sku)
        return ''

def get_company(consignee_name, old_company, delivery_style):
    if pd.isnull(old_company):
        old_company = ''
    split_names = re.split(r'[,]', consignee_name)

    champion = ''
    for names in split_names:
        get_split_name = names.split(' ')
        for name in reversed(get_split_name):
            if name in champion_name:
                champion = names
    if old_company:
        champion = str(old_company) + champion
    if champion:
        consignee_name = consignee_name.replace(champion, '')
    if 'DHL' in delivery_style and len(champion) > 61:
        return '||||||||||||||||||||||||||||||', consignee_name
    if 'GLS' in delivery_style and len(champion) > 41:
        return '||||||||||||||||||||||||||||||', consignee_name
    return champion, consignee_name

def get_email(text):
    if text:
        return text
    else:
        return '/'

def remake_null(text):
    if not pd.isnull(text):
        return text
    else:
        return ' '

def get_street(doorplate_number, street):
    rule = '(.*)' + doorplate_number + '[,]{0,1}$'
    re_street = re.findall(rule, doorplate_number)
    if re_street:
        return re_street[0]
    return street

def clear_comma(street):
    span_front = re.search(r'(,*$)', street).span()
    street = street[:span_front[0]]
    span_behind = re.search(r'(^,*)', street).span()
    street = street[span_behind[1]:]
    comma_word = re.findall(',,', street)
    if comma_word:
        street = street.replace(comma_word[0], ',')
    street += ','
    return street

def get_ez_street(street, zip_code, consignee_company, city, doorplate):
    street = clear_comma(street)
    print(street)
    ez_street = setting.ez_street
    for word in list(ez_street.keys()):
        capital_words = re.findall(word, street, flags=re.IGNORECASE)
        if capital_words:
            street = street.replace(capital_words[0], ez_street[word])
    zip_code_word = ''
    consignee_company_word = ''
    city_word = ''
    doorplate_word = ''
    # consignee_company += ' '
    print(street, zip_code, consignee_company, city, 123)
    if not pd.isnull(zip_code):
        try:
            zip_code_word = re.findall(zip_code, street, flags=re.IGNORECASE)
            consignee_company_word = re.findall(consignee_company, street, flags=re.IGNORECASE)  # 公司
            city_word = re.findall(city, street, flags=re.IGNORECASE)  # city
            doorplate_word = re.findall(doorplate, street, flags=re.IGNORECASE)  # doorplate
        except:
            print('122211')
    if zip_code_word:
        street = street.replace(zip_code_word[0], '')
    if consignee_company_word:
        # if consignee_company_word[0] != ' ':
        street = street.replace(consignee_company_word[0], '')
    if city_word:
        street = street.replace(city_word[0] + ' ', '')
    if doorplate_word:
        try:
            doorplate_re = doorplate_word[0]
            if '-' in doorplate_word[0]:
                doorplate_re = doorplate_word[0].replace('-', '\-')
            word_split = street.split(',')

            re_sql = doorplate_re + ',(.*)'
            val = re.findall(re_sql, street)
            # if not val:
            # re_s = '[^' + doorplate_re + '](,.*)'
            #  = re.findall(re_sql_2, street)
            mark_mes = val
            mark_mes = mark_mes[0] if mark_mes else ''
        except:
            mark_mes = ''
        consignee_company += mark_mes
        street = street.replace(doorplate_word[0], '')
        street = street.replace(mark_mes, '')
        # print(doorplate_word[0])
        # street = mark_mes
        print(mark_mes)
    consignee_company = consignee_company.replace(doorplate, '')
    street = clear_comma(street)
    consignee_company = clear_comma(consignee_company)
    if consignee_company == ',':
        consignee_company = ''

    if pd.isnull(consignee_company):
        consignee_company = ''
    if len(street) > 35:
        return '||||||||||||||||||||||||||||||', consignee_company
    return street, consignee_company

def get_task():
    db_helper = DBPipeline()  # get DB config
    db_helper.cursor.execute("""select * from view_address_check_bigdate""")  # run sql
    db_data = db_helper.cursor.fetchall()  # get result
    if not db_data:
        return ''
    fields = [field[0] for field in db_helper.cursor.description]  # get dict key in list
    data_list = [dict(zip(fields, item)) for item in db_data]
    df_data = pd.DataFrame(data_list)
    db_df_data = df_data[
        ['name', 'receiveman', 'countryname', 'city', 'street', 'receivezip', 'phone',
         'statezone', 'statezone', 'statezone', 'email', 'company', 'ebname', 'province']]
    db_df_data.columns = setting.ssh_old_columns  # get normal col
    print(db_df_data, 123222)

    db_helper.cursor.close()

    data = db_df_data
    data['Delivery Address Line 1'] = data.apply(lambda x: remake_null(x['Delivery Address Line 1']), axis=1)
    data_dict = dict(data)
    predict_adderss = []
    for i in data_dict['Delivery Address Line 1']:
        predict_adderss.append(i)
    item_list = predict(predict_adderss)  # predict
    tmp_dict = {'flat_number': []}

    b = dict(Counter(data_dict['Order Number']))
    order_tmp_dict = {key: value for key, value in b.items() if value > 1}  # 只展示重复元素
    index = 0
    for order_number in data_dict['Order Number']:
        if order_number in order_tmp_dict.keys():
            order_tmp_dict[order_number] = order_tmp_dict[order_number] - 1
            data_dict['Order Number'][index] = str(data_dict['Order Number'][index]) + str(
                word_number[order_tmp_dict[order_number]])
        tmp_dict['flat_number'].append(
            read_excel(data_dict['Delivery Address Line 1'][index], data_dict['Delivery Country'][index],
                       data_dict['Delivery Postcode'][index]))
        index += 1

    index = 0
    for i in item_list:
        if not tmp_dict['flat_number'][index]:
            if i:
                flat_number = ''
                if 'flat_number' in i.keys():
                    flat_number = i['flat_number']
                    tmp_dict['flat_number'][index] = fun_isdigit(flat_number)
        index += 1
    data_dict.update(tmp_dict)

    data = pd.DataFrame(data_dict)
    data_new = data[
        ['Order Number', 'Delivery Receive Name', 'Delivery Country', 'Delivery Town',
         'Delivery Address Line 1',
         'Delivery Postcode', 'Telephone Number', 'Product Code', 'Quantity', 'flat_number', 'Email Address',
         'Delivery Company', 'ebname', 'province']]
    data_new.columns = setting.ssh_columns
    data_new2 = data_new.reindex(columns=name_list)
    data_new2[['街道2/Street2']] = data_new2[['街道/Street']]
    data_new2[['街道3/Street3']] = data_new2[['收件人公司/Consignee Company']]
    data_new2[['仓库代码/Warehouse Code']] = 'EUKL'
    data_new2[['自动分配仓库']] = 'EUKL'
    data_new2['销售平台/Sales Platform'] = data_new2.apply(lambda x: function(x['参考编号/Reference Code']), axis=1)
    data_new2['收件人国家/Consignee Country'] = data_new2.apply(lambda x: get_country(x['收件人国家/Consignee Country']),
                                                           axis=1)
    data_new2['派送方式/Delivery Style'] = data_new2.apply(
        lambda x: get_delivery(x['SKU1'], x['收件人国家/Consignee Country']), axis=1)
    data_new2['门牌号/Doorplate'] = data_new2.apply(lambda x: get_email(x['门牌号/Doorplate']),
                                                 axis=1)
    try:
        data_new2['收件人公司/Consignee Company'], data_new2['街道/Street'] = zip(*data_new2.apply(
            lambda x: get_company(x['街道/Street'], x['收件人公司/Consignee Company'], x['派送方式/Delivery Style']),
            axis=1))

        # data_new2['街道/Street'] = data_new2.apply(lambda x: get_street(x['门牌号/Doorplate'], x['街道/Street']),
        #                                          axis=1)
        data_new2['街道/Street'], data_new2['收件人公司/Consignee Company'] = zip(*data_new2.apply(
            lambda x: get_ez_street(x['街道/Street'], x['邮编/Zip Code'],
                                    x['收件人公司/Consignee Company'],
                                    x['城市/City'], x['门牌号/Doorplate']
                                    ), axis=1))
    except Exception as error:
        print(error)
    # --- new ---

    # data_new2.to_excel('asdasd.xls', index=False)
    data_clear = data_new2[[
        '参考编号/Reference Code', 'ebname', '街道2/Street2', '城市/City', '邮编/Zip Code', 'province',
        '收件人国家/Consignee Country', '收件人电话/Consignee Phone', '收件人Email/Consignee Email',
        '街道3/Street3', '街道/Street', '门牌号/Doorplate', '收件人公司/Consignee Company'
    ]]
    data_clear.columns = setting.columns_to_dict
    data_clear = data_clear.to_dict(orient='records')
    # data_new2.to_excel('asdasd.xls', index=False)

    for item in data_clear:
        print(item)
        DBPipeline().process_item(item)
    return 'ok'

if __name__ == "__main__":
    get_task()
    scheduler = BlockingScheduler()
    scheduler.add_job(get_task, 'interval', hours=8)
    scheduler.start()


"""Initial setup

Revision ID: 1ad4d4de5fb9
Revises:
Create Date: 2020-08-13 11:06:17.871892

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '1ad4d4de5fb9'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('user',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('social_id', sa.String(length=64), nullable=False),
    sa.Column('nickname', sa.String(length=64), nullable=True),
    sa.Column('email', sa.String(length=64), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('social_id')
    )
    op.create_table('route',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('user_id', sa.Integer(), nullable=True),
    sa.Column('public', sa.Boolean(), nullable=True),
    sa.Column('title', sa.String(length=120), nullable=True),
    sa.Column('distance', sa.Integer(), nullable=True),
    sa.Column('duration', sa.Integer(), nullable=True),
    sa.Column('bbox', sa.String(length=120), nullable=True),
    sa.Column('polyline', sa.String(length=10000), nullable=True),
    sa.Column('timestamp', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_route_public'), 'route', ['public'], unique=False)
    op.create_index(op.f('ix_route_timestamp'), 'route', ['timestamp'], unique=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_route_timestamp'), table_name='route')
    op.drop_index(op.f('ix_route_public'), table_name='route')
    op.drop_table('route')
    op.drop_table('user')
    # ### end Alembic commands ###


from aocd.models import Puzzle
import numpy as np
from collections import Counter

DAY = 3
YEAR = 2021

def main():
    puzzle = Puzzle(year=YEAR, day=DAY)
    puzzle.answer_a = solve_a(puzzle.input_data)
    puzzle.answer_b = solve_b(puzzle.input_data)

def parse_report(raw_input):
    return np.array([list(row) for row in raw_input.splitlines()]).astype(int)

def binary_to_int(binary):
    return int("".join(binary.astype(str)), 2)

def calculate_gamma_rate(report):
    column_sum = report.sum(axis=0)
    threshold = len(report) / 2
    column_sum[column_sum < threshold] = 0
    column_sum[column_sum > threshold] = 1
    return binary_to_int(column_sum)

def calculate_epsilon_rate(report):
    column_sum = report.sum(axis=0)
    threshold = len(report) / 2
    column_sum[column_sum < threshold] = 1
    column_sum[column_sum > threshold] = 0
    return binary_to_int(column_sum)

def calculate_oxygen_rating(report):
    candidates = report.copy()
    for i in range(len(report[0])):
        new_candidates = []
        digits = candidates[:, i]
        most_common_picks = Counter(digits).most_common(2)
        if most_common_picks[0][1] == most_common_picks[1][1]:
            most_common = 1
        else:
            most_common = most_common_picks[0][0]

        for candidate in candidates:
            if candidate[i] == most_common:
                new_candidates.append(candidate)

        if len(new_candidates) == 1:
            return binary_to_int(new_candidates[0])
        candidates = np.array(new_candidates)

def calculate_carbon_rating(report):
    candidates = report.copy()
    for i in range(len(report[0])):
        new_candidates = []
        digits = candidates[:, i]
        most_common_picks = Counter(digits).most_common(2)
        if most_common_picks[0][1] == most_common_picks[1][1]:
            least_common = 0
        else:
            least_common = most_common_picks[1][0]

        for candidate in candidates:
            if candidate[i] == least_common:
                new_candidates.append(candidate)

        if len(new_candidates) == 1:
            return binary_to_int(new_candidates[0])
        candidates = np.array(new_candidates)

def solve_a(puzzle_input):
    report = parse_report(puzzle_input)
    gamma = calculate_gamma_rate(report)
    epsilon = calculate_epsilon_rate(report)
    return gamma * epsilon

def solve_b(puzzle_input):
    report = parse_report(puzzle_input)
    oxygen = calculate_oxygen_rating(report)
    carbon = calculate_carbon_rating(report)
    return oxygen * carbon

if __name__ == "__main__":
    main()


import random
import shutil
from collections import defaultdict
from itertools import chain
from pathlib import Path
from typing import Set, Tuple, List, Dict

import Levenshtein
from tqdm import tqdm

from paraphone.tasks.dictionaries import DictionaryCSV
from paraphone.tasks.filters.base import FilteringTaskMixin, CandidatesPairCSV, WordPair, CorpusFinalFilteringTask
from paraphone.tasks.phonemize import PhonemizedWordsCSV
from paraphone.tasks.tokenize import TokenizedWordsCSV
from paraphone.tasks.wuggy_gen import FakeWordsCandidatesCSV
from paraphone.utils import logger
from paraphone.workspace import Workspace

class InitFilteringTask(FilteringTaskMixin):
    requires = [
        "wuggy/candidates.csv",
    ]

    creates = [
        "candidates_filtering/steps/",
        "candidates_filtering/steps/step_1_init.csv"
    ]

    def run(self, workspace: Workspace):
        steps_folder = workspace.candidates_filtering / Path("steps")
        # cleaning up folder in case of re-initing
        shutil.rmtree(steps_folder, ignore_errors=True)
        steps_folder.mkdir(parents=True, exist_ok=True)

        wuggy_candidates_csv = FakeWordsCandidatesCSV(workspace.wuggy / Path("candidates.csv"))
        step_init_csv = CandidatesPairCSV(steps_folder / Path("step_1_init.csv"))

        logger.info(f"Initializing filtering pipeline with {wuggy_candidates_csv.file_path}")
        with step_init_csv.dict_writer as dict_writer:
            dict_writer.writeheader()
            for word, word_pho, _, fake_word_pho, _ in tqdm(wuggy_candidates_csv,
                                                            total=wuggy_candidates_csv.lines_count):
                dict_writer.writerow({
                    "word": word,
                    "word_pho": " ".join(word_pho),
                    "fake_word_pho": " ".join(fake_word_pho)
                })

class PassThroughFinalFilter(CorpusFinalFilteringTask):
    """Just copies the last step of the filtering pipeline into
    the respective corpora"""

    def run_for_corpus(self, workspace: Workspace, corpus_id: int):
        self.filter(workspace, corpus_id)

class RandomFilterTask(FilteringTaskMixin):
    """Keeps a random split of the candidates determined by ratio."""
    step_name = "random"

    def __init__(self, ratio: float):
        super().__init__()
        assert 0 < ratio < 1.0
        self.ratio = ratio

    def keep_pair(self, word_pair: WordPair) -> bool:
        return random.random() < self.ratio

    def run(self, workspace: Workspace):
        random.seed(4577)
        self.filter(workspace)

class RandomPairFilterTask(FilteringTaskMixin):
    """Keeps only one of the word/nonword pairs, at random"""
    step_name = "random-pairs"

    def __init__(self):
        super().__init__()
        self._chosen_pairs: Set[Tuple[str, str]] = set()

    def keep_pair(self, word_pair: WordPair) -> bool:
        return (word_pair.word_pho, word_pair.fake_word_pho) in self._chosen_pairs

    def run(self, workspace: Workspace):
        random.seed(4577)

        previous_step_csv = self.previous_step_csv(workspace)
        word_nonword = defaultdict(list)  # word -> list(nonwords)
        for _, word_pho, fake_word_pho in tqdm(previous_step_csv):
            word_nonword[word_pho].append(fake_word_pho)

        for word, fake_words in tqdm(word_nonword.items()):
            chosen_fake_word = random.choice(fake_words)
            self._chosen_pairs.add((word, chosen_fake_word))
        self.filter(workspace)

class EqualsFilterTask(FilteringTaskMixin):
    """Filters out equal phonetic pairs"""
    step_name = "equals"

    def keep_pair(self, word_pair: WordPair) -> bool:
        return word_pair.word_pho != word_pair.fake_word_pho

    def run(self, workspace: Workspace):
        self.filter(workspace)

class MostFrequentHomophoneFilterTask(FilteringTaskMixin):
    """Filter homophones based on grapheme frequency (the grapheme with
    the largest frequency is the one that "wins" and other graphemic forms
    are filtered out) """
    requires = [
        "datasets/tokenized/all.csv"
    ]
    step_name = "homophones"

    def __init__(self):
        super().__init__()
        # phonetic forms and the frequency of the word that is associated with it
        # for two words with the same word_pho (homophones), only the one
        # with the most frequent occurences is kept
        self.word_pho_freq: Dict[str, int] = defaultdict(int)
        # word_pho -> word
        self.kept_word_pho: Dict[str, str] = dict()
        self.kept_words: Set[str] = set()

    def keep_pair(self, word_pair: WordPair) -> bool:
        return word_pair.word in self.kept_words

    def run(self, workspace: Workspace):
        tokenized_words_csv = TokenizedWordsCSV(
            workspace.tokenized / Path("all.csv")
        )
        tokenized_words = tokenized_words_csv.to_dict()
        previous_step_csv = self.previous_step_csv(workspace)
        for word, word_pho, _ in previous_step_csv:
            if self.word_pho_freq[word_pho] < tokenized_words[word]:
                self.word_pho_freq[word_pho] = tokenized_words[word]
                self.kept_word_pho[word_pho] = word

        self.kept_words = set(self.kept_word_pho.values())
        self.filter(workspace)

class WuggyHomophonesFilterTask(FilteringTaskMixin):
    """Filters out fake word whose phonetic form matches that of a real word."""
    requires = [
        "dictionaries/*/dict_folded.csv",
        "phonemized/all.csv"
    ]
    step_name = "wuggy-homophones"

    def __init__(self):
        super().__init__()
        self.all_words_phonemized: Set[str] = set()

    def keep_pair(self, word_pair: WordPair) -> bool:
        return word_pair.fake_word_pho not in self.all_words_phonemized

    def run(self, workspace: Workspace):
        self.all_words_phonemized = set()
        for dict_filepath in workspace.dictionaries.glob("**/dict_folded.csv"):
            dict_csv = DictionaryCSV(dict_filepath)
            self.all_words_phonemized.update({
                " ".join(pho) for _, pho, _ in dict_csv
            })
        phonemized_words_csv = PhonemizedWordsCSV(workspace.phonemized / Path("all.csv"))
        self.all_words_phonemized.update({
            " ".join(pho) for _, pho in phonemized_words_csv
        })

        self.filter(workspace)

class LevenshteinFilterTask(FilteringTaskMixin):
    """Keeps pairs whose edit distance is lower than a threshold"""
    step_name = "levenshtein"

    def __init__(self, max_distance: int):
        super().__init__()
        assert max_distance > 0
        self.max_distance = max_distance

    def pho_to_str(self, pho_a: List[str], pho_b: List[str]) -> Tuple[str, str]:
        """Remaps phoneme lists to ASCII strings for levenshtein edit distance"""
        pho_set = set(chain.from_iterable([pho_a, pho_b]))
        pho_map = {pho: chr(i) for i, pho in enumerate(pho_set)}
        return "".join(pho_map[pho] for pho in pho_a), "".join(pho_map[pho] for pho in pho_b)

    def keep_pair(self, word_pair: WordPair) -> bool:
        word_pho, fake_word_pho = self.pho_to_str(word_pair.word_pho.split(" "),
                                                  word_pair.fake_word_pho.split(" "))
        return Levenshtein.distance(word_pho, fake_word_pho) <= self.max_distance

    def run(self, workspace: Workspace):
        self.filter(workspace)


from typing import List
from typing import Tuple

import chainer
import chainer.functions as F
import chainer.links as L
from chainer import Parameter
from chainer import Variable
import numpy as np

from cnmt.misc.constants import EOS
from cnmt.misc.constants import PAD
from cnmt.misc.typing import ndarray
from cnmt.models.attention import AttentionModule
from cnmt.models.attention import DynamicAttentionModule

class Decoder(chainer.Chain):
    def __init__(self,
                 vocabulary_size: int,
                 word_embeddings_size: int,
                 hidden_layer_size: int,
                 attention_hidden_layer_size: int,
                 encoder_output_size: int,
                 maxout_layer_size: int,
                 maxout_pool_size: int = 2,
                 ignore_label: int = -1,
                 dynamic_attention: bool = False):
        super(Decoder, self).__init__()
        with self.init_scope():
            self.embed_id = L.EmbedID(vocabulary_size,
                                      word_embeddings_size,
                                      ignore_label=ignore_label)
            self.rnn = L.StatelessLSTM(
                word_embeddings_size + encoder_output_size,
                hidden_layer_size
            )
            self.maxout = L.Maxout(word_embeddings_size +
                                   encoder_output_size +
                                   hidden_layer_size,
                                   maxout_layer_size,
                                   maxout_pool_size)
            self.linear = L.Linear(maxout_layer_size, vocabulary_size)
            if dynamic_attention:
                self.attention = DynamicAttentionModule(
                    encoder_output_size,
                    attention_hidden_layer_size,
                    hidden_layer_size,
                    word_embeddings_size
                )
            else:
                self.attention = AttentionModule(
                    encoder_output_size,
                    attention_hidden_layer_size,
                    hidden_layer_size,
                    word_embeddings_size
                )
            self.bos_state = Parameter(
                initializer=self.xp.random.randn(
                    1,
                    hidden_layer_size
                ).astype('f')
            )
        self.vocabulary_size = vocabulary_size
        self.word_embeddings_size = word_embeddings_size
        self.hidden_layer_size = hidden_layer_size
        self.encoder_output_size = encoder_output_size

    def __call__(
            self,
            encoded: Variable,
            target: ndarray
    ) -> Variable:
        minibatch_size, max_sentence_size, encoder_output_size = encoded.shape
        assert encoder_output_size == self.encoder_output_size
        assert target.shape[0] == minibatch_size

        self.setup(encoded)
        cell, state, previous_words = self.get_initial_states(minibatch_size)

        total_loss = Variable(self.xp.array(0, 'f'))
        total_predictions = 0

        for target_id in self.xp.hsplit(target, target.shape[1]):
            target_id = target_id.reshape((minibatch_size,))
            cell, state, context, concatenated = \
                self.advance_one_step(cell, state, previous_words)
            logit, state = self.compute_logit(concatenated, state, context)

            current_sentence_count = self.xp.sum(target_id != PAD)

            loss = F.softmax_cross_entropy(logit, target_id, ignore_label=PAD)
            total_loss += loss * current_sentence_count
            total_predictions += current_sentence_count

            previous_words = target_id

        return total_loss / total_predictions

    def setup(self, encoded: Variable):
        if self.bos_state.array is None:
            self.bos_state.initialize((1, self.hidden_layer_size))
        self.attention.precompute(encoded)

    def get_initial_states(
            self,
            minibatch_size: int
    ) -> Tuple[Variable, Variable, ndarray]:
        cell = Variable(
            self.xp.zeros((minibatch_size, self.hidden_layer_size), 'f')
        )
        state = F.broadcast_to(
            self.bos_state, (minibatch_size, self.hidden_layer_size)
        )
        previous_words = self.xp.full((minibatch_size,), EOS, 'i')
        return cell, state, previous_words

    def advance_one_step(
            self,
            cell: Variable,
            state: Variable,
            previous_words: ndarray
    ) -> Tuple[Variable, Variable, Variable, Variable]:
        minibatch_size = cell.shape[0]
        previous_embedding = self.embed_id(previous_words)
        context = self.attention(state, previous_embedding)
        assert context.shape == (minibatch_size, self.encoder_output_size)
        concatenated = F.concat((previous_embedding, context))
        cell, state = self.rnn(cell, state, concatenated)
        return cell, state, context, concatenated

    def compute_logit(
            self,
            concatenated: Variable,
            state: Variable,
            context: Variable
    ) -> Tuple[Variable, Variable]:
        all_concatenated = F.concat((concatenated, state))
        logit = self.linear(self.maxout(all_concatenated))
        return logit, state

    def translate(
            self,
            encoded: Variable,
            max_length: int = 100
    ) -> List[ndarray]:
        sentence_count = encoded.shape[0]

        self.setup(encoded)
        cell, state, previous_words = self.get_initial_states(sentence_count)

        result = []
        for _ in range(max_length):
            cell, state, context, concatenated = \
                self.advance_one_step(cell, state, previous_words)
            logit, state = self.compute_logit(concatenated, state, context)

            output_id = F.reshape(F.argmax(logit, axis=1), (sentence_count,))
            result.append(output_id)

            previous_words = output_id

        # Remove words after <EOS>
        outputs = F.separate(F.transpose(F.vstack(result)), axis=0)
        assert len(outputs) == sentence_count
        output_sentences = []
        for output in outputs:
            assert output.shape == (max_length,)
            indexes = np.argwhere(output.data == EOS)
            if len(indexes) > 0:
                output = output[:indexes[0, 0] + 1]
            output_sentences.append(output.data)

        return output_sentences


from random import randint

# Plotly
from plotly import offline
# recommended starting point
import plotly.express as px
# in-depth access point
import plotly.graph_objects as pgo

class Die:
    """A single die"""

    def __init__(self, sides:int = 6) -> None:
        """Initialise a new die"""
        self.sides = sides

    def roll(self) -> int:
        """Roll the die"""
        return randint(1, self.sides)

    def roll(self, sides) -> int:
        """Roll the die, with a specified number of sides"""
        return randint(1, sides)

    def roll_multiple(self, roll_count:int) -> list:
        """Roll the die a specified number of times"""
        results = []
        for x in range(roll_count):
            results.append(self.roll())
        return results

    def roll_multiple(self, roll_count:int, die_count:int) -> list:
        """Roll a specified number of times with the specified number of dice"""
        results = []
        for roll in range(roll_count):
            results.append(0)
            for die in range(die_count):
                results[-1] += self.roll()
        return results

    def roll_multiple(self, roll_count:int, dice:list = []) -> list:
        """Roll a specified number of times with the specified number of dice, each with the specified number of sides"""
        results = []
        for roll in range(roll_count):
            results.append(0)
            for die in range(len(dice)):
                results[-1] += self.roll(dice[die])
        return results

# ---- Input values ----
number_of_rolls = 1000
die_sides_default = 6
dice = [6, 10]

die = Die(die_sides_default)
# Generate some data
results = die.roll_multiple(number_of_rolls, dice)

# Analyse the result
frequencies = []
for value in range(len(dice), (sum(dice)) + 1):
    frequencies.append(results.count(value))

# Visualise the result
x_values = list(range(len(dice), sum(dice) + 1))

fig = pgo.Figure([pgo.Bar(x=x_values, y=frequencies)])
fig.update_layout(title_text=f'Die rolls ({number_of_rolls}x)')
fig.show()


import requests  # used for sending requests to the API
import json      # used for manipulating JSON data
import pprint  # used for formatting the output of JSON objects received in API responses

import os
import sys
base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
sys.path.insert(0, base_path)

from common.base_operations import Config

# Initialize config and read all required values form JSON config
cfg = Config(full=False)

with open(os.path.join(base_path, 'api_client.json')) as api_client_file:
    api_client_json = json.load(api_client_file)

client_id = api_client_json["client_id"]
client_secret = api_client_json["client_secret"]

response = requests.post(
    f'{cfg.base_url}api/2/idp/token',
    headers={**cfg.header, **{'Content-Type': 'application/x-www-form-urlencoded'}},
    auth=(client_id, client_secret),
    data={'grant_type': 'client_credentials'}
)

if response.ok:
    with open(os.path.join(base_path, 'api_token.json'), 'w') as outfile:
        json.dump(response.json(), outfile)
else:
    pprint.pprint(response.json())


import glob
import http.server
import io
import logging
import os
import pathlib
import socketserver
import shutil
from datetime import datetime
from PIL import Image, ImageFont, ImageDraw
from pilapse.ip_finder import get_local_ip

PATH = '.'
logger = logging.getLogger('server')

class H(http.server.BaseHTTPRequestHandler):

    def do_GET(self):
        logger.debug("{}: {} - {}\n".format(self.command,
                                            self.client_address, self.path))
        if self.path == '/last.jpg':
            self.serve_latest_image()
        else:
            self.serve_index()

    def serve_index(self):
        self.send_response(200)
        self.send_header('Content-Type', 'text/html; charset=utf-8')
        self.end_headers()
        f = open(pathlib.Path(__file__).parent.joinpath(
            'index.html').absolute(), 'rb')
        try:
            shutil.copyfileobj(f, self.wfile)
        finally:
            f.close()

    def serve_latest_image(self):
        byteIO = io.BytesIO()
        path = self.get_latest_image_path()
        image = self.get_image(path)
        image.resize((960, 540))
        image.save(byteIO, format='JPEG')
        self.send_response(200, "")
        self.send_header("'Content-Length'", str(len(byteIO.getvalue())))
        self.send_header("Content-Type", "image/jpeg")
        self.end_headers()
        self.wfile.write(byteIO.getvalue())

    def get_latest_image_path(self):
        list_of_files = glob.glob(os.path.join(PATH, '*'))
        latest_file = max(list_of_files, key=os.path.getctime)
        logger.debug("latest image: {}".format(latest_file))
        return latest_file

    def get_file_mtime(self, path):
        fname = pathlib.Path(path)
        assert fname.exists(), f'No such file: {fname}'
        ts = fname.stat().st_mtime
        logger.debug(datetime.fromtimestamp(ts))
        return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")

    def get_image(self, path):
        font = ImageFont.truetype(str(pathlib.Path(__file__).parent.joinpath(
            'Product Sans Regular.ttf').absolute()), 25)
        with Image.open(path) as img:
            draw = ImageDraw.Draw(img)
            text = self.get_file_mtime(path)
            draw.text((0, 0), text, (255, 255, 255), font=font)
            return img

def start_server(port, path):
    with socketserver.TCPServer(("", port), H) as httpd:
        global PATH
        PATH = pathlib.Path(path).expanduser().absolute()
        ip = get_local_ip()
        logger.info(
            "Serving images via http://{}:{}/ from {}".format(ip, port, PATH))
        httpd.serve_forever()

if __name__ == '__main__':
    start_server(8080, '/tmp/images/')


from selenium import webdriver
import time
import os

#указываем какой и откуда брать файл
current_dir = os.path.abspath(os.path.dirname(__file__))
file_name = "test.txt"
file_path = os.path.join(current_dir, file_name)

try:
    link = "http://suninjuly.github.io/file_input.html"
    browser = webdriver.Chrome()
    browser.get(link)

    browser.find_element_by_css_selector('[name="firstname"]').send_keys('Leo')
    browser.find_element_by_css_selector('[name="lastname"]').send_keys('Messi')
    browser.find_element_by_css_selector('[name="email"]').send_keys('psg@fc.com')

    element = browser.find_element_by_css_selector("[type='file']")
    element.send_keys(file_path)

    browser.find_element_by_css_selector(".btn.btn-primary").click()

finally:
    # ожидание чтобы визуально оценить результаты прохождения скрипта
    time.sleep(5)
    # закрываем браузер после всех манипуляций
    browser.quit()


import requests
from bs4 import BeautifulSoup
import csv

driving_side_url = 'https://en.wikipedia.org/wiki/Left-_and_right-hand_traffic'
html_text = requests.get(driving_side_url).text
soup = BeautifulSoup(html_text, 'html.parser')

tables = soup.find_all('table')
table = tables[1]
table_body = table.find('tbody')

data = []
rows = table_body.find_all('tr')
for row in rows:
    cols = row.find_all('td')
    cols = [ele.text.strip() for ele in cols]
    data.append([ele for ele in cols if ele])

print(data)

with open('wiki_table.csv', 'w', newline='', encoding="utf-8") as f:
    writer = csv.writer(f)
    row_num = 0
    for row in data:
        row_num += 1
        writer.writerow(row)

        # to make sure I haven't missed any countries with weird results
        # points me to CSV row numbers
        if len(row) > 1 and len(row[1]) >= 3:
            if row[1][1:3] != 'HT':
                print(row_num)


import tempfile
import io
from colorama import Back
from flask import Flask
from flask import request, render_template, url_for, jsonify, send_file
from pytube import YouTube
from pytube.exceptions import RegexMatchError, VideoUnavailable
import iuliia

app = Flask(__name__)

@app.route("/", methods=["GET"])
def index():
    return render_template(
        "index.html",
        get_video_metadata_url=url_for("get_video_metadata"),
        install_video_url=url_for("install_video")
    )

@app.route("/validate-link/", methods=["POST"])
def get_video_metadata():
    try:
        pytube_obj = YouTube(url=request.json["current_url"])
    except RegexMatchError:
        return jsonify({"fail_message" : "Ссылка на YouTube-видео указана неверно!"})
    else:
        try:
            return jsonify({
                "video_title" : pytube_obj.title,
                "transliterated_title" : iuliia.translate(pytube_obj.title, iuliia.TELEGRAM),
                "thumbnail_url" : pytube_obj.thumbnail_url,
            })
        except VideoUnavailable: # user can pass url for another site, so pytube won't raise RegexMatchError or smth else, it just won't be able to get video's metadata
            return jsonify({"fail_message" : "Ссылка на YouTube-видео указана неверно!"})

@app.route("/install/", methods=["POST"])
def install_video():
    try:
        yt = YouTube(url=request.json["video_url"])
        stream_obj = yt.streams.get_by_itag(22) # 720p / 30fps by default

        with tempfile.NamedTemporaryFile(mode="w+b", prefix="yt_", suffix="_video", dir="video_tempfiles/") as temp:
            video_binary_buffer = io.BytesIO()
            stream_obj.stream_to_buffer(video_binary_buffer)
            video_binary_buffer.seek(0)
            temp.write(video_binary_buffer.getvalue()) # method .getvalue() returns 'bytes-like' object instead of '_ioBytes' obj
            temp.seek(0)

            return send_file(
                temp.name, as_attachment=True
            )
    except Exception as exc:
        print(Back.RED + f"Server Error: {exc}")

if __name__ == "__main__":
    app.run(port=8000)


from django.contrib import admin
from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static
from rest_framework_simplejwt.views import (
    TokenObtainPairView,
    TokenRefreshView,
    TokenVerifyView,
)

urlpatterns = [
    path('admin/', admin.site.urls),
    path('users/', include('users.urls')),
    path('newsletter/', include('newsletter.urls')),
    path('ckeditor/', include('ckeditor_uploader.urls')),
    path('api/', include('magazine.urls', namespace='magazine_api')),
    path('api/token/', TokenObtainPairView.as_view(), name='token_obtain_pair'),
    path('api/token/refresh/', TokenRefreshView.as_view(), name='token_refresh'),
    path('api/token/verify/', TokenVerifyView.as_view(), name='token_verify'),
]

urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)